Browse Source

Bug 1580778 - Migrate build tasks to taskgraph and retire the old decision task (#5488)

master
Johan Lorenzo 2 years ago
committed by GitHub
parent
commit
6a8f7224dc
No known key found for this signature in database GPG Key ID: 4AEE18F83AFDEB23
  1. 2
      .cron.yml
  2. 1
      .gitignore
  3. 87
      automation/taskcluster/decision_task.py
  4. 20
      automation/taskcluster/lib/chain_of_trust.py
  5. 352
      automation/taskcluster/lib/tasks.py
  6. 15
      automation/taskcluster/lib/util.py
  7. 34
      automation/taskcluster/lib/variant.py
  8. 105
      automation/taskcluster/schedule_nightly_graph.py
  9. 108
      taskcluster/ci/build/kind.yml
  10. 2
      taskcluster/ci/config.yml
  11. 7
      taskcluster/ci/nimbledroid/kind.yml
  12. 2
      taskcluster/ci/pr/kind.yml
  13. 18
      taskcluster/ci/push-apk/kind.yml
  14. 6
      taskcluster/ci/raptor/kind.yml
  15. 115
      taskcluster/ci/signing/kind.yml
  16. 18
      taskcluster/ci/test/kind.yml
  17. 21
      taskcluster/docker/base/Dockerfile
  18. 16
      taskcluster/fenix_taskgraph/gradle.py
  19. 3
      taskcluster/fenix_taskgraph/job.py
  20. 81
      taskcluster/fenix_taskgraph/loader/old_decision.py
  21. 27
      taskcluster/fenix_taskgraph/nimbledroid.py
  22. 75
      taskcluster/fenix_taskgraph/pushapk.py
  23. 67
      taskcluster/fenix_taskgraph/signing.py
  24. 44
      taskcluster/fenix_taskgraph/target_tasks.py
  25. 0
      taskcluster/fenix_taskgraph/transforms/__init__.py
  26. 128
      taskcluster/fenix_taskgraph/transforms/build.py
  27. 46
      taskcluster/fenix_taskgraph/transforms/push_apk.py
  28. 0
      taskcluster/fenix_taskgraph/transforms/raptor.py
  29. 46
      taskcluster/fenix_taskgraph/transforms/signing.py
  30. 62
      taskcluster/fenix_taskgraph/transforms/single_dep.py
  31. 10
      taskcluster/fenix_taskgraph/util.py

2
.cron.yml

@ -8,7 +8,6 @@ jobs:
job:
type: decision-task
treeherder-symbol: Nd
# TODO change target method once first tasks are migrated off "old-decision"
target-tasks-method: nightly
when: [{hour: 6, minute: 0}]
when: [{hour: 18, minute: 0}]
@ -16,6 +15,5 @@ jobs:
job:
type: decision-task
treeherder-symbol: raptor-D
# TODO change target method once first tasks are migrated off "old-decision"
target-tasks-method: raptor
when: [{hour: 1, minute: 0}]

1
.gitignore

@ -22,6 +22,7 @@ out/
# Gradle files
.gradle/
build/
!taskcluster/ci/build
# Local configuration file (sdk path, etc)
local.properties

87
automation/taskcluster/decision_task.py

@ -1,87 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Decision task for nightly releases.
"""
from __future__ import print_function
import argparse
import datetime
import os
import re
import taskcluster
from lib.gradle import get_variant
from lib.tasks import (
fetch_mozharness_task_id,
schedule_task_graph,
TaskBuilder,
)
from lib.chain_of_trust import (
populate_chain_of_trust_task_graph,
populate_chain_of_trust_required_but_unused_files
)
def pr(builder):
tasks = []
variant = get_variant('debug', 'geckoNightly')
tasks.append(builder.craft_assemble_pr_task(variant))
tasks.append(builder.craft_test_pr_task(variant))
for task in tasks:
task['attributes']['code-review'] = True
return tasks
def push(builder):
# We want the same tasks on pushes than on PRs, for now.
return pr(builder)
def raptor(builder, is_staging):
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
}).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
build_task = builder.craft_assemble_raptor_task(variant)
# Signing and Raptor tasks are generated in taskgraph
return [build_task]
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
# The signing push-apk tasks are generated by taskgraph
return [build_task]
def release_as_fennec(builder, is_staging, version_name):
variant = get_variant('fennecProduction', 'geckoBeta')
channel = 'fennec-production'
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
# The signing task is generated by taskgraph
return [build_task]
def nightly_to_production_app(builder, is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
variant = get_variant('fenixNightlyLegacy', 'geckoNightly')
taskcluster_apk_paths = variant.upstream_artifacts()
build_task = builder.craft_assemble_release_task(
variant, 'nightly-legacy', is_staging, version_name)
# Nimbledroid, signing and push-apk tasks are generated by taskgraph
return [build_task]

20
automation/taskcluster/lib/chain_of_trust.py

@ -1,20 +0,0 @@
import json
def populate_chain_of_trust_required_but_unused_files():
# These files are needed to keep chainOfTrust happy. However, they have no
# need for android-components, at the moment. For more details, see:
# https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
for file_names in ('actions.json', 'parameters.yml'):
with open(file_names, 'w') as f:
json.dump({}, f) # Yaml is a super-set of JSON.
def populate_chain_of_trust_task_graph(full_task_graph):
# full_task_graph must follow the format:
# {
# task_id: full_task_definition
# }
with open('task-graph.json', 'w') as f:
json.dump(full_task_graph, f)

352
automation/taskcluster/lib/tasks.py

@ -2,360 +2,8 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
import arrow
import datetime
import json
import os
import taskcluster
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
DEFAULT_EXPIRES_IN = '1 year'
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix'
_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task'
GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json'
# Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases
ARM_RAPTOR_URL_PARAMS = [
"exclude=1,2,9,10,17,18,21,22,26,28,30,32,39,40,47,"
"48,55,56,63,64,71,72,79,80,83,84,89,90,95,96",
]
class TaskBuilder(object):
def __init__(
self,
task_id,
repo_url,
git_ref,
short_head_branch, commit, owner, source, scheduler_id, date_string,
tasks_priority='lowest',
trust_level=1
):
self.task_id = task_id
self.repo_url = repo_url
self.git_ref = git_ref
self.short_head_branch = short_head_branch
self.commit = commit
self.owner = owner
self.source = source
self.scheduler_id = scheduler_id
self.trust_level = trust_level
self.tasks_priority = tasks_priority
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
self.trust_level = trust_level
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
if is_staging:
secret_index = 'garbage/staging/project/mobile/fenix'
else:
secret_index = 'project/mobile/fenix/{}'.format(channel)
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
secret_index, key, target_file
)
for key, target_file in (
('sentry_dsn', '.sentry_token'),
('leanplum', '.leanplum_token'),
('adjust', '.adjust_token'),
('digital_asset_links', '.digital_asset_links_token'),
('firebase', 'app/src/{}/res/values/firebase.xml'.format(variant.build_type)),
)
)
capitalized_build_type = upper_case_first_letter(variant.build_type)
gradle_commands = (
'./gradlew --no-daemon -PversionName="{}" clean test assemble{}'.format(
version_name, capitalized_build_type),
)
command = ' && '.join(
cmd
for commands in (pre_gradle_commands, gradle_commands)
for cmd in commands
if cmd
)
routes = [] if is_staging else [
"notify.email.fenix-eng-notifications@mozilla.com.on-failed"
]
return self._craft_build_ish_task(
name='Build {} task'.format(capitalized_build_type),
description='Build Fenix {} from source code'.format(capitalized_build_type),
command=command,
scopes=[
"secrets:get:{}".format(secret_index)
],
artifacts=variant.artifacts(),
routes=routes,
treeherder={
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': '{}-A'.format(variant.build_type),
'tier': 1,
},
attributes={
'apks': variant.upstream_artifacts_per_abi,
}
)
def craft_assemble_raptor_task(self, variant):
command = ' && '.join((
'echo "https://fake@sentry.prod.mozaws.net/368" > .sentry_token',
'echo "--" > .adjust_token',
'echo "-:-" > .leanplum_token',
'touch .digital_asset_links_token',
'./gradlew --no-daemon clean assemble{}'.format(variant.name),
))
return self._craft_build_ish_task(
name='assemble: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
command=command,
artifacts=variant.artifacts(),
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'A',
'tier': 1,
},
attributes={
'build-type': 'raptor',
'apks': variant.upstream_artifacts_per_abi,
},
)
def craft_assemble_pr_task(self, variant):
return self._craft_clean_gradle_task(
name='assemble: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
gradle_task='assemble{}'.format(variant.name),
artifacts=variant.artifacts(),
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'build',
'machine': {
'platform': 'android-all',
},
'symbol': 'A',
'tier': 1,
},
)
def craft_test_pr_task(self, variant):
# upload coverage only once, if the variant is arm64
secret_index = 'project/mobile/fenix/public-tokens'
pre_gradle_commands = (
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
secret_index, key, target_file
)
for key, target_file in (
('codecov', '.cc_token'),
)
)
gradle_commands = (
'./gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport',
)
post_gradle_commands = (
'automation/taskcluster/upload_coverage_report.sh',
)
command = ' && '.join(
cmd
for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands)
for cmd in commands
if cmd
)
return self._craft_build_ish_task(
name='test: {}'.format(variant.name),
description='Building and testing variant {}'.format(variant.name),
command=command,
treeherder={
'groupSymbol': variant.build_type,
'jobKind': 'test',
'machine': {
'platform': 'android-all',
},
'symbol': 'T',
'tier': 1,
},
scopes=[
'secrets:get:{}'.format(secret_index)
]
)
def _craft_clean_gradle_task(
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
):
return self._craft_build_ish_task(
name=name,
description=description,
command='./gradlew --no-daemon clean {}'.format(gradle_task),
artifacts=artifacts,
routes=routes,
treeherder=treeherder,
scopes=scopes,
)
def _craft_build_ish_task(
self, name, description, command, dependencies=None, artifacts=None,
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
):
artifacts = {} if artifacts is None else artifacts
scopes = [] if scopes is None else scopes
env_vars = {} if env_vars is None else env_vars
checkout_command = ' && '.join([
"export TERM=dumb",
"git fetch {} {}".format(self.repo_url, self.git_ref),
"git config advice.detachedHead false",
"git checkout FETCH_HEAD",
])
command = '{} && {}'.format(checkout_command, command)
features = {}
if artifacts:
features['chainOfTrust'] = True
if any(scope.startswith('secrets:') for scope in scopes):
features['taskclusterProxy'] = True
payload = {
"features": features,
"env": env_vars,
"maxRunTime": 7200,
"image": "mozillamobile/fenix:1.4",
"command": [
"/bin/bash",
"--login",
"-cx",
# Some tasks like nimbledroid do have tasks references
{'task-reference': command},
],
"artifacts": artifacts,
}
return self._craft_default_task_definition(
worker_type='mobile-{}-b-fenix'.format(self.trust_level),
provisioner_id='aws-provisioner-v1',
name=name,
description=description,
payload=payload,
dependencies=dependencies,
routes=routes,
scopes=scopes,
treeherder=treeherder,
attributes=attributes,
)
def _craft_default_task_definition(
self,
worker_type,
provisioner_id,
name,
description,
payload,
dependencies=None,
routes=None,
scopes=None,
treeherder=None,
notify=None,
attributes=None
):
dependencies = {} if dependencies is None else dependencies
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
treeherder = {} if treeherder is None else treeherder
attributes = {} if attributes is None else attributes
created = datetime.datetime.now()
deadline = taskcluster.fromNow('1 day')
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
routes.append('checks')
if self.trust_level == 3:
routes.append('tc-treeherder.v2.fenix.{}'.format(self.commit))
extra = {
"treeherder": treeherder,
}
if notify:
extra['notify'] = notify
return {
"attributes": attributes,
"dependencies": dependencies,
"label": name,
"task": {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
},
}
def schedule_task(queue, taskId, task):
print("TASK", taskId)
print(json.dumps(task, indent=4, separators=(',', ': ')))
result = queue.createTask(taskId, task)
print("RESULT", taskId)
print(json.dumps(result))
def schedule_task_graph(ordered_groups_of_tasks):
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
full_task_graph = {}
# TODO: Switch to async python to speed up submission
for group_of_tasks in ordered_groups_of_tasks:
for task_id, task_definition in group_of_tasks.items():
schedule_task(queue, task_id, task_definition)
full_task_graph[task_id] = {
# Some values of the task definition are automatically filled. Querying the task
# allows to have the full definition. This is needed to make Chain of Trust happy
'task': queue.task(task_id),
}
return full_task_graph
def fetch_mozharness_task_id():
# We now want to use the latest available raptor
raptor_index = 'gecko.v2.mozilla-central.nightly.latest.mobile.android-x86_64-opt'
return taskcluster.Index({
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
}).findTask(raptor_index)['taskId']

15
automation/taskcluster/lib/util.py

@ -1,15 +0,0 @@
import re
def convert_camel_case_into_kebab_case(string):
# Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501
first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string)
return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower()
def lower_case_first_letter(string):
return '{}{}'.format(string[0].lower(), string[1:])
def upper_case_first_letter(string):
return string[0].upper() + string[1:]

34
automation/taskcluster/lib/variant.py

@ -1,34 +0,0 @@
import taskcluster
class VariantApk:
def __init__(self, build_type, abi, engine, file_name):
self.abi = abi
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
class Variant:
def __init__(self, name, build_type, apks):
self.name = name
self.build_type = build_type
self._apks = apks
def get_apk(self, abi):
return [apk for apk in self._apks if apk.abi == abi][0]
def artifacts(self):
return {
apk.taskcluster_path: {
'type': 'file',
'path': apk.absolute_path,
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
} for apk in self._apks
}
def upstream_artifacts(self):
return [apk.taskcluster_path for apk in self._apks]
@property
def upstream_artifacts_per_abi(self):
return {apk.abi: apk.taskcluster_path for apk in self._apks}

105
automation/taskcluster/schedule_nightly_graph.py

@ -1,105 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse
import datetime
import jsone
import os
import slugid
import taskcluster
import yaml
from git import Repo
from lib.tasks import schedule_task
ROOT = os.path.join(os.path.dirname(__file__), '../..')
class InvalidGithubRepositoryError(Exception):
pass
def calculate_git_references(root):
repo = Repo(root)
remote = repo.remote()
branch = repo.head.reference
if not remote.url.startswith('https://github.com'):
raise InvalidGithubRepositoryError('expected remote to be a GitHub repository (accessed via HTTPs)')
html_url = remote.url[:-4] if remote.url.endswith('.git') else remote.url
return html_url, str(branch), str(branch.commit)
def make_decision_task(params):
"""Generate a basic decision task, based on the root .taskcluster.yml"""
with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f:
taskcluster_yml = yaml.safe_load(f)
slugids = {}
def as_slugid(name):
if name not in slugids:
slugids[name] = slugid.nice()
return slugids[name]
repository_parts = params['html_url'].split('/')
repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1]))
# provide a similar JSON-e context to what taskcluster-github provides
context = {
'tasks_for': 'cron',
'cron': {
'task_id': params['cron_task_id'],
'name': params['name'],
},
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
'as_slugid': as_slugid,
'event': {
'repository': {
'html_url': params['html_url'],
'full_name': repository_full_name,
},
'release': {
'tag_name': params['head_rev'],
'target_commitish': params['branch'],
},
'sender': {
'login': 'TaskclusterHook',
}
}
}
rendered = jsone.render(taskcluster_yml, context)
if len(rendered['tasks']) != 1:
raise Exception('Expected .taskcluster.yml to only produce one cron task')
task = rendered['tasks'][0]
task_id = task.pop('taskId')
return task_id, task
def schedule():
parser = argparse.ArgumentParser(
description='Creates and submit a graph of tasks on Taskcluster.'
)
parser.add_argument('name', choices=['nightly', 'raptor'])
result = parser.parse_args()
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
html_url, branch, head_rev = calculate_git_references(ROOT)
params = {
'html_url': html_url,
'head_rev': head_rev,
'branch': branch,
'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
'name': result.name,
}
decision_task_id, decision_task = make_decision_task(params)
schedule_task(queue, decision_task_id, decision_task)
print('All scheduled!')
if __name__ == '__main__':
schedule()

108
taskcluster/ci/build/kind.yml

@ -0,0 +1,108 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.transforms.build:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
description: Build Fenix from source code.
treeherder:
kind: build
symbol: B
platform: android-all/opt
tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 7200
chain-of-trust: true
run:
using: gradlew
use-caches: false
run-on-tasks-for: []
# Builds generate multiple APKs with different ABIs. For each APK described
# by `gradlew printVariant`, an artifact will be generated. `variant` and
# the per-apk config from `printVariant` can be used as substitutions in
# `name` and `path`.
apk-artifact-template:
type: file
name: public/build/{abi}/{geckoview_engine}/target.apk
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
jobs:
debug:
attributes:
code-review: true
run-on-tasks-for: [github-pull-request, github-push]
run:
geckoview-engine: geckoNightly
gradle-build-type: debug
treeherder:
symbol: debug(B)
performance-test:
run:
geckoview-engine: geckoNightly
gradle-build-type: forPerformanceTest
treeherder:
symbol: forPerformanceTest(B)
nightly:
attributes:
nightly: true
include-nightly-version: true
include-shippable-secrets: true
run:
geckoview-engine: geckoNightly
gradle-build-type: fenixNightly
treeherder:
symbol: nightly(B)
nightly-legacy:
attributes:
nightly: true
include-nightly-version: true
include-shippable-secrets: true
run:
geckoview-engine: geckoNightly
gradle-build-type: fenixNightlyLegacy
treeherder:
symbol: nightlyLegacy(B)
fennec-production:
attributes:
nightly: true
include-shippable-secrets: true
run:
geckoview-engine: geckoBeta
gradle-build-type: fennecProduction
treeherder:
symbol: nightlyFennec(B)
beta:
attributes:
build-type: production
release-type: beta
include-shippable-secrets: true
run:
geckoview-engine: geckoBeta
gradle-build-type: fenixBeta
run-on-tasks-for: [github-release]
treeherder:
symbol: beta(B)
production:
attributes:
release-type: production
include-shippable-secrets: true
run:
geckoview-engine: geckoBeta
gradle-build-type: fenixProduction
run-on-tasks-for: [github-release]
treeherder:
symbol: production(B)

2
taskcluster/ci/config.yml

@ -2,6 +2,8 @@
trust-domain: mobile
treeherder:
group-names:
'beta': 'Nightly-related tasks'
'debug': 'Builds made for testing'
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
'I': 'Docker Image Builds'
'nightly': 'Nightly-related tasks'

7
taskcluster/ci/nimbledroid/kind.yml

@ -5,12 +5,11 @@
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.nimbledroid:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
- build
job-defaults:
description: Upload APKs to Nimbledroid for performance measurement and tracking
@ -37,7 +36,7 @@ jobs:
nightly:
attributes:
nightly: true
dependencies:
build: Build FenixNightlyLegacy task # Comes from the old decision task
treeherder:
symbol: nimbledroid
dependencies:
build: build-nightly-legacy

2
taskcluster/ci/pr/kind.yml

@ -5,7 +5,7 @@
loader: taskgraph.loader.transform:loader
kind-dependencies:
- old-decision
- build
- test
transforms:

18
taskcluster/ci/push-apk/kind.yml

@ -5,7 +5,8 @@
loader: fenix_taskgraph.loader.single_dep:loader
transforms:
- fenix_taskgraph.pushapk:transforms
- fenix_taskgraph.transforms.single_dep:transforms
- fenix_taskgraph.transforms.push_apk:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
@ -21,6 +22,21 @@ job-template:
worker-type: push-apk
worker:
commit: true
channel:
by-build-type:
nightly: nightly
nightly-legacy: production
production: production
dep:
by-level:
'3': false
default: true
google-play-track:
by-build-type:
nightly-legacy: nightly
default: ''
product: fenix
treeherder:
job-symbol: gp
kind: build

6
taskcluster/ci/raptor/kind.yml

@ -1,14 +1,16 @@
---
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.raptor:transforms
- fenix_taskgraph.transforms.raptor:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- signing
only-for-build-types:
- performance-test
only-for-abis:
- armeabi-v7a
- arm64-v8a
@ -49,8 +51,6 @@ job-defaults:
- name: public/test_info/
path: workspace/build/blobber_upload_dir
type: directory
run:
using: run-task
checkout: false

115
taskcluster/ci/signing/kind.yml

@ -2,101 +2,54 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
loader: fenix_taskgraph.loader.single_dep:loader
transforms:
- fenix_taskgraph.signing:transforms
- fenix_taskgraph.transforms.single_dep:transforms
- fenix_taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
- build
job-defaults:
job-template:
description: Sign Fenix
worker-type:
by-variant:
by-build-type:
(fennec-production|nightly|nightly-legacy|production):
by-level:
'3': signing
default: dep-signing
default: dep-signing
worker:
signing-type:
by-build-type:
fennec-production:
by-level:
'3': fennec-production-signing
default: dep-signing
nightly:
by-level:
'3': nightly-signing
default: dep-signing
nightly-legacy:
by-level:
'3': production-signing
default: dep-signing
performance-test: dep-signing
production:
by-level:
'3': production-signing
default: dep-signing
default: dep-signing
index:
by-build-type:
(fennec-production|nightly|performance-test|production):
type: signing
default: {}
run-on-tasks-for: []
treeherder:
job-symbol: Bs
kind: build
tier: 2
platform: android-all/opt
jobs:
performance-test:
attributes:
build-type: performance-test
index:
type: signing
worker:
signing-type: dep-signing
dependencies:
build: 'assemble: geckoNightlyForPerformanceTest' # comes from the old-decision task
treeherder:
symbol: forPerformanceTest(s)
production:
attributes:
build-type: production
index:
type: signing
worker:
signing-type:
by-level:
'3': production-signing
default: dep-signing
dependencies:
build: 'Build FenixProduction task' # comes from the old-decision task
run-on-tasks-for: [github-release]
treeherder:
symbol: production(s)
fennec-production:
attributes:
build-type: fennec-production
nightly: true
index:
type: signing
worker:
signing-type:
by-level:
'3': fennec-production-signing
default: dep-signing
dependencies:
build: 'Build FennecProduction task' # comes from the old-decision task
treeherder:
symbol: nightlyFennec(s)
nightly:
attributes:
build-type: nightly
nightly: true
index:
type: signing
worker:
signing-type:
by-level:
'3': nightly-signing
default: dep-signing
dependencies:
build: 'Build FenixNightly task' # comes from the old-decision task
treeherder:
symbol: nightly(s)
nightly-legacy:
attributes:
build-type: nightly-legacy
nightly: true
worker:
signing-type:
by-level:
'3': production-signing
default: dep-signing
dependencies:
build: 'Build FenixNightlyLegacy task' # comes from the old-decision task
treeherder:
symbol: nightlyLegacy(s)
tier: 2

18
taskcluster/ci/test/kind.yml

@ -15,12 +15,30 @@ job-defaults:
tier: 2
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 7200
run:
using: gradlew
use-caches: false
jobs:
debug:
attributes:
build-type: debug
geckoview-engine: geckoNightly
code-review: true
treeherder:
platform: 'android-all/opt'
symbol: debug(T)
tier: 1
run:
gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport']
post-gradlew:
- ['automation/taskcluster/upload_coverage_report.sh']
secrets:
- name: project/mobile/fenix/public-tokens
key: codecov
path: .cc_token
ui:
attributes:
build-type: debug

21
taskcluster/docker/base/Dockerfile

@ -19,12 +19,12 @@ WORKDIR /builds/worker/
#-- Configuration -----------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------------------
ENV LANG en_US.UTF-8
# Do not use fancy output on taskcluster
ENV TERM dumb
ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false
ENV ANDROID_SDK_VERSION='3859397' \
ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
LANG='en_US.UTF-8' \
TERM='dumb' \
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
#----------------------------------------------------------------------------------------------------------------------
#-- System ------------------------------------------------------------------------------------------------------------
@ -50,7 +50,14 @@ RUN apt-get update -qq \
RUN pip install --upgrade pip
RUN pip install taskcluster
RUN locale-gen en_US.UTF-8
RUN locale-gen "$LANG"
RUN curl -o "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
&& unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
&& rm "$SDK_ZIP_LOCATION" \
&& yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
&& chown -R worker:worker "$ANDROID_SDK_ROOT"
# %include-run-task

16
automation/taskcluster/lib/gradle.py → taskcluster/fenix_taskgraph/gradle.py

@ -2,23 +2,19 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
from __future__ import absolute_import, print_function, unicode_literals
import json
import subprocess
from ..lib.variant import Variant, VariantApk
from taskgraph.util.memoize import memoize
@memoize
def get_variant(build_type, engine):
print("Fetching variant information for build_type='{}', engine='{}'".format(build_type, engine))
output = _run_gradle_process('printVariant', variantBuildType=build_type, variantEngine=engine)
content = _extract_content_from_command_output(output, prefix='variant: ')
raw_variant = json.loads(content)
return Variant(
raw_variant['name'],
build_type,
[VariantApk(build_type, raw_apk['abi'], engine, raw_apk['fileName']) for raw_apk in raw_variant['apks']]
)
return json.loads(content)
def _run_gradle_process(gradle_command, **kwargs):
@ -32,7 +28,7 @@ def _run_gradle_process(gradle_command, **kwargs):
exit_code = process.wait()
if exit_code is not 0:
print("Gradle command returned error: {}".format(exit_code))
raise RuntimeError("Gradle command returned error: {}".format(exit_code))
return output

3
taskcluster/fenix_taskgraph/job.py

@ -15,6 +15,7 @@ from pipes import quote as shell_quote
gradlew_schema = Schema(
{
Required("using"): "gradlew",
Optional("pre-gradlew"): [[text_type]],
Required("gradlew"): [text_type],
Optional("post-gradlew"): [[text_type]],
# Base work directory used to set up the task.
@ -53,7 +54,7 @@ def configure_gradlew(config, job, taskdesc):
def _extract_command(run):
pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]]
pre_gradle_commands = run.pop("pre-gradlew", [])
pre_gradle_commands += [
_generate_secret_command(secret) for secret in run.get("secrets", [])
]

81
taskcluster/fenix_taskgraph/loader/old_decision.py

@ -1,81 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import datetime
import os
import re
import sys
current_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
sys.path.append(project_dir)
from automation.taskcluster.decision_task import (
pr,
push,
raptor,
nightly_to_production_app,
release,
release_as_fennec,
)
from automation.taskcluster.lib.tasks import TaskBuilder
def loader(kind, path, config, params, loaded_tasks):
repo_url = params['head_repository']
commit = params['head_rev']
trust_level = int(params['level'])
builder = TaskBuilder(
task_id=os.environ.get('TASK_ID'),
repo_url=repo_url,
git_ref=params['head_ref'],
short_head_branch=params['head_ref'],
commit=commit,
owner=params['owner'],
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
scheduler_id='mobile-level-{}'.format(trust_level),
tasks_priority='highest', # TODO parametrize
date_string=params['moz_build_date'],
trust_level=trust_level,
)
is_staging = trust_level != 3
tasks_for = params['tasks_for']
if tasks_for == 'github-pull-request':
ordered_groups_of_tasks = pr(builder)
elif tasks_for == 'github-push':
ordered_groups_of_tasks = push(builder)
elif tasks_for == 'github-release':
git_tag = os.environ['GIT_TAG']
version = git_tag[1:] # remove prefixed "v"
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
if beta_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
elif production_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
elif tasks_for == 'cron':
target_tasks_method = params['target_tasks_method']
if target_tasks_method == 'raptor':
ordered_groups_of_tasks = raptor(builder, is_staging)
elif target_tasks_method == 'nightly':
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
nightly_version = 'Nightly {}'.format(now)
ordered_groups_of_tasks = release(builder, 'nightly', 'geckoNightly', is_staging, nightly_version) \
+ nightly_to_production_app(builder, is_staging, nightly_version)
ordered_groups_of_tasks += release_as_fennec(builder, is_staging, 'Signed-as-Fennec Nightly {}'.format(now))
else:
raise NotImplementedError('Unsupported task_name "{}"'.format(params))
else:
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
for task in ordered_groups_of_tasks:
yield task

27
taskcluster/fenix_taskgraph/nimbledroid.py

@ -1,27 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the nimbledroid
kind. This transform won't be necessary anymore once the full tasgraph migration is
done
"""
from __future__ import absolute_import, print_function, unicode_literals
import copy
import json
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.util.schema import resolve_keyed_by
transforms = TransformSequence()
@transforms.add
def filter_out_non_nightly_legacy(config, tasks):
for dep_task in config.kind_dependencies_tasks:
if dep_task.label == 'Build FenixNightlyLegacy task':
for task in tasks:
yield task

75
taskcluster/fenix_taskgraph/pushapk.py

@ -1,75 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
transforms = TransformSequence()
_CHANNEL_PER_TASK_NAME = {
'nightly': 'nightly',
'nightly-legacy': 'production',
'production': 'production',
}
@transforms.add
def build_name_and_attributes(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["dependencies"] = {"signing": dep.label}
task.setdefault("attributes", {}).update(dep.attributes.copy())
task["name"] = _get_dependent_job_name_without_its_kind(dep)
yield task
def _get_dependent_job_name_without_its_kind(dependent_job):
return dependent_job.label[len(dependent_job.kind) + 1:]
@transforms.add
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
treeherder_symbol = join_symbol(treeherder_group, 'gp')
task["treeherder"]["symbol"] = treeherder_symbol
yield task
@transforms.add
def build_worker_definition(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task_name = task["name"]
worker_definition = {}
worker_definition["upstream-artifacts"] = [{
"taskId": {"task-reference": "<signing>"},
"taskType": "signing",
"paths": dep.attributes["apks"].values(),
}]
worker_definition["dep"] = config.params["level"] != "3"
worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name]
# Fenix production doesn't follow the rule {product}-{channel}
worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \
"{}-{}".format(task["worker"]["product"], worker_definition["channel"])
if task_name == 'nightly-legacy':
worker_definition["google-play-track"] = 'nightly'
task["worker"].update(worker_definition)
yield task

67
taskcluster/fenix_taskgraph/signing.py

@ -1,67 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
transforms = TransformSequence()
# TODO remove this transform once build tasks are migrated to taskgraph
@transforms.add
def fetch_old_decision_dependency(config, tasks):
for task in tasks:
for dep_task in config.kind_dependencies_tasks:
expected_signing_dep_label = task['dependencies']['build']
if dep_task.label != expected_signing_dep_label:
continue
task['primary-dependency'] = dep_task
yield task
@transforms.add
def define_signing_attributes(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
attributes = task.setdefault("attributes", {})
upstream_attributes = dep.attributes.copy()
upstream_attributes.update(attributes)
task["attributes"] = upstream_attributes
task["attributes"]["signed"] = True
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
for key in ("worker-type", "worker.signing-type"):
resolve_keyed_by(
task,
key,
item_name=task["name"],
variant=task["attributes"]["build-type"],
level=config.params["level"],
)
yield task
@transforms.add
def build_signing_task(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task["dependencies"] = {"build": dep.label}
task["worker"]["upstream-artifacts"] = [
{
"taskId": {"task-reference": "<build>"},
"taskType": "build",
"paths": dep.attributes["apks"].values(),
"formats": ["autograph_apk"],
}
]
yield task

44
taskcluster/fenix_taskgraph/target_tasks.py

@ -4,22 +4,38 @@
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.target_tasks import _target_task, standard_filter
import os
import re
from taskgraph.target_tasks import _target_task, filter_for_tasks_for
BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
@_target_task('default')
def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
return [l for l, t in full_task_graph.tasks.iteritems() if _old_decision_filter(t, parameters)]
def _old_decision_filter(task, parameters):
if task.kind == 'old-decision':
return True
filter = filter_for_tasks_for
if parameters["tasks_for"] == 'github-release':
# TODO Move GIT_TAG as to a parameter
git_tag = os.environ['GIT_TAG']
version = git_tag[1:] # remove prefixed "v"
if BETA_SEMVER.match(git_tag):
def filter(task, params):
return task.attributes.get("release-type", "") == "beta"
elif PRODUCTION_SEMVER.match(git_tag):
def filter(task, params):
return task.attributes.get("release-type", "") == "production"
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
return standard_filter(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
@_target_task("nightly")
@ -27,20 +43,14 @@ def target_tasks_nightly(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build."""
def filter(task, parameters):
if task.attributes.get("nightly", False):
return True
return _old_decision_filter(task, parameters)
return task.attributes.get("nightly", False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
@_target_task('raptor')
def target_tasks_raptor(full_task_graph, parameters, graph_config):
def filter(t, params):
if t.kind == 'raptor':
return True
return _old_decision_filter(t, params)
def filter(task, params):
return task.kind == 'raptor'
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]

0
taskcluster/fenix_taskgraph/transforms/__init__.py

128
taskcluster/fenix_taskgraph/transforms/build.py

@ -0,0 +1,128 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
from __future__ import absolute_import, print_function, unicode_literals
import datetime
from taskgraph.transforms.base import TransformSequence
from fenix_taskgraph.gradle import get_variant
from fenix_taskgraph.util import upper_case_first_letter
transforms = TransformSequence()
@transforms.add