diff --git a/.cron.yml b/.cron.yml index 6db5221a4..24d107e03 100644 --- a/.cron.yml +++ b/.cron.yml @@ -8,7 +8,6 @@ jobs: job: type: decision-task treeherder-symbol: Nd - # TODO change target method once first tasks are migrated off "old-decision" target-tasks-method: nightly when: [{hour: 6, minute: 0}] when: [{hour: 18, minute: 0}] @@ -16,6 +15,5 @@ jobs: job: type: decision-task treeherder-symbol: raptor-D - # TODO change target method once first tasks are migrated off "old-decision" target-tasks-method: raptor when: [{hour: 1, minute: 0}] diff --git a/.gitignore b/.gitignore index 8d93c55d7..698e059b7 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ out/ # Gradle files .gradle/ build/ +!taskcluster/ci/build # Local configuration file (sdk path, etc) local.properties diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py deleted file mode 100644 index 9b042e29d..000000000 --- a/automation/taskcluster/decision_task.py +++ /dev/null @@ -1,87 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -""" -Decision task for nightly releases. -""" - -from __future__ import print_function - -import argparse -import datetime -import os -import re - -import taskcluster - -from lib.gradle import get_variant -from lib.tasks import ( - fetch_mozharness_task_id, - schedule_task_graph, - TaskBuilder, -) -from lib.chain_of_trust import ( - populate_chain_of_trust_task_graph, - populate_chain_of_trust_required_but_unused_files -) - -def pr(builder): - tasks = [] - - variant = get_variant('debug', 'geckoNightly') - tasks.append(builder.craft_assemble_pr_task(variant)) - tasks.append(builder.craft_test_pr_task(variant)) - - for task in tasks: - task['attributes']['code-review'] = True - - return tasks - -def push(builder): - # We want the same tasks on pushes than on PRs, for now. - return pr(builder) - -def raptor(builder, is_staging): - mozharness_task_id = fetch_mozharness_task_id() - gecko_revision = taskcluster.Queue({ - 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), - }).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] - - variant = get_variant('forPerformanceTest', 'geckoNightly') - build_task = builder.craft_assemble_raptor_task(variant) - - # Signing and Raptor tasks are generated in taskgraph - return [build_task] - - -def release(builder, channel, engine, is_staging, version_name): - variant = get_variant('fenix' + channel.capitalize(), engine) - build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - - # The signing push-apk tasks are generated by taskgraph - return [build_task] - - -def release_as_fennec(builder, is_staging, version_name): - variant = get_variant('fennecProduction', 'geckoBeta') - channel = 'fennec-production' - - build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - - # The signing task is generated by taskgraph - return [build_task] - - -def nightly_to_production_app(builder, is_staging, version_name): - # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the - # "nightly" track. We're moving towards having each channel be published to its own app, but we need to - # keep updating this "backwards-compatible" nightly for a while yet - variant = get_variant('fenixNightlyLegacy', 'geckoNightly') - taskcluster_apk_paths = variant.upstream_artifacts() - - build_task = builder.craft_assemble_release_task( - variant, 'nightly-legacy', is_staging, version_name) - - # Nimbledroid, signing and push-apk tasks are generated by taskgraph - return [build_task] diff --git a/automation/taskcluster/lib/chain_of_trust.py b/automation/taskcluster/lib/chain_of_trust.py deleted file mode 100644 index 49e4ac190..000000000 --- a/automation/taskcluster/lib/chain_of_trust.py +++ /dev/null @@ -1,20 +0,0 @@ -import json - - -def populate_chain_of_trust_required_but_unused_files(): - # These files are needed to keep chainOfTrust happy. However, they have no - # need for android-components, at the moment. For more details, see: - # https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585 - - for file_names in ('actions.json', 'parameters.yml'): - with open(file_names, 'w') as f: - json.dump({}, f) # Yaml is a super-set of JSON. - - -def populate_chain_of_trust_task_graph(full_task_graph): - # full_task_graph must follow the format: - # { - # task_id: full_task_definition - # } - with open('task-graph.json', 'w') as f: - json.dump(full_task_graph, f) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index d0538ec4a..917b9c8c1 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -2,360 +2,8 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function - -import arrow -import datetime -import json -import os -import taskcluster - -from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter - -DEFAULT_EXPIRES_IN = '1 year' -DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk' -_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix' -_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task' -GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json' # Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases ARM_RAPTOR_URL_PARAMS = [ "exclude=1,2,9,10,17,18,21,22,26,28,30,32,39,40,47," "48,55,56,63,64,71,72,79,80,83,84,89,90,95,96", ] - - -class TaskBuilder(object): - def __init__( - self, - task_id, - repo_url, - git_ref, - short_head_branch, commit, owner, source, scheduler_id, date_string, - tasks_priority='lowest', - trust_level=1 - ): - self.task_id = task_id - self.repo_url = repo_url - self.git_ref = git_ref - self.short_head_branch = short_head_branch - self.commit = commit - self.owner = owner - self.source = source - self.scheduler_id = scheduler_id - self.trust_level = trust_level - self.tasks_priority = tasks_priority - self.date = arrow.get(date_string, 'YYYYMMDDHHmmss') - self.trust_level = trust_level - - def craft_assemble_release_task(self, variant, channel, is_staging, version_name): - if is_staging: - secret_index = 'garbage/staging/project/mobile/fenix' - else: - secret_index = 'project/mobile/fenix/{}'.format(channel) - - pre_gradle_commands = ( - 'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format( - secret_index, key, target_file - ) - for key, target_file in ( - ('sentry_dsn', '.sentry_token'), - ('leanplum', '.leanplum_token'), - ('adjust', '.adjust_token'), - ('digital_asset_links', '.digital_asset_links_token'), - ('firebase', 'app/src/{}/res/values/firebase.xml'.format(variant.build_type)), - ) - ) - - capitalized_build_type = upper_case_first_letter(variant.build_type) - gradle_commands = ( - './gradlew --no-daemon -PversionName="{}" clean test assemble{}'.format( - version_name, capitalized_build_type), - ) - - command = ' && '.join( - cmd - for commands in (pre_gradle_commands, gradle_commands) - for cmd in commands - if cmd - ) - - routes = [] if is_staging else [ - "notify.email.fenix-eng-notifications@mozilla.com.on-failed" - ] - - return self._craft_build_ish_task( - name='Build {} task'.format(capitalized_build_type), - description='Build Fenix {} from source code'.format(capitalized_build_type), - command=command, - scopes=[ - "secrets:get:{}".format(secret_index) - ], - artifacts=variant.artifacts(), - routes=routes, - treeherder={ - 'jobKind': 'build', - 'machine': { - 'platform': 'android-all', - }, - 'collection': { - 'opt': True, - }, - 'symbol': '{}-A'.format(variant.build_type), - 'tier': 1, - }, - attributes={ - 'apks': variant.upstream_artifacts_per_abi, - } - ) - - def craft_assemble_raptor_task(self, variant): - command = ' && '.join(( - 'echo "https://fake@sentry.prod.mozaws.net/368" > .sentry_token', - 'echo "--" > .adjust_token', - 'echo "-:-" > .leanplum_token', - 'touch .digital_asset_links_token', - './gradlew --no-daemon clean assemble{}'.format(variant.name), - )) - - return self._craft_build_ish_task( - name='assemble: {}'.format(variant.name), - description='Building and testing variant {}'.format(variant.name), - command=command, - artifacts=variant.artifacts(), - treeherder={ - 'groupSymbol': variant.build_type, - 'jobKind': 'build', - 'machine': { - 'platform': 'android-all', - }, - 'symbol': 'A', - 'tier': 1, - }, - attributes={ - 'build-type': 'raptor', - 'apks': variant.upstream_artifacts_per_abi, - }, - ) - - def craft_assemble_pr_task(self, variant): - return self._craft_clean_gradle_task( - name='assemble: {}'.format(variant.name), - description='Building and testing variant {}'.format(variant.name), - gradle_task='assemble{}'.format(variant.name), - artifacts=variant.artifacts(), - treeherder={ - 'groupSymbol': variant.build_type, - 'jobKind': 'build', - 'machine': { - 'platform': 'android-all', - }, - 'symbol': 'A', - 'tier': 1, - }, - ) - - def craft_test_pr_task(self, variant): - # upload coverage only once, if the variant is arm64 - secret_index = 'project/mobile/fenix/public-tokens' - pre_gradle_commands = ( - 'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format( - secret_index, key, target_file - ) - for key, target_file in ( - ('codecov', '.cc_token'), - ) - ) - - gradle_commands = ( - './gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport', - ) - - post_gradle_commands = ( - 'automation/taskcluster/upload_coverage_report.sh', - ) - - command = ' && '.join( - cmd - for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands) - for cmd in commands - if cmd - ) - - return self._craft_build_ish_task( - name='test: {}'.format(variant.name), - description='Building and testing variant {}'.format(variant.name), - command=command, - treeherder={ - 'groupSymbol': variant.build_type, - 'jobKind': 'test', - 'machine': { - 'platform': 'android-all', - }, - 'symbol': 'T', - 'tier': 1, - }, - scopes=[ - 'secrets:get:{}'.format(secret_index) - ] - ) - - def _craft_clean_gradle_task( - self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None - ): - return self._craft_build_ish_task( - name=name, - description=description, - command='./gradlew --no-daemon clean {}'.format(gradle_task), - artifacts=artifacts, - routes=routes, - treeherder=treeherder, - scopes=scopes, - ) - - def _craft_build_ish_task( - self, name, description, command, dependencies=None, artifacts=None, - routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None - ): - artifacts = {} if artifacts is None else artifacts - scopes = [] if scopes is None else scopes - env_vars = {} if env_vars is None else env_vars - - checkout_command = ' && '.join([ - "export TERM=dumb", - "git fetch {} {}".format(self.repo_url, self.git_ref), - "git config advice.detachedHead false", - "git checkout FETCH_HEAD", - ]) - - command = '{} && {}'.format(checkout_command, command) - - features = {} - if artifacts: - features['chainOfTrust'] = True - if any(scope.startswith('secrets:') for scope in scopes): - features['taskclusterProxy'] = True - payload = { - "features": features, - "env": env_vars, - "maxRunTime": 7200, - "image": "mozillamobile/fenix:1.4", - "command": [ - "/bin/bash", - "--login", - "-cx", - # Some tasks like nimbledroid do have tasks references - {'task-reference': command}, - ], - "artifacts": artifacts, - } - - return self._craft_default_task_definition( - worker_type='mobile-{}-b-fenix'.format(self.trust_level), - provisioner_id='aws-provisioner-v1', - name=name, - description=description, - payload=payload, - dependencies=dependencies, - routes=routes, - scopes=scopes, - treeherder=treeherder, - attributes=attributes, - ) - - def _craft_default_task_definition( - self, - worker_type, - provisioner_id, - name, - description, - payload, - dependencies=None, - routes=None, - scopes=None, - treeherder=None, - notify=None, - attributes=None - ): - dependencies = {} if dependencies is None else dependencies - scopes = [] if scopes is None else scopes - routes = [] if routes is None else routes - treeherder = {} if treeherder is None else treeherder - attributes = {} if attributes is None else attributes - - created = datetime.datetime.now() - deadline = taskcluster.fromNow('1 day') - expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN) - - routes.append('checks') - if self.trust_level == 3: - routes.append('tc-treeherder.v2.fenix.{}'.format(self.commit)) - - extra = { - "treeherder": treeherder, - } - if notify: - extra['notify'] = notify - - return { - "attributes": attributes, - "dependencies": dependencies, - "label": name, - "task": { - "provisionerId": provisioner_id, - "workerType": worker_type, - "taskGroupId": self.task_id, - "schedulerId": self.scheduler_id, - "created": taskcluster.stringDate(created), - "deadline": taskcluster.stringDate(deadline), - "expires": taskcluster.stringDate(expires), - "retries": 5, - "tags": {}, - "priority": self.tasks_priority, - "requires": "all-completed", - "routes": routes, - "scopes": scopes, - "payload": payload, - "extra": extra, - "metadata": { - "name": "Fenix - {}".format(name), - "description": description, - "owner": self.owner, - "source": self.source, - }, - }, - } - - -def schedule_task(queue, taskId, task): - print("TASK", taskId) - print(json.dumps(task, indent=4, separators=(',', ': '))) - - result = queue.createTask(taskId, task) - print("RESULT", taskId) - print(json.dumps(result)) - - -def schedule_task_graph(ordered_groups_of_tasks): - queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) - full_task_graph = {} - - # TODO: Switch to async python to speed up submission - for group_of_tasks in ordered_groups_of_tasks: - for task_id, task_definition in group_of_tasks.items(): - schedule_task(queue, task_id, task_definition) - - full_task_graph[task_id] = { - # Some values of the task definition are automatically filled. Querying the task - # allows to have the full definition. This is needed to make Chain of Trust happy - 'task': queue.task(task_id), - } - - return full_task_graph - - -def fetch_mozharness_task_id(): - # We now want to use the latest available raptor - raptor_index = 'gecko.v2.mozilla-central.nightly.latest.mobile.android-x86_64-opt' - return taskcluster.Index({ - 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), - }).findTask(raptor_index)['taskId'] diff --git a/automation/taskcluster/lib/util.py b/automation/taskcluster/lib/util.py deleted file mode 100644 index 09e5931eb..000000000 --- a/automation/taskcluster/lib/util.py +++ /dev/null @@ -1,15 +0,0 @@ -import re - - -def convert_camel_case_into_kebab_case(string): - # Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501 - first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string) - return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower() - - -def lower_case_first_letter(string): - return '{}{}'.format(string[0].lower(), string[1:]) - - -def upper_case_first_letter(string): - return string[0].upper() + string[1:] diff --git a/automation/taskcluster/lib/variant.py b/automation/taskcluster/lib/variant.py deleted file mode 100644 index a2a8a23c2..000000000 --- a/automation/taskcluster/lib/variant.py +++ /dev/null @@ -1,34 +0,0 @@ -import taskcluster - - -class VariantApk: - def __init__(self, build_type, abi, engine, file_name): - self.abi = abi - self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine) - self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name) - - -class Variant: - def __init__(self, name, build_type, apks): - self.name = name - self.build_type = build_type - self._apks = apks - - def get_apk(self, abi): - return [apk for apk in self._apks if apk.abi == abi][0] - - def artifacts(self): - return { - apk.taskcluster_path: { - 'type': 'file', - 'path': apk.absolute_path, - 'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')), - } for apk in self._apks - } - - def upstream_artifacts(self): - return [apk.taskcluster_path for apk in self._apks] - - @property - def upstream_artifacts_per_abi(self): - return {apk.abi: apk.taskcluster_path for apk in self._apks} diff --git a/automation/taskcluster/schedule_nightly_graph.py b/automation/taskcluster/schedule_nightly_graph.py deleted file mode 100644 index 2311f3fd3..000000000 --- a/automation/taskcluster/schedule_nightly_graph.py +++ /dev/null @@ -1,105 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -import argparse -import datetime -import jsone -import os -import slugid -import taskcluster -import yaml - -from git import Repo -from lib.tasks import schedule_task - -ROOT = os.path.join(os.path.dirname(__file__), '../..') - - -class InvalidGithubRepositoryError(Exception): - pass - - -def calculate_git_references(root): - repo = Repo(root) - remote = repo.remote() - branch = repo.head.reference - - if not remote.url.startswith('https://github.com'): - raise InvalidGithubRepositoryError('expected remote to be a GitHub repository (accessed via HTTPs)') - - html_url = remote.url[:-4] if remote.url.endswith('.git') else remote.url - return html_url, str(branch), str(branch.commit) - - -def make_decision_task(params): - """Generate a basic decision task, based on the root .taskcluster.yml""" - with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f: - taskcluster_yml = yaml.safe_load(f) - - slugids = {} - - def as_slugid(name): - if name not in slugids: - slugids[name] = slugid.nice() - return slugids[name] - - repository_parts = params['html_url'].split('/') - repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1])) - - # provide a similar JSON-e context to what taskcluster-github provides - context = { - 'tasks_for': 'cron', - 'cron': { - 'task_id': params['cron_task_id'], - 'name': params['name'], - }, - 'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z', - 'as_slugid': as_slugid, - 'event': { - 'repository': { - 'html_url': params['html_url'], - 'full_name': repository_full_name, - }, - 'release': { - 'tag_name': params['head_rev'], - 'target_commitish': params['branch'], - }, - 'sender': { - 'login': 'TaskclusterHook', - } - } - } - - rendered = jsone.render(taskcluster_yml, context) - if len(rendered['tasks']) != 1: - raise Exception('Expected .taskcluster.yml to only produce one cron task') - task = rendered['tasks'][0] - - task_id = task.pop('taskId') - return task_id, task - - -def schedule(): - parser = argparse.ArgumentParser( - description='Creates and submit a graph of tasks on Taskcluster.' - ) - - parser.add_argument('name', choices=['nightly', 'raptor']) - result = parser.parse_args() - queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) - - html_url, branch, head_rev = calculate_git_references(ROOT) - params = { - 'html_url': html_url, - 'head_rev': head_rev, - 'branch': branch, - 'cron_task_id': os.environ.get('CRON_TASK_ID', ''), - 'name': result.name, - } - decision_task_id, decision_task = make_decision_task(params) - schedule_task(queue, decision_task_id, decision_task) - print('All scheduled!') - - -if __name__ == '__main__': - schedule() diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml new file mode 100644 index 000000000..b41deb5ea --- /dev/null +++ b/taskcluster/ci/build/kind.yml @@ -0,0 +1,108 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - fenix_taskgraph.transforms.build:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: Build Fenix from source code. + treeherder: + kind: build + symbol: B + platform: android-all/opt + tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 + chain-of-trust: true + run: + using: gradlew + use-caches: false + run-on-tasks-for: [] + # Builds generate multiple APKs with different ABIs. For each APK described + # by `gradlew printVariant`, an artifact will be generated. `variant` and + # the per-apk config from `printVariant` can be used as substitutions in + # `name` and `path`. + apk-artifact-template: + type: file + name: public/build/{abi}/{geckoview_engine}/target.apk + path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}' + +jobs: + debug: + attributes: + code-review: true + run-on-tasks-for: [github-pull-request, github-push] + run: + geckoview-engine: geckoNightly + gradle-build-type: debug + treeherder: + symbol: debug(B) + + performance-test: + run: + geckoview-engine: geckoNightly + gradle-build-type: forPerformanceTest + treeherder: + symbol: forPerformanceTest(B) + + nightly: + attributes: + nightly: true + include-nightly-version: true + include-shippable-secrets: true + run: + geckoview-engine: geckoNightly + gradle-build-type: fenixNightly + treeherder: + symbol: nightly(B) + + nightly-legacy: + attributes: + nightly: true + include-nightly-version: true + include-shippable-secrets: true + run: + geckoview-engine: geckoNightly + gradle-build-type: fenixNightlyLegacy + treeherder: + symbol: nightlyLegacy(B) + + fennec-production: + attributes: + nightly: true + include-shippable-secrets: true + run: + geckoview-engine: geckoBeta + gradle-build-type: fennecProduction + treeherder: + symbol: nightlyFennec(B) + + beta: + attributes: + build-type: production + release-type: beta + include-shippable-secrets: true + run: + geckoview-engine: geckoBeta + gradle-build-type: fenixBeta + run-on-tasks-for: [github-release] + treeherder: + symbol: beta(B) + + production: + attributes: + release-type: production + include-shippable-secrets: true + run: + geckoview-engine: geckoBeta + gradle-build-type: fenixProduction + run-on-tasks-for: [github-release] + treeherder: + symbol: production(B) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 8ea82e63a..41406b77e 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -2,6 +2,8 @@ trust-domain: mobile treeherder: group-names: + 'beta': 'Nightly-related tasks' + 'debug': 'Builds made for testing' 'forPerformanceTest': 'Builds made for Raptor and other performance tests' 'I': 'Docker Image Builds' 'nightly': 'Nightly-related tasks' diff --git a/taskcluster/ci/nimbledroid/kind.yml b/taskcluster/ci/nimbledroid/kind.yml index 408c2b2c0..c916e7c9c 100644 --- a/taskcluster/ci/nimbledroid/kind.yml +++ b/taskcluster/ci/nimbledroid/kind.yml @@ -5,12 +5,11 @@ loader: taskgraph.loader.transform:loader transforms: - - fenix_taskgraph.nimbledroid:transforms - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms kind-dependencies: - - old-decision + - build job-defaults: description: Upload APKs to Nimbledroid for performance measurement and tracking @@ -37,7 +36,7 @@ jobs: nightly: attributes: nightly: true - dependencies: - build: Build FenixNightlyLegacy task # Comes from the old decision task treeherder: symbol: nimbledroid + dependencies: + build: build-nightly-legacy diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml index 36d7e1927..f7d882c8a 100644 --- a/taskcluster/ci/pr/kind.yml +++ b/taskcluster/ci/pr/kind.yml @@ -5,7 +5,7 @@ loader: taskgraph.loader.transform:loader kind-dependencies: - - old-decision + - build - test transforms: diff --git a/taskcluster/ci/push-apk/kind.yml b/taskcluster/ci/push-apk/kind.yml index bc769b501..01fb3568c 100644 --- a/taskcluster/ci/push-apk/kind.yml +++ b/taskcluster/ci/push-apk/kind.yml @@ -5,7 +5,8 @@ loader: fenix_taskgraph.loader.single_dep:loader transforms: - - fenix_taskgraph.pushapk:transforms + - fenix_taskgraph.transforms.single_dep:transforms + - fenix_taskgraph.transforms.push_apk:transforms - taskgraph.transforms.task:transforms kind-dependencies: @@ -21,6 +22,21 @@ job-template: worker-type: push-apk worker: commit: true + channel: + by-build-type: + nightly: nightly + nightly-legacy: production + production: production + dep: + by-level: + '3': false + default: true + google-play-track: + by-build-type: + nightly-legacy: nightly + default: '' product: fenix + treeherder: + job-symbol: gp kind: build diff --git a/taskcluster/ci/raptor/kind.yml b/taskcluster/ci/raptor/kind.yml index 6c0b4927a..f49beaff3 100644 --- a/taskcluster/ci/raptor/kind.yml +++ b/taskcluster/ci/raptor/kind.yml @@ -1,14 +1,16 @@ --- loader: taskgraph.loader.transform:loader transforms: - - fenix_taskgraph.raptor:transforms + - fenix_taskgraph.transforms.raptor:transforms - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms kind-dependencies: - signing + only-for-build-types: - performance-test + only-for-abis: - armeabi-v7a - arm64-v8a @@ -49,8 +51,6 @@ job-defaults: - name: public/test_info/ path: workspace/build/blobber_upload_dir type: directory - - run: using: run-task checkout: false diff --git a/taskcluster/ci/signing/kind.yml b/taskcluster/ci/signing/kind.yml index 39dc4ca34..3dafe677b 100644 --- a/taskcluster/ci/signing/kind.yml +++ b/taskcluster/ci/signing/kind.yml @@ -2,101 +2,54 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: taskgraph.loader.transform:loader +loader: fenix_taskgraph.loader.single_dep:loader transforms: - - fenix_taskgraph.signing:transforms + - fenix_taskgraph.transforms.single_dep:transforms + - fenix_taskgraph.transforms.signing:transforms - taskgraph.transforms.task:transforms kind-dependencies: - - old-decision + - build -job-defaults: +job-template: description: Sign Fenix worker-type: - by-variant: + by-build-type: (fennec-production|nightly|nightly-legacy|production): by-level: '3': signing default: dep-signing default: dep-signing + worker: + signing-type: + by-build-type: + fennec-production: + by-level: + '3': fennec-production-signing + default: dep-signing + nightly: + by-level: + '3': nightly-signing + default: dep-signing + nightly-legacy: + by-level: + '3': production-signing + default: dep-signing + performance-test: dep-signing + production: + by-level: + '3': production-signing + default: dep-signing + default: dep-signing + index: + by-build-type: + (fennec-production|nightly|performance-test|production): + type: signing + default: {} run-on-tasks-for: [] treeherder: + job-symbol: Bs kind: build - tier: 2 platform: android-all/opt - -jobs: - performance-test: - attributes: - build-type: performance-test - index: - type: signing - worker: - signing-type: dep-signing - dependencies: - build: 'assemble: geckoNightlyForPerformanceTest' # comes from the old-decision task - treeherder: - symbol: forPerformanceTest(s) - - production: - attributes: - build-type: production - index: - type: signing - worker: - signing-type: - by-level: - '3': production-signing - default: dep-signing - dependencies: - build: 'Build FenixProduction task' # comes from the old-decision task - run-on-tasks-for: [github-release] - treeherder: - symbol: production(s) - - fennec-production: - attributes: - build-type: fennec-production - nightly: true - index: - type: signing - worker: - signing-type: - by-level: - '3': fennec-production-signing - default: dep-signing - dependencies: - build: 'Build FennecProduction task' # comes from the old-decision task - treeherder: - symbol: nightlyFennec(s) - - nightly: - attributes: - build-type: nightly - nightly: true - index: - type: signing - worker: - signing-type: - by-level: - '3': nightly-signing - default: dep-signing - dependencies: - build: 'Build FenixNightly task' # comes from the old-decision task - treeherder: - symbol: nightly(s) - - nightly-legacy: - attributes: - build-type: nightly-legacy - nightly: true - worker: - signing-type: - by-level: - '3': production-signing - default: dep-signing - dependencies: - build: 'Build FenixNightlyLegacy task' # comes from the old-decision task - treeherder: - symbol: nightlyLegacy(s) + tier: 2 diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index 0fa11c5cf..c47ef11c3 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -15,12 +15,30 @@ job-defaults: tier: 2 worker-type: b-android worker: + docker-image: {in-tree: base} max-run-time: 7200 run: using: gradlew use-caches: false jobs: + debug: + attributes: + build-type: debug + geckoview-engine: geckoNightly + code-review: true + treeherder: + platform: 'android-all/opt' + symbol: debug(T) + tier: 1 + run: + gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport'] + post-gradlew: + - ['automation/taskcluster/upload_coverage_report.sh'] + secrets: + - name: project/mobile/fenix/public-tokens + key: codecov + path: .cc_token ui: attributes: build-type: debug diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile index 93fac01c9..1eb02870d 100644 --- a/taskcluster/docker/base/Dockerfile +++ b/taskcluster/docker/base/Dockerfile @@ -19,12 +19,12 @@ WORKDIR /builds/worker/ #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV LANG en_US.UTF-8 - -# Do not use fancy output on taskcluster -ENV TERM dumb - -ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false +ENV ANDROID_SDK_VERSION='3859397' \ + ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \ + GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ + LANG='en_US.UTF-8' \ + TERM='dumb' \ + SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -50,7 +50,14 @@ RUN apt-get update -qq \ RUN pip install --upgrade pip RUN pip install taskcluster -RUN locale-gen en_US.UTF-8 +RUN locale-gen "$LANG" + +RUN curl -o "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \ + && unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \ + && rm "$SDK_ZIP_LOCATION" \ + && yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \ + && chown -R worker:worker "$ANDROID_SDK_ROOT" + # %include-run-task diff --git a/automation/taskcluster/lib/gradle.py b/taskcluster/fenix_taskgraph/gradle.py similarity index 69% rename from automation/taskcluster/lib/gradle.py rename to taskcluster/fenix_taskgraph/gradle.py index 2b496e59a..fe53744ff 100644 --- a/automation/taskcluster/lib/gradle.py +++ b/taskcluster/fenix_taskgraph/gradle.py @@ -2,23 +2,19 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. -from __future__ import print_function +from __future__ import absolute_import, print_function, unicode_literals + import json import subprocess -from ..lib.variant import Variant, VariantApk +from taskgraph.util.memoize import memoize +@memoize def get_variant(build_type, engine): - print("Fetching variant information for build_type='{}', engine='{}'".format(build_type, engine)) output = _run_gradle_process('printVariant', variantBuildType=build_type, variantEngine=engine) content = _extract_content_from_command_output(output, prefix='variant: ') - raw_variant = json.loads(content) - return Variant( - raw_variant['name'], - build_type, - [VariantApk(build_type, raw_apk['abi'], engine, raw_apk['fileName']) for raw_apk in raw_variant['apks']] - ) + return json.loads(content) def _run_gradle_process(gradle_command, **kwargs): @@ -32,7 +28,7 @@ def _run_gradle_process(gradle_command, **kwargs): exit_code = process.wait() if exit_code is not 0: - print("Gradle command returned error: {}".format(exit_code)) + raise RuntimeError("Gradle command returned error: {}".format(exit_code)) return output diff --git a/taskcluster/fenix_taskgraph/job.py b/taskcluster/fenix_taskgraph/job.py index cc5b97aa5..80abc9aa2 100644 --- a/taskcluster/fenix_taskgraph/job.py +++ b/taskcluster/fenix_taskgraph/job.py @@ -15,6 +15,7 @@ from pipes import quote as shell_quote gradlew_schema = Schema( { Required("using"): "gradlew", + Optional("pre-gradlew"): [[text_type]], Required("gradlew"): [text_type], Optional("post-gradlew"): [[text_type]], # Base work directory used to set up the task. @@ -53,7 +54,7 @@ def configure_gradlew(config, job, taskdesc): def _extract_command(run): - pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]] + pre_gradle_commands = run.pop("pre-gradlew", []) pre_gradle_commands += [ _generate_secret_command(secret) for secret in run.get("secrets", []) ] diff --git a/taskcluster/fenix_taskgraph/loader/old_decision.py b/taskcluster/fenix_taskgraph/loader/old_decision.py deleted file mode 100644 index 30cb553d5..000000000 --- a/taskcluster/fenix_taskgraph/loader/old_decision.py +++ /dev/null @@ -1,81 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import print_function, unicode_literals - -import datetime -import os -import re -import sys - -current_dir = os.path.dirname(os.path.realpath(__file__)) -project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..')) -sys.path.append(project_dir) - -from automation.taskcluster.decision_task import ( - pr, - push, - raptor, - nightly_to_production_app, - release, - release_as_fennec, -) -from automation.taskcluster.lib.tasks import TaskBuilder - - -def loader(kind, path, config, params, loaded_tasks): - repo_url = params['head_repository'] - commit = params['head_rev'] - trust_level = int(params['level']) - - builder = TaskBuilder( - task_id=os.environ.get('TASK_ID'), - repo_url=repo_url, - git_ref=params['head_ref'], - short_head_branch=params['head_ref'], - commit=commit, - owner=params['owner'], - source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit), - scheduler_id='mobile-level-{}'.format(trust_level), - tasks_priority='highest', # TODO parametrize - date_string=params['moz_build_date'], - trust_level=trust_level, - ) - - is_staging = trust_level != 3 - - tasks_for = params['tasks_for'] - if tasks_for == 'github-pull-request': - ordered_groups_of_tasks = pr(builder) - elif tasks_for == 'github-push': - ordered_groups_of_tasks = push(builder) - elif tasks_for == 'github-release': - git_tag = os.environ['GIT_TAG'] - version = git_tag[1:] # remove prefixed "v" - beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') - production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') - if beta_semver.match(git_tag): - ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version) - elif production_semver.match(git_tag): - ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version) - else: - raise ValueError('Github tag must be in semver format and prefixed with a "v", ' - 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') - elif tasks_for == 'cron': - target_tasks_method = params['target_tasks_method'] - if target_tasks_method == 'raptor': - ordered_groups_of_tasks = raptor(builder, is_staging) - elif target_tasks_method == 'nightly': - now = datetime.datetime.now().strftime('%y%m%d %H:%M') - nightly_version = 'Nightly {}'.format(now) - ordered_groups_of_tasks = release(builder, 'nightly', 'geckoNightly', is_staging, nightly_version) \ - + nightly_to_production_app(builder, is_staging, nightly_version) - ordered_groups_of_tasks += release_as_fennec(builder, is_staging, 'Signed-as-Fennec Nightly {}'.format(now)) - else: - raise NotImplementedError('Unsupported task_name "{}"'.format(params)) - else: - raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for)) - - for task in ordered_groups_of_tasks: - yield task diff --git a/taskcluster/fenix_taskgraph/nimbledroid.py b/taskcluster/fenix_taskgraph/nimbledroid.py deleted file mode 100644 index edf900cf9..000000000 --- a/taskcluster/fenix_taskgraph/nimbledroid.py +++ /dev/null @@ -1,27 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Apply some defaults and minor modifications to the jobs defined in the nimbledroid -kind. This transform won't be necessary anymore once the full tasgraph migration is -done -""" - -from __future__ import absolute_import, print_function, unicode_literals - -import copy -import json - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.treeherder import inherit_treeherder_from_dep -from taskgraph.util.schema import resolve_keyed_by - -transforms = TransformSequence() - - -@transforms.add -def filter_out_non_nightly_legacy(config, tasks): - for dep_task in config.kind_dependencies_tasks: - if dep_task.label == 'Build FenixNightlyLegacy task': - for task in tasks: - yield task diff --git a/taskcluster/fenix_taskgraph/pushapk.py b/taskcluster/fenix_taskgraph/pushapk.py deleted file mode 100644 index e4646d7e6..000000000 --- a/taskcluster/fenix_taskgraph/pushapk.py +++ /dev/null @@ -1,75 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Apply some defaults and minor modifications to the jobs defined in the build -kind. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol - - -transforms = TransformSequence() - - -_CHANNEL_PER_TASK_NAME = { - 'nightly': 'nightly', - 'nightly-legacy': 'production', - 'production': 'production', -} - - -@transforms.add -def build_name_and_attributes(config, tasks): - for task in tasks: - dep = task["primary-dependency"] - task["dependencies"] = {"signing": dep.label} - task.setdefault("attributes", {}).update(dep.attributes.copy()) - task["name"] = _get_dependent_job_name_without_its_kind(dep) - - yield task - - -def _get_dependent_job_name_without_its_kind(dependent_job): - return dependent_job.label[len(dependent_job.kind) + 1:] - - -@transforms.add -def build_treeherder_definition(config, tasks): - for task in tasks: - dep = task["primary-dependency"] - task["treeherder"] = inherit_treeherder_from_dep(task, dep) - treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"] - treeherder_symbol = join_symbol(treeherder_group, 'gp') - task["treeherder"]["symbol"] = treeherder_symbol - - yield task - - -@transforms.add -def build_worker_definition(config, tasks): - for task in tasks: - dep = task.pop("primary-dependency") - task_name = task["name"] - - worker_definition = {} - worker_definition["upstream-artifacts"] = [{ - "taskId": {"task-reference": ""}, - "taskType": "signing", - "paths": dep.attributes["apks"].values(), - }] - worker_definition["dep"] = config.params["level"] != "3" - worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name] - - # Fenix production doesn't follow the rule {product}-{channel} - worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \ - "{}-{}".format(task["worker"]["product"], worker_definition["channel"]) - - if task_name == 'nightly-legacy': - worker_definition["google-play-track"] = 'nightly' - - task["worker"].update(worker_definition) - yield task diff --git a/taskcluster/fenix_taskgraph/signing.py b/taskcluster/fenix_taskgraph/signing.py deleted file mode 100644 index 0f1400832..000000000 --- a/taskcluster/fenix_taskgraph/signing.py +++ /dev/null @@ -1,67 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. -""" -Apply some defaults and minor modifications to the jobs defined in the build -kind. -""" - -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.transforms.base import TransformSequence -from taskgraph.util.schema import resolve_keyed_by - - -transforms = TransformSequence() - -# TODO remove this transform once build tasks are migrated to taskgraph -@transforms.add -def fetch_old_decision_dependency(config, tasks): - for task in tasks: - for dep_task in config.kind_dependencies_tasks: - expected_signing_dep_label = task['dependencies']['build'] - if dep_task.label != expected_signing_dep_label: - continue - - task['primary-dependency'] = dep_task - yield task - - -@transforms.add -def define_signing_attributes(config, tasks): - for task in tasks: - dep = task["primary-dependency"] - attributes = task.setdefault("attributes", {}) - upstream_attributes = dep.attributes.copy() - upstream_attributes.update(attributes) - task["attributes"] = upstream_attributes - - task["attributes"]["signed"] = True - if "run_on_tasks_for" in task["attributes"]: - task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"] - - for key in ("worker-type", "worker.signing-type"): - resolve_keyed_by( - task, - key, - item_name=task["name"], - variant=task["attributes"]["build-type"], - level=config.params["level"], - ) - yield task - - -@transforms.add -def build_signing_task(config, tasks): - for task in tasks: - dep = task.pop("primary-dependency") - task["dependencies"] = {"build": dep.label} - task["worker"]["upstream-artifacts"] = [ - { - "taskId": {"task-reference": ""}, - "taskType": "build", - "paths": dep.attributes["apks"].values(), - "formats": ["autograph_apk"], - } - ] - yield task diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index a2ad210d1..4fcb227ae 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -4,22 +4,38 @@ from __future__ import absolute_import, print_function, unicode_literals -from taskgraph.target_tasks import _target_task, standard_filter +import os +import re + +from taskgraph.target_tasks import _target_task, filter_for_tasks_for + + +BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') +PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') -# XXX We're overwritting the default target_task while all tasks are ported to taskgraph @_target_task('default') def target_tasks_default(full_task_graph, parameters, graph_config): """Target the tasks which have indicated they should be run on this project via the `run_on_projects` attributes.""" - return [l for l, t in full_task_graph.tasks.iteritems() if _old_decision_filter(t, parameters)] + filter = filter_for_tasks_for + if parameters["tasks_for"] == 'github-release': + # TODO Move GIT_TAG as to a parameter + git_tag = os.environ['GIT_TAG'] + version = git_tag[1:] # remove prefixed "v" -def _old_decision_filter(task, parameters): - if task.kind == 'old-decision': - return True + if BETA_SEMVER.match(git_tag): + def filter(task, params): + return task.attributes.get("release-type", "") == "beta" + elif PRODUCTION_SEMVER.match(git_tag): + def filter(task, params): + return task.attributes.get("release-type", "") == "production" + else: + raise ValueError('Github tag must be in semver format and prefixed with a "v", ' + 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') - return standard_filter(task, parameters) + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] @_target_task("nightly") @@ -27,20 +43,14 @@ def target_tasks_nightly(full_task_graph, parameters, graph_config): """Select the set of tasks required for a nightly build.""" def filter(task, parameters): - if task.attributes.get("nightly", False): - return True - - return _old_decision_filter(task, parameters) + return task.attributes.get("nightly", False) return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] @_target_task('raptor') def target_tasks_raptor(full_task_graph, parameters, graph_config): - def filter(t, params): - if t.kind == 'raptor': - return True - - return _old_decision_filter(t, params) + def filter(task, params): + return task.kind == 'raptor' return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] diff --git a/taskcluster/fenix_taskgraph/transforms/__init__.py b/taskcluster/fenix_taskgraph/transforms/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/fenix_taskgraph/transforms/build.py b/taskcluster/fenix_taskgraph/transforms/build.py new file mode 100644 index 000000000..4c6a9e7e0 --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/build.py @@ -0,0 +1,128 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import datetime + +from taskgraph.transforms.base import TransformSequence +from fenix_taskgraph.gradle import get_variant +from fenix_taskgraph.util import upper_case_first_letter + + +transforms = TransformSequence() + + +@transforms.add +def add_variant_config(config, tasks): + for task in tasks: + attributes = task.setdefault("attributes", {}) + if not attributes.get("build-type"): + attributes["build-type"] = task["name"] + + yield task + + +@transforms.add +def add_shippable_secrets(config, tasks): + for task in tasks: + secrets = task["run"].setdefault("secrets", []) + + if task.pop("include-shippable-secrets", False) and config.params["level"] == "3": + build_type = task["attributes"]["build-type"] + secret_index = 'project/mobile/fenix/{}'.format(build_type) + secrets.extend([{ + "key": key, + "name": secret_index, + "path": target_file, + } for key, target_file in ( + ('adjust', '.adjust_token'), + ('firebase', 'app/src/{}/res/values/firebase.xml'.format(build_type)), + ('digital_asset_links', '.digital_asset_links_token'), + ('leanplum', '.leanplum_token'), + ('sentry_dsn', '.sentry_token'), + )]) + else: + task["run"]["pre-gradlew"] = [[ + "echo", '"{}"'.format(fake_value), ">", target_file + ] for fake_value, target_file in ( + ("--", ".adjust_token"), + ("", ".digital_asset_links_token"), + ("-:-", ".leanplum_token"), + ("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"), + )] + + yield task + + +@transforms.add +def build_gradle_command(config, tasks): + for task in tasks: + gradle_build_type = task["run"]["gradle-build-type"] + geckoview_engine = task["run"]["geckoview-engine"] + variant_config = get_variant(gradle_build_type, geckoview_engine) + + task["run"]["gradlew"] = [ + "clean", + "assemble{}".format(upper_case_first_letter(variant_config["name"])) + ] + + yield task + + +@transforms.add +def add_nightly_version(config, tasks): + push_date_string = config.params["moz_build_date"] + push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S") + formated_date_time = 'Nightly {}'.format(push_date_time.strftime('%y%m%d %H:%M')) + + for task in tasks: + if task.pop("include-nightly-version", False): + task["run"]["gradlew"].append('-PversionName="{}"'.format(formated_date_time)) + yield task + + +@transforms.add +def add_release_version(config, tasks): + for task in tasks: + if task.pop("include-release-version", False): + # TODO Move GIT_TAG to a parameter + git_tag = os.environ['GIT_TAG'] + version = git_tag[1:] # remove prefixed "v" + + task["run"]["gradlew"].append('-PversionName="{}"'.format(version)) + yield task + + +@transforms.add +def add_artifacts(config, tasks): + for task in tasks: + gradle_build_type = task["run"].pop("gradle-build-type") + geckoview_engine = task["run"].pop("geckoview-engine") + variant_config = get_variant(gradle_build_type, geckoview_engine) + artifacts = task.setdefault("worker", {}).setdefault("artifacts", []) + task["attributes"]["apks"] = apks = {} + + if "apk-artifact-template" in task: + artifact_template = task.pop("apk-artifact-template") + for apk in variant_config["apks"]: + apk_name = artifact_template["name"].format( + geckoview_engine=geckoview_engine, **apk + ) + artifacts.append({ + "type": artifact_template["type"], + "name": apk_name, + "path": artifact_template["path"].format( + geckoview_engine=geckoview_engine, + gradle_build_type=gradle_build_type, + **apk + ), + }) + apks[apk["abi"]] = apk_name + + yield task diff --git a/taskcluster/fenix_taskgraph/transforms/push_apk.py b/taskcluster/fenix_taskgraph/transforms/push_apk.py new file mode 100644 index 000000000..8ccdcbe9a --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/push_apk.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("worker.channel", "worker.dep", "worker.google-play-track"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + 'build-type': task["attributes"]["build-type"], + 'level': config.params["level"], + } + ) + yield task + + +@transforms.add +def build_worker_definition(config, tasks): + for task in tasks: + worker_definition = {} + worker_definition["certificate-alias"] = "{}-{}".format( + task["worker"]["product"], task["worker"]["channel"] + ) + # Fenix production doesn't follow the rule {product}-{channel} + if task["attributes"]["build-type"] == "production": + worker_definition["certificate-alias"] = "fenix" + + task["worker"].update(worker_definition) + yield task diff --git a/taskcluster/fenix_taskgraph/raptor.py b/taskcluster/fenix_taskgraph/transforms/raptor.py similarity index 100% rename from taskcluster/fenix_taskgraph/raptor.py rename to taskcluster/fenix_taskgraph/transforms/raptor.py diff --git a/taskcluster/fenix_taskgraph/transforms/signing.py b/taskcluster/fenix_taskgraph/transforms/signing.py new file mode 100644 index 000000000..82ece300c --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/signing.py @@ -0,0 +1,46 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + + +transforms = TransformSequence() + + +@transforms.add +def resolve_keys(config, tasks): + for task in tasks: + for key in ("index", "worker-type", "worker.signing-type"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + **{ + 'build-type': task["attributes"]["build-type"], + 'level': config.params["level"], + } + ) + yield task + + +@transforms.add +def set_signing_attributes(config, tasks): + for task in tasks: + task["attributes"]["signed"] = True + yield task + + +@transforms.add +def set_signing_format(config, tasks): + for task in tasks: + for upstream_artifact in task["worker"]["upstream-artifacts"]: + upstream_artifact["formats"] = ["autograph_apk"] + yield task diff --git a/taskcluster/fenix_taskgraph/transforms/single_dep.py b/taskcluster/fenix_taskgraph/transforms/single_dep.py new file mode 100644 index 000000000..ba0f9ddc9 --- /dev/null +++ b/taskcluster/fenix_taskgraph/transforms/single_dep.py @@ -0,0 +1,62 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the single_dep jobs. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol + + +transforms = TransformSequence() + + +@transforms.add +def build_name_and_attributes(config, tasks): + for task in tasks: + dep = task["primary-dependency"] + task["dependencies"] = {dep.kind: dep.label} + copy_of_attributes = dep.attributes.copy() + task.setdefault("attributes", copy_of_attributes) + # run_on_tasks_for is set as an attribute later in the pipeline + task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for']) + task["name"] = _get_dependent_job_name_without_its_kind(dep) + + yield task + + +def _get_dependent_job_name_without_its_kind(dependent_job): + return dependent_job.label[len(dependent_job.kind) + 1:] + + +@transforms.add +def build_upstream_artifacts(config, tasks): + for task in tasks: + dep = task["primary-dependency"] + + worker_definition = {} + worker_definition["upstream-artifacts"] = [{ + "taskId": {"task-reference": "<{}>".format(dep.kind)}, + "taskType": dep.kind, + "paths": sorted(dep.attributes["apks"].values()), + }] + + task["worker"].update(worker_definition) + yield task + + +@transforms.add +def build_treeherder_definition(config, tasks): + for task in tasks: + dep = task.pop("primary-dependency") + + task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep)) + job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?") + job_symbol = task["treeherder"].pop("job-symbol") + full_symbol = join_symbol(job_group, job_symbol) + task["treeherder"]["symbol"] = full_symbol + + yield task diff --git a/taskcluster/ci/old-decision/kind.yml b/taskcluster/fenix_taskgraph/util.py similarity index 55% rename from taskcluster/ci/old-decision/kind.yml rename to taskcluster/fenix_taskgraph/util.py index 04682b195..d613fc2f7 100644 --- a/taskcluster/ci/old-decision/kind.yml +++ b/taskcluster/fenix_taskgraph/util.py @@ -1,9 +1,11 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. ---- -loader: fenix_taskgraph.loader.old_decision:loader -transforms: [] +from __future__ import absolute_import, print_function, unicode_literals -# XXX Everything is done in the loader until everything is migrated +import re + + +def upper_case_first_letter(string): + return string[0].upper() + string[1:]