diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index ea2e35138..60a696ab9 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -59,25 +59,17 @@ def raptor(builder, is_staging): variant = get_variant('forPerformanceTest', 'geckoNightly') build_task = builder.craft_assemble_raptor_task(variant) - signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging) - # Raptor tests are generated in taskgraph - return [build_task, signing_task] + # Signing and Raptor tasks are generated in taskgraph + return [build_task] def release(builder, channel, engine, is_staging, version_name): variant = get_variant('fenix' + channel.capitalize(), engine) build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - signing_task = builder.craft_release_signing_task( - build_task['label'], - variant, - channel=channel, - is_staging=is_staging, - ) - - # The push-apk task is generated by taskgraph - return [build_task, signing_task] + # The signing push-apk tasks are generated by taskgraph + return [build_task] def release_as_fennec(builder, is_staging, version_name): @@ -85,15 +77,9 @@ def release_as_fennec(builder, is_staging, version_name): channel = 'fennec-production' build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - signing_task = builder.craft_release_signing_task( - build_task['label'], - variant, - channel, - variant, - is_staging, - ) - return [build_task, signing_task] + # The signing task is generated by taskgraph + return [build_task] def nightly_to_production_app(builder, is_staging, version_name): @@ -106,16 +92,8 @@ def nightly_to_production_app(builder, is_staging, version_name): build_task = builder.craft_assemble_release_task( variant, 'nightly-legacy', is_staging, version_name) - signing_task = builder.craft_release_signing_task( - build_task['label'], - variant, - channel='production', # Since we're publishing to the "production" app, we need to sign for production - is_staging=is_staging, - publish_to_index=False, - ) - - # The push-apk task is generated by taskgraph - tasks = [build_task, signing_task] + # The signing and push-apk tasks are generated by taskgraph + tasks = [build_task] if not is_staging: tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label'])) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index acb05163e..3e27155dd 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -97,9 +97,15 @@ class TaskBuilder(object): 'machine': { 'platform': 'android-all', }, + 'collection': { + 'opt': True, + }, 'symbol': '{}-A'.format(variant.build_type), 'tier': 1, }, + attributes={ + 'apks': variant.upstream_artifacts_per_abi, + } ) def craft_assemble_raptor_task(self, variant): @@ -125,7 +131,10 @@ class TaskBuilder(object): 'symbol': 'A', 'tier': 1, }, - attributes={'build-type': 'raptor'}, + attributes={ + 'build-type': 'raptor', + 'apks': variant.upstream_artifacts_per_abi, + }, ) def craft_assemble_pr_task(self, variant): @@ -142,7 +151,7 @@ class TaskBuilder(object): }, 'symbol': 'A', 'tier': 1, - } + }, ) def craft_test_pr_task(self, variant): @@ -367,33 +376,6 @@ class TaskBuilder(object): attributes=attributes, ) - def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder, attributes=None): - signing_format = "autograph_apk" - payload = { - 'upstreamArtifacts': [{ - 'paths': apk_paths, - 'formats': [signing_format], - 'taskId': {'task-reference': ''}, - 'taskType': 'build' - }] - } - - return self._craft_default_task_definition( - worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1', - provisioner_id='scriptworker-prov-v1', - dependencies={'build': assemble_task_label}, - routes=routes, - scopes=[ - "project:mobile:fenix:releng:signing:format:{}".format(signing_format), - "project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type), - ], - name=name, - description=description, - payload=payload, - treeherder=treeherder, - attributes=attributes, - ) - def _craft_default_task_definition( self, worker_type, @@ -457,86 +439,6 @@ class TaskBuilder(object): }, } - def craft_raptor_signing_task( - self, assemble_task_label, variant, is_staging, - ): - staging_prefix = '.staging' if is_staging else '' - routes = [ - "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.latest".format( - staging_prefix, self.date.year, self.date.month, self.date.day - ), - "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.revision.{}".format( - staging_prefix, self.date.year, self.date.month, self.date.day, self.commit - ), - "index.project.mobile.fenix.v2{}.performance-test.latest".format(staging_prefix), - ] - - return self._craft_signing_task( - name='sign: {}'.format('forPerformanceTest'), - description='Dep-signing variant {}'.format('forPerformanceTest'), - signing_type="dep", - assemble_task_label=assemble_task_label, - apk_paths=variant.upstream_artifacts(), - routes=routes, - treeherder={ - 'groupSymbol': 'forPerformanceTest', - 'jobKind': 'other', - 'machine': { - 'platform': 'android-all', - }, - 'collection': { - 'opt': True, - }, - 'symbol': 'As', - 'tier': 1, - }, - attributes={ - 'build-type': 'raptor', - 'apks': variant.upstream_artifacts_per_abi, - }, - ) - - def craft_release_signing_task( - self, build_task_id, variant, channel, is_staging, publish_to_index=True - ): - if publish_to_index: - staging_prefix = '.staging' if is_staging else '' - routes = [ - "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format( - staging_prefix, channel, self.date.year, self.date.month, self.date.day - ), - "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format( - staging_prefix, channel, self.date.year, self.date.month, self.date.day, self.commit - ), - "index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, channel), - ] - else: - routes = [] - - capitalized_channel = upper_case_first_letter(channel) - return self._craft_signing_task( - name="Signing {} task".format(capitalized_channel), - description="Sign {} builds of Fenix".format(capitalized_channel), - signing_type="dep" if is_staging else channel, - assemble_task_label=build_task_id, - apk_paths=variant.upstream_artifacts(), - routes=routes, - treeherder={ - 'jobKind': 'other', - 'machine': { - 'platform': 'android-all', - }, - 'collection': { - 'opt': True, - }, - 'symbol': '{}-s'.format(channel), - 'tier': 1, - }, - attributes={ - 'apks': variant.upstream_artifacts_per_abi, - }, - ) - def schedule_task(queue, taskId, task): print("TASK", taskId) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 5bad2f8cc..7c49a6ccc 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -2,7 +2,11 @@ trust-domain: mobile treeherder: group-names: + 'forPerformanceTest': 'Builds made for Raptor and other performance tests' 'I': 'Docker Image Builds' + 'nightly': 'Nightly-related tasks' + 'nightlyLegacy': 'Nightly-related tasks that ship to https://play.google.com/store/apps/details?id=org.mozilla.fenix' + 'production': 'Release-related tasks' 'Rap': 'Raptor tests' 'Rap-P': 'Raptor power tests' diff --git a/taskcluster/ci/push-apk/kind.yml b/taskcluster/ci/push-apk/kind.yml index 7a8ef02f6..bc769b501 100644 --- a/taskcluster/ci/push-apk/kind.yml +++ b/taskcluster/ci/push-apk/kind.yml @@ -2,55 +2,25 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: taskgraph.loader.transform:loader +loader: fenix_taskgraph.loader.single_dep:loader transforms: - fenix_taskgraph.pushapk:transforms - taskgraph.transforms.task:transforms kind-dependencies: - - old-decision + - signing -job-defaults: +only-for-build-types: + - nightly + - nightly-legacy + - production + +job-template: description: Publish Fenix worker-type: push-apk worker: commit: true product: fenix - run-on-tasks-for: [] treeherder: kind: build - tier: 1 - -jobs: - release: - dependencies: - signing: 'Signing Production task' # comes from the old-decision task - worker: - channel: production - certificate-alias: fenix - run-on-tasks-for: [github-release] - treeherder: - symbol: release-gp - - nightly: - attributes: - nightly: true - dependencies: - signing: 'Signing Nightly task' # comes from the old-decision task - worker: - channel: nightly - google-play-track: internal - treeherder: - symbol: nightly-gp - - nightly-legacy: - attributes: - nightly: true - dependencies: - signing: 'Signing Production task' # comes from the old-decision task - worker: - channel: production - google-play-track: nightly - treeherder: - symbol: production-gp diff --git a/taskcluster/ci/raptor/kind.yml b/taskcluster/ci/raptor/kind.yml index 289fe2bf1..c653e589c 100644 --- a/taskcluster/ci/raptor/kind.yml +++ b/taskcluster/ci/raptor/kind.yml @@ -6,7 +6,7 @@ transforms: - taskgraph.transforms.task:transforms kind-dependencies: - - old-decision + - signing only-for-build-types: - raptor only-for-abis: @@ -28,7 +28,6 @@ job-defaults: armeabi-v7a: android-hw-g5-7-0-arm7-api-16/opt dependencies: geckoview-nightly: geckoview-nightly - signing: 'sign: forPerformanceTest' # comes from the old-decision task worker: max-run-time: 3600 env: diff --git a/taskcluster/ci/signing/kind.yml b/taskcluster/ci/signing/kind.yml new file mode 100644 index 000000000..2ef95f085 --- /dev/null +++ b/taskcluster/ci/signing/kind.yml @@ -0,0 +1,74 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - fenix_taskgraph.signing:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - old-decision + +job-defaults: + description: Sign Fenix + index: + type: signing + worker-type: + by-variant: + (nightly|nightly-legacy|production): + by-level: + '3': signing + default: dep-signing + default: dep-signing + worker: + signing-type: + by-variant: + (nightly|nightly-legacy|production): + by-level: + '3': release-signing + default: dep-signing + default: dep-signing + run-on-tasks-for: [] + treeherder: + kind: build + tier: 2 + platform: android-all/opt + +jobs: + performance-test: + attributes: + build-type: performance-test + worker-type: dep-signing + dependencies: + build: 'assemble: geckoNightlyForPerformanceTest' # comes from the old-decision task + treeherder: + symbol: forPerformanceTest(s) + + production: + attributes: + build-type: production + dependencies: + build: 'Build FenixProduction task' # comes from the old-decision task + run-on-tasks-for: [github-release] + treeherder: + symbol: production(s) + + nightly: + attributes: + build-type: nightly + nightly: true + dependencies: + build: 'Build FenixNightly task' # comes from the old-decision task + treeherder: + symbol: nightly(s) + + nightly-legacy: + attributes: + build-type: nightly-legacy + nightly: true + dependencies: + build: 'Build FenixNightlyLegacy task' # comes from the old-decision task + treeherder: + symbol: nightly-legacy(s) diff --git a/taskcluster/fenix_taskgraph/loader/single_dep.py b/taskcluster/fenix_taskgraph/loader/single_dep.py new file mode 100644 index 000000000..aead8403a --- /dev/null +++ b/taskcluster/fenix_taskgraph/loader/single_dep.py @@ -0,0 +1,52 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy + +from voluptuous import Required + +from taskgraph.task import Task +from taskgraph.util.schema import Schema + +schema = Schema({Required("primary-dependency", "primary dependency task"): Task}) + + +def loader(kind, path, config, params, loaded_tasks): + """ + Load tasks based on the jobs dependant kinds. + + Optional `only-for-attributes` kind configuration, if specified, will limit + the jobs chosen to ones which have the specified attribute, with the specified + value. + + Optional `job-template` kind configuration value, if specified, will be used to + pass configuration down to the specified transforms used. + """ + only_attributes = config.get("only-for-attributes") + only_build_type = config.get('only-for-build-types') + job_template = config.get("job-template") + + for task in loaded_tasks: + if task.kind not in config.get("kind-dependencies", []): + continue + + if only_attributes: + config_attrs = set(only_attributes) + if not config_attrs & set(task.attributes): + # make sure any attribute exists + continue + + if only_build_type: + build_type = task.attributes.get('build-type') + if build_type not in only_build_type: + continue + + job = {"primary-dependency": task} + + if job_template: + job.update(copy.deepcopy(job_template)) + + yield job diff --git a/taskcluster/fenix_taskgraph/pushapk.py b/taskcluster/fenix_taskgraph/pushapk.py index a931e8d7c..9ab65cb00 100644 --- a/taskcluster/fenix_taskgraph/pushapk.py +++ b/taskcluster/fenix_taskgraph/pushapk.py @@ -9,43 +9,72 @@ kind. from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.base import TransformSequence -from taskgraph.util.treeherder import inherit_treeherder_from_dep +from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol transforms = TransformSequence() -# TODO remove this transform once signing task are migrated to taskgraph +_CHANNEL_PER_TASK_NAME = { + 'nightly': 'nightly', + 'nightly-legacy': 'production', + 'production': 'production', +} + +_GOOGLE_PLAY_TRACK_PER_TASK_NAME = { + 'nightly': 'internal', + 'nightly-legacy': 'nightly', +} + + @transforms.add -def fetch_old_decision_dependency(config, tasks): +def build_name_and_attributes(config, tasks): for task in tasks: - for dep_task in config.kind_dependencies_tasks: - expected_signing_dep_label = task['dependencies']['signing'] - if dep_task.label != expected_signing_dep_label: - continue + dep = task["primary-dependency"] + task["dependencies"] = {"signing": dep.label} + task.setdefault("attributes", {}).update(dep.attributes.copy()) + task["name"] = _get_dependent_job_name_without_its_kind(dep) - task['primary-dependency'] = dep_task - yield task + yield task + + +def _get_dependent_job_name_without_its_kind(dependent_job): + return dependent_job.label[len(dependent_job.kind) + 1:] @transforms.add -def build_pushapk_task(config, tasks): +def build_treeherder_definition(config, tasks): + for task in tasks: + dep = task["primary-dependency"] + task["treeherder"] = inherit_treeherder_from_dep(task, dep) + treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"] + treeherder_symbol = join_symbol(treeherder_group, 'gp') + task["treeherder"]["symbol"] = treeherder_symbol + + yield task + + +@transforms.add +def build_worker_definition(config, tasks): for task in tasks: dep = task.pop("primary-dependency") - task.setdefault("attributes", {}).update(dep.attributes.copy()) - if "run_on_tasks_for" in task["attributes"]: - task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"] + task_name = task["name"] - task["treeherder"] = inherit_treeherder_from_dep(task, dep) - task["worker"]["upstream-artifacts"] = [{ + worker_definition = {} + worker_definition["upstream-artifacts"] = [{ "taskId": {"task-reference": ""}, "taskType": "signing", "paths": dep.attributes["apks"].values(), }] - task["worker"]["dep"] = config.params["level"] != "3" - if not task["worker"].get("certificate-alias"): - task["worker"]["certificate-alias"] = "{}-{}".format( - task["worker"]["product"], task["worker"]["channel"] - ) + worker_definition["dep"] = config.params["level"] != "3" + worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name] + # Fenix production doesn't follow the rule {product}-{channel} + worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \ + "{}-{}".format(task["worker"]["product"], worker_definition["channel"]) + + if _GOOGLE_PLAY_TRACK_PER_TASK_NAME.get(task_name): + worker_definition["google-play-track"] = _GOOGLE_PLAY_TRACK_PER_TASK_NAME[task_name] + + task["worker"].update(worker_definition) yield task diff --git a/taskcluster/fenix_taskgraph/raptor.py b/taskcluster/fenix_taskgraph/raptor.py index 4ebb86cb3..f8fab1069 100644 --- a/taskcluster/fenix_taskgraph/raptor.py +++ b/taskcluster/fenix_taskgraph/raptor.py @@ -30,10 +30,6 @@ def add_variants(config, tasks): if build_type not in only_types: continue - # TODO remove this if statement once signing tasks are fully created in taskgraph - if not dep_task.label.startswith('sign: '): - continue - for abi, apk_path in dep_task.attributes["apks"].items(): if abi not in only_abis: continue @@ -52,6 +48,7 @@ def add_variants(config, tasks): def build_raptor_task(config, tasks): for task in tasks: signing = task.pop("primary-dependency") + task["dependencies"]["signing"] = signing.label build_type = task["attributes"]["build-type"] abi = task["attributes"]["abi"] apk = task["attributes"]["apk"] diff --git a/taskcluster/fenix_taskgraph/routes.py b/taskcluster/fenix_taskgraph/routes.py index 00c407cf9..06cc325a7 100644 --- a/taskcluster/fenix_taskgraph/routes.py +++ b/taskcluster/fenix_taskgraph/routes.py @@ -9,9 +9,9 @@ import time from taskgraph.transforms.task import index_builder SIGNING_ROUTE_TEMPLATES = [ - "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}", - "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest", - "index.project.{trust-domain}.{project}.v3.{variant}.latest", + "index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.revision.{head_rev}", + "index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.latest", + "index.project.{trust-domain}.{project}.v2.{variant}.latest", ] diff --git a/taskcluster/fenix_taskgraph/signing.py b/taskcluster/fenix_taskgraph/signing.py new file mode 100644 index 000000000..57e5f0615 --- /dev/null +++ b/taskcluster/fenix_taskgraph/signing.py @@ -0,0 +1,64 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.schema import resolve_keyed_by + + +transforms = TransformSequence() + +# TODO remove this transform once build tasks are migrated to taskgraph +@transforms.add +def fetch_old_decision_dependency(config, tasks): + for task in tasks: + for dep_task in config.kind_dependencies_tasks: + expected_signing_dep_label = task['dependencies']['build'] + if dep_task.label != expected_signing_dep_label: + continue + + task['primary-dependency'] = dep_task + yield task + + +@transforms.add +def define_signing_flags(config, tasks): + for task in tasks: + dep = task["primary-dependency"] + # Current kind will be prepended later in the transform chain. + task.setdefault("attributes", {}).update(dep.attributes.copy()) + task["attributes"]["signed"] = True + if "run_on_tasks_for" in task["attributes"]: + task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"] + + for key in ("worker-type", "worker.signing-type"): + resolve_keyed_by( + task, + key, + item_name=task["name"], + variant=task["attributes"]["build-type"], + level=config.params["level"], + ) + yield task + + +@transforms.add +def build_signing_task(config, tasks): + for task in tasks: + dep = task.pop("primary-dependency") + task["dependencies"] = {"build": dep.label} + task["worker"]["upstream-artifacts"] = [ + { + "taskId": {"task-reference": ""}, + "taskType": "build", + "paths": dep.attributes["apks"].values(), + "formats": ["autograph_apk"], + } + ] + yield task