diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 9a3133353..ea2e35138 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -67,28 +67,17 @@ def raptor(builder, is_staging): def release(builder, channel, engine, is_staging, version_name): variant = get_variant('fenix' + channel.capitalize(), engine) - taskcluster_apk_paths = variant.upstream_artifacts() - build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) signing_task = builder.craft_release_signing_task( build_task['label'], - taskcluster_apk_paths, + variant, channel=channel, is_staging=is_staging, ) - push_task = builder.craft_push_task( - signing_task['label'], - taskcluster_apk_paths, - channel=channel, - variant=variant, - # TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track - override_google_play_track=None if channel != "nightly" else "internal", - is_staging=is_staging, - ) - - return [build_task, signing_task, push_task] + # The push-apk task is generated by taskgraph + return [build_task, signing_task] def release_as_fennec(builder, is_staging, version_name): @@ -98,7 +87,7 @@ def release_as_fennec(builder, is_staging, version_name): build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) signing_task = builder.craft_release_signing_task( build_task['label'], - variant.upstream_artifacts(), + variant, channel, variant, is_staging, @@ -119,22 +108,14 @@ def nightly_to_production_app(builder, is_staging, version_name): signing_task = builder.craft_release_signing_task( build_task['label'], - taskcluster_apk_paths, + variant, channel='production', # Since we're publishing to the "production" app, we need to sign for production is_staging=is_staging, publish_to_index=False, ) - push_task = builder.craft_push_task( - signing_task['label'], - taskcluster_apk_paths, - channel='production', # We're publishing to the "production" app on the "nightly" track - variant=variant, - override_google_play_track='nightly', - is_staging=is_staging, - ) - - tasks = [build_task, signing_task, push_task] + # The push-apk task is generated by taskgraph + tasks = [build_task, signing_task] if not is_staging: tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label'])) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index ec941bdeb..0e2572d31 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -495,7 +495,7 @@ class TaskBuilder(object): ) def craft_release_signing_task( - self, build_task_id, apk_paths, channel, is_staging, publish_to_index=True + self, build_task_id, variant, channel, is_staging, publish_to_index=True ): if publish_to_index: staging_prefix = '.staging' if is_staging else '' @@ -517,60 +517,25 @@ class TaskBuilder(object): description="Sign {} builds of Fenix".format(capitalized_channel), signing_type="dep" if is_staging else channel, assemble_task_label=build_task_id, - apk_paths=apk_paths, + apk_paths=variant.upstream_artifacts(), routes=routes, treeherder={ 'jobKind': 'other', 'machine': { 'platform': 'android-all', }, + 'collection': { + 'opt': True, + }, 'symbol': '{}-s'.format(channel), 'tier': 1, }, - ) - - def craft_push_task( - self, signing_task_label, apk_paths, channel, variant, is_staging=False, override_google_play_track=None, - ): - payload = { - "commit": True, - "channel": channel, - "certificate_alias": 'fenix' if is_staging else 'fenix-{}'.format(channel), - "upstreamArtifacts": [ - { - "paths": apk_paths, - "taskId": {'task-reference': ''}, - "taskType": "signing" - } - ] - } - - if override_google_play_track: - payload['google_play_track'] = override_google_play_track - - return self._craft_default_task_definition( - worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1', - provisioner_id='scriptworker-prov-v1', - dependencies={'signing': signing_task_label}, - routes=[], - scopes=[ - "project:mobile:fenix:releng:googleplay:product:fenix{}".format( - ':dep' if is_staging else '' - ) - ], - name="Push task {}".format(variant.name), - description="Upload signed release builds of Fenix to Google Play", - payload=payload, - treeherder={ - 'jobKind': 'other', - 'machine': { - 'platform': 'android-all', - }, - 'symbol': '{}-gp'.format(channel), - 'tier': 1, + attributes={ + 'apks': variant.upstream_artifacts_per_abi, }, ) + def schedule_task(queue, taskId, task): print("TASK", taskId) print(json.dumps(task, indent=4, separators=(',', ': '))) diff --git a/automation/taskcluster/lib/variant.py b/automation/taskcluster/lib/variant.py index b3392c500..a2a8a23c2 100644 --- a/automation/taskcluster/lib/variant.py +++ b/automation/taskcluster/lib/variant.py @@ -4,7 +4,7 @@ import taskcluster class VariantApk: def __init__(self, build_type, abi, engine, file_name): self.abi = abi - self.taskcluster_path = 'public/build/{}/{}/target.apk'.format(abi, engine) + self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine) self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name) diff --git a/taskcluster/ci/push-apk/kind.yml b/taskcluster/ci/push-apk/kind.yml new file mode 100644 index 000000000..7a8ef02f6 --- /dev/null +++ b/taskcluster/ci/push-apk/kind.yml @@ -0,0 +1,56 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - fenix_taskgraph.pushapk:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - old-decision + +job-defaults: + description: Publish Fenix + worker-type: push-apk + worker: + commit: true + product: fenix + run-on-tasks-for: [] + treeherder: + kind: build + tier: 1 + +jobs: + release: + dependencies: + signing: 'Signing Production task' # comes from the old-decision task + worker: + channel: production + certificate-alias: fenix + run-on-tasks-for: [github-release] + treeherder: + symbol: release-gp + + nightly: + attributes: + nightly: true + dependencies: + signing: 'Signing Nightly task' # comes from the old-decision task + worker: + channel: nightly + google-play-track: internal + treeherder: + symbol: nightly-gp + + nightly-legacy: + attributes: + nightly: true + dependencies: + signing: 'Signing Production task' # comes from the old-decision task + worker: + channel: production + google-play-track: nightly + treeherder: + symbol: production-gp diff --git a/taskcluster/fenix_taskgraph/__init__.py b/taskcluster/fenix_taskgraph/__init__.py index 21bfd6c13..da3941992 100644 --- a/taskcluster/fenix_taskgraph/__init__.py +++ b/taskcluster/fenix_taskgraph/__init__.py @@ -12,7 +12,7 @@ def register(graph_config): Import all modules that are siblings of this one, triggering decorators in the process. """ - _import_modules(["job", "worker_types", "routes", "target", "target_tasks"]) + _import_modules(["job", "worker_types", "routes", "target_tasks"]) def _import_modules(modules): diff --git a/taskcluster/fenix_taskgraph/pushapk.py b/taskcluster/fenix_taskgraph/pushapk.py new file mode 100644 index 000000000..a931e8d7c --- /dev/null +++ b/taskcluster/fenix_taskgraph/pushapk.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the build +kind. +""" + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep + + +transforms = TransformSequence() + + +# TODO remove this transform once signing task are migrated to taskgraph +@transforms.add +def fetch_old_decision_dependency(config, tasks): + for task in tasks: + for dep_task in config.kind_dependencies_tasks: + expected_signing_dep_label = task['dependencies']['signing'] + if dep_task.label != expected_signing_dep_label: + continue + + task['primary-dependency'] = dep_task + yield task + + +@transforms.add +def build_pushapk_task(config, tasks): + for task in tasks: + dep = task.pop("primary-dependency") + task.setdefault("attributes", {}).update(dep.attributes.copy()) + if "run_on_tasks_for" in task["attributes"]: + task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"] + + task["treeherder"] = inherit_treeherder_from_dep(task, dep) + task["worker"]["upstream-artifacts"] = [{ + "taskId": {"task-reference": ""}, + "taskType": "signing", + "paths": dep.attributes["apks"].values(), + }] + task["worker"]["dep"] = config.params["level"] != "3" + if not task["worker"].get("certificate-alias"): + task["worker"]["certificate-alias"] = "{}-{}".format( + task["worker"]["product"], task["worker"]["channel"] + ) + + yield task diff --git a/taskcluster/fenix_taskgraph/target.py b/taskcluster/fenix_taskgraph/target.py deleted file mode 100644 index e7a23a8a4..000000000 --- a/taskcluster/fenix_taskgraph/target.py +++ /dev/null @@ -1,17 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -from taskgraph.target_tasks import _target_task as target_task - - -@target_task("nightly") -def target_tasks_nightly(full_task_graph, parameters, graph_config): - """Select the set of tasks required for a nightly build.""" - - def filter(task, parameters): - return task.attributes.get("nightly", False) - - return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index 0e9de3ec9..a2ad210d1 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -22,10 +22,17 @@ def _old_decision_filter(task, parameters): return standard_filter(task, parameters) -@_target_task('nightly') -def target_tasks_raptor(full_task_graph, parameters, graph_config): - # TODO Change this target task method once old-decision loader is no more - return target_tasks_default(full_task_graph, parameters, graph_config) +@_target_task("nightly") +def target_tasks_nightly(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build.""" + + def filter(task, parameters): + if task.attributes.get("nightly", False): + return True + + return _old_decision_filter(task, parameters) + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] @_target_task('raptor') diff --git a/taskcluster/fenix_taskgraph/worker_types.py b/taskcluster/fenix_taskgraph/worker_types.py index 3ab708a5e..705934c25 100644 --- a/taskcluster/fenix_taskgraph/worker_types.py +++ b/taskcluster/fenix_taskgraph/worker_types.py @@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals from six import text_type -from voluptuous import Required +from voluptuous import Required, Optional from taskgraph.util.schema import taskref_or_string from taskgraph.transforms.task import payload_builder @@ -69,8 +69,10 @@ def build_scriptworker_signing_payload(config, task, task_def): Required("paths"): [text_type], } ], + Required("certificate-alias"): text_type, Required("channel"): text_type, Required("commit"): bool, + Optional("google-play-track"): text_type, Required("product"): text_type, Required("dep"): bool, }, @@ -81,11 +83,15 @@ def build_push_apk_payload(config, task, task_def): task_def["tags"]["worker-implementation"] = "scriptworker" task_def["payload"] = { + "certificate_alias": worker["certificate-alias"], + "channel": worker["channel"], "commit": worker["commit"], "upstreamArtifacts": worker["upstream-artifacts"], - "channel": worker["channel"], } + if worker.get("google-play-track"): + task_def["payload"]["google_play_track"] = worker["google-play-track"] + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] task_def["scopes"].append( "{}:googleplay:product:{}{}".format(