blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Bug 1580778 - Migrate push-apk tasks to taskgraph (#5377)

master
Johan Lorenzo 2019-09-18 12:49:22 +02:00 committed by GitHub
parent 4462899763
commit 39da162dc9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 143 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
automation/taskcluster/decision_task.py
View File

@ -67,28 +67,17 @@ def raptor(builder, is_staging):
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
taskcluster_apk_paths = variant.upstream_artifacts()
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task = builder.craft_release_signing_task(
build_task['label'],
taskcluster_apk_paths,
variant,
channel=channel,
is_staging=is_staging,
)
push_task = builder.craft_push_task(
signing_task['label'],
taskcluster_apk_paths,
channel=channel,
variant=variant,
# TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track
override_google_play_track=None if channel != "nightly" else "internal",
is_staging=is_staging,
)
return [build_task, signing_task, push_task]
# The push-apk task is generated by taskgraph
return [build_task, signing_task]
def release_as_fennec(builder, is_staging, version_name):
@ -98,7 +87,7 @@ def release_as_fennec(builder, is_staging, version_name):
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task = builder.craft_release_signing_task(
build_task['label'],
variant.upstream_artifacts(),
variant,
channel,
variant,
is_staging,
@ -119,22 +108,14 @@ def nightly_to_production_app(builder, is_staging, version_name):
signing_task = builder.craft_release_signing_task(
build_task['label'],
taskcluster_apk_paths,
variant,
channel='production', # Since we're publishing to the "production" app, we need to sign for production
is_staging=is_staging,
publish_to_index=False,
)
push_task = builder.craft_push_task(
signing_task['label'],
taskcluster_apk_paths,
channel='production', # We're publishing to the "production" app on the "nightly" track
variant=variant,
override_google_play_track='nightly',
is_staging=is_staging,
)
tasks = [build_task, signing_task, push_task]
# The push-apk task is generated by taskgraph
tasks = [build_task, signing_task]
if not is_staging:
tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label']))

51
automation/taskcluster/lib/tasks.py
View File

@ -495,7 +495,7 @@ class TaskBuilder(object):
)
def craft_release_signing_task(
self, build_task_id, apk_paths, channel, is_staging, publish_to_index=True
self, build_task_id, variant, channel, is_staging, publish_to_index=True
):
if publish_to_index:
staging_prefix = '.staging' if is_staging else ''
@ -517,60 +517,25 @@ class TaskBuilder(object):
description="Sign {} builds of Fenix".format(capitalized_channel),
signing_type="dep" if is_staging else channel,
assemble_task_label=build_task_id,
apk_paths=apk_paths,
apk_paths=variant.upstream_artifacts(),
routes=routes,
treeherder={
'jobKind': 'other',
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': '{}-s'.format(channel),
'tier': 1,
},
)
def craft_push_task(
self, signing_task_label, apk_paths, channel, variant, is_staging=False, override_google_play_track=None,
):
payload = {
"commit": True,
"channel": channel,
"certificate_alias": 'fenix' if is_staging else 'fenix-{}'.format(channel),
"upstreamArtifacts": [
{
"paths": apk_paths,
"taskId": {'task-reference': '<signing>'},
"taskType": "signing"
}
]
}
if override_google_play_track:
payload['google_play_track'] = override_google_play_track
return self._craft_default_task_definition(
worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
provisioner_id='scriptworker-prov-v1',
dependencies={'signing': signing_task_label},
routes=[],
scopes=[
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(
':dep' if is_staging else ''
)
],
name="Push task {}".format(variant.name),
description="Upload signed release builds of Fenix to Google Play",
payload=payload,
treeherder={
'jobKind': 'other',
'machine': {
'platform': 'android-all',
},
'symbol': '{}-gp'.format(channel),
'tier': 1,
attributes={
'apks': variant.upstream_artifacts_per_abi,
},
)
def schedule_task(queue, taskId, task):
print("TASK", taskId)
print(json.dumps(task, indent=4, separators=(',', ': ')))

2
automation/taskcluster/lib/variant.py
View File

@ -4,7 +4,7 @@ import taskcluster
class VariantApk:
def __init__(self, build_type, abi, engine, file_name):
self.abi = abi
self.taskcluster_path = 'public/build/{}/{}/target.apk'.format(abi, engine)
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)

56
taskcluster/ci/push-apk/kind.yml 100644
View File

@ -0,0 +1,56 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.pushapk:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
job-defaults:
description: Publish Fenix
worker-type: push-apk
worker:
commit: true
product: fenix
run-on-tasks-for: []
treeherder:
kind: build
tier: 1
jobs:
release:
dependencies:
signing: 'Signing Production task' # comes from the old-decision task
worker:
channel: production
certificate-alias: fenix
run-on-tasks-for: [github-release]
treeherder:
symbol: release-gp
nightly:
attributes:
nightly: true
dependencies:
signing: 'Signing Nightly task' # comes from the old-decision task
worker:
channel: nightly
google-play-track: internal
treeherder:
symbol: nightly-gp
nightly-legacy:
attributes:
nightly: true
dependencies:
signing: 'Signing Production task' # comes from the old-decision task
worker:
channel: production
google-play-track: nightly
treeherder:
symbol: production-gp

2
taskcluster/fenix_taskgraph/__init__.py
View File

@ -12,7 +12,7 @@ def register(graph_config):
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
_import_modules(["job", "worker_types", "routes", "target_tasks"])
def _import_modules(modules):

51
taskcluster/fenix_taskgraph/pushapk.py 100644
View File

@ -0,0 +1,51 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.treeherder import inherit_treeherder_from_dep
transforms = TransformSequence()
# TODO remove this transform once signing task are migrated to taskgraph
@transforms.add
def fetch_old_decision_dependency(config, tasks):
for task in tasks:
for dep_task in config.kind_dependencies_tasks:
expected_signing_dep_label = task['dependencies']['signing']
if dep_task.label != expected_signing_dep_label:
continue
task['primary-dependency'] = dep_task
yield task
@transforms.add
def build_pushapk_task(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task.setdefault("attributes", {}).update(dep.attributes.copy())
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
task["worker"]["upstream-artifacts"] = [{
"taskId": {"task-reference": "<signing>"},
"taskType": "signing",
"paths": dep.attributes["apks"].values(),
}]
task["worker"]["dep"] = config.params["level"] != "3"
if not task["worker"].get("certificate-alias"):
task["worker"]["certificate-alias"] = "{}-{}".format(
task["worker"]["product"], task["worker"]["channel"]
)
yield task

17
taskcluster/fenix_taskgraph/target.py
View File

@ -1,17 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.target_tasks import _target_task as target_task
@target_task("nightly")
def target_tasks_nightly(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build."""
def filter(task, parameters):
return task.attributes.get("nightly", False)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]

15
taskcluster/fenix_taskgraph/target_tasks.py
View File

@ -22,10 +22,17 @@ def _old_decision_filter(task, parameters):
return standard_filter(task, parameters)
@_target_task('nightly')
def target_tasks_raptor(full_task_graph, parameters, graph_config):
# TODO Change this target task method once old-decision loader is no more
return target_tasks_default(full_task_graph, parameters, graph_config)
@_target_task("nightly")
def target_tasks_nightly(full_task_graph, parameters, graph_config):
"""Select the set of tasks required for a nightly build."""
def filter(task, parameters):
if task.attributes.get("nightly", False):
return True
return _old_decision_filter(task, parameters)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
@_target_task('raptor')

10
taskcluster/fenix_taskgraph/worker_types.py
View File

@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
from six import text_type
from voluptuous import Required
from voluptuous import Required, Optional
from taskgraph.util.schema import taskref_or_string
from taskgraph.transforms.task import payload_builder
@ -69,8 +69,10 @@ def build_scriptworker_signing_payload(config, task, task_def):
Required("paths"): [text_type],
}
],
Required("certificate-alias"): text_type,
Required("channel"): text_type,
Required("commit"): bool,
Optional("google-play-track"): text_type,
Required("product"): text_type,
Required("dep"): bool,
},
@ -81,11 +83,15 @@ def build_push_apk_payload(config, task, task_def):
task_def["tags"]["worker-implementation"] = "scriptworker"
task_def["payload"] = {
"certificate_alias": worker["certificate-alias"],
"channel": worker["channel"],
"commit": worker["commit"],
"upstreamArtifacts": worker["upstream-artifacts"],
"channel": worker["channel"],
}
if worker.get("google-play-track"):
task_def["payload"]["google_play_track"] = worker["google-play-track"]
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
task_def["scopes"].append(
"{}:googleplay:product:{}{}".format(