Bug 1580778 - Migrate push-apk tasks to taskgraph (#5377)
parent
4462899763
commit
39da162dc9
|
@ -67,28 +67,17 @@ def raptor(builder, is_staging):
|
||||||
|
|
||||||
def release(builder, channel, engine, is_staging, version_name):
|
def release(builder, channel, engine, is_staging, version_name):
|
||||||
variant = get_variant('fenix' + channel.capitalize(), engine)
|
variant = get_variant('fenix' + channel.capitalize(), engine)
|
||||||
taskcluster_apk_paths = variant.upstream_artifacts()
|
|
||||||
|
|
||||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||||
|
|
||||||
signing_task = builder.craft_release_signing_task(
|
signing_task = builder.craft_release_signing_task(
|
||||||
build_task['label'],
|
build_task['label'],
|
||||||
taskcluster_apk_paths,
|
variant,
|
||||||
channel=channel,
|
channel=channel,
|
||||||
is_staging=is_staging,
|
is_staging=is_staging,
|
||||||
)
|
)
|
||||||
|
|
||||||
push_task = builder.craft_push_task(
|
# The push-apk task is generated by taskgraph
|
||||||
signing_task['label'],
|
return [build_task, signing_task]
|
||||||
taskcluster_apk_paths,
|
|
||||||
channel=channel,
|
|
||||||
variant=variant,
|
|
||||||
# TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track
|
|
||||||
override_google_play_track=None if channel != "nightly" else "internal",
|
|
||||||
is_staging=is_staging,
|
|
||||||
)
|
|
||||||
|
|
||||||
return [build_task, signing_task, push_task]
|
|
||||||
|
|
||||||
|
|
||||||
def release_as_fennec(builder, is_staging, version_name):
|
def release_as_fennec(builder, is_staging, version_name):
|
||||||
|
@ -98,7 +87,7 @@ def release_as_fennec(builder, is_staging, version_name):
|
||||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||||
signing_task = builder.craft_release_signing_task(
|
signing_task = builder.craft_release_signing_task(
|
||||||
build_task['label'],
|
build_task['label'],
|
||||||
variant.upstream_artifacts(),
|
variant,
|
||||||
channel,
|
channel,
|
||||||
variant,
|
variant,
|
||||||
is_staging,
|
is_staging,
|
||||||
|
@ -119,22 +108,14 @@ def nightly_to_production_app(builder, is_staging, version_name):
|
||||||
|
|
||||||
signing_task = builder.craft_release_signing_task(
|
signing_task = builder.craft_release_signing_task(
|
||||||
build_task['label'],
|
build_task['label'],
|
||||||
taskcluster_apk_paths,
|
variant,
|
||||||
channel='production', # Since we're publishing to the "production" app, we need to sign for production
|
channel='production', # Since we're publishing to the "production" app, we need to sign for production
|
||||||
is_staging=is_staging,
|
is_staging=is_staging,
|
||||||
publish_to_index=False,
|
publish_to_index=False,
|
||||||
)
|
)
|
||||||
|
|
||||||
push_task = builder.craft_push_task(
|
# The push-apk task is generated by taskgraph
|
||||||
signing_task['label'],
|
tasks = [build_task, signing_task]
|
||||||
taskcluster_apk_paths,
|
|
||||||
channel='production', # We're publishing to the "production" app on the "nightly" track
|
|
||||||
variant=variant,
|
|
||||||
override_google_play_track='nightly',
|
|
||||||
is_staging=is_staging,
|
|
||||||
)
|
|
||||||
|
|
||||||
tasks = [build_task, signing_task, push_task]
|
|
||||||
if not is_staging:
|
if not is_staging:
|
||||||
tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label']))
|
tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label']))
|
||||||
|
|
||||||
|
|
|
@ -495,7 +495,7 @@ class TaskBuilder(object):
|
||||||
)
|
)
|
||||||
|
|
||||||
def craft_release_signing_task(
|
def craft_release_signing_task(
|
||||||
self, build_task_id, apk_paths, channel, is_staging, publish_to_index=True
|
self, build_task_id, variant, channel, is_staging, publish_to_index=True
|
||||||
):
|
):
|
||||||
if publish_to_index:
|
if publish_to_index:
|
||||||
staging_prefix = '.staging' if is_staging else ''
|
staging_prefix = '.staging' if is_staging else ''
|
||||||
|
@ -517,60 +517,25 @@ class TaskBuilder(object):
|
||||||
description="Sign {} builds of Fenix".format(capitalized_channel),
|
description="Sign {} builds of Fenix".format(capitalized_channel),
|
||||||
signing_type="dep" if is_staging else channel,
|
signing_type="dep" if is_staging else channel,
|
||||||
assemble_task_label=build_task_id,
|
assemble_task_label=build_task_id,
|
||||||
apk_paths=apk_paths,
|
apk_paths=variant.upstream_artifacts(),
|
||||||
routes=routes,
|
routes=routes,
|
||||||
treeherder={
|
treeherder={
|
||||||
'jobKind': 'other',
|
'jobKind': 'other',
|
||||||
'machine': {
|
'machine': {
|
||||||
'platform': 'android-all',
|
'platform': 'android-all',
|
||||||
},
|
},
|
||||||
|
'collection': {
|
||||||
|
'opt': True,
|
||||||
|
},
|
||||||
'symbol': '{}-s'.format(channel),
|
'symbol': '{}-s'.format(channel),
|
||||||
'tier': 1,
|
'tier': 1,
|
||||||
},
|
},
|
||||||
)
|
attributes={
|
||||||
|
'apks': variant.upstream_artifacts_per_abi,
|
||||||
def craft_push_task(
|
|
||||||
self, signing_task_label, apk_paths, channel, variant, is_staging=False, override_google_play_track=None,
|
|
||||||
):
|
|
||||||
payload = {
|
|
||||||
"commit": True,
|
|
||||||
"channel": channel,
|
|
||||||
"certificate_alias": 'fenix' if is_staging else 'fenix-{}'.format(channel),
|
|
||||||
"upstreamArtifacts": [
|
|
||||||
{
|
|
||||||
"paths": apk_paths,
|
|
||||||
"taskId": {'task-reference': '<signing>'},
|
|
||||||
"taskType": "signing"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
if override_google_play_track:
|
|
||||||
payload['google_play_track'] = override_google_play_track
|
|
||||||
|
|
||||||
return self._craft_default_task_definition(
|
|
||||||
worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
|
|
||||||
provisioner_id='scriptworker-prov-v1',
|
|
||||||
dependencies={'signing': signing_task_label},
|
|
||||||
routes=[],
|
|
||||||
scopes=[
|
|
||||||
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(
|
|
||||||
':dep' if is_staging else ''
|
|
||||||
)
|
|
||||||
],
|
|
||||||
name="Push task {}".format(variant.name),
|
|
||||||
description="Upload signed release builds of Fenix to Google Play",
|
|
||||||
payload=payload,
|
|
||||||
treeherder={
|
|
||||||
'jobKind': 'other',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'android-all',
|
|
||||||
},
|
|
||||||
'symbol': '{}-gp'.format(channel),
|
|
||||||
'tier': 1,
|
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def schedule_task(queue, taskId, task):
|
def schedule_task(queue, taskId, task):
|
||||||
print("TASK", taskId)
|
print("TASK", taskId)
|
||||||
print(json.dumps(task, indent=4, separators=(',', ': ')))
|
print(json.dumps(task, indent=4, separators=(',', ': ')))
|
||||||
|
|
|
@ -4,7 +4,7 @@ import taskcluster
|
||||||
class VariantApk:
|
class VariantApk:
|
||||||
def __init__(self, build_type, abi, engine, file_name):
|
def __init__(self, build_type, abi, engine, file_name):
|
||||||
self.abi = abi
|
self.abi = abi
|
||||||
self.taskcluster_path = 'public/build/{}/{}/target.apk'.format(abi, engine)
|
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
|
||||||
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
|
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
---
|
||||||
|
loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
- fenix_taskgraph.pushapk:transforms
|
||||||
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
|
kind-dependencies:
|
||||||
|
- old-decision
|
||||||
|
|
||||||
|
job-defaults:
|
||||||
|
description: Publish Fenix
|
||||||
|
worker-type: push-apk
|
||||||
|
worker:
|
||||||
|
commit: true
|
||||||
|
product: fenix
|
||||||
|
run-on-tasks-for: []
|
||||||
|
treeherder:
|
||||||
|
kind: build
|
||||||
|
tier: 1
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
release:
|
||||||
|
dependencies:
|
||||||
|
signing: 'Signing Production task' # comes from the old-decision task
|
||||||
|
worker:
|
||||||
|
channel: production
|
||||||
|
certificate-alias: fenix
|
||||||
|
run-on-tasks-for: [github-release]
|
||||||
|
treeherder:
|
||||||
|
symbol: release-gp
|
||||||
|
|
||||||
|
nightly:
|
||||||
|
attributes:
|
||||||
|
nightly: true
|
||||||
|
dependencies:
|
||||||
|
signing: 'Signing Nightly task' # comes from the old-decision task
|
||||||
|
worker:
|
||||||
|
channel: nightly
|
||||||
|
google-play-track: internal
|
||||||
|
treeherder:
|
||||||
|
symbol: nightly-gp
|
||||||
|
|
||||||
|
nightly-legacy:
|
||||||
|
attributes:
|
||||||
|
nightly: true
|
||||||
|
dependencies:
|
||||||
|
signing: 'Signing Production task' # comes from the old-decision task
|
||||||
|
worker:
|
||||||
|
channel: production
|
||||||
|
google-play-track: nightly
|
||||||
|
treeherder:
|
||||||
|
symbol: production-gp
|
|
@ -12,7 +12,7 @@ def register(graph_config):
|
||||||
Import all modules that are siblings of this one, triggering decorators in
|
Import all modules that are siblings of this one, triggering decorators in
|
||||||
the process.
|
the process.
|
||||||
"""
|
"""
|
||||||
_import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
|
_import_modules(["job", "worker_types", "routes", "target_tasks"])
|
||||||
|
|
||||||
|
|
||||||
def _import_modules(modules):
|
def _import_modules(modules):
|
||||||
|
|
|
@ -0,0 +1,51 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
"""
|
||||||
|
Apply some defaults and minor modifications to the jobs defined in the build
|
||||||
|
kind.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
from taskgraph.transforms.base import TransformSequence
|
||||||
|
from taskgraph.util.treeherder import inherit_treeherder_from_dep
|
||||||
|
|
||||||
|
|
||||||
|
transforms = TransformSequence()
|
||||||
|
|
||||||
|
|
||||||
|
# TODO remove this transform once signing task are migrated to taskgraph
|
||||||
|
@transforms.add
|
||||||
|
def fetch_old_decision_dependency(config, tasks):
|
||||||
|
for task in tasks:
|
||||||
|
for dep_task in config.kind_dependencies_tasks:
|
||||||
|
expected_signing_dep_label = task['dependencies']['signing']
|
||||||
|
if dep_task.label != expected_signing_dep_label:
|
||||||
|
continue
|
||||||
|
|
||||||
|
task['primary-dependency'] = dep_task
|
||||||
|
yield task
|
||||||
|
|
||||||
|
|
||||||
|
@transforms.add
|
||||||
|
def build_pushapk_task(config, tasks):
|
||||||
|
for task in tasks:
|
||||||
|
dep = task.pop("primary-dependency")
|
||||||
|
task.setdefault("attributes", {}).update(dep.attributes.copy())
|
||||||
|
if "run_on_tasks_for" in task["attributes"]:
|
||||||
|
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
|
||||||
|
|
||||||
|
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
|
||||||
|
task["worker"]["upstream-artifacts"] = [{
|
||||||
|
"taskId": {"task-reference": "<signing>"},
|
||||||
|
"taskType": "signing",
|
||||||
|
"paths": dep.attributes["apks"].values(),
|
||||||
|
}]
|
||||||
|
task["worker"]["dep"] = config.params["level"] != "3"
|
||||||
|
if not task["worker"].get("certificate-alias"):
|
||||||
|
task["worker"]["certificate-alias"] = "{}-{}".format(
|
||||||
|
task["worker"]["product"], task["worker"]["channel"]
|
||||||
|
)
|
||||||
|
|
||||||
|
yield task
|
|
@ -1,17 +0,0 @@
|
||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
from __future__ import absolute_import, print_function, unicode_literals
|
|
||||||
|
|
||||||
from taskgraph.target_tasks import _target_task as target_task
|
|
||||||
|
|
||||||
|
|
||||||
@target_task("nightly")
|
|
||||||
def target_tasks_nightly(full_task_graph, parameters, graph_config):
|
|
||||||
"""Select the set of tasks required for a nightly build."""
|
|
||||||
|
|
||||||
def filter(task, parameters):
|
|
||||||
return task.attributes.get("nightly", False)
|
|
||||||
|
|
||||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
|
|
@ -22,10 +22,17 @@ def _old_decision_filter(task, parameters):
|
||||||
return standard_filter(task, parameters)
|
return standard_filter(task, parameters)
|
||||||
|
|
||||||
|
|
||||||
@_target_task('nightly')
|
@_target_task("nightly")
|
||||||
def target_tasks_raptor(full_task_graph, parameters, graph_config):
|
def target_tasks_nightly(full_task_graph, parameters, graph_config):
|
||||||
# TODO Change this target task method once old-decision loader is no more
|
"""Select the set of tasks required for a nightly build."""
|
||||||
return target_tasks_default(full_task_graph, parameters, graph_config)
|
|
||||||
|
def filter(task, parameters):
|
||||||
|
if task.attributes.get("nightly", False):
|
||||||
|
return True
|
||||||
|
|
||||||
|
return _old_decision_filter(task, parameters)
|
||||||
|
|
||||||
|
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
||||||
|
|
||||||
|
|
||||||
@_target_task('raptor')
|
@_target_task('raptor')
|
||||||
|
|
|
@ -6,7 +6,7 @@ from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
from six import text_type
|
from six import text_type
|
||||||
|
|
||||||
from voluptuous import Required
|
from voluptuous import Required, Optional
|
||||||
|
|
||||||
from taskgraph.util.schema import taskref_or_string
|
from taskgraph.util.schema import taskref_or_string
|
||||||
from taskgraph.transforms.task import payload_builder
|
from taskgraph.transforms.task import payload_builder
|
||||||
|
@ -69,8 +69,10 @@ def build_scriptworker_signing_payload(config, task, task_def):
|
||||||
Required("paths"): [text_type],
|
Required("paths"): [text_type],
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
|
Required("certificate-alias"): text_type,
|
||||||
Required("channel"): text_type,
|
Required("channel"): text_type,
|
||||||
Required("commit"): bool,
|
Required("commit"): bool,
|
||||||
|
Optional("google-play-track"): text_type,
|
||||||
Required("product"): text_type,
|
Required("product"): text_type,
|
||||||
Required("dep"): bool,
|
Required("dep"): bool,
|
||||||
},
|
},
|
||||||
|
@ -81,11 +83,15 @@ def build_push_apk_payload(config, task, task_def):
|
||||||
task_def["tags"]["worker-implementation"] = "scriptworker"
|
task_def["tags"]["worker-implementation"] = "scriptworker"
|
||||||
|
|
||||||
task_def["payload"] = {
|
task_def["payload"] = {
|
||||||
|
"certificate_alias": worker["certificate-alias"],
|
||||||
|
"channel": worker["channel"],
|
||||||
"commit": worker["commit"],
|
"commit": worker["commit"],
|
||||||
"upstreamArtifacts": worker["upstream-artifacts"],
|
"upstreamArtifacts": worker["upstream-artifacts"],
|
||||||
"channel": worker["channel"],
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if worker.get("google-play-track"):
|
||||||
|
task_def["payload"]["google_play_track"] = worker["google-play-track"]
|
||||||
|
|
||||||
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
|
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
|
||||||
task_def["scopes"].append(
|
task_def["scopes"].append(
|
||||||
"{}:googleplay:product:{}{}".format(
|
"{}:googleplay:product:{}{}".format(
|
||||||
|
|
Loading…
Reference in New Issue