blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Bug 1580778 - Migrate signing tasks to taskgraph (#5409)

master
Johan Lorenzo 2019-09-19 10:40:05 +02:00 committed by GitHub
parent 1777b5a830
commit bcdc22a2f4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 275 additions and 206 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
automation/taskcluster/decision_task.py
View File

@ -59,25 +59,17 @@ def raptor(builder, is_staging):
variant = get_variant('forPerformanceTest', 'geckoNightly')
build_task = builder.craft_assemble_raptor_task(variant)
signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
# Raptor tests are generated in taskgraph
return [build_task, signing_task]
# Signing and Raptor tasks are generated in taskgraph
return [build_task]
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task = builder.craft_release_signing_task(
build_task['label'],
variant,
channel=channel,
is_staging=is_staging,
)
# The push-apk task is generated by taskgraph
return [build_task, signing_task]
# The signing push-apk tasks are generated by taskgraph
return [build_task]
def release_as_fennec(builder, is_staging, version_name):
@ -85,15 +77,9 @@ def release_as_fennec(builder, is_staging, version_name):
channel = 'fennec-production'
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task = builder.craft_release_signing_task(
build_task['label'],
variant,
channel,
variant,
is_staging,
)
return [build_task, signing_task]
# The signing task is generated by taskgraph
return [build_task]
def nightly_to_production_app(builder, is_staging, version_name):
@ -106,16 +92,8 @@ def nightly_to_production_app(builder, is_staging, version_name):
build_task = builder.craft_assemble_release_task(
variant, 'nightly-legacy', is_staging, version_name)
signing_task = builder.craft_release_signing_task(
build_task['label'],
variant,
channel='production', # Since we're publishing to the "production" app, we need to sign for production
is_staging=is_staging,
publish_to_index=False,
)
# The push-apk task is generated by taskgraph
tasks = [build_task, signing_task]
# The signing and push-apk tasks are generated by taskgraph
tasks = [build_task]
if not is_staging:
tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label']))

120
automation/taskcluster/lib/tasks.py
View File

@ -97,9 +97,15 @@ class TaskBuilder(object):
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': '{}-A'.format(variant.build_type),
'tier': 1,
},
attributes={
'apks': variant.upstream_artifacts_per_abi,
}
)
def craft_assemble_raptor_task(self, variant):
@ -125,7 +131,10 @@ class TaskBuilder(object):
'symbol': 'A',
'tier': 1,
},
attributes={'build-type': 'raptor'},
attributes={
'build-type': 'raptor',
'apks': variant.upstream_artifacts_per_abi,
},
)
def craft_assemble_pr_task(self, variant):
@ -142,7 +151,7 @@ class TaskBuilder(object):
},
'symbol': 'A',
'tier': 1,
}
},
)
def craft_test_pr_task(self, variant):
@ -367,33 +376,6 @@ class TaskBuilder(object):
attributes=attributes,
)
def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder, attributes=None):
signing_format = "autograph_apk"
payload = {
'upstreamArtifacts': [{
'paths': apk_paths,
'formats': [signing_format],
'taskId': {'task-reference': '<build>'},
'taskType': 'build'
}]
}
return self._craft_default_task_definition(
worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
provisioner_id='scriptworker-prov-v1',
dependencies={'build': assemble_task_label},
routes=routes,
scopes=[
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
"project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type),
],
name=name,
description=description,
payload=payload,
treeherder=treeherder,
attributes=attributes,
)
def _craft_default_task_definition(
self,
worker_type,
@ -457,86 +439,6 @@ class TaskBuilder(object):
},
}
def craft_raptor_signing_task(
self, assemble_task_label, variant, is_staging,
):
staging_prefix = '.staging' if is_staging else ''
routes = [
"index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.latest".format(
staging_prefix, self.date.year, self.date.month, self.date.day
),
"index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.revision.{}".format(
staging_prefix, self.date.year, self.date.month, self.date.day, self.commit
),
"index.project.mobile.fenix.v2{}.performance-test.latest".format(staging_prefix),
]
return self._craft_signing_task(
name='sign: {}'.format('forPerformanceTest'),
description='Dep-signing variant {}'.format('forPerformanceTest'),
signing_type="dep",
assemble_task_label=assemble_task_label,
apk_paths=variant.upstream_artifacts(),
routes=routes,
treeherder={
'groupSymbol': 'forPerformanceTest',
'jobKind': 'other',
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': 'As',
'tier': 1,
},
attributes={
'build-type': 'raptor',
'apks': variant.upstream_artifacts_per_abi,
},
)
def craft_release_signing_task(
self, build_task_id, variant, channel, is_staging, publish_to_index=True
):
if publish_to_index:
staging_prefix = '.staging' if is_staging else ''
routes = [
"index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format(
staging_prefix, channel, self.date.year, self.date.month, self.date.day
),
"index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format(
staging_prefix, channel, self.date.year, self.date.month, self.date.day, self.commit
),
"index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, channel),
]
else:
routes = []
capitalized_channel = upper_case_first_letter(channel)
return self._craft_signing_task(
name="Signing {} task".format(capitalized_channel),
description="Sign {} builds of Fenix".format(capitalized_channel),
signing_type="dep" if is_staging else channel,
assemble_task_label=build_task_id,
apk_paths=variant.upstream_artifacts(),
routes=routes,
treeherder={
'jobKind': 'other',
'machine': {
'platform': 'android-all',
},
'collection': {
'opt': True,
},
'symbol': '{}-s'.format(channel),
'tier': 1,
},
attributes={
'apks': variant.upstream_artifacts_per_abi,
},
)
def schedule_task(queue, taskId, task):
print("TASK", taskId)

4
taskcluster/ci/config.yml
View File

@ -2,7 +2,11 @@
trust-domain: mobile
treeherder:
group-names:
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
'I': 'Docker Image Builds'
'nightly': 'Nightly-related tasks'
'nightlyLegacy': 'Nightly-related tasks that ship to https://play.google.com/store/apps/details?id=org.mozilla.fenix'
'production': 'Release-related tasks'
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'

46
taskcluster/ci/push-apk/kind.yml
View File

@ -2,55 +2,25 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
loader: fenix_taskgraph.loader.single_dep:loader
transforms:
- fenix_taskgraph.pushapk:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
- signing
job-defaults:
only-for-build-types:
- nightly
- nightly-legacy
- production
job-template:
description: Publish Fenix
worker-type: push-apk
worker:
commit: true
product: fenix
run-on-tasks-for: []
treeherder:
kind: build
tier: 1
jobs:
release:
dependencies:
signing: 'Signing Production task' # comes from the old-decision task
worker:
channel: production
certificate-alias: fenix
run-on-tasks-for: [github-release]
treeherder:
symbol: release-gp
nightly:
attributes:
nightly: true
dependencies:
signing: 'Signing Nightly task' # comes from the old-decision task
worker:
channel: nightly
google-play-track: internal
treeherder:
symbol: nightly-gp
nightly-legacy:
attributes:
nightly: true
dependencies:
signing: 'Signing Production task' # comes from the old-decision task
worker:
channel: production
google-play-track: nightly
treeherder:
symbol: production-gp

3
taskcluster/ci/raptor/kind.yml
View File

@ -6,7 +6,7 @@ transforms:
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
- signing
only-for-build-types:
- raptor
only-for-abis:
@ -28,7 +28,6 @@ job-defaults:
armeabi-v7a: android-hw-g5-7-0-arm7-api-16/opt
dependencies:
geckoview-nightly: geckoview-nightly
signing: 'sign: forPerformanceTest' # comes from the old-decision task
worker:
max-run-time: 3600
env:

74
taskcluster/ci/signing/kind.yml 100644
View File

@ -0,0 +1,74 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
transforms:
- fenix_taskgraph.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- old-decision
job-defaults:
description: Sign Fenix
index:
type: signing
worker-type:
by-variant:
(nightly|nightly-legacy|production):
by-level:
'3': signing
default: dep-signing
default: dep-signing
worker:
signing-type:
by-variant:
(nightly|nightly-legacy|production):
by-level:
'3': release-signing
default: dep-signing
default: dep-signing
run-on-tasks-for: []
treeherder:
kind: build
tier: 2
platform: android-all/opt
jobs:
performance-test:
attributes:
build-type: performance-test
worker-type: dep-signing
dependencies:
build: 'assemble: geckoNightlyForPerformanceTest' # comes from the old-decision task
treeherder:
symbol: forPerformanceTest(s)
production:
attributes:
build-type: production
dependencies:
build: 'Build FenixProduction task' # comes from the old-decision task
run-on-tasks-for: [github-release]
treeherder:
symbol: production(s)
nightly:
attributes:
build-type: nightly
nightly: true
dependencies:
build: 'Build FenixNightly task' # comes from the old-decision task
treeherder:
symbol: nightly(s)
nightly-legacy:
attributes:
build-type: nightly-legacy
nightly: true
dependencies:
build: 'Build FenixNightlyLegacy task' # comes from the old-decision task
treeherder:
symbol: nightly-legacy(s)

52
taskcluster/fenix_taskgraph/loader/single_dep.py 100644
View File

@ -0,0 +1,52 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
from voluptuous import Required
from taskgraph.task import Task
from taskgraph.util.schema import Schema
schema = Schema({Required("primary-dependency", "primary dependency task"): Task})
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds.
Optional `only-for-attributes` kind configuration, if specified, will limit
the jobs chosen to ones which have the specified attribute, with the specified
value.
Optional `job-template` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
only_attributes = config.get("only-for-attributes")
only_build_type = config.get('only-for-build-types')
job_template = config.get("job-template")
for task in loaded_tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if only_attributes:
config_attrs = set(only_attributes)
if not config_attrs & set(task.attributes):
# make sure any attribute exists
continue
if only_build_type:
build_type = task.attributes.get('build-type')
if build_type not in only_build_type:
continue
job = {"primary-dependency": task}
if job_template:
job.update(copy.deepcopy(job_template))
yield job

69
taskcluster/fenix_taskgraph/pushapk.py
View File

@ -9,43 +9,72 @@ kind.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.treeherder import inherit_treeherder_from_dep
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
transforms = TransformSequence()
# TODO remove this transform once signing task are migrated to taskgraph
_CHANNEL_PER_TASK_NAME = {
'nightly': 'nightly',
'nightly-legacy': 'production',
'production': 'production',
}
_GOOGLE_PLAY_TRACK_PER_TASK_NAME = {
'nightly': 'internal',
'nightly-legacy': 'nightly',
}
@transforms.add
def fetch_old_decision_dependency(config, tasks):
def build_name_and_attributes(config, tasks):
for task in tasks:
for dep_task in config.kind_dependencies_tasks:
expected_signing_dep_label = task['dependencies']['signing']
if dep_task.label != expected_signing_dep_label:
continue
dep = task["primary-dependency"]
task["dependencies"] = {"signing": dep.label}
task.setdefault("attributes", {}).update(dep.attributes.copy())
task["name"] = _get_dependent_job_name_without_its_kind(dep)
task['primary-dependency'] = dep_task
yield task
yield task
def _get_dependent_job_name_without_its_kind(dependent_job):
return dependent_job.label[len(dependent_job.kind) + 1:]
@transforms.add
def build_pushapk_task(config, tasks):
def build_treeherder_definition(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
treeherder_symbol = join_symbol(treeherder_group, 'gp')
task["treeherder"]["symbol"] = treeherder_symbol
yield task
@transforms.add
def build_worker_definition(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task.setdefault("attributes", {}).update(dep.attributes.copy())
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
task_name = task["name"]
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
task["worker"]["upstream-artifacts"] = [{
worker_definition = {}
worker_definition["upstream-artifacts"] = [{
"taskId": {"task-reference": "<signing>"},
"taskType": "signing",
"paths": dep.attributes["apks"].values(),
}]
task["worker"]["dep"] = config.params["level"] != "3"
if not task["worker"].get("certificate-alias"):
task["worker"]["certificate-alias"] = "{}-{}".format(
task["worker"]["product"], task["worker"]["channel"]
)
worker_definition["dep"] = config.params["level"] != "3"
worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name]
# Fenix production doesn't follow the rule {product}-{channel}
worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \
"{}-{}".format(task["worker"]["product"], worker_definition["channel"])
if _GOOGLE_PLAY_TRACK_PER_TASK_NAME.get(task_name):
worker_definition["google-play-track"] = _GOOGLE_PLAY_TRACK_PER_TASK_NAME[task_name]
task["worker"].update(worker_definition)
yield task

5
taskcluster/fenix_taskgraph/raptor.py
View File

@ -30,10 +30,6 @@ def add_variants(config, tasks):
if build_type not in only_types:
continue
# TODO remove this if statement once signing tasks are fully created in taskgraph
if not dep_task.label.startswith('sign: '):
continue
for abi, apk_path in dep_task.attributes["apks"].items():
if abi not in only_abis:
continue
@ -52,6 +48,7 @@ def add_variants(config, tasks):
def build_raptor_task(config, tasks):
for task in tasks:
signing = task.pop("primary-dependency")
task["dependencies"]["signing"] = signing.label
build_type = task["attributes"]["build-type"]
abi = task["attributes"]["abi"]
apk = task["attributes"]["apk"]

6
taskcluster/fenix_taskgraph/routes.py
View File

@ -9,9 +9,9 @@ import time
from taskgraph.transforms.task import index_builder
SIGNING_ROUTE_TEMPLATES = [
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}",
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest",
"index.project.{trust-domain}.{project}.v3.{variant}.latest",
"index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.revision.{head_rev}",
"index.project.{trust-domain}.{project}.v2.{variant}.{build_date}.latest",
"index.project.{trust-domain}.{project}.v2.{variant}.latest",
]

64
taskcluster/fenix_taskgraph/signing.py 100644
View File

@ -0,0 +1,64 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
Apply some defaults and minor modifications to the jobs defined in the build
kind.
"""
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.transforms.base import TransformSequence
from taskgraph.util.schema import resolve_keyed_by
transforms = TransformSequence()
# TODO remove this transform once build tasks are migrated to taskgraph
@transforms.add
def fetch_old_decision_dependency(config, tasks):
for task in tasks:
for dep_task in config.kind_dependencies_tasks:
expected_signing_dep_label = task['dependencies']['build']
if dep_task.label != expected_signing_dep_label:
continue
task['primary-dependency'] = dep_task
yield task
@transforms.add
def define_signing_flags(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
# Current kind will be prepended later in the transform chain.
task.setdefault("attributes", {}).update(dep.attributes.copy())
task["attributes"]["signed"] = True
if "run_on_tasks_for" in task["attributes"]:
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
for key in ("worker-type", "worker.signing-type"):
resolve_keyed_by(
task,
key,
item_name=task["name"],
variant=task["attributes"]["build-type"],
level=config.params["level"],
)
yield task
@transforms.add
def build_signing_task(config, tasks):
for task in tasks:
dep = task.pop("primary-dependency")
task["dependencies"] = {"build": dep.label}
task["worker"]["upstream-artifacts"] = [
{
"taskId": {"task-reference": "<build>"},
"taskType": "build",
"paths": dep.attributes["apks"].values(),
"formats": ["autograph_apk"],
}
]
yield task