blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Fixes #1321: dep-signs builds on-master-push

master
Mitchell Hentges 2019-04-05 16:21:51 +02:00 committed by Colin Lee
parent f36da2d8f2
commit 6ed0c26dd5
6 changed files with 179 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

109
.taskcluster.yml
View File

@ -104,7 +104,21 @@ tasks:
TASKS_PRIORITY: ${tasks_priority}
TRUST_LEVEL: ${trust_level}
features:
chainOfTrust: true
taskclusterProxy: true
artifacts:
public/task-graph.json:
type: file
path: /opt/fenix/task-graph.json
expires: ${expires_in}
public/actions.json:
type: file
path: /opt/fenix/actions.json
expires: ${expires_in}
public/parameters.yml:
type: file
path: /opt/fenix/parameters.yml
expires: ${expires_in}
extra:
tasks_for: ${tasks_for}
treeherder:
@ -116,50 +130,51 @@ tasks:
in:
$if: 'tasks_for in ["github-pull-request", "github-push"]'
then:
$let:
pr_or_push_parameters:
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout ${head_rev}
&& python automation/taskcluster/decision_task.py pr-or-push
in:
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
then:
$let:
pull_request_title: ${event.pull_request.title}
pull_request_number: ${event.pull_request.number}
pull_request_url: ${event.pull_request.html_url}
in:
$mergeDeep:
- {$eval: 'default_task_definition'}
- {$eval: 'pr_or_push_parameters'}
- scopes:
- ${assume_scope_prefix}:pull-request
payload:
env:
GITHUB_PULL_TITLE: ${pull_request_title}
extra:
treeherder:
symbol: D-PR
metadata:
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
then:
$let:
pull_request_title: ${event.pull_request.title}
pull_request_number: ${event.pull_request.number}
pull_request_url: ${event.pull_request.html_url}
in:
$mergeDeep:
- {$eval: 'default_task_definition'}
- {$eval: 'pr_or_push_parameters'}
- scopes:
- ${assume_scope_prefix}:branch:${short_head_branch}
- ${assume_scope_prefix}:pull-request
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py pull-request
env:
GITHUB_PULL_TITLE: ${pull_request_title}
extra:
treeherder:
symbol: D
symbol: D-PR
metadata:
name: Fenix - Decision task
description: Schedules the build and test tasks for Fenix.
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:branch:${short_head_branch}
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py push
extra:
treeherder:
symbol: D
metadata:
name: Fenix - Decision task
description: Schedules the build and test tasks for Fenix.
else:
- $if: 'tasks_for == "cron"'
then:
@ -172,31 +187,15 @@ tasks:
routes:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
features:
taskclusterProxy: true
chainOfTrust: true
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout ${head_rev}
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py \
release \
--nightly \
--track ${track}
artifacts:
public/task-graph.json:
type: file
path: /opt/fenix/task-graph.json
expires: ${expires_in}
public/actions.json:
type: file
path: /opt/fenix/actions.json
expires: ${expires_in}
public/parameters.yml:
type: file
path: /opt/fenix/parameters.yml
expires: ${expires_in}
extra:
cron: {$json: {$eval: 'cron'}}
treeherder:

52
automation/taskcluster/decision_task.py
View File

@ -13,7 +13,7 @@ import os
import taskcluster
from lib import build_variants
from lib.tasks import TaskBuilder, schedule_task_graph
from lib.tasks import TaskBuilder, schedule_task_graph, _get_architecture_and_build_type_and_product_from_variant
from lib.chain_of_trust import (
populate_chain_of_trust_task_graph,
populate_chain_of_trust_required_but_unused_files
@ -42,27 +42,31 @@ BUILDER = TaskBuilder(
)
def pr_or_push():
if SKIP_TASKS_TRIGGER in PR_TITLE:
def pr_or_push(is_master_push):
if not is_master_push and SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
print("Fetching build variants from gradle")
variants = build_variants.from_gradle()
if len(variants) == 0:
raise ValueError("Could not get build variants from gradle")
print("Got variants: {}".format(' '.join(variants)))
build_tasks = {}
signing_tasks = {}
other_tasks = {}
for variant in variants:
build_tasks[taskcluster.slugId()] = BUILDER.craft_assemble_task(variant)
for variant in build_variants.from_gradle():
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
arch, build_type, _ = _get_architecture_and_build_type_and_product_from_variant(variant)
# autophone only supports arm and aarch64, so only sign/perftest those builds
if (
build_type == 'releaseRaptor' and
arch in ('arm', 'aarch64') and
is_master_push
):
signing_tasks[taskcluster.slugId()] = BUILDER.craft_master_commit_signing_task(assemble_task_id, variant)
# raptor task will be added in follow-up
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
@ -71,12 +75,13 @@ def pr_or_push():
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, other_tasks)
return (build_tasks, signing_tasks, other_tasks)
def nightly(track):
is_staging = track == 'staging-nightly'
architectures = ['x86', 'arm', 'aarch64']
apk_paths = ["public/target.{}.apk".format(arch) for arch in architectures]
build_tasks = {}
signing_tasks = {}
@ -85,19 +90,18 @@ def nightly(track):
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, is_staging)
artifacts = ["public/target.{}.apk".format(arch) for arch in architectures]
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_signing_task(
signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task(
build_task_id,
apks=artifacts,
apk_paths=apk_paths,
is_staging=is_staging,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
signing_task_id,
apks=artifacts,
is_staging=is_staging
apks=apk_paths,
is_staging=is_staging,
)
return (build_tasks, signing_tasks, push_tasks)
@ -110,7 +114,8 @@ if __name__ == "__main__":
subparsers = parser.add_subparsers(dest='command')
subparsers.add_parser('pr-or-push')
subparsers.add_parser('pull-request')
subparsers.add_parser('push')
release_parser = subparsers.add_parser('release')
release_parser.add_argument('--nightly', action="store_true", default=False)
@ -121,9 +126,12 @@ if __name__ == "__main__":
result = parser.parse_args()
command = result.command
taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
if command == 'pr-or-push':
ordered_groups_of_tasks = pr_or_push()
if command == 'pull-request':
ordered_groups_of_tasks = pr_or_push(False)
elif command == 'push':
ordered_groups_of_tasks = pr_or_push(True)
elif command == 'release':
ordered_groups_of_tasks = nightly(result.track)
else:

5
automation/taskcluster/lib/build_variants.py
View File

@ -8,6 +8,7 @@ import subprocess
def from_gradle():
print('Fetching build variants from gradle')
process = subprocess.Popen([
"./gradlew", "--no-daemon", "--quiet", "printBuildVariants"
], stdout=subprocess.PIPE)
@ -21,4 +22,8 @@ def from_gradle():
variants_json = variants_line.split(' ', 1)[1]
variants = json.loads(variants_json)
if len(variants) == 0:
raise RuntimeError('Expected at least one build variant from gradle')
print("Got variants: " + ' '.join(variants))
return variants

2
automation/taskcluster/lib/chain_of_trust.py
View File

@ -12,7 +12,7 @@ def populate_chain_of_trust_required_but_unused_files():
def populate_chain_of_trust_task_graph(full_task_graph):
# taskgraph must follow the format:
# full_task_graph must follow the format:
# {
# task_id: full_task_definition
# }

154
automation/taskcluster/lib/tasks.py
View File

@ -10,7 +10,7 @@ import json
import os
import taskcluster
from lib.util import convert_camel_case_into_kebab_case
from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter
DEFAULT_EXPIRES_IN = '1 year'
_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix'
@ -22,12 +22,7 @@ class TaskBuilder(object):
task_id,
repo_url,
git_ref,
short_head_branch,
commit,
owner,
source,
scheduler_id,
date_string,
short_head_branch, commit, owner, source, scheduler_id, date_string,
tasks_priority='lowest',
trust_level=1
):
@ -39,6 +34,7 @@ class TaskBuilder(object):
self.owner = owner
self.source = source
self.scheduler_id = scheduler_id
self.trust_level = trust_level
self.tasks_priority = tasks_priority
self.date = arrow.get(date_string)
self.trust_level = trust_level
@ -111,7 +107,6 @@ class TaskBuilder(object):
description='Building and testing variant {}'.format(variant),
gradle_task='assemble{}'.format(variant.capitalize()),
artifacts=_craft_artifacts_from_variant(variant),
routes=self._craft_branch_routes(variant),
treeherder={
'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant),
'jobKind': 'build',
@ -139,33 +134,6 @@ class TaskBuilder(object):
},
)
def _craft_branch_routes(self, variant):
routes = []
if self.repo_url == _OFFICIAL_REPO_URL and self.short_head_branch == 'master':
architecture, build_type, product = \
_get_architecture_and_build_type_and_product_from_variant(variant)
product = convert_camel_case_into_kebab_case(product)
postfix = convert_camel_case_into_kebab_case('{}-{}'.format(architecture, build_type))
routes = [
'index.project.mobile.fenix.branch.{}.revision.{}.{}.{}'.format(
self.short_head_branch, self.commit, product, postfix
),
'index.project.mobile.fenix.branch.{}.latest.{}.{}'.format(
self.short_head_branch, product, postfix
),
'index.project.mobile.fenix.branch.{}.pushdate.{}.{}.{}.revision.{}.{}.{}'.format(
self.short_head_branch, self.date.year, self.date.month, self.date.day,
self.commit, product, postfix
),
'index.project.mobile.fenix.branch.{}.pushdate.{}.{}.{}.latest.{}.{}'.format(
self.short_head_branch, self.date.year, self.date.month, self.date.day,
product, postfix
),
]
return routes
def craft_detekt_task(self):
return self._craft_clean_gradle_task(
name='detekt',
@ -251,14 +219,12 @@ class TaskBuilder(object):
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
checkout_command = (
"export TERM=dumb && "
"git fetch {} {} --tags && "
"git config advice.detachedHead false && "
"git checkout {}".format(
self.repo_url, self.git_ref, self.commit
)
)
checkout_command = ' && '.join([
"export TERM=dumb",
"git fetch {} {}".format(self.repo_url, self.git_ref),
"git config advice.detachedHead false",
"git checkout FETCH_HEAD",
])
command = '{} && {}'.format(checkout_command, command)
@ -293,6 +259,32 @@ class TaskBuilder(object):
treeherder=treeherder,
)
def _craft_signing_task(self, name, description, signing_type, assemble_task_id, apk_paths, routes, treeherder):
signing_format = "autograph_apk"
payload = {
'upstreamArtifacts': [{
'paths': apk_paths,
'formats': [signing_format],
'taskId': assemble_task_id,
'taskType': 'build'
}]
}
return self._craft_default_task_definition(
worker_type='mobile-signing-dep-v1' if signing_format == 'dep' else 'mobile-signing-v1',
provisioner_id='scriptworker-prov-v1',
dependencies=[assemble_task_id],
routes=routes,
scopes=[
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
"project:mobile:fenix:releng:signing:cert:{}".format(signing_type),
],
name=name,
description=description,
payload=payload,
treeherder=treeherder,
)
def _craft_default_task_definition(
self, worker_type, provisioner_id, dependencies, routes, scopes, name, description,
payload, treeherder=None
@ -332,19 +324,49 @@ class TaskBuilder(object):
},
}
def craft_signing_task(
self, build_task_id, apks, is_staging=True,
def craft_master_commit_signing_task(
self, assemble_task_id, variant
):
signing_format = 'autograph_apk'
payload = {
"upstreamArtifacts": [{
"paths": apks,
"formats": [signing_format],
"taskId": build_task_id,
"taskType": "build",
}],
}
architecture, build_type, product = _get_architecture_and_build_type_and_product_from_variant(variant)
product = convert_camel_case_into_kebab_case(product)
postfix = convert_camel_case_into_kebab_case('{}-{}'.format(architecture, build_type))
routes = [
'index.project.mobile.fenix.branch.master.revision.{}.{}.{}'.format(
self.commit, product, postfix
),
'index.project.mobile.fenix.branch.master.latest.{}.{}'.format(
product, postfix
),
'index.project.mobile.fenix.branch.master.pushdate.{}.{}.{}.revision.{}.{}.{}'.format(
self.date.year, self.date.month, self.date.day, self.commit,
product, postfix
),
'index.project.mobile.fenix.branch.master.pushdate.{}.{}.{}.latest.{}.{}'.format(
self.date.year, self.date.month, self.date.day, product, postfix
),
]
return self._craft_signing_task(
name='sign: {}'.format(variant),
description='Dep-signing variant {}'.format(variant),
signing_type="dep-signing",
assemble_task_id=assemble_task_id,
apk_paths=["public/target.apk"],
routes=routes,
treeherder={
'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant),
'jobKind': 'other',
'machine': {
'platform': _craft_treeherder_platform_from_variant(variant),
},
'symbol': 'As',
'tier': 1,
},
)
def craft_nightly_signing_task(
self, build_task_id, apk_paths, is_staging=True,
):
index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'
routes = [
"index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(
@ -356,20 +378,13 @@ class TaskBuilder(object):
"index.project.mobile.fenix.{}.nightly.latest".format(index_release),
]
return self._craft_default_task_definition(
worker_type='mobile-signing-dep-v1' if is_staging else 'mobile-signing-v1',
provisioner_id='scriptworker-prov-v1',
dependencies=[build_task_id],
routes=routes,
scopes=[
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
"project:mobile:fenix:releng:signing:cert:{}".format(
'dep-signing' if is_staging else 'release-signing'
)
],
return self._craft_signing_task(
name="Signing task",
description="Sign release builds of Fenix",
payload=payload,
signing_type="dep-signing" if is_staging else "release-signing",
assemble_task_id=build_task_id,
apk_paths=apk_paths,
routes=routes,
treeherder={
'jobKind': 'other',
'machine': {
@ -447,8 +462,8 @@ def _craft_apk_full_path_from_variant(variant):
)
short_variant = variant[:-len(build_type)]
postfix = '-unsigned' if build_type == 'release' else ''
product = '{}{}'.format(product[0].lower(), product[1:])
postfix = '-unsigned' if build_type.startswith('release') else ''
product = lower_case_first_letter(product)
return '/opt/fenix/app/build/outputs/apk/{short_variant}/{build_type}/app-{architecture}-{product}-{build_type}{postfix}.apk'.format( # noqa: E501
architecture=architecture,
@ -479,7 +494,7 @@ def _get_architecture_and_build_type_and_product_from_variant(variant):
for supported_build_type in _SUPPORTED_BUILD_TYPES:
if variant.endswith(supported_build_type):
build_type = supported_build_type.lower()
build_type = lower_case_first_letter(supported_build_type)
break
else:
raise ValueError(
@ -528,4 +543,5 @@ def schedule_task_graph(ordered_groups_of_tasks):
# allows to have the full definition. This is needed to make Chain of Trust happy
'task': queue.task(task_id),
}
return full_task_graph

4
automation/taskcluster/lib/util.py
View File

@ -5,3 +5,7 @@ def convert_camel_case_into_kebab_case(string):
# Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501
first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string)
return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower()
def lower_case_first_letter(string):
return '{}{}'.format(string[0].lower(), string[1:])