1
0
Fork 0

Let taskgraph load tasks for releases

master
Johan Lorenzo 2019-09-13 10:08:51 +02:00 committed by Sebastian Kaspari
parent d26ef6087d
commit 294ac50f3a
4 changed files with 128 additions and 103 deletions

View File

@ -11,72 +11,81 @@ tasks:
revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614 revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614
trustDomain: mobile trustDomain: mobile
in: in:
$if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]' $let:
then: # Github events have this stuff in different places...
$let: ownerEmail:
# Github events have this stuff in different places... $if: 'tasks_for == "github-push"'
ownerEmail: then: '${event.pusher.email}'
$if: 'tasks_for == "github-push"' # Assume Pull Request
then: '${event.pusher.email}' else:
# Assume Pull Request $if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.user.login}@users.noreply.github.com'
else: else:
$if: 'tasks_for == "github-pull-request"' $if: 'tasks_for == "github-release"'
then: '${event.pull_request.user.login}@users.noreply.github.com' then: '${event.sender.login}@users.noreply.github.com'
else: else:
$if: 'tasks_for in ["cron", "action"]' $if: 'tasks_for in ["cron", "action"]'
then: '${tasks_for}@noreply.mozilla.org' then: '${tasks_for}@noreply.mozilla.org'
baseRepoUrl: baseRepoUrl:
$if: 'tasks_for == "github-push"' $if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.html_url}' then: '${event.repository.html_url}'
else: else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.base.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
repoUrl:
$if: 'tasks_for == "github-push"'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
project:
$if: 'tasks_for == "github-push"'
then: '${event.repository.name}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.name}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.project}'
head_branch:
$if: 'tasks_for == "github-pull-request"' $if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.ref} then: '${event.pull_request.base.repo.html_url}'
else: else:
$if: 'tasks_for == "github-push"' $if: 'tasks_for in ["cron", "action"]'
then: ${event.ref} then: '${repository.url}'
repoUrl:
$if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
project:
$if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.name}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.name}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.project}'
head_branch:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.ref}
else:
$if: 'tasks_for == "github-push"'
then: ${event.ref}
else:
$if: 'tasks_for == "github-release"'
then: '${event.release.target_commitish}'
else: else:
$if: 'tasks_for in ["cron", "action"]' $if: 'tasks_for in ["cron", "action"]'
then: '${push.branch}' then: '${push.branch}'
head_sha: head_sha:
$if: 'tasks_for == "github-push"' $if: 'tasks_for == "github-push"'
then: '${event.after}' then: '${event.after}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.sha}'
else: else:
$if: 'tasks_for == "github-pull-request"' $if: 'tasks_for == "github-release"'
then: '${event.pull_request.head.sha}' then: '${event.release.tag_name}'
else: else:
$if: 'tasks_for in ["cron", "action"]' $if: 'tasks_for in ["cron", "action"]'
then: '${push.revision}' then: '${push.revision}'
ownTaskId: ownTaskId:
$if: '"github" in tasks_for' $if: '"github" in tasks_for'
then: {$eval: as_slugid("decision_task")} then: {$eval: as_slugid("decision_task")}
else: else:
$if: 'tasks_for == "cron"' $if: 'tasks_for == "cron"'
then: '${ownTaskId}' then: '${ownTaskId}'
in: in:
$if: 'tasks_for in ["github-pull-request", "github-release", "action", "cron"] || tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$let: $let:
level: level:
$if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"' $if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"'
@ -101,7 +110,7 @@ tasks:
$merge: $merge:
- owner: "${ownerEmail}" - owner: "${ownerEmail}"
source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml'
- $if: 'tasks_for in ["github-push", "github-pull-request"]' - $if: 'tasks_for in ["github-push", "github-pull-request", "github-release"]'
then: then:
name: "Decision Task" name: "Decision Task"
description: 'The task that creates all of the other tasks in the task graph' description: 'The task that creates all of the other tasks in the task graph'
@ -140,12 +149,9 @@ tasks:
then: then:
$let: $let:
short_head_branch: short_head_branch:
$if: 'head_branch[:10] == "refs/tags/"' $if: 'head_branch[:11] == "refs/heads/"'
then: {$eval: 'head_branch[10:]'} then: {$eval: 'head_branch[11:]'}
else: else: ${head_branch}
$if: 'head_branch[:11] == "refs/heads/"'
then: {$eval: 'head_branch[11:]'}
else: ${head_branch}
in: in:
- 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}'
else: else:
@ -153,12 +159,16 @@ tasks:
then: then:
- 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request'
else: else:
$if: 'tasks_for == "action"' $if: 'tasks_for == "github-release"'
then: then:
# when all actions are hooks, we can calculate this directly rather than using a variable - 'assume:repo:${repoUrl[8:]}:release'
- '${action.repo_scope}'
else: else:
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' $if: 'tasks_for == "action"'
then:
# when all actions are hooks, we can calculate this directly rather than using a variable
- '${action.repo_scope}'
else:
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
requires: all-completed requires: all-completed
priority: lowest priority: lowest
@ -190,6 +200,9 @@ tasks:
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
ACTION_INPUT: {$json: {$eval: 'input'}} ACTION_INPUT: {$json: {$eval: 'input'}}
ACTION_CALLBACK: '${action.cb_name}' ACTION_CALLBACK: '${action.cb_name}'
- $if: 'tasks_for == "github-release"'
then:
GIT_TAG: '${event.release.tag_name}'
features: features:
taskclusterProxy: true taskclusterProxy: true
chainOfTrust: true chainOfTrust: true
@ -252,14 +265,18 @@ tasks:
then: then:
symbol: D symbol: D
else: else:
$if: 'tasks_for == "action"' $if: 'tasks_for == "github-release"'
then: then:
groupName: 'action-callback' symbol: 'ship_fenix'
groupSymbol: AC
symbol: "${action.symbol}"
else: else:
groupSymbol: cron $if: 'tasks_for == "action"'
symbol: "${cron.job_symbol}" then:
groupName: 'action-callback'
groupSymbol: AC
symbol: "${action.symbol}"
else:
groupSymbol: cron
symbol: "${cron.job_symbol}"
- $if: 'tasks_for == "action"' - $if: 'tasks_for == "action"'
then: then:
parent: '${action.taskGroupId}' parent: '${action.taskGroupId}'

View File

@ -51,18 +51,16 @@ def push(builder):
def raptor(builder, is_staging): def raptor(builder, is_staging):
build_tasks = {} tasks = []
signing_tasks = {}
other_tasks = {}
mozharness_task_id = fetch_mozharness_task_id() mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly') variant = get_variant('forPerformanceTest', 'geckoNightly')
assemble_task_id = _generate_slug_id() build_task = builder.craft_assemble_raptor_task(variant)
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant) tasks.append(build_task)
signing_task_id = _generate_slug_id() signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging) tasks.append(signing_task)
for abi in ('armeabi-v7a', 'arm64-v8a'): for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi) variant_apk = variant.get_apk(abi)
@ -73,34 +71,29 @@ def raptor(builder, is_staging):
builder.craft_raptor_youtube_playback_task, builder.craft_raptor_youtube_playback_task,
] ]
for craft_function in all_raptor_craft_functions: for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging) raptor_task = craft_function(
other_tasks[_generate_slug_id()] = craft_function(*args) signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging
)
tasks.append(raptor_task)
return (build_tasks, signing_tasks, other_tasks) return tasks
def release(builder, channel, engine, is_staging, version_name): def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine) variant = get_variant('fenix' + channel.capitalize(), engine)
taskcluster_apk_paths = variant.upstream_artifacts() taskcluster_apk_paths = variant.upstream_artifacts()
build_tasks = {} build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_tasks = {}
push_tasks = {}
build_task_id = _generate_slug_id() signing_task = builder.craft_release_signing_task(
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) build_task['label'],
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths, taskcluster_apk_paths,
channel=channel, channel=channel,
is_staging=is_staging, is_staging=is_staging,
) )
push_task_id = _generate_slug_id() push_task = builder.craft_push_task(
push_tasks[push_task_id] = builder.craft_push_task( signing_task['label'],
signing_task_id,
taskcluster_apk_paths, taskcluster_apk_paths,
channel=channel, channel=channel,
# TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track # TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track
@ -108,7 +101,7 @@ def release(builder, channel, engine, is_staging, version_name):
is_staging=is_staging, is_staging=is_staging,
) )
return (build_tasks, signing_tasks, push_tasks) return [build_task, signing_task, push_task]
def release_as_fennec(builder, is_staging, version_name): def release_as_fennec(builder, is_staging, version_name):

View File

@ -361,13 +361,13 @@ class TaskBuilder(object):
treeherder=treeherder, treeherder=treeherder,
) )
def _craft_signing_task(self, name, description, signing_type, assemble_task_id, apk_paths, routes, treeherder): def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder):
signing_format = "autograph_apk" signing_format = "autograph_apk"
payload = { payload = {
'upstreamArtifacts': [{ 'upstreamArtifacts': [{
'paths': apk_paths, 'paths': apk_paths,
'formats': [signing_format], 'formats': [signing_format],
'taskId': assemble_task_id, 'taskId': {'task-reference': '<build>'},
'taskType': 'build' 'taskType': 'build'
}] }]
} }
@ -375,7 +375,7 @@ class TaskBuilder(object):
return self._craft_default_task_definition( return self._craft_default_task_definition(
worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1', worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
provisioner_id='scriptworker-prov-v1', provisioner_id='scriptworker-prov-v1',
dependencies=[assemble_task_id], dependencies={'build': assemble_task_label},
routes=routes, routes=routes,
scopes=[ scopes=[
"project:mobile:fenix:releng:signing:format:{}".format(signing_format), "project:mobile:fenix:releng:signing:format:{}".format(signing_format),
@ -448,7 +448,7 @@ class TaskBuilder(object):
} }
def craft_raptor_signing_task( def craft_raptor_signing_task(
self, assemble_task_id, variant, is_staging, self, assemble_task_label, variant, is_staging,
): ):
staging_prefix = '.staging' if is_staging else '' staging_prefix = '.staging' if is_staging else ''
routes = [ routes = [
@ -465,7 +465,7 @@ class TaskBuilder(object):
name='sign: {}'.format('forPerformanceTest'), name='sign: {}'.format('forPerformanceTest'),
description='Dep-signing variant {}'.format('forPerformanceTest'), description='Dep-signing variant {}'.format('forPerformanceTest'),
signing_type="dep", signing_type="dep",
assemble_task_id=assemble_task_id, assemble_task_label=assemble_task_label,
apk_paths=variant.upstream_artifacts(), apk_paths=variant.upstream_artifacts(),
routes=routes, routes=routes,
treeherder={ treeherder={
@ -501,7 +501,7 @@ class TaskBuilder(object):
name="Signing {} task".format(capitalized_channel), name="Signing {} task".format(capitalized_channel),
description="Sign {} builds of Fenix".format(capitalized_channel), description="Sign {} builds of Fenix".format(capitalized_channel),
signing_type="dep" if is_staging else channel, signing_type="dep" if is_staging else channel,
assemble_task_id=build_task_id, assemble_task_label=build_task_id,
apk_paths=apk_paths, apk_paths=apk_paths,
routes=routes, routes=routes,
treeherder={ treeherder={
@ -515,7 +515,7 @@ class TaskBuilder(object):
) )
def craft_push_task( def craft_push_task(
self, signing_task_id, apk_paths, channel, is_staging=False, override_google_play_track=None self, signing_task_label, apk_paths, channel, is_staging=False, override_google_play_track=None
): ):
payload = { payload = {
"commit": True, "commit": True,
@ -524,7 +524,7 @@ class TaskBuilder(object):
"upstreamArtifacts": [ "upstreamArtifacts": [
{ {
"paths": apk_paths, "paths": apk_paths,
"taskId": signing_task_id, "taskId": {'task-reference': '<signing>'},
"taskType": "signing" "taskType": "signing"
} }
] ]
@ -536,7 +536,7 @@ class TaskBuilder(object):
return self._craft_default_task_definition( return self._craft_default_task_definition(
worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1', worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
provisioner_id='scriptworker-prov-v1', provisioner_id='scriptworker-prov-v1',
dependencies=[signing_task_id], dependencies={'signing': signing_task_label},
routes=[], routes=[],
scopes=[ scopes=[
"project:mobile:fenix:releng:googleplay:product:fenix{}".format( "project:mobile:fenix:releng:googleplay:product:fenix{}".format(

View File

@ -5,13 +5,14 @@
from __future__ import print_function, unicode_literals from __future__ import print_function, unicode_literals
import os import os
import re
import sys import sys
current_dir = os.path.dirname(os.path.realpath(__file__)) current_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..')) project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
sys.path.append(project_dir) sys.path.append(project_dir)
from automation.taskcluster.decision_task import pr, push from automation.taskcluster.decision_task import pr, push, release
from automation.taskcluster.lib.tasks import TaskBuilder from automation.taskcluster.lib.tasks import TaskBuilder
@ -34,11 +35,25 @@ def loader(kind, path, config, params, loaded_tasks):
trust_level=trust_level, trust_level=trust_level,
) )
is_staging = trust_level != 3
tasks_for = params['tasks_for'] tasks_for = params['tasks_for']
if tasks_for == 'github-pull-request': if tasks_for == 'github-pull-request':
ordered_groups_of_tasks = pr(builder) ordered_groups_of_tasks = pr(builder)
elif tasks_for == 'github-push': elif tasks_for == 'github-push':
ordered_groups_of_tasks = push(builder) ordered_groups_of_tasks = push(builder)
elif tasks_for == 'github-release':
git_tag = os.environ['GIT_TAG']
version = git_tag[1:] # remove prefixed "v"
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
if beta_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
elif production_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
else: else:
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for)) raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))