1
0
Fork 0

Let taskgraph load tasks for releases

master
Johan Lorenzo 2019-09-13 10:08:51 +02:00 committed by Sebastian Kaspari
parent d26ef6087d
commit 294ac50f3a
4 changed files with 128 additions and 103 deletions

View File

@ -11,72 +11,81 @@ tasks:
revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614
trustDomain: mobile
in:
$if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
then:
$let:
# Github events have this stuff in different places...
ownerEmail:
$if: 'tasks_for == "github-push"'
then: '${event.pusher.email}'
# Assume Pull Request
$let:
# Github events have this stuff in different places...
ownerEmail:
$if: 'tasks_for == "github-push"'
then: '${event.pusher.email}'
# Assume Pull Request
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.user.login}@users.noreply.github.com'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.user.login}@users.noreply.github.com'
$if: 'tasks_for == "github-release"'
then: '${event.sender.login}@users.noreply.github.com'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${tasks_for}@noreply.mozilla.org'
baseRepoUrl:
$if: 'tasks_for == "github-push"'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.base.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
repoUrl:
$if: 'tasks_for == "github-push"'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
project:
$if: 'tasks_for == "github-push"'
then: '${event.repository.name}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.name}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.project}'
head_branch:
baseRepoUrl:
$if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.ref}
then: '${event.pull_request.base.repo.html_url}'
else:
$if: 'tasks_for == "github-push"'
then: ${event.ref}
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
repoUrl:
$if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.html_url}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.html_url}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.url}'
project:
$if: 'tasks_for in ["github-push", "github-release"]'
then: '${event.repository.name}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.repo.name}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${repository.project}'
head_branch:
$if: 'tasks_for == "github-pull-request"'
then: ${event.pull_request.head.ref}
else:
$if: 'tasks_for == "github-push"'
then: ${event.ref}
else:
$if: 'tasks_for == "github-release"'
then: '${event.release.target_commitish}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${push.branch}'
head_sha:
$if: 'tasks_for == "github-push"'
then: '${event.after}'
head_sha:
$if: 'tasks_for == "github-push"'
then: '${event.after}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.sha}'
else:
$if: 'tasks_for == "github-pull-request"'
then: '${event.pull_request.head.sha}'
$if: 'tasks_for == "github-release"'
then: '${event.release.tag_name}'
else:
$if: 'tasks_for in ["cron", "action"]'
then: '${push.revision}'
ownTaskId:
$if: '"github" in tasks_for'
then: {$eval: as_slugid("decision_task")}
else:
$if: 'tasks_for == "cron"'
then: '${ownTaskId}'
in:
ownTaskId:
$if: '"github" in tasks_for'
then: {$eval: as_slugid("decision_task")}
else:
$if: 'tasks_for == "cron"'
then: '${ownTaskId}'
in:
$if: 'tasks_for in ["github-pull-request", "github-release", "action", "cron"] || tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$let:
level:
$if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"'
@ -101,7 +110,7 @@ tasks:
$merge:
- owner: "${ownerEmail}"
source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml'
- $if: 'tasks_for in ["github-push", "github-pull-request"]'
- $if: 'tasks_for in ["github-push", "github-pull-request", "github-release"]'
then:
name: "Decision Task"
description: 'The task that creates all of the other tasks in the task graph'
@ -140,12 +149,9 @@ tasks:
then:
$let:
short_head_branch:
$if: 'head_branch[:10] == "refs/tags/"'
then: {$eval: 'head_branch[10:]'}
else:
$if: 'head_branch[:11] == "refs/heads/"'
then: {$eval: 'head_branch[11:]'}
else: ${head_branch}
$if: 'head_branch[:11] == "refs/heads/"'
then: {$eval: 'head_branch[11:]'}
else: ${head_branch}
in:
- 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}'
else:
@ -153,12 +159,16 @@ tasks:
then:
- 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request'
else:
$if: 'tasks_for == "action"'
$if: 'tasks_for == "github-release"'
then:
# when all actions are hooks, we can calculate this directly rather than using a variable
- '${action.repo_scope}'
- 'assume:repo:${repoUrl[8:]}:release'
else:
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
$if: 'tasks_for == "action"'
then:
# when all actions are hooks, we can calculate this directly rather than using a variable
- '${action.repo_scope}'
else:
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
requires: all-completed
priority: lowest
@ -190,6 +200,9 @@ tasks:
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
ACTION_INPUT: {$json: {$eval: 'input'}}
ACTION_CALLBACK: '${action.cb_name}'
- $if: 'tasks_for == "github-release"'
then:
GIT_TAG: '${event.release.tag_name}'
features:
taskclusterProxy: true
chainOfTrust: true
@ -252,14 +265,18 @@ tasks:
then:
symbol: D
else:
$if: 'tasks_for == "action"'
$if: 'tasks_for == "github-release"'
then:
groupName: 'action-callback'
groupSymbol: AC
symbol: "${action.symbol}"
symbol: 'ship_fenix'
else:
groupSymbol: cron
symbol: "${cron.job_symbol}"
$if: 'tasks_for == "action"'
then:
groupName: 'action-callback'
groupSymbol: AC
symbol: "${action.symbol}"
else:
groupSymbol: cron
symbol: "${cron.job_symbol}"
- $if: 'tasks_for == "action"'
then:
parent: '${action.taskGroupId}'

View File

@ -51,18 +51,16 @@ def push(builder):
def raptor(builder, is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
tasks = []
mozharness_task_id = fetch_mozharness_task_id()
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
assemble_task_id = _generate_slug_id()
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
build_task = builder.craft_assemble_raptor_task(variant)
tasks.append(build_task)
signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
tasks.append(signing_task)
for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi)
@ -73,34 +71,29 @@ def raptor(builder, is_staging):
builder.craft_raptor_youtube_playback_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
other_tasks[_generate_slug_id()] = craft_function(*args)
raptor_task = craft_function(
signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging
)
tasks.append(raptor_task)
return (build_tasks, signing_tasks, other_tasks)
return tasks
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
taskcluster_apk_paths = variant.upstream_artifacts()
build_tasks = {}
signing_tasks = {}
push_tasks = {}
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
signing_task = builder.craft_release_signing_task(
build_task['label'],
taskcluster_apk_paths,
channel=channel,
is_staging=is_staging,
)
push_task_id = _generate_slug_id()
push_tasks[push_task_id] = builder.craft_push_task(
signing_task_id,
push_task = builder.craft_push_task(
signing_task['label'],
taskcluster_apk_paths,
channel=channel,
# TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track
@ -108,7 +101,7 @@ def release(builder, channel, engine, is_staging, version_name):
is_staging=is_staging,
)
return (build_tasks, signing_tasks, push_tasks)
return [build_task, signing_task, push_task]
def release_as_fennec(builder, is_staging, version_name):

View File

@ -361,13 +361,13 @@ class TaskBuilder(object):
treeherder=treeherder,
)
def _craft_signing_task(self, name, description, signing_type, assemble_task_id, apk_paths, routes, treeherder):
def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder):
signing_format = "autograph_apk"
payload = {
'upstreamArtifacts': [{
'paths': apk_paths,
'formats': [signing_format],
'taskId': assemble_task_id,
'taskId': {'task-reference': '<build>'},
'taskType': 'build'
}]
}
@ -375,7 +375,7 @@ class TaskBuilder(object):
return self._craft_default_task_definition(
worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
provisioner_id='scriptworker-prov-v1',
dependencies=[assemble_task_id],
dependencies={'build': assemble_task_label},
routes=routes,
scopes=[
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
@ -448,7 +448,7 @@ class TaskBuilder(object):
}
def craft_raptor_signing_task(
self, assemble_task_id, variant, is_staging,
self, assemble_task_label, variant, is_staging,
):
staging_prefix = '.staging' if is_staging else ''
routes = [
@ -465,7 +465,7 @@ class TaskBuilder(object):
name='sign: {}'.format('forPerformanceTest'),
description='Dep-signing variant {}'.format('forPerformanceTest'),
signing_type="dep",
assemble_task_id=assemble_task_id,
assemble_task_label=assemble_task_label,
apk_paths=variant.upstream_artifacts(),
routes=routes,
treeherder={
@ -501,7 +501,7 @@ class TaskBuilder(object):
name="Signing {} task".format(capitalized_channel),
description="Sign {} builds of Fenix".format(capitalized_channel),
signing_type="dep" if is_staging else channel,
assemble_task_id=build_task_id,
assemble_task_label=build_task_id,
apk_paths=apk_paths,
routes=routes,
treeherder={
@ -515,7 +515,7 @@ class TaskBuilder(object):
)
def craft_push_task(
self, signing_task_id, apk_paths, channel, is_staging=False, override_google_play_track=None
self, signing_task_label, apk_paths, channel, is_staging=False, override_google_play_track=None
):
payload = {
"commit": True,
@ -524,7 +524,7 @@ class TaskBuilder(object):
"upstreamArtifacts": [
{
"paths": apk_paths,
"taskId": signing_task_id,
"taskId": {'task-reference': '<signing>'},
"taskType": "signing"
}
]
@ -536,7 +536,7 @@ class TaskBuilder(object):
return self._craft_default_task_definition(
worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
provisioner_id='scriptworker-prov-v1',
dependencies=[signing_task_id],
dependencies={'signing': signing_task_label},
routes=[],
scopes=[
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(

View File

@ -5,13 +5,14 @@
from __future__ import print_function, unicode_literals
import os
import re
import sys
current_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
sys.path.append(project_dir)
from automation.taskcluster.decision_task import pr, push
from automation.taskcluster.decision_task import pr, push, release
from automation.taskcluster.lib.tasks import TaskBuilder
@ -34,11 +35,25 @@ def loader(kind, path, config, params, loaded_tasks):
trust_level=trust_level,
)
is_staging = trust_level != 3
tasks_for = params['tasks_for']
if tasks_for == 'github-pull-request':
ordered_groups_of_tasks = pr(builder)
elif tasks_for == 'github-push':
ordered_groups_of_tasks = push(builder)
elif tasks_for == 'github-release':
git_tag = os.environ['GIT_TAG']
version = git_tag[1:] # remove prefixed "v"
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
if beta_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
elif production_semver.match(git_tag):
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
else:
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))