Let taskgraph load tasks for releases
parent
d26ef6087d
commit
294ac50f3a
157
.taskcluster.yml
157
.taskcluster.yml
|
@ -11,72 +11,81 @@ tasks:
|
|||
revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614
|
||||
trustDomain: mobile
|
||||
in:
|
||||
$if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
|
||||
then:
|
||||
$let:
|
||||
# Github events have this stuff in different places...
|
||||
ownerEmail:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.pusher.email}'
|
||||
# Assume Pull Request
|
||||
$let:
|
||||
# Github events have this stuff in different places...
|
||||
ownerEmail:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.pusher.email}'
|
||||
# Assume Pull Request
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.user.login}@users.noreply.github.com'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.user.login}@users.noreply.github.com'
|
||||
$if: 'tasks_for == "github-release"'
|
||||
then: '${event.sender.login}@users.noreply.github.com'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${tasks_for}@noreply.mozilla.org'
|
||||
baseRepoUrl:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.repository.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.base.repo.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.url}'
|
||||
repoUrl:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.repository.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.repo.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.url}'
|
||||
project:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.repository.name}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.repo.name}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.project}'
|
||||
head_branch:
|
||||
baseRepoUrl:
|
||||
$if: 'tasks_for in ["github-push", "github-release"]'
|
||||
then: '${event.repository.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: ${event.pull_request.head.ref}
|
||||
then: '${event.pull_request.base.repo.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: ${event.ref}
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.url}'
|
||||
repoUrl:
|
||||
$if: 'tasks_for in ["github-push", "github-release"]'
|
||||
then: '${event.repository.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.repo.html_url}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.url}'
|
||||
project:
|
||||
$if: 'tasks_for in ["github-push", "github-release"]'
|
||||
then: '${event.repository.name}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.repo.name}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${repository.project}'
|
||||
head_branch:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: ${event.pull_request.head.ref}
|
||||
else:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: ${event.ref}
|
||||
else:
|
||||
$if: 'tasks_for == "github-release"'
|
||||
then: '${event.release.target_commitish}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${push.branch}'
|
||||
head_sha:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.after}'
|
||||
head_sha:
|
||||
$if: 'tasks_for == "github-push"'
|
||||
then: '${event.after}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.sha}'
|
||||
else:
|
||||
$if: 'tasks_for == "github-pull-request"'
|
||||
then: '${event.pull_request.head.sha}'
|
||||
$if: 'tasks_for == "github-release"'
|
||||
then: '${event.release.tag_name}'
|
||||
else:
|
||||
$if: 'tasks_for in ["cron", "action"]'
|
||||
then: '${push.revision}'
|
||||
ownTaskId:
|
||||
$if: '"github" in tasks_for'
|
||||
then: {$eval: as_slugid("decision_task")}
|
||||
else:
|
||||
$if: 'tasks_for == "cron"'
|
||||
then: '${ownTaskId}'
|
||||
in:
|
||||
ownTaskId:
|
||||
$if: '"github" in tasks_for'
|
||||
then: {$eval: as_slugid("decision_task")}
|
||||
else:
|
||||
$if: 'tasks_for == "cron"'
|
||||
then: '${ownTaskId}'
|
||||
in:
|
||||
$if: 'tasks_for in ["github-pull-request", "github-release", "action", "cron"] || tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
|
||||
then:
|
||||
$let:
|
||||
level:
|
||||
$if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"'
|
||||
|
@ -101,7 +110,7 @@ tasks:
|
|||
$merge:
|
||||
- owner: "${ownerEmail}"
|
||||
source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml'
|
||||
- $if: 'tasks_for in ["github-push", "github-pull-request"]'
|
||||
- $if: 'tasks_for in ["github-push", "github-pull-request", "github-release"]'
|
||||
then:
|
||||
name: "Decision Task"
|
||||
description: 'The task that creates all of the other tasks in the task graph'
|
||||
|
@ -140,12 +149,9 @@ tasks:
|
|||
then:
|
||||
$let:
|
||||
short_head_branch:
|
||||
$if: 'head_branch[:10] == "refs/tags/"'
|
||||
then: {$eval: 'head_branch[10:]'}
|
||||
else:
|
||||
$if: 'head_branch[:11] == "refs/heads/"'
|
||||
then: {$eval: 'head_branch[11:]'}
|
||||
else: ${head_branch}
|
||||
$if: 'head_branch[:11] == "refs/heads/"'
|
||||
then: {$eval: 'head_branch[11:]'}
|
||||
else: ${head_branch}
|
||||
in:
|
||||
- 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}'
|
||||
else:
|
||||
|
@ -153,12 +159,16 @@ tasks:
|
|||
then:
|
||||
- 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request'
|
||||
else:
|
||||
$if: 'tasks_for == "action"'
|
||||
$if: 'tasks_for == "github-release"'
|
||||
then:
|
||||
# when all actions are hooks, we can calculate this directly rather than using a variable
|
||||
- '${action.repo_scope}'
|
||||
- 'assume:repo:${repoUrl[8:]}:release'
|
||||
else:
|
||||
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
||||
$if: 'tasks_for == "action"'
|
||||
then:
|
||||
# when all actions are hooks, we can calculate this directly rather than using a variable
|
||||
- '${action.repo_scope}'
|
||||
else:
|
||||
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
||||
|
||||
requires: all-completed
|
||||
priority: lowest
|
||||
|
@ -190,6 +200,9 @@ tasks:
|
|||
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
|
||||
ACTION_INPUT: {$json: {$eval: 'input'}}
|
||||
ACTION_CALLBACK: '${action.cb_name}'
|
||||
- $if: 'tasks_for == "github-release"'
|
||||
then:
|
||||
GIT_TAG: '${event.release.tag_name}'
|
||||
features:
|
||||
taskclusterProxy: true
|
||||
chainOfTrust: true
|
||||
|
@ -252,14 +265,18 @@ tasks:
|
|||
then:
|
||||
symbol: D
|
||||
else:
|
||||
$if: 'tasks_for == "action"'
|
||||
$if: 'tasks_for == "github-release"'
|
||||
then:
|
||||
groupName: 'action-callback'
|
||||
groupSymbol: AC
|
||||
symbol: "${action.symbol}"
|
||||
symbol: 'ship_fenix'
|
||||
else:
|
||||
groupSymbol: cron
|
||||
symbol: "${cron.job_symbol}"
|
||||
$if: 'tasks_for == "action"'
|
||||
then:
|
||||
groupName: 'action-callback'
|
||||
groupSymbol: AC
|
||||
symbol: "${action.symbol}"
|
||||
else:
|
||||
groupSymbol: cron
|
||||
symbol: "${cron.job_symbol}"
|
||||
- $if: 'tasks_for == "action"'
|
||||
then:
|
||||
parent: '${action.taskGroupId}'
|
||||
|
|
|
@ -51,18 +51,16 @@ def push(builder):
|
|||
|
||||
|
||||
def raptor(builder, is_staging):
|
||||
build_tasks = {}
|
||||
signing_tasks = {}
|
||||
other_tasks = {}
|
||||
tasks = []
|
||||
|
||||
mozharness_task_id = fetch_mozharness_task_id()
|
||||
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
|
||||
|
||||
variant = get_variant('forPerformanceTest', 'geckoNightly')
|
||||
assemble_task_id = _generate_slug_id()
|
||||
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
|
||||
build_task = builder.craft_assemble_raptor_task(variant)
|
||||
tasks.append(build_task)
|
||||
signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging)
|
||||
tasks.append(signing_task)
|
||||
|
||||
for abi in ('armeabi-v7a', 'arm64-v8a'):
|
||||
variant_apk = variant.get_apk(abi)
|
||||
|
@ -73,34 +71,29 @@ def raptor(builder, is_staging):
|
|||
builder.craft_raptor_youtube_playback_task,
|
||||
]
|
||||
for craft_function in all_raptor_craft_functions:
|
||||
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
|
||||
other_tasks[_generate_slug_id()] = craft_function(*args)
|
||||
raptor_task = craft_function(
|
||||
signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging
|
||||
)
|
||||
tasks.append(raptor_task)
|
||||
|
||||
return (build_tasks, signing_tasks, other_tasks)
|
||||
return tasks
|
||||
|
||||
|
||||
def release(builder, channel, engine, is_staging, version_name):
|
||||
variant = get_variant('fenix' + channel.capitalize(), engine)
|
||||
taskcluster_apk_paths = variant.upstream_artifacts()
|
||||
|
||||
build_tasks = {}
|
||||
signing_tasks = {}
|
||||
push_tasks = {}
|
||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
build_task_id = _generate_slug_id()
|
||||
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
|
||||
build_task_id,
|
||||
signing_task = builder.craft_release_signing_task(
|
||||
build_task['label'],
|
||||
taskcluster_apk_paths,
|
||||
channel=channel,
|
||||
is_staging=is_staging,
|
||||
)
|
||||
|
||||
push_task_id = _generate_slug_id()
|
||||
push_tasks[push_task_id] = builder.craft_push_task(
|
||||
signing_task_id,
|
||||
push_task = builder.craft_push_task(
|
||||
signing_task['label'],
|
||||
taskcluster_apk_paths,
|
||||
channel=channel,
|
||||
# TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track
|
||||
|
@ -108,7 +101,7 @@ def release(builder, channel, engine, is_staging, version_name):
|
|||
is_staging=is_staging,
|
||||
)
|
||||
|
||||
return (build_tasks, signing_tasks, push_tasks)
|
||||
return [build_task, signing_task, push_task]
|
||||
|
||||
|
||||
def release_as_fennec(builder, is_staging, version_name):
|
||||
|
|
|
@ -361,13 +361,13 @@ class TaskBuilder(object):
|
|||
treeherder=treeherder,
|
||||
)
|
||||
|
||||
def _craft_signing_task(self, name, description, signing_type, assemble_task_id, apk_paths, routes, treeherder):
|
||||
def _craft_signing_task(self, name, description, signing_type, assemble_task_label, apk_paths, routes, treeherder):
|
||||
signing_format = "autograph_apk"
|
||||
payload = {
|
||||
'upstreamArtifacts': [{
|
||||
'paths': apk_paths,
|
||||
'formats': [signing_format],
|
||||
'taskId': assemble_task_id,
|
||||
'taskId': {'task-reference': '<build>'},
|
||||
'taskType': 'build'
|
||||
}]
|
||||
}
|
||||
|
@ -375,7 +375,7 @@ class TaskBuilder(object):
|
|||
return self._craft_default_task_definition(
|
||||
worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
|
||||
provisioner_id='scriptworker-prov-v1',
|
||||
dependencies=[assemble_task_id],
|
||||
dependencies={'build': assemble_task_label},
|
||||
routes=routes,
|
||||
scopes=[
|
||||
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
|
||||
|
@ -448,7 +448,7 @@ class TaskBuilder(object):
|
|||
}
|
||||
|
||||
def craft_raptor_signing_task(
|
||||
self, assemble_task_id, variant, is_staging,
|
||||
self, assemble_task_label, variant, is_staging,
|
||||
):
|
||||
staging_prefix = '.staging' if is_staging else ''
|
||||
routes = [
|
||||
|
@ -465,7 +465,7 @@ class TaskBuilder(object):
|
|||
name='sign: {}'.format('forPerformanceTest'),
|
||||
description='Dep-signing variant {}'.format('forPerformanceTest'),
|
||||
signing_type="dep",
|
||||
assemble_task_id=assemble_task_id,
|
||||
assemble_task_label=assemble_task_label,
|
||||
apk_paths=variant.upstream_artifacts(),
|
||||
routes=routes,
|
||||
treeherder={
|
||||
|
@ -501,7 +501,7 @@ class TaskBuilder(object):
|
|||
name="Signing {} task".format(capitalized_channel),
|
||||
description="Sign {} builds of Fenix".format(capitalized_channel),
|
||||
signing_type="dep" if is_staging else channel,
|
||||
assemble_task_id=build_task_id,
|
||||
assemble_task_label=build_task_id,
|
||||
apk_paths=apk_paths,
|
||||
routes=routes,
|
||||
treeherder={
|
||||
|
@ -515,7 +515,7 @@ class TaskBuilder(object):
|
|||
)
|
||||
|
||||
def craft_push_task(
|
||||
self, signing_task_id, apk_paths, channel, is_staging=False, override_google_play_track=None
|
||||
self, signing_task_label, apk_paths, channel, is_staging=False, override_google_play_track=None
|
||||
):
|
||||
payload = {
|
||||
"commit": True,
|
||||
|
@ -524,7 +524,7 @@ class TaskBuilder(object):
|
|||
"upstreamArtifacts": [
|
||||
{
|
||||
"paths": apk_paths,
|
||||
"taskId": signing_task_id,
|
||||
"taskId": {'task-reference': '<signing>'},
|
||||
"taskType": "signing"
|
||||
}
|
||||
]
|
||||
|
@ -536,7 +536,7 @@ class TaskBuilder(object):
|
|||
return self._craft_default_task_definition(
|
||||
worker_type='mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
|
||||
provisioner_id='scriptworker-prov-v1',
|
||||
dependencies=[signing_task_id],
|
||||
dependencies={'signing': signing_task_label},
|
||||
routes=[],
|
||||
scopes=[
|
||||
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(
|
||||
|
|
|
@ -5,13 +5,14 @@
|
|||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
|
||||
sys.path.append(project_dir)
|
||||
|
||||
from automation.taskcluster.decision_task import pr, push
|
||||
from automation.taskcluster.decision_task import pr, push, release
|
||||
from automation.taskcluster.lib.tasks import TaskBuilder
|
||||
|
||||
|
||||
|
@ -34,11 +35,25 @@ def loader(kind, path, config, params, loaded_tasks):
|
|||
trust_level=trust_level,
|
||||
)
|
||||
|
||||
is_staging = trust_level != 3
|
||||
|
||||
tasks_for = params['tasks_for']
|
||||
if tasks_for == 'github-pull-request':
|
||||
ordered_groups_of_tasks = pr(builder)
|
||||
elif tasks_for == 'github-push':
|
||||
ordered_groups_of_tasks = push(builder)
|
||||
elif tasks_for == 'github-release':
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
|
||||
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
|
||||
if beta_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
|
||||
elif production_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
|
||||
else:
|
||||
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
|
||||
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
|
||||
else:
|
||||
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
|
||||
|
||||
|
|
Loading…
Reference in New Issue