Taskgraph skeleton
parent
6710e4604d
commit
daa1693f23
519
.taskcluster.yml
519
.taskcluster.yml
|
@ -1,265 +1,274 @@
|
||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
---
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
version: 1
|
version: 1
|
||||||
|
reporting: checks-v1
|
||||||
policy:
|
policy:
|
||||||
# XXX We restrict taskcluster to collaborators so priviledged tests (like UI tests) can run on PRs
|
# XXX We restrict taskcluster to collaborators so priviledged tests (like UI tests) can run on PRs
|
||||||
pullRequests: collaborators
|
pullRequests: collaborators
|
||||||
tasks:
|
tasks:
|
||||||
$let:
|
- $let:
|
||||||
decision_task_id: {$eval: as_slugid("decision_task")}
|
taskgraph:
|
||||||
expires_in: {$fromNow: '1 year'}
|
branch: taskgraph
|
||||||
user: ${event.sender.login}
|
revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614
|
||||||
project_name: fenix
|
trustDomain: mobile
|
||||||
|
|
||||||
# We define the following variable at the very top, because they are used in the
|
|
||||||
# default definition
|
|
||||||
head_branch:
|
|
||||||
$if: 'tasks_for == "github-pull-request"'
|
|
||||||
then: ${event.pull_request.head.ref}
|
|
||||||
else:
|
|
||||||
$if: 'tasks_for == "github-push"'
|
|
||||||
then: ${event.ref}
|
|
||||||
else: ${event.release.target_commitish}
|
|
||||||
|
|
||||||
head_rev:
|
|
||||||
$if: 'tasks_for == "github-pull-request"'
|
|
||||||
then: ${event.pull_request.head.sha}
|
|
||||||
else:
|
|
||||||
$if: 'tasks_for == "github-push"'
|
|
||||||
then: ${event.after}
|
|
||||||
else: ${event.release.tag_name}
|
|
||||||
|
|
||||||
repository:
|
|
||||||
$if: 'tasks_for == "github-pull-request"'
|
|
||||||
then: ${event.pull_request.head.repo.html_url}
|
|
||||||
else: ${event.repository.html_url}
|
|
||||||
|
|
||||||
scheduler_id:
|
|
||||||
$if: 'tasks_for == "cron"'
|
|
||||||
then: focus-nightly-sched # TODO: Rename to mobile-nightly-sched
|
|
||||||
else: taskcluster-github
|
|
||||||
|
|
||||||
github_repository_full_name:
|
|
||||||
$if: 'tasks_for == "github-pull-request"'
|
|
||||||
then: ${event.pull_request.base.repo.full_name}
|
|
||||||
else: ${event.repository.full_name}
|
|
||||||
|
|
||||||
trust_level:
|
|
||||||
# Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review
|
|
||||||
$if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/fenix"'
|
|
||||||
then: 3
|
|
||||||
else: 1
|
|
||||||
in:
|
|
||||||
$let:
|
|
||||||
# TODO: revisit once bug 1533314 is done to possibly infer better priorities
|
|
||||||
tasks_priority: highest
|
|
||||||
|
|
||||||
short_head_branch:
|
|
||||||
$if: 'head_branch[:10] == "refs/tags/"'
|
|
||||||
then: {$eval: 'head_branch[10:]'}
|
|
||||||
else:
|
|
||||||
$if: 'head_branch[:11] == "refs/heads/"'
|
|
||||||
then: {$eval: 'head_branch[11:]'}
|
|
||||||
else: ${head_branch}
|
|
||||||
|
|
||||||
assume_scope_prefix: assume:repo:github.com/${github_repository_full_name}
|
|
||||||
in:
|
|
||||||
$let:
|
|
||||||
default_task_definition:
|
|
||||||
taskId: ${decision_task_id}
|
|
||||||
taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust
|
|
||||||
schedulerId: ${scheduler_id}
|
|
||||||
created: {$fromNow: ''}
|
|
||||||
deadline: {$fromNow: '2 hours'}
|
|
||||||
expires: ${expires_in}
|
|
||||||
provisionerId: aws-provisioner-v1
|
|
||||||
workerType: mobile-${trust_level}-decision
|
|
||||||
priority: ${tasks_priority}
|
|
||||||
requires: all-completed # Must be explicit because of Chain of Trust
|
|
||||||
retries: 5
|
|
||||||
routes:
|
|
||||||
$flatten:
|
|
||||||
- statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust
|
|
||||||
- $if: 'trust_level == 3'
|
|
||||||
then:
|
|
||||||
- tc-treeherder.v2.${project_name}.${head_rev}
|
|
||||||
else: []
|
|
||||||
payload:
|
|
||||||
maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks
|
|
||||||
image: mozillamobile/${project_name}:1.4
|
|
||||||
command:
|
|
||||||
- /bin/bash
|
|
||||||
- --login
|
|
||||||
- -cx
|
|
||||||
# The rest of the command must be defined below
|
|
||||||
env:
|
|
||||||
BUILD_DATE: ${now}
|
|
||||||
MOBILE_HEAD_BRANCH: ${head_branch}
|
|
||||||
MOBILE_HEAD_REPOSITORY: ${repository}
|
|
||||||
MOBILE_HEAD_REV: ${head_rev}
|
|
||||||
MOBILE_TRIGGERED_BY: ${user}
|
|
||||||
SCHEDULER_ID: ${scheduler_id}
|
|
||||||
SHORT_HEAD_BRANCH: ${short_head_branch}
|
|
||||||
TASK_ID: ${decision_task_id}
|
|
||||||
TASKS_PRIORITY: ${tasks_priority}
|
|
||||||
TRUST_LEVEL: ${trust_level}
|
|
||||||
features:
|
|
||||||
chainOfTrust: true
|
|
||||||
taskclusterProxy: true
|
|
||||||
artifacts:
|
|
||||||
public/task-graph.json:
|
|
||||||
type: file
|
|
||||||
path: /opt/${project_name}/task-graph.json
|
|
||||||
expires: ${expires_in}
|
|
||||||
public/actions.json:
|
|
||||||
type: file
|
|
||||||
path: /opt/${project_name}/actions.json
|
|
||||||
expires: ${expires_in}
|
|
||||||
public/parameters.yml:
|
|
||||||
type: file
|
|
||||||
path: /opt/${project_name}/parameters.yml
|
|
||||||
expires: ${expires_in}
|
|
||||||
extra:
|
|
||||||
tasks_for: ${tasks_for}
|
|
||||||
treeherder:
|
|
||||||
machine:
|
|
||||||
platform: mobile-decision
|
|
||||||
metadata:
|
|
||||||
owner: ${user}@users.noreply.github.com
|
|
||||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
|
||||||
in:
|
in:
|
||||||
$flatten:
|
$if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]'
|
||||||
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]'
|
then:
|
||||||
then:
|
|
||||||
$let:
|
$let:
|
||||||
pull_request_title: ${event.pull_request.title}
|
# Github events have this stuff in different places...
|
||||||
pull_request_number: ${event.pull_request.number}
|
ownerEmail:
|
||||||
pull_request_url: ${event.pull_request.html_url}
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: '${event.pusher.email}'
|
||||||
|
# Assume Pull Request
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: '${event.pull_request.user.login}@users.noreply.github.com'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${tasks_for}@noreply.mozilla.org'
|
||||||
|
baseRepoUrl:
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: '${event.repository.html_url}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: '${event.pull_request.base.repo.html_url}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${repository.url}'
|
||||||
|
repoUrl:
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: '${event.repository.html_url}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: '${event.pull_request.head.repo.html_url}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${repository.url}'
|
||||||
|
project:
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: '${event.repository.name}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: '${event.pull_request.head.repo.name}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${repository.project}'
|
||||||
|
head_branch:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: ${event.pull_request.head.ref}
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: ${event.ref}
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${push.branch}'
|
||||||
|
head_sha:
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then: '${event.after}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then: '${event.pull_request.head.sha}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
|
then: '${push.revision}'
|
||||||
|
ownTaskId:
|
||||||
|
$if: '"github" in tasks_for'
|
||||||
|
then: {$eval: as_slugid("decision_task")}
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "cron"'
|
||||||
|
then: '${ownTaskId}'
|
||||||
in:
|
in:
|
||||||
$mergeDeep:
|
$let:
|
||||||
- {$eval: 'default_task_definition'}
|
level:
|
||||||
- scopes:
|
$if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"'
|
||||||
- ${assume_scope_prefix}:pull-request
|
then: '3'
|
||||||
payload:
|
else:
|
||||||
command:
|
$if: 'tasks_for in ["cron", "action"]'
|
||||||
- >-
|
then: '${repository.level}'
|
||||||
git fetch ${repository} ${head_branch}
|
else: 1
|
||||||
&& git config advice.detachedHead false
|
in:
|
||||||
&& git checkout FETCH_HEAD
|
taskId: '${ownTaskId}'
|
||||||
&& python automation/taskcluster/decision_task.py pull-request
|
taskGroupId:
|
||||||
env:
|
$if: 'tasks_for == "action"'
|
||||||
GITHUB_PULL_TITLE: ${pull_request_title}
|
then:
|
||||||
MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number}
|
'${action.taskGroupId}'
|
||||||
extra:
|
else:
|
||||||
treeherder:
|
'${ownTaskId}' # same as taskId; this is how automation identifies a decision task
|
||||||
symbol: D-PR
|
schedulerId: '${trustDomain}-level-${level}'
|
||||||
metadata:
|
created: {$fromNow: ''}
|
||||||
name: '${project_name} - Decision task (Pull Request #${pull_request_number})'
|
deadline: {$fromNow: '1 day'}
|
||||||
description: 'Building and testing ${project_name} - triggered by [#${pull_request_number}](${pull_request_url})'
|
expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors
|
||||||
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
|
|
||||||
then:
|
|
||||||
$mergeDeep:
|
|
||||||
- {$eval: 'default_task_definition'}
|
|
||||||
- scopes:
|
|
||||||
- ${assume_scope_prefix}:branch:${short_head_branch}
|
|
||||||
payload:
|
|
||||||
command:
|
|
||||||
- >-
|
|
||||||
git fetch ${repository} ${head_branch}
|
|
||||||
&& git config advice.detachedHead false
|
|
||||||
&& git checkout FETCH_HEAD
|
|
||||||
&& python automation/taskcluster/decision_task.py push
|
|
||||||
extra:
|
|
||||||
treeherder:
|
|
||||||
symbol: D
|
|
||||||
metadata:
|
|
||||||
name: ${project_name} VCS-Push Decision task
|
|
||||||
description: Schedules the build and test tasks for ${project_name}.
|
|
||||||
- $if: 'tasks_for == "github-release" && event["action"] == "published"'
|
|
||||||
then:
|
|
||||||
$mergeDeep:
|
|
||||||
- {$eval: 'default_task_definition'}
|
|
||||||
- scopes:
|
|
||||||
- ${assume_scope_prefix}:release
|
|
||||||
payload:
|
|
||||||
command:
|
|
||||||
- >-
|
|
||||||
git fetch ${repository} refs/tags/${head_rev}
|
|
||||||
&& git config advice.detachedHead false
|
|
||||||
&& git checkout FETCH_HEAD
|
|
||||||
&& python automation/taskcluster/decision_task.py github-release ${event.release.tag_name}
|
|
||||||
extra:
|
|
||||||
treeherder:
|
|
||||||
symbol: D-github-release
|
|
||||||
metadata:
|
|
||||||
name: ${project_name} Github Release Decision Task
|
|
||||||
description: Building and releasing ${project_name} ${event.release.tag_name}
|
|
||||||
- $if: 'tasks_for == "cron"'
|
|
||||||
then:
|
|
||||||
$let:
|
|
||||||
staging_flag:
|
|
||||||
$if: 'trust_level == 3'
|
|
||||||
then: ''
|
|
||||||
else: '--staging'
|
|
||||||
in:
|
|
||||||
- $if: 'cron.name == "nightly"'
|
|
||||||
then:
|
|
||||||
$mergeDeep:
|
|
||||||
- {$eval: 'default_task_definition'}
|
|
||||||
- scopes:
|
|
||||||
- $if: 'trust_level == 3'
|
|
||||||
then: assume:hook-id:project-mobile/${project_name}-nightly
|
|
||||||
else: assume:hook-id:project-mobile/${project_name}-nightly-staging
|
|
||||||
routes:
|
|
||||||
$if: 'trust_level == 3'
|
|
||||||
then:
|
|
||||||
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
|
|
||||||
payload:
|
|
||||||
command:
|
|
||||||
- >-
|
|
||||||
git fetch ${repository} ${head_branch}
|
|
||||||
&& git config advice.detachedHead false
|
|
||||||
&& git checkout FETCH_HEAD
|
|
||||||
&& python automation/taskcluster/decision_task.py nightly ${staging_flag}
|
|
||||||
extra:
|
|
||||||
cron: {$json: {$eval: 'cron'}}
|
|
||||||
treeherder:
|
|
||||||
symbol: nightly-D
|
|
||||||
metadata:
|
metadata:
|
||||||
name: ${project_name} Nightly Decision Task
|
$merge:
|
||||||
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
|
- owner: "${ownerEmail}"
|
||||||
- $if: 'cron.name == "raptor"'
|
source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml'
|
||||||
then:
|
- $if: 'tasks_for in ["github-push", "github-pull-request"]'
|
||||||
$mergeDeep:
|
then:
|
||||||
- {$eval: 'default_task_definition'}
|
name: "Decision Task"
|
||||||
- scopes:
|
description: 'The task that creates all of the other tasks in the task graph'
|
||||||
- $if: 'trust_level == 3'
|
else:
|
||||||
then: assume:hook-id:project-mobile/${project_name}-raptor
|
$if: 'tasks_for == "action"'
|
||||||
else: assume:hook-id:project-mobile/${project_name}-raptor-staging
|
then:
|
||||||
|
name: "Action: ${action.title}"
|
||||||
|
description: '${action.description}'
|
||||||
|
else:
|
||||||
|
name: "Decision Task for cron job ${cron.job_name}"
|
||||||
|
description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})'
|
||||||
|
provisionerId: "aws-provisioner-v1"
|
||||||
|
workerType: "mobile-${level}-decision"
|
||||||
|
tags:
|
||||||
|
$if: 'tasks_for in ["github-push", "github-pull-request"]'
|
||||||
|
then:
|
||||||
|
kind: decision-task
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "action"'
|
||||||
|
then:
|
||||||
|
kind: 'action-callback'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "cron"'
|
||||||
|
then:
|
||||||
|
kind: cron-task
|
||||||
routes:
|
routes:
|
||||||
$if: 'trust_level == 3'
|
$flatten:
|
||||||
then:
|
- checks
|
||||||
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
|
- $if: 'tasks_for != "github-pull-request"'
|
||||||
- notify.email.perftest-alerts@mozilla.com.on-failed
|
then:
|
||||||
|
- "tc-treeherder.v2.${project}.${head_sha}"
|
||||||
|
else: []
|
||||||
|
scopes:
|
||||||
|
# `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol.
|
||||||
|
$if: 'tasks_for == "github-push"'
|
||||||
|
then:
|
||||||
|
$let:
|
||||||
|
short_head_branch:
|
||||||
|
$if: 'head_branch[:10] == "refs/tags/"'
|
||||||
|
then: {$eval: 'head_branch[10:]'}
|
||||||
|
else:
|
||||||
|
$if: 'head_branch[:11] == "refs/heads/"'
|
||||||
|
then: {$eval: 'head_branch[11:]'}
|
||||||
|
else: ${head_branch}
|
||||||
|
in:
|
||||||
|
- 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "github-pull-request"'
|
||||||
|
then:
|
||||||
|
- 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request'
|
||||||
|
else:
|
||||||
|
$if: 'tasks_for == "action"'
|
||||||
|
then:
|
||||||
|
# when all actions are hooks, we can calculate this directly rather than using a variable
|
||||||
|
- '${action.repo_scope}'
|
||||||
|
else:
|
||||||
|
- 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}'
|
||||||
|
|
||||||
|
requires: all-completed
|
||||||
|
priority: lowest
|
||||||
|
retries: 5
|
||||||
|
|
||||||
payload:
|
payload:
|
||||||
command:
|
env:
|
||||||
- >-
|
# run-task uses these to check out the source; the inputs
|
||||||
git fetch ${repository} ${head_branch}
|
# to `mach taskgraph decision` are all on the command line.
|
||||||
&& git config advice.detachedHead false
|
$merge:
|
||||||
&& git checkout FETCH_HEAD
|
- MOBILE_BASE_REPOSITORY: '${baseRepoUrl}'
|
||||||
&& python automation/taskcluster/decision_task.py raptor ${staging_flag}
|
MOBILE_HEAD_REPOSITORY: '${repoUrl}'
|
||||||
|
MOBILE_HEAD_REF: '${head_branch}'
|
||||||
|
MOBILE_HEAD_REV: '${head_sha}'
|
||||||
|
MOBILE_REPOSITORY_TYPE: git
|
||||||
|
TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph
|
||||||
|
TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch}
|
||||||
|
TASKGRAPH_HEAD_REV: ${taskgraph.revision}
|
||||||
|
TASKGRAPH_REPOSITORY_TYPE: hg
|
||||||
|
REPOSITORIES: {$json: {mobile: "Fenix", taskgraph: "Taskgraph"}}
|
||||||
|
HG_STORE_PATH: /builds/worker/checkouts/hg-store
|
||||||
|
ANDROID_SDK_ROOT: /builds/worker/android-sdk
|
||||||
|
- $if: 'tasks_for in ["github-pull-request"]'
|
||||||
|
then:
|
||||||
|
MOBILE_PULL_REQUEST_NUMBER: '${event.pull_request.number}'
|
||||||
|
- $if: 'tasks_for == "action"'
|
||||||
|
then:
|
||||||
|
ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task
|
||||||
|
ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded)
|
||||||
|
ACTION_INPUT: {$json: {$eval: 'input'}}
|
||||||
|
ACTION_CALLBACK: '${action.cb_name}'
|
||||||
|
features:
|
||||||
|
taskclusterProxy: true
|
||||||
|
chainOfTrust: true
|
||||||
|
# Note: This task is built server side without the context or tooling that
|
||||||
|
# exist in tree so we must hard code the hash
|
||||||
|
image:
|
||||||
|
mozillareleases/taskgraph:decision-mobile-6020473b1a928d8df50e234a7ca2e81ade2220a4fb5fbe16b02477dd64a49728@sha256:98d226736b7d03907114bf37938002b90e8a37cbe3a297690e349f1ddddb1d7c
|
||||||
|
|
||||||
|
maxRunTime: 1800
|
||||||
|
|
||||||
|
command:
|
||||||
|
- /usr/local/bin/run-task
|
||||||
|
- '--mobile-checkout=/builds/worker/checkouts/src'
|
||||||
|
- '--taskgraph-checkout=/builds/worker/checkouts/taskgraph'
|
||||||
|
- '--task-cwd=/builds/worker/checkouts/src'
|
||||||
|
- '--'
|
||||||
|
- bash
|
||||||
|
- -cx
|
||||||
|
- $let:
|
||||||
|
extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''}
|
||||||
|
in:
|
||||||
|
$if: 'tasks_for == "action"'
|
||||||
|
then: >
|
||||||
|
cd /builds/worker/checkouts/src &&
|
||||||
|
ln -s /builds/worker/artifacts artifacts &&
|
||||||
|
taskgraph action-callback
|
||||||
|
else: >
|
||||||
|
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
|
||||||
|
taskcluster/scripts/install-sdk.sh &&
|
||||||
|
ln -s /builds/worker/artifacts artifacts &&
|
||||||
|
~/.local/bin/taskgraph decision
|
||||||
|
--pushlog-id='0'
|
||||||
|
--pushdate='0'
|
||||||
|
--project='${project}'
|
||||||
|
--message=""
|
||||||
|
--owner='${ownerEmail}'
|
||||||
|
--level='${level}'
|
||||||
|
--base-repository="$MOBILE_BASE_REPOSITORY"
|
||||||
|
--head-repository="$MOBILE_HEAD_REPOSITORY"
|
||||||
|
--head-ref="$MOBILE_HEAD_REF"
|
||||||
|
--head-rev="$MOBILE_HEAD_REV"
|
||||||
|
--repository-type="$MOBILE_REPOSITORY_TYPE"
|
||||||
|
--tasks-for='${tasks_for}'
|
||||||
|
${extraArgs}
|
||||||
|
|
||||||
|
artifacts:
|
||||||
|
'public':
|
||||||
|
type: 'directory'
|
||||||
|
path: '/builds/worker/artifacts'
|
||||||
|
expires: {$fromNow: '1 year'}
|
||||||
|
|
||||||
extra:
|
extra:
|
||||||
cron: {$json: {$eval: 'cron'}}
|
$merge:
|
||||||
treeherder:
|
- treeherder:
|
||||||
symbol: raptor-D
|
$merge:
|
||||||
notify:
|
- machine:
|
||||||
email:
|
platform: gecko-decision
|
||||||
link:
|
- $if: 'tasks_for in ["github-push", "github-pull-request"]'
|
||||||
text: Treeherder Job
|
then:
|
||||||
href: "https://treeherder.mozilla.org/#/jobs?repo=${project_name}&revision=${head_rev}"
|
symbol: D
|
||||||
subject: "[${project_name}] Raptor decision task failed"
|
else:
|
||||||
content: "This calls for an action of the development team. If needed, escalate to the release engineering team in #releaseduty-mobile on Slack. Use the link to view it on Treeherder."
|
$if: 'tasks_for == "action"'
|
||||||
metadata:
|
then:
|
||||||
name: ${project_name} Raptor Decision Task
|
groupName: 'action-callback'
|
||||||
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
|
groupSymbol: AC
|
||||||
|
symbol: "${action.symbol}"
|
||||||
|
else:
|
||||||
|
groupSymbol: cron
|
||||||
|
symbol: "${cron.job_symbol}"
|
||||||
|
- $if: 'tasks_for == "action"'
|
||||||
|
then:
|
||||||
|
parent: '${action.taskGroupId}'
|
||||||
|
action:
|
||||||
|
name: '${action.name}'
|
||||||
|
context:
|
||||||
|
taskGroupId: '${action.taskGroupId}'
|
||||||
|
taskId: {$eval: 'taskId'}
|
||||||
|
input: {$eval: 'input'}
|
||||||
|
- $if: 'tasks_for == "cron"'
|
||||||
|
then:
|
||||||
|
cron: {$json: {$eval: 'cron'}}
|
||||||
|
- tasks_for: '${tasks_for}'
|
||||||
|
|
|
@ -258,7 +258,4 @@ if __name__ == "__main__":
|
||||||
else:
|
else:
|
||||||
raise Exception('Unsupported command "{}"'.format(command))
|
raise Exception('Unsupported command "{}"'.format(command))
|
||||||
|
|
||||||
full_task_graph = schedule_task_graph(ordered_groups_of_tasks)
|
|
||||||
|
|
||||||
populate_chain_of_trust_task_graph(full_task_graph)
|
populate_chain_of_trust_task_graph(full_task_graph)
|
||||||
populate_chain_of_trust_required_but_unused_files()
|
|
||||||
|
|
|
@ -0,0 +1,58 @@
|
||||||
|
---
|
||||||
|
trust-domain: mobile
|
||||||
|
treeherder:
|
||||||
|
group-names:
|
||||||
|
'I': 'Docker Image Builds'
|
||||||
|
'Rap': 'Raptor tests'
|
||||||
|
'Rap-P': 'Raptor power tests'
|
||||||
|
|
||||||
|
task-priority: lowest
|
||||||
|
|
||||||
|
taskgraph:
|
||||||
|
register: fenix_taskgraph:register
|
||||||
|
repositories:
|
||||||
|
mobile:
|
||||||
|
name: "Fenix"
|
||||||
|
cached-task-prefix: project.mobile.fenix
|
||||||
|
|
||||||
|
workers:
|
||||||
|
aliases:
|
||||||
|
b-android:
|
||||||
|
provisioner: aws-provisioner-v1
|
||||||
|
implementation: docker-worker
|
||||||
|
os: linux
|
||||||
|
worker-type: 'mobile-{level}-b-ref-browser'
|
||||||
|
images:
|
||||||
|
provisioner: aws-provisioner-v1
|
||||||
|
implementation: docker-worker
|
||||||
|
os: linux
|
||||||
|
worker-type: 'mobile-{level}-images'
|
||||||
|
dep-signing:
|
||||||
|
provisioner: scriptworker-prov-v1
|
||||||
|
implementation: scriptworker-signing
|
||||||
|
os: scriptworker
|
||||||
|
worker-type: mobile-signing-dep-v1
|
||||||
|
signing:
|
||||||
|
provisioner: scriptworker-prov-v1
|
||||||
|
implementation: scriptworker-signing
|
||||||
|
os: scriptworker
|
||||||
|
worker-type:
|
||||||
|
by-level:
|
||||||
|
"3": mobile-signing-v1
|
||||||
|
default: mobile-signing-dep-v1
|
||||||
|
push-apk:
|
||||||
|
provisioner: scriptworker-prov-v1
|
||||||
|
implementation: scriptworker-pushapk
|
||||||
|
os: scriptworker
|
||||||
|
worker-type:
|
||||||
|
by-level:
|
||||||
|
"3": mobile-pushapk-v1
|
||||||
|
default: mobile-pushapk-dep-v1
|
||||||
|
t-bitbar.*:
|
||||||
|
provisioner: proj-autophone
|
||||||
|
implementation: generic-worker
|
||||||
|
os: linux-bitbar
|
||||||
|
worker-type: 'gecko-{alias}'
|
||||||
|
|
||||||
|
scriptworker:
|
||||||
|
scope-prefix: project:mobile:fenix:releng
|
|
@ -0,0 +1,20 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
from importlib import import_module
|
||||||
|
|
||||||
|
|
||||||
|
def register(graph_config):
|
||||||
|
"""
|
||||||
|
Import all modules that are siblings of this one, triggering decorators in
|
||||||
|
the process.
|
||||||
|
"""
|
||||||
|
_import_modules(["job", "worker_types", "routes", "target"])
|
||||||
|
|
||||||
|
|
||||||
|
def _import_modules(modules):
|
||||||
|
for module in modules:
|
||||||
|
import_module(".{}".format(module), package=__name__)
|
|
@ -0,0 +1,79 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
|
||||||
|
from taskgraph.util import path
|
||||||
|
from taskgraph.util.schema import Schema
|
||||||
|
from voluptuous import Required, Optional
|
||||||
|
from six import text_type
|
||||||
|
|
||||||
|
from pipes import quote as shell_quote
|
||||||
|
|
||||||
|
gradlew_schema = Schema(
|
||||||
|
{
|
||||||
|
Required("using"): "gradlew",
|
||||||
|
Required("gradlew"): [text_type],
|
||||||
|
Optional("post-gradlew"): [[text_type]],
|
||||||
|
# Base work directory used to set up the task.
|
||||||
|
Required("workdir"): text_type,
|
||||||
|
Optional("use-caches"): bool,
|
||||||
|
Optional("secrets"): [
|
||||||
|
{
|
||||||
|
Required("name"): text_type,
|
||||||
|
Required("path"): text_type,
|
||||||
|
Required("key"): text_type,
|
||||||
|
Optional("json"): bool,
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
|
||||||
|
def configure_gradlew(config, job, taskdesc):
|
||||||
|
run = job["run"]
|
||||||
|
worker = taskdesc["worker"] = job["worker"]
|
||||||
|
|
||||||
|
worker.setdefault("env", {}).update(
|
||||||
|
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
|
||||||
|
)
|
||||||
|
|
||||||
|
# defer to the run_task implementation
|
||||||
|
run["command"] = _extract_command(run)
|
||||||
|
secrets = run.pop("secrets", [])
|
||||||
|
scopes = taskdesc.setdefault("scopes", [])
|
||||||
|
scopes.extend(["secrets:get:{}".format(secret["name"]) for secret in secrets])
|
||||||
|
|
||||||
|
run["cwd"] = "{checkout}"
|
||||||
|
run["using"] = "run-task"
|
||||||
|
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_command(run):
|
||||||
|
pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]]
|
||||||
|
pre_gradle_commands += [
|
||||||
|
_generate_secret_command(secret) for secret in run.get("secrets", [])
|
||||||
|
]
|
||||||
|
|
||||||
|
gradle_command = ["./gradlew"] + run.pop("gradlew")
|
||||||
|
post_gradle_commands = run.pop("post-gradlew", [])
|
||||||
|
|
||||||
|
commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
|
||||||
|
shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands]
|
||||||
|
return " && ".join(shell_quoted_commands)
|
||||||
|
|
||||||
|
|
||||||
|
def _generate_secret_command(secret):
|
||||||
|
secret_command = [
|
||||||
|
"taskcluster/scripts/get-secret.py",
|
||||||
|
"-s", secret["name"],
|
||||||
|
"-k", secret["key"],
|
||||||
|
"-f", secret["path"],
|
||||||
|
]
|
||||||
|
if secret.get("json"):
|
||||||
|
secret_command.append("--json")
|
||||||
|
|
||||||
|
return secret_command
|
|
@ -0,0 +1,34 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
import time
|
||||||
|
|
||||||
|
from taskgraph.transforms.task import index_builder
|
||||||
|
|
||||||
|
SIGNING_ROUTE_TEMPLATES = [
|
||||||
|
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}",
|
||||||
|
"index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest",
|
||||||
|
"index.project.{trust-domain}.{project}.v3.{variant}.latest",
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@index_builder("signing")
|
||||||
|
def add_signing_indexes(config, task):
|
||||||
|
routes = task.setdefault("routes", [])
|
||||||
|
|
||||||
|
if config.params["level"] != "3":
|
||||||
|
return task
|
||||||
|
|
||||||
|
subs = config.params.copy()
|
||||||
|
subs["build_date"] = time.strftime(
|
||||||
|
"%Y.%m.%d", time.gmtime(config.params["build_date"])
|
||||||
|
)
|
||||||
|
subs["trust-domain"] = config.graph_config["trust-domain"]
|
||||||
|
subs["variant"] = task["attributes"]["build-type"]
|
||||||
|
|
||||||
|
for tpl in SIGNING_ROUTE_TEMPLATES:
|
||||||
|
routes.append(tpl.format(**subs))
|
||||||
|
return task
|
|
@ -0,0 +1,17 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
from taskgraph.target_tasks import _target_task as target_task
|
||||||
|
|
||||||
|
|
||||||
|
@target_task("nightly")
|
||||||
|
def target_tasks_nightly(full_task_graph, parameters, graph_config):
|
||||||
|
"""Select the set of tasks required for a nightly build."""
|
||||||
|
|
||||||
|
def filter(task, parameters):
|
||||||
|
return task.attributes.get("nightly", False)
|
||||||
|
|
||||||
|
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
|
@ -0,0 +1,94 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
from six import text_type
|
||||||
|
|
||||||
|
from voluptuous import Required
|
||||||
|
|
||||||
|
from taskgraph.util.schema import taskref_or_string
|
||||||
|
from taskgraph.transforms.task import payload_builder
|
||||||
|
|
||||||
|
|
||||||
|
@payload_builder(
|
||||||
|
"scriptworker-signing",
|
||||||
|
schema={
|
||||||
|
# the maximum time to run, in seconds
|
||||||
|
Required("max-run-time"): int,
|
||||||
|
Required("signing-type"): text_type,
|
||||||
|
# list of artifact URLs for the artifacts that should be signed
|
||||||
|
Required("upstream-artifacts"): [
|
||||||
|
{
|
||||||
|
# taskId of the task with the artifact
|
||||||
|
Required("taskId"): taskref_or_string,
|
||||||
|
# type of signing task (for CoT)
|
||||||
|
Required("taskType"): text_type,
|
||||||
|
# Paths to the artifacts to sign
|
||||||
|
Required("paths"): [text_type],
|
||||||
|
# Signing formats to use on each of the paths
|
||||||
|
Required("formats"): [text_type],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def build_scriptworker_signing_payload(config, task, task_def):
|
||||||
|
worker = task["worker"]
|
||||||
|
|
||||||
|
task_def["tags"]["worker-implementation"] = "scriptworker"
|
||||||
|
|
||||||
|
task_def["payload"] = {
|
||||||
|
"maxRunTime": worker["max-run-time"],
|
||||||
|
"upstreamArtifacts": worker["upstream-artifacts"],
|
||||||
|
}
|
||||||
|
|
||||||
|
formats = set()
|
||||||
|
for artifacts in worker["upstream-artifacts"]:
|
||||||
|
formats.update(artifacts["formats"])
|
||||||
|
|
||||||
|
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
|
||||||
|
task_def["scopes"].append(
|
||||||
|
"{}:signing:cert:{}".format(scope_prefix, worker["signing-type"])
|
||||||
|
)
|
||||||
|
task_def["scopes"].extend(
|
||||||
|
[
|
||||||
|
"{}:signing:format:{}".format(scope_prefix, format)
|
||||||
|
for format in sorted(formats)
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@payload_builder(
|
||||||
|
"scriptworker-pushapk",
|
||||||
|
schema={
|
||||||
|
Required("upstream-artifacts"): [
|
||||||
|
{
|
||||||
|
Required("taskId"): taskref_or_string,
|
||||||
|
Required("taskType"): text_type,
|
||||||
|
Required("paths"): [text_type],
|
||||||
|
}
|
||||||
|
],
|
||||||
|
Required("channel"): text_type,
|
||||||
|
Required("commit"): bool,
|
||||||
|
Required("product"): text_type,
|
||||||
|
Required("dep"): bool,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
def build_push_apk_payload(config, task, task_def):
|
||||||
|
worker = task["worker"]
|
||||||
|
|
||||||
|
task_def["tags"]["worker-implementation"] = "scriptworker"
|
||||||
|
|
||||||
|
task_def["payload"] = {
|
||||||
|
"commit": worker["commit"],
|
||||||
|
"upstreamArtifacts": worker["upstream-artifacts"],
|
||||||
|
"channel": worker["channel"],
|
||||||
|
}
|
||||||
|
|
||||||
|
scope_prefix = config.graph_config["scriptworker"]["scope-prefix"]
|
||||||
|
task_def["scopes"].append(
|
||||||
|
"{}:googleplay:product:{}{}".format(
|
||||||
|
scope_prefix, worker["product"], ":dep" if worker["dep"] else ""
|
||||||
|
)
|
||||||
|
)
|
|
@ -0,0 +1,9 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set +x
|
||||||
|
|
||||||
|
ANDROID_SDK_VERSION=3859397
|
||||||
|
|
||||||
|
curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
|
||||||
|
unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
|
||||||
|
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
|
Loading…
Reference in New Issue