From daa1693f231be88e823531c6b2962387c78a5f56 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 12 Sep 2019 14:06:28 +0200 Subject: [PATCH] Taskgraph skeleton --- .taskcluster.yml | 519 ++++++++++---------- automation/taskcluster/decision_task.py | 3 - taskcluster/ci/config.yml | 58 +++ taskcluster/fenix_taskgraph/__init__.py | 20 + taskcluster/fenix_taskgraph/job.py | 79 +++ taskcluster/fenix_taskgraph/routes.py | 34 ++ taskcluster/fenix_taskgraph/target.py | 17 + taskcluster/fenix_taskgraph/worker_types.py | 94 ++++ taskcluster/scripts/install-sdk.sh | 9 + 9 files changed, 575 insertions(+), 258 deletions(-) create mode 100644 taskcluster/ci/config.yml create mode 100644 taskcluster/fenix_taskgraph/__init__.py create mode 100644 taskcluster/fenix_taskgraph/job.py create mode 100644 taskcluster/fenix_taskgraph/routes.py create mode 100644 taskcluster/fenix_taskgraph/target.py create mode 100644 taskcluster/fenix_taskgraph/worker_types.py create mode 100755 taskcluster/scripts/install-sdk.sh diff --git a/.taskcluster.yml b/.taskcluster.yml index 80cd192a0..4a1491a2b 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -1,265 +1,274 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- version: 1 +reporting: checks-v1 policy: - # XXX We restrict taskcluster to collaborators so priviledged tests (like UI tests) can run on PRs - pullRequests: collaborators + # XXX We restrict taskcluster to collaborators so priviledged tests (like UI tests) can run on PRs + pullRequests: collaborators tasks: - $let: - decision_task_id: {$eval: as_slugid("decision_task")} - expires_in: {$fromNow: '1 year'} - user: ${event.sender.login} - project_name: fenix - - # We define the following variable at the very top, because they are used in the - # default definition - head_branch: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.ref} - else: - $if: 'tasks_for == "github-push"' - then: ${event.ref} - else: ${event.release.target_commitish} - - head_rev: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.sha} - else: - $if: 'tasks_for == "github-push"' - then: ${event.after} - else: ${event.release.tag_name} - - repository: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.head.repo.html_url} - else: ${event.repository.html_url} - - scheduler_id: - $if: 'tasks_for == "cron"' - then: focus-nightly-sched # TODO: Rename to mobile-nightly-sched - else: taskcluster-github - - github_repository_full_name: - $if: 'tasks_for == "github-pull-request"' - then: ${event.pull_request.base.repo.full_name} - else: ${event.repository.full_name} - - trust_level: - # Pull requests on main repository can't be trusted because anybody can open a PR on it, without a review - $if: 'tasks_for in ["github-push", "github-release", "cron"] && event.repository.html_url == "https://github.com/mozilla-mobile/fenix"' - then: 3 - else: 1 - in: - $let: - # TODO: revisit once bug 1533314 is done to possibly infer better priorities - tasks_priority: highest - - short_head_branch: - $if: 'head_branch[:10] == "refs/tags/"' - then: {$eval: 'head_branch[10:]'} - else: - $if: 'head_branch[:11] == "refs/heads/"' - then: {$eval: 'head_branch[11:]'} - else: ${head_branch} - - assume_scope_prefix: assume:repo:github.com/${github_repository_full_name} - in: - $let: - default_task_definition: - taskId: ${decision_task_id} - taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust - schedulerId: ${scheduler_id} - created: {$fromNow: ''} - deadline: {$fromNow: '2 hours'} - expires: ${expires_in} - provisionerId: aws-provisioner-v1 - workerType: mobile-${trust_level}-decision - priority: ${tasks_priority} - requires: all-completed # Must be explicit because of Chain of Trust - retries: 5 - routes: - $flatten: - - statuses # Automatically added by taskcluster-github. It must be explicit because of Chain of Trust - - $if: 'trust_level == 3' - then: - - tc-treeherder.v2.${project_name}.${head_rev} - else: [] - payload: - maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks - image: mozillamobile/${project_name}:1.4 - command: - - /bin/bash - - --login - - -cx - # The rest of the command must be defined below - env: - BUILD_DATE: ${now} - MOBILE_HEAD_BRANCH: ${head_branch} - MOBILE_HEAD_REPOSITORY: ${repository} - MOBILE_HEAD_REV: ${head_rev} - MOBILE_TRIGGERED_BY: ${user} - SCHEDULER_ID: ${scheduler_id} - SHORT_HEAD_BRANCH: ${short_head_branch} - TASK_ID: ${decision_task_id} - TASKS_PRIORITY: ${tasks_priority} - TRUST_LEVEL: ${trust_level} - features: - chainOfTrust: true - taskclusterProxy: true - artifacts: - public/task-graph.json: - type: file - path: /opt/${project_name}/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /opt/${project_name}/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /opt/${project_name}/parameters.yml - expires: ${expires_in} - extra: - tasks_for: ${tasks_for} - treeherder: - machine: - platform: mobile-decision - metadata: - owner: ${user}@users.noreply.github.com - source: ${repository}/raw/${head_rev}/.taskcluster.yml + - $let: + taskgraph: + branch: taskgraph + revision: 0c5a68749f9a7672a7e56604b69a7bd41b036614 + trustDomain: mobile in: - $flatten: - - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "edited", "synchronize"]' - then: + $if: 'tasks_for in ["github-pull-request", "github-push", "action", "cron"]' + then: $let: - pull_request_title: ${event.pull_request.title} - pull_request_number: ${event.pull_request.number} - pull_request_url: ${event.pull_request.html_url} + # Github events have this stuff in different places... + ownerEmail: + $if: 'tasks_for == "github-push"' + then: '${event.pusher.email}' + # Assume Pull Request + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.user.login}@users.noreply.github.com' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${tasks_for}@noreply.mozilla.org' + baseRepoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.base.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + repoUrl: + $if: 'tasks_for == "github-push"' + then: '${event.repository.html_url}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.html_url}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.url}' + project: + $if: 'tasks_for == "github-push"' + then: '${event.repository.name}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.repo.name}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.project}' + head_branch: + $if: 'tasks_for == "github-pull-request"' + then: ${event.pull_request.head.ref} + else: + $if: 'tasks_for == "github-push"' + then: ${event.ref} + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.branch}' + head_sha: + $if: 'tasks_for == "github-push"' + then: '${event.after}' + else: + $if: 'tasks_for == "github-pull-request"' + then: '${event.pull_request.head.sha}' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${push.revision}' + ownTaskId: + $if: '"github" in tasks_for' + then: {$eval: as_slugid("decision_task")} + else: + $if: 'tasks_for == "cron"' + then: '${ownTaskId}' in: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:pull-request - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py pull-request - env: - GITHUB_PULL_TITLE: ${pull_request_title} - MOBILE_PULL_REQUEST_NUMBER: ${pull_request_number} - extra: - treeherder: - symbol: D-PR - metadata: - name: '${project_name} - Decision task (Pull Request #${pull_request_number})' - description: 'Building and testing ${project_name} - triggered by [#${pull_request_number}](${pull_request_url})' - - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:branch:${short_head_branch} - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py push - extra: - treeherder: - symbol: D - metadata: - name: ${project_name} VCS-Push Decision task - description: Schedules the build and test tasks for ${project_name}. - - $if: 'tasks_for == "github-release" && event["action"] == "published"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:release - payload: - command: - - >- - git fetch ${repository} refs/tags/${head_rev} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py github-release ${event.release.tag_name} - extra: - treeherder: - symbol: D-github-release - metadata: - name: ${project_name} Github Release Decision Task - description: Building and releasing ${project_name} ${event.release.tag_name} - - $if: 'tasks_for == "cron"' - then: - $let: - staging_flag: - $if: 'trust_level == 3' - then: '' - else: '--staging' - in: - - $if: 'cron.name == "nightly"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - $if: 'trust_level == 3' - then: assume:hook-id:project-mobile/${project_name}-nightly - else: assume:hook-id:project-mobile/${project_name}-nightly-staging - routes: - $if: 'trust_level == 3' - then: - - notify.email.fenix-eng-notifications@mozilla.com.on-failed - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py nightly ${staging_flag} - extra: - cron: {$json: {$eval: 'cron'}} - treeherder: - symbol: nightly-D + $let: + level: + $if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"' + then: '3' + else: + $if: 'tasks_for in ["cron", "action"]' + then: '${repository.level}' + else: 1 + in: + taskId: '${ownTaskId}' + taskGroupId: + $if: 'tasks_for == "action"' + then: + '${action.taskGroupId}' + else: + '${ownTaskId}' # same as taskId; this is how automation identifies a decision task + schedulerId: '${trustDomain}-level-${level}' + created: {$fromNow: ''} + deadline: {$fromNow: '1 day'} + expires: {$fromNow: '1 year 1 second'} # 1 second so artifacts expire first, despite rounding errors metadata: - name: ${project_name} Nightly Decision Task - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) - - $if: 'cron.name == "raptor"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - $if: 'trust_level == 3' - then: assume:hook-id:project-mobile/${project_name}-raptor - else: assume:hook-id:project-mobile/${project_name}-raptor-staging + $merge: + - owner: "${ownerEmail}" + source: '${repoUrl}/raw/${head_sha}/.taskcluster.yml' + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + name: "Decision Task" + description: 'The task that creates all of the other tasks in the task graph' + else: + $if: 'tasks_for == "action"' + then: + name: "Action: ${action.title}" + description: '${action.description}' + else: + name: "Decision Task for cron job ${cron.job_name}" + description: 'Created by a [cron task](https://tools.taskcluster.net/tasks/${cron.task_id})' + provisionerId: "aws-provisioner-v1" + workerType: "mobile-${level}-decision" + tags: + $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + kind: decision-task + else: + $if: 'tasks_for == "action"' + then: + kind: 'action-callback' + else: + $if: 'tasks_for == "cron"' + then: + kind: cron-task routes: - $if: 'trust_level == 3' - then: - - notify.email.fenix-eng-notifications@mozilla.com.on-failed - - notify.email.perftest-alerts@mozilla.com.on-failed + $flatten: + - checks + - $if: 'tasks_for != "github-pull-request"' + then: + - "tc-treeherder.v2.${project}.${head_sha}" + else: [] + scopes: + # `https://` is 8 characters so, ${repoUrl[8:]} is the repository without the protocol. + $if: 'tasks_for == "github-push"' + then: + $let: + short_head_branch: + $if: 'head_branch[:10] == "refs/tags/"' + then: {$eval: 'head_branch[10:]'} + else: + $if: 'head_branch[:11] == "refs/heads/"' + then: {$eval: 'head_branch[11:]'} + else: ${head_branch} + in: + - 'assume:repo:${repoUrl[8:]}:branch:${short_head_branch}' + else: + $if: 'tasks_for == "github-pull-request"' + then: + - 'assume:repo:github.com/${event.pull_request.base.repo.full_name}:pull-request' + else: + $if: 'tasks_for == "action"' + then: + # when all actions are hooks, we can calculate this directly rather than using a variable + - '${action.repo_scope}' + else: + - 'assume:repo:${repoUrl[8:]}:cron:${cron.job_name}' + + requires: all-completed + priority: lowest + retries: 5 + payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py raptor ${staging_flag} + env: + # run-task uses these to check out the source; the inputs + # to `mach taskgraph decision` are all on the command line. + $merge: + - MOBILE_BASE_REPOSITORY: '${baseRepoUrl}' + MOBILE_HEAD_REPOSITORY: '${repoUrl}' + MOBILE_HEAD_REF: '${head_branch}' + MOBILE_HEAD_REV: '${head_sha}' + MOBILE_REPOSITORY_TYPE: git + TASKGRAPH_BASE_REPOSITORY: https://hg.mozilla.org/ci/taskgraph + TASKGRAPH_HEAD_REPOSITORY: https://hg.mozilla.org/ci/${taskgraph.branch} + TASKGRAPH_HEAD_REV: ${taskgraph.revision} + TASKGRAPH_REPOSITORY_TYPE: hg + REPOSITORIES: {$json: {mobile: "Fenix", taskgraph: "Taskgraph"}} + HG_STORE_PATH: /builds/worker/checkouts/hg-store + ANDROID_SDK_ROOT: /builds/worker/android-sdk + - $if: 'tasks_for in ["github-pull-request"]' + then: + MOBILE_PULL_REQUEST_NUMBER: '${event.pull_request.number}' + - $if: 'tasks_for == "action"' + then: + ACTION_TASK_GROUP_ID: '${action.taskGroupId}' # taskGroupId of the target task + ACTION_TASK_ID: {$json: {$eval: 'taskId'}} # taskId of the target task (JSON-encoded) + ACTION_INPUT: {$json: {$eval: 'input'}} + ACTION_CALLBACK: '${action.cb_name}' + features: + taskclusterProxy: true + chainOfTrust: true + # Note: This task is built server side without the context or tooling that + # exist in tree so we must hard code the hash + image: + mozillareleases/taskgraph:decision-mobile-6020473b1a928d8df50e234a7ca2e81ade2220a4fb5fbe16b02477dd64a49728@sha256:98d226736b7d03907114bf37938002b90e8a37cbe3a297690e349f1ddddb1d7c + + maxRunTime: 1800 + + command: + - /usr/local/bin/run-task + - '--mobile-checkout=/builds/worker/checkouts/src' + - '--taskgraph-checkout=/builds/worker/checkouts/taskgraph' + - '--task-cwd=/builds/worker/checkouts/src' + - '--' + - bash + - -cx + - $let: + extraArgs: {$if: 'tasks_for == "cron"', then: '${cron.quoted_args}', else: ''} + in: + $if: 'tasks_for == "action"' + then: > + cd /builds/worker/checkouts/src && + ln -s /builds/worker/artifacts artifacts && + taskgraph action-callback + else: > + PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + taskcluster/scripts/install-sdk.sh && + ln -s /builds/worker/artifacts artifacts && + ~/.local/bin/taskgraph decision + --pushlog-id='0' + --pushdate='0' + --project='${project}' + --message="" + --owner='${ownerEmail}' + --level='${level}' + --base-repository="$MOBILE_BASE_REPOSITORY" + --head-repository="$MOBILE_HEAD_REPOSITORY" + --head-ref="$MOBILE_HEAD_REF" + --head-rev="$MOBILE_HEAD_REV" + --repository-type="$MOBILE_REPOSITORY_TYPE" + --tasks-for='${tasks_for}' + ${extraArgs} + + artifacts: + 'public': + type: 'directory' + path: '/builds/worker/artifacts' + expires: {$fromNow: '1 year'} + extra: - cron: {$json: {$eval: 'cron'}} - treeherder: - symbol: raptor-D - notify: - email: - link: - text: Treeherder Job - href: "https://treeherder.mozilla.org/#/jobs?repo=${project_name}&revision=${head_rev}" - subject: "[${project_name}] Raptor decision task failed" - content: "This calls for an action of the development team. If needed, escalate to the release engineering team in #releaseduty-mobile on Slack. Use the link to view it on Treeherder." - metadata: - name: ${project_name} Raptor Decision Task - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + $merge: + - treeherder: + $merge: + - machine: + platform: gecko-decision + - $if: 'tasks_for in ["github-push", "github-pull-request"]' + then: + symbol: D + else: + $if: 'tasks_for == "action"' + then: + groupName: 'action-callback' + groupSymbol: AC + symbol: "${action.symbol}" + else: + groupSymbol: cron + symbol: "${cron.job_symbol}" + - $if: 'tasks_for == "action"' + then: + parent: '${action.taskGroupId}' + action: + name: '${action.name}' + context: + taskGroupId: '${action.taskGroupId}' + taskId: {$eval: 'taskId'} + input: {$eval: 'input'} + - $if: 'tasks_for == "cron"' + then: + cron: {$json: {$eval: 'cron'}} + - tasks_for: '${tasks_for}' diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index fca6a6075..8f6d2a551 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -258,7 +258,4 @@ if __name__ == "__main__": else: raise Exception('Unsupported command "{}"'.format(command)) - full_task_graph = schedule_task_graph(ordered_groups_of_tasks) - populate_chain_of_trust_task_graph(full_task_graph) - populate_chain_of_trust_required_but_unused_files() diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml new file mode 100644 index 000000000..7434fc00b --- /dev/null +++ b/taskcluster/ci/config.yml @@ -0,0 +1,58 @@ +--- +trust-domain: mobile +treeherder: + group-names: + 'I': 'Docker Image Builds' + 'Rap': 'Raptor tests' + 'Rap-P': 'Raptor power tests' + +task-priority: lowest + +taskgraph: + register: fenix_taskgraph:register + repositories: + mobile: + name: "Fenix" + cached-task-prefix: project.mobile.fenix + +workers: + aliases: + b-android: + provisioner: aws-provisioner-v1 + implementation: docker-worker + os: linux + worker-type: 'mobile-{level}-b-ref-browser' + images: + provisioner: aws-provisioner-v1 + implementation: docker-worker + os: linux + worker-type: 'mobile-{level}-images' + dep-signing: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: scriptworker + worker-type: mobile-signing-dep-v1 + signing: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-signing + os: scriptworker + worker-type: + by-level: + "3": mobile-signing-v1 + default: mobile-signing-dep-v1 + push-apk: + provisioner: scriptworker-prov-v1 + implementation: scriptworker-pushapk + os: scriptworker + worker-type: + by-level: + "3": mobile-pushapk-v1 + default: mobile-pushapk-dep-v1 + t-bitbar.*: + provisioner: proj-autophone + implementation: generic-worker + os: linux-bitbar + worker-type: 'gecko-{alias}' + +scriptworker: + scope-prefix: project:mobile:fenix:releng diff --git a/taskcluster/fenix_taskgraph/__init__.py b/taskcluster/fenix_taskgraph/__init__.py new file mode 100644 index 000000000..90e56ee5a --- /dev/null +++ b/taskcluster/fenix_taskgraph/__init__.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from importlib import import_module + + +def register(graph_config): + """ + Import all modules that are siblings of this one, triggering decorators in + the process. + """ + _import_modules(["job", "worker_types", "routes", "target"]) + + +def _import_modules(modules): + for module in modules: + import_module(".{}".format(module), package=__name__) diff --git a/taskcluster/fenix_taskgraph/job.py b/taskcluster/fenix_taskgraph/job.py new file mode 100644 index 000000000..33b058b6e --- /dev/null +++ b/taskcluster/fenix_taskgraph/job.py @@ -0,0 +1,79 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run +from taskgraph.util import path +from taskgraph.util.schema import Schema +from voluptuous import Required, Optional +from six import text_type + +from pipes import quote as shell_quote + +gradlew_schema = Schema( + { + Required("using"): "gradlew", + Required("gradlew"): [text_type], + Optional("post-gradlew"): [[text_type]], + # Base work directory used to set up the task. + Required("workdir"): text_type, + Optional("use-caches"): bool, + Optional("secrets"): [ + { + Required("name"): text_type, + Required("path"): text_type, + Required("key"): text_type, + Optional("json"): bool, + } + ], + } +) + + +@run_job_using("docker-worker", "gradlew", schema=gradlew_schema) +def configure_gradlew(config, job, taskdesc): + run = job["run"] + worker = taskdesc["worker"] = job["worker"] + + worker.setdefault("env", {}).update( + {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")} + ) + + # defer to the run_task implementation + run["command"] = _extract_command(run) + secrets = run.pop("secrets", []) + scopes = taskdesc.setdefault("scopes", []) + scopes.extend(["secrets:get:{}".format(secret["name"]) for secret in secrets]) + + run["cwd"] = "{checkout}" + run["using"] = "run-task" + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) + + +def _extract_command(run): + pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]] + pre_gradle_commands += [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + + gradle_command = ["./gradlew"] + run.pop("gradlew") + post_gradle_commands = run.pop("post-gradlew", []) + + commands = pre_gradle_commands + [gradle_command] + post_gradle_commands + shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands] + return " && ".join(shell_quoted_commands) + + +def _generate_secret_command(secret): + secret_command = [ + "taskcluster/scripts/get-secret.py", + "-s", secret["name"], + "-k", secret["key"], + "-f", secret["path"], + ] + if secret.get("json"): + secret_command.append("--json") + + return secret_command diff --git a/taskcluster/fenix_taskgraph/routes.py b/taskcluster/fenix_taskgraph/routes.py new file mode 100644 index 000000000..00c407cf9 --- /dev/null +++ b/taskcluster/fenix_taskgraph/routes.py @@ -0,0 +1,34 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import time + +from taskgraph.transforms.task import index_builder + +SIGNING_ROUTE_TEMPLATES = [ + "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.revision.{head_rev}", + "index.project.{trust-domain}.{project}.v3.{variant}.{build_date}.latest", + "index.project.{trust-domain}.{project}.v3.{variant}.latest", +] + + +@index_builder("signing") +def add_signing_indexes(config, task): + routes = task.setdefault("routes", []) + + if config.params["level"] != "3": + return task + + subs = config.params.copy() + subs["build_date"] = time.strftime( + "%Y.%m.%d", time.gmtime(config.params["build_date"]) + ) + subs["trust-domain"] = config.graph_config["trust-domain"] + subs["variant"] = task["attributes"]["build-type"] + + for tpl in SIGNING_ROUTE_TEMPLATES: + routes.append(tpl.format(**subs)) + return task diff --git a/taskcluster/fenix_taskgraph/target.py b/taskcluster/fenix_taskgraph/target.py new file mode 100644 index 000000000..e7a23a8a4 --- /dev/null +++ b/taskcluster/fenix_taskgraph/target.py @@ -0,0 +1,17 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.target_tasks import _target_task as target_task + + +@target_task("nightly") +def target_tasks_nightly(full_task_graph, parameters, graph_config): + """Select the set of tasks required for a nightly build.""" + + def filter(task, parameters): + return task.attributes.get("nightly", False) + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] diff --git a/taskcluster/fenix_taskgraph/worker_types.py b/taskcluster/fenix_taskgraph/worker_types.py new file mode 100644 index 000000000..3ab708a5e --- /dev/null +++ b/taskcluster/fenix_taskgraph/worker_types.py @@ -0,0 +1,94 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from six import text_type + +from voluptuous import Required + +from taskgraph.util.schema import taskref_or_string +from taskgraph.transforms.task import payload_builder + + +@payload_builder( + "scriptworker-signing", + schema={ + # the maximum time to run, in seconds + Required("max-run-time"): int, + Required("signing-type"): text_type, + # list of artifact URLs for the artifacts that should be signed + Required("upstream-artifacts"): [ + { + # taskId of the task with the artifact + Required("taskId"): taskref_or_string, + # type of signing task (for CoT) + Required("taskType"): text_type, + # Paths to the artifacts to sign + Required("paths"): [text_type], + # Signing formats to use on each of the paths + Required("formats"): [text_type], + } + ], + }, +) +def build_scriptworker_signing_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "maxRunTime": worker["max-run-time"], + "upstreamArtifacts": worker["upstream-artifacts"], + } + + formats = set() + for artifacts in worker["upstream-artifacts"]: + formats.update(artifacts["formats"]) + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].append( + "{}:signing:cert:{}".format(scope_prefix, worker["signing-type"]) + ) + task_def["scopes"].extend( + [ + "{}:signing:format:{}".format(scope_prefix, format) + for format in sorted(formats) + ] + ) + + +@payload_builder( + "scriptworker-pushapk", + schema={ + Required("upstream-artifacts"): [ + { + Required("taskId"): taskref_or_string, + Required("taskType"): text_type, + Required("paths"): [text_type], + } + ], + Required("channel"): text_type, + Required("commit"): bool, + Required("product"): text_type, + Required("dep"): bool, + }, +) +def build_push_apk_payload(config, task, task_def): + worker = task["worker"] + + task_def["tags"]["worker-implementation"] = "scriptworker" + + task_def["payload"] = { + "commit": worker["commit"], + "upstreamArtifacts": worker["upstream-artifacts"], + "channel": worker["channel"], + } + + scope_prefix = config.graph_config["scriptworker"]["scope-prefix"] + task_def["scopes"].append( + "{}:googleplay:product:{}{}".format( + scope_prefix, worker["product"], ":dep" if worker["dep"] else "" + ) + ) diff --git a/taskcluster/scripts/install-sdk.sh b/taskcluster/scripts/install-sdk.sh new file mode 100755 index 000000000..7a431a34a --- /dev/null +++ b/taskcluster/scripts/install-sdk.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set +x + +ANDROID_SDK_VERSION=3859397 + +curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" +unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip" +yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses