From bd6001b1d189c88ca4ba02f960cd6208fbddcf18 Mon Sep 17 00:00:00 2001 From: Mitchell Hentges Date: Mon, 27 May 2019 17:31:06 +0200 Subject: [PATCH] Adds raptor hook ability to schedule_nightly_graph (#2341) --- .taskcluster.yml | 189 ++++++++++-------- automation/taskcluster/decision_task.py | 55 +++-- automation/taskcluster/lib/tasks.py | 42 ++-- .../taskcluster/schedule_nightly_graph.py | 14 +- 4 files changed, 172 insertions(+), 128 deletions(-) diff --git a/.taskcluster.yml b/.taskcluster.yml index dcbb262c7..3b3171d54 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -123,99 +123,128 @@ tasks: owner: ${user}@users.noreply.github.com source: ${repository}/raw/${head_rev}/.taskcluster.yml in: - - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' - then: - $let: - pull_request_title: ${event.pull_request.title} - pull_request_number: ${event.pull_request.number} - pull_request_url: ${event.pull_request.html_url} - in: + $flatten: + - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: + $let: + pull_request_title: ${event.pull_request.title} + pull_request_number: ${event.pull_request.number} + pull_request_url: ${event.pull_request.html_url} + in: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - ${assume_scope_prefix}:pull-request + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py pull-request + env: + GITHUB_PULL_TITLE: ${pull_request_title} + extra: + treeherder: + symbol: D-PR + metadata: + name: 'Fenix - Decision task (Pull Request #${pull_request_number})' + description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' + - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' + then: $mergeDeep: - {$eval: 'default_task_definition'} - scopes: - - ${assume_scope_prefix}:pull-request + - ${assume_scope_prefix}:branch:${short_head_branch} payload: command: - >- git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py pull-request - env: - GITHUB_PULL_TITLE: ${pull_request_title} + && python automation/taskcluster/decision_task.py push extra: treeherder: - symbol: D-PR + symbol: D metadata: - name: 'Fenix - Decision task (Pull Request #${pull_request_number})' - description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' - - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:branch:${short_head_branch} - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py push - extra: - treeherder: - symbol: D - metadata: - name: Fenix - Decision task - description: Schedules the build and test tasks for Fenix. - - $if: 'tasks_for == "github-release" && event["action"] == "published"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:release - payload: - command: - - >- - git fetch ${repository} refs/tags/${head_rev} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py beta ${event.release.tag_name} - extra: - treeherder: - symbol: beta-D - metadata: - name: Fenix Beta Decision Task - description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name} - - $if: 'tasks_for == "cron"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - $if: 'trust_level == 3' - then: assume:hook-id:project-mobile/fenix-nightly - else: assume:hook-id:project-mobile/fenix-nightly-staging - routes: - $if: 'trust_level == 3' - then: - - notify.email.fenix-eng-notifications@mozilla.com.on-failed - payload: - $let: - staging_flag: - $if: 'trust_level == 3' - then: '' - else: '--staging' - in: + name: Fenix VCS-Push Decision task + description: Schedules the build and test tasks for Fenix. + - $if: 'tasks_for == "github-release" && event["action"] == "published"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - ${assume_scope_prefix}:release + payload: command: - >- - git fetch ${repository} ${head_branch} + git fetch ${repository} refs/tags/${head_rev} && git config advice.detachedHead false && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py nightly ${staging_flag} - extra: - cron: {$json: {$eval: 'cron'}} - treeherder: - symbol: nightly-D - metadata: - name: Fenix Nightly Decision Task - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + && python automation/taskcluster/decision_task.py beta ${event.release.tag_name} + extra: + treeherder: + symbol: beta-D + metadata: + name: Fenix Beta Decision Task + description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name} + - $if: 'tasks_for == "cron"' + then: + $let: + staging_flag: + $if: 'trust_level == 3' + then: '' + else: '--staging' + in: + - $if: 'cron.name == "nightly"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - $if: 'trust_level == 3' + then: assume:hook-id:project-mobile/fenix-nightly + else: assume:hook-id:project-mobile/fenix-nightly-staging + routes: + $if: 'trust_level == 3' + then: + - notify.email.fenix-eng-notifications@mozilla.com.on-failed + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py nightly ${staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + treeherder: + symbol: nightly-D + metadata: + name: Fenix Nightly Decision Task + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + - $if: 'cron.name == "raptor"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - $if: 'trust_level == 3' + then: assume:hook-id:project-mobile/fenix-raptor + else: assume:hook-id:project-mobile/fenix-raptor-staging + routes: + $if: 'trust_level == 3' + then: + - notify.email.fenix-eng-notifications@mozilla.com.on-failed + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py raptor ${staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + treeherder: + symbol: raptor-D + metadata: + name: Fenix Raptor Decision Task + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index a11025ce5..91c1f8d88 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -57,10 +57,6 @@ def pr_or_push(is_push): print("Exit") return {} - geckoview_nightly_version = get_geckoview_versions()['nightly'] - mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) - gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] - build_tasks = {} signing_tasks = {} other_tasks = {} @@ -70,23 +66,6 @@ def pr_or_push(is_push): build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) - if is_push and SHORT_HEAD_BRANCH == 'master': - other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() - - for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]: - assemble_task_id = taskcluster.slugId() - build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant) - - ALL_RAPTOR_CRAFT_FUNCTIONS = [ - BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) - for i in range(1, 15) - ] - for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS: - args = (signing_task_id, mozharness_task_id, variant, gecko_revision) - other_tasks[taskcluster.slugId()] = craft_function(*args) - for craft_function in ( BUILDER.craft_detekt_task, BUILDER.craft_ktlint_task, @@ -95,6 +74,35 @@ def pr_or_push(is_push): ): other_tasks[taskcluster.slugId()] = craft_function() + if is_push and SHORT_HEAD_BRANCH == 'master': + other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() + + return (build_tasks, signing_tasks, other_tasks) + + +def raptor(is_staging): + build_tasks = {} + signing_tasks = {} + other_tasks = {} + + geckoview_nightly_version = get_geckoview_versions()['nightly'] + mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) + gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] + + for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]: + assemble_task_id = taskcluster.slugId() + build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) + signing_task_id = taskcluster.slugId() + signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging) + + all_raptor_craft_functions = [ + BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) + for i in range(1, 15) + ] + for craft_function in all_raptor_craft_functions: + args = (signing_task_id, mozharness_task_id, variant, gecko_revision) + other_tasks[taskcluster.slugId()] = craft_function(*args) + return (build_tasks, signing_tasks, other_tasks) @@ -138,6 +146,9 @@ if __name__ == "__main__": subparsers.add_parser('pull-request') subparsers.add_parser('push') + raptor_parser = subparsers.add_parser('raptor') + raptor_parser.add_argument('--staging', action='store_true') + nightly_parser = subparsers.add_parser('nightly') nightly_parser.add_argument('--staging', action='store_true') @@ -152,6 +163,8 @@ if __name__ == "__main__": ordered_groups_of_tasks = pr_or_push(False) elif command == 'push': ordered_groups_of_tasks = pr_or_push(True) + elif command == 'raptor': + ordered_groups_of_tasks = raptor(result.staging) elif command == 'nightly': formatted_date = datetime.datetime.now().strftime('%y%V') ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date)) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index d2e7efd69..822443a3f 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -375,24 +375,18 @@ class TaskBuilder(object): } def craft_raptor_signing_task( - self, assemble_task_id, variant + self, assemble_task_id, variant, is_staging, ): - routes = [] - if self.repo_url == _OFFICIAL_REPO_URL: - routes = [ - 'index.project.mobile.fenix.v2.branch.master.revision.{}.raptor.{}'.format( - self.commit, variant.abi - ), - 'index.project.mobile.fenix.v2.branch.master.latest.raptor.{}'.format( - variant.abi - ), - 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.raptor.{}'.format( - self.date.year, self.date.month, self.date.day, self.commit, variant.abi - ), - 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.raptor.{}'.format( - self.date.year, self.date.month, self.date.day, variant.abi - ), - ] + staging_prefix = '.staging' if is_staging else '' + routes = [ + "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.latest.{}".format( + staging_prefix, self.date.year, self.date.month, self.date.day, variant.abi + ), + "index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.revision.{}.{}".format( + staging_prefix, self.date.year, self.date.month, self.date.day, self.commit, variant.abi + ), + "index.project.mobile.fenix.v2{}.raptor.latest.{}".format(staging_prefix, variant.abi), + ] return self._craft_signing_task( name='sign: {}'.format(variant.raw), @@ -413,19 +407,19 @@ class TaskBuilder(object): ) def craft_release_signing_task( - self, build_task_id, apk_paths, track, is_staging=False, + self, build_task_id, apk_paths, track, is_staging, ): capitalized_track = upper_case_first_letter(track) - index_release = 'staging.{}'.format(track) if is_staging else track + staging_prefix = '.staging' if is_staging else '' routes = [ - "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format( - index_release, self.date.year, self.date.month, self.date.day + "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format( + staging_prefix, track, self.date.year, self.date.month, self.date.day ), - "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format( - index_release, self.date.year, self.date.month, self.date.day, self.commit + "index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format( + staging_prefix, track, self.date.year, self.date.month, self.date.day, self.commit ), - "index.project.mobile.fenix.v2.{}.latest".format(index_release), + "index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, track), ] return self._craft_signing_task( diff --git a/automation/taskcluster/schedule_nightly_graph.py b/automation/taskcluster/schedule_nightly_graph.py index cf4b5e449..2311f3fd3 100644 --- a/automation/taskcluster/schedule_nightly_graph.py +++ b/automation/taskcluster/schedule_nightly_graph.py @@ -1,7 +1,7 @@ # This Source Code Form is subject to the terms of the Mozilla Public # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. - +import argparse import datetime import jsone import os @@ -50,7 +50,8 @@ def make_decision_task(params): context = { 'tasks_for': 'cron', 'cron': { - 'task_id': params['cron_task_id'] + 'task_id': params['cron_task_id'], + 'name': params['name'], }, 'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z', 'as_slugid': as_slugid, @@ -79,6 +80,12 @@ def make_decision_task(params): def schedule(): + parser = argparse.ArgumentParser( + description='Creates and submit a graph of tasks on Taskcluster.' + ) + + parser.add_argument('name', choices=['nightly', 'raptor']) + result = parser.parse_args() queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) html_url, branch, head_rev = calculate_git_references(ROOT) @@ -86,7 +93,8 @@ def schedule(): 'html_url': html_url, 'head_rev': head_rev, 'branch': branch, - 'cron_task_id': os.environ.get('CRON_TASK_ID', '') + 'cron_task_id': os.environ.get('CRON_TASK_ID', ''), + 'name': result.name, } decision_task_id, decision_task = make_decision_task(params) schedule_task(queue, decision_task_id, decision_task)