blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Adds raptor hook ability to schedule_nightly_graph (#2341)

master
Mitchell Hentges 2019-05-27 17:31:06 +02:00 committed by GitHub
parent 77ff164e30
commit bd6001b1d1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 172 additions and 128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

189
.taskcluster.yml
View File

@ -123,99 +123,128 @@ tasks:
owner: ${user}@users.noreply.github.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
in:
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
then:
$let:
pull_request_title: ${event.pull_request.title}
pull_request_number: ${event.pull_request.number}
pull_request_url: ${event.pull_request.html_url}
in:
$flatten:
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
then:
$let:
pull_request_title: ${event.pull_request.title}
pull_request_number: ${event.pull_request.number}
pull_request_url: ${event.pull_request.html_url}
in:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:pull-request
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py pull-request
env:
GITHUB_PULL_TITLE: ${pull_request_title}
extra:
treeherder:
symbol: D-PR
metadata:
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:pull-request
- ${assume_scope_prefix}:branch:${short_head_branch}
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py pull-request
env:
GITHUB_PULL_TITLE: ${pull_request_title}
&& python automation/taskcluster/decision_task.py push
extra:
treeherder:
symbol: D-PR
symbol: D
metadata:
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:branch:${short_head_branch}
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py push
extra:
treeherder:
symbol: D
metadata:
name: Fenix - Decision task
description: Schedules the build and test tasks for Fenix.
- $if: 'tasks_for == "github-release" && event["action"] == "published"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:release
payload:
command:
- >-
git fetch ${repository} refs/tags/${head_rev}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py beta ${event.release.tag_name}
extra:
treeherder:
symbol: beta-D
metadata:
name: Fenix Beta Decision Task
description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
- $if: 'tasks_for == "cron"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- $if: 'trust_level == 3'
then: assume:hook-id:project-mobile/fenix-nightly
else: assume:hook-id:project-mobile/fenix-nightly-staging
routes:
$if: 'trust_level == 3'
then:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
$let:
staging_flag:
$if: 'trust_level == 3'
then: ''
else: '--staging'
in:
name: Fenix VCS-Push Decision task
description: Schedules the build and test tasks for Fenix.
- $if: 'tasks_for == "github-release" && event["action"] == "published"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- ${assume_scope_prefix}:release
payload:
command:
- >-
git fetch ${repository} ${head_branch}
git fetch ${repository} refs/tags/${head_rev}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py nightly ${staging_flag}
extra:
cron: {$json: {$eval: 'cron'}}
treeherder:
symbol: nightly-D
metadata:
name: Fenix Nightly Decision Task
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
&& python automation/taskcluster/decision_task.py beta ${event.release.tag_name}
extra:
treeherder:
symbol: beta-D
metadata:
name: Fenix Beta Decision Task
description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
- $if: 'tasks_for == "cron"'
then:
$let:
staging_flag:
$if: 'trust_level == 3'
then: ''
else: '--staging'
in:
- $if: 'cron.name == "nightly"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- $if: 'trust_level == 3'
then: assume:hook-id:project-mobile/fenix-nightly
else: assume:hook-id:project-mobile/fenix-nightly-staging
routes:
$if: 'trust_level == 3'
then:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py nightly ${staging_flag}
extra:
cron: {$json: {$eval: 'cron'}}
treeherder:
symbol: nightly-D
metadata:
name: Fenix Nightly Decision Task
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
- $if: 'cron.name == "raptor"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- $if: 'trust_level == 3'
then: assume:hook-id:project-mobile/fenix-raptor
else: assume:hook-id:project-mobile/fenix-raptor-staging
routes:
$if: 'trust_level == 3'
then:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout FETCH_HEAD
&& python automation/taskcluster/decision_task.py raptor ${staging_flag}
extra:
cron: {$json: {$eval: 'cron'}}
treeherder:
symbol: raptor-D
metadata:
name: Fenix Raptor Decision Task
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

55
automation/taskcluster/decision_task.py
View File

@ -57,10 +57,6 @@ def pr_or_push(is_push):
print("Exit")
return {}
geckoview_nightly_version = get_geckoview_versions()['nightly']
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
build_tasks = {}
signing_tasks = {}
other_tasks = {}
@ -70,23 +66,6 @@ def pr_or_push(is_push):
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
if is_push and SHORT_HEAD_BRANCH == 'master':
other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant)
ALL_RAPTOR_CRAFT_FUNCTIONS = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 15)
]
for craft_function in ALL_RAPTOR_CRAFT_FUNCTIONS:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
@ -95,6 +74,35 @@ def pr_or_push(is_push):
):
other_tasks[taskcluster.slugId()] = craft_function()
if is_push and SHORT_HEAD_BRANCH == 'master':
other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task()
return (build_tasks, signing_tasks, other_tasks)
def raptor(is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
geckoview_nightly_version = get_geckoview_versions()['nightly']
mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version)
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]:
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 15)
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant, gecko_revision)
other_tasks[taskcluster.slugId()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
@ -138,6 +146,9 @@ if __name__ == "__main__":
subparsers.add_parser('pull-request')
subparsers.add_parser('push')
raptor_parser = subparsers.add_parser('raptor')
raptor_parser.add_argument('--staging', action='store_true')
nightly_parser = subparsers.add_parser('nightly')
nightly_parser.add_argument('--staging', action='store_true')
@ -152,6 +163,8 @@ if __name__ == "__main__":
ordered_groups_of_tasks = pr_or_push(False)
elif command == 'push':
ordered_groups_of_tasks = pr_or_push(True)
elif command == 'raptor':
ordered_groups_of_tasks = raptor(result.staging)
elif command == 'nightly':
formatted_date = datetime.datetime.now().strftime('%y%V')
ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date))

42
automation/taskcluster/lib/tasks.py
View File

@ -375,24 +375,18 @@ class TaskBuilder(object):
}
def craft_raptor_signing_task(
self, assemble_task_id, variant
self, assemble_task_id, variant, is_staging,
):
routes = []
if self.repo_url == _OFFICIAL_REPO_URL:
routes = [
'index.project.mobile.fenix.v2.branch.master.revision.{}.raptor.{}'.format(
self.commit, variant.abi
),
'index.project.mobile.fenix.v2.branch.master.latest.raptor.{}'.format(
variant.abi
),
'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.raptor.{}'.format(
self.date.year, self.date.month, self.date.day, self.commit, variant.abi
),
'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.raptor.{}'.format(
self.date.year, self.date.month, self.date.day, variant.abi
),
]
staging_prefix = '.staging' if is_staging else ''
routes = [
"index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.latest.{}".format(
staging_prefix, self.date.year, self.date.month, self.date.day, variant.abi
),
"index.project.mobile.fenix.v2{}.raptor.{}.{}.{}.revision.{}.{}".format(
staging_prefix, self.date.year, self.date.month, self.date.day, self.commit, variant.abi
),
"index.project.mobile.fenix.v2{}.raptor.latest.{}".format(staging_prefix, variant.abi),
]
return self._craft_signing_task(
name='sign: {}'.format(variant.raw),
@ -413,19 +407,19 @@ class TaskBuilder(object):
)
def craft_release_signing_task(
self, build_task_id, apk_paths, track, is_staging=False,
self, build_task_id, apk_paths, track, is_staging,
):
capitalized_track = upper_case_first_letter(track)
index_release = 'staging.{}'.format(track) if is_staging else track
staging_prefix = '.staging' if is_staging else ''
routes = [
"index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format(
index_release, self.date.year, self.date.month, self.date.day
"index.project.mobile.fenix.v2{}.{}.{}.{}.{}.latest".format(
staging_prefix, track, self.date.year, self.date.month, self.date.day
),
"index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format(
index_release, self.date.year, self.date.month, self.date.day, self.commit
"index.project.mobile.fenix.v2{}.{}.{}.{}.{}.revision.{}".format(
staging_prefix, track, self.date.year, self.date.month, self.date.day, self.commit
),
"index.project.mobile.fenix.v2.{}.latest".format(index_release),
"index.project.mobile.fenix.v2{}.{}.latest".format(staging_prefix, track),
]
return self._craft_signing_task(

14
automation/taskcluster/schedule_nightly_graph.py
View File

@ -1,7 +1,7 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import argparse
import datetime
import jsone
import os
@ -50,7 +50,8 @@ def make_decision_task(params):
context = {
'tasks_for': 'cron',
'cron': {
'task_id': params['cron_task_id']
'task_id': params['cron_task_id'],
'name': params['name'],
},
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
'as_slugid': as_slugid,
@ -79,6 +80,12 @@ def make_decision_task(params):
def schedule():
parser = argparse.ArgumentParser(
description='Creates and submit a graph of tasks on Taskcluster.'
)
parser.add_argument('name', choices=['nightly', 'raptor'])
result = parser.parse_args()
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
html_url, branch, head_rev = calculate_git_references(ROOT)
@ -86,7 +93,8 @@ def schedule():
'html_url': html_url,
'head_rev': head_rev,
'branch': branch,
'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>')
'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
'name': result.name,
}
decision_task_id, decision_task = make_decision_task(params)
schedule_task(queue, decision_task_id, decision_task)