blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Let taskgraph load tasks from decision_tasks.py in PRs

master
Johan Lorenzo 2019-09-12 18:45:26 +02:00 committed by Sebastian Kaspari
parent daa1693f23
commit 2a4f48109d
10 changed files with 148 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.taskcluster.yml
View File

@ -218,6 +218,7 @@ tasks:
taskgraph action-callback
else: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
taskcluster/scripts/install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph decision

166
automation/taskcluster/decision_task.py
View File

@ -26,65 +26,33 @@ from lib.chain_of_trust import (
populate_chain_of_trust_required_but_unused_files
)
REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY')
COMMIT = os.environ.get('MOBILE_HEAD_REV')
PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '')
SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH')
# If we see this text inside a pull request title then we will not execute any tasks for this PR.
SKIP_TASKS_TRIGGER = '[ci skip]'
BUILDER = TaskBuilder(
task_id=os.environ.get('TASK_ID'),
repo_url=REPO_URL,
git_ref=os.environ.get('MOBILE_HEAD_BRANCH'),
short_head_branch=SHORT_HEAD_BRANCH,
commit=COMMIT,
owner="fenix-eng-notifications@mozilla.com",
source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT),
scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'),
tasks_priority=os.environ.get('TASKS_PRIORITY'),
date_string=os.environ.get('BUILD_DATE'),
trust_level=int(os.environ.get('TRUST_LEVEL')),
)
def pr():
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
build_tasks = {}
signing_tasks = {}
other_tasks = {}
def pr(builder):
tasks = []
variant = get_variant('debug', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant)
tasks.append(builder.craft_assemble_pr_task(variant))
tasks.append(builder.craft_test_pr_task(variant))
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
builder.craft_detekt_task,
builder.craft_ktlint_task,
builder.craft_lint_task,
builder.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
tasks.append(craft_function())
return (build_tasks, signing_tasks, other_tasks)
return tasks
def push():
def push(builder):
all_tasks = pr()
other_tasks = all_tasks[-1]
other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task()
other_tasks[_generate_slug_id()] = builder.craft_ui_tests_task()
return all_tasks
def raptor(is_staging):
def raptor(builder, is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
@ -93,27 +61,27 @@ def raptor(is_staging):
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
assemble_task_id = _generate_slug_id()
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
builder.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 28)
] + [
BUILDER.craft_raptor_youtube_playback_task,
builder.craft_raptor_youtube_playback_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
other_tasks[taskcluster.slugId()] = craft_function(*args)
other_tasks[_generate_slug_id()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
def release(channel, engine, is_staging, version_name):
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
taskcluster_apk_paths = variant.upstream_artifacts()
@ -121,19 +89,19 @@ def release(channel, engine, is_staging, version_name):
signing_tasks = {}
push_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths,
channel=channel,
is_staging=is_staging,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
push_task_id = _generate_slug_id()
push_tasks[push_task_id] = builder.craft_push_task(
signing_task_id,
taskcluster_apk_paths,
channel=channel,
@ -145,18 +113,18 @@ def release(channel, engine, is_staging, version_name):
return (build_tasks, signing_tasks, push_tasks)
def release_as_fennec(is_staging, version_name):
def release_as_fennec(builder, is_staging, version_name):
variant = get_variant('fennecProduction', 'geckoBeta')
channel = 'fennec-production'
build_tasks = {}
signing_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
variant.upstream_artifacts(),
channel,
@ -166,7 +134,7 @@ def release_as_fennec(is_staging, version_name):
return (build_tasks, signing_tasks)
def nightly_to_production_app(is_staging, version_name):
def nightly_to_production_app(builder, is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
@ -178,12 +146,12 @@ def nightly_to_production_app(is_staging, version_name):
push_tasks = {}
other_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(
variant, 'nightly-legacy', is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths,
channel='production', # Since we're publishing to the "production" app, we need to sign for production
@ -191,8 +159,8 @@ def nightly_to_production_app(is_staging, version_name):
publish_to_index=False,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
push_task_id = _generate_slug_id()
push_tasks[push_task_id] = builder.craft_push_task(
signing_task_id,
taskcluster_apk_paths,
channel='production', # We're publishing to the "production" app on the "nightly" track
@ -201,61 +169,13 @@ def nightly_to_production_app(is_staging, version_name):
)
if not is_staging:
nimbledroid_task_id = taskcluster.slugId()
other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
nimbledroid_task_id = _generate_slug_id()
other_tasks[nimbledroid_task_id] = builder.craft_upload_apk_nimbledroid_task(
build_task_id
)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Creates and submit a graph of tasks on Taskcluster.'
)
subparsers = parser.add_subparsers(dest='command')
subparsers.add_parser('pull-request')
subparsers.add_parser('push')
raptor_parser = subparsers.add_parser('raptor')
raptor_parser.add_argument('--staging', action='store_true')
nightly_parser = subparsers.add_parser('nightly')
nightly_parser.add_argument('--staging', action='store_true')
release_parser = subparsers.add_parser('github-release')
release_parser.add_argument('tag')
release_parser.add_argument('--staging', action='store_true')
result = parser.parse_args()
command = result.command
if command == 'pull-request':
ordered_groups_of_tasks = pr()
elif command == 'push':
ordered_groups_of_tasks = push()
elif command == 'raptor':
ordered_groups_of_tasks = raptor(result.staging)
elif command == 'nightly':
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
nightly_version = 'Nightly {}'.format(now)
ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \
+ nightly_to_production_app(result.staging, nightly_version)
ordered_groups_of_tasks += release_as_fennec(result.staging, 'Signed-as-Fennec Nightly {}'.format(now))
elif command == 'github-release':
version = result.tag[1:] # remove prefixed "v"
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
if beta_semver.match(result.tag):
ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version)
elif production_semver.match(result.tag):
ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version)
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
else:
raise Exception('Unsupported command "{}"'.format(command))
populate_chain_of_trust_task_graph(full_task_graph)
def _generate_slug_id():
return taskcluster.slugId()

2
automation/taskcluster/lib/gradle.py
View File

@ -6,7 +6,7 @@ from __future__ import print_function
import json
import subprocess
from lib.variant import Variant, VariantApk
from ..lib.variant import Variant, VariantApk
def get_variant(build_type, engine):

54
automation/taskcluster/lib/tasks.py
View File

@ -9,7 +9,7 @@ import datetime
import json
import taskcluster
from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
DEFAULT_EXPIRES_IN = '1 year'
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
@ -43,7 +43,7 @@ class TaskBuilder(object):
self.scheduler_id = scheduler_id
self.trust_level = trust_level
self.tasks_priority = tasks_priority
self.date = arrow.get(date_string)
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
self.trust_level = trust_level
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
@ -317,10 +317,8 @@ class TaskBuilder(object):
self, name, description, command, dependencies=None, artifacts=None,
routes=None, treeherder=None, env_vars=None, scopes=None
):
dependencies = [] if dependencies is None else dependencies
artifacts = {} if artifacts is None else artifacts
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
env_vars = {} if env_vars is None else env_vars
checkout_command = ' && '.join([
@ -402,7 +400,7 @@ class TaskBuilder(object):
treeherder=None,
notify=None,
):
dependencies = [] if dependencies is None else dependencies
dependencies = {} if dependencies is None else dependencies
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
treeherder = {} if treeherder is None else treeherder
@ -421,27 +419,31 @@ class TaskBuilder(object):
extra['notify'] = notify
return {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"dependencies": [self.task_id] + dependencies,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
"attributes": {},
"dependencies": dependencies,
"label": name,
"task": {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
},
}

2
taskcluster/ci/config.yml
View File

@ -6,7 +6,7 @@ treeherder:
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'
task-priority: lowest
task-priority: highest
taskgraph:
register: fenix_taskgraph:register

9
taskcluster/ci/old-decision/kind.yml 100644
View File

@ -0,0 +1,9 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: fenix_taskgraph.loader.old_decision:loader
transforms: []
# XXX Everything is done in the loader until everything is migrated

2
taskcluster/fenix_taskgraph/__init__.py
View File

@ -12,7 +12,7 @@ def register(graph_config):
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules(["job", "worker_types", "routes", "target"])
_import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
def _import_modules(modules):

0
taskcluster/fenix_taskgraph/loader/__init__.py 100644
View File

44
taskcluster/fenix_taskgraph/loader/old_decision.py 100644
View File

@ -0,0 +1,44 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import os
import sys
current_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
sys.path.append(project_dir)
from automation.taskcluster.decision_task import pr
from automation.taskcluster.lib.tasks import TaskBuilder
def loader(kind, path, config, params, loaded_tasks):
repo_url = params['head_repository']
commit = params['head_rev']
trust_level = int(params['level'])
builder = TaskBuilder(
task_id=os.environ.get('TASK_ID'),
repo_url=repo_url,
git_ref=params['head_ref'],
short_head_branch=params['head_ref'],
commit=commit,
owner=params['owner'],
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
scheduler_id='mobile-level-{}'.format(trust_level),
tasks_priority='highest', # TODO parametrize
date_string=params['moz_build_date'],
trust_level=trust_level,
)
tasks_for = params['tasks_for']
if tasks_for == 'github-pull-request':
ordered_groups_of_tasks = pr(builder)
else:
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
for task in ordered_groups_of_tasks:
yield task

20
taskcluster/fenix_taskgraph/target_tasks.py 100644
View File

@ -0,0 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.target_tasks import _target_task, standard_filter
# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
@_target_task('default')
def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
def filter(t, params):
if t.kind == 'old-decision':
return True
return standard_filter(t, params)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]