Let taskgraph load tasks from decision_tasks.py in PRs
parent
daa1693f23
commit
2a4f48109d
|
@ -218,6 +218,7 @@ tasks:
|
|||
taskgraph action-callback
|
||||
else: >
|
||||
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
|
||||
PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
|
||||
taskcluster/scripts/install-sdk.sh &&
|
||||
ln -s /builds/worker/artifacts artifacts &&
|
||||
~/.local/bin/taskgraph decision
|
||||
|
|
|
@ -26,65 +26,33 @@ from lib.chain_of_trust import (
|
|||
populate_chain_of_trust_required_but_unused_files
|
||||
)
|
||||
|
||||
REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY')
|
||||
COMMIT = os.environ.get('MOBILE_HEAD_REV')
|
||||
PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '')
|
||||
SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH')
|
||||
|
||||
# If we see this text inside a pull request title then we will not execute any tasks for this PR.
|
||||
SKIP_TASKS_TRIGGER = '[ci skip]'
|
||||
|
||||
|
||||
BUILDER = TaskBuilder(
|
||||
task_id=os.environ.get('TASK_ID'),
|
||||
repo_url=REPO_URL,
|
||||
git_ref=os.environ.get('MOBILE_HEAD_BRANCH'),
|
||||
short_head_branch=SHORT_HEAD_BRANCH,
|
||||
commit=COMMIT,
|
||||
owner="fenix-eng-notifications@mozilla.com",
|
||||
source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT),
|
||||
scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'),
|
||||
tasks_priority=os.environ.get('TASKS_PRIORITY'),
|
||||
date_string=os.environ.get('BUILD_DATE'),
|
||||
trust_level=int(os.environ.get('TRUST_LEVEL')),
|
||||
)
|
||||
|
||||
|
||||
def pr():
|
||||
if SKIP_TASKS_TRIGGER in PR_TITLE:
|
||||
print("Pull request title contains", SKIP_TASKS_TRIGGER)
|
||||
print("Exit")
|
||||
return {}
|
||||
|
||||
build_tasks = {}
|
||||
signing_tasks = {}
|
||||
other_tasks = {}
|
||||
def pr(builder):
|
||||
tasks = []
|
||||
|
||||
variant = get_variant('debug', 'geckoNightly')
|
||||
assemble_task_id = taskcluster.slugId()
|
||||
build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant)
|
||||
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant)
|
||||
tasks.append(builder.craft_assemble_pr_task(variant))
|
||||
tasks.append(builder.craft_test_pr_task(variant))
|
||||
|
||||
for craft_function in (
|
||||
BUILDER.craft_detekt_task,
|
||||
BUILDER.craft_ktlint_task,
|
||||
BUILDER.craft_lint_task,
|
||||
BUILDER.craft_compare_locales_task,
|
||||
builder.craft_detekt_task,
|
||||
builder.craft_ktlint_task,
|
||||
builder.craft_lint_task,
|
||||
builder.craft_compare_locales_task,
|
||||
):
|
||||
other_tasks[taskcluster.slugId()] = craft_function()
|
||||
tasks.append(craft_function())
|
||||
|
||||
return (build_tasks, signing_tasks, other_tasks)
|
||||
return tasks
|
||||
|
||||
|
||||
def push():
|
||||
def push(builder):
|
||||
all_tasks = pr()
|
||||
other_tasks = all_tasks[-1]
|
||||
other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task()
|
||||
other_tasks[_generate_slug_id()] = builder.craft_ui_tests_task()
|
||||
|
||||
return all_tasks
|
||||
|
||||
|
||||
def raptor(is_staging):
|
||||
def raptor(builder, is_staging):
|
||||
build_tasks = {}
|
||||
signing_tasks = {}
|
||||
other_tasks = {}
|
||||
|
@ -93,27 +61,27 @@ def raptor(is_staging):
|
|||
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
|
||||
|
||||
variant = get_variant('forPerformanceTest', 'geckoNightly')
|
||||
assemble_task_id = taskcluster.slugId()
|
||||
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
|
||||
signing_task_id = taskcluster.slugId()
|
||||
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
|
||||
assemble_task_id = _generate_slug_id()
|
||||
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
|
||||
|
||||
for abi in ('armeabi-v7a', 'arm64-v8a'):
|
||||
variant_apk = variant.get_apk(abi)
|
||||
all_raptor_craft_functions = [
|
||||
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
|
||||
builder.craft_raptor_tp6m_cold_task(for_suite=i)
|
||||
for i in range(1, 28)
|
||||
] + [
|
||||
BUILDER.craft_raptor_youtube_playback_task,
|
||||
builder.craft_raptor_youtube_playback_task,
|
||||
]
|
||||
for craft_function in all_raptor_craft_functions:
|
||||
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
|
||||
other_tasks[taskcluster.slugId()] = craft_function(*args)
|
||||
other_tasks[_generate_slug_id()] = craft_function(*args)
|
||||
|
||||
return (build_tasks, signing_tasks, other_tasks)
|
||||
|
||||
|
||||
def release(channel, engine, is_staging, version_name):
|
||||
def release(builder, channel, engine, is_staging, version_name):
|
||||
variant = get_variant('fenix' + channel.capitalize(), engine)
|
||||
taskcluster_apk_paths = variant.upstream_artifacts()
|
||||
|
||||
|
@ -121,19 +89,19 @@ def release(channel, engine, is_staging, version_name):
|
|||
signing_tasks = {}
|
||||
push_tasks = {}
|
||||
|
||||
build_task_id = taskcluster.slugId()
|
||||
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
build_task_id = _generate_slug_id()
|
||||
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
signing_task_id = taskcluster.slugId()
|
||||
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
|
||||
build_task_id,
|
||||
taskcluster_apk_paths,
|
||||
channel=channel,
|
||||
is_staging=is_staging,
|
||||
)
|
||||
|
||||
push_task_id = taskcluster.slugId()
|
||||
push_tasks[push_task_id] = BUILDER.craft_push_task(
|
||||
push_task_id = _generate_slug_id()
|
||||
push_tasks[push_task_id] = builder.craft_push_task(
|
||||
signing_task_id,
|
||||
taskcluster_apk_paths,
|
||||
channel=channel,
|
||||
|
@ -145,18 +113,18 @@ def release(channel, engine, is_staging, version_name):
|
|||
return (build_tasks, signing_tasks, push_tasks)
|
||||
|
||||
|
||||
def release_as_fennec(is_staging, version_name):
|
||||
def release_as_fennec(builder, is_staging, version_name):
|
||||
variant = get_variant('fennecProduction', 'geckoBeta')
|
||||
channel = 'fennec-production'
|
||||
|
||||
build_tasks = {}
|
||||
signing_tasks = {}
|
||||
|
||||
build_task_id = taskcluster.slugId()
|
||||
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
build_task_id = _generate_slug_id()
|
||||
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
signing_task_id = taskcluster.slugId()
|
||||
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
|
||||
build_task_id,
|
||||
variant.upstream_artifacts(),
|
||||
channel,
|
||||
|
@ -166,7 +134,7 @@ def release_as_fennec(is_staging, version_name):
|
|||
return (build_tasks, signing_tasks)
|
||||
|
||||
|
||||
def nightly_to_production_app(is_staging, version_name):
|
||||
def nightly_to_production_app(builder, is_staging, version_name):
|
||||
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
|
||||
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
|
||||
# keep updating this "backwards-compatible" nightly for a while yet
|
||||
|
@ -178,12 +146,12 @@ def nightly_to_production_app(is_staging, version_name):
|
|||
push_tasks = {}
|
||||
other_tasks = {}
|
||||
|
||||
build_task_id = taskcluster.slugId()
|
||||
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
|
||||
build_task_id = _generate_slug_id()
|
||||
build_tasks[build_task_id] = builder.craft_assemble_release_task(
|
||||
variant, 'nightly-legacy', is_staging, version_name)
|
||||
|
||||
signing_task_id = taskcluster.slugId()
|
||||
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
|
||||
signing_task_id = _generate_slug_id()
|
||||
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
|
||||
build_task_id,
|
||||
taskcluster_apk_paths,
|
||||
channel='production', # Since we're publishing to the "production" app, we need to sign for production
|
||||
|
@ -191,8 +159,8 @@ def nightly_to_production_app(is_staging, version_name):
|
|||
publish_to_index=False,
|
||||
)
|
||||
|
||||
push_task_id = taskcluster.slugId()
|
||||
push_tasks[push_task_id] = BUILDER.craft_push_task(
|
||||
push_task_id = _generate_slug_id()
|
||||
push_tasks[push_task_id] = builder.craft_push_task(
|
||||
signing_task_id,
|
||||
taskcluster_apk_paths,
|
||||
channel='production', # We're publishing to the "production" app on the "nightly" track
|
||||
|
@ -201,61 +169,13 @@ def nightly_to_production_app(is_staging, version_name):
|
|||
)
|
||||
|
||||
if not is_staging:
|
||||
nimbledroid_task_id = taskcluster.slugId()
|
||||
other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
|
||||
nimbledroid_task_id = _generate_slug_id()
|
||||
other_tasks[nimbledroid_task_id] = builder.craft_upload_apk_nimbledroid_task(
|
||||
build_task_id
|
||||
)
|
||||
|
||||
return (build_tasks, signing_tasks, push_tasks, other_tasks)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Creates and submit a graph of tasks on Taskcluster.'
|
||||
)
|
||||
|
||||
subparsers = parser.add_subparsers(dest='command')
|
||||
|
||||
subparsers.add_parser('pull-request')
|
||||
subparsers.add_parser('push')
|
||||
|
||||
raptor_parser = subparsers.add_parser('raptor')
|
||||
raptor_parser.add_argument('--staging', action='store_true')
|
||||
|
||||
nightly_parser = subparsers.add_parser('nightly')
|
||||
nightly_parser.add_argument('--staging', action='store_true')
|
||||
|
||||
release_parser = subparsers.add_parser('github-release')
|
||||
release_parser.add_argument('tag')
|
||||
release_parser.add_argument('--staging', action='store_true')
|
||||
|
||||
result = parser.parse_args()
|
||||
command = result.command
|
||||
|
||||
if command == 'pull-request':
|
||||
ordered_groups_of_tasks = pr()
|
||||
elif command == 'push':
|
||||
ordered_groups_of_tasks = push()
|
||||
elif command == 'raptor':
|
||||
ordered_groups_of_tasks = raptor(result.staging)
|
||||
elif command == 'nightly':
|
||||
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
|
||||
nightly_version = 'Nightly {}'.format(now)
|
||||
ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \
|
||||
+ nightly_to_production_app(result.staging, nightly_version)
|
||||
ordered_groups_of_tasks += release_as_fennec(result.staging, 'Signed-as-Fennec Nightly {}'.format(now))
|
||||
elif command == 'github-release':
|
||||
version = result.tag[1:] # remove prefixed "v"
|
||||
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
|
||||
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
|
||||
if beta_semver.match(result.tag):
|
||||
ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version)
|
||||
elif production_semver.match(result.tag):
|
||||
ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version)
|
||||
else:
|
||||
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
|
||||
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
|
||||
else:
|
||||
raise Exception('Unsupported command "{}"'.format(command))
|
||||
|
||||
populate_chain_of_trust_task_graph(full_task_graph)
|
||||
def _generate_slug_id():
|
||||
return taskcluster.slugId()
|
||||
|
|
|
@ -6,7 +6,7 @@ from __future__ import print_function
|
|||
import json
|
||||
import subprocess
|
||||
|
||||
from lib.variant import Variant, VariantApk
|
||||
from ..lib.variant import Variant, VariantApk
|
||||
|
||||
|
||||
def get_variant(build_type, engine):
|
||||
|
|
|
@ -9,7 +9,7 @@ import datetime
|
|||
import json
|
||||
import taskcluster
|
||||
|
||||
from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
|
||||
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
|
||||
|
||||
DEFAULT_EXPIRES_IN = '1 year'
|
||||
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
|
||||
|
@ -43,7 +43,7 @@ class TaskBuilder(object):
|
|||
self.scheduler_id = scheduler_id
|
||||
self.trust_level = trust_level
|
||||
self.tasks_priority = tasks_priority
|
||||
self.date = arrow.get(date_string)
|
||||
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
|
||||
self.trust_level = trust_level
|
||||
|
||||
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
|
||||
|
@ -317,10 +317,8 @@ class TaskBuilder(object):
|
|||
self, name, description, command, dependencies=None, artifacts=None,
|
||||
routes=None, treeherder=None, env_vars=None, scopes=None
|
||||
):
|
||||
dependencies = [] if dependencies is None else dependencies
|
||||
artifacts = {} if artifacts is None else artifacts
|
||||
scopes = [] if scopes is None else scopes
|
||||
routes = [] if routes is None else routes
|
||||
env_vars = {} if env_vars is None else env_vars
|
||||
|
||||
checkout_command = ' && '.join([
|
||||
|
@ -402,7 +400,7 @@ class TaskBuilder(object):
|
|||
treeherder=None,
|
||||
notify=None,
|
||||
):
|
||||
dependencies = [] if dependencies is None else dependencies
|
||||
dependencies = {} if dependencies is None else dependencies
|
||||
scopes = [] if scopes is None else scopes
|
||||
routes = [] if routes is None else routes
|
||||
treeherder = {} if treeherder is None else treeherder
|
||||
|
@ -421,27 +419,31 @@ class TaskBuilder(object):
|
|||
extra['notify'] = notify
|
||||
|
||||
return {
|
||||
"provisionerId": provisioner_id,
|
||||
"workerType": worker_type,
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"tags": {},
|
||||
"priority": self.tasks_priority,
|
||||
"dependencies": [self.task_id] + dependencies,
|
||||
"requires": "all-completed",
|
||||
"routes": routes,
|
||||
"scopes": scopes,
|
||||
"payload": payload,
|
||||
"extra": extra,
|
||||
"metadata": {
|
||||
"name": "Fenix - {}".format(name),
|
||||
"description": description,
|
||||
"owner": self.owner,
|
||||
"source": self.source,
|
||||
"attributes": {},
|
||||
"dependencies": dependencies,
|
||||
"label": name,
|
||||
"task": {
|
||||
"provisionerId": provisioner_id,
|
||||
"workerType": worker_type,
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"tags": {},
|
||||
"priority": self.tasks_priority,
|
||||
"requires": "all-completed",
|
||||
"routes": routes,
|
||||
"scopes": scopes,
|
||||
"payload": payload,
|
||||
"extra": extra,
|
||||
"metadata": {
|
||||
"name": "Fenix - {}".format(name),
|
||||
"description": description,
|
||||
"owner": self.owner,
|
||||
"source": self.source,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -6,7 +6,7 @@ treeherder:
|
|||
'Rap': 'Raptor tests'
|
||||
'Rap-P': 'Raptor power tests'
|
||||
|
||||
task-priority: lowest
|
||||
task-priority: highest
|
||||
|
||||
taskgraph:
|
||||
register: fenix_taskgraph:register
|
||||
|
|
|
@ -0,0 +1,9 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: fenix_taskgraph.loader.old_decision:loader
|
||||
|
||||
transforms: []
|
||||
|
||||
# XXX Everything is done in the loader until everything is migrated
|
|
@ -12,7 +12,7 @@ def register(graph_config):
|
|||
Import all modules that are siblings of this one, triggering decorators in
|
||||
the process.
|
||||
"""
|
||||
_import_modules(["job", "worker_types", "routes", "target"])
|
||||
_import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
|
||||
|
||||
|
||||
def _import_modules(modules):
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
|
||||
sys.path.append(project_dir)
|
||||
|
||||
from automation.taskcluster.decision_task import pr
|
||||
from automation.taskcluster.lib.tasks import TaskBuilder
|
||||
|
||||
|
||||
def loader(kind, path, config, params, loaded_tasks):
|
||||
repo_url = params['head_repository']
|
||||
commit = params['head_rev']
|
||||
trust_level = int(params['level'])
|
||||
|
||||
builder = TaskBuilder(
|
||||
task_id=os.environ.get('TASK_ID'),
|
||||
repo_url=repo_url,
|
||||
git_ref=params['head_ref'],
|
||||
short_head_branch=params['head_ref'],
|
||||
commit=commit,
|
||||
owner=params['owner'],
|
||||
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
|
||||
scheduler_id='mobile-level-{}'.format(trust_level),
|
||||
tasks_priority='highest', # TODO parametrize
|
||||
date_string=params['moz_build_date'],
|
||||
trust_level=trust_level,
|
||||
)
|
||||
|
||||
tasks_for = params['tasks_for']
|
||||
if tasks_for == 'github-pull-request':
|
||||
ordered_groups_of_tasks = pr(builder)
|
||||
else:
|
||||
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
|
||||
|
||||
for task in ordered_groups_of_tasks:
|
||||
yield task
|
|
@ -0,0 +1,20 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.target_tasks import _target_task, standard_filter
|
||||
|
||||
# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
|
||||
@_target_task('default')
|
||||
def target_tasks_default(full_task_graph, parameters, graph_config):
|
||||
"""Target the tasks which have indicated they should be run on this project
|
||||
via the `run_on_projects` attributes."""
|
||||
def filter(t, params):
|
||||
if t.kind == 'old-decision':
|
||||
return True
|
||||
|
||||
return standard_filter(t, params)
|
||||
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
Loading…
Reference in New Issue