Browse Source

Let taskgraph load tasks from decision_tasks.py in PRs

master
Johan Lorenzo 2 years ago
committed by Sebastian Kaspari
parent
commit
2a4f48109d
  1. 1
      .taskcluster.yml
  2. 166
      automation/taskcluster/decision_task.py
  3. 2
      automation/taskcluster/lib/gradle.py
  4. 54
      automation/taskcluster/lib/tasks.py
  5. 2
      taskcluster/ci/config.yml
  6. 9
      taskcluster/ci/old-decision/kind.yml
  7. 2
      taskcluster/fenix_taskgraph/__init__.py
  8. 0
      taskcluster/fenix_taskgraph/loader/__init__.py
  9. 44
      taskcluster/fenix_taskgraph/loader/old_decision.py
  10. 20
      taskcluster/fenix_taskgraph/target_tasks.py

1
.taskcluster.yml

@ -218,6 +218,7 @@ tasks:
taskgraph action-callback
else: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
taskcluster/scripts/install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph decision

166
automation/taskcluster/decision_task.py

@ -26,65 +26,33 @@ from lib.chain_of_trust import (
populate_chain_of_trust_required_but_unused_files
)
REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY')
COMMIT = os.environ.get('MOBILE_HEAD_REV')
PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '')
SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH')
# If we see this text inside a pull request title then we will not execute any tasks for this PR.
SKIP_TASKS_TRIGGER = '[ci skip]'
BUILDER = TaskBuilder(
task_id=os.environ.get('TASK_ID'),
repo_url=REPO_URL,
git_ref=os.environ.get('MOBILE_HEAD_BRANCH'),
short_head_branch=SHORT_HEAD_BRANCH,
commit=COMMIT,
owner="fenix-eng-notifications@mozilla.com",
source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT),
scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'),
tasks_priority=os.environ.get('TASKS_PRIORITY'),
date_string=os.environ.get('BUILD_DATE'),
trust_level=int(os.environ.get('TRUST_LEVEL')),
)
def pr():
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
return {}
build_tasks = {}
signing_tasks = {}
other_tasks = {}
def pr(builder):
tasks = []
variant = get_variant('debug', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant)
tasks.append(builder.craft_assemble_pr_task(variant))
tasks.append(builder.craft_test_pr_task(variant))
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
builder.craft_detekt_task,
builder.craft_ktlint_task,
builder.craft_lint_task,
builder.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
tasks.append(craft_function())
return (build_tasks, signing_tasks, other_tasks)
return tasks
def push():
def push(builder):
all_tasks = pr()
other_tasks = all_tasks[-1]
other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task()
other_tasks[_generate_slug_id()] = builder.craft_ui_tests_task()
return all_tasks
def raptor(is_staging):
def raptor(builder, is_staging):
build_tasks = {}
signing_tasks = {}
other_tasks = {}
@ -93,27 +61,27 @@ def raptor(is_staging):
gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
variant = get_variant('forPerformanceTest', 'geckoNightly')
assemble_task_id = taskcluster.slugId()
build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
assemble_task_id = _generate_slug_id()
build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
for abi in ('armeabi-v7a', 'arm64-v8a'):
variant_apk = variant.get_apk(abi)
all_raptor_craft_functions = [
BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
builder.craft_raptor_tp6m_cold_task(for_suite=i)
for i in range(1, 28)
] + [
BUILDER.craft_raptor_youtube_playback_task,
builder.craft_raptor_youtube_playback_task,
]
for craft_function in all_raptor_craft_functions:
args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
other_tasks[taskcluster.slugId()] = craft_function(*args)
other_tasks[_generate_slug_id()] = craft_function(*args)
return (build_tasks, signing_tasks, other_tasks)
def release(channel, engine, is_staging, version_name):
def release(builder, channel, engine, is_staging, version_name):
variant = get_variant('fenix' + channel.capitalize(), engine)
taskcluster_apk_paths = variant.upstream_artifacts()
@ -121,19 +89,19 @@ def release(channel, engine, is_staging, version_name):
signing_tasks = {}
push_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths,
channel=channel,
is_staging=is_staging,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
push_task_id = _generate_slug_id()
push_tasks[push_task_id] = builder.craft_push_task(
signing_task_id,
taskcluster_apk_paths,
channel=channel,
@ -145,18 +113,18 @@ def release(channel, engine, is_staging, version_name):
return (build_tasks, signing_tasks, push_tasks)
def release_as_fennec(is_staging, version_name):
def release_as_fennec(builder, is_staging, version_name):
variant = get_variant('fennecProduction', 'geckoBeta')
channel = 'fennec-production'
build_tasks = {}
signing_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
variant.upstream_artifacts(),
channel,
@ -166,7 +134,7 @@ def release_as_fennec(is_staging, version_name):
return (build_tasks, signing_tasks)
def nightly_to_production_app(is_staging, version_name):
def nightly_to_production_app(builder, is_staging, version_name):
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
# keep updating this "backwards-compatible" nightly for a while yet
@ -178,12 +146,12 @@ def nightly_to_production_app(is_staging, version_name):
push_tasks = {}
other_tasks = {}
build_task_id = taskcluster.slugId()
build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
build_task_id = _generate_slug_id()
build_tasks[build_task_id] = builder.craft_assemble_release_task(
variant, 'nightly-legacy', is_staging, version_name)
signing_task_id = taskcluster.slugId()
signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
signing_task_id = _generate_slug_id()
signing_tasks[signing_task_id] = builder.craft_release_signing_task(
build_task_id,
taskcluster_apk_paths,
channel='production', # Since we're publishing to the "production" app, we need to sign for production
@ -191,8 +159,8 @@ def nightly_to_production_app(is_staging, version_name):
publish_to_index=False,
)
push_task_id = taskcluster.slugId()
push_tasks[push_task_id] = BUILDER.craft_push_task(
push_task_id = _generate_slug_id()
push_tasks[push_task_id] = builder.craft_push_task(
signing_task_id,
taskcluster_apk_paths,
channel='production', # We're publishing to the "production" app on the "nightly" track
@ -201,61 +169,13 @@ def nightly_to_production_app(is_staging, version_name):
)
if not is_staging:
nimbledroid_task_id = taskcluster.slugId()
other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
nimbledroid_task_id = _generate_slug_id()
other_tasks[nimbledroid_task_id] = builder.craft_upload_apk_nimbledroid_task(
build_task_id
)
return (build_tasks, signing_tasks, push_tasks, other_tasks)
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description='Creates and submit a graph of tasks on Taskcluster.'
)
subparsers = parser.add_subparsers(dest='command')
subparsers.add_parser('pull-request')
subparsers.add_parser('push')
raptor_parser = subparsers.add_parser('raptor')
raptor_parser.add_argument('--staging', action='store_true')
nightly_parser = subparsers.add_parser('nightly')
nightly_parser.add_argument('--staging', action='store_true')
release_parser = subparsers.add_parser('github-release')
release_parser.add_argument('tag')
release_parser.add_argument('--staging', action='store_true')
result = parser.parse_args()
command = result.command
if command == 'pull-request':
ordered_groups_of_tasks = pr()
elif command == 'push':
ordered_groups_of_tasks = push()
elif command == 'raptor':
ordered_groups_of_tasks = raptor(result.staging)
elif command == 'nightly':
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
nightly_version = 'Nightly {}'.format(now)
ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \
+ nightly_to_production_app(result.staging, nightly_version)
ordered_groups_of_tasks += release_as_fennec(result.staging, 'Signed-as-Fennec Nightly {}'.format(now))
elif command == 'github-release':
version = result.tag[1:] # remove prefixed "v"
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
if beta_semver.match(result.tag):
ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version)
elif production_semver.match(result.tag):
ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version)
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
else:
raise Exception('Unsupported command "{}"'.format(command))
populate_chain_of_trust_task_graph(full_task_graph)
def _generate_slug_id():
return taskcluster.slugId()

2
automation/taskcluster/lib/gradle.py

@ -6,7 +6,7 @@ from __future__ import print_function
import json
import subprocess
from lib.variant import Variant, VariantApk
from ..lib.variant import Variant, VariantApk
def get_variant(build_type, engine):

54
automation/taskcluster/lib/tasks.py

@ -9,7 +9,7 @@ import datetime
import json
import taskcluster
from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
DEFAULT_EXPIRES_IN = '1 year'
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
@ -43,7 +43,7 @@ class TaskBuilder(object):
self.scheduler_id = scheduler_id
self.trust_level = trust_level
self.tasks_priority = tasks_priority
self.date = arrow.get(date_string)
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
self.trust_level = trust_level
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
@ -317,10 +317,8 @@ class TaskBuilder(object):
self, name, description, command, dependencies=None, artifacts=None,
routes=None, treeherder=None, env_vars=None, scopes=None
):
dependencies = [] if dependencies is None else dependencies
artifacts = {} if artifacts is None else artifacts
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
env_vars = {} if env_vars is None else env_vars
checkout_command = ' && '.join([
@ -402,7 +400,7 @@ class TaskBuilder(object):
treeherder=None,
notify=None,
):
dependencies = [] if dependencies is None else dependencies
dependencies = {} if dependencies is None else dependencies
scopes = [] if scopes is None else scopes
routes = [] if routes is None else routes
treeherder = {} if treeherder is None else treeherder
@ -421,27 +419,31 @@ class TaskBuilder(object):
extra['notify'] = notify
return {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"dependencies": [self.task_id] + dependencies,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
"attributes": {},
"dependencies": dependencies,
"label": name,
"task": {
"provisionerId": provisioner_id,
"workerType": worker_type,
"taskGroupId": self.task_id,
"schedulerId": self.scheduler_id,
"created": taskcluster.stringDate(created),
"deadline": taskcluster.stringDate(deadline),
"expires": taskcluster.stringDate(expires),
"retries": 5,
"tags": {},
"priority": self.tasks_priority,
"requires": "all-completed",
"routes": routes,
"scopes": scopes,
"payload": payload,
"extra": extra,
"metadata": {
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
},
},
}

2
taskcluster/ci/config.yml

@ -6,7 +6,7 @@ treeherder:
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'
task-priority: lowest
task-priority: highest
taskgraph:
register: fenix_taskgraph:register

9
taskcluster/ci/old-decision/kind.yml

@ -0,0 +1,9 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: fenix_taskgraph.loader.old_decision:loader
transforms: []
# XXX Everything is done in the loader until everything is migrated

2
taskcluster/fenix_taskgraph/__init__.py

@ -12,7 +12,7 @@ def register(graph_config):
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules(["job", "worker_types", "routes", "target"])
_import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
def _import_modules(modules):

0
taskcluster/fenix_taskgraph/loader/__init__.py

44
taskcluster/fenix_taskgraph/loader/old_decision.py

@ -0,0 +1,44 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function, unicode_literals
import os
import sys
current_dir = os.path.dirname(os.path.realpath(__file__))
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
sys.path.append(project_dir)
from automation.taskcluster.decision_task import pr
from automation.taskcluster.lib.tasks import TaskBuilder
def loader(kind, path, config, params, loaded_tasks):
repo_url = params['head_repository']
commit = params['head_rev']
trust_level = int(params['level'])
builder = TaskBuilder(
task_id=os.environ.get('TASK_ID'),
repo_url=repo_url,
git_ref=params['head_ref'],
short_head_branch=params['head_ref'],
commit=commit,
owner=params['owner'],
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
scheduler_id='mobile-level-{}'.format(trust_level),
tasks_priority='highest', # TODO parametrize
date_string=params['moz_build_date'],
trust_level=trust_level,
)
tasks_for = params['tasks_for']
if tasks_for == 'github-pull-request':
ordered_groups_of_tasks = pr(builder)
else:
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
for task in ordered_groups_of_tasks:
yield task

20
taskcluster/fenix_taskgraph/target_tasks.py

@ -0,0 +1,20 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.target_tasks import _target_task, standard_filter
# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
@_target_task('default')
def target_tasks_default(full_task_graph, parameters, graph_config):
"""Target the tasks which have indicated they should be run on this project
via the `run_on_projects` attributes."""
def filter(t, params):
if t.kind == 'old-decision':
return True
return standard_filter(t, params)
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
Loading…
Cancel
Save