From 2a4f48109d30c0a05738e7b25a30716e26c12985 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 12 Sep 2019 18:45:26 +0200 Subject: [PATCH] Let taskgraph load tasks from decision_tasks.py in PRs --- .taskcluster.yml | 1 + automation/taskcluster/decision_task.py | 166 +++++------------- automation/taskcluster/lib/gradle.py | 2 +- automation/taskcluster/lib/tasks.py | 54 +++--- taskcluster/ci/config.yml | 2 +- taskcluster/ci/old-decision/kind.yml | 9 + taskcluster/fenix_taskgraph/__init__.py | 2 +- .../fenix_taskgraph/loader/__init__.py | 0 .../fenix_taskgraph/loader/old_decision.py | 44 +++++ taskcluster/fenix_taskgraph/target_tasks.py | 20 +++ 10 files changed, 148 insertions(+), 152 deletions(-) create mode 100644 taskcluster/ci/old-decision/kind.yml create mode 100644 taskcluster/fenix_taskgraph/loader/__init__.py create mode 100644 taskcluster/fenix_taskgraph/loader/old_decision.py create mode 100644 taskcluster/fenix_taskgraph/target_tasks.py diff --git a/.taskcluster.yml b/.taskcluster.yml index 4a1491a2b..bfc755f63 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -218,6 +218,7 @@ tasks: taskgraph action-callback else: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && + PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml && taskcluster/scripts/install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 8f6d2a551..cab290c2a 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -26,65 +26,33 @@ from lib.chain_of_trust import ( populate_chain_of_trust_required_but_unused_files ) -REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') -COMMIT = os.environ.get('MOBILE_HEAD_REV') -PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '') -SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH') - -# If we see this text inside a pull request title then we will not execute any tasks for this PR. -SKIP_TASKS_TRIGGER = '[ci skip]' - - -BUILDER = TaskBuilder( - task_id=os.environ.get('TASK_ID'), - repo_url=REPO_URL, - git_ref=os.environ.get('MOBILE_HEAD_BRANCH'), - short_head_branch=SHORT_HEAD_BRANCH, - commit=COMMIT, - owner="fenix-eng-notifications@mozilla.com", - source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT), - scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'), - tasks_priority=os.environ.get('TASKS_PRIORITY'), - date_string=os.environ.get('BUILD_DATE'), - trust_level=int(os.environ.get('TRUST_LEVEL')), -) - - -def pr(): - if SKIP_TASKS_TRIGGER in PR_TITLE: - print("Pull request title contains", SKIP_TASKS_TRIGGER) - print("Exit") - return {} - - build_tasks = {} - signing_tasks = {} - other_tasks = {} +def pr(builder): + tasks = [] variant = get_variant('debug', 'geckoNightly') - assemble_task_id = taskcluster.slugId() - build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant) - build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant) + tasks.append(builder.craft_assemble_pr_task(variant)) + tasks.append(builder.craft_test_pr_task(variant)) for craft_function in ( - BUILDER.craft_detekt_task, - BUILDER.craft_ktlint_task, - BUILDER.craft_lint_task, - BUILDER.craft_compare_locales_task, + builder.craft_detekt_task, + builder.craft_ktlint_task, + builder.craft_lint_task, + builder.craft_compare_locales_task, ): - other_tasks[taskcluster.slugId()] = craft_function() + tasks.append(craft_function()) - return (build_tasks, signing_tasks, other_tasks) + return tasks -def push(): +def push(builder): all_tasks = pr() other_tasks = all_tasks[-1] - other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task() + other_tasks[_generate_slug_id()] = builder.craft_ui_tests_task() return all_tasks -def raptor(is_staging): +def raptor(builder, is_staging): build_tasks = {} signing_tasks = {} other_tasks = {} @@ -93,27 +61,27 @@ def raptor(is_staging): gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] variant = get_variant('forPerformanceTest', 'geckoNightly') - assemble_task_id = taskcluster.slugId() - build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging) + assemble_task_id = _generate_slug_id() + build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant) + signing_task_id = _generate_slug_id() + signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging) for abi in ('armeabi-v7a', 'arm64-v8a'): variant_apk = variant.get_apk(abi) all_raptor_craft_functions = [ - BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) + builder.craft_raptor_tp6m_cold_task(for_suite=i) for i in range(1, 28) ] + [ - BUILDER.craft_raptor_youtube_playback_task, + builder.craft_raptor_youtube_playback_task, ] for craft_function in all_raptor_craft_functions: args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging) - other_tasks[taskcluster.slugId()] = craft_function(*args) + other_tasks[_generate_slug_id()] = craft_function(*args) return (build_tasks, signing_tasks, other_tasks) -def release(channel, engine, is_staging, version_name): +def release(builder, channel, engine, is_staging, version_name): variant = get_variant('fenix' + channel.capitalize(), engine) taskcluster_apk_paths = variant.upstream_artifacts() @@ -121,19 +89,19 @@ def release(channel, engine, is_staging, version_name): signing_tasks = {} push_tasks = {} - build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name) + build_task_id = _generate_slug_id() + build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( + signing_task_id = _generate_slug_id() + signing_tasks[signing_task_id] = builder.craft_release_signing_task( build_task_id, taskcluster_apk_paths, channel=channel, is_staging=is_staging, ) - push_task_id = taskcluster.slugId() - push_tasks[push_task_id] = BUILDER.craft_push_task( + push_task_id = _generate_slug_id() + push_tasks[push_task_id] = builder.craft_push_task( signing_task_id, taskcluster_apk_paths, channel=channel, @@ -145,18 +113,18 @@ def release(channel, engine, is_staging, version_name): return (build_tasks, signing_tasks, push_tasks) -def release_as_fennec(is_staging, version_name): +def release_as_fennec(builder, is_staging, version_name): variant = get_variant('fennecProduction', 'geckoBeta') channel = 'fennec-production' build_tasks = {} signing_tasks = {} - build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name) + build_task_id = _generate_slug_id() + build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( + signing_task_id = _generate_slug_id() + signing_tasks[signing_task_id] = builder.craft_release_signing_task( build_task_id, variant.upstream_artifacts(), channel, @@ -166,7 +134,7 @@ def release_as_fennec(is_staging, version_name): return (build_tasks, signing_tasks) -def nightly_to_production_app(is_staging, version_name): +def nightly_to_production_app(builder, is_staging, version_name): # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the # "nightly" track. We're moving towards having each channel be published to its own app, but we need to # keep updating this "backwards-compatible" nightly for a while yet @@ -178,12 +146,12 @@ def nightly_to_production_app(is_staging, version_name): push_tasks = {} other_tasks = {} - build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_release_task( + build_task_id = _generate_slug_id() + build_tasks[build_task_id] = builder.craft_assemble_release_task( variant, 'nightly-legacy', is_staging, version_name) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( + signing_task_id = _generate_slug_id() + signing_tasks[signing_task_id] = builder.craft_release_signing_task( build_task_id, taskcluster_apk_paths, channel='production', # Since we're publishing to the "production" app, we need to sign for production @@ -191,8 +159,8 @@ def nightly_to_production_app(is_staging, version_name): publish_to_index=False, ) - push_task_id = taskcluster.slugId() - push_tasks[push_task_id] = BUILDER.craft_push_task( + push_task_id = _generate_slug_id() + push_tasks[push_task_id] = builder.craft_push_task( signing_task_id, taskcluster_apk_paths, channel='production', # We're publishing to the "production" app on the "nightly" track @@ -201,61 +169,13 @@ def nightly_to_production_app(is_staging, version_name): ) if not is_staging: - nimbledroid_task_id = taskcluster.slugId() - other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task( + nimbledroid_task_id = _generate_slug_id() + other_tasks[nimbledroid_task_id] = builder.craft_upload_apk_nimbledroid_task( build_task_id ) return (build_tasks, signing_tasks, push_tasks, other_tasks) -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description='Creates and submit a graph of tasks on Taskcluster.' - ) - - subparsers = parser.add_subparsers(dest='command') - - subparsers.add_parser('pull-request') - subparsers.add_parser('push') - - raptor_parser = subparsers.add_parser('raptor') - raptor_parser.add_argument('--staging', action='store_true') - - nightly_parser = subparsers.add_parser('nightly') - nightly_parser.add_argument('--staging', action='store_true') - - release_parser = subparsers.add_parser('github-release') - release_parser.add_argument('tag') - release_parser.add_argument('--staging', action='store_true') - - result = parser.parse_args() - command = result.command - - if command == 'pull-request': - ordered_groups_of_tasks = pr() - elif command == 'push': - ordered_groups_of_tasks = push() - elif command == 'raptor': - ordered_groups_of_tasks = raptor(result.staging) - elif command == 'nightly': - now = datetime.datetime.now().strftime('%y%m%d %H:%M') - nightly_version = 'Nightly {}'.format(now) - ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \ - + nightly_to_production_app(result.staging, nightly_version) - ordered_groups_of_tasks += release_as_fennec(result.staging, 'Signed-as-Fennec Nightly {}'.format(now)) - elif command == 'github-release': - version = result.tag[1:] # remove prefixed "v" - beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') - production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') - if beta_semver.match(result.tag): - ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version) - elif production_semver.match(result.tag): - ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version) - else: - raise ValueError('Github tag must be in semver format and prefixed with a "v", ' - 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') - else: - raise Exception('Unsupported command "{}"'.format(command)) - - populate_chain_of_trust_task_graph(full_task_graph) +def _generate_slug_id(): + return taskcluster.slugId() diff --git a/automation/taskcluster/lib/gradle.py b/automation/taskcluster/lib/gradle.py index df79265d0..2b496e59a 100644 --- a/automation/taskcluster/lib/gradle.py +++ b/automation/taskcluster/lib/gradle.py @@ -6,7 +6,7 @@ from __future__ import print_function import json import subprocess -from lib.variant import Variant, VariantApk +from ..lib.variant import Variant, VariantApk def get_variant(build_type, engine): diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 98c08f675..c2d01c89f 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -9,7 +9,7 @@ import datetime import json import taskcluster -from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter +from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter DEFAULT_EXPIRES_IN = '1 year' DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk' @@ -43,7 +43,7 @@ class TaskBuilder(object): self.scheduler_id = scheduler_id self.trust_level = trust_level self.tasks_priority = tasks_priority - self.date = arrow.get(date_string) + self.date = arrow.get(date_string, 'YYYYMMDDHHmmss') self.trust_level = trust_level def craft_assemble_release_task(self, variant, channel, is_staging, version_name): @@ -317,10 +317,8 @@ class TaskBuilder(object): self, name, description, command, dependencies=None, artifacts=None, routes=None, treeherder=None, env_vars=None, scopes=None ): - dependencies = [] if dependencies is None else dependencies artifacts = {} if artifacts is None else artifacts scopes = [] if scopes is None else scopes - routes = [] if routes is None else routes env_vars = {} if env_vars is None else env_vars checkout_command = ' && '.join([ @@ -402,7 +400,7 @@ class TaskBuilder(object): treeherder=None, notify=None, ): - dependencies = [] if dependencies is None else dependencies + dependencies = {} if dependencies is None else dependencies scopes = [] if scopes is None else scopes routes = [] if routes is None else routes treeherder = {} if treeherder is None else treeherder @@ -421,27 +419,31 @@ class TaskBuilder(object): extra['notify'] = notify return { - "provisionerId": provisioner_id, - "workerType": worker_type, - "taskGroupId": self.task_id, - "schedulerId": self.scheduler_id, - "created": taskcluster.stringDate(created), - "deadline": taskcluster.stringDate(deadline), - "expires": taskcluster.stringDate(expires), - "retries": 5, - "tags": {}, - "priority": self.tasks_priority, - "dependencies": [self.task_id] + dependencies, - "requires": "all-completed", - "routes": routes, - "scopes": scopes, - "payload": payload, - "extra": extra, - "metadata": { - "name": "Fenix - {}".format(name), - "description": description, - "owner": self.owner, - "source": self.source, + "attributes": {}, + "dependencies": dependencies, + "label": name, + "task": { + "provisionerId": provisioner_id, + "workerType": worker_type, + "taskGroupId": self.task_id, + "schedulerId": self.scheduler_id, + "created": taskcluster.stringDate(created), + "deadline": taskcluster.stringDate(deadline), + "expires": taskcluster.stringDate(expires), + "retries": 5, + "tags": {}, + "priority": self.tasks_priority, + "requires": "all-completed", + "routes": routes, + "scopes": scopes, + "payload": payload, + "extra": extra, + "metadata": { + "name": "Fenix - {}".format(name), + "description": description, + "owner": self.owner, + "source": self.source, + }, }, } diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 7434fc00b..5bad2f8cc 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -6,7 +6,7 @@ treeherder: 'Rap': 'Raptor tests' 'Rap-P': 'Raptor power tests' -task-priority: lowest +task-priority: highest taskgraph: register: fenix_taskgraph:register diff --git a/taskcluster/ci/old-decision/kind.yml b/taskcluster/ci/old-decision/kind.yml new file mode 100644 index 000000000..04682b195 --- /dev/null +++ b/taskcluster/ci/old-decision/kind.yml @@ -0,0 +1,9 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: fenix_taskgraph.loader.old_decision:loader + +transforms: [] + +# XXX Everything is done in the loader until everything is migrated diff --git a/taskcluster/fenix_taskgraph/__init__.py b/taskcluster/fenix_taskgraph/__init__.py index 90e56ee5a..21bfd6c13 100644 --- a/taskcluster/fenix_taskgraph/__init__.py +++ b/taskcluster/fenix_taskgraph/__init__.py @@ -12,7 +12,7 @@ def register(graph_config): Import all modules that are siblings of this one, triggering decorators in the process. """ - _import_modules(["job", "worker_types", "routes", "target"]) + _import_modules(["job", "worker_types", "routes", "target", "target_tasks"]) def _import_modules(modules): diff --git a/taskcluster/fenix_taskgraph/loader/__init__.py b/taskcluster/fenix_taskgraph/loader/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/taskcluster/fenix_taskgraph/loader/old_decision.py b/taskcluster/fenix_taskgraph/loader/old_decision.py new file mode 100644 index 000000000..9963a4ce6 --- /dev/null +++ b/taskcluster/fenix_taskgraph/loader/old_decision.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function, unicode_literals + +import os +import sys + +current_dir = os.path.dirname(os.path.realpath(__file__)) +project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..')) +sys.path.append(project_dir) + +from automation.taskcluster.decision_task import pr +from automation.taskcluster.lib.tasks import TaskBuilder + + +def loader(kind, path, config, params, loaded_tasks): + repo_url = params['head_repository'] + commit = params['head_rev'] + trust_level = int(params['level']) + + builder = TaskBuilder( + task_id=os.environ.get('TASK_ID'), + repo_url=repo_url, + git_ref=params['head_ref'], + short_head_branch=params['head_ref'], + commit=commit, + owner=params['owner'], + source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit), + scheduler_id='mobile-level-{}'.format(trust_level), + tasks_priority='highest', # TODO parametrize + date_string=params['moz_build_date'], + trust_level=trust_level, + ) + + tasks_for = params['tasks_for'] + if tasks_for == 'github-pull-request': + ordered_groups_of_tasks = pr(builder) + else: + raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for)) + + for task in ordered_groups_of_tasks: + yield task diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py new file mode 100644 index 000000000..e4b3ec935 --- /dev/null +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -0,0 +1,20 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.target_tasks import _target_task, standard_filter + +# XXX We're overwritting the default target_task while all tasks are ported to taskgraph +@_target_task('default') +def target_tasks_default(full_task_graph, parameters, graph_config): + """Target the tasks which have indicated they should be run on this project + via the `run_on_projects` attributes.""" + def filter(t, params): + if t.kind == 'old-decision': + return True + + return standard_filter(t, params) + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]