From aa6021959e0eee4ce14b34513ce3a895a2c85487 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Fri, 13 Sep 2019 12:48:22 +0200 Subject: [PATCH] Let taskgraph load tasks for raptor --- .cron.yml | 13 +++++++++ .taskcluster.yml | 5 +--- automation/taskcluster/decision_task.py | 12 ++++---- automation/taskcluster/lib/tasks.py | 29 ++++++++++--------- .../fenix_taskgraph/loader/old_decision.py | 7 ++++- taskcluster/fenix_taskgraph/target_tasks.py | 6 ++++ 6 files changed, 48 insertions(+), 24 deletions(-) create mode 100644 .cron.yml diff --git a/.cron.yml b/.cron.yml new file mode 100644 index 000000000..a2a7859e7 --- /dev/null +++ b/.cron.yml @@ -0,0 +1,13 @@ +# Definitions for jobs that run periodically. For details on the format, see +# `taskcluster/taskgraph/cron/schema.py`. For documentation, see +# `taskcluster/docs/cron.rst`. +--- + +jobs: + - name: raptor + job: + type: decision-task + treeherder-symbol: raptor-D + # TODO change target method once first tasks are migrated off "old-decision" + target-tasks-method: raptor + when: [{hour: 1, minute: 0}] diff --git a/.taskcluster.yml b/.taskcluster.yml index 6233d3d36..b95eae672 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -90,10 +90,7 @@ tasks: level: $if: 'tasks_for in ["github-push", "github-release", "action", "cron"] && repoUrl == "https://github.com/mozilla-mobile/fenix"' then: '3' - else: - $if: 'tasks_for in ["cron", "action"]' - then: '${repository.level}' - else: '1' + else: '1' in: taskId: '${ownTaskId}' taskGroupId: diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 5858681f7..93b06b7eb 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -51,16 +51,16 @@ def push(builder): def raptor(builder, is_staging): - tasks = [] - mozharness_task_id = fetch_mozharness_task_id() - gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] + gecko_revision = taskcluster.Queue({ + 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), + }).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] variant = get_variant('forPerformanceTest', 'geckoNightly') build_task = builder.craft_assemble_raptor_task(variant) - tasks.append(build_task) signing_task = builder.craft_raptor_signing_task(build_task['label'], variant, is_staging) - tasks.append(signing_task) + + tasks = [build_task, signing_task] for abi in ('armeabi-v7a', 'arm64-v8a'): variant_apk = variant.get_apk(abi) @@ -72,7 +72,7 @@ def raptor(builder, is_staging): ] for craft_function in all_raptor_craft_functions: raptor_task = craft_function( - signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging + signing_task['label'], mozharness_task_id, variant_apk, gecko_revision, is_staging ) tasks.append(raptor_task) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index ebe3e1561..2b79def9d 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -7,6 +7,7 @@ from __future__ import print_function import arrow import datetime import json +import os import taskcluster from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter @@ -558,9 +559,9 @@ class TaskBuilder(object): def craft_raptor_tp6m_cold_task(self, for_suite): - def craft_function(signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging, force_run_on_64_bit_device=False): + def craft_function(signing_task_label, mozharness_task_id, variant_apk, gecko_revision, is_staging, force_run_on_64_bit_device=False): return self._craft_raptor_task( - signing_task_id, + signing_task_label, mozharness_task_id, variant_apk, gecko_revision, @@ -573,10 +574,10 @@ class TaskBuilder(object): ) return craft_function - def craft_raptor_youtube_playback_task(self, signing_task_id, mozharness_task_id, variant_apk, gecko_revision, + def craft_raptor_youtube_playback_task(self, signing_task_label, mozharness_task_id, variant_apk, gecko_revision, is_staging, force_run_on_64_bit_device=False): return self._craft_raptor_task( - signing_task_id, + signing_task_label, mozharness_task_id, variant_apk, gecko_revision, @@ -591,7 +592,7 @@ class TaskBuilder(object): def _craft_raptor_task( self, - signing_task_id, + signing_task_label, mozharness_task_id, variant_apk, gecko_revision, @@ -614,11 +615,11 @@ class TaskBuilder(object): else: raise ValueError('Unsupported architecture "{}"'.format(variant_apk.abi)) - task_name = '{}: forPerformanceTest {}'.format( - name_prefix, '(on 64-bit-device)' if force_run_on_64_bit_device else '' + task_name = '{} {}: forPerformanceTest {}'.format( + name_prefix, variant_apk.abi, '(on 64-bit-device)' if force_run_on_64_bit_device else '' ) - apk_url = '{}/{}/artifacts/{}'.format(_DEFAULT_TASK_URL, signing_task_id, variant_apk.taskcluster_path) + apk_url = '{}//artifacts/{}'.format(_DEFAULT_TASK_URL, variant_apk.taskcluster_path) command = [[ "/builds/taskcluster/script.py", "bash", @@ -639,7 +640,7 @@ class TaskBuilder(object): return self._craft_default_task_definition( worker_type=worker_type, provisioner_id='proj-autophone', - dependencies=[signing_task_id], + dependencies={'signing': signing_task_label}, name=task_name, description=description, routes=['notify.email.perftest-alerts@mozilla.com.on-failed'] if not is_staging else [], @@ -657,10 +658,10 @@ class TaskBuilder(object): )], "command": command, "env": { - "EXTRA_MOZHARNESS_CONFIG": json.dumps({ + "EXTRA_MOZHARNESS_CONFIG": {'task-reference': json.dumps({ "test_packages_url": "{}/{}/artifacts/public/build/en-US/target.test_packages.json".format(_DEFAULT_TASK_URL, mozharness_task_id), "installer_url": apk_url, - }), + })}, "GECKO_HEAD_REPOSITORY": "https://hg.mozilla.org/mozilla-central", "GECKO_HEAD_REV": gecko_revision, "MOZ_AUTOMATION": "1", @@ -670,7 +671,7 @@ class TaskBuilder(object): "MOZHARNESS_CONFIG": "raptor/android_hw_config.py", "MOZHARNESS_SCRIPT": "raptor_script.py", "MOZHARNESS_URL": "{}/{}/artifacts/public/build/en-US/mozharness.zip".format(_DEFAULT_TASK_URL, mozharness_task_id), - "MOZILLA_BUILD_URL": apk_url, + "MOZILLA_BUILD_URL": {'task-reference': apk_url}, "NEED_XVFB": "false", "NO_FAIL_ON_TEST_ERRORS": "1", "SCCACHE_DISABLE": "1", @@ -738,4 +739,6 @@ def schedule_task_graph(ordered_groups_of_tasks): def fetch_mozharness_task_id(): # We now want to use the latest available raptor raptor_index = 'gecko.v2.mozilla-central.nightly.latest.mobile.android-x86_64-opt' - return taskcluster.Index().findTask(raptor_index)['taskId'] + return taskcluster.Index({ + 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), + }).findTask(raptor_index)['taskId'] diff --git a/taskcluster/fenix_taskgraph/loader/old_decision.py b/taskcluster/fenix_taskgraph/loader/old_decision.py index d27b610db..aa1c7403b 100644 --- a/taskcluster/fenix_taskgraph/loader/old_decision.py +++ b/taskcluster/fenix_taskgraph/loader/old_decision.py @@ -12,7 +12,7 @@ current_dir = os.path.dirname(os.path.realpath(__file__)) project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..')) sys.path.append(project_dir) -from automation.taskcluster.decision_task import pr, push, release +from automation.taskcluster.decision_task import pr, push, release, raptor from automation.taskcluster.lib.tasks import TaskBuilder @@ -54,6 +54,11 @@ def loader(kind, path, config, params, loaded_tasks): else: raise ValueError('Github tag must be in semver format and prefixed with a "v", ' 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') + elif tasks_for == 'cron': + if params['target_tasks_method'] == 'raptor': + ordered_groups_of_tasks = raptor(builder, is_staging) + else: + raise NotImplementedError('Unsupported task_name "{}"'.format(params)) else: raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for)) diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index e4b3ec935..1ca38fba6 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -18,3 +18,9 @@ def target_tasks_default(full_task_graph, parameters, graph_config): return standard_filter(t, params) return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] + + +@_target_task('raptor') +def target_tasks_raptor(full_task_graph, parameters, graph_config): + # TODO Change this target task method once old-decision loader is no more + return target_tasks_default(full_task_graph, parameters, graph_config)