Let taskgraph load tasks from decision_tasks.py in PRs

.taskcluster.yml

@@ -218,6 +218,7 @@ tasks:
                                         taskgraph action-callback
                                     else: >
                                         PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
+                                        PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
                                         taskcluster/scripts/install-sdk.sh &&
                                         ln -s /builds/worker/artifacts artifacts &&
                                         ~/.local/bin/taskgraph decision

automation/taskcluster/decision_task.py

@@ -26,65 +26,33 @@ from lib.chain_of_trust import (
     populate_chain_of_trust_required_but_unused_files
 )
 
-REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY')
+def pr(builder):
-COMMIT = os.environ.get('MOBILE_HEAD_REV')
+    tasks = []
-PR_TITLE = os.environ.get('GITHUB_PULL_TITLE', '')
-SHORT_HEAD_BRANCH = os.environ.get('SHORT_HEAD_BRANCH')
-
-# If we see this text inside a pull request title then we will not execute any tasks for this PR.
-SKIP_TASKS_TRIGGER = '[ci skip]'
-
-
-BUILDER = TaskBuilder(
-    task_id=os.environ.get('TASK_ID'),
-    repo_url=REPO_URL,
-    git_ref=os.environ.get('MOBILE_HEAD_BRANCH'),
-    short_head_branch=SHORT_HEAD_BRANCH,
-    commit=COMMIT,
-    owner="fenix-eng-notifications@mozilla.com",
-    source='{}/raw/{}/.taskcluster.yml'.format(REPO_URL, COMMIT),
-    scheduler_id=os.environ.get('SCHEDULER_ID', 'taskcluster-github'),
-    tasks_priority=os.environ.get('TASKS_PRIORITY'),
-    date_string=os.environ.get('BUILD_DATE'),
-    trust_level=int(os.environ.get('TRUST_LEVEL')),
-)
-
-
-def pr():
-    if SKIP_TASKS_TRIGGER in PR_TITLE:
-        print("Pull request title contains", SKIP_TASKS_TRIGGER)
-        print("Exit")
-        return {}
-
-    build_tasks = {}
-    signing_tasks = {}
-    other_tasks = {}
 
     variant = get_variant('debug', 'geckoNightly')
-    assemble_task_id = taskcluster.slugId()
+    tasks.append(builder.craft_assemble_pr_task(variant))
-    build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant)
+    tasks.append(builder.craft_test_pr_task(variant))
-    build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant)
 
     for craft_function in (
-        BUILDER.craft_detekt_task,
+        builder.craft_detekt_task,
-        BUILDER.craft_ktlint_task,
+        builder.craft_ktlint_task,
-        BUILDER.craft_lint_task,
+        builder.craft_lint_task,
-        BUILDER.craft_compare_locales_task,
+        builder.craft_compare_locales_task,
     ):
-        other_tasks[taskcluster.slugId()] = craft_function()
+        tasks.append(craft_function())
 
-    return (build_tasks, signing_tasks, other_tasks)
+    return tasks
 
 
-def push():
+def push(builder):
     all_tasks = pr()
     other_tasks = all_tasks[-1]
-    other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task()
+    other_tasks[_generate_slug_id()] = builder.craft_ui_tests_task()
 
     return all_tasks
 
 
-def raptor(is_staging):
+def raptor(builder, is_staging):
     build_tasks = {}
     signing_tasks = {}
     other_tasks = {}
@@ -93,27 +61,27 @@ def raptor(is_staging):
     gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
 
     variant = get_variant('forPerformanceTest', 'geckoNightly')
-    assemble_task_id = taskcluster.slugId()
+    assemble_task_id = _generate_slug_id()
-    build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant)
+    build_tasks[assemble_task_id] = builder.craft_assemble_raptor_task(variant)
-    signing_task_id = taskcluster.slugId()
+    signing_task_id = _generate_slug_id()
-    signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
+    signing_tasks[signing_task_id] = builder.craft_raptor_signing_task(assemble_task_id, variant, is_staging)
 
     for abi in ('armeabi-v7a', 'arm64-v8a'):
         variant_apk = variant.get_apk(abi)
         all_raptor_craft_functions = [
-            BUILDER.craft_raptor_tp6m_cold_task(for_suite=i)
+            builder.craft_raptor_tp6m_cold_task(for_suite=i)
                 for i in range(1, 28)
             ] + [
-                BUILDER.craft_raptor_youtube_playback_task,
+                builder.craft_raptor_youtube_playback_task,
             ]
         for craft_function in all_raptor_craft_functions:
             args = (signing_task_id, mozharness_task_id, variant_apk, gecko_revision, is_staging)
-            other_tasks[taskcluster.slugId()] = craft_function(*args)
+            other_tasks[_generate_slug_id()] = craft_function(*args)
 
     return (build_tasks, signing_tasks, other_tasks)
 
 
-def release(channel, engine, is_staging, version_name):
+def release(builder, channel, engine, is_staging, version_name):
     variant = get_variant('fenix' + channel.capitalize(), engine)
     taskcluster_apk_paths = variant.upstream_artifacts()
 
@@ -121,19 +89,19 @@ def release(channel, engine, is_staging, version_name):
     signing_tasks = {}
     push_tasks = {}
 
-    build_task_id = taskcluster.slugId()
+    build_task_id = _generate_slug_id()
-    build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
+    build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
 
-    signing_task_id = taskcluster.slugId()
+    signing_task_id = _generate_slug_id()
-    signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
+    signing_tasks[signing_task_id] = builder.craft_release_signing_task(
         build_task_id,
         taskcluster_apk_paths,
         channel=channel,
         is_staging=is_staging,
     )
 
-    push_task_id = taskcluster.slugId()
+    push_task_id = _generate_slug_id()
-    push_tasks[push_task_id] = BUILDER.craft_push_task(
+    push_tasks[push_task_id] = builder.craft_push_task(
         signing_task_id,
         taskcluster_apk_paths,
         channel=channel,
@@ -145,18 +113,18 @@ def release(channel, engine, is_staging, version_name):
     return (build_tasks, signing_tasks, push_tasks)
 
 
-def release_as_fennec(is_staging, version_name):
+def release_as_fennec(builder, is_staging, version_name):
     variant = get_variant('fennecProduction', 'geckoBeta')
     channel = 'fennec-production'
 
     build_tasks = {}
     signing_tasks = {}
 
-    build_task_id = taskcluster.slugId()
+    build_task_id = _generate_slug_id()
-    build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, channel, is_staging, version_name)
+    build_tasks[build_task_id] = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
 
-    signing_task_id = taskcluster.slugId()
+    signing_task_id = _generate_slug_id()
-    signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
+    signing_tasks[signing_task_id] = builder.craft_release_signing_task(
         build_task_id,
         variant.upstream_artifacts(),
         channel,
@@ -166,7 +134,7 @@ def release_as_fennec(is_staging, version_name):
     return (build_tasks, signing_tasks)
 
 
-def nightly_to_production_app(is_staging, version_name):
+def nightly_to_production_app(builder, is_staging, version_name):
     # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
     # "nightly" track. We're moving towards having each channel be published to its own app, but we need to
     # keep updating this "backwards-compatible" nightly for a while yet
@@ -178,12 +146,12 @@ def nightly_to_production_app(is_staging, version_name):
     push_tasks = {}
     other_tasks = {}
 
-    build_task_id = taskcluster.slugId()
+    build_task_id = _generate_slug_id()
-    build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(
+    build_tasks[build_task_id] = builder.craft_assemble_release_task(
         variant, 'nightly-legacy', is_staging, version_name)
 
-    signing_task_id = taskcluster.slugId()
+    signing_task_id = _generate_slug_id()
-    signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
+    signing_tasks[signing_task_id] = builder.craft_release_signing_task(
         build_task_id,
         taskcluster_apk_paths,
         channel='production',  # Since we're publishing to the "production" app, we need to sign for production
@@ -191,8 +159,8 @@ def nightly_to_production_app(is_staging, version_name):
         publish_to_index=False,
     )
 
-    push_task_id = taskcluster.slugId()
+    push_task_id = _generate_slug_id()
-    push_tasks[push_task_id] = BUILDER.craft_push_task(
+    push_tasks[push_task_id] = builder.craft_push_task(
         signing_task_id,
         taskcluster_apk_paths,
         channel='production',  # We're publishing to the "production" app on the "nightly" track
@@ -201,61 +169,13 @@ def nightly_to_production_app(is_staging, version_name):
     )
 
     if not is_staging:
-        nimbledroid_task_id = taskcluster.slugId()
+        nimbledroid_task_id = _generate_slug_id()
-        other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task(
+        other_tasks[nimbledroid_task_id] = builder.craft_upload_apk_nimbledroid_task(
             build_task_id
         )
 
     return (build_tasks, signing_tasks, push_tasks, other_tasks)
 
 
-if __name__ == "__main__":
+def _generate_slug_id():
-    parser = argparse.ArgumentParser(
+    return taskcluster.slugId()
-        description='Creates and submit a graph of tasks on Taskcluster.'
-    )
-
-    subparsers = parser.add_subparsers(dest='command')
-
-    subparsers.add_parser('pull-request')
-    subparsers.add_parser('push')
-
-    raptor_parser = subparsers.add_parser('raptor')
-    raptor_parser.add_argument('--staging', action='store_true')
-
-    nightly_parser = subparsers.add_parser('nightly')
-    nightly_parser.add_argument('--staging', action='store_true')
-
-    release_parser = subparsers.add_parser('github-release')
-    release_parser.add_argument('tag')
-    release_parser.add_argument('--staging', action='store_true')
-
-    result = parser.parse_args()
-    command = result.command
-
-    if command == 'pull-request':
-        ordered_groups_of_tasks = pr()
-    elif command == 'push':
-        ordered_groups_of_tasks = push()
-    elif command == 'raptor':
-        ordered_groups_of_tasks = raptor(result.staging)
-    elif command == 'nightly':
-        now = datetime.datetime.now().strftime('%y%m%d %H:%M')
-        nightly_version = 'Nightly {}'.format(now)
-        ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \
-            + nightly_to_production_app(result.staging, nightly_version)
-        ordered_groups_of_tasks += release_as_fennec(result.staging, 'Signed-as-Fennec Nightly {}'.format(now))
-    elif command == 'github-release':
-        version = result.tag[1:]  # remove prefixed "v"
-        beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
-        production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
-        if beta_semver.match(result.tag):
-            ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version)
-        elif production_semver.match(result.tag):
-            ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version)
-        else:
-            raise ValueError('Github tag must be in semver format and prefixed with a "v", '
-                             'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
-    else:
-        raise Exception('Unsupported command "{}"'.format(command))
-
-    populate_chain_of_trust_task_graph(full_task_graph)

automation/taskcluster/lib/gradle.py

@@ -6,7 +6,7 @@ from __future__ import print_function
 import json
 import subprocess
 
-from lib.variant import Variant, VariantApk
+from ..lib.variant import Variant, VariantApk
 
 
 def get_variant(build_type, engine):

automation/taskcluster/lib/tasks.py

@@ -9,7 +9,7 @@ import datetime
 import json
 import taskcluster
 
-from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
+from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
 
 DEFAULT_EXPIRES_IN = '1 year'
 DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
@@ -43,7 +43,7 @@ class TaskBuilder(object):
         self.scheduler_id = scheduler_id
         self.trust_level = trust_level
         self.tasks_priority = tasks_priority
-        self.date = arrow.get(date_string)
+        self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
         self.trust_level = trust_level
 
     def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
@@ -317,10 +317,8 @@ class TaskBuilder(object):
         self, name, description, command, dependencies=None, artifacts=None,
         routes=None, treeherder=None, env_vars=None, scopes=None
     ):
-        dependencies = [] if dependencies is None else dependencies
         artifacts = {} if artifacts is None else artifacts
         scopes = [] if scopes is None else scopes
-        routes = [] if routes is None else routes
         env_vars = {} if env_vars is None else env_vars
 
         checkout_command = ' && '.join([
@@ -402,7 +400,7 @@ class TaskBuilder(object):
         treeherder=None,
         notify=None,
     ):
-        dependencies = [] if dependencies is None else dependencies
+        dependencies = {} if dependencies is None else dependencies
         scopes = [] if scopes is None else scopes
         routes = [] if routes is None else routes
         treeherder = {} if treeherder is None else treeherder
@@ -421,27 +419,31 @@ class TaskBuilder(object):
             extra['notify'] = notify
 
         return {
-            "provisionerId": provisioner_id,
+            "attributes": {},
-            "workerType": worker_type,
+            "dependencies": dependencies,
-            "taskGroupId": self.task_id,
+            "label": name,
-            "schedulerId": self.scheduler_id,
+            "task": {
-            "created": taskcluster.stringDate(created),
+                "provisionerId": provisioner_id,
-            "deadline": taskcluster.stringDate(deadline),
+                "workerType": worker_type,
-            "expires": taskcluster.stringDate(expires),
+                "taskGroupId": self.task_id,
-            "retries": 5,
+                "schedulerId": self.scheduler_id,
-            "tags": {},
+                "created": taskcluster.stringDate(created),
-            "priority": self.tasks_priority,
+                "deadline": taskcluster.stringDate(deadline),
-            "dependencies": [self.task_id] + dependencies,
+                "expires": taskcluster.stringDate(expires),
-            "requires": "all-completed",
+                "retries": 5,
-            "routes": routes,
+                "tags": {},
-            "scopes": scopes,
+                "priority": self.tasks_priority,
-            "payload": payload,
+                "requires": "all-completed",
-            "extra": extra,
+                "routes": routes,
-            "metadata": {
+                "scopes": scopes,
-                "name": "Fenix - {}".format(name),
+                "payload": payload,
-                "description": description,
+                "extra": extra,
-                "owner": self.owner,
+                "metadata": {
-                "source": self.source,
+                    "name": "Fenix - {}".format(name),
+                    "description": description,
+                    "owner": self.owner,
+                    "source": self.source,
+                },
             },
         }
 

taskcluster/ci/config.yml

@@ -6,7 +6,7 @@ treeherder:
         'Rap': 'Raptor tests'
         'Rap-P': 'Raptor power tests'
 
-task-priority: lowest
+task-priority: highest
 
 taskgraph:
     register: fenix_taskgraph:register

taskcluster/ci/old-decision/kind.yml

@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: fenix_taskgraph.loader.old_decision:loader
+
+transforms: []
+
+# XXX Everything is done in the loader until everything is migrated

taskcluster/fenix_taskgraph/__init__.py

@@ -12,7 +12,7 @@ def register(graph_config):
     Import all modules that are siblings of this one, triggering decorators in
     the process.
     """
-    _import_modules(["job", "worker_types", "routes", "target"])
+    _import_modules(["job", "worker_types", "routes", "target", "target_tasks"])
 
 
 def _import_modules(modules):

taskcluster/fenix_taskgraph/loader/old_decision.py

@@ -0,0 +1,44 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import sys
+
+current_dir = os.path.dirname(os.path.realpath(__file__))
+project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
+sys.path.append(project_dir)
+
+from automation.taskcluster.decision_task import pr
+from automation.taskcluster.lib.tasks import TaskBuilder
+
+
+def loader(kind, path, config, params, loaded_tasks):
+    repo_url = params['head_repository']
+    commit = params['head_rev']
+    trust_level = int(params['level'])
+
+    builder = TaskBuilder(
+        task_id=os.environ.get('TASK_ID'),
+        repo_url=repo_url,
+        git_ref=params['head_ref'],
+        short_head_branch=params['head_ref'],
+        commit=commit,
+        owner=params['owner'],
+        source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
+        scheduler_id='mobile-level-{}'.format(trust_level),
+        tasks_priority='highest',    # TODO parametrize
+        date_string=params['moz_build_date'],
+        trust_level=trust_level,
+    )
+
+    tasks_for = params['tasks_for']
+    if tasks_for == 'github-pull-request':
+        ordered_groups_of_tasks = pr(builder)
+    else:
+        raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
+
+    for task in ordered_groups_of_tasks:
+        yield task

taskcluster/fenix_taskgraph/target_tasks.py

@@ -0,0 +1,20 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.target_tasks import _target_task, standard_filter
+
+# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
+@_target_task('default')
+def target_tasks_default(full_task_graph, parameters, graph_config):
+    """Target the tasks which have indicated they should be run on this project
+    via the `run_on_projects` attributes."""
+    def filter(t, params):
+        if t.kind == 'old-decision':
+            return True
+
+        return standard_filter(t, params)
+
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]