diff --git a/taskcluster/ci/push-apk/kind.yml b/taskcluster/ci/push-apk/kind.yml index 1476721b1..d46d73cdd 100644 --- a/taskcluster/ci/push-apk/kind.yml +++ b/taskcluster/ci/push-apk/kind.yml @@ -2,16 +2,20 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: fenix_taskgraph.loader.single_dep:loader +loader: fenix_taskgraph.loader.multi_dep:loader transforms: - - fenix_taskgraph.transforms.single_dep:transforms + - fenix_taskgraph.transforms.multi_dep:transforms - fenix_taskgraph.transforms.push_apk:transforms - taskgraph.transforms.task:transforms kind-dependencies: - signing +primary-dependency: signing + +group-by: build-type + only-for-build-types: - nightly - beta diff --git a/taskcluster/ci/signing/kind.yml b/taskcluster/ci/signing/kind.yml index d3d0b01aa..9a11e69b2 100644 --- a/taskcluster/ci/signing/kind.yml +++ b/taskcluster/ci/signing/kind.yml @@ -2,16 +2,19 @@ # License, v. 2.0. If a copy of the MPL was not distributed with this # file, You can obtain one at http://mozilla.org/MPL/2.0/. --- -loader: fenix_taskgraph.loader.single_dep:loader +loader: fenix_taskgraph.loader.multi_dep:loader transforms: - - fenix_taskgraph.transforms.single_dep:transforms + - fenix_taskgraph.transforms.multi_dep:transforms - fenix_taskgraph.transforms.signing:transforms - taskgraph.transforms.task:transforms kind-dependencies: - build +primary-dependency: build + +group-by: build-type job-template: description: Sign Fenix diff --git a/taskcluster/fenix_taskgraph/loader/__init__.py b/taskcluster/fenix_taskgraph/loader/__init__.py index e69de29bb..c73dd12b4 100644 --- a/taskcluster/fenix_taskgraph/loader/__init__.py +++ b/taskcluster/fenix_taskgraph/loader/__init__.py @@ -0,0 +1,51 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy + + +# Define a collection of group_by functions +GROUP_BY_MAP = {} + + +def group_by(name): + def wrapper(func): + GROUP_BY_MAP[name] = func + return func + return wrapper + + + +def group_tasks(config, tasks): + group_by_fn = GROUP_BY_MAP[config['group-by']] + + groups = group_by_fn(config, tasks) + + for combinations in groups.itervalues(): + dependencies = [copy.deepcopy(t) for t in combinations] + yield dependencies + + +@group_by('build-type') +def build_type_grouping(config, tasks): + groups = {} + kind_dependencies = config.get('kind-dependencies') + only_build_type = config.get('only-for-build-types') + + for task in tasks: + if task.kind not in kind_dependencies: + continue + + if only_build_type: + build_type = task.attributes.get('build-type') + if build_type not in only_build_type: + continue + + build_type = task.attributes.get('build-type') + + groups.setdefault(build_type, []).append(task) + + return groups diff --git a/taskcluster/fenix_taskgraph/loader/multi_dep.py b/taskcluster/fenix_taskgraph/loader/multi_dep.py new file mode 100644 index 000000000..0c9f4dbac --- /dev/null +++ b/taskcluster/fenix_taskgraph/loader/multi_dep.py @@ -0,0 +1,87 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import copy + +from voluptuous import Required + +from taskgraph.task import Task +from taskgraph.util.attributes import sorted_unique_list +from taskgraph.util.schema import Schema + +from . import group_tasks + + +schema = Schema({ + Required('primary-dependency', 'primary dependency task'): Task, + Required( + 'dependent-tasks', + 'dictionary of dependent tasks, keyed by kind', + ): {basestring: Task}, +}) + + + +def loader(kind, path, config, params, loaded_tasks): + """ + Load tasks based on the jobs dependant kinds, designed for use as + multiple-dependent needs. + Required ``group-by-fn`` is used to define how we coalesce the + multiple deps together to pass to transforms, e.g. all kinds specified get + collapsed by platform with `platform` + Optional ``primary-dependency`` (ordered list or string) is used to determine + which upstream kind to inherit attrs from. See ``get_primary_dep``. + The `only-for-build-type` kind configuration, if specified, will limit + the build types for which a job will be created. + Optional ``job-template`` kind configuration value, if specified, will be used to + pass configuration down to the specified transforms used. + """ + job_template = config.get('job-template') + + for dep_tasks in group_tasks(config, loaded_tasks): + kinds = [dep.kind for dep in dep_tasks] + kinds_occurrences = {kind: kinds.count(kind) for kind in kinds} + + dep_tasks_per_unique_key = { + dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep + for dep in dep_tasks + } + + job = {'dependent-tasks': dep_tasks_per_unique_key} + job['primary-dependency'] = get_primary_dep(config, dep_tasks_per_unique_key) + if job_template: + job.update(copy.deepcopy(job_template)) + + yield job + + +def get_primary_dep(config, dep_tasks): + """Find the dependent task to inherit attributes from. + If ``primary-dependency`` is defined in ``kind.yml`` and is a string, + then find the first dep with that task kind and return it. If it is + defined and is a list, the first kind in that list with a matching dep + is the primary dependency. If it's undefined, return the first dep. + """ + primary_dependencies = config.get('primary-dependency') + if isinstance(primary_dependencies, basestring): + primary_dependencies = [primary_dependencies] + if not primary_dependencies: + assert len(dep_tasks) == 1, "Must define a primary-dependency!" + return dep_tasks.values()[0] + primary_dep = None + for primary_kind in primary_dependencies: + for dep_kind in dep_tasks: + if dep_kind == primary_kind: + assert primary_dep is None, \ + "Too many primary dependent tasks in dep_tasks: {}!".format( + [t.label for t in dep_tasks] + ) + primary_dep = dep_tasks[dep_kind] + if primary_dep is None: + raise Exception("Can't find dependency of {}: {}".format( + config['primary-dependency'], config + )) + return primary_dep diff --git a/taskcluster/fenix_taskgraph/loader/single_dep.py b/taskcluster/fenix_taskgraph/loader/single_dep.py deleted file mode 100644 index aead8403a..000000000 --- a/taskcluster/fenix_taskgraph/loader/single_dep.py +++ /dev/null @@ -1,52 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -from __future__ import absolute_import, print_function, unicode_literals - -import copy - -from voluptuous import Required - -from taskgraph.task import Task -from taskgraph.util.schema import Schema - -schema = Schema({Required("primary-dependency", "primary dependency task"): Task}) - - -def loader(kind, path, config, params, loaded_tasks): - """ - Load tasks based on the jobs dependant kinds. - - Optional `only-for-attributes` kind configuration, if specified, will limit - the jobs chosen to ones which have the specified attribute, with the specified - value. - - Optional `job-template` kind configuration value, if specified, will be used to - pass configuration down to the specified transforms used. - """ - only_attributes = config.get("only-for-attributes") - only_build_type = config.get('only-for-build-types') - job_template = config.get("job-template") - - for task in loaded_tasks: - if task.kind not in config.get("kind-dependencies", []): - continue - - if only_attributes: - config_attrs = set(only_attributes) - if not config_attrs & set(task.attributes): - # make sure any attribute exists - continue - - if only_build_type: - build_type = task.attributes.get('build-type') - if build_type not in only_build_type: - continue - - job = {"primary-dependency": task} - - if job_template: - job.update(copy.deepcopy(job_template)) - - yield job diff --git a/taskcluster/fenix_taskgraph/transforms/single_dep.py b/taskcluster/fenix_taskgraph/transforms/multi_dep.py similarity index 65% rename from taskcluster/fenix_taskgraph/transforms/single_dep.py rename to taskcluster/fenix_taskgraph/transforms/multi_dep.py index f201a48af..4924182a2 100644 --- a/taskcluster/fenix_taskgraph/transforms/single_dep.py +++ b/taskcluster/fenix_taskgraph/transforms/multi_dep.py @@ -18,13 +18,16 @@ transforms = TransformSequence() @transforms.add def build_name_and_attributes(config, tasks): for task in tasks: - dep = task["primary-dependency"] - task["dependencies"] = {dep.kind: dep.label} - copy_of_attributes = dep.attributes.copy() + task["dependencies"] = { + dep_key: dep.label + for dep_key, dep in _get_all_deps(task).iteritems() + } + primary_dep = task["primary-dependency"] + copy_of_attributes = primary_dep.attributes.copy() task.setdefault("attributes", copy_of_attributes) # run_on_tasks_for is set as an attribute later in the pipeline task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for']) - task["name"] = _get_dependent_job_name_without_its_kind(dep) + task["name"] = _get_dependent_job_name_without_its_kind(primary_dep) yield task @@ -33,6 +36,13 @@ def _get_dependent_job_name_without_its_kind(dependent_job): return dependent_job.label[len(dependent_job.kind) + 1:] +def _get_all_deps(task): + if task.get("dependent-tasks"): + return task["dependent-tasks"] + + return {task["primary-dependency"].kind: task["primary-dependency"]} + + @transforms.add def resolve_keys(config, tasks): for task in tasks: @@ -51,14 +61,18 @@ def resolve_keys(config, tasks): @transforms.add def build_upstream_artifacts(config, tasks): for task in tasks: - dep = task["primary-dependency"] + worker_definition = { + "upstream-artifacts": [], + } - worker_definition = {} - worker_definition["upstream-artifacts"] = [{ - "taskId": {"task-reference": "<{}>".format(dep.kind)}, - "taskType": dep.kind, - "paths": sorted(dep.attributes["apks"].values()), - }] + for dep in _get_all_deps(task).values(): + paths = sorted(dep.attributes.get("apks", {}).values()) + if paths: + worker_definition["upstream-artifacts"].append({ + "taskId": {"task-reference": "<{}>".format(dep.kind)}, + "taskType": dep.kind, + "paths": paths, + }) task["worker"].update(worker_definition) yield task @@ -76,3 +90,14 @@ def build_treeherder_definition(config, tasks): task["treeherder"]["symbol"] = full_symbol yield task + + +@transforms.add +def remove_dependent_tasks(config, tasks): + for task in tasks: + try: + del task["dependent-tasks"] + except KeyError: + pass + + yield task