1
0
Fork 0

Use multi_dep instead of single_dep (#6923)

master
Johan Lorenzo 2019-12-05 05:14:58 +01:00 committed by liuche
parent 561ec5088b
commit 11c7d3eb02
6 changed files with 185 additions and 67 deletions

View File

@ -2,16 +2,20 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: fenix_taskgraph.loader.single_dep:loader
loader: fenix_taskgraph.loader.multi_dep:loader
transforms:
- fenix_taskgraph.transforms.single_dep:transforms
- fenix_taskgraph.transforms.multi_dep:transforms
- fenix_taskgraph.transforms.push_apk:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- signing
primary-dependency: signing
group-by: build-type
only-for-build-types:
- nightly
- beta

View File

@ -2,16 +2,19 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: fenix_taskgraph.loader.single_dep:loader
loader: fenix_taskgraph.loader.multi_dep:loader
transforms:
- fenix_taskgraph.transforms.single_dep:transforms
- fenix_taskgraph.transforms.multi_dep:transforms
- fenix_taskgraph.transforms.signing:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- build
primary-dependency: build
group-by: build-type
job-template:
description: Sign Fenix

View File

@ -0,0 +1,51 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
# Define a collection of group_by functions
GROUP_BY_MAP = {}
def group_by(name):
def wrapper(func):
GROUP_BY_MAP[name] = func
return func
return wrapper
def group_tasks(config, tasks):
group_by_fn = GROUP_BY_MAP[config['group-by']]
groups = group_by_fn(config, tasks)
for combinations in groups.itervalues():
dependencies = [copy.deepcopy(t) for t in combinations]
yield dependencies
@group_by('build-type')
def build_type_grouping(config, tasks):
groups = {}
kind_dependencies = config.get('kind-dependencies')
only_build_type = config.get('only-for-build-types')
for task in tasks:
if task.kind not in kind_dependencies:
continue
if only_build_type:
build_type = task.attributes.get('build-type')
if build_type not in only_build_type:
continue
build_type = task.attributes.get('build-type')
groups.setdefault(build_type, []).append(task)
return groups

View File

@ -0,0 +1,87 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
from voluptuous import Required
from taskgraph.task import Task
from taskgraph.util.attributes import sorted_unique_list
from taskgraph.util.schema import Schema
from . import group_tasks
schema = Schema({
Required('primary-dependency', 'primary dependency task'): Task,
Required(
'dependent-tasks',
'dictionary of dependent tasks, keyed by kind',
): {basestring: Task},
})
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds, designed for use as
multiple-dependent needs.
Required ``group-by-fn`` is used to define how we coalesce the
multiple deps together to pass to transforms, e.g. all kinds specified get
collapsed by platform with `platform`
Optional ``primary-dependency`` (ordered list or string) is used to determine
which upstream kind to inherit attrs from. See ``get_primary_dep``.
The `only-for-build-type` kind configuration, if specified, will limit
the build types for which a job will be created.
Optional ``job-template`` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
job_template = config.get('job-template')
for dep_tasks in group_tasks(config, loaded_tasks):
kinds = [dep.kind for dep in dep_tasks]
kinds_occurrences = {kind: kinds.count(kind) for kind in kinds}
dep_tasks_per_unique_key = {
dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep
for dep in dep_tasks
}
job = {'dependent-tasks': dep_tasks_per_unique_key}
job['primary-dependency'] = get_primary_dep(config, dep_tasks_per_unique_key)
if job_template:
job.update(copy.deepcopy(job_template))
yield job
def get_primary_dep(config, dep_tasks):
"""Find the dependent task to inherit attributes from.
If ``primary-dependency`` is defined in ``kind.yml`` and is a string,
then find the first dep with that task kind and return it. If it is
defined and is a list, the first kind in that list with a matching dep
is the primary dependency. If it's undefined, return the first dep.
"""
primary_dependencies = config.get('primary-dependency')
if isinstance(primary_dependencies, basestring):
primary_dependencies = [primary_dependencies]
if not primary_dependencies:
assert len(dep_tasks) == 1, "Must define a primary-dependency!"
return dep_tasks.values()[0]
primary_dep = None
for primary_kind in primary_dependencies:
for dep_kind in dep_tasks:
if dep_kind == primary_kind:
assert primary_dep is None, \
"Too many primary dependent tasks in dep_tasks: {}!".format(
[t.label for t in dep_tasks]
)
primary_dep = dep_tasks[dep_kind]
if primary_dep is None:
raise Exception("Can't find dependency of {}: {}".format(
config['primary-dependency'], config
))
return primary_dep

View File

@ -1,52 +0,0 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import copy
from voluptuous import Required
from taskgraph.task import Task
from taskgraph.util.schema import Schema
schema = Schema({Required("primary-dependency", "primary dependency task"): Task})
def loader(kind, path, config, params, loaded_tasks):
"""
Load tasks based on the jobs dependant kinds.
Optional `only-for-attributes` kind configuration, if specified, will limit
the jobs chosen to ones which have the specified attribute, with the specified
value.
Optional `job-template` kind configuration value, if specified, will be used to
pass configuration down to the specified transforms used.
"""
only_attributes = config.get("only-for-attributes")
only_build_type = config.get('only-for-build-types')
job_template = config.get("job-template")
for task in loaded_tasks:
if task.kind not in config.get("kind-dependencies", []):
continue
if only_attributes:
config_attrs = set(only_attributes)
if not config_attrs & set(task.attributes):
# make sure any attribute exists
continue
if only_build_type:
build_type = task.attributes.get('build-type')
if build_type not in only_build_type:
continue
job = {"primary-dependency": task}
if job_template:
job.update(copy.deepcopy(job_template))
yield job

View File

@ -18,13 +18,16 @@ transforms = TransformSequence()
@transforms.add
def build_name_and_attributes(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
task["dependencies"] = {dep.kind: dep.label}
copy_of_attributes = dep.attributes.copy()
task["dependencies"] = {
dep_key: dep.label
for dep_key, dep in _get_all_deps(task).iteritems()
}
primary_dep = task["primary-dependency"]
copy_of_attributes = primary_dep.attributes.copy()
task.setdefault("attributes", copy_of_attributes)
# run_on_tasks_for is set as an attribute later in the pipeline
task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
task["name"] = _get_dependent_job_name_without_its_kind(dep)
task["name"] = _get_dependent_job_name_without_its_kind(primary_dep)
yield task
@ -33,6 +36,13 @@ def _get_dependent_job_name_without_its_kind(dependent_job):
return dependent_job.label[len(dependent_job.kind) + 1:]
def _get_all_deps(task):
if task.get("dependent-tasks"):
return task["dependent-tasks"]
return {task["primary-dependency"].kind: task["primary-dependency"]}
@transforms.add
def resolve_keys(config, tasks):
for task in tasks:
@ -51,14 +61,18 @@ def resolve_keys(config, tasks):
@transforms.add
def build_upstream_artifacts(config, tasks):
for task in tasks:
dep = task["primary-dependency"]
worker_definition = {
"upstream-artifacts": [],
}
worker_definition = {}
worker_definition["upstream-artifacts"] = [{
for dep in _get_all_deps(task).values():
paths = sorted(dep.attributes.get("apks", {}).values())
if paths:
worker_definition["upstream-artifacts"].append({
"taskId": {"task-reference": "<{}>".format(dep.kind)},
"taskType": dep.kind,
"paths": sorted(dep.attributes["apks"].values()),
}]
"paths": paths,
})
task["worker"].update(worker_definition)
yield task
@ -76,3 +90,14 @@ def build_treeherder_definition(config, tasks):
task["treeherder"]["symbol"] = full_symbol
yield task
@transforms.add
def remove_dependent_tasks(config, tasks):
for task in tasks:
try:
del task["dependent-tasks"]
except KeyError:
pass
yield task