Use multi_dep instead of single_dep (#6923)
parent
561ec5088b
commit
11c7d3eb02
|
@ -2,16 +2,20 @@
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
---
|
---
|
||||||
loader: fenix_taskgraph.loader.single_dep:loader
|
loader: fenix_taskgraph.loader.multi_dep:loader
|
||||||
|
|
||||||
transforms:
|
transforms:
|
||||||
- fenix_taskgraph.transforms.single_dep:transforms
|
- fenix_taskgraph.transforms.multi_dep:transforms
|
||||||
- fenix_taskgraph.transforms.push_apk:transforms
|
- fenix_taskgraph.transforms.push_apk:transforms
|
||||||
- taskgraph.transforms.task:transforms
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
kind-dependencies:
|
kind-dependencies:
|
||||||
- signing
|
- signing
|
||||||
|
|
||||||
|
primary-dependency: signing
|
||||||
|
|
||||||
|
group-by: build-type
|
||||||
|
|
||||||
only-for-build-types:
|
only-for-build-types:
|
||||||
- nightly
|
- nightly
|
||||||
- beta
|
- beta
|
||||||
|
|
|
@ -2,16 +2,19 @@
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
---
|
---
|
||||||
loader: fenix_taskgraph.loader.single_dep:loader
|
loader: fenix_taskgraph.loader.multi_dep:loader
|
||||||
|
|
||||||
transforms:
|
transforms:
|
||||||
- fenix_taskgraph.transforms.single_dep:transforms
|
- fenix_taskgraph.transforms.multi_dep:transforms
|
||||||
- fenix_taskgraph.transforms.signing:transforms
|
- fenix_taskgraph.transforms.signing:transforms
|
||||||
- taskgraph.transforms.task:transforms
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
kind-dependencies:
|
kind-dependencies:
|
||||||
- build
|
- build
|
||||||
|
|
||||||
|
primary-dependency: build
|
||||||
|
|
||||||
|
group-by: build-type
|
||||||
|
|
||||||
job-template:
|
job-template:
|
||||||
description: Sign Fenix
|
description: Sign Fenix
|
||||||
|
|
|
@ -0,0 +1,51 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
import copy
|
||||||
|
|
||||||
|
|
||||||
|
# Define a collection of group_by functions
|
||||||
|
GROUP_BY_MAP = {}
|
||||||
|
|
||||||
|
|
||||||
|
def group_by(name):
|
||||||
|
def wrapper(func):
|
||||||
|
GROUP_BY_MAP[name] = func
|
||||||
|
return func
|
||||||
|
return wrapper
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def group_tasks(config, tasks):
|
||||||
|
group_by_fn = GROUP_BY_MAP[config['group-by']]
|
||||||
|
|
||||||
|
groups = group_by_fn(config, tasks)
|
||||||
|
|
||||||
|
for combinations in groups.itervalues():
|
||||||
|
dependencies = [copy.deepcopy(t) for t in combinations]
|
||||||
|
yield dependencies
|
||||||
|
|
||||||
|
|
||||||
|
@group_by('build-type')
|
||||||
|
def build_type_grouping(config, tasks):
|
||||||
|
groups = {}
|
||||||
|
kind_dependencies = config.get('kind-dependencies')
|
||||||
|
only_build_type = config.get('only-for-build-types')
|
||||||
|
|
||||||
|
for task in tasks:
|
||||||
|
if task.kind not in kind_dependencies:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if only_build_type:
|
||||||
|
build_type = task.attributes.get('build-type')
|
||||||
|
if build_type not in only_build_type:
|
||||||
|
continue
|
||||||
|
|
||||||
|
build_type = task.attributes.get('build-type')
|
||||||
|
|
||||||
|
groups.setdefault(build_type, []).append(task)
|
||||||
|
|
||||||
|
return groups
|
|
@ -0,0 +1,87 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
import copy
|
||||||
|
|
||||||
|
from voluptuous import Required
|
||||||
|
|
||||||
|
from taskgraph.task import Task
|
||||||
|
from taskgraph.util.attributes import sorted_unique_list
|
||||||
|
from taskgraph.util.schema import Schema
|
||||||
|
|
||||||
|
from . import group_tasks
|
||||||
|
|
||||||
|
|
||||||
|
schema = Schema({
|
||||||
|
Required('primary-dependency', 'primary dependency task'): Task,
|
||||||
|
Required(
|
||||||
|
'dependent-tasks',
|
||||||
|
'dictionary of dependent tasks, keyed by kind',
|
||||||
|
): {basestring: Task},
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def loader(kind, path, config, params, loaded_tasks):
|
||||||
|
"""
|
||||||
|
Load tasks based on the jobs dependant kinds, designed for use as
|
||||||
|
multiple-dependent needs.
|
||||||
|
Required ``group-by-fn`` is used to define how we coalesce the
|
||||||
|
multiple deps together to pass to transforms, e.g. all kinds specified get
|
||||||
|
collapsed by platform with `platform`
|
||||||
|
Optional ``primary-dependency`` (ordered list or string) is used to determine
|
||||||
|
which upstream kind to inherit attrs from. See ``get_primary_dep``.
|
||||||
|
The `only-for-build-type` kind configuration, if specified, will limit
|
||||||
|
the build types for which a job will be created.
|
||||||
|
Optional ``job-template`` kind configuration value, if specified, will be used to
|
||||||
|
pass configuration down to the specified transforms used.
|
||||||
|
"""
|
||||||
|
job_template = config.get('job-template')
|
||||||
|
|
||||||
|
for dep_tasks in group_tasks(config, loaded_tasks):
|
||||||
|
kinds = [dep.kind for dep in dep_tasks]
|
||||||
|
kinds_occurrences = {kind: kinds.count(kind) for kind in kinds}
|
||||||
|
|
||||||
|
dep_tasks_per_unique_key = {
|
||||||
|
dep.kind if kinds_occurrences[dep.kind] == 1 else dep.label: dep
|
||||||
|
for dep in dep_tasks
|
||||||
|
}
|
||||||
|
|
||||||
|
job = {'dependent-tasks': dep_tasks_per_unique_key}
|
||||||
|
job['primary-dependency'] = get_primary_dep(config, dep_tasks_per_unique_key)
|
||||||
|
if job_template:
|
||||||
|
job.update(copy.deepcopy(job_template))
|
||||||
|
|
||||||
|
yield job
|
||||||
|
|
||||||
|
|
||||||
|
def get_primary_dep(config, dep_tasks):
|
||||||
|
"""Find the dependent task to inherit attributes from.
|
||||||
|
If ``primary-dependency`` is defined in ``kind.yml`` and is a string,
|
||||||
|
then find the first dep with that task kind and return it. If it is
|
||||||
|
defined and is a list, the first kind in that list with a matching dep
|
||||||
|
is the primary dependency. If it's undefined, return the first dep.
|
||||||
|
"""
|
||||||
|
primary_dependencies = config.get('primary-dependency')
|
||||||
|
if isinstance(primary_dependencies, basestring):
|
||||||
|
primary_dependencies = [primary_dependencies]
|
||||||
|
if not primary_dependencies:
|
||||||
|
assert len(dep_tasks) == 1, "Must define a primary-dependency!"
|
||||||
|
return dep_tasks.values()[0]
|
||||||
|
primary_dep = None
|
||||||
|
for primary_kind in primary_dependencies:
|
||||||
|
for dep_kind in dep_tasks:
|
||||||
|
if dep_kind == primary_kind:
|
||||||
|
assert primary_dep is None, \
|
||||||
|
"Too many primary dependent tasks in dep_tasks: {}!".format(
|
||||||
|
[t.label for t in dep_tasks]
|
||||||
|
)
|
||||||
|
primary_dep = dep_tasks[dep_kind]
|
||||||
|
if primary_dep is None:
|
||||||
|
raise Exception("Can't find dependency of {}: {}".format(
|
||||||
|
config['primary-dependency'], config
|
||||||
|
))
|
||||||
|
return primary_dep
|
|
@ -1,52 +0,0 @@
|
||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
||||||
|
|
||||||
from __future__ import absolute_import, print_function, unicode_literals
|
|
||||||
|
|
||||||
import copy
|
|
||||||
|
|
||||||
from voluptuous import Required
|
|
||||||
|
|
||||||
from taskgraph.task import Task
|
|
||||||
from taskgraph.util.schema import Schema
|
|
||||||
|
|
||||||
schema = Schema({Required("primary-dependency", "primary dependency task"): Task})
|
|
||||||
|
|
||||||
|
|
||||||
def loader(kind, path, config, params, loaded_tasks):
|
|
||||||
"""
|
|
||||||
Load tasks based on the jobs dependant kinds.
|
|
||||||
|
|
||||||
Optional `only-for-attributes` kind configuration, if specified, will limit
|
|
||||||
the jobs chosen to ones which have the specified attribute, with the specified
|
|
||||||
value.
|
|
||||||
|
|
||||||
Optional `job-template` kind configuration value, if specified, will be used to
|
|
||||||
pass configuration down to the specified transforms used.
|
|
||||||
"""
|
|
||||||
only_attributes = config.get("only-for-attributes")
|
|
||||||
only_build_type = config.get('only-for-build-types')
|
|
||||||
job_template = config.get("job-template")
|
|
||||||
|
|
||||||
for task in loaded_tasks:
|
|
||||||
if task.kind not in config.get("kind-dependencies", []):
|
|
||||||
continue
|
|
||||||
|
|
||||||
if only_attributes:
|
|
||||||
config_attrs = set(only_attributes)
|
|
||||||
if not config_attrs & set(task.attributes):
|
|
||||||
# make sure any attribute exists
|
|
||||||
continue
|
|
||||||
|
|
||||||
if only_build_type:
|
|
||||||
build_type = task.attributes.get('build-type')
|
|
||||||
if build_type not in only_build_type:
|
|
||||||
continue
|
|
||||||
|
|
||||||
job = {"primary-dependency": task}
|
|
||||||
|
|
||||||
if job_template:
|
|
||||||
job.update(copy.deepcopy(job_template))
|
|
||||||
|
|
||||||
yield job
|
|
|
@ -18,13 +18,16 @@ transforms = TransformSequence()
|
||||||
@transforms.add
|
@transforms.add
|
||||||
def build_name_and_attributes(config, tasks):
|
def build_name_and_attributes(config, tasks):
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
dep = task["primary-dependency"]
|
task["dependencies"] = {
|
||||||
task["dependencies"] = {dep.kind: dep.label}
|
dep_key: dep.label
|
||||||
copy_of_attributes = dep.attributes.copy()
|
for dep_key, dep in _get_all_deps(task).iteritems()
|
||||||
|
}
|
||||||
|
primary_dep = task["primary-dependency"]
|
||||||
|
copy_of_attributes = primary_dep.attributes.copy()
|
||||||
task.setdefault("attributes", copy_of_attributes)
|
task.setdefault("attributes", copy_of_attributes)
|
||||||
# run_on_tasks_for is set as an attribute later in the pipeline
|
# run_on_tasks_for is set as an attribute later in the pipeline
|
||||||
task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
|
task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
|
||||||
task["name"] = _get_dependent_job_name_without_its_kind(dep)
|
task["name"] = _get_dependent_job_name_without_its_kind(primary_dep)
|
||||||
|
|
||||||
yield task
|
yield task
|
||||||
|
|
||||||
|
@ -33,6 +36,13 @@ def _get_dependent_job_name_without_its_kind(dependent_job):
|
||||||
return dependent_job.label[len(dependent_job.kind) + 1:]
|
return dependent_job.label[len(dependent_job.kind) + 1:]
|
||||||
|
|
||||||
|
|
||||||
|
def _get_all_deps(task):
|
||||||
|
if task.get("dependent-tasks"):
|
||||||
|
return task["dependent-tasks"]
|
||||||
|
|
||||||
|
return {task["primary-dependency"].kind: task["primary-dependency"]}
|
||||||
|
|
||||||
|
|
||||||
@transforms.add
|
@transforms.add
|
||||||
def resolve_keys(config, tasks):
|
def resolve_keys(config, tasks):
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
|
@ -51,14 +61,18 @@ def resolve_keys(config, tasks):
|
||||||
@transforms.add
|
@transforms.add
|
||||||
def build_upstream_artifacts(config, tasks):
|
def build_upstream_artifacts(config, tasks):
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
dep = task["primary-dependency"]
|
worker_definition = {
|
||||||
|
"upstream-artifacts": [],
|
||||||
|
}
|
||||||
|
|
||||||
worker_definition = {}
|
for dep in _get_all_deps(task).values():
|
||||||
worker_definition["upstream-artifacts"] = [{
|
paths = sorted(dep.attributes.get("apks", {}).values())
|
||||||
"taskId": {"task-reference": "<{}>".format(dep.kind)},
|
if paths:
|
||||||
"taskType": dep.kind,
|
worker_definition["upstream-artifacts"].append({
|
||||||
"paths": sorted(dep.attributes["apks"].values()),
|
"taskId": {"task-reference": "<{}>".format(dep.kind)},
|
||||||
}]
|
"taskType": dep.kind,
|
||||||
|
"paths": paths,
|
||||||
|
})
|
||||||
|
|
||||||
task["worker"].update(worker_definition)
|
task["worker"].update(worker_definition)
|
||||||
yield task
|
yield task
|
||||||
|
@ -76,3 +90,14 @@ def build_treeherder_definition(config, tasks):
|
||||||
task["treeherder"]["symbol"] = full_symbol
|
task["treeherder"]["symbol"] = full_symbol
|
||||||
|
|
||||||
yield task
|
yield task
|
||||||
|
|
||||||
|
|
||||||
|
@transforms.add
|
||||||
|
def remove_dependent_tasks(config, tasks):
|
||||||
|
for task in tasks:
|
||||||
|
try:
|
||||||
|
del task["dependent-tasks"]
|
||||||
|
except KeyError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
yield task
|
Loading…
Reference in New Issue