blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Bug 1568466 - part 3: Enable shipit graph (#7314)

master
Johan Lorenzo 2020-04-09 14:33:13 +02:00 committed by GitHub
parent ed466d34f7
commit 58fc0e5338
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 224 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
taskcluster/ci/config.yml
View File

@ -26,7 +26,7 @@ taskgraph:
mobile:
name: "Fenix"
cached-task-prefix: project.mobile.fenix
decision-parameters: 'fenix_taskgraph:get_decision_parameters'
decision-parameters: 'fenix_taskgraph.parameters:get_decision_parameters'
workers:
aliases:
@ -71,3 +71,9 @@ workers:
scriptworker:
scope-prefix: project:mobile:fenix:releng
release-promotion:
flavors:
ship:
target-tasks-method: release

62
taskcluster/fenix_taskgraph/__init__.py
View File

@ -4,17 +4,7 @@
from __future__ import absolute_import, print_function, unicode_literals
import os
import re
from importlib import import_module
from six import text_type
from voluptuous import All, Any, Range, Required
from taskgraph.parameters import extend_parameters_schema
BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
def register(graph_config):
@ -22,52 +12,16 @@ def register(graph_config):
Import all modules that are siblings of this one, triggering decorators in
the process.
"""
_import_modules(["job", "worker_types", "routes", "target_tasks"])
extend_parameters_schema({
Required("pull_request_number"): Any(All(int, Range(min=1)), None),
Required("release_type"): text_type,
Required("release_version"): text_type,
})
_import_modules([
"job",
"parameters",
"release_promotion",
"routes",
"target_tasks",
"worker_types",
])
def _import_modules(modules):
for module in modules:
import_module(".{}".format(module), package=__name__)
def get_decision_parameters(graph_config, parameters):
head_tag = parameters["head_tag"].decode("utf-8")
parameters["release_type"] = _resolve_release_type(head_tag)
parameters["release_version"] = read_version_file()
if head_tag:
parameters["release_version"] = head_tag[1:]
pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None)
parameters["pull_request_number"] = None if pr_number is None else int(pr_number)
if parameters["tasks_for"] == "github-release":
for param_name in ("release_type", "release_version"):
if not parameters[param_name]:
raise ValueError(
'Cannot run github-release if "{}" is not defined. Got: {}'.format(
param_name, parameters[param_name]
)
)
parameters["target_tasks_method"] = "release"
def read_version_file():
with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f:
return f.read().strip().decode('utf-8')
def _resolve_release_type(head_tag):
if not head_tag:
return ""
elif BETA_SEMVER.match(head_tag):
return "beta"
elif PRODUCTION_SEMVER.match(head_tag):
return "production"
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')

55
taskcluster/fenix_taskgraph/parameters.py 100644
View File

@ -0,0 +1,55 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
import os
import re
from six import text_type
from taskgraph.parameters import extend_parameters_schema
from voluptuous import All, Any, Optional, Range, Required
BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
extend_parameters_schema({
Required("pull_request_number"): Any(All(int, Range(min=1)), None),
Required("release_type"): text_type,
Optional("shipping_phase"): Any('build', 'ship', None),
Required("version"): text_type,
})
def get_decision_parameters(graph_config, parameters):
head_tag = parameters["head_tag"].decode("utf-8")
parameters["release_type"] = resolve_release_type(head_tag)
parameters["version"] = head_tag[1:] if head_tag else ""
pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None)
parameters["pull_request_number"] = None if pr_number is None else int(pr_number)
if parameters["tasks_for"] == "github-release":
for param_name in ("release_type", "version"):
if not parameters[param_name]:
raise ValueError(
'Cannot run github-release if "{}" is not defined. Got: {}'.format(
param_name, parameters[param_name]
)
)
parameters["target_tasks_method"] = "release"
def resolve_release_type(head_tag):
if not head_tag:
return ""
elif BETA_SEMVER.match(head_tag):
return "beta"
elif PRODUCTION_SEMVER.match(head_tag):
return "production"
else:
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')

150
taskcluster/fenix_taskgraph/release_promotion.py 100644
View File

@ -0,0 +1,150 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import absolute_import, print_function, unicode_literals
from taskgraph.actions.registry import register_callback_action
from taskgraph.util.taskcluster import get_artifact
from taskgraph.taskgraph import TaskGraph
from taskgraph.decision import taskgraph_decision
from taskgraph.parameters import Parameters
from taskgraph.util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds
from .parameters import resolve_release_type
RELEASE_PROMOTION_PROJECTS = (
"https://github.com/JohanLorenzo/fenix",
)
def is_release_promotion_available(parameters):
return parameters['head_repository'] in RELEASE_PROMOTION_PROJECTS
@register_callback_action(
name='release-promotion',
title='Ship Fenix',
symbol='${input.release_promotion_flavor}',
description="Ship Fenix",
generic=False,
order=500,
context=[],
available=is_release_promotion_available,
schema=lambda graph_config: {
'type': 'object',
'properties': {
'build_number': {
'type': 'integer',
'default': 1,
'minimum': 1,
'title': 'The release build number',
'description': ('The release build number. Starts at 1 per '
'release version, and increments on rebuild.'),
},
'do_not_optimize': {
'type': 'array',
'description': ('Optional: a list of labels to avoid optimizing out '
'of the graph (to force a rerun of, say, '
'funsize docker-image tasks).'),
'items': {
'type': 'string',
},
},
'revision': {
'type': 'string',
'title': 'Optional: revision to ship',
'description': ('Optional: the revision to ship.'),
},
'release_promotion_flavor': {
'type': 'string',
'description': 'The flavor of release promotion to perform.',
'default': 'build',
'enum': sorted(graph_config['release-promotion']['flavors'].keys()),
},
'rebuild_kinds': {
'type': 'array',
'description': ('Optional: an array of kinds to ignore from the previous '
'graph(s).'),
'items': {
'type': 'string',
},
},
'previous_graph_ids': {
'type': 'array',
'description': ('Optional: an array of taskIds of decision or action '
'tasks from the previous graph(s) to use to populate '
'our `previous_graph_kinds`.'),
'items': {
'type': 'string',
},
},
'version': {
'type': 'string',
'description': ('Optional: override the version for release promotion. '
"Occasionally we'll land a taskgraph fix in a later "
'commit, but want to act on a build from a previous '
'commit. If a version bump has landed in the meantime, '
'relying on the in-tree version will break things.'),
'default': '',
},
},
"required": ['release_promotion_flavor', 'version', 'build_number'],
}
)
def release_promotion_action(parameters, graph_config, input, task_group_id, task_id):
release_promotion_flavor = input['release_promotion_flavor']
promotion_config = graph_config['release-promotion']['flavors'][release_promotion_flavor]
target_tasks_method = promotion_config['target-tasks-method'].format(
project=parameters['project']
)
rebuild_kinds = input.get('rebuild_kinds') or promotion_config.get('rebuild-kinds', [])
do_not_optimize = input.get('do_not_optimize') or promotion_config.get('do-not-optimize', [])
# make parameters read-write
parameters = dict(parameters)
# Build previous_graph_ids from ``previous_graph_ids`` or ``revision``.
previous_graph_ids = input.get('previous_graph_ids')
if not previous_graph_ids:
previous_graph_ids = [find_decision_task(parameters, graph_config)]
# Download parameters from the first decision task
parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml")
# Download and combine full task graphs from each of the previous_graph_ids.
# Sometimes previous relpro action tasks will add tasks, like partials,
# that didn't exist in the first full_task_graph, so combining them is
# important. The rightmost graph should take precedence in the case of
# conflicts.
combined_full_task_graph = {}
for graph_id in previous_graph_ids:
full_task_graph = get_artifact(graph_id, "public/full-task-graph.json")
combined_full_task_graph.update(full_task_graph)
_, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph)
parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds(
combined_full_task_graph, previous_graph_ids, rebuild_kinds
)
parameters['do_not_optimize'] = do_not_optimize
parameters['target_tasks_method'] = target_tasks_method
parameters['build_number'] = int(input['build_number'])
# When doing staging releases on try, we still want to re-use tasks from
# previous graphs.
parameters['optimize_target_tasks'] = True
parameters['shipping_phase'] = input['release_promotion_flavor']
parameters['version'] = input['version'] if input.get('version') else read_version_file()
parameters['head_tag'] = 'v{}'.format(parameters['version'])
parameters['release_type'] = resolve_release_type(parameters['head_tag'])
parameters['pull_request_number'] = None
# make parameters read-only
parameters = Parameters(**parameters)
taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters)
def read_version_file():
with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f:
return f.read().strip().decode('utf-8')

1
taskcluster/fenix_taskgraph/target_tasks.py
View File

@ -19,6 +19,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config):
@_target_task('release')
def target_tasks_default(full_task_graph, parameters, graph_config):
# TODO Use shipping-phase once we retire github-releases
def filter(task, parameters):
return task.attributes.get("release-type", "") == parameters["release_type"]

2
taskcluster/fenix_taskgraph/transforms/build.py
View File

@ -97,7 +97,7 @@ def add_release_version(config, tasks):
for task in tasks:
if task.pop("include-release-version", False):
task["run"]["gradlew"].append(
'-PversionName={}'.format(config.params["release_version"])
'-PversionName={}'.format(config.params["version"])
)
yield task

4
taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py
View File

@ -19,10 +19,10 @@ transforms = TransformSequence()
def make_task_description(config, jobs):
for job in jobs:
product = "Fenix"
version = config.params['release_version'] or "{ver}"
version = config.params['version'] or "{ver}"
job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format(
product=product,
version=version,
build_number=config.params.get('build_number', 1)
)
yield job
yield job