From 58fc0e5338787e122379b31a81544c9e89a83722 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 9 Apr 2020 14:33:13 +0200 Subject: [PATCH] Bug 1568466 - part 3: Enable shipit graph (#7314) --- taskcluster/ci/config.yml | 8 +- taskcluster/fenix_taskgraph/__init__.py | 62 +------- taskcluster/fenix_taskgraph/parameters.py | 55 +++++++ .../fenix_taskgraph/release_promotion.py | 150 ++++++++++++++++++ taskcluster/fenix_taskgraph/target_tasks.py | 1 + .../fenix_taskgraph/transforms/build.py | 2 +- .../transforms/mark_as_shipped.py | 4 +- 7 files changed, 224 insertions(+), 58 deletions(-) create mode 100644 taskcluster/fenix_taskgraph/parameters.py create mode 100644 taskcluster/fenix_taskgraph/release_promotion.py diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 5245fa9a7..8d68d9331 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -26,7 +26,7 @@ taskgraph: mobile: name: "Fenix" cached-task-prefix: project.mobile.fenix - decision-parameters: 'fenix_taskgraph:get_decision_parameters' + decision-parameters: 'fenix_taskgraph.parameters:get_decision_parameters' workers: aliases: @@ -71,3 +71,9 @@ workers: scriptworker: scope-prefix: project:mobile:fenix:releng + + +release-promotion: + flavors: + ship: + target-tasks-method: release diff --git a/taskcluster/fenix_taskgraph/__init__.py b/taskcluster/fenix_taskgraph/__init__.py index 7a6f6e098..37cd6ff80 100644 --- a/taskcluster/fenix_taskgraph/__init__.py +++ b/taskcluster/fenix_taskgraph/__init__.py @@ -4,17 +4,7 @@ from __future__ import absolute_import, print_function, unicode_literals -import os -import re - from importlib import import_module -from six import text_type -from voluptuous import All, Any, Range, Required - -from taskgraph.parameters import extend_parameters_schema - -BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') -PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') def register(graph_config): @@ -22,52 +12,16 @@ def register(graph_config): Import all modules that are siblings of this one, triggering decorators in the process. """ - _import_modules(["job", "worker_types", "routes", "target_tasks"]) - extend_parameters_schema({ - Required("pull_request_number"): Any(All(int, Range(min=1)), None), - Required("release_type"): text_type, - Required("release_version"): text_type, - }) + _import_modules([ + "job", + "parameters", + "release_promotion", + "routes", + "target_tasks", + "worker_types", + ]) def _import_modules(modules): for module in modules: import_module(".{}".format(module), package=__name__) - - -def get_decision_parameters(graph_config, parameters): - head_tag = parameters["head_tag"].decode("utf-8") - parameters["release_type"] = _resolve_release_type(head_tag) - parameters["release_version"] = read_version_file() - if head_tag: - parameters["release_version"] = head_tag[1:] - - pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None) - parameters["pull_request_number"] = None if pr_number is None else int(pr_number) - - if parameters["tasks_for"] == "github-release": - for param_name in ("release_type", "release_version"): - if not parameters[param_name]: - raise ValueError( - 'Cannot run github-release if "{}" is not defined. Got: {}'.format( - param_name, parameters[param_name] - ) - ) - parameters["target_tasks_method"] = "release" - - -def read_version_file(): - with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f: - return f.read().strip().decode('utf-8') - - -def _resolve_release_type(head_tag): - if not head_tag: - return "" - elif BETA_SEMVER.match(head_tag): - return "beta" - elif PRODUCTION_SEMVER.match(head_tag): - return "production" - else: - raise ValueError('Github tag must be in semver format and prefixed with a "v", ' - 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') diff --git a/taskcluster/fenix_taskgraph/parameters.py b/taskcluster/fenix_taskgraph/parameters.py new file mode 100644 index 000000000..11a4d9a75 --- /dev/null +++ b/taskcluster/fenix_taskgraph/parameters.py @@ -0,0 +1,55 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import os +import re + +from six import text_type +from taskgraph.parameters import extend_parameters_schema +from voluptuous import All, Any, Optional, Range, Required + + +BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') +PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') + + +extend_parameters_schema({ + Required("pull_request_number"): Any(All(int, Range(min=1)), None), + Required("release_type"): text_type, + Optional("shipping_phase"): Any('build', 'ship', None), + Required("version"): text_type, +}) + + +def get_decision_parameters(graph_config, parameters): + head_tag = parameters["head_tag"].decode("utf-8") + parameters["release_type"] = resolve_release_type(head_tag) + parameters["version"] = head_tag[1:] if head_tag else "" + + pr_number = os.environ.get("MOBILE_PULL_REQUEST_NUMBER", None) + parameters["pull_request_number"] = None if pr_number is None else int(pr_number) + + if parameters["tasks_for"] == "github-release": + for param_name in ("release_type", "version"): + if not parameters[param_name]: + raise ValueError( + 'Cannot run github-release if "{}" is not defined. Got: {}'.format( + param_name, parameters[param_name] + ) + ) + parameters["target_tasks_method"] = "release" + + +def resolve_release_type(head_tag): + if not head_tag: + return "" + elif BETA_SEMVER.match(head_tag): + return "beta" + elif PRODUCTION_SEMVER.match(head_tag): + return "production" + else: + raise ValueError('Github tag must be in semver format and prefixed with a "v", ' + 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') diff --git a/taskcluster/fenix_taskgraph/release_promotion.py b/taskcluster/fenix_taskgraph/release_promotion.py new file mode 100644 index 000000000..9a34417a7 --- /dev/null +++ b/taskcluster/fenix_taskgraph/release_promotion.py @@ -0,0 +1,150 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +from taskgraph.actions.registry import register_callback_action + +from taskgraph.util.taskcluster import get_artifact +from taskgraph.taskgraph import TaskGraph +from taskgraph.decision import taskgraph_decision +from taskgraph.parameters import Parameters +from taskgraph.util.taskgraph import find_decision_task, find_existing_tasks_from_previous_kinds + +from .parameters import resolve_release_type + +RELEASE_PROMOTION_PROJECTS = ( + "https://github.com/JohanLorenzo/fenix", +) + + +def is_release_promotion_available(parameters): + return parameters['head_repository'] in RELEASE_PROMOTION_PROJECTS + + +@register_callback_action( + name='release-promotion', + title='Ship Fenix', + symbol='${input.release_promotion_flavor}', + description="Ship Fenix", + generic=False, + order=500, + context=[], + available=is_release_promotion_available, + schema=lambda graph_config: { + 'type': 'object', + 'properties': { + 'build_number': { + 'type': 'integer', + 'default': 1, + 'minimum': 1, + 'title': 'The release build number', + 'description': ('The release build number. Starts at 1 per ' + 'release version, and increments on rebuild.'), + }, + 'do_not_optimize': { + 'type': 'array', + 'description': ('Optional: a list of labels to avoid optimizing out ' + 'of the graph (to force a rerun of, say, ' + 'funsize docker-image tasks).'), + 'items': { + 'type': 'string', + }, + }, + 'revision': { + 'type': 'string', + 'title': 'Optional: revision to ship', + 'description': ('Optional: the revision to ship.'), + }, + 'release_promotion_flavor': { + 'type': 'string', + 'description': 'The flavor of release promotion to perform.', + 'default': 'build', + 'enum': sorted(graph_config['release-promotion']['flavors'].keys()), + }, + 'rebuild_kinds': { + 'type': 'array', + 'description': ('Optional: an array of kinds to ignore from the previous ' + 'graph(s).'), + 'items': { + 'type': 'string', + }, + }, + 'previous_graph_ids': { + 'type': 'array', + 'description': ('Optional: an array of taskIds of decision or action ' + 'tasks from the previous graph(s) to use to populate ' + 'our `previous_graph_kinds`.'), + 'items': { + 'type': 'string', + }, + }, + 'version': { + 'type': 'string', + 'description': ('Optional: override the version for release promotion. ' + "Occasionally we'll land a taskgraph fix in a later " + 'commit, but want to act on a build from a previous ' + 'commit. If a version bump has landed in the meantime, ' + 'relying on the in-tree version will break things.'), + 'default': '', + }, + }, + "required": ['release_promotion_flavor', 'version', 'build_number'], + } +) +def release_promotion_action(parameters, graph_config, input, task_group_id, task_id): + release_promotion_flavor = input['release_promotion_flavor'] + promotion_config = graph_config['release-promotion']['flavors'][release_promotion_flavor] + + target_tasks_method = promotion_config['target-tasks-method'].format( + project=parameters['project'] + ) + rebuild_kinds = input.get('rebuild_kinds') or promotion_config.get('rebuild-kinds', []) + do_not_optimize = input.get('do_not_optimize') or promotion_config.get('do-not-optimize', []) + + # make parameters read-write + parameters = dict(parameters) + # Build previous_graph_ids from ``previous_graph_ids`` or ``revision``. + previous_graph_ids = input.get('previous_graph_ids') + if not previous_graph_ids: + previous_graph_ids = [find_decision_task(parameters, graph_config)] + + # Download parameters from the first decision task + parameters = get_artifact(previous_graph_ids[0], "public/parameters.yml") + # Download and combine full task graphs from each of the previous_graph_ids. + # Sometimes previous relpro action tasks will add tasks, like partials, + # that didn't exist in the first full_task_graph, so combining them is + # important. The rightmost graph should take precedence in the case of + # conflicts. + combined_full_task_graph = {} + for graph_id in previous_graph_ids: + full_task_graph = get_artifact(graph_id, "public/full-task-graph.json") + combined_full_task_graph.update(full_task_graph) + _, combined_full_task_graph = TaskGraph.from_json(combined_full_task_graph) + parameters['existing_tasks'] = find_existing_tasks_from_previous_kinds( + combined_full_task_graph, previous_graph_ids, rebuild_kinds + ) + parameters['do_not_optimize'] = do_not_optimize + parameters['target_tasks_method'] = target_tasks_method + parameters['build_number'] = int(input['build_number']) + # When doing staging releases on try, we still want to re-use tasks from + # previous graphs. + parameters['optimize_target_tasks'] = True + parameters['shipping_phase'] = input['release_promotion_flavor'] + + parameters['version'] = input['version'] if input.get('version') else read_version_file() + parameters['head_tag'] = 'v{}'.format(parameters['version']) + parameters['release_type'] = resolve_release_type(parameters['head_tag']) + + parameters['pull_request_number'] = None + + # make parameters read-only + parameters = Parameters(**parameters) + + taskgraph_decision({'root': graph_config.root_dir}, parameters=parameters) + + +def read_version_file(): + with open(os.path.join(os.path.dirname(__file__), '..', '..', 'version.txt')) as f: + return f.read().strip().decode('utf-8') diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index 04939da48..e28706517 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -19,6 +19,7 @@ def target_tasks_default(full_task_graph, parameters, graph_config): @_target_task('release') def target_tasks_default(full_task_graph, parameters, graph_config): + # TODO Use shipping-phase once we retire github-releases def filter(task, parameters): return task.attributes.get("release-type", "") == parameters["release_type"] diff --git a/taskcluster/fenix_taskgraph/transforms/build.py b/taskcluster/fenix_taskgraph/transforms/build.py index c97d64da4..3136b1b91 100644 --- a/taskcluster/fenix_taskgraph/transforms/build.py +++ b/taskcluster/fenix_taskgraph/transforms/build.py @@ -97,7 +97,7 @@ def add_release_version(config, tasks): for task in tasks: if task.pop("include-release-version", False): task["run"]["gradlew"].append( - '-PversionName={}'.format(config.params["release_version"]) + '-PversionName={}'.format(config.params["version"]) ) yield task diff --git a/taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py b/taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py index 1b1a9bcb6..ac963b915 100644 --- a/taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py +++ b/taskcluster/fenix_taskgraph/transforms/mark_as_shipped.py @@ -19,10 +19,10 @@ transforms = TransformSequence() def make_task_description(config, jobs): for job in jobs: product = "Fenix" - version = config.params['release_version'] or "{ver}" + version = config.params['version'] or "{ver}" job['worker']['release-name'] = '{product}-{version}-build{build_number}'.format( product=product, version=version, build_number=config.params.get('build_number', 1) ) - yield job \ No newline at end of file + yield job