From 7f772404ce90be7b4bd9c141c3b858b5c3db3456 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 27 Mar 2019 15:23:03 +0100 Subject: [PATCH] Run build and tests on PRs and pushes --- .taskcluster.yml | 144 +++++++++++++------ app/build.gradle | 12 ++ automation/taskcluster/decision_task.py | 38 ++++- automation/taskcluster/lib/build_variants.py | 24 ++++ automation/taskcluster/lib/tasks.py | 115 ++++++++++++++- 5 files changed, 280 insertions(+), 53 deletions(-) create mode 100644 automation/taskcluster/lib/build_variants.py diff --git a/.taskcluster.yml b/.taskcluster.yml index 50d9cfca3..b489fb131 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -108,54 +108,104 @@ tasks: taskclusterProxy: true extra: tasks_for: ${tasks_for} + treeherder: + machine: + platform: mobile-decision metadata: owner: ${user}@users.noreply.github.com source: ${repository}/raw/${head_rev}/.taskcluster.yml in: - - $if: 'tasks_for == "cron"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - $if: is_repo_trusted - then: assume:hook-id:project-mobile/fenix-nightly - else: assume:hook-id:project-mobile/fenix-nightly-staging - routes: - - notify.email.fenix-eng-notifications@mozilla.com.on-failed - payload: - features: - taskclusterProxy: true - chainOfTrust: true - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout ${head_rev} - && python automation/taskcluster/decision_task.py \ - release \ - --nightly \ - --track ${track} \ - --commit \ - --output /opt/fenix/app/build/outputs/apk \ - --apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \ - --apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \ - --apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \ - --date ${now} - artifacts: - public/task-graph.json: - type: file - path: /opt/fenix/task-graph.json - expires: ${expires_in} - public/actions.json: - type: file - path: /opt/fenix/actions.json - expires: ${expires_in} - public/parameters.yml: - type: file - path: /opt/fenix/parameters.yml - expires: ${expires_in} - extra: - cron: {$json: {$eval: 'cron'}} - metadata: - name: Fenix Nightly Decision Task - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + $if: 'tasks_for in ["github-pull-request", "github-push"]' + then: + $let: + pr_or_push_parameters: + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout ${head_rev} + && python automation/taskcluster/decision_task.py pr-or-push + in: + - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: + $let: + pull_request_title: ${event.pull_request.title} + pull_request_number: ${event.pull_request.number} + pull_request_url: ${event.pull_request.html_url} + in: + $mergeDeep: + - {$eval: 'default_task_definition'} + - {$eval: 'pr_or_push_parameters'} + - scopes: + - ${assume_scope_prefix}:pull-request + payload: + env: + GITHUB_PULL_TITLE: ${pull_request_title} + extra: + treeherder: + symbol: D-PR + metadata: + name: 'Fenix - Decision task (Pull Request #${pull_request_number})' + description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' + - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - {$eval: 'pr_or_push_parameters'} + - scopes: + - ${assume_scope_prefix}:branch:${short_head_branch} + extra: + treeherder: + symbol: D + metadata: + name: Fenix - Decision task + description: Schedules the build and test tasks for Fenix. + else: + - $if: 'tasks_for == "cron"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - $if: is_repo_trusted + then: assume:hook-id:project-mobile/fenix-nightly + else: assume:hook-id:project-mobile/fenix-nightly-staging + routes: + - notify.email.fenix-eng-notifications@mozilla.com.on-failed + payload: + features: + taskclusterProxy: true + chainOfTrust: true + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout ${head_rev} + && python automation/taskcluster/decision_task.py \ + release \ + --nightly \ + --track ${track} \ + --commit \ + --output /opt/fenix/app/build/outputs/apk \ + --apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \ + --apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \ + --apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \ + --date ${now} + artifacts: + public/task-graph.json: + type: file + path: /opt/fenix/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /opt/fenix/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /opt/fenix/parameters.yml + expires: ${expires_in} + extra: + cron: {$json: {$eval: 'cron'}} + metadata: + name: Fenix Nightly Decision Task + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) diff --git a/app/build.gradle b/app/build.gradle index 5585cb139..bb8d61ccb 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -333,6 +333,18 @@ if (project.hasProperty("raptor")) { android.defaultConfig.manifestPlaceholders.isRaptorEnabled = "true" } +// ------------------------------------------------------------------------------------------------- +// Task for printing all build variants to build variants in parallel in automation +// ------------------------------------------------------------------------------------------------- +task printBuildVariants { + doLast { + def buildVariants = android.applicationVariants.collect { variant -> + variant.name + } + println "variants: " + groovy.json.JsonOutput.toJson(buildVariants) + } +} + // Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we // can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best. apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle' diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 6694875a2..b51923a73 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -10,8 +10,10 @@ from __future__ import print_function import argparse import os +import sys import taskcluster +from lib import build_variants from lib.tasks import TaskBuilder, schedule_task_graph from lib.util import ( populate_chain_of_trust_task_graph, @@ -38,6 +40,39 @@ BUILDER = TaskBuilder( ) +def pr_or_push(): + if SKIP_TASKS_TRIGGER in PR_TITLE: + print("Pull request title contains", SKIP_TASKS_TRIGGER) + print("Exit") + exit(0) + + print("Fetching build variants from gradle") + variants = build_variants.from_gradle() + + if len(variants) == 0: + print("Could not get build variants from gradle") + sys.exit(2) + + print("Got variants: {}".format(' '.join(variants))) + + build_tasks = {} + other_tasks = {} + + for variant in variants: + build_tasks[taskcluster.slugId()] = BUILDER.craft_assemble_task(variant) + build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) + + for craft_function in ( + BUILDER.craft_detekt_task, + BUILDER.craft_ktlint_task, + BUILDER.craft_lint_task, + BUILDER.craft_compare_locales_task, + ): + other_tasks[taskcluster.slugId()] = craft_function() + + return (build_tasks, other_tasks) + + def nightly(apks, track, commit, date_string): is_staging = track == 'staging-nightly' @@ -99,8 +134,7 @@ if __name__ == "__main__": command = result.command if command == 'pr-or-push': - # TODO - ordered_groups_of_tasks = {} + ordered_groups_of_tasks = pr_or_push() elif command == 'release': apks = ["{}/{}".format(result.output, apk) for apk in result.apks] # nightly(apks, result.track, result.commit, result.date) diff --git a/automation/taskcluster/lib/build_variants.py b/automation/taskcluster/lib/build_variants.py new file mode 100644 index 000000000..e4c7ad95b --- /dev/null +++ b/automation/taskcluster/lib/build_variants.py @@ -0,0 +1,24 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import print_function +import json +import subprocess + + +def from_gradle(): + process = subprocess.Popen([ + "./gradlew", "--no-daemon", "--quiet", "printBuildVariants" + ], stdout=subprocess.PIPE) + (output, err) = process.communicate() + exit_code = process.wait() + + if exit_code != 0: + print("Gradle command returned error: {}".format(exit_code)) + + variants_line = [line for line in output.split('\n') if line.startswith('variants: ')][0] + variants_json = variants_line.split(' ', 1)[1] + variants = json.loads(variants_json) + + return variants diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index c4aadfcc8..22ab9f97a 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -70,7 +70,7 @@ class TaskBuilder(object): ] return self._craft_build_ish_task( - name='Fenix - Build task', + name='Build task', description='Build Fenix from source code', command=command, scopes=[ @@ -81,6 +81,60 @@ class TaskBuilder(object): is_staging=is_staging, ) + def craft_assemble_task(self, variant): + return self._craft_gradle_clean_task( + name='assemble: {}'.format(variant), + description='Building and testing variant {}'.format(variant), + gradle_task='assemble{}'.format(variant.capitalize()), + artifacts=_craft_artifacts_from_variant(variant), + ) + + def craft_test_task(self, variant): + return self._craft_gradle_clean_task( + name='test: {}'.format(variant), + description='Building and testing variant {}'.format(variant), + gradle_task='test{}UnitTest'.format(variant.capitalize()), + ) + + def craft_detekt_task(self): + return self._craft_gradle_clean_task( + name='detekt', + description='Running detekt over all modules', + gradle_task='detekt' + ) + + def craft_ktlint_task(self): + return self._craft_gradle_clean_task( + name='ktlint', + description='Running ktlint over all modules', + gradle_task='ktlint' + ) + + def craft_lint_task(self): + return self._craft_gradle_clean_task( + name='lint', + description='Running ktlint over all modules', + gradle_task='lint' + ) + + def _craft_gradle_clean_task(self, name, description, gradle_task, artifacts=None): + return self._craft_build_ish_task( + name=name, + description=description, + command='./gradlew --no-daemon clean {}'.format(gradle_task), + artifacts=artifacts, + ) + + def craft_compare_locales_task(self): + return self._craft_build_ish_task( + name='compare-locales', + description='Validate strings.xml with compare-locales', + command=( + 'pip install "compare-locales>=5.0.2,<6.0" && ' + 'compare-locales --validate l10n.toml .' + ) + ) + def _craft_build_ish_task( self, name, description, command, dependencies=None, artifacts=None, scopes=None, routes=None, is_staging=True @@ -155,7 +209,7 @@ class TaskBuilder(object): "scopes": scopes, "payload": payload, "metadata": { - "name": name, + "name": "Fenix - {}".format(name), "description": description, "owner": self.owner, "source": self.source, @@ -198,7 +252,7 @@ class TaskBuilder(object): 'dep-signing' if is_staging else 'release-signing' ) ], - name="Fenix - Signing task", + name="Signing task", description="Sign release builds of Fenix", payload=payload ) @@ -228,12 +282,65 @@ class TaskBuilder(object): ':dep' if is_staging else '' ) ], - name="Fenix - Push task", + name="Push task", description="Upload signed release builds of Fenix to Google Play", payload=payload ) +def _craft_artifacts_from_variant(variant): + return { + 'public/target.apk': { + 'type': 'file', + 'path': _craft_apk_full_path_from_variant(variant), + 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), + } + } + + +def _craft_apk_full_path_from_variant(variant): + architecture, build_type = _get_architecture_and_build_type_from_variant(variant) + + short_variant = variant[:-len(build_type)] + postfix = '-unsigned' if build_type == 'release' else '' + product = '{}{}'.format(product[0].lower(), product[1:]) + + return '/opt/fenix/app/build/outputs/apk/{short_variant}/{build_type}/app-{architecture}-{product}-{build_type}{postfix}.apk'.format( # noqa: E501 + architecture=architecture, + build_type=build_type, + product=product, + short_variant=short_variant, + postfix=postfix + ) + + +def _get_architecture_and_build_type_from_variant(variant): + variant = variant.lower() + + architecture = None + if 'aarch64' in variant: + architecture = 'aarch64' + elif 'x86' in variant: + architecture = 'x86' + elif 'arm' in variant: + architecture = 'arm' + + build_type = None + if variant.endswith('debug'): + build_type = 'debug' + elif variant.endswith('release'): + build_type = 'release' + + if not architecture or not build_type: + raise ValueError( + 'Unsupported variant "{}". Found architecture, build_type: {}'.format( + variant, (architecture, build_type) + ) + ) + + return architecture, build_type + + def schedule_task(queue, taskId, task): print("TASK", taskId) print(json.dumps(task, indent=4, separators=(',', ': ')))