From 77ff164e3065ad2d396a0c3e00781b33e7ca535c Mon Sep 17 00:00:00 2001 From: Mitchell Hentges Date: Mon, 27 May 2019 15:00:44 +0200 Subject: [PATCH] Passes variants without parsing (#2408) --- app/build.gradle | 11 +-- automation/taskcluster/decision_task.py | 9 ++- automation/taskcluster/lib/gradle.py | 11 +-- automation/taskcluster/lib/tasks.py | 92 +++++++------------------ automation/taskcluster/lib/variant.py | 20 ++++++ 5 files changed, 62 insertions(+), 81 deletions(-) create mode 100644 automation/taskcluster/lib/variant.py diff --git a/app/build.gradle b/app/build.gradle index 4bcb65103..090bc2ac9 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -393,10 +393,13 @@ if (project.hasProperty("raptor")) { // ------------------------------------------------------------------------------------------------- task printBuildVariants { doLast { - def buildVariants = android.applicationVariants.collect { variant -> - variant.name - } - println "variants: " + groovy.json.JsonOutput.toJson(buildVariants) + def variantData = android.applicationVariants.collect { variant -> [ + name: variant.name, + buildType: variant.buildType.name, + abi: variant.productFlavors.find { it.dimension == 'abi' }.name, + isSigned: variant.signingReady, + ]} + println "variants: " + groovy.json.JsonOutput.toJson(variantData) } } diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index cccf29b31..a11025ce5 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -15,10 +15,9 @@ import re import taskcluster -from lib.gradle import get_build_variants, get_geckoview_versions +from lib.gradle import get_debug_variants, get_geckoview_versions from lib.tasks import ( fetch_mozharness_task_id, - get_architecture_and_build_type_from_variant, schedule_task_graph, TaskBuilder, ) @@ -26,6 +25,7 @@ from lib.chain_of_trust import ( populate_chain_of_trust_task_graph, populate_chain_of_trust_required_but_unused_files ) +from lib.variant import Variant REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') @@ -57,7 +57,6 @@ def pr_or_push(is_push): print("Exit") return {} - debug_variants = [variant for variant in get_build_variants() if variant.endswith('Debug')] geckoview_nightly_version = get_geckoview_versions()['nightly'] mozharness_task_id = fetch_mozharness_task_id(geckoview_nightly_version) gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] @@ -66,7 +65,7 @@ def pr_or_push(is_push): signing_tasks = {} other_tasks = {} - for variant in debug_variants: + for variant in get_debug_variants(): assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant) @@ -74,7 +73,7 @@ def pr_or_push(is_push): if is_push and SHORT_HEAD_BRANCH == 'master': other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() - for variant in ('armRaptor', 'aarch64Raptor'): + for variant in [Variant.from_values(abi, False, 'raptor') for abi in ('aarch64', 'arm')]: assemble_task_id = taskcluster.slugId() build_tasks[assemble_task_id] = BUILDER.craft_assemble_task(variant) signing_task_id = taskcluster.slugId() diff --git a/automation/taskcluster/lib/gradle.py b/automation/taskcluster/lib/gradle.py index 6242c9180..e0ff24437 100644 --- a/automation/taskcluster/lib/gradle.py +++ b/automation/taskcluster/lib/gradle.py @@ -6,8 +6,10 @@ from __future__ import print_function import json import subprocess +from lib.variant import Variant -def get_build_variants(): + +def get_debug_variants(): print("Fetching build variants from gradle") output = _run_gradle_process('printBuildVariants') content = _extract_content_from_command_output(output, prefix='variants: ') @@ -16,9 +18,10 @@ def get_build_variants(): if len(variants) == 0: raise ValueError("Could not get build variants from gradle") - print("Got variants: {}".format(' '.join(variants))) - - return variants + print("Got variants: {}".format(variants)) + return [Variant(variant_dict['name'], variant_dict['abi'], variant_dict['isSigned'], variant_dict['buildType']) + for variant_dict in variants + if variant_dict['buildType'] == 'debug'] def get_geckoview_versions(): diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 21659fa1c..d2e7efd69 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -113,15 +113,15 @@ class TaskBuilder(object): def craft_assemble_task(self, variant): return self._craft_clean_gradle_task( - name='assemble: {}'.format(variant), - description='Building and testing variant {}'.format(variant), - gradle_task='assemble{}'.format(variant.capitalize()), + name='assemble: {}'.format(variant.raw), + description='Building and testing variant {}'.format(variant.raw), + gradle_task='assemble{}'.format(variant.for_gradle_command), artifacts=_craft_artifacts_from_variant(variant), treeherder={ - 'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant), + 'groupSymbol': variant.build_type, 'jobKind': 'build', 'machine': { - 'platform': _craft_treeherder_platform_from_variant(variant), + 'platform': variant.platform, }, 'symbol': 'A', 'tier': 1, @@ -130,14 +130,14 @@ class TaskBuilder(object): def craft_test_task(self, variant): return self._craft_clean_gradle_task( - name='test: {}'.format(variant), - description='Building and testing variant {}'.format(variant), - gradle_task='test{}UnitTest'.format(variant.capitalize()), + name='test: {}'.format(variant.raw), + description='Building and testing variant {}'.format(variant.raw), + gradle_task='test{}UnitTest'.format(variant.for_gradle_command), treeherder={ - 'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant), + 'groupSymbol': variant.build_type, 'jobKind': 'test', 'machine': { - 'platform': _craft_treeherder_platform_from_variant(variant), + 'platform': variant.platform, }, 'symbol': 'T', 'tier': 1, @@ -377,36 +377,35 @@ class TaskBuilder(object): def craft_raptor_signing_task( self, assemble_task_id, variant ): - architecture, _ = get_architecture_and_build_type_from_variant(variant) routes = [] if self.repo_url == _OFFICIAL_REPO_URL: routes = [ 'index.project.mobile.fenix.v2.branch.master.revision.{}.raptor.{}'.format( - self.commit, architecture + self.commit, variant.abi ), 'index.project.mobile.fenix.v2.branch.master.latest.raptor.{}'.format( - architecture + variant.abi ), 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.revision.{}.raptor.{}'.format( - self.date.year, self.date.month, self.date.day, self.commit, architecture + self.date.year, self.date.month, self.date.day, self.commit, variant.abi ), 'index.project.mobile.fenix.v2.branch.master.pushdate.{}.{}.{}.latest.raptor.{}'.format( - self.date.year, self.date.month, self.date.day, architecture + self.date.year, self.date.month, self.date.day, variant.abi ), ] return self._craft_signing_task( - name='sign: {}'.format(variant), - description='Dep-signing variant {}'.format(variant), + name='sign: {}'.format(variant.raw), + description='Dep-signing variant {}'.format(variant.raw), signing_type="dep", assemble_task_id=assemble_task_id, apk_paths=["public/target.apk"], routes=routes, treeherder={ - 'groupSymbol': _craft_treeherder_group_symbol_from_variant(variant), + 'groupSymbol': variant.build_type, 'jobKind': 'other', 'machine': { - 'platform': _craft_treeherder_platform_from_variant(variant), + 'platform': variant.platform, }, 'symbol': 'As', 'tier': 1, @@ -514,20 +513,19 @@ class TaskBuilder(object): group_symbol=None, force_run_on_64_bit_device=False, ): - architecture, _ = get_architecture_and_build_type_from_variant(variant) - worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or architecture == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5' + worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or variant.abi == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5' if force_run_on_64_bit_device: treeherder_platform = 'android-hw-p2-8-0-arm7-api-16' - elif architecture == 'arm': + elif variant.abi == 'arm': treeherder_platform = 'android-hw-g5-7-0-arm7-api-16' - elif architecture == 'aarch64': + elif variant.abi == 'aarch64': treeherder_platform = 'android-hw-p2-8-0-aarch64' else: - raise ValueError('Unsupported architecture "{}"'.format(architecture)) + raise ValueError('Unsupported architecture "{}"'.format(variant.abi)) task_name = '{}: {} {}'.format( - name_prefix, variant, '(on 64-bit-device)' if force_run_on_64_bit_device else '' + name_prefix, variant.raw, '(on 64-bit-device)' if force_run_on_64_bit_device else '' ) apk_url = '{}/{}/artifacts/{}'.format(_DEFAULT_TASK_URL, signing_task_id, @@ -603,58 +601,16 @@ class TaskBuilder(object): ) - -def _craft_treeherder_platform_from_variant(variant): - architecture, build_type = get_architecture_and_build_type_from_variant(variant) - return 'android-{}-{}'.format(architecture, build_type) - - -def _craft_treeherder_group_symbol_from_variant(variant): - _, build_type = get_architecture_and_build_type_from_variant(variant) - return build_type - - def _craft_artifacts_from_variant(variant): return { DEFAULT_APK_ARTIFACT_LOCATION: { 'type': 'file', - 'path': _craft_apk_full_path_from_variant(variant), + 'path': variant.apk_absolute_path(), 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), } } -def _craft_apk_full_path_from_variant(variant): - architecture, build_type = get_architecture_and_build_type_from_variant(variant) - postfix = '' if build_type == 'debug' else '-unsigned' - return '/opt/fenix/app/build/outputs/apk/{architecture}/{build_type}/app-{architecture}-{build_type}{postfix}.apk'.format( # noqa: E501 - architecture=architecture, - build_type=build_type, - postfix=postfix - ) - - -_SUPPORTED_ARCHITECTURES = ('aarch64', 'arm', 'x86') - - -def get_architecture_and_build_type_from_variant(variant): - for supported_architecture in _SUPPORTED_ARCHITECTURES: - if variant.startswith(supported_architecture): - architecture = supported_architecture - break - else: - raise ValueError( - 'Cannot identify architecture in "{}". ' - 'Expected to find one of these supported ones: {}'.format( - variant, _SUPPORTED_ARCHITECTURES - ) - ) - - build_type = variant[len(architecture):] - build_type = lower_case_first_letter(build_type) - return architecture, build_type - - def schedule_task(queue, taskId, task): print("TASK", taskId) print(json.dumps(task, indent=4, separators=(',', ': '))) diff --git a/automation/taskcluster/lib/variant.py b/automation/taskcluster/lib/variant.py new file mode 100644 index 000000000..271abb508 --- /dev/null +++ b/automation/taskcluster/lib/variant.py @@ -0,0 +1,20 @@ +class Variant: + def __init__(self, raw, abi, is_signed, build_type): + self.raw = raw + self.abi = abi + self.build_type = build_type + self._is_signed = is_signed + self.for_gradle_command = raw[:1].upper() + raw[1:] + self.platform = 'android-{}-{}'.format(self.abi, self.build_type) + + def apk_absolute_path(self): + return '/opt/fenix/app/build/outputs/apk/{abi}/{build_type}/app-{abi}-{build_type}{unsigned}.apk'.format( + build_type=self.build_type, + abi=self.abi, + unsigned='' if self._is_signed else '-unsigned', + ) + + @staticmethod + def from_values(abi, is_signed, build_type): + raw = abi + build_type[:1].upper() + build_type[1:] + return Variant(raw, abi, is_signed, build_type)