From 6c020a0bc45a1157dbeba3da81da8a4346343364 Mon Sep 17 00:00:00 2001 From: Mitchell Hentges Date: Wed, 21 Aug 2019 08:32:01 -0700 Subject: [PATCH] Configure either geckoview beta or nightly at compile-time (#4851) * Remove "abi" product flavor and introduce "engine" product flavor. This patch will allow us to build Fenix against GeckoView Nightly and GeckoView Beta by introducing a new flavor dimension: engine = [geckoNightly, geckoBeta]. In addition to that it adds a "fenix" prefix to the nightly, beta and production flavors to reduce the ambiguity between fenix beta/nightly and GeckoView beta/nightly. For now the build types have the following engine variants enabled: **debug**: geckoNightly, geckoBeta Both variants enabled for local development and testing. **forPerformanceTest**: geckoNightly, geckoBeta Both variants enabled unless the perf team only cares about Nightly (tbd) **fenixNightlyLegacy**: geckoBeta Uses GeckoView Beta for now - the same version we ship production builds with (same behavior as before). This release type will eventualyl be decommissioned once we switch to a separate Nightly app on Google Play. **fenixNightly**: geckoBeta Uses GeckoView Beta for now - the same version we ship production builds with (same behavior as before). Changing this build to use GeckoView Nightly is currently being discussed. **fenixBeta**: geckoBeta Fenix Beta uses GeckoView Beta. **fenixProduction** Fenix Production uses GeckoView Beta (69) currently. * gradle.py/variant.py: Replace "abi" with "engine". * Disable enableUnitTestBinaryResources until we can switch to Android Gradle plugin 3.5. * Fenix nightly should use both geckoview nightly and beta * Updates automation to use apk splitting and support different engine --- app/build.gradle | 128 +++++++++++++++--------- automation/taskcluster/decision_task.py | 81 ++++++++------- automation/taskcluster/lib/gradle.py | 34 ++++--- automation/taskcluster/lib/tasks.py | 126 +++++++++-------------- automation/taskcluster/lib/variant.py | 39 +++++--- config/pre-push-recommended.sh | 2 +- gradle.properties | 2 +- 7 files changed, 205 insertions(+), 207 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index eb88e7bac..f5d97d2ba 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -14,6 +14,7 @@ import com.android.build.gradle.internal.tasks.AppPreBuildTask import org.gradle.internal.logging.text.StyledTextOutput.Style import org.gradle.internal.logging.text.StyledTextOutputFactory import static org.gradle.api.tasks.testing.TestResult.ResultType +import com.android.build.OutputFile android { compileSdkVersion 28 @@ -52,18 +53,18 @@ android { applicationIdSuffix ".performancetest" debuggable true } - nightlyLegacy releaseTemplate >> { + fenixNightlyLegacy releaseTemplate >> { buildConfigField "boolean", "USE_RELEASE_VERSIONING", "true" } - nightly releaseTemplate >> { + fenixNightly releaseTemplate >> { applicationIdSuffix ".nightly" buildConfigField "boolean", "USE_RELEASE_VERSIONING", "true" } - beta releaseTemplate >> { + fenixBeta releaseTemplate >> { applicationIdSuffix ".beta" buildConfigField "boolean", "USE_RELEASE_VERSIONING", "true" } - production releaseTemplate >> { + fenixProduction releaseTemplate >> { buildConfigField "boolean", "USE_RELEASE_VERSIONING", "true" } } @@ -72,6 +73,28 @@ android { if (buildType.name == 'release') { setIgnore true } + + // Current build variant setup: + // + // | geckoNightly | geckoBeta | + // |--------------------|---------------|-----------| + // | debug | ✅ | ✅ | Both variants for testing and development. + // | forPerformanceTest | ✅ | ✅ | Both variants unless the perf team only cares about Nightly (TBD). + // | fenixNightlyLegacy | ✅ | ✅ | Release type will be decommissioned soon. + // | fenixNightly | ✅ | ✅ | Built with both, but only the "geckoNightly" one is published to Google Play + // | fenixBeta | ❌ | ✅ | Fenix Beta ships with GV Beta + // | fenixProduction | ❌ | ✅ | Fenix Production ships with GV Beta + // + + def flavors = flavors*.name.toString().toLowerCase() + + if (buildType.name == 'fenixBeta' && flavors.contains("geckonightly")) { + setIgnore true + } + + if (buildType.name == 'fenixProduction' && flavors.contains("geckonightly")) { + setIgnore true + } } testOptions { @@ -79,32 +102,25 @@ android { unitTests.includeAndroidResources = true } - flavorDimensions "abi" + flavorDimensions "engine" productFlavors { - arm { - dimension "abi" - ndk { - abiFilter "armeabi-v7a" - } + geckoNightly { + dimension "engine" } - aarch64 { - dimension "abi" - ndk { - abiFilter "arm64-v8a" - } + + geckoBeta { + dimension "engine" } - x86 { - dimension "abi" - ndk { - abiFilter "x86" - } - } - x86_64 { - dimension "abi" - ndk { - abiFilter "x86_64" - } + } + + splits { + abi { + enable true + + reset() + + include "x86", "armeabi-v7a", "arm64-v8a", "x86_64" } } @@ -126,6 +142,8 @@ android { } } +def baseVersionCode = generatedVersionCode + android.applicationVariants.all { variant -> // ------------------------------------------------------------------------------------------------- @@ -144,14 +162,10 @@ android.applicationVariants.all { variant -> // Generate version codes for builds // ------------------------------------------------------------------------------------------------- - def buildType = variant.buildType.name - def versionCode = null def isDebug = variant.buildType.resValues['IS_DEBUG']?.value ?: false def useReleaseVersioning = variant.buildType.buildConfigFields['USE_RELEASE_VERSIONING']?.value ?: false if (useReleaseVersioning) { - versionCode = generatedVersionCode - // The Google Play Store does not allow multiple APKs for the same app that all have the // same version code. Therefore we need to have different version codes for our ARM and x86 // builds. @@ -160,17 +174,24 @@ android.applicationVariants.all { variant -> // Our x86 builds need a higher version code to avoid installing ARM builds on an x86 device // with ARM compatibility mode. - if (variant.flavorName.contains("x86_64")) { - versionCode = versionCode + 3 - } else if (variant.flavorName.contains("x86")) { - versionCode = versionCode + 2 - } else if (variant.flavorName.contains("aarch64")) { - versionCode = versionCode + 1 - }// else variant.flavorName.contains("Arm")) use generated version code + variant.outputs.each { output -> + def abi = output.getFilter(OutputFile.ABI) - variant.outputs.all { - versionCodeOverride = versionCode - versionNameOverride = Config.releaseVersionName(project) + def versionCodeOverride + + if (abi == "x86_64") { + versionCodeOverride = baseVersionCode + 3 + } else if (abi == "x86") { + versionCodeOverride = baseVersionCode + 2 + } else if (abi == "arm64-v8a") { + versionCodeOverride = baseVersionCode + 1 + } else if (abi == "armeabi-v7a") { + versionCodeOverride = baseVersionCode + } else { + throw RuntimeException("Unknown ABI: $abi") + } + + println("versionCode for $abi = $versionCodeOverride") } // If this is a release build, validate that "versionName" is set @@ -277,6 +298,9 @@ androidExtensions { dependencies { implementation project(':architecture') + geckoNightlyImplementation Deps.mozilla_browser_engine_gecko_nightly + geckoBetaImplementation Deps.mozilla_browser_engine_gecko_beta + implementation Deps.kotlin_stdlib implementation Deps.kotlin_coroutines implementation Deps.androidx_appcompat @@ -309,7 +333,6 @@ dependencies { implementation Deps.mozilla_browser_domains implementation Deps.mozilla_browser_icons implementation Deps.mozilla_browser_menu - implementation Deps.mozilla_browser_engine_gecko_beta implementation Deps.mozilla_browser_search implementation Deps.mozilla_browser_session implementation Deps.mozilla_browser_storage_sync @@ -483,17 +506,22 @@ if (project.hasProperty("coverage")) { } // ------------------------------------------------------------------------------------------------- -// Task for printing all build variants to build variants in parallel in automation +// Task for printing APK information for the requested variant +// Usage: "./gradlew printVariant -PvariantBuildType=nightly -PvariantEngine=geckoNightly" // ------------------------------------------------------------------------------------------------- -task printBuildVariants { +task printVariant { doLast { - def variantData = android.applicationVariants.collect { variant -> [ - name: variant.name, - buildType: variant.buildType.name, - abi: variant.productFlavors.find { it.dimension == 'abi' }.name, - isSigned: variant.signingReady, - ]} - println "variants: " + groovy.json.JsonOutput.toJson(variantData) + def rawVariant = android.applicationVariants.find { + it.buildType.name == variantBuildType && + it.productFlavors.find { it.dimension == 'engine' }.name == variantEngine + } + println 'variant: ' + groovy.json.JsonOutput.toJson([ + name: rawVariant.name, + apks: rawVariant.variantData.outputScope.apkDatas.collect { [ + abi: it.filters.find { it.filterType == 'ABI' }.identifier, + fileName: it.outputFileName, + ]} + ]) } } diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 26c96c1a5..3b8a3d1c8 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -15,7 +15,7 @@ import re import taskcluster -from lib.gradle import get_variants_for_build_type +from lib.gradle import get_variant from lib.tasks import ( fetch_mozharness_task_id, schedule_task_graph, @@ -25,7 +25,6 @@ from lib.chain_of_trust import ( populate_chain_of_trust_task_graph, populate_chain_of_trust_required_but_unused_files ) -from lib.variant import Variant REPO_URL = os.environ.get('MOBILE_HEAD_REPOSITORY') COMMIT = os.environ.get('MOBILE_HEAD_REV') @@ -61,10 +60,10 @@ def pr(): signing_tasks = {} other_tasks = {} - for variant in get_variants_for_build_type('debug'): - assemble_task_id = taskcluster.slugId() - build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant) - build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant, True) + variant = get_variant('debug', 'geckoNightly') + assemble_task_id = taskcluster.slugId() + build_tasks[assemble_task_id] = BUILDER.craft_assemble_pr_task(variant) + build_tasks[taskcluster.slugId()] = BUILDER.craft_test_pr_task(variant) for craft_function in ( BUILDER.craft_detekt_task, @@ -80,12 +79,12 @@ def pr(): def push(): all_tasks = pr() other_tasks = all_tasks[-1] - other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task() + other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task() if SHORT_HEAD_BRANCH == 'master': other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() - return all_tasks + return all_tasks def raptor(is_staging): @@ -96,41 +95,41 @@ def raptor(is_staging): mozharness_task_id = fetch_mozharness_task_id() gecko_revision = taskcluster.Queue().task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV'] - for variant in [Variant.from_values(abi, False, 'forPerformanceTest') for abi in ('aarch64', 'arm')]: - assemble_task_id = taskcluster.slugId() - build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant) - signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging) + variant = get_variant('forPerformanceTest', 'geckoNightly') + assemble_task_id = taskcluster.slugId() + build_tasks[assemble_task_id] = BUILDER.craft_assemble_raptor_task(variant) + signing_task_id = taskcluster.slugId() + signing_tasks[signing_task_id] = BUILDER.craft_raptor_signing_task(assemble_task_id, variant, is_staging) + for abi in ('aarch64', 'arm'): all_raptor_craft_functions = [ BUILDER.craft_raptor_tp6m_cold_task(for_suite=i) - for i in range(1, 28) - ] + [ - BUILDER.craft_raptor_youtube_playback_task, - ] + for i in range(1, 28) + ] + [ + BUILDER.craft_raptor_youtube_playback_task, + ] for craft_function in all_raptor_craft_functions: - args = (signing_task_id, mozharness_task_id, variant, gecko_revision) + args = (signing_task_id, mozharness_task_id, abi, gecko_revision) other_tasks[taskcluster.slugId()] = craft_function(*args) return (build_tasks, signing_tasks, other_tasks) -def release(channel, is_staging, version_name): - variants = get_variants_for_build_type(channel) - architectures = [variant.abi for variant in variants] - apk_paths = ["public/build/{}/target.apk".format(arch) for arch in architectures] +def release(channel, engine, is_staging, version_name): + variant = get_variant('fenix' + channel.capitalize(), engine) + taskcluster_apk_paths = variant.upstream_artifacts() build_tasks = {} signing_tasks = {} push_tasks = {} build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, channel, is_staging, version_name) + build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, is_staging, version_name) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, - apk_paths=apk_paths, + taskcluster_apk_paths, channel=channel, is_staging=is_staging, ) @@ -138,7 +137,7 @@ def release(channel, is_staging, version_name): push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, - apks=apk_paths, + taskcluster_apk_paths, channel=channel, # TODO until org.mozilla.fenix.nightly is made public, put it on the internally-testable track override_google_play_track=None if channel != "nightly" else "internal", @@ -152,10 +151,8 @@ def nightly_to_production_app(is_staging, version_name): # Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the # "nightly" track. We're moving towards having each channel be published to its own app, but we need to # keep updating this "backwards-compatible" nightly for a while yet - build_type = 'nightlyLegacy' - variants = get_variants_for_build_type(build_type) - architectures = [variant.abi for variant in variants] - apk_paths = ["public/build/{}/target.apk".format(arch) for arch in architectures] + variant = get_variant('fenixNightlyLegacy', 'geckoNightly') + taskcluster_apk_paths = variant.upstream_artifacts() build_tasks = {} signing_tasks = {} @@ -163,12 +160,12 @@ def nightly_to_production_app(is_staging, version_name): other_tasks = {} build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, build_type, is_staging, version_name) + build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(variant, is_staging, version_name) signing_task_id = taskcluster.slugId() signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, - apk_paths=apk_paths, + taskcluster_apk_paths, channel='production', # Since we're publishing to the "production" app, we need to sign for production index_channel='nightly', is_staging=is_staging, @@ -177,16 +174,17 @@ def nightly_to_production_app(is_staging, version_name): push_task_id = taskcluster.slugId() push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, - apks=apk_paths, + taskcluster_apk_paths, channel='production', # We're publishing to the "production" app on the "nightly" track override_google_play_track='nightly', is_staging=is_staging, ) - nimbledroid_task_id = taskcluster.slugId() - other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task( - build_task_id - ) + if not is_staging: + nimbledroid_task_id = taskcluster.slugId() + other_tasks[nimbledroid_task_id] = BUILDER.craft_upload_apk_nimbledroid_task( + build_task_id + ) return (build_tasks, signing_tasks, push_tasks, other_tasks) @@ -213,26 +211,25 @@ if __name__ == "__main__": result = parser.parse_args() command = result.command - taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) - if command in ('pull-request'): + if command == 'pull-request': ordered_groups_of_tasks = pr() - elif command in ('push'): + elif command == 'push': ordered_groups_of_tasks = push() elif command == 'raptor': ordered_groups_of_tasks = raptor(result.staging) elif command == 'nightly': nightly_version = datetime.datetime.now().strftime('Nightly %y%m%d %H:%M') - ordered_groups_of_tasks = release('nightly', result.staging, nightly_version) \ + ordered_groups_of_tasks = release('nightly', 'geckoNightly', result.staging, nightly_version) \ + nightly_to_production_app(result.staging, nightly_version) elif command == 'github-release': version = result.tag[1:] # remove prefixed "v" beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$') production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$') if beta_semver.match(result.tag): - ordered_groups_of_tasks = release('beta', result.staging, version) + ordered_groups_of_tasks = release('beta', 'geckoBeta', result.staging, version) elif production_semver.match(result.tag): - ordered_groups_of_tasks = release('production', result.staging, version) + ordered_groups_of_tasks = release('production', 'geckoBeta', result.staging, version) else: raise ValueError('Github tag must be in semver format and prefixed with a "v", ' 'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)') diff --git a/automation/taskcluster/lib/gradle.py b/automation/taskcluster/lib/gradle.py index 2df005108..df79265d0 100644 --- a/automation/taskcluster/lib/gradle.py +++ b/automation/taskcluster/lib/gradle.py @@ -6,26 +6,28 @@ from __future__ import print_function import json import subprocess -from lib.variant import Variant +from lib.variant import Variant, VariantApk -def get_variants_for_build_type(build_type): - print("Fetching build variants from gradle") - output = _run_gradle_process('printBuildVariants') - content = _extract_content_from_command_output(output, prefix='variants: ') - variants = json.loads(content) - - if len(variants) == 0: - raise ValueError("Could not get build variants from gradle") - - print("Got variants: {}".format(variants)) - return [Variant(variant_dict['name'], variant_dict['abi'], variant_dict['isSigned'], variant_dict['buildType']) - for variant_dict in variants - if variant_dict['buildType'] == build_type] +def get_variant(build_type, engine): + print("Fetching variant information for build_type='{}', engine='{}'".format(build_type, engine)) + output = _run_gradle_process('printVariant', variantBuildType=build_type, variantEngine=engine) + content = _extract_content_from_command_output(output, prefix='variant: ') + raw_variant = json.loads(content) + return Variant( + raw_variant['name'], + build_type, + [VariantApk(build_type, raw_apk['abi'], engine, raw_apk['fileName']) for raw_apk in raw_variant['apks']] + ) -def _run_gradle_process(gradle_command): - process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command], stdout=subprocess.PIPE) +def _run_gradle_process(gradle_command, **kwargs): + gradle_properties = [ + '-P{property_name}={value}'.format(property_name=property_name, value=value) + for property_name, value in kwargs.iteritems() + ] + + process = subprocess.Popen(["./gradlew", "--no-daemon", "--quiet", gradle_command] + gradle_properties, stdout=subprocess.PIPE) output, err = process.communicate() exit_code = process.wait() diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index e7725ca78..6c5f62d8a 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -46,21 +46,11 @@ class TaskBuilder(object): self.date = arrow.get(date_string) self.trust_level = trust_level - def craft_assemble_release_task(self, architectures, build_type, is_staging, version_name): - artifacts = { - 'public/build/{}/target.apk'.format(arch): { - "type": 'file', - "path": '/opt/fenix/app/build/outputs/apk/' - '{arch}/{build_type}/app-{arch}-{build_type}-unsigned.apk'.format(arch=arch, build_type=build_type), - "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), - } - for arch in architectures - } - + def craft_assemble_release_task(self, variant, is_staging, version_name): if is_staging: secret_index = 'garbage/staging/project/mobile/fenix' else: - secret_index = 'project/mobile/fenix/{}'.format(build_type) + secret_index = 'project/mobile/fenix/{}'.format(variant.build_type) pre_gradle_commands = ( 'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format( @@ -70,11 +60,11 @@ class TaskBuilder(object): ('sentry_dsn', '.sentry_token'), ('leanplum', '.leanplum_token'), ('adjust', '.adjust_token'), - ('firebase', 'app/src/{}/res/values/firebase.xml'.format(build_type)), + ('firebase', 'app/src/{}/res/values/firebase.xml'.format(variant.build_type)), ) ) - capitalized_build_type = upper_case_first_letter(build_type) + capitalized_build_type = upper_case_first_letter(variant.build_type) gradle_commands = ( './gradlew --no-daemon -PversionName="{}" clean test assemble{}'.format( version_name, capitalized_build_type), @@ -98,14 +88,14 @@ class TaskBuilder(object): scopes=[ "secrets:get:{}".format(secret_index) ], - artifacts=artifacts, + artifacts=variant.artifacts(), routes=routes, treeherder={ 'jobKind': 'build', 'machine': { 'platform': 'android-all', }, - 'symbol': '{}-A'.format(build_type), + 'symbol': '{}-A'.format(variant.build_type), 'tier': 1, }, ) @@ -115,19 +105,19 @@ class TaskBuilder(object): 'echo "https://fake@sentry.prod.mozaws.net/368" > .sentry_token', 'echo "--" > .adjust_token', 'echo "-:-" > .leanplum_token', - './gradlew --no-daemon clean assemble{}'.format(variant.for_gradle_command), + './gradlew --no-daemon clean assemble{}'.format(variant.name), )) return self._craft_build_ish_task( - name='assemble: {}'.format(variant.raw), - description='Building and testing variant {}'.format(variant.raw), + name='assemble: {}'.format(variant.name), + description='Building and testing variant {}'.format(variant.name), command=command, - artifacts=_craft_artifacts_from_variant(variant), + artifacts=variant.artifacts(), treeherder={ 'groupSymbol': variant.build_type, 'jobKind': 'build', 'machine': { - 'platform': variant.platform, + 'platform': 'android-all', }, 'symbol': 'A', 'tier': 1, @@ -135,49 +125,33 @@ class TaskBuilder(object): ) def craft_assemble_pr_task(self, variant): - assemble_gradle_command = 'assemble{}'.format(variant.for_gradle_command) - return self._craft_clean_gradle_task( - name='assemble: {}'.format(variant.raw), - description='Building and testing variant {}'.format(variant.raw), - gradle_task=assemble_gradle_command, - artifacts=_craft_artifacts_from_variant(variant), + name='assemble: {}'.format(variant.name), + description='Building and testing variant {}'.format(variant.name), + gradle_task='assemble{}'.format(variant.name), + artifacts=variant.artifacts(), treeherder={ 'groupSymbol': variant.build_type, 'jobKind': 'build', 'machine': { - 'platform': variant.platform, + 'platform': 'android-all', }, 'symbol': 'A', 'tier': 1, } ) - def craft_test_pr_task(self, variant, run_coverage=False): - test_gradle_command = '-Pcoverage jacoco{}TestReport'.format(variant.for_gradle_command) \ - if (run_coverage and variant.abi == 'aarch64') \ - else 'test{}UnitTest'.format(variant.for_gradle_command) - post_gradle_command = ('automation/taskcluster/upload_coverage_report.sh' if run_coverage else '',) - - if variant.abi == 'aarch64': - command = ' && '.join( - cmd - for commands in ((test_gradle_command,), post_gradle_command) - for cmd in commands - if cmd - ) - else: - command = test_gradle_command - + def craft_test_pr_task(self, variant): + command = 'test{}UnitTest'.format(variant.name) return self._craft_clean_gradle_task( - name='test: {}'.format(variant.raw), - description='Building and testing variant {}'.format(variant.raw), + name='test: {}'.format(variant.name), + description='Building and testing variant {}'.format(variant.name), gradle_task=command, treeherder={ 'groupSymbol': variant.build_type, 'jobKind': 'test', 'machine': { - 'platform': variant.platform, + 'platform': 'android-all', }, 'symbol': 'T', 'tier': 1, @@ -491,27 +465,27 @@ class TaskBuilder(object): ): staging_prefix = '.staging' if is_staging else '' routes = [ - "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.latest.{}".format( - staging_prefix, self.date.year, self.date.month, self.date.day, variant.abi + "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.latest".format( + staging_prefix, self.date.year, self.date.month, self.date.day ), - "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.revision.{}.{}".format( - staging_prefix, self.date.year, self.date.month, self.date.day, self.commit, variant.abi + "index.project.mobile.fenix.v2{}.performance-test.{}.{}.{}.revision.{}".format( + staging_prefix, self.date.year, self.date.month, self.date.day, self.commit ), - "index.project.mobile.fenix.v2{}.performance-test.latest.{}".format(staging_prefix, variant.abi), + "index.project.mobile.fenix.v2{}.performance-test.latest".format(staging_prefix), ] return self._craft_signing_task( - name='sign: {}'.format(variant.raw), - description='Dep-signing variant {}'.format(variant.raw), + name='sign: {}'.format('forPerformanceTest'), + description='Dep-signing variant {}'.format('forPerformanceTest'), signing_type="dep", assemble_task_id=assemble_task_id, - apk_paths=[DEFAULT_APK_ARTIFACT_LOCATION], + apk_paths=variant.upstream_artifacts(), routes=routes, treeherder={ - 'groupSymbol': variant.build_type, + 'groupSymbol': 'forPerformanceTest', 'jobKind': 'other', 'machine': { - 'platform': variant.platform, + 'platform': 'android-all', }, 'symbol': 'As', 'tier': 1, @@ -553,7 +527,7 @@ class TaskBuilder(object): ) def craft_push_task( - self, signing_task_id, apks, channel, is_staging=False, override_google_play_track=None + self, signing_task_id, apk_paths, channel, is_staging=False, override_google_play_track=None ): payload = { "commit": True, @@ -561,7 +535,7 @@ class TaskBuilder(object): "certificate_alias": 'fenix' if is_staging else 'fenix-{}'.format(channel), "upstreamArtifacts": [ { - "paths": apks, + "paths": apk_paths, "taskId": signing_task_id, "taskType": "signing" } @@ -596,11 +570,11 @@ class TaskBuilder(object): def craft_raptor_tp6m_cold_task(self, for_suite): - def craft_function(signing_task_id, mozharness_task_id, variant, gecko_revision, force_run_on_64_bit_device=False): + def craft_function(signing_task_id, mozharness_task_id, abi, gecko_revision, force_run_on_64_bit_device=False): return self._craft_raptor_task( signing_task_id, mozharness_task_id, - variant, + abi, gecko_revision, name_prefix='raptor tp6m-cold-{}'.format(for_suite), description='Raptor tp6m cold on Fenix', @@ -610,12 +584,12 @@ class TaskBuilder(object): ) return craft_function - def craft_raptor_youtube_playback_task(self, signing_task_id, mozharness_task_id, variant, gecko_revision, + def craft_raptor_youtube_playback_task(self, signing_task_id, mozharness_task_id, abi, gecko_revision, force_run_on_64_bit_device=False): return self._craft_raptor_task( signing_task_id, mozharness_task_id, - variant, + abi, gecko_revision, name_prefix='raptor youtube playback', description='Raptor YouTube Playback on Fenix', @@ -629,7 +603,7 @@ class TaskBuilder(object): self, signing_task_id, mozharness_task_id, - variant, + abi, gecko_revision, name_prefix, description, @@ -638,19 +612,19 @@ class TaskBuilder(object): group_symbol=None, force_run_on_64_bit_device=False, ): - worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or variant.abi == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5' + worker_type = 'gecko-t-bitbar-gw-perf-p2' if force_run_on_64_bit_device or abi == 'aarch64' else 'gecko-t-bitbar-gw-perf-g5' if force_run_on_64_bit_device: treeherder_platform = 'android-hw-p2-8-0-arm7-api-16' - elif variant.abi == 'arm': + elif abi == 'arm': treeherder_platform = 'android-hw-g5-7-0-arm7-api-16' - elif variant.abi == 'aarch64': + elif abi == 'aarch64': treeherder_platform = 'android-hw-p2-8-0-android-aarch64' else: - raise ValueError('Unsupported architecture "{}"'.format(variant.abi)) + raise ValueError('Unsupported architecture "{}"'.format(abi)) - task_name = '{}: {} {}'.format( - name_prefix, variant.raw, '(on 64-bit-device)' if force_run_on_64_bit_device else '' + task_name = '{}: forPerformanceTest {}'.format( + name_prefix, '(on 64-bit-device)' if force_run_on_64_bit_device else '' ) apk_url = '{}/{}/artifacts/{}'.format(_DEFAULT_TASK_URL, signing_task_id, @@ -667,7 +641,7 @@ class TaskBuilder(object): "--download-symbols=ondemand", ]] # Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases - if variant.abi == 'arm' and test_name == 'raptor-youtube-playback': + if abi == 'arm' and test_name == 'raptor-youtube-playback': params_query = '&'.join(ARM_RAPTOR_URL_PARAMS) add_extra_params_option = "--test-url-params={}".format(params_query) command[0].append(add_extra_params_option) @@ -744,16 +718,6 @@ class TaskBuilder(object): ) -def _craft_artifacts_from_variant(variant): - return { - DEFAULT_APK_ARTIFACT_LOCATION: { - 'type': 'file', - 'path': variant.apk_absolute_path(), - 'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), - } - } - - def schedule_task(queue, taskId, task): print("TASK", taskId) print(json.dumps(task, indent=4, separators=(',', ': '))) diff --git a/automation/taskcluster/lib/variant.py b/automation/taskcluster/lib/variant.py index 271abb508..8d1744bbc 100644 --- a/automation/taskcluster/lib/variant.py +++ b/automation/taskcluster/lib/variant.py @@ -1,20 +1,27 @@ -class Variant: - def __init__(self, raw, abi, is_signed, build_type): - self.raw = raw +import taskcluster + + +class VariantApk: + def __init__(self, build_type, abi, engine, file_name): self.abi = abi + self.taskcluster_path = 'public/build/{}/{}/target.apk'.format(abi, engine) + self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name) + + +class Variant: + def __init__(self, name, build_type, apks): + self.name = name self.build_type = build_type - self._is_signed = is_signed - self.for_gradle_command = raw[:1].upper() + raw[1:] - self.platform = 'android-{}-{}'.format(self.abi, self.build_type) + self._apks = apks - def apk_absolute_path(self): - return '/opt/fenix/app/build/outputs/apk/{abi}/{build_type}/app-{abi}-{build_type}{unsigned}.apk'.format( - build_type=self.build_type, - abi=self.abi, - unsigned='' if self._is_signed else '-unsigned', - ) + def artifacts(self): + return { + apk.taskcluster_path: { + 'type': 'file', + 'path': apk.absolute_path, + 'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')), + } for apk in self._apks + } - @staticmethod - def from_values(abi, is_signed, build_type): - raw = abi + build_type[:1].upper() + build_type[1:] - return Variant(raw, abi, is_signed, build_type) + def upstream_artifacts(self): + return [apk.taskcluster_path for apk in self._apks] diff --git a/config/pre-push-recommended.sh b/config/pre-push-recommended.sh index 364dc9f7f..bc8943c84 100755 --- a/config/pre-push-recommended.sh +++ b/config/pre-push-recommended.sh @@ -17,4 +17,4 @@ ./gradlew -q \ ktlint \ detekt \ - app:assembleX86Debug + app:assembleDebug diff --git a/gradle.properties b/gradle.properties index ea86ad3c4..03d546425 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,5 +17,5 @@ android.useAndroidX=true android.enableJetifier=true android.enableR8=true android.enableR8.fullMode=true -android.enableUnitTestBinaryResources=true +android.enableUnitTestBinaryResources=false org.gradle.parallel=false