diff --git a/.taskcluster.yml b/.taskcluster.yml index ec40ffd89..4eee64d92 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -50,11 +50,6 @@ tasks: else: 1 in: $let: - track: - $if: 'trust_level == 3' - then: 'nightly' - else: 'staging-nightly' - # TODO: revisit once bug 1533314 is done to possibly infer better priorities tasks_priority: highest @@ -128,78 +123,99 @@ tasks: owner: ${user}@users.noreply.github.com source: ${repository}/raw/${head_rev}/.taskcluster.yml in: - $if: 'tasks_for in ["github-pull-request", "github-push"]' - then: - - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' - then: - $let: - pull_request_title: ${event.pull_request.title} - pull_request_number: ${event.pull_request.number} - pull_request_url: ${event.pull_request.html_url} - in: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - ${assume_scope_prefix}:pull-request - payload: - command: - - >- - git fetch ${repository} ${head_branch} - && git config advice.detachedHead false - && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py pull-request - env: - GITHUB_PULL_TITLE: ${pull_request_title} - extra: - treeherder: - symbol: D-PR - metadata: - name: 'Fenix - Decision task (Pull Request #${pull_request_number})' - description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' - - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' - then: + - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]' + then: + $let: + pull_request_title: ${event.pull_request.title} + pull_request_number: ${event.pull_request.number} + pull_request_url: ${event.pull_request.html_url} + in: $mergeDeep: - {$eval: 'default_task_definition'} - scopes: - - ${assume_scope_prefix}:branch:${short_head_branch} + - ${assume_scope_prefix}:pull-request payload: command: - >- git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py push + && python automation/taskcluster/decision_task.py pull-request + env: + GITHUB_PULL_TITLE: ${pull_request_title} extra: treeherder: - symbol: D + symbol: D-PR metadata: - name: Fenix - Decision task - description: Schedules the build and test tasks for Fenix. - else: - - $if: 'tasks_for == "cron"' - then: - $mergeDeep: - - {$eval: 'default_task_definition'} - - scopes: - - $if: 'trust_level == 3' - then: assume:hook-id:project-mobile/fenix-nightly - else: assume:hook-id:project-mobile/fenix-nightly-staging - routes: + name: 'Fenix - Decision task (Pull Request #${pull_request_number})' + description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})' + - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - ${assume_scope_prefix}:branch:${short_head_branch} + payload: + command: + - >- + git fetch ${repository} ${head_branch} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py push + extra: + treeherder: + symbol: D + metadata: + name: Fenix - Decision task + description: Schedules the build and test tasks for Fenix. + - $if: 'tasks_for == "github-release"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - ${assume_scope_prefix}:release + payload: + command: + - >- + git fetch ${repository} refs/tags/${head_rev} + && git config advice.detachedHead false + && git checkout FETCH_HEAD + && python automation/taskcluster/decision_task.py beta ${event.release.tag_name} + extra: + treeherder: + symbol: beta-D + metadata: + name: Fenix Beta Decision Task + description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name} + - $if: 'tasks_for == "cron"' + then: + $mergeDeep: + - {$eval: 'default_task_definition'} + - scopes: + - $if: 'trust_level == 3' + then: assume:hook-id:project-mobile/fenix-nightly + else: assume:hook-id:project-mobile/fenix-nightly-staging + routes: + $if: 'trust_level == 3' + then: - notify.email.fenix-eng-notifications@mozilla.com.on-failed - payload: + payload: + $let: + staging_flag: + $if: 'trust_level == 3' + then: '' + else: '--staging' + in: command: - >- git fetch ${repository} ${head_branch} && git config advice.detachedHead false && git checkout FETCH_HEAD - && python automation/taskcluster/decision_task.py \ - release \ - --nightly \ - --track ${track} - extra: - cron: {$json: {$eval: 'cron'}} - treeherder: - symbol: N - metadata: - name: Fenix Nightly Decision Task - description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + && python automation/taskcluster/decision_task.py nightly ${staging_flag} + extra: + cron: {$json: {$eval: 'cron'}} + treeherder: + symbol: nightly-D + metadata: + name: Fenix Nightly Decision Task + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) diff --git a/README.md b/README.md index 7fa3737c0..7ef8ccbc9 100644 --- a/README.md +++ b/README.md @@ -60,4 +60,4 @@ git push --no-verify This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this - file, You can obtain one at http://mozilla.org/MPL/2.0/ \ No newline at end of file + file, You can obtain one at http://mozilla.org/MPL/2.0/ diff --git a/app/build.gradle b/app/build.gradle index b5881b0b4..5fe6a6ff4 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -10,6 +10,8 @@ apply plugin: 'androidx.navigation.safeargs.kotlin' apply plugin: 'org.mozilla.appservices' +import com.android.build.gradle.internal.tasks.AppPreBuildTask + appservices { defaultConfig { megazord = 'fenix' @@ -25,8 +27,8 @@ android { applicationId "org.mozilla.fenix" minSdkVersion Config.minSdkVersion targetSdkVersion Config.targetSdkVersion - versionCode Config.versionCode - versionName Config.versionName + Config.generateVersionSuffix() + versionCode 1 + versionName Config.generateDebugVersionName() testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" testInstrumentationRunnerArguments clearPackageData: 'true' manifestPlaceholders.isRaptorEnabled = "false" @@ -57,6 +59,7 @@ android { resValue "bool", "IS_NOT_RELEASED", "false" } beta releaseTemplate >> { + applicationIdSuffix ".beta" buildConfigField "boolean", "IS_RELEASED", "true" resValue "bool", "IS_NOT_RELEASED", "false" } @@ -129,10 +132,11 @@ android.applicationVariants.all { variant -> def buildType = variant.buildType.name def versionCode = null + def isReleased = variant.buildType.buildConfigFields['IS_RELEASED']?.value ?: false buildConfigField 'Boolean', 'COLLECTIONS_ENABLED', (buildType != "release").toString() - if (buildType == "nightly") { + if (isReleased) { versionCode = generatedVersionCode // The Google Play Store does not allow multiple APKs for the same app that all have the @@ -150,7 +154,25 @@ android.applicationVariants.all { variant -> }// else variant.flavorName.contains("Arm")) use generated version code variant.outputs.all { - setVersionCodeOverride(versionCode) + versionCodeOverride = versionCode + versionNameOverride = Config.releaseVersionName(project) + } + + // If this is a release build, validate that "versionName" is set + tasks.withType(AppPreBuildTask) { prebuildTask -> + // You can't add a closure to a variant, so we need to look for an early variant-specific type + // of task (AppPreBuildTask is the first) and filter to make sure we're looking at the task for + // this variant that we're currently configuring + if (prebuildTask.variantName != variant.name) { + return + } + + // Append to the task so the first thing it does is run our validation + prebuildTask.doFirst { + if (!project.hasProperty('versionName')) { + throw new RuntimeException("Release builds require the 'versionName' property, e.g.: './gradlew -PversionName=<...> assembleNightly'") + } + } } } @@ -197,7 +219,7 @@ android.applicationVariants.all { variant -> print("Adjust token: ") - if (variant.buildType.buildConfigFields['IS_RELEASED']?.value) { + if (isReleased) { try { def token = new File("${rootDir}/.adjust_token").text.trim() buildConfigField 'String', 'ADJUST_TOKEN', '"' + token + '"' @@ -390,4 +412,4 @@ task printGeckoviewVersions { // Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we // can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best. -apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle' +apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle' \ No newline at end of file diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 1823a5035..a7036dce8 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -9,7 +9,10 @@ Decision task for nightly releases. from __future__ import print_function import argparse +import datetime import os +import re + import taskcluster from lib.gradle import get_build_variants, get_geckoview_versions @@ -94,8 +97,7 @@ def pr_or_push(is_push): return (build_tasks, signing_tasks, other_tasks) -def nightly(track): - is_staging = track == 'staging-nightly' +def release(track, is_staging, version_name): architectures = ['x86', 'arm', 'aarch64'] apk_paths = ["public/target.{}.apk".format(arch) for arch in architectures] @@ -104,12 +106,13 @@ def nightly(track): push_tasks = {} build_task_id = taskcluster.slugId() - build_tasks[build_task_id] = BUILDER.craft_assemble_nightly_task(architectures, is_staging) + build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, track, is_staging, version_name) signing_task_id = taskcluster.slugId() - signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task( + signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task( build_task_id, apk_paths=apk_paths, + track=track, is_staging=is_staging, ) @@ -117,6 +120,7 @@ def nightly(track): push_tasks[push_task_id] = BUILDER.craft_push_task( signing_task_id, apks=apk_paths, + track=track, is_staging=is_staging, ) @@ -132,15 +136,14 @@ if __name__ == "__main__": subparsers.add_parser('pull-request') subparsers.add_parser('push') - release_parser = subparsers.add_parser('release') - release_parser.add_argument('--nightly', action="store_true", default=False) - release_parser.add_argument( - '--track', action="store", choices=['nightly', 'staging-nightly'], required=True - ) + nightly_parser = subparsers.add_parser('nightly') + nightly_parser.add_argument('--staging', action='store_true') + + release_parser = subparsers.add_parser('beta') + release_parser.add_argument('tag') result = parser.parse_args() - command = result.command taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) @@ -148,8 +151,14 @@ if __name__ == "__main__": ordered_groups_of_tasks = pr_or_push(False) elif command == 'push': ordered_groups_of_tasks = pr_or_push(True) - elif command == 'release': - ordered_groups_of_tasks = nightly(result.track) + elif command == 'nightly': + formatted_date = datetime.datetime.now().strftime('%y%V') + ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date)) + elif command == 'beta': + semver = re.compile(r'^\d+\.\d+\.\d+-beta\.\d+$') + if not semver.match(result.tag): + raise ValueError('Github tag must be in beta semver format, e.g.: "1.0.0-beta.0') + ordered_groups_of_tasks = release('beta', False, result.tag) else: raise Exception('Unsupported command "{}"'.format(command)) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 553fa0d20..41224eb10 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -7,10 +7,9 @@ from __future__ import print_function import arrow import datetime import json -import os import taskcluster -from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter +from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter DEFAULT_EXPIRES_IN = '1 year' DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk' @@ -41,26 +40,29 @@ class TaskBuilder(object): self.date = arrow.get(date_string) self.trust_level = trust_level - def craft_assemble_nightly_task(self, architectures, is_staging=False): + def craft_assemble_release_task(self, architectures, track, is_staging, version_name): artifacts = { 'public/target.{}.apk'.format(arch): { "type": 'file', "path": '/opt/fenix/app/build/outputs/apk/' - '{}/nightly/app-{}-nightly-unsigned.apk'.format(arch, arch), + '{arch}/{track}/app-{arch}-{track}-unsigned.apk'.format(arch=arch, track=track), "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)), } for arch in architectures } - sentry_secret = '{}project/mobile/fenix/sentry'.format( - 'garbage/staging/' if is_staging else '' - ) - leanplum_secret = '{}project/mobile/fenix/leanplum'.format( - 'garbage/staging/' if is_staging else '' - ) - adjust_secret = '{}project/mobile/fenix/adjust'.format( - 'garbage/staging/' if is_staging else '' - ) + def secret_index(name): + if is_staging: + return 'garbage/staging/project/mobile/fenix/{}'.format(name) + elif track == 'nightly': + # TODO: Move nightly secrets to "project/mobile/fenix/nightly/..." + return 'project/mobile/fenix/{}'.format(name) + else: + return 'project/mobile/fenix/{}/{}'.format(track, name) + + sentry_secret = secret_index('sentry') + leanplum_secret = secret_index('leanplum') + adjust_secret = secret_index('adjust') pre_gradle_commands = ( 'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format( @@ -69,12 +71,14 @@ class TaskBuilder(object): for secret, key, target_file in ( (sentry_secret, 'dsn', '.sentry_token'), (leanplum_secret, 'production', '.leanplum_token'), - (adjust_secret, 'Greenfield', '.adjust_token'), + (adjust_secret, 'adjust', '.adjust_token'), ) ) + capitalized_track = upper_case_first_letter(track) gradle_commands = ( - './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true clean test assembleNightly', + './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true -PversionName={} clean test assemble{}'.format( + version_name, capitalized_track), ) command = ' && '.join( @@ -89,8 +93,8 @@ class TaskBuilder(object): ] return self._craft_build_ish_task( - name='Build task', - description='Build Fenix from source code', + name='Build {} task'.format(capitalized_track), + description='Build Fenix {} from source code'.format(capitalized_track), command=command, scopes=[ "secrets:get:{}".format(secret) for secret in (sentry_secret, leanplum_secret, adjust_secret) @@ -100,9 +104,9 @@ class TaskBuilder(object): treeherder={ 'jobKind': 'build', 'machine': { - 'platform': 'android-all', + 'platform': 'android-all', }, - 'symbol': 'NA', + 'symbol': '{}-A'.format(track), 'tier': 1, }, ) @@ -175,7 +179,7 @@ class TaskBuilder(object): return self._craft_clean_gradle_task( name='lint', description='Running lint for aarch64 release variant', - gradle_task='lintAarch64Nightly', + gradle_task='lintDebug', treeherder={ 'jobKind': 'test', 'machine': { @@ -277,13 +281,13 @@ class TaskBuilder(object): } return self._craft_default_task_definition( - worker_type='mobile-signing-dep-v1' if signing_type == 'dep-signing' else 'mobile-signing-v1', + worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1', provisioner_id='scriptworker-prov-v1', dependencies=[assemble_task_id], routes=routes, scopes=[ "project:mobile:fenix:releng:signing:format:{}".format(signing_format), - "project:mobile:fenix:releng:signing:cert:{}".format(signing_type), + "project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type), ], name=name, description=description, @@ -365,7 +369,7 @@ class TaskBuilder(object): return self._craft_signing_task( name='sign: {}'.format(variant), description='Dep-signing variant {}'.format(variant), - signing_type="dep-signing", + signing_type="dep", assemble_task_id=assemble_task_id, apk_paths=["public/target.apk"], routes=routes, @@ -380,24 +384,26 @@ class TaskBuilder(object): }, ) - def craft_nightly_signing_task( - self, build_task_id, apk_paths, is_staging=True, + def craft_release_signing_task( + self, build_task_id, apk_paths, track, is_staging=False, ): - index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly' + capitalized_track = upper_case_first_letter(track) + index_release = 'staging.{}'.format(track) if is_staging else track + routes = [ - "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format( + "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format( index_release, self.date.year, self.date.month, self.date.day ), - "index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format( + "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format( index_release, self.date.year, self.date.month, self.date.day, self.commit ), - "index.project.mobile.fenix.{}.nightly.latest".format(index_release), + "index.project.mobile.fenix.v2.{}.latest".format(index_release), ] return self._craft_signing_task( - name="Signing task", - description="Sign release builds of Fenix", - signing_type="dep-signing" if is_staging else "release-signing", + name="Signing {} task".format(capitalized_track), + description="Sign {} builds of Fenix".format(capitalized_track), + signing_type="dep" if is_staging else track, assemble_task_id=build_task_id, apk_paths=apk_paths, routes=routes, @@ -406,17 +412,17 @@ class TaskBuilder(object): 'machine': { 'platform': 'android-all', }, - 'symbol': 'Ns', + 'symbol': '{}-s'.format(track), 'tier': 1, }, ) def craft_push_task( - self, signing_task_id, apks, is_staging=True + self, signing_task_id, apks, track, is_staging=False ): payload = { "commit": True, - "google_play_track": 'nightly', + "google_play_track": track, "upstreamArtifacts": [ { "paths": apks, @@ -444,7 +450,7 @@ class TaskBuilder(object): 'machine': { 'platform': 'android-all', }, - 'symbol': 'gp', + 'symbol': '{}-gp'.format(track), 'tier': 1, }, ) diff --git a/automation/taskcluster/lib/util.py b/automation/taskcluster/lib/util.py index e887af63c..09e5931eb 100644 --- a/automation/taskcluster/lib/util.py +++ b/automation/taskcluster/lib/util.py @@ -9,3 +9,7 @@ def convert_camel_case_into_kebab_case(string): def lower_case_first_letter(string): return '{}{}'.format(string[0].lower(), string[1:]) + + +def upper_case_first_letter(string): + return string[0].upper() + string[1:] diff --git a/buildSrc/src/main/java/Config.kt b/buildSrc/src/main/java/Config.kt index 57c64b746..3aa931c96 100644 --- a/buildSrc/src/main/java/Config.kt +++ b/buildSrc/src/main/java/Config.kt @@ -1,3 +1,5 @@ +import org.gradle.api.Project +import java.lang.RuntimeException import java.text.SimpleDateFormat import java.time.LocalDateTime import java.time.format.DateTimeFormatter @@ -9,22 +11,27 @@ import java.util.Locale * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ object Config { - const val versionCode = 1 - const val versionName = "1.0" - // Synchronized build configuration for all modules const val compileSdkVersion = 28 const val minSdkVersion = 21 const val targetSdkVersion = 28 @JvmStatic - fun generateVersionSuffix(): String { + private fun generateDebugVersionName(): String { val today = Date() // Append the year (2 digits) and week in year (2 digits). This will make it easier to distinguish versions and // identify ancient versions when debugging issues. However this will still keep the same version number during // the week so that we do not end up with a lot of versions in tools like Sentry. As an extra this matches the // sections we use in the changelog (weeks). - return SimpleDateFormat(".yyww", Locale.US).format(today) + return SimpleDateFormat("1.0.yyww", Locale.US).format(today) + } + + @JvmStatic + fun releaseVersionName(project: Project): String { + // This function is called in the configuration phase, before gradle knows which variants we'll use. + // So, validation that "versionName" has been set happens elsewhere (at time of writing, we staple + // validation to tasks of type "AppPreBuildTask" + return if (project.hasProperty("versionName")) project.property("versionName") as String else "" } @JvmStatic