Updates Fenix taskcluster tasks to support beta release (#1893)

* Updates Fenix taskcluster tasks to support beta release

* Throw error if -PversionName isn't set for release builds

* Uses beta secrets for beta

* Improves nightly and beta treeherder symbols

.taskcluster.yml

@@ -50,11 +50,6 @@ tasks:
       else: 1
   in:
     $let:
-      track:
-        $if: 'trust_level == 3'
-        then: 'nightly'
-        else: 'staging-nightly'
-
       # TODO: revisit once bug 1533314 is done to possibly infer better priorities
       tasks_priority: highest
 
@@ -128,78 +123,99 @@ tasks:
             owner: ${user}@users.noreply.github.com
             source: ${repository}/raw/${head_rev}/.taskcluster.yml
       in:
-        $if: 'tasks_for in ["github-pull-request", "github-push"]'
+        - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
-        then:
+          then:
-          - $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
+            $let:
-            then:
+              pull_request_title: ${event.pull_request.title}
-              $let:
+              pull_request_number: ${event.pull_request.number}
-                pull_request_title: ${event.pull_request.title}
+              pull_request_url: ${event.pull_request.html_url}
-                pull_request_number: ${event.pull_request.number}
+            in:
-                pull_request_url: ${event.pull_request.html_url}
-              in:
-                $mergeDeep:
-                  - {$eval: 'default_task_definition'}
-                  - scopes:
-                      - ${assume_scope_prefix}:pull-request
-                    payload:
-                      command:
-                        - >-
-                          git fetch ${repository} ${head_branch}
-                          && git config advice.detachedHead false
-                          && git checkout FETCH_HEAD
-                          && python automation/taskcluster/decision_task.py pull-request
-                      env:
-                        GITHUB_PULL_TITLE: ${pull_request_title}
-                    extra:
-                      treeherder:
-                        symbol: D-PR
-                    metadata:
-                      name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
-                      description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
-          - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
-            then:
               $mergeDeep:
                 - {$eval: 'default_task_definition'}
                 - scopes:
-                    - ${assume_scope_prefix}:branch:${short_head_branch}
+                    - ${assume_scope_prefix}:pull-request
                   payload:
                     command:
                       - >-
                         git fetch ${repository} ${head_branch}
                         && git config advice.detachedHead false
                         && git checkout FETCH_HEAD
-                        && python automation/taskcluster/decision_task.py push
+                        && python automation/taskcluster/decision_task.py pull-request
+                    env:
+                      GITHUB_PULL_TITLE: ${pull_request_title}
                   extra:
                     treeherder:
-                      symbol: D
+                      symbol: D-PR
                   metadata:
-                    name: Fenix - Decision task
+                    name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
-                    description: Schedules the build and test tasks for Fenix.
+                    description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
-        else:
+        - $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
-          - $if: 'tasks_for == "cron"'
+          then:
-            then:
+            $mergeDeep:
-              $mergeDeep:
+              - {$eval: 'default_task_definition'}
-                - {$eval: 'default_task_definition'}
+              - scopes:
-                - scopes:
+                  - ${assume_scope_prefix}:branch:${short_head_branch}
-                    - $if: 'trust_level == 3'
+                payload:
-                      then: assume:hook-id:project-mobile/fenix-nightly
+                  command:
-                      else: assume:hook-id:project-mobile/fenix-nightly-staging
+                    - >-
-                  routes:
+                      git fetch ${repository} ${head_branch}
+                      && git config advice.detachedHead false
+                      && git checkout FETCH_HEAD
+                      && python automation/taskcluster/decision_task.py push
+                extra:
+                  treeherder:
+                    symbol: D
+                metadata:
+                  name: Fenix - Decision task
+                  description: Schedules the build and test tasks for Fenix.
+        - $if: 'tasks_for == "github-release"'
+          then:
+            $mergeDeep:
+              - {$eval: 'default_task_definition'}
+              - scopes:
+                - ${assume_scope_prefix}:release
+                payload:
+                  command:
+                    - >-
+                      git fetch ${repository} refs/tags/${head_rev}
+                      && git config advice.detachedHead false
+                      && git checkout FETCH_HEAD
+                      && python automation/taskcluster/decision_task.py beta ${event.release.tag_name}
+                extra:
+                  treeherder:
+                    symbol: beta-D
+                metadata:
+                  name: Fenix Beta Decision Task
+                  description: Building and releasing Fenix to the beta channel - triggered by release ${event.release.tag_name}
+        - $if: 'tasks_for == "cron"'
+          then:
+            $mergeDeep:
+              - {$eval: 'default_task_definition'}
+              - scopes:
+                  - $if: 'trust_level == 3'
+                    then: assume:hook-id:project-mobile/fenix-nightly
+                    else: assume:hook-id:project-mobile/fenix-nightly-staging
+                routes:
+                  $if: 'trust_level == 3'
+                  then:
                     - notify.email.fenix-eng-notifications@mozilla.com.on-failed
-                  payload:
+                payload:
+                  $let:
+                    staging_flag:
+                      $if: 'trust_level == 3'
+                      then: ''
+                      else: '--staging'
+                  in:
                     command:
                       - >-
                         git fetch ${repository} ${head_branch}
                         && git config advice.detachedHead false
                         && git checkout FETCH_HEAD
-                        && python automation/taskcluster/decision_task.py \
+                        && python automation/taskcluster/decision_task.py nightly ${staging_flag}
-                          release \
+                extra:
-                          --nightly \
+                  cron: {$json: {$eval: 'cron'}}
-                          --track ${track}
+                  treeherder:
-                  extra:
+                    symbol: nightly-D
-                    cron: {$json: {$eval: 'cron'}}
+                metadata:
-                    treeherder:
+                  name: Fenix Nightly Decision Task
-                      symbol: N
+                  description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
-                  metadata:
-                    name: Fenix Nightly Decision Task
-                    description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

README.md

@@ -60,4 +60,4 @@ git push <remote> --no-verify
 
     This Source Code Form is subject to the terms of the Mozilla Public
     License, v. 2.0. If a copy of the MPL was not distributed with this
-    file, You can obtain one at http://mozilla.org/MPL/2.0/
+    file, You can obtain one at http://mozilla.org/MPL/2.0/

app/build.gradle

@@ -10,6 +10,8 @@ apply plugin: 'androidx.navigation.safeargs.kotlin'
 
 apply plugin: 'org.mozilla.appservices'
 
+import com.android.build.gradle.internal.tasks.AppPreBuildTask
+
 appservices {
     defaultConfig {
         megazord = 'fenix'
@@ -25,8 +27,8 @@ android {
         applicationId "org.mozilla.fenix"
         minSdkVersion Config.minSdkVersion
         targetSdkVersion Config.targetSdkVersion
-        versionCode Config.versionCode
+        versionCode 1
-        versionName Config.versionName + Config.generateVersionSuffix()
+        versionName Config.generateDebugVersionName()
         testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
         testInstrumentationRunnerArguments clearPackageData: 'true'
         manifestPlaceholders.isRaptorEnabled = "false"
@@ -57,6 +59,7 @@ android {
             resValue "bool", "IS_NOT_RELEASED", "false"
         }
         beta releaseTemplate >> {
+            applicationIdSuffix ".beta"
             buildConfigField "boolean", "IS_RELEASED", "true"
             resValue "bool", "IS_NOT_RELEASED", "false"
         }
@@ -129,10 +132,11 @@ android.applicationVariants.all { variant ->
 
     def buildType = variant.buildType.name
     def versionCode = null
+    def isReleased = variant.buildType.buildConfigFields['IS_RELEASED']?.value ?: false
 
     buildConfigField 'Boolean', 'COLLECTIONS_ENABLED', (buildType != "release").toString()
 
-    if (buildType == "nightly") {
+    if (isReleased) {
         versionCode = generatedVersionCode
 
         // The Google Play Store does not allow multiple APKs for the same app that all have the
@@ -150,7 +154,25 @@ android.applicationVariants.all { variant ->
         }// else variant.flavorName.contains("Arm")) use generated version code
 
         variant.outputs.all {
-            setVersionCodeOverride(versionCode)
+            versionCodeOverride = versionCode
+            versionNameOverride = Config.releaseVersionName(project)
+        }
+
+        // If this is a release build, validate that "versionName" is set
+        tasks.withType(AppPreBuildTask) { prebuildTask ->
+            // You can't add a closure to a variant, so we need to look for an early variant-specific type
+            // of task (AppPreBuildTask is the first) and filter to make sure we're looking at the task for
+            // this variant that we're currently configuring
+            if (prebuildTask.variantName != variant.name) {
+                return
+            }
+
+            // Append to the task so the first thing it does is run our validation
+            prebuildTask.doFirst {
+                if (!project.hasProperty('versionName')) {
+                    throw new RuntimeException("Release builds require the 'versionName' property, e.g.: './gradlew -PversionName=<...> assembleNightly'")
+                }
+            }
         }
     }
 
@@ -197,7 +219,7 @@ android.applicationVariants.all { variant ->
 
     print("Adjust token: ")
 
-    if (variant.buildType.buildConfigFields['IS_RELEASED']?.value) {
+    if (isReleased) {
         try {
             def token = new File("${rootDir}/.adjust_token").text.trim()
             buildConfigField 'String', 'ADJUST_TOKEN', '"' + token + '"'
@@ -390,4 +412,4 @@ task printGeckoviewVersions {
 
 // Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we
 // can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best.
-apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle'
+apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle'

automation/taskcluster/decision_task.py

@@ -9,7 +9,10 @@ Decision task for nightly releases.
 from __future__ import print_function
 
 import argparse
+import datetime
 import os
+import re
+
 import taskcluster
 
 from lib.gradle import get_build_variants, get_geckoview_versions
@@ -94,8 +97,7 @@ def pr_or_push(is_push):
     return (build_tasks, signing_tasks, other_tasks)
 
 
-def nightly(track):
+def release(track, is_staging, version_name):
-    is_staging = track == 'staging-nightly'
     architectures = ['x86', 'arm', 'aarch64']
     apk_paths = ["public/target.{}.apk".format(arch) for arch in architectures]
 
@@ -104,12 +106,13 @@ def nightly(track):
     push_tasks = {}
 
     build_task_id = taskcluster.slugId()
-    build_tasks[build_task_id] = BUILDER.craft_assemble_nightly_task(architectures, is_staging)
+    build_tasks[build_task_id] = BUILDER.craft_assemble_release_task(architectures, track, is_staging, version_name)
 
     signing_task_id = taskcluster.slugId()
-    signing_tasks[signing_task_id] = BUILDER.craft_nightly_signing_task(
+    signing_tasks[signing_task_id] = BUILDER.craft_release_signing_task(
         build_task_id,
         apk_paths=apk_paths,
+        track=track,
         is_staging=is_staging,
     )
 
@@ -117,6 +120,7 @@ def nightly(track):
     push_tasks[push_task_id] = BUILDER.craft_push_task(
         signing_task_id,
         apks=apk_paths,
+        track=track,
         is_staging=is_staging,
     )
 
@@ -132,15 +136,14 @@ if __name__ == "__main__":
 
     subparsers.add_parser('pull-request')
     subparsers.add_parser('push')
-    release_parser = subparsers.add_parser('release')
 
-    release_parser.add_argument('--nightly', action="store_true", default=False)
+    nightly_parser = subparsers.add_parser('nightly')
-    release_parser.add_argument(
+    nightly_parser.add_argument('--staging', action='store_true')
-        '--track', action="store", choices=['nightly', 'staging-nightly'], required=True
+
-    )
+    release_parser = subparsers.add_parser('beta')
+    release_parser.add_argument('tag')
 
     result = parser.parse_args()
-
     command = result.command
     taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
 
@@ -148,8 +151,14 @@ if __name__ == "__main__":
         ordered_groups_of_tasks = pr_or_push(False)
     elif command == 'push':
         ordered_groups_of_tasks = pr_or_push(True)
-    elif command == 'release':
+    elif command == 'nightly':
-        ordered_groups_of_tasks = nightly(result.track)
+        formatted_date = datetime.datetime.now().strftime('%y%V')
+        ordered_groups_of_tasks = release('nightly', result.staging, '1.0.{}'.format(formatted_date))
+    elif command == 'beta':
+        semver = re.compile(r'^\d+\.\d+\.\d+-beta\.\d+$')
+        if not semver.match(result.tag):
+            raise ValueError('Github tag must be in beta semver format, e.g.: "1.0.0-beta.0')
+        ordered_groups_of_tasks = release('beta', False, result.tag)
     else:
         raise Exception('Unsupported command "{}"'.format(command))
 

automation/taskcluster/lib/tasks.py

@@ -7,10 +7,9 @@ from __future__ import print_function
 import arrow
 import datetime
 import json
-import os
 import taskcluster
 
-from lib.util import convert_camel_case_into_kebab_case, lower_case_first_letter
+from lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
 
 DEFAULT_EXPIRES_IN = '1 year'
 DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
@@ -41,26 +40,29 @@ class TaskBuilder(object):
         self.date = arrow.get(date_string)
         self.trust_level = trust_level
 
-    def craft_assemble_nightly_task(self, architectures, is_staging=False):
+    def craft_assemble_release_task(self, architectures, track, is_staging, version_name):
         artifacts = {
             'public/target.{}.apk'.format(arch): {
                 "type": 'file',
                 "path": '/opt/fenix/app/build/outputs/apk/'
-                        '{}/nightly/app-{}-nightly-unsigned.apk'.format(arch, arch),
+                        '{arch}/{track}/app-{arch}-{track}-unsigned.apk'.format(arch=arch, track=track),
                 "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
             }
             for arch in architectures
         }
 
-        sentry_secret = '{}project/mobile/fenix/sentry'.format(
+        def secret_index(name):
-            'garbage/staging/' if is_staging else ''
+            if is_staging:
-        )
+                return 'garbage/staging/project/mobile/fenix/{}'.format(name)
-        leanplum_secret = '{}project/mobile/fenix/leanplum'.format(
+            elif track == 'nightly':
-            'garbage/staging/' if is_staging else ''
+                # TODO: Move nightly secrets to "project/mobile/fenix/nightly/..."
-        )
+                return 'project/mobile/fenix/{}'.format(name)
-        adjust_secret = '{}project/mobile/fenix/adjust'.format(
+            else:
-            'garbage/staging/' if is_staging else ''
+                return 'project/mobile/fenix/{}/{}'.format(track, name)
-        )
+
+        sentry_secret = secret_index('sentry')
+        leanplum_secret = secret_index('leanplum')
+        adjust_secret = secret_index('adjust')
 
         pre_gradle_commands = (
             'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
@@ -69,12 +71,14 @@ class TaskBuilder(object):
             for secret, key, target_file in (
                 (sentry_secret, 'dsn', '.sentry_token'),
                 (leanplum_secret, 'production', '.leanplum_token'),
-                (adjust_secret, 'Greenfield', '.adjust_token'),
+                (adjust_secret, 'adjust', '.adjust_token'),
             )
         )
 
+        capitalized_track = upper_case_first_letter(track)
         gradle_commands = (
-            './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true clean test assembleNightly',
+            './gradlew --no-daemon -PcrashReports=true -Ptelemetry=true -PversionName={} clean test assemble{}'.format(
+                version_name, capitalized_track),
         )
 
         command = ' && '.join(
@@ -89,8 +93,8 @@ class TaskBuilder(object):
         ]
 
         return self._craft_build_ish_task(
-            name='Build task',
+            name='Build {} task'.format(capitalized_track),
-            description='Build Fenix from source code',
+            description='Build Fenix {} from source code'.format(capitalized_track),
             command=command,
             scopes=[
                 "secrets:get:{}".format(secret) for secret in (sentry_secret, leanplum_secret, adjust_secret)
@@ -100,9 +104,9 @@ class TaskBuilder(object):
             treeherder={
                 'jobKind': 'build',
                 'machine': {
-                  'platform': 'android-all',
+                    'platform': 'android-all',
                 },
-                'symbol': 'NA',
+                'symbol': '{}-A'.format(track),
                 'tier': 1,
             },
         )
@@ -175,7 +179,7 @@ class TaskBuilder(object):
         return self._craft_clean_gradle_task(
             name='lint',
             description='Running lint for aarch64 release variant',
-            gradle_task='lintAarch64Nightly',
+            gradle_task='lintDebug',
             treeherder={
                 'jobKind': 'test',
                 'machine': {
@@ -277,13 +281,13 @@ class TaskBuilder(object):
         }
 
         return self._craft_default_task_definition(
-            worker_type='mobile-signing-dep-v1' if signing_type == 'dep-signing' else 'mobile-signing-v1',
+            worker_type='mobile-signing-dep-v1' if signing_type == 'dep' else 'mobile-signing-v1',
             provisioner_id='scriptworker-prov-v1',
             dependencies=[assemble_task_id],
             routes=routes,
             scopes=[
                 "project:mobile:fenix:releng:signing:format:{}".format(signing_format),
-                "project:mobile:fenix:releng:signing:cert:{}".format(signing_type),
+                "project:mobile:fenix:releng:signing:cert:{}-signing".format(signing_type),
             ],
             name=name,
             description=description,
@@ -365,7 +369,7 @@ class TaskBuilder(object):
         return self._craft_signing_task(
             name='sign: {}'.format(variant),
             description='Dep-signing variant {}'.format(variant),
-            signing_type="dep-signing",
+            signing_type="dep",
             assemble_task_id=assemble_task_id,
             apk_paths=["public/target.apk"],
             routes=routes,
@@ -380,24 +384,26 @@ class TaskBuilder(object):
             },
         )
 
-    def craft_nightly_signing_task(
+    def craft_release_signing_task(
-        self, build_task_id, apk_paths, is_staging=True,
+        self, build_task_id, apk_paths, track, is_staging=False,
     ):
-        index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'
+        capitalized_track = upper_case_first_letter(track)
+        index_release = 'staging.{}'.format(track) if is_staging else track
+
         routes = [
-            "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(
+            "index.project.mobile.fenix.v2.{}.{}.{}.{}.latest".format(
                 index_release, self.date.year, self.date.month, self.date.day
             ),
-            "index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(
+            "index.project.mobile.fenix.v2.{}.{}.{}.{}.revision.{}".format(
                 index_release, self.date.year, self.date.month, self.date.day, self.commit
             ),
-            "index.project.mobile.fenix.{}.nightly.latest".format(index_release),
+            "index.project.mobile.fenix.v2.{}.latest".format(index_release),
         ]
 
         return self._craft_signing_task(
-            name="Signing task",
+            name="Signing {} task".format(capitalized_track),
-            description="Sign release builds of Fenix",
+            description="Sign {} builds of Fenix".format(capitalized_track),
-            signing_type="dep-signing" if is_staging else "release-signing",
+            signing_type="dep" if is_staging else track,
             assemble_task_id=build_task_id,
             apk_paths=apk_paths,
             routes=routes,
@@ -406,17 +412,17 @@ class TaskBuilder(object):
                 'machine': {
                   'platform': 'android-all',
                 },
-                'symbol': 'Ns',
+                'symbol': '{}-s'.format(track),
                 'tier': 1,
             },
         )
 
     def craft_push_task(
-        self, signing_task_id, apks, is_staging=True
+        self, signing_task_id, apks, track, is_staging=False
     ):
         payload = {
             "commit": True,
-            "google_play_track": 'nightly',
+            "google_play_track": track,
             "upstreamArtifacts": [
                 {
                     "paths": apks,
@@ -444,7 +450,7 @@ class TaskBuilder(object):
                 'machine': {
                   'platform': 'android-all',
                 },
-                'symbol': 'gp',
+                'symbol': '{}-gp'.format(track),
                 'tier': 1,
             },
         )

automation/taskcluster/lib/util.py

@@ -9,3 +9,7 @@ def convert_camel_case_into_kebab_case(string):
 
 def lower_case_first_letter(string):
     return '{}{}'.format(string[0].lower(), string[1:])
+
+
+def upper_case_first_letter(string):
+    return string[0].upper() + string[1:]

buildSrc/src/main/java/Config.kt

@@ -1,3 +1,5 @@
+import org.gradle.api.Project
+import java.lang.RuntimeException
 import java.text.SimpleDateFormat
 import java.time.LocalDateTime
 import java.time.format.DateTimeFormatter
@@ -9,22 +11,27 @@ import java.util.Locale
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 object Config {
-    const val versionCode = 1
-    const val versionName = "1.0"
-
     // Synchronized build configuration for all modules
     const val compileSdkVersion = 28
     const val minSdkVersion = 21
     const val targetSdkVersion = 28
 
     @JvmStatic
-    fun generateVersionSuffix(): String {
+    private fun generateDebugVersionName(): String {
         val today = Date()
         // Append the year (2 digits) and week in year (2 digits). This will make it easier to distinguish versions and
         // identify ancient versions when debugging issues. However this will still keep the same version number during
         // the week so that we do not end up with a lot of versions in tools like Sentry. As an extra this matches the
         // sections we use in the changelog (weeks).
-        return SimpleDateFormat(".yyww", Locale.US).format(today)
+        return SimpleDateFormat("1.0.yyww", Locale.US).format(today)
+    }
+
+    @JvmStatic
+    fun releaseVersionName(project: Project): String {
+        // This function is called in the configuration phase, before gradle knows which variants we'll use.
+        // So, validation that "versionName" has been set happens elsewhere (at time of writing, we staple
+        // validation to tasks of type "AppPreBuildTask"
+        return if (project.hasProperty("versionName")) project.property("versionName") as String else ""
     }
 
     @JvmStatic