Bug 1622339 - Move android-sdk installation into a toolchain task (#9394)

.taskcluster.yml

@@ -8,7 +8,7 @@ tasks:
     - $let:
           taskgraph:
               branch: taskgraph
-              revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0
+              revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57
           trustDomain: mobile
       in:
           $let:
@@ -255,13 +255,13 @@ tasks:
                                         $if: 'tasks_for == "action"'
                                         then: >
                                             PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
-                                            taskcluster/scripts/install-sdk.sh &&
+                                            taskcluster/scripts/decision-install-sdk.sh &&
                                             ln -s /builds/worker/artifacts artifacts &&
                                             ~/.local/bin/taskgraph action-callback
                                         else: >
                                             PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
                                             PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
-                                            taskcluster/scripts/install-sdk.sh &&
+                                            taskcluster/scripts/decision-install-sdk.sh &&
                                             ln -s /builds/worker/artifacts artifacts &&
                                             ~/.local/bin/taskgraph decision
                                             --pushlog-id='0'

taskcluster/ci/build/kind.yml

@@ -9,8 +9,27 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
+    # Builds generate multiple APKs with different ABIs. For each APK described
+    # by `gradlew printVariant`, an artifact will be generated. `variant` and
+    # the per-apk config from `printVariant` can be used as substitutions in
+    # `name` and `path`.
+    apk-artifact-template:
+        type: file
+        name: public/build/{abi}/{geckoview_engine}/target.apk
+        path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
     description: Build Fenix from source code.
+    fetches:
+        toolchain:
+            - android-sdk-linux
+    run:
+        using: gradlew
+        use-caches: false
+    run-on-tasks-for: []
     treeherder:
         kind: build
         symbol: B
@@ -21,18 +40,7 @@ job-defaults:
         docker-image: {in-tree: base}
         max-run-time: 7200
         chain-of-trust: true
-    run:
+
-        using: gradlew
-        use-caches: false
-    run-on-tasks-for: []
-    # Builds generate multiple APKs with different ABIs. For each APK described
-    # by `gradlew printVariant`, an artifact will be generated. `variant` and
-    # the per-apk config from `printVariant` can be used as substitutions in
-    # `name` and `path`.
-    apk-artifact-template:
-        type: file
-        name: public/build/{abi}/{geckoview_engine}/target.apk
-        path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
 
 jobs:
     debug:

taskcluster/ci/config.yml

@@ -4,8 +4,10 @@ treeherder:
     group-names:
         'beta': 'Beta-related tasks'
         'betaFennec': 'Beta-related tasks with same APK configuration as Fennec'
+        'Btime': 'Raptor-Browsertime tests'
         'bump': 'Bump dependencies'
         'debug': 'Builds made for testing'
+        'Fetch': 'Fetch and store content'
         'forPerformanceTest': 'Builds made for Raptor and other performance tests'
         'I': 'Docker Image Builds'
         'nightly': 'Nightly-related tasks'
@@ -14,7 +16,7 @@ treeherder:
         'productionFennec': 'Production-related tasks with same APK configuration as Fennec'
         'Rap': 'Raptor tests'
         'Rap-P': 'Raptor power tests'
-        'Btime': 'Raptor-Browsertime tests'
+        'TL': 'Toolchain builds for Linux 64-bits'
 
 task-priority: highest
 

taskcluster/ci/fetch/kind.yml

@@ -0,0 +1,25 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+loader: taskgraph.loader.transform:loader
+
+transforms:
+    - taskgraph.transforms.fetch:transforms
+    - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.task:transforms
+
+job-defaults:
+    docker-image: {in-tree: base}
+
+jobs:
+    android-sdk-3859397:
+        description: Android SDK
+        fetch:
+            type: static-url
+            url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip
+            artifact-name: sdk-tools-linux.zip
+            sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
+            size: 136964098
+        artifact-prefix: mobile/android-sdk
+        fetch-alias: android-sdk

taskcluster/ci/lint/kind.yml

@@ -9,14 +9,17 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
     attributes:
         code-review: true
         retrigger: true
-    worker-type: b-android
+    fetches:
-    worker:
+        toolchain:
-        docker-image: {in-tree: base}
+            - android-sdk-linux
-        max-run-time: 7200
     run:
         use-caches: false
     run-on-tasks-for: [github-pull-request, github-push]
@@ -24,6 +27,10 @@ job-defaults:
         kind: test
         platform: 'lint/opt'
         tier: 1
+    worker-type: b-android
+    worker:
+        docker-image: {in-tree: base}
+        max-run-time: 7200
 
 jobs:
     compare-locales:

taskcluster/ci/test/kind.yml

@@ -9,11 +9,18 @@ transforms:
     - taskgraph.transforms.job:transforms
     - taskgraph.transforms.task:transforms
 
+kind-dependencies:
+    - toolchain
+
+
 job-defaults:
-    description: Test Fenix
-    include-pull-request-number: false
     attributes:
         retrigger: true
+    description: Test Fenix
+    fetches:
+        toolchain:
+            - android-sdk-linux
+    include-pull-request-number: false
     run:
         using: gradlew
         use-caches: false
@@ -31,10 +38,6 @@ jobs:
             build-type: debug
             geckoview-engine: geckoNightly
             code-review: true
-        treeherder:
-            platform: 'android-all/opt'
-            symbol: debug(T)
-            tier: 1
         run:
             gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport']
             post-gradlew:
@@ -43,6 +46,10 @@ jobs:
                 - name: project/mobile/fenix/public-tokens
                   key: codecov
                   path: .cc_token
+        treeherder:
+            platform: 'android-all/opt'
+            symbol: debug(T)
+            tier: 1
         worker:
             artifacts:
                 - name: public/reports/index.html

taskcluster/ci/toolchain/android.yml

@@ -0,0 +1,31 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    run:
+        using: toolchain-script
+    treeherder:
+        kind: build
+        platform: toolchains/opt
+        tier: 1
+    worker-type: b-android
+    worker:
+        docker-image: {in-tree: base}
+        max-run-time: 1800
+
+
+linux64-android-sdk-linux-repack:
+    attributes:
+        artifact_prefix: mobile/android-sdk
+    description: "Android SDK (Linux) repack toolchain build"
+    fetches:
+        fetch:
+            - android-sdk
+    run:
+        script: repack-android-sdk-linux.sh
+        resources: []
+        toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz
+        toolchain-alias: android-sdk-linux
+    treeherder:
+        symbol: TL(android-sdk-linux)

taskcluster/ci/toolchain/gecko-derived.yml

@@ -0,0 +1,58 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+---
+job-defaults:
+    run:
+        using: index-search
+    run-on-projects: []
+    worker-type: always-optimized
+
+
+browsertime:
+    attributes:
+        toolchain-artifact: public/build/browsertime.tar.bz2
+    description: "Browsertime toolchain"
+    run:
+        index-search:
+            - gecko.cache.level-3.toolchains.v3.browsertime.latest
+
+linux64-ffmpeg-4.1.4:
+    attributes:
+        toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
+    description: "FFMPEG fetch"
+    run:
+        index-search:
+            - gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
+
+linux64-geckodriver:
+    attributes:
+        toolchain-artifact: public/build/geckodriver.tar.xz
+    description: "Geckodriver toolchain"
+    run:
+        index-search:
+            - gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
+
+linux64-minidump-stackwalk:
+    attributes:
+        toolchain-artifact: public/build/minidump_stackwalk.tar.xz
+    description: "minidump-stackwalk toolchain"
+    run:
+        index-search:
+            - gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
+
+linux64-node:
+    attributes:
+        toolchain-artifact: public/build/node.tar.xz
+    description: "Node.js toolchain"
+    run:
+        index-search:
+            - gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
+
+visual-metrics:
+    attributes:
+        toolchain-artifact: public/visualmetrics.py
+    description: "Browsertime visual metrics analysis script"
+    run:
+        index-search:
+            - gecko.cache.level-3.content.v1.visual-metrics.latest

taskcluster/ci/toolchain/kind.yml

@@ -1,54 +1,18 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
 loader: taskgraph.loader.transform:loader
+
+kind-dependencies:
+    - fetch
+
 transforms:
     - taskgraph.transforms.job:transforms
+    - taskgraph.transforms.cached_tasks:transforms
     - taskgraph.transforms.task:transforms
 
-job-defaults:
-    worker-type: always-optimized
-    run:
-        using: index-search
 
-jobs:
+jobs-from:
-    browsertime:
+    - android.yml
-        description: "Browsertime toolchain"
+    - gecko-derived.yml
-        attributes:
-            toolchain-artifact: public/build/browsertime.tar.bz2
-        run:
-            index-search:
-                - gecko.cache.level-3.toolchains.v3.browsertime.latest
-    linux64-ffmpeg-4.1.4:
-        description: "FFMPEG fetch"
-        attributes:
-            toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
-        run:
-            index-search:
-                - gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
-    linux64-geckodriver:
-        description: "Geckodriver toolchain"
-        attributes:
-            toolchain-artifact: public/build/geckodriver.tar.xz
-        run:
-            index-search:
-                - gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
-    linux64-minidump-stackwalk:
-        description: "minidump_stackwalk toolchain"
-        attributes:
-            toolchain-artifact: public/build/minidump_stackwalk.tar.xz
-        run:
-            index-search:
-                - gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
-    linux64-node:
-        description: "Node.js toolchain"
-        attributes:
-            toolchain-artifact: public/build/node.tar.xz
-        run:
-            index-search:
-                - gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
-    visual-metrics:
-        description: "Browsertime visual metrics analsyis script"
-        attributes:
-            toolchain-artifact: public/visualmetrics.py
-        run:
-            index-search:
-                - gecko.cache.level-3.content.v1.visual-metrics.latest

taskcluster/docker/base/Dockerfile

@@ -19,14 +19,9 @@ WORKDIR /builds/worker/
 #-- Configuration -----------------------------------------------------------------------------------------------------
 #----------------------------------------------------------------------------------------------------------------------
 
-ENV ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' \
+ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
-    ANDROID_SDK_VERSION='3859397' \
-    ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
-    CURL='curl --location --retry 5' \
-    GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
     LANG='en_US.UTF-8' \
-    TERM='dumb' \
+    TERM='dumb'
-    SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
 
 #----------------------------------------------------------------------------------------------------------------------
 #-- System ------------------------------------------------------------------------------------------------------------
@@ -54,14 +49,6 @@ RUN pip install taskcluster
 
 RUN locale-gen "$LANG"
 
-RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
-    && echo "$ANDROID_SDK_SHA256  $SDK_ZIP_LOCATION" | sha256sum --check \
-    && unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
-    && rm "$SDK_ZIP_LOCATION" \
-    && yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
-    && chown -R worker:worker "$ANDROID_SDK_ROOT"
-
-
 # %include-run-task
 
 ENV SHELL=/bin/bash \

taskcluster/docker/bump/Dockerfile

@@ -5,7 +5,8 @@ MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
 
 USER worker:worker
 
-ENV HUB_ARCHIVE='hub.tgz' \
+ENV CURL='curl --location --retry 5' \
+  HUB_ARCHIVE='hub.tgz' \
   HUB_ROOT='/builds/worker/hub' \
   HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \
   HUB_VERSION='2.14.1'

taskcluster/fenix_taskgraph/job.py

@@ -72,9 +72,11 @@ def configure_gradlew(config, job, taskdesc):
     run = job["run"]
     worker = taskdesc["worker"] = job["worker"]
 
-    worker.setdefault("env", {}).update(
+    worker.setdefault("env", {}).update({
-        {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
+        "ANDROID_SDK_ROOT": path.join(
-    )
+            run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
+        )
+    })
 
     run["command"] = _extract_gradlew_command(run)
     _inject_secrets_scopes(run, taskdesc)

taskcluster/scripts/decision-install-sdk.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+set -ex
+
+export CURL='curl --location --retry 5'
+
+ANDROID_SDK_VERSION='3859397'
+ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
+SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
+
+$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
+echo "$ANDROID_SDK_SHA256  $SDK_ZIP_LOCATION" | sha256sum --check
+unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION"
+rm "$SDK_ZIP_LOCATION"
+yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

taskcluster/scripts/install-sdk.sh

@@ -1,9 +0,0 @@
-#!/bin/bash
-
-set +x
-
-ANDROID_SDK_VERSION=3859397
-
-curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
-unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
-yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

taskcluster/scripts/toolchain/repack-android-sdk-linux.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR
+
+yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
+
+# It's nice to have the build logs include the state of the world upon completion.
+"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list
+
+tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"