From 44432d281e822814cb3db74c8ff092144ca0d2b1 Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Wed, 1 Apr 2020 18:29:35 +0200 Subject: [PATCH] Bug 1622339 - Move android-sdk installation into a toolchain task (#9394) --- .taskcluster.yml | 6 +- taskcluster/ci/build/kind.yml | 32 ++++++---- taskcluster/ci/config.yml | 4 +- taskcluster/ci/fetch/kind.yml | 25 ++++++++ taskcluster/ci/lint/kind.yml | 15 +++-- taskcluster/ci/test/kind.yml | 19 ++++-- taskcluster/ci/toolchain/android.yml | 31 ++++++++++ taskcluster/ci/toolchain/gecko-derived.yml | 58 +++++++++++++++++++ taskcluster/ci/toolchain/kind.yml | 58 ++++--------------- taskcluster/docker/base/Dockerfile | 17 +----- taskcluster/docker/bump/Dockerfile | 3 +- taskcluster/fenix_taskgraph/job.py | 8 ++- taskcluster/scripts/decision-install-sdk.sh | 15 +++++ taskcluster/scripts/install-sdk.sh | 9 --- .../toolchain/repack-android-sdk-linux.sh | 10 ++++ 15 files changed, 209 insertions(+), 101 deletions(-) create mode 100644 taskcluster/ci/fetch/kind.yml create mode 100644 taskcluster/ci/toolchain/android.yml create mode 100644 taskcluster/ci/toolchain/gecko-derived.yml create mode 100755 taskcluster/scripts/decision-install-sdk.sh delete mode 100755 taskcluster/scripts/install-sdk.sh create mode 100755 taskcluster/scripts/toolchain/repack-android-sdk-linux.sh diff --git a/.taskcluster.yml b/.taskcluster.yml index 968c9c9c4..5aa266a88 100644 --- a/.taskcluster.yml +++ b/.taskcluster.yml @@ -8,7 +8,7 @@ tasks: - $let: taskgraph: branch: taskgraph - revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0 + revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57 trustDomain: mobile in: $let: @@ -255,13 +255,13 @@ tasks: $if: 'tasks_for == "action"' then: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && - taskcluster/scripts/install-sdk.sh && + taskcluster/scripts/decision-install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph action-callback else: > PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph && PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml && - taskcluster/scripts/install-sdk.sh && + taskcluster/scripts/decision-install-sdk.sh && ln -s /builds/worker/artifacts artifacts && ~/.local/bin/taskgraph decision --pushlog-id='0' diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index bb41c2701..17dddfaa0 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -9,8 +9,27 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain + + job-defaults: + # Builds generate multiple APKs with different ABIs. For each APK described + # by `gradlew printVariant`, an artifact will be generated. `variant` and + # the per-apk config from `printVariant` can be used as substitutions in + # `name` and `path`. + apk-artifact-template: + type: file + name: public/build/{abi}/{geckoview_engine}/target.apk + path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}' description: Build Fenix from source code. + fetches: + toolchain: + - android-sdk-linux + run: + using: gradlew + use-caches: false + run-on-tasks-for: [] treeherder: kind: build symbol: B @@ -21,18 +40,7 @@ job-defaults: docker-image: {in-tree: base} max-run-time: 7200 chain-of-trust: true - run: - using: gradlew - use-caches: false - run-on-tasks-for: [] - # Builds generate multiple APKs with different ABIs. For each APK described - # by `gradlew printVariant`, an artifact will be generated. `variant` and - # the per-apk config from `printVariant` can be used as substitutions in - # `name` and `path`. - apk-artifact-template: - type: file - name: public/build/{abi}/{geckoview_engine}/target.apk - path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}' + jobs: debug: diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 07bd7bde9..5245fa9a7 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -4,8 +4,10 @@ treeherder: group-names: 'beta': 'Beta-related tasks' 'betaFennec': 'Beta-related tasks with same APK configuration as Fennec' + 'Btime': 'Raptor-Browsertime tests' 'bump': 'Bump dependencies' 'debug': 'Builds made for testing' + 'Fetch': 'Fetch and store content' 'forPerformanceTest': 'Builds made for Raptor and other performance tests' 'I': 'Docker Image Builds' 'nightly': 'Nightly-related tasks' @@ -14,7 +16,7 @@ treeherder: 'productionFennec': 'Production-related tasks with same APK configuration as Fennec' 'Rap': 'Raptor tests' 'Rap-P': 'Raptor power tests' - 'Btime': 'Raptor-Browsertime tests' + 'TL': 'Toolchain builds for Linux 64-bits' task-priority: highest diff --git a/taskcluster/ci/fetch/kind.yml b/taskcluster/ci/fetch/kind.yml new file mode 100644 index 000000000..f49abb5f3 --- /dev/null +++ b/taskcluster/ci/fetch/kind.yml @@ -0,0 +1,25 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.fetch:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + docker-image: {in-tree: base} + +jobs: + android-sdk-3859397: + description: Android SDK + fetch: + type: static-url + url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip + artifact-name: sdk-tools-linux.zip + sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0 + size: 136964098 + artifact-prefix: mobile/android-sdk + fetch-alias: android-sdk diff --git a/taskcluster/ci/lint/kind.yml b/taskcluster/ci/lint/kind.yml index fe756b216..d4dfd8885 100644 --- a/taskcluster/ci/lint/kind.yml +++ b/taskcluster/ci/lint/kind.yml @@ -9,14 +9,17 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain + + job-defaults: attributes: code-review: true retrigger: true - worker-type: b-android - worker: - docker-image: {in-tree: base} - max-run-time: 7200 + fetches: + toolchain: + - android-sdk-linux run: use-caches: false run-on-tasks-for: [github-pull-request, github-push] @@ -24,6 +27,10 @@ job-defaults: kind: test platform: 'lint/opt' tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 7200 jobs: compare-locales: diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index 0c12706e9..f676bcbbd 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -9,11 +9,18 @@ transforms: - taskgraph.transforms.job:transforms - taskgraph.transforms.task:transforms +kind-dependencies: + - toolchain + + job-defaults: - description: Test Fenix - include-pull-request-number: false attributes: retrigger: true + description: Test Fenix + fetches: + toolchain: + - android-sdk-linux + include-pull-request-number: false run: using: gradlew use-caches: false @@ -31,10 +38,6 @@ jobs: build-type: debug geckoview-engine: geckoNightly code-review: true - treeherder: - platform: 'android-all/opt' - symbol: debug(T) - tier: 1 run: gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport'] post-gradlew: @@ -43,6 +46,10 @@ jobs: - name: project/mobile/fenix/public-tokens key: codecov path: .cc_token + treeherder: + platform: 'android-all/opt' + symbol: debug(T) + tier: 1 worker: artifacts: - name: public/reports/index.html diff --git a/taskcluster/ci/toolchain/android.yml b/taskcluster/ci/toolchain/android.yml new file mode 100644 index 000000000..e5d0a248d --- /dev/null +++ b/taskcluster/ci/toolchain/android.yml @@ -0,0 +1,31 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: toolchain-script + treeherder: + kind: build + platform: toolchains/opt + tier: 1 + worker-type: b-android + worker: + docker-image: {in-tree: base} + max-run-time: 1800 + + +linux64-android-sdk-linux-repack: + attributes: + artifact_prefix: mobile/android-sdk + description: "Android SDK (Linux) repack toolchain build" + fetches: + fetch: + - android-sdk + run: + script: repack-android-sdk-linux.sh + resources: [] + toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz + toolchain-alias: android-sdk-linux + treeherder: + symbol: TL(android-sdk-linux) diff --git a/taskcluster/ci/toolchain/gecko-derived.yml b/taskcluster/ci/toolchain/gecko-derived.yml new file mode 100644 index 000000000..5595d6629 --- /dev/null +++ b/taskcluster/ci/toolchain/gecko-derived.yml @@ -0,0 +1,58 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +job-defaults: + run: + using: index-search + run-on-projects: [] + worker-type: always-optimized + + +browsertime: + attributes: + toolchain-artifact: public/build/browsertime.tar.bz2 + description: "Browsertime toolchain" + run: + index-search: + - gecko.cache.level-3.toolchains.v3.browsertime.latest + +linux64-ffmpeg-4.1.4: + attributes: + toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz' + description: "FFMPEG fetch" + run: + index-search: + - gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest + +linux64-geckodriver: + attributes: + toolchain-artifact: public/build/geckodriver.tar.xz + description: "Geckodriver toolchain" + run: + index-search: + - gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest + +linux64-minidump-stackwalk: + attributes: + toolchain-artifact: public/build/minidump_stackwalk.tar.xz + description: "minidump-stackwalk toolchain" + run: + index-search: + - gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest + +linux64-node: + attributes: + toolchain-artifact: public/build/node.tar.xz + description: "Node.js toolchain" + run: + index-search: + - gecko.cache.level-3.toolchains.v3.linux64-node-10.latest + +visual-metrics: + attributes: + toolchain-artifact: public/visualmetrics.py + description: "Browsertime visual metrics analysis script" + run: + index-search: + - gecko.cache.level-3.content.v1.visual-metrics.latest diff --git a/taskcluster/ci/toolchain/kind.yml b/taskcluster/ci/toolchain/kind.yml index a5b3442a5..8a9ec803c 100644 --- a/taskcluster/ci/toolchain/kind.yml +++ b/taskcluster/ci/toolchain/kind.yml @@ -1,54 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. --- loader: taskgraph.loader.transform:loader + +kind-dependencies: + - fetch + transforms: - taskgraph.transforms.job:transforms + - taskgraph.transforms.cached_tasks:transforms - taskgraph.transforms.task:transforms -job-defaults: - worker-type: always-optimized - run: - using: index-search -jobs: - browsertime: - description: "Browsertime toolchain" - attributes: - toolchain-artifact: public/build/browsertime.tar.bz2 - run: - index-search: - - gecko.cache.level-3.toolchains.v3.browsertime.latest - linux64-ffmpeg-4.1.4: - description: "FFMPEG fetch" - attributes: - toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz' - run: - index-search: - - gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest - linux64-geckodriver: - description: "Geckodriver toolchain" - attributes: - toolchain-artifact: public/build/geckodriver.tar.xz - run: - index-search: - - gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest - linux64-minidump-stackwalk: - description: "minidump_stackwalk toolchain" - attributes: - toolchain-artifact: public/build/minidump_stackwalk.tar.xz - run: - index-search: - - gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest - linux64-node: - description: "Node.js toolchain" - attributes: - toolchain-artifact: public/build/node.tar.xz - run: - index-search: - - gecko.cache.level-3.toolchains.v3.linux64-node-10.latest - visual-metrics: - description: "Browsertime visual metrics analsyis script" - attributes: - toolchain-artifact: public/visualmetrics.py - run: - index-search: - - gecko.cache.level-3.content.v1.visual-metrics.latest +jobs-from: + - android.yml + - gecko-derived.yml diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile index ed7d80056..3f70100c8 100644 --- a/taskcluster/docker/base/Dockerfile +++ b/taskcluster/docker/base/Dockerfile @@ -19,14 +19,9 @@ WORKDIR /builds/worker/ #-- Configuration ----------------------------------------------------------------------------------------------------- #---------------------------------------------------------------------------------------------------------------------- -ENV ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' \ - ANDROID_SDK_VERSION='3859397' \ - ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \ - CURL='curl --location --retry 5' \ - GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ +ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \ LANG='en_US.UTF-8' \ - TERM='dumb' \ - SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" + TERM='dumb' #---------------------------------------------------------------------------------------------------------------------- #-- System ------------------------------------------------------------------------------------------------------------ @@ -54,14 +49,6 @@ RUN pip install taskcluster RUN locale-gen "$LANG" -RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \ - && echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check \ - && unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \ - && rm "$SDK_ZIP_LOCATION" \ - && yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \ - && chown -R worker:worker "$ANDROID_SDK_ROOT" - - # %include-run-task ENV SHELL=/bin/bash \ diff --git a/taskcluster/docker/bump/Dockerfile b/taskcluster/docker/bump/Dockerfile index 7b18080a4..af4873db8 100644 --- a/taskcluster/docker/bump/Dockerfile +++ b/taskcluster/docker/bump/Dockerfile @@ -5,7 +5,8 @@ MAINTAINER Johan Lorenzo USER worker:worker -ENV HUB_ARCHIVE='hub.tgz' \ +ENV CURL='curl --location --retry 5' \ + HUB_ARCHIVE='hub.tgz' \ HUB_ROOT='/builds/worker/hub' \ HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \ HUB_VERSION='2.14.1' diff --git a/taskcluster/fenix_taskgraph/job.py b/taskcluster/fenix_taskgraph/job.py index 1b98d7e8b..514dbdbaa 100644 --- a/taskcluster/fenix_taskgraph/job.py +++ b/taskcluster/fenix_taskgraph/job.py @@ -72,9 +72,11 @@ def configure_gradlew(config, job, taskdesc): run = job["run"] worker = taskdesc["worker"] = job["worker"] - worker.setdefault("env", {}).update( - {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")} - ) + worker.setdefault("env", {}).update({ + "ANDROID_SDK_ROOT": path.join( + run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux" + ) + }) run["command"] = _extract_gradlew_command(run) _inject_secrets_scopes(run, taskdesc) diff --git a/taskcluster/scripts/decision-install-sdk.sh b/taskcluster/scripts/decision-install-sdk.sh new file mode 100755 index 000000000..9094d9609 --- /dev/null +++ b/taskcluster/scripts/decision-install-sdk.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -ex + +export CURL='curl --location --retry 5' + +ANDROID_SDK_VERSION='3859397' +ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' +SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip" + +$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" +echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check +unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" +rm "$SDK_ZIP_LOCATION" +yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses diff --git a/taskcluster/scripts/install-sdk.sh b/taskcluster/scripts/install-sdk.sh deleted file mode 100755 index 7a431a34a..000000000 --- a/taskcluster/scripts/install-sdk.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set +x - -ANDROID_SDK_VERSION=3859397 - -curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" -unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip" -yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses diff --git a/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh new file mode 100755 index 000000000..9eb108709 --- /dev/null +++ b/taskcluster/scripts/toolchain/repack-android-sdk-linux.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR + +yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses + +# It's nice to have the build logs include the state of the world upon completion. +"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list + +tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"