1
0
Fork 0

Bug 1622339 - Move android-sdk installation into a toolchain task (#9394)

master
Johan Lorenzo 2020-04-01 18:29:35 +02:00 committed by GitHub
parent 24596fd3cd
commit 44432d281e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 209 additions and 101 deletions

View File

@ -8,7 +8,7 @@ tasks:
- $let:
taskgraph:
branch: taskgraph
revision: a9a010487bf10fca6dc34ba53f40d34a8ab51fc0
revision: f949fb56d96b9509b071ba55853f2d60e8dbbf57
trustDomain: mobile
in:
$let:
@ -255,13 +255,13 @@ tasks:
$if: 'tasks_for == "action"'
then: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
taskcluster/scripts/install-sdk.sh &&
taskcluster/scripts/decision-install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph action-callback
else: >
PIP_IGNORE_INSTALLED=0 pip install --user /builds/worker/checkouts/taskgraph &&
PIP_IGNORE_INSTALLED=0 pip install --user arrow taskcluster pyyaml &&
taskcluster/scripts/install-sdk.sh &&
taskcluster/scripts/decision-install-sdk.sh &&
ln -s /builds/worker/artifacts artifacts &&
~/.local/bin/taskgraph decision
--pushlog-id='0'

View File

@ -9,8 +9,27 @@ transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults:
# Builds generate multiple APKs with different ABIs. For each APK described
# by `gradlew printVariant`, an artifact will be generated. `variant` and
# the per-apk config from `printVariant` can be used as substitutions in
# `name` and `path`.
apk-artifact-template:
type: file
name: public/build/{abi}/{geckoview_engine}/target.apk
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
description: Build Fenix from source code.
fetches:
toolchain:
- android-sdk-linux
run:
using: gradlew
use-caches: false
run-on-tasks-for: []
treeherder:
kind: build
symbol: B
@ -21,18 +40,7 @@ job-defaults:
docker-image: {in-tree: base}
max-run-time: 7200
chain-of-trust: true
run:
using: gradlew
use-caches: false
run-on-tasks-for: []
# Builds generate multiple APKs with different ABIs. For each APK described
# by `gradlew printVariant`, an artifact will be generated. `variant` and
# the per-apk config from `printVariant` can be used as substitutions in
# `name` and `path`.
apk-artifact-template:
type: file
name: public/build/{abi}/{geckoview_engine}/target.apk
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
jobs:
debug:

View File

@ -4,8 +4,10 @@ treeherder:
group-names:
'beta': 'Beta-related tasks'
'betaFennec': 'Beta-related tasks with same APK configuration as Fennec'
'Btime': 'Raptor-Browsertime tests'
'bump': 'Bump dependencies'
'debug': 'Builds made for testing'
'Fetch': 'Fetch and store content'
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
'I': 'Docker Image Builds'
'nightly': 'Nightly-related tasks'
@ -14,7 +16,7 @@ treeherder:
'productionFennec': 'Production-related tasks with same APK configuration as Fennec'
'Rap': 'Raptor tests'
'Rap-P': 'Raptor power tests'
'Btime': 'Raptor-Browsertime tests'
'TL': 'Toolchain builds for Linux 64-bits'
task-priority: highest

View File

@ -0,0 +1,25 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
transforms:
- taskgraph.transforms.fetch:transforms
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
job-defaults:
docker-image: {in-tree: base}
jobs:
android-sdk-3859397:
description: Android SDK
fetch:
type: static-url
url: https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip
artifact-name: sdk-tools-linux.zip
sha256: 444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0
size: 136964098
artifact-prefix: mobile/android-sdk
fetch-alias: android-sdk

View File

@ -9,14 +9,17 @@ transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults:
attributes:
code-review: true
retrigger: true
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 7200
fetches:
toolchain:
- android-sdk-linux
run:
use-caches: false
run-on-tasks-for: [github-pull-request, github-push]
@ -24,6 +27,10 @@ job-defaults:
kind: test
platform: 'lint/opt'
tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 7200
jobs:
compare-locales:

View File

@ -9,11 +9,18 @@ transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.task:transforms
kind-dependencies:
- toolchain
job-defaults:
description: Test Fenix
include-pull-request-number: false
attributes:
retrigger: true
description: Test Fenix
fetches:
toolchain:
- android-sdk-linux
include-pull-request-number: false
run:
using: gradlew
use-caches: false
@ -31,10 +38,6 @@ jobs:
build-type: debug
geckoview-engine: geckoNightly
code-review: true
treeherder:
platform: 'android-all/opt'
symbol: debug(T)
tier: 1
run:
gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport']
post-gradlew:
@ -43,6 +46,10 @@ jobs:
- name: project/mobile/fenix/public-tokens
key: codecov
path: .cc_token
treeherder:
platform: 'android-all/opt'
symbol: debug(T)
tier: 1
worker:
artifacts:
- name: public/reports/index.html

View File

@ -0,0 +1,31 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
run:
using: toolchain-script
treeherder:
kind: build
platform: toolchains/opt
tier: 1
worker-type: b-android
worker:
docker-image: {in-tree: base}
max-run-time: 1800
linux64-android-sdk-linux-repack:
attributes:
artifact_prefix: mobile/android-sdk
description: "Android SDK (Linux) repack toolchain build"
fetches:
fetch:
- android-sdk
run:
script: repack-android-sdk-linux.sh
resources: []
toolchain-artifact: mobile/android-sdk/android-sdk-linux.tar.xz
toolchain-alias: android-sdk-linux
treeherder:
symbol: TL(android-sdk-linux)

View File

@ -0,0 +1,58 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
job-defaults:
run:
using: index-search
run-on-projects: []
worker-type: always-optimized
browsertime:
attributes:
toolchain-artifact: public/build/browsertime.tar.bz2
description: "Browsertime toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.browsertime.latest
linux64-ffmpeg-4.1.4:
attributes:
toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
description: "FFMPEG fetch"
run:
index-search:
- gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
linux64-geckodriver:
attributes:
toolchain-artifact: public/build/geckodriver.tar.xz
description: "Geckodriver toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
linux64-minidump-stackwalk:
attributes:
toolchain-artifact: public/build/minidump_stackwalk.tar.xz
description: "minidump-stackwalk toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
linux64-node:
attributes:
toolchain-artifact: public/build/node.tar.xz
description: "Node.js toolchain"
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
visual-metrics:
attributes:
toolchain-artifact: public/visualmetrics.py
description: "Browsertime visual metrics analysis script"
run:
index-search:
- gecko.cache.level-3.content.v1.visual-metrics.latest

View File

@ -1,54 +1,18 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
---
loader: taskgraph.loader.transform:loader
kind-dependencies:
- fetch
transforms:
- taskgraph.transforms.job:transforms
- taskgraph.transforms.cached_tasks:transforms
- taskgraph.transforms.task:transforms
job-defaults:
worker-type: always-optimized
run:
using: index-search
jobs:
browsertime:
description: "Browsertime toolchain"
attributes:
toolchain-artifact: public/build/browsertime.tar.bz2
run:
index-search:
- gecko.cache.level-3.toolchains.v3.browsertime.latest
linux64-ffmpeg-4.1.4:
description: "FFMPEG fetch"
attributes:
toolchain-artifact: 'public/ffmpeg-4.1.4-i686-static.tar.xz'
run:
index-search:
- gecko.cache.level-3.content.v1.linux64-ffmpeg-4.1.4.latest
linux64-geckodriver:
description: "Geckodriver toolchain"
attributes:
toolchain-artifact: public/build/geckodriver.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-geckodriver.latest
linux64-minidump-stackwalk:
description: "minidump_stackwalk toolchain"
attributes:
toolchain-artifact: public/build/minidump_stackwalk.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-minidump-stackwalk.latest
linux64-node:
description: "Node.js toolchain"
attributes:
toolchain-artifact: public/build/node.tar.xz
run:
index-search:
- gecko.cache.level-3.toolchains.v3.linux64-node-10.latest
visual-metrics:
description: "Browsertime visual metrics analsyis script"
attributes:
toolchain-artifact: public/visualmetrics.py
run:
index-search:
- gecko.cache.level-3.content.v1.visual-metrics.latest
jobs-from:
- android.yml
- gecko-derived.yml

View File

@ -19,14 +19,9 @@ WORKDIR /builds/worker/
#-- Configuration -----------------------------------------------------------------------------------------------------
#----------------------------------------------------------------------------------------------------------------------
ENV ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0' \
ANDROID_SDK_VERSION='3859397' \
ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
CURL='curl --location --retry 5' \
GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
ENV GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
LANG='en_US.UTF-8' \
TERM='dumb' \
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
TERM='dumb'
#----------------------------------------------------------------------------------------------------------------------
#-- System ------------------------------------------------------------------------------------------------------------
@ -54,14 +49,6 @@ RUN pip install taskcluster
RUN locale-gen "$LANG"
RUN $CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
&& echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check \
&& unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
&& rm "$SDK_ZIP_LOCATION" \
&& yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
&& chown -R worker:worker "$ANDROID_SDK_ROOT"
# %include-run-task
ENV SHELL=/bin/bash \

View File

@ -5,7 +5,8 @@ MAINTAINER Johan Lorenzo <jlorenzo+tc@mozilla.com>
USER worker:worker
ENV HUB_ARCHIVE='hub.tgz' \
ENV CURL='curl --location --retry 5' \
HUB_ARCHIVE='hub.tgz' \
HUB_ROOT='/builds/worker/hub' \
HUB_SHA256='734733c9d807715a4ec26ccce0f9987bd19f1c3f84dd35e56451711766930ef0' \
HUB_VERSION='2.14.1'

View File

@ -72,9 +72,11 @@ def configure_gradlew(config, job, taskdesc):
run = job["run"]
worker = taskdesc["worker"] = job["worker"]
worker.setdefault("env", {}).update(
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
worker.setdefault("env", {}).update({
"ANDROID_SDK_ROOT": path.join(
run["workdir"], worker["env"]["MOZ_FETCHES_DIR"], "android-sdk-linux"
)
})
run["command"] = _extract_gradlew_command(run)
_inject_secrets_scopes(run, taskdesc)

View File

@ -0,0 +1,15 @@
#!/bin/bash
set -ex
export CURL='curl --location --retry 5'
ANDROID_SDK_VERSION='3859397'
ANDROID_SDK_SHA256='444e22ce8ca0f67353bda4b85175ed3731cae3ffa695ca18119cbacef1c1bea0'
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
$CURL --output "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
echo "$ANDROID_SDK_SHA256 $SDK_ZIP_LOCATION" | sha256sum --check
unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION"
rm "$SDK_ZIP_LOCATION"
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

View File

@ -1,9 +0,0 @@
#!/bin/bash
set +x
ANDROID_SDK_VERSION=3859397
curl -o "$HOME/sdk-tools-linux.zip" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip"
unzip -d "$ANDROID_SDK_ROOT" "$HOME/sdk-tools-linux.zip"
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses

View File

@ -0,0 +1,10 @@
#!/bin/bash
export ANDROID_SDK_ROOT=$MOZ_FETCHES_DIR
yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses
# It's nice to have the build logs include the state of the world upon completion.
"${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --list
tar cf - -C "$ANDROID_SDK_ROOT" . --transform 's,^\./,android-sdk-linux/,' | xz > "$UPLOAD_DIR/android-sdk-linux.tar.xz"