Bug 1580778 - Migrate lint and ui-test tasks to taskgraph (#5430)
parent
057451178c
commit
c47f0f57ce
|
@ -48,14 +48,7 @@ fi
|
||||||
JAVA_BIN="/usr/bin/java"
|
JAVA_BIN="/usr/bin/java"
|
||||||
PATH_TEST="./automation/taskcluster/androidTest"
|
PATH_TEST="./automation/taskcluster/androidTest"
|
||||||
PATH_APK="./app/build/outputs/apk/geckoNightly/debug"
|
PATH_APK="./app/build/outputs/apk/geckoNightly/debug"
|
||||||
FLANK_BIN="/build/test-tools/flank.jar"
|
FLANK_BIN="/builds/worker/test-tools/flank.jar"
|
||||||
|
|
||||||
echo
|
|
||||||
echo "RETRIEVE SERVICE ACCT TOKEN"
|
|
||||||
echo
|
|
||||||
python automation/taskcluster/helper/get-secret.py --json -s project/mobile/fenix/firebase -k firebaseToken -f $GOOGLE_APPLICATION_CREDENTIALS
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "ACTIVATE SERVICE ACCT"
|
echo "ACTIVATE SERVICE ACCT"
|
||||||
|
|
|
@ -33,15 +33,6 @@ def pr(builder):
|
||||||
tasks.append(builder.craft_assemble_pr_task(variant))
|
tasks.append(builder.craft_assemble_pr_task(variant))
|
||||||
tasks.append(builder.craft_test_pr_task(variant))
|
tasks.append(builder.craft_test_pr_task(variant))
|
||||||
|
|
||||||
for craft_function in (
|
|
||||||
builder.craft_detekt_task,
|
|
||||||
builder.craft_ktlint_task,
|
|
||||||
builder.craft_lint_task,
|
|
||||||
builder.craft_compare_locales_task,
|
|
||||||
builder.craft_ui_tests_task,
|
|
||||||
):
|
|
||||||
tasks.append(craft_function())
|
|
||||||
|
|
||||||
for task in tasks:
|
for task in tasks:
|
||||||
task['attributes']['code-review'] = True
|
task['attributes']['code-review'] = True
|
||||||
|
|
||||||
|
|
|
@ -1,3 +1,5 @@
|
||||||
|
#!/usr/bin/env python
|
||||||
|
|
||||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
@ -28,7 +30,17 @@ def write_secret_to_file(path, data, key, base64decode=False, json_secret=False,
|
||||||
|
|
||||||
|
|
||||||
def fetch_secret_from_taskcluster(name):
|
def fetch_secret_from_taskcluster(name):
|
||||||
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
|
try:
|
||||||
|
secrets = taskcluster.Secrets({
|
||||||
|
# BaseUrl is still needed for tasks that haven't migrated to taskgraph yet.
|
||||||
|
'baseUrl': 'http://taskcluster/secrets/v1',
|
||||||
|
})
|
||||||
|
except taskcluster.exceptions.TaskclusterFailure:
|
||||||
|
# taskcluster library >=5 errors out when `baseUrl` is used
|
||||||
|
secrets = taskcluster.Secrets({
|
||||||
|
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
|
||||||
|
})
|
||||||
|
|
||||||
return secrets.get(name)
|
return secrets.get(name)
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -156,13 +156,35 @@ class TaskBuilder(object):
|
||||||
|
|
||||||
def craft_test_pr_task(self, variant):
|
def craft_test_pr_task(self, variant):
|
||||||
# upload coverage only once, if the variant is arm64
|
# upload coverage only once, if the variant is arm64
|
||||||
test_gradle_command = \
|
secret_index = 'project/mobile/fenix/public-tokens'
|
||||||
'-Pcoverage jacocoGeckoNightlyDebugTestReport && automation/taskcluster/upload_coverage_report.sh'
|
pre_gradle_commands = (
|
||||||
|
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
|
||||||
|
secret_index, key, target_file
|
||||||
|
)
|
||||||
|
for key, target_file in (
|
||||||
|
('codecov', '.cc_token'),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
return self._craft_clean_gradle_task(
|
gradle_commands = (
|
||||||
|
'./gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport',
|
||||||
|
)
|
||||||
|
|
||||||
|
post_gradle_commands = (
|
||||||
|
'automation/taskcluster/upload_coverage_report.sh',
|
||||||
|
)
|
||||||
|
|
||||||
|
command = ' && '.join(
|
||||||
|
cmd
|
||||||
|
for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands)
|
||||||
|
for cmd in commands
|
||||||
|
if cmd
|
||||||
|
)
|
||||||
|
|
||||||
|
return self._craft_build_ish_task(
|
||||||
name='test: {}'.format(variant.name),
|
name='test: {}'.format(variant.name),
|
||||||
description='Building and testing variant {}'.format(variant.name),
|
description='Building and testing variant {}'.format(variant.name),
|
||||||
gradle_task=test_gradle_command,
|
command=command,
|
||||||
treeherder={
|
treeherder={
|
||||||
'groupSymbol': variant.build_type,
|
'groupSymbol': variant.build_type,
|
||||||
'jobKind': 'test',
|
'jobKind': 'test',
|
||||||
|
@ -173,106 +195,10 @@ class TaskBuilder(object):
|
||||||
'tier': 1,
|
'tier': 1,
|
||||||
},
|
},
|
||||||
scopes=[
|
scopes=[
|
||||||
'secrets:get:project/mobile/fenix/public-tokens'
|
'secrets:get:{}'.format(secret_index)
|
||||||
]
|
]
|
||||||
)
|
)
|
||||||
|
|
||||||
def craft_ui_tests_task(self):
|
|
||||||
artifacts = {
|
|
||||||
"public": {
|
|
||||||
"type": "directory",
|
|
||||||
"path": "/build/fenix/results",
|
|
||||||
"expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
env_vars = {
|
|
||||||
"GOOGLE_PROJECT": "moz-fenix",
|
|
||||||
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
|
|
||||||
}
|
|
||||||
|
|
||||||
gradle_commands = (
|
|
||||||
'./gradlew --no-daemon clean assemble assembleAndroidTest',
|
|
||||||
)
|
|
||||||
|
|
||||||
test_commands = (
|
|
||||||
'automation/taskcluster/androidTest/ui-test.sh x86 -1',
|
|
||||||
)
|
|
||||||
|
|
||||||
command = ' && '.join(
|
|
||||||
cmd
|
|
||||||
for commands in (gradle_commands, test_commands)
|
|
||||||
for cmd in commands
|
|
||||||
if cmd
|
|
||||||
)
|
|
||||||
|
|
||||||
treeherder = {
|
|
||||||
'jobKind': 'test',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'ui-test',
|
|
||||||
},
|
|
||||||
'symbol': 'ui-test',
|
|
||||||
'tier': 2,
|
|
||||||
}
|
|
||||||
|
|
||||||
return self._craft_build_ish_task(
|
|
||||||
name='Fenix - UI test',
|
|
||||||
description='Execute Gradle tasks for UI tests',
|
|
||||||
command=command,
|
|
||||||
scopes=[
|
|
||||||
'secrets:get:project/mobile/fenix/firebase'
|
|
||||||
],
|
|
||||||
artifacts=artifacts,
|
|
||||||
env_vars=env_vars,
|
|
||||||
treeherder=treeherder,
|
|
||||||
)
|
|
||||||
|
|
||||||
def craft_detekt_task(self):
|
|
||||||
return self._craft_clean_gradle_task(
|
|
||||||
name='detekt',
|
|
||||||
description='Running detekt code quality checks',
|
|
||||||
gradle_task='detekt',
|
|
||||||
treeherder={
|
|
||||||
'jobKind': 'test',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'lint',
|
|
||||||
},
|
|
||||||
'symbol': 'detekt',
|
|
||||||
'tier': 1,
|
|
||||||
}
|
|
||||||
|
|
||||||
)
|
|
||||||
|
|
||||||
def craft_ktlint_task(self):
|
|
||||||
return self._craft_clean_gradle_task(
|
|
||||||
name='ktlint',
|
|
||||||
description='Running ktlint code quality checks',
|
|
||||||
gradle_task='ktlint',
|
|
||||||
treeherder={
|
|
||||||
'jobKind': 'test',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'lint',
|
|
||||||
},
|
|
||||||
'symbol': 'ktlint',
|
|
||||||
'tier': 1,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def craft_lint_task(self):
|
|
||||||
return self._craft_clean_gradle_task(
|
|
||||||
name='lint',
|
|
||||||
description='Running lint for aarch64 release variant',
|
|
||||||
gradle_task='lintDebug',
|
|
||||||
treeherder={
|
|
||||||
'jobKind': 'test',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'lint',
|
|
||||||
},
|
|
||||||
'symbol': 'lint',
|
|
||||||
'tier': 1,
|
|
||||||
},
|
|
||||||
)
|
|
||||||
|
|
||||||
def _craft_clean_gradle_task(
|
def _craft_clean_gradle_task(
|
||||||
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
|
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
|
||||||
):
|
):
|
||||||
|
@ -286,24 +212,6 @@ class TaskBuilder(object):
|
||||||
scopes=scopes,
|
scopes=scopes,
|
||||||
)
|
)
|
||||||
|
|
||||||
def craft_compare_locales_task(self):
|
|
||||||
return self._craft_build_ish_task(
|
|
||||||
name='compare-locales',
|
|
||||||
description='Validate strings.xml with compare-locales',
|
|
||||||
command=(
|
|
||||||
'pip install "compare-locales>=5.0.2,<6.0" && '
|
|
||||||
'compare-locales --validate l10n.toml .'
|
|
||||||
),
|
|
||||||
treeherder={
|
|
||||||
'jobKind': 'test',
|
|
||||||
'machine': {
|
|
||||||
'platform': 'lint',
|
|
||||||
},
|
|
||||||
'symbol': 'compare-locale',
|
|
||||||
'tier': 2,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
def _craft_build_ish_task(
|
def _craft_build_ish_task(
|
||||||
self, name, description, command, dependencies=None, artifacts=None,
|
self, name, description, command, dependencies=None, artifacts=None,
|
||||||
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
|
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
|
||||||
|
|
|
@ -5,12 +5,6 @@
|
||||||
# If a command fails then do not proceed and fail this script too.
|
# If a command fails then do not proceed and fail this script too.
|
||||||
set -ex
|
set -ex
|
||||||
|
|
||||||
# Get token for uploading to codecov and append it to codecov.yml
|
|
||||||
python automation/taskcluster/helper/get-secret.py \
|
|
||||||
-s project/mobile/fenix/public-tokens \
|
|
||||||
-k codecov \
|
|
||||||
-f .cc_token \
|
|
||||||
|
|
||||||
# Set some environment variables that will help codecov detect the CI
|
# Set some environment variables that will help codecov detect the CI
|
||||||
export CI_BUILD_URL="https://tools.taskcluster.net/tasks/$TASK_ID"
|
export CI_BUILD_URL="https://tools.taskcluster.net/tasks/$TASK_ID"
|
||||||
|
|
||||||
|
|
|
@ -16,3 +16,6 @@ jobs:
|
||||||
nimbledroid:
|
nimbledroid:
|
||||||
parent: base
|
parent: base
|
||||||
symbol: I(nimbledroid)
|
symbol: I(nimbledroid)
|
||||||
|
ui-tests:
|
||||||
|
parent: base
|
||||||
|
symbol: I(ui-tests)
|
||||||
|
|
|
@ -0,0 +1,44 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
---
|
||||||
|
|
||||||
|
loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
- taskgraph.transforms.job:transforms
|
||||||
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
|
job-defaults:
|
||||||
|
attributes:
|
||||||
|
code-review: true
|
||||||
|
worker-type: b-android
|
||||||
|
worker:
|
||||||
|
docker-image: {in-tree: base}
|
||||||
|
max-run-time: 7200
|
||||||
|
run:
|
||||||
|
use-caches: false
|
||||||
|
run-on-tasks-for: [github-pull-request, github-push]
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
detekt:
|
||||||
|
description: 'Running detekt over all modules'
|
||||||
|
run:
|
||||||
|
using: gradlew
|
||||||
|
gradlew: [detekt]
|
||||||
|
ktlint:
|
||||||
|
description: 'Running ktlint over all modules'
|
||||||
|
run:
|
||||||
|
using: gradlew
|
||||||
|
gradlew: [ktlint]
|
||||||
|
compare-locales:
|
||||||
|
description: 'Validate strings.xml with compare-locales'
|
||||||
|
run:
|
||||||
|
using: run-task
|
||||||
|
cwd: '{checkout}'
|
||||||
|
command: 'pip install --user "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .'
|
||||||
|
lint:
|
||||||
|
description: 'Running tlint over all modules'
|
||||||
|
run:
|
||||||
|
using: gradlew
|
||||||
|
gradlew: [lintDebug]
|
|
@ -6,6 +6,7 @@ loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
kind-dependencies:
|
kind-dependencies:
|
||||||
- old-decision
|
- old-decision
|
||||||
|
- test
|
||||||
|
|
||||||
transforms:
|
transforms:
|
||||||
- taskgraph.transforms.code_review:transforms
|
- taskgraph.transforms.code_review:transforms
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
---
|
||||||
|
loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
- taskgraph.transforms.job:transforms
|
||||||
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
|
job-defaults:
|
||||||
|
description: Test Reference Browser
|
||||||
|
treeherder:
|
||||||
|
kind: test
|
||||||
|
tier: 2
|
||||||
|
worker-type: b-android
|
||||||
|
worker:
|
||||||
|
max-run-time: 7200
|
||||||
|
run:
|
||||||
|
using: gradlew
|
||||||
|
use-caches: false
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
ui:
|
||||||
|
attributes:
|
||||||
|
build-type: debug
|
||||||
|
code-review: true
|
||||||
|
treeherder:
|
||||||
|
symbol: ui
|
||||||
|
platform: 'ui-test/opt'
|
||||||
|
run-on-tasks-for: [github-pull-request, github-push]
|
||||||
|
run:
|
||||||
|
# TODO Generate APKs in a build task instead
|
||||||
|
gradlew: ['clean', 'assembleDebug', 'assembleAndroidTest']
|
||||||
|
post-gradlew:
|
||||||
|
- ['automation/taskcluster/androidTest/ui-test.sh', 'x86', '-1']
|
||||||
|
secrets:
|
||||||
|
- name: project/mobile/fenix/firebase
|
||||||
|
key: firebaseToken
|
||||||
|
path: .firebase_token.json
|
||||||
|
json: true
|
||||||
|
worker:
|
||||||
|
docker-image: {in-tree: ui-tests}
|
||||||
|
env:
|
||||||
|
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
||||||
|
GOOGLE_PROJECT: moz-fenix
|
||||||
|
artifacts:
|
||||||
|
- name: public
|
||||||
|
path: /build/fenix/results
|
||||||
|
type: directory
|
|
@ -48,6 +48,7 @@ RUN apt-get update -qq \
|
||||||
&& apt-get clean
|
&& apt-get clean
|
||||||
|
|
||||||
RUN pip install --upgrade pip
|
RUN pip install --upgrade pip
|
||||||
|
RUN pip install taskcluster
|
||||||
|
|
||||||
RUN locale-gen en_US.UTF-8
|
RUN locale-gen en_US.UTF-8
|
||||||
|
|
||||||
|
|
|
@ -3,4 +3,4 @@ FROM $DOCKER_IMAGE_PARENT
|
||||||
|
|
||||||
MAINTAINER Kate Glazko <kglazko@mozilla.com>
|
MAINTAINER Kate Glazko <kglazko@mozilla.com>
|
||||||
|
|
||||||
RUN pip install taskcluster requests
|
RUN pip install requests
|
||||||
|
|
|
@ -0,0 +1,32 @@
|
||||||
|
# %ARG DOCKER_IMAGE_PARENT
|
||||||
|
FROM $DOCKER_IMAGE_PARENT
|
||||||
|
MAINTAINER Richard Pappalardo <rpappalax@gmail.com>
|
||||||
|
|
||||||
|
#----------------------------------------------------------------------------------------------------------------------
|
||||||
|
#-- Test tools --------------------------------------------------------------------------------------------------------
|
||||||
|
#----------------------------------------------------------------------------------------------------------------------
|
||||||
|
|
||||||
|
USER worker:worker
|
||||||
|
|
||||||
|
ENV GOOGLE_SDK_DOWNLOAD ./gcloud.tar.gz
|
||||||
|
ENV GOOGLE_SDK_VERSION 233
|
||||||
|
|
||||||
|
ENV TEST_TOOLS /builds/worker/test-tools
|
||||||
|
ENV PATH ${PATH}:${TEST_TOOLS}:${TEST_TOOLS}/google-cloud-sdk/bin
|
||||||
|
|
||||||
|
RUN mkdir -p ${TEST_TOOLS} && \
|
||||||
|
mkdir -p ${HOME}/.config/gcloud
|
||||||
|
|
||||||
|
RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}.0.0-linux-x86_64.tar.gz --output ${GOOGLE_SDK_DOWNLOAD} \
|
||||||
|
&& tar -xvf ${GOOGLE_SDK_DOWNLOAD} -C ${TEST_TOOLS} \
|
||||||
|
&& rm -f ${GOOGLE_SDK_DOWNLOAD} \
|
||||||
|
&& ${TEST_TOOLS}/google-cloud-sdk/install.sh --quiet \
|
||||||
|
&& ${TEST_TOOLS}/google-cloud-sdk/bin/gcloud --quiet components update
|
||||||
|
|
||||||
|
RUN URL_FLANK_BIN=$(curl -s "https://api.github.com/repos/TestArmada/flank/releases/latest" | grep "browser_download_url*" | sed -r "s/\"//g" | cut -d ":" -f3) \
|
||||||
|
&& wget "https:${URL_FLANK_BIN}" -O ${TEST_TOOLS}/flank.jar \
|
||||||
|
&& chmod +x ${TEST_TOOLS}/flank.jar
|
||||||
|
|
||||||
|
|
||||||
|
# run-task expects to run as root
|
||||||
|
USER root
|
|
@ -68,7 +68,7 @@ def _extract_command(run):
|
||||||
|
|
||||||
def _generate_secret_command(secret):
|
def _generate_secret_command(secret):
|
||||||
secret_command = [
|
secret_command = [
|
||||||
"taskcluster/scripts/get-secret.py",
|
"automation/taskcluster/helper/get-secret.py",
|
||||||
"-s", secret["name"],
|
"-s", secret["name"],
|
||||||
"-k", secret["key"],
|
"-k", secret["key"],
|
||||||
"-f", secret["path"],
|
"-f", secret["path"],
|
||||||
|
|
Loading…
Reference in New Issue