Bug 1580778 - Migrate lint and ui-test tasks to taskgraph (#5430)
parent
057451178c
commit
c47f0f57ce
|
@ -48,14 +48,7 @@ fi
|
|||
JAVA_BIN="/usr/bin/java"
|
||||
PATH_TEST="./automation/taskcluster/androidTest"
|
||||
PATH_APK="./app/build/outputs/apk/geckoNightly/debug"
|
||||
FLANK_BIN="/build/test-tools/flank.jar"
|
||||
|
||||
echo
|
||||
echo "RETRIEVE SERVICE ACCT TOKEN"
|
||||
echo
|
||||
python automation/taskcluster/helper/get-secret.py --json -s project/mobile/fenix/firebase -k firebaseToken -f $GOOGLE_APPLICATION_CREDENTIALS
|
||||
echo
|
||||
echo
|
||||
FLANK_BIN="/builds/worker/test-tools/flank.jar"
|
||||
|
||||
echo
|
||||
echo "ACTIVATE SERVICE ACCT"
|
||||
|
|
|
@ -33,15 +33,6 @@ def pr(builder):
|
|||
tasks.append(builder.craft_assemble_pr_task(variant))
|
||||
tasks.append(builder.craft_test_pr_task(variant))
|
||||
|
||||
for craft_function in (
|
||||
builder.craft_detekt_task,
|
||||
builder.craft_ktlint_task,
|
||||
builder.craft_lint_task,
|
||||
builder.craft_compare_locales_task,
|
||||
builder.craft_ui_tests_task,
|
||||
):
|
||||
tasks.append(craft_function())
|
||||
|
||||
for task in tasks:
|
||||
task['attributes']['code-review'] = True
|
||||
|
||||
|
|
|
@ -1,3 +1,5 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
@ -28,7 +30,17 @@ def write_secret_to_file(path, data, key, base64decode=False, json_secret=False,
|
|||
|
||||
|
||||
def fetch_secret_from_taskcluster(name):
|
||||
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
|
||||
try:
|
||||
secrets = taskcluster.Secrets({
|
||||
# BaseUrl is still needed for tasks that haven't migrated to taskgraph yet.
|
||||
'baseUrl': 'http://taskcluster/secrets/v1',
|
||||
})
|
||||
except taskcluster.exceptions.TaskclusterFailure:
|
||||
# taskcluster library >=5 errors out when `baseUrl` is used
|
||||
secrets = taskcluster.Secrets({
|
||||
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
|
||||
})
|
||||
|
||||
return secrets.get(name)
|
||||
|
||||
|
||||
|
|
|
@ -156,13 +156,35 @@ class TaskBuilder(object):
|
|||
|
||||
def craft_test_pr_task(self, variant):
|
||||
# upload coverage only once, if the variant is arm64
|
||||
test_gradle_command = \
|
||||
'-Pcoverage jacocoGeckoNightlyDebugTestReport && automation/taskcluster/upload_coverage_report.sh'
|
||||
secret_index = 'project/mobile/fenix/public-tokens'
|
||||
pre_gradle_commands = (
|
||||
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
|
||||
secret_index, key, target_file
|
||||
)
|
||||
for key, target_file in (
|
||||
('codecov', '.cc_token'),
|
||||
)
|
||||
)
|
||||
|
||||
return self._craft_clean_gradle_task(
|
||||
gradle_commands = (
|
||||
'./gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport',
|
||||
)
|
||||
|
||||
post_gradle_commands = (
|
||||
'automation/taskcluster/upload_coverage_report.sh',
|
||||
)
|
||||
|
||||
command = ' && '.join(
|
||||
cmd
|
||||
for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands)
|
||||
for cmd in commands
|
||||
if cmd
|
||||
)
|
||||
|
||||
return self._craft_build_ish_task(
|
||||
name='test: {}'.format(variant.name),
|
||||
description='Building and testing variant {}'.format(variant.name),
|
||||
gradle_task=test_gradle_command,
|
||||
command=command,
|
||||
treeherder={
|
||||
'groupSymbol': variant.build_type,
|
||||
'jobKind': 'test',
|
||||
|
@ -173,106 +195,10 @@ class TaskBuilder(object):
|
|||
'tier': 1,
|
||||
},
|
||||
scopes=[
|
||||
'secrets:get:project/mobile/fenix/public-tokens'
|
||||
'secrets:get:{}'.format(secret_index)
|
||||
]
|
||||
)
|
||||
|
||||
def craft_ui_tests_task(self):
|
||||
artifacts = {
|
||||
"public": {
|
||||
"type": "directory",
|
||||
"path": "/build/fenix/results",
|
||||
"expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN))
|
||||
}
|
||||
}
|
||||
|
||||
env_vars = {
|
||||
"GOOGLE_PROJECT": "moz-fenix",
|
||||
"GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json"
|
||||
}
|
||||
|
||||
gradle_commands = (
|
||||
'./gradlew --no-daemon clean assemble assembleAndroidTest',
|
||||
)
|
||||
|
||||
test_commands = (
|
||||
'automation/taskcluster/androidTest/ui-test.sh x86 -1',
|
||||
)
|
||||
|
||||
command = ' && '.join(
|
||||
cmd
|
||||
for commands in (gradle_commands, test_commands)
|
||||
for cmd in commands
|
||||
if cmd
|
||||
)
|
||||
|
||||
treeherder = {
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'ui-test',
|
||||
},
|
||||
'symbol': 'ui-test',
|
||||
'tier': 2,
|
||||
}
|
||||
|
||||
return self._craft_build_ish_task(
|
||||
name='Fenix - UI test',
|
||||
description='Execute Gradle tasks for UI tests',
|
||||
command=command,
|
||||
scopes=[
|
||||
'secrets:get:project/mobile/fenix/firebase'
|
||||
],
|
||||
artifacts=artifacts,
|
||||
env_vars=env_vars,
|
||||
treeherder=treeherder,
|
||||
)
|
||||
|
||||
def craft_detekt_task(self):
|
||||
return self._craft_clean_gradle_task(
|
||||
name='detekt',
|
||||
description='Running detekt code quality checks',
|
||||
gradle_task='detekt',
|
||||
treeherder={
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'lint',
|
||||
},
|
||||
'symbol': 'detekt',
|
||||
'tier': 1,
|
||||
}
|
||||
|
||||
)
|
||||
|
||||
def craft_ktlint_task(self):
|
||||
return self._craft_clean_gradle_task(
|
||||
name='ktlint',
|
||||
description='Running ktlint code quality checks',
|
||||
gradle_task='ktlint',
|
||||
treeherder={
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'lint',
|
||||
},
|
||||
'symbol': 'ktlint',
|
||||
'tier': 1,
|
||||
}
|
||||
)
|
||||
|
||||
def craft_lint_task(self):
|
||||
return self._craft_clean_gradle_task(
|
||||
name='lint',
|
||||
description='Running lint for aarch64 release variant',
|
||||
gradle_task='lintDebug',
|
||||
treeherder={
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'lint',
|
||||
},
|
||||
'symbol': 'lint',
|
||||
'tier': 1,
|
||||
},
|
||||
)
|
||||
|
||||
def _craft_clean_gradle_task(
|
||||
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
|
||||
):
|
||||
|
@ -286,24 +212,6 @@ class TaskBuilder(object):
|
|||
scopes=scopes,
|
||||
)
|
||||
|
||||
def craft_compare_locales_task(self):
|
||||
return self._craft_build_ish_task(
|
||||
name='compare-locales',
|
||||
description='Validate strings.xml with compare-locales',
|
||||
command=(
|
||||
'pip install "compare-locales>=5.0.2,<6.0" && '
|
||||
'compare-locales --validate l10n.toml .'
|
||||
),
|
||||
treeherder={
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'lint',
|
||||
},
|
||||
'symbol': 'compare-locale',
|
||||
'tier': 2,
|
||||
}
|
||||
)
|
||||
|
||||
def _craft_build_ish_task(
|
||||
self, name, description, command, dependencies=None, artifacts=None,
|
||||
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
|
||||
|
|
|
@ -5,12 +5,6 @@
|
|||
# If a command fails then do not proceed and fail this script too.
|
||||
set -ex
|
||||
|
||||
# Get token for uploading to codecov and append it to codecov.yml
|
||||
python automation/taskcluster/helper/get-secret.py \
|
||||
-s project/mobile/fenix/public-tokens \
|
||||
-k codecov \
|
||||
-f .cc_token \
|
||||
|
||||
# Set some environment variables that will help codecov detect the CI
|
||||
export CI_BUILD_URL="https://tools.taskcluster.net/tasks/$TASK_ID"
|
||||
|
||||
|
|
|
@ -16,3 +16,6 @@ jobs:
|
|||
nimbledroid:
|
||||
parent: base
|
||||
symbol: I(nimbledroid)
|
||||
ui-tests:
|
||||
parent: base
|
||||
symbol: I(ui-tests)
|
||||
|
|
|
@ -0,0 +1,44 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
attributes:
|
||||
code-review: true
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: base}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
use-caches: false
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
|
||||
jobs:
|
||||
detekt:
|
||||
description: 'Running detekt over all modules'
|
||||
run:
|
||||
using: gradlew
|
||||
gradlew: [detekt]
|
||||
ktlint:
|
||||
description: 'Running ktlint over all modules'
|
||||
run:
|
||||
using: gradlew
|
||||
gradlew: [ktlint]
|
||||
compare-locales:
|
||||
description: 'Validate strings.xml with compare-locales'
|
||||
run:
|
||||
using: run-task
|
||||
cwd: '{checkout}'
|
||||
command: 'pip install --user "compare-locales>=5.0.2,<6.0" && compare-locales --validate l10n.toml .'
|
||||
lint:
|
||||
description: 'Running tlint over all modules'
|
||||
run:
|
||||
using: gradlew
|
||||
gradlew: [lintDebug]
|
|
@ -6,6 +6,7 @@ loader: taskgraph.loader.transform:loader
|
|||
|
||||
kind-dependencies:
|
||||
- old-decision
|
||||
- test
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.code_review:transforms
|
||||
|
|
|
@ -0,0 +1,50 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
description: Test Reference Browser
|
||||
treeherder:
|
||||
kind: test
|
||||
tier: 2
|
||||
worker-type: b-android
|
||||
worker:
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: gradlew
|
||||
use-caches: false
|
||||
|
||||
jobs:
|
||||
ui:
|
||||
attributes:
|
||||
build-type: debug
|
||||
code-review: true
|
||||
treeherder:
|
||||
symbol: ui
|
||||
platform: 'ui-test/opt'
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
run:
|
||||
# TODO Generate APKs in a build task instead
|
||||
gradlew: ['clean', 'assembleDebug', 'assembleAndroidTest']
|
||||
post-gradlew:
|
||||
- ['automation/taskcluster/androidTest/ui-test.sh', 'x86', '-1']
|
||||
secrets:
|
||||
- name: project/mobile/fenix/firebase
|
||||
key: firebaseToken
|
||||
path: .firebase_token.json
|
||||
json: true
|
||||
worker:
|
||||
docker-image: {in-tree: ui-tests}
|
||||
env:
|
||||
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
||||
GOOGLE_PROJECT: moz-fenix
|
||||
artifacts:
|
||||
- name: public
|
||||
path: /build/fenix/results
|
||||
type: directory
|
|
@ -48,6 +48,7 @@ RUN apt-get update -qq \
|
|||
&& apt-get clean
|
||||
|
||||
RUN pip install --upgrade pip
|
||||
RUN pip install taskcluster
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
|
||||
|
|
|
@ -3,4 +3,4 @@ FROM $DOCKER_IMAGE_PARENT
|
|||
|
||||
MAINTAINER Kate Glazko <kglazko@mozilla.com>
|
||||
|
||||
RUN pip install taskcluster requests
|
||||
RUN pip install requests
|
||||
|
|
|
@ -0,0 +1,32 @@
|
|||
# %ARG DOCKER_IMAGE_PARENT
|
||||
FROM $DOCKER_IMAGE_PARENT
|
||||
MAINTAINER Richard Pappalardo <rpappalax@gmail.com>
|
||||
|
||||
#----------------------------------------------------------------------------------------------------------------------
|
||||
#-- Test tools --------------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
USER worker:worker
|
||||
|
||||
ENV GOOGLE_SDK_DOWNLOAD ./gcloud.tar.gz
|
||||
ENV GOOGLE_SDK_VERSION 233
|
||||
|
||||
ENV TEST_TOOLS /builds/worker/test-tools
|
||||
ENV PATH ${PATH}:${TEST_TOOLS}:${TEST_TOOLS}/google-cloud-sdk/bin
|
||||
|
||||
RUN mkdir -p ${TEST_TOOLS} && \
|
||||
mkdir -p ${HOME}/.config/gcloud
|
||||
|
||||
RUN curl https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-${GOOGLE_SDK_VERSION}.0.0-linux-x86_64.tar.gz --output ${GOOGLE_SDK_DOWNLOAD} \
|
||||
&& tar -xvf ${GOOGLE_SDK_DOWNLOAD} -C ${TEST_TOOLS} \
|
||||
&& rm -f ${GOOGLE_SDK_DOWNLOAD} \
|
||||
&& ${TEST_TOOLS}/google-cloud-sdk/install.sh --quiet \
|
||||
&& ${TEST_TOOLS}/google-cloud-sdk/bin/gcloud --quiet components update
|
||||
|
||||
RUN URL_FLANK_BIN=$(curl -s "https://api.github.com/repos/TestArmada/flank/releases/latest" | grep "browser_download_url*" | sed -r "s/\"//g" | cut -d ":" -f3) \
|
||||
&& wget "https:${URL_FLANK_BIN}" -O ${TEST_TOOLS}/flank.jar \
|
||||
&& chmod +x ${TEST_TOOLS}/flank.jar
|
||||
|
||||
|
||||
# run-task expects to run as root
|
||||
USER root
|
|
@ -68,7 +68,7 @@ def _extract_command(run):
|
|||
|
||||
def _generate_secret_command(secret):
|
||||
secret_command = [
|
||||
"taskcluster/scripts/get-secret.py",
|
||||
"automation/taskcluster/helper/get-secret.py",
|
||||
"-s", secret["name"],
|
||||
"-k", secret["key"],
|
||||
"-f", secret["path"],
|
||||
|
|
Loading…
Reference in New Issue