Split UI test
parent
f099552ae3
commit
de8bc2f8a5
|
@ -552,6 +552,16 @@ task printVariants {
|
||||||
engine: it.productFlavors.find { it.dimension == 'engine' }.name,
|
engine: it.productFlavors.find { it.dimension == 'engine' }.name,
|
||||||
name: it.name,
|
name: it.name,
|
||||||
]}
|
]}
|
||||||
|
// AndroidTest is a special case not included above
|
||||||
|
variants.add([
|
||||||
|
apks: [[
|
||||||
|
abi: 'noarch',
|
||||||
|
fileName: 'app-geckoNightly-debug-androidTest.apk',
|
||||||
|
]],
|
||||||
|
build_type: 'androidTest',
|
||||||
|
engine: 'geckoNightly',
|
||||||
|
name: 'androidTest',
|
||||||
|
])
|
||||||
println 'variants: ' + groovy.json.JsonOutput.toJson(variants)
|
println 'variants: ' + groovy.json.JsonOutput.toJson(variants)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -33,6 +33,13 @@ display_help() {
|
||||||
echo
|
echo
|
||||||
}
|
}
|
||||||
|
|
||||||
|
get_abs_filename() {
|
||||||
|
# $1 : relative filename
|
||||||
|
relative_filename="$1"
|
||||||
|
echo "$(cd "$(dirname "$relative_filename")" && pwd)/$(basename "$relative_filename")"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
# Basic parameter check
|
# Basic parameter check
|
||||||
if [[ $# -lt 1 ]]; then
|
if [[ $# -lt 1 ]]; then
|
||||||
echo "Error: please provide at least one build variant (arm|x86)"
|
echo "Error: please provide at least one build variant (arm|x86)"
|
||||||
|
@ -41,23 +48,24 @@ if [[ $# -lt 1 ]]; then
|
||||||
fi
|
fi
|
||||||
|
|
||||||
device_type="$1" # arm64-v8a | armeabi-v7a | x86_64 | x86
|
device_type="$1" # arm64-v8a | armeabi-v7a | x86_64 | x86
|
||||||
if [[ ! -z "$2" ]]; then
|
APK_APP="$2"
|
||||||
num_shards=$2
|
APK_TEST="$3"
|
||||||
|
if [[ ! -z "$4" ]]; then
|
||||||
|
num_shards=$4
|
||||||
fi
|
fi
|
||||||
|
|
||||||
JAVA_BIN="/usr/bin/java"
|
JAVA_BIN="/usr/bin/java"
|
||||||
PATH_TEST="./automation/taskcluster/androidTest"
|
PATH_TEST="./automation/taskcluster/androidTest"
|
||||||
PATH_APK="./app/build/outputs/apk/geckoNightly/debug"
|
|
||||||
FLANK_BIN="/builds/worker/test-tools/flank.jar"
|
FLANK_BIN="/builds/worker/test-tools/flank.jar"
|
||||||
|
|
||||||
echo
|
echo
|
||||||
echo "ACTIVATE SERVICE ACCT"
|
echo "ACTIVATE SERVICE ACCT"
|
||||||
echo
|
echo
|
||||||
# this is where the Google Testcloud project ID is set
|
# this is where the Google Testcloud project ID is set
|
||||||
gcloud config set project "$GOOGLE_PROJECT"
|
gcloud config set project "$GOOGLE_PROJECT"
|
||||||
echo
|
echo
|
||||||
|
|
||||||
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
|
gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS"
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
@ -66,7 +74,6 @@ echo
|
||||||
set +e
|
set +e
|
||||||
|
|
||||||
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
|
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
|
||||||
APK_APP="${PATH_APK}/app-geckoNightly-${device_type}-debug.apk"
|
|
||||||
flank_template="${PATH_TEST}/flank-${device_type}.yml"
|
flank_template="${PATH_TEST}/flank-${device_type}.yml"
|
||||||
echo "device_type: ${device_type}"
|
echo "device_type: ${device_type}"
|
||||||
else
|
else
|
||||||
|
@ -74,8 +81,9 @@ else
|
||||||
exitcode=1
|
exitcode=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
APK_TEST="./app/build/outputs/apk/androidTest/geckoNightly/debug/app-geckoNightly-debug-androidTest.apk"
|
APK_APP="$(get_abs_filename $APK_APP)"
|
||||||
echo "APK_PATH: ${APK_PATH}"
|
APK_TEST="$(get_abs_filename $APK_TEST)"
|
||||||
|
echo "APK_APP: ${APK_APP}"
|
||||||
echo "APK_TEST: ${APK_TEST}"
|
echo "APK_TEST: ${APK_TEST}"
|
||||||
|
|
||||||
# function to exit script with exit code from test run.
|
# function to exit script with exit code from test run.
|
||||||
|
@ -92,7 +100,7 @@ function failure_check() {
|
||||||
echo
|
echo
|
||||||
echo "FLANK VERSION"
|
echo "FLANK VERSION"
|
||||||
echo
|
echo
|
||||||
$JAVA_BIN -jar $FLANK_BIN --version
|
$JAVA_BIN -jar $FLANK_BIN --version
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
@ -118,12 +126,9 @@ echo
|
||||||
echo "RESULTS"
|
echo "RESULTS"
|
||||||
echo
|
echo
|
||||||
ls -la ./results
|
ls -la ./results
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
|
|
||||||
echo "All UI test(s) have passed!"
|
echo "All UI test(s) have passed!"
|
||||||
echo
|
echo
|
||||||
echo
|
echo
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -45,6 +45,19 @@ jobs:
|
||||||
treeherder:
|
treeherder:
|
||||||
symbol: debug(B)
|
symbol: debug(B)
|
||||||
|
|
||||||
|
android-test:
|
||||||
|
attributes:
|
||||||
|
code-review: true
|
||||||
|
run-on-tasks-for: [github-pull-request, github-push]
|
||||||
|
run:
|
||||||
|
geckoview-engine: geckoNightly
|
||||||
|
gradle-build-type: androidTest
|
||||||
|
apk-artifact-template:
|
||||||
|
# 2 differences here: "androidTest/" is added and "{gradle_build_type}" is forced to "debug"
|
||||||
|
path: '/builds/worker/checkouts/src/app/build/outputs/apk/androidTest/{geckoview_engine}/debug/{fileName}'
|
||||||
|
treeherder:
|
||||||
|
symbol: androidTest(B)
|
||||||
|
|
||||||
performance-test:
|
performance-test:
|
||||||
run:
|
run:
|
||||||
geckoview-engine: geckoNightly
|
geckoview-engine: geckoNightly
|
||||||
|
|
|
@ -2,6 +2,7 @@
|
||||||
trust-domain: mobile
|
trust-domain: mobile
|
||||||
treeherder:
|
treeherder:
|
||||||
group-names:
|
group-names:
|
||||||
|
'androidTest': 'Tasks related to the androidTest APK'
|
||||||
'beta': 'Nightly-related tasks'
|
'beta': 'Nightly-related tasks'
|
||||||
'debug': 'Builds made for testing'
|
'debug': 'Builds made for testing'
|
||||||
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
|
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
|
||||||
|
|
|
@ -8,6 +8,7 @@ kind-dependencies:
|
||||||
- build
|
- build
|
||||||
- lint
|
- lint
|
||||||
- test
|
- test
|
||||||
|
- ui-test
|
||||||
|
|
||||||
transforms:
|
transforms:
|
||||||
- taskgraph.transforms.code_review:transforms
|
- taskgraph.transforms.code_review:transforms
|
||||||
|
|
|
@ -46,31 +46,3 @@ jobs:
|
||||||
- name: public/reports/index.html
|
- name: public/reports/index.html
|
||||||
path: /builds/worker/checkouts/src/app/build/reports/tests/testGeckoNightlyDebugUnitTest/index.html
|
path: /builds/worker/checkouts/src/app/build/reports/tests/testGeckoNightlyDebugUnitTest/index.html
|
||||||
type: file
|
type: file
|
||||||
ui:
|
|
||||||
attributes:
|
|
||||||
build-type: debug
|
|
||||||
code-review: true
|
|
||||||
include-pull-request-number: true
|
|
||||||
run-on-tasks-for: [github-pull-request, github-push]
|
|
||||||
run:
|
|
||||||
# TODO Generate APKs in a build task instead
|
|
||||||
gradlew: ['clean', 'assembleDebug', 'assembleAndroidTest']
|
|
||||||
post-gradlew:
|
|
||||||
- ['automation/taskcluster/androidTest/ui-test.sh', 'x86', '-1']
|
|
||||||
secrets:
|
|
||||||
- name: project/mobile/fenix/firebase
|
|
||||||
key: firebaseToken
|
|
||||||
path: .firebase_token.json
|
|
||||||
json: true
|
|
||||||
treeherder:
|
|
||||||
symbol: ui
|
|
||||||
platform: 'ui-test/opt'
|
|
||||||
worker:
|
|
||||||
docker-image: {in-tree: ui-tests}
|
|
||||||
env:
|
|
||||||
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
|
||||||
GOOGLE_PROJECT: moz-fenix
|
|
||||||
artifacts:
|
|
||||||
- name: public
|
|
||||||
path: /build/fenix/results
|
|
||||||
type: directory
|
|
||||||
|
|
|
@ -0,0 +1,50 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
---
|
||||||
|
loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
- fenix_taskgraph.transforms.test:transforms
|
||||||
|
- taskgraph.transforms.job:transforms
|
||||||
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
x86:
|
||||||
|
attributes:
|
||||||
|
build-type: debug
|
||||||
|
code-review: true
|
||||||
|
dependencies:
|
||||||
|
signing: signing-debug
|
||||||
|
signing-android-test: signing-android-test
|
||||||
|
description: Test Fenix
|
||||||
|
include-pull-request-number: true
|
||||||
|
run-on-tasks-for: [github-pull-request, github-push]
|
||||||
|
run:
|
||||||
|
commands:
|
||||||
|
- [wget, {artifact-reference: '<signing/public/build/x86/geckoNightly/target.apk>'}, '-O', app.apk]
|
||||||
|
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/geckoNightly/target.apk>'}, '-O', android-test.apk]
|
||||||
|
- [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1']
|
||||||
|
secrets:
|
||||||
|
- name: project/mobile/fenix/firebase
|
||||||
|
key: firebaseToken
|
||||||
|
path: .firebase_token.json
|
||||||
|
json: true
|
||||||
|
using: run-commands
|
||||||
|
use-caches: false
|
||||||
|
treeherder:
|
||||||
|
kind: test
|
||||||
|
platform: 'ui-test/opt'
|
||||||
|
symbol: debug(ui-test-x86)
|
||||||
|
tier: 2
|
||||||
|
worker:
|
||||||
|
docker-image: {in-tree: ui-tests}
|
||||||
|
max-run-time: 7200
|
||||||
|
env:
|
||||||
|
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
||||||
|
GOOGLE_PROJECT: moz-fenix
|
||||||
|
artifacts:
|
||||||
|
- name: public
|
||||||
|
path: /build/fenix/results
|
||||||
|
type: directory
|
||||||
|
worker-type: b-android
|
|
@ -6,31 +6,51 @@ from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
|
from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run
|
||||||
from taskgraph.util import path
|
from taskgraph.util import path
|
||||||
from taskgraph.util.schema import Schema
|
from taskgraph.util.schema import Schema, taskref_or_string
|
||||||
from voluptuous import Required, Optional
|
from voluptuous import Required, Optional
|
||||||
from six import text_type
|
from six import text_type
|
||||||
|
|
||||||
from pipes import quote as shell_quote
|
from pipes import quote as shell_quote
|
||||||
|
|
||||||
gradlew_schema = Schema(
|
secret_schema = {
|
||||||
{
|
Required("name"): text_type,
|
||||||
Required("using"): "gradlew",
|
Required("path"): text_type,
|
||||||
Optional("pre-gradlew"): [[text_type]],
|
Required("key"): text_type,
|
||||||
Required("gradlew"): [text_type],
|
Optional("json"): bool,
|
||||||
Optional("post-gradlew"): [[text_type]],
|
}
|
||||||
# Base work directory used to set up the task.
|
|
||||||
Required("workdir"): text_type,
|
gradlew_schema = Schema({
|
||||||
Optional("use-caches"): bool,
|
Required("using"): "gradlew",
|
||||||
Optional("secrets"): [
|
Optional("pre-gradlew"): [[text_type]],
|
||||||
{
|
Required("gradlew"): [text_type],
|
||||||
Required("name"): text_type,
|
Optional("post-gradlew"): [[text_type]],
|
||||||
Required("path"): text_type,
|
# Base work directory used to set up the task.
|
||||||
Required("key"): text_type,
|
Required("workdir"): text_type,
|
||||||
Optional("json"): bool,
|
Optional("use-caches"): bool,
|
||||||
}
|
Optional("secrets"): [secret_schema],
|
||||||
],
|
})
|
||||||
}
|
|
||||||
)
|
run_commands_schema = Schema({
|
||||||
|
Required("using"): "run-commands",
|
||||||
|
Required("commands"): [[taskref_or_string]],
|
||||||
|
Required("workdir"): text_type,
|
||||||
|
Optional("use-caches"): bool,
|
||||||
|
Optional("secrets"): [secret_schema],
|
||||||
|
})
|
||||||
|
|
||||||
|
|
||||||
|
@run_job_using("docker-worker", "run-commands", schema=run_commands_schema)
|
||||||
|
def configure_run_commands_schema(config, job, taskdesc):
|
||||||
|
run = job["run"]
|
||||||
|
pre_commands = [
|
||||||
|
_generate_secret_command(secret) for secret in run.get("secrets", [])
|
||||||
|
]
|
||||||
|
all_commands = pre_commands + run.pop("commands", [])
|
||||||
|
|
||||||
|
run["command"] = _convert_commands_to_string(all_commands)
|
||||||
|
_inject_secrets_scopes(run, taskdesc)
|
||||||
|
_set_run_task_attributes(job)
|
||||||
|
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
|
||||||
|
|
||||||
|
|
||||||
@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
|
@run_job_using("docker-worker", "gradlew", schema=gradlew_schema)
|
||||||
|
@ -42,20 +62,13 @@ def configure_gradlew(config, job, taskdesc):
|
||||||
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
|
{"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")}
|
||||||
)
|
)
|
||||||
|
|
||||||
# defer to the run_task implementation
|
run["command"] = _extract_gradlew_command(run)
|
||||||
run["command"] = _extract_command(run)
|
_inject_secrets_scopes(run, taskdesc)
|
||||||
secrets = run.pop("secrets", [])
|
_set_run_task_attributes(job)
|
||||||
scopes = taskdesc.setdefault("scopes", [])
|
|
||||||
new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets]
|
|
||||||
new_secret_scopes = list(set(new_secret_scopes)) # Scopes must not have any duplicates
|
|
||||||
scopes.extend(new_secret_scopes)
|
|
||||||
|
|
||||||
run["cwd"] = "{checkout}"
|
|
||||||
run["using"] = "run-task"
|
|
||||||
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
|
configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"])
|
||||||
|
|
||||||
|
|
||||||
def _extract_command(run):
|
def _extract_gradlew_command(run):
|
||||||
pre_gradle_commands = run.pop("pre-gradlew", [])
|
pre_gradle_commands = run.pop("pre-gradlew", [])
|
||||||
pre_gradle_commands += [
|
pre_gradle_commands += [
|
||||||
_generate_secret_command(secret) for secret in run.get("secrets", [])
|
_generate_secret_command(secret) for secret in run.get("secrets", [])
|
||||||
|
@ -65,8 +78,7 @@ def _extract_command(run):
|
||||||
post_gradle_commands = run.pop("post-gradlew", [])
|
post_gradle_commands = run.pop("post-gradlew", [])
|
||||||
|
|
||||||
commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
|
commands = pre_gradle_commands + [gradle_command] + post_gradle_commands
|
||||||
shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands]
|
return _convert_commands_to_string(commands)
|
||||||
return " && ".join(shell_quoted_commands)
|
|
||||||
|
|
||||||
|
|
||||||
def _generate_secret_command(secret):
|
def _generate_secret_command(secret):
|
||||||
|
@ -80,3 +92,53 @@ def _generate_secret_command(secret):
|
||||||
secret_command.append("--json")
|
secret_command.append("--json")
|
||||||
|
|
||||||
return secret_command
|
return secret_command
|
||||||
|
|
||||||
|
|
||||||
|
def _convert_commands_to_string(commands):
|
||||||
|
should_artifact_reference = False
|
||||||
|
should_task_reference = False
|
||||||
|
|
||||||
|
sanitized_commands = []
|
||||||
|
for command in commands:
|
||||||
|
sanitized_parts = []
|
||||||
|
for part in command:
|
||||||
|
if isinstance(part, dict):
|
||||||
|
if "artifact-reference" in part:
|
||||||
|
part_string = part["artifact-reference"]
|
||||||
|
should_artifact_reference = True
|
||||||
|
elif "task-reference" in part:
|
||||||
|
part_string = part["task-reference"]
|
||||||
|
should_task_reference = True
|
||||||
|
else:
|
||||||
|
raise ValueError('Unsupported dict: {}'.format(part))
|
||||||
|
else:
|
||||||
|
part_string = part
|
||||||
|
|
||||||
|
sanitized_parts.append(part_string)
|
||||||
|
sanitized_commands.append(sanitized_parts)
|
||||||
|
|
||||||
|
shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in sanitized_commands]
|
||||||
|
full_string_command = " && ".join(shell_quoted_commands)
|
||||||
|
|
||||||
|
if should_artifact_reference and should_task_reference:
|
||||||
|
raise NotImplementedError('"arifact-reference" and "task-reference" cannot be both used')
|
||||||
|
elif should_artifact_reference:
|
||||||
|
return {"artifact-reference": full_string_command}
|
||||||
|
elif should_task_reference:
|
||||||
|
return {"task-reference": full_string_command}
|
||||||
|
else:
|
||||||
|
return full_string_command
|
||||||
|
|
||||||
|
|
||||||
|
def _inject_secrets_scopes(run, taskdesc):
|
||||||
|
secrets = run.pop("secrets", [])
|
||||||
|
scopes = taskdesc.setdefault("scopes", [])
|
||||||
|
new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets]
|
||||||
|
new_secret_scopes = list(set(new_secret_scopes)) # Scopes must not have any duplicates
|
||||||
|
scopes.extend(new_secret_scopes)
|
||||||
|
|
||||||
|
|
||||||
|
def _set_run_task_attributes(job):
|
||||||
|
run = job["run"]
|
||||||
|
run["cwd"] = "{checkout}"
|
||||||
|
run["using"] = "run-task"
|
||||||
|
|
Loading…
Reference in New Issue