Add Nightly startup-test (#6569)
parent
7208ce95a5
commit
85d222d2b1
|
@ -21,6 +21,10 @@ gcloud:
|
||||||
# see: https://github.com/TestArmada/flank/issues/341
|
# see: https://github.com/TestArmada/flank/issues/341
|
||||||
#results-history-name: tmp_parallel
|
#results-history-name: tmp_parallel
|
||||||
|
|
||||||
|
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
||||||
|
# The maximum number of reruns allowed is 10. Default is 0, which implies noi reruns.
|
||||||
|
num-flaky-test-attempts: 2
|
||||||
|
|
||||||
# test and app are the only required args
|
# test and app are the only required args
|
||||||
app: /APP/PATH
|
app: /APP/PATH
|
||||||
test: /TEST/PATH
|
test: /TEST/PATH
|
||||||
|
@ -44,10 +48,6 @@ flank:
|
||||||
# test shards - the amount of groups to split the test suite into
|
# test shards - the amount of groups to split the test suite into
|
||||||
# set to -1 to use one shard per test.
|
# set to -1 to use one shard per test.
|
||||||
max-test-shards: -1
|
max-test-shards: -1
|
||||||
# repeat tests - the amount of times to run the tests.
|
# num-test-runs- the amount of times to run the tests.
|
||||||
# 1 runs the tests once. 10 runs all the tests 10x
|
# 1 runs the tests once. 10 runs all the tests 10x
|
||||||
repeat-tests: 1
|
num-test-runs: 1
|
||||||
|
|
||||||
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
|
||||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
|
||||||
num-flaky-test-attempts: 3
|
|
||||||
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
# gcloud args match the official gcloud cli
|
||||||
|
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||||
|
gcloud:
|
||||||
|
results-bucket: fenix_test_artifacts
|
||||||
|
record-video: true
|
||||||
|
|
||||||
|
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||||
|
timeout: 30m
|
||||||
|
# will start test then close socket. no reports will be generated.
|
||||||
|
# to retrieve results later, use the "refresh" command
|
||||||
|
# reports will be generated from /results/matrix_ids.json
|
||||||
|
#async: true
|
||||||
|
# will start test then leave socket open. reports will be published
|
||||||
|
# to /results
|
||||||
|
# see: https://github.com/TestArmada/flank/issues/339
|
||||||
|
async: false
|
||||||
|
|
||||||
|
# results-history-name
|
||||||
|
# by default, set to app name
|
||||||
|
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||||
|
# see: https://github.com/TestArmada/flank/issues/341
|
||||||
|
#results-history-name: tmp_parallel
|
||||||
|
|
||||||
|
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
||||||
|
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||||
|
num-flaky-test-attempts: 2
|
||||||
|
|
||||||
|
# test and app are the only required args
|
||||||
|
app: /app/path
|
||||||
|
test: /test/path
|
||||||
|
|
||||||
|
auto-google-login: true
|
||||||
|
use-orchestrator: true
|
||||||
|
environment-variables:
|
||||||
|
clearPackageData: true
|
||||||
|
directories-to-pull:
|
||||||
|
- /sdcard/screenshots
|
||||||
|
performance-metrics: true
|
||||||
|
|
||||||
|
test-targets:
|
||||||
|
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||||
|
|
||||||
|
device:
|
||||||
|
- model: sailfish
|
||||||
|
version: 25
|
||||||
|
- model: sailfish
|
||||||
|
version: 28
|
||||||
|
|
||||||
|
flank:
|
||||||
|
project: GOOGLE_PROJECT
|
||||||
|
# num-test-runs: the amount of times to run the tests.
|
||||||
|
# 1 runs the tests once. 10 runs all the tests 10x
|
||||||
|
num-test-runs: 1
|
||||||
|
|
|
@ -0,0 +1,54 @@
|
||||||
|
# gcloud args match the official gcloud cli
|
||||||
|
# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run
|
||||||
|
gcloud:
|
||||||
|
results-bucket: fenix_test_artifacts
|
||||||
|
record-video: true
|
||||||
|
|
||||||
|
# The maximum possible testing time is 30m on physical devices and 60m on virtual devices.
|
||||||
|
timeout: 30m
|
||||||
|
# will start test then close socket. no reports will be generated.
|
||||||
|
# to retrieve results later, use the "refresh" command
|
||||||
|
# reports will be generated from /results/matrix_ids.json
|
||||||
|
#async: true
|
||||||
|
# will start test then leave socket open. reports will be published
|
||||||
|
# to /results
|
||||||
|
# see: https://github.com/TestArmada/flank/issues/339
|
||||||
|
async: false
|
||||||
|
|
||||||
|
# results-history-name
|
||||||
|
# by default, set to app name
|
||||||
|
# declare results-history-name to create a separate dropdown menu in Firebase
|
||||||
|
# see: https://github.com/TestArmada/flank/issues/341
|
||||||
|
#results-history-name: tmp_parallel
|
||||||
|
|
||||||
|
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
||||||
|
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||||
|
num-flaky-test-attempts: 2
|
||||||
|
|
||||||
|
# test and app are the only required args
|
||||||
|
app: /app/path
|
||||||
|
test: /test/path
|
||||||
|
|
||||||
|
auto-google-login: true
|
||||||
|
use-orchestrator: true
|
||||||
|
environment-variables:
|
||||||
|
clearPackageData: true
|
||||||
|
directories-to-pull:
|
||||||
|
- /sdcard/screenshots
|
||||||
|
performance-metrics: true
|
||||||
|
|
||||||
|
test-targets:
|
||||||
|
- class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest
|
||||||
|
|
||||||
|
device:
|
||||||
|
- model: Nexus6
|
||||||
|
version: 25
|
||||||
|
- model: Pixel2
|
||||||
|
version: 28
|
||||||
|
|
||||||
|
flank:
|
||||||
|
project: GOOGLE_PROJECT
|
||||||
|
# num-test-runs: the amount of times to run the tests.
|
||||||
|
# 1 runs the tests once. 10 runs all the tests 10x
|
||||||
|
num-test-runs: 1
|
||||||
|
|
|
@ -23,7 +23,7 @@ gcloud:
|
||||||
|
|
||||||
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
# The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason.
|
||||||
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
# The maximum number of reruns allowed is 10. Default is 0, which implies no reruns.
|
||||||
num-flaky-test-attempts: 3
|
num-flaky-test-attempts: 2
|
||||||
|
|
||||||
# test and app are the only required args
|
# test and app are the only required args
|
||||||
app: /app/path
|
app: /app/path
|
||||||
|
@ -51,7 +51,7 @@ flank:
|
||||||
# test shards - the amount of groups to split the test suite into
|
# test shards - the amount of groups to split the test suite into
|
||||||
# set to -1 to use one shard per test.
|
# set to -1 to use one shard per test.
|
||||||
max-test-shards: -1
|
max-test-shards: -1
|
||||||
# repeat tests - the amount of times to run the tests.
|
# num-test-runs: the amount of times to run the tests.
|
||||||
# 1 runs the tests once. 10 runs all the tests 10x
|
# 1 runs the tests once. 10 runs all the tests 10x
|
||||||
repeat-tests: 1
|
num-test-runs: 1
|
||||||
|
|
||||||
|
|
|
@ -74,7 +74,10 @@ set +e
|
||||||
|
|
||||||
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
|
if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then
|
||||||
flank_template="${PATH_TEST}/flank-${device_type}.yml"
|
flank_template="${PATH_TEST}/flank-${device_type}.yml"
|
||||||
echo "device_type: ${device_type}"
|
elif [[ "${device_type}" == "x86-start-test" ]]; then
|
||||||
|
flank_template="${PATH_TEST}/flank-x86-start-test.yml"
|
||||||
|
elif [[ "${device_type}" == "arm-start-test" ]]; then
|
||||||
|
flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml"
|
||||||
else
|
else
|
||||||
echo "NOT FOUND"
|
echo "NOT FOUND"
|
||||||
exitcode=1
|
exitcode=1
|
||||||
|
@ -82,18 +85,32 @@ fi
|
||||||
|
|
||||||
APK_APP="$(get_abs_filename $APK_APP)"
|
APK_APP="$(get_abs_filename $APK_APP)"
|
||||||
APK_TEST="$(get_abs_filename $APK_TEST)"
|
APK_TEST="$(get_abs_filename $APK_TEST)"
|
||||||
|
echo "device_type: ${device_type}"
|
||||||
echo "APK_APP: ${APK_APP}"
|
echo "APK_APP: ${APK_APP}"
|
||||||
echo "APK_TEST: ${APK_TEST}"
|
echo "APK_TEST: ${APK_TEST}"
|
||||||
|
|
||||||
# function to exit script with exit code from test run.
|
# function to exit script with exit code from test run.
|
||||||
# (Only 0 if all test executions passed)
|
# (Only 0 if all test executions passed)
|
||||||
function failure_check() {
|
function failure_check() {
|
||||||
|
echo
|
||||||
|
echo
|
||||||
if [[ $exitcode -ne 0 ]]; then
|
if [[ $exitcode -ne 0 ]]; then
|
||||||
echo
|
|
||||||
echo
|
|
||||||
echo "ERROR: UI test run failed, please check above URL"
|
echo "ERROR: UI test run failed, please check above URL"
|
||||||
|
else
|
||||||
|
echo "All UI test(s) have passed!"
|
||||||
fi
|
fi
|
||||||
exit $exitcode
|
|
||||||
|
echo
|
||||||
|
echo "COPY ARTIFACTS"
|
||||||
|
echo
|
||||||
|
cp -r ./results /builds/worker/artifacts
|
||||||
|
|
||||||
|
echo
|
||||||
|
echo "RESULTS"
|
||||||
|
echo
|
||||||
|
ls -la ./results
|
||||||
|
echo
|
||||||
|
echo
|
||||||
}
|
}
|
||||||
|
|
||||||
echo
|
echo
|
||||||
|
@ -109,25 +126,5 @@ echo
|
||||||
$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT
|
$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT
|
||||||
exitcode=$?
|
exitcode=$?
|
||||||
failure_check
|
failure_check
|
||||||
echo
|
|
||||||
echo
|
|
||||||
|
|
||||||
echo
|
exit $exitcode
|
||||||
echo "COPY ARTIFACTS"
|
|
||||||
echo
|
|
||||||
cp -r ./results ./test_artifacts
|
|
||||||
exitcode=$?
|
|
||||||
failure_check
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
|
|
||||||
echo
|
|
||||||
echo "RESULTS"
|
|
||||||
echo
|
|
||||||
ls -la ./results
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
|
|
||||||
echo "All UI test(s) have passed!"
|
|
||||||
echo
|
|
||||||
echo
|
|
||||||
|
|
|
@ -0,0 +1,60 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
---
|
||||||
|
loader: taskgraph.loader.transform:loader
|
||||||
|
|
||||||
|
transforms:
|
||||||
|
- taskgraph.transforms.job:transforms
|
||||||
|
- taskgraph.transforms.task:transforms
|
||||||
|
|
||||||
|
job-defaults:
|
||||||
|
description: Run a simple UI test to verify Nightly build can start
|
||||||
|
treeherder:
|
||||||
|
kind: test
|
||||||
|
platform: 'nightly-start-test/opt'
|
||||||
|
tier: 2
|
||||||
|
worker-type: b-android
|
||||||
|
worker:
|
||||||
|
docker-image: {in-tree: ui-tests}
|
||||||
|
max-run-time: 7200
|
||||||
|
env:
|
||||||
|
GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json'
|
||||||
|
GOOGLE_PROJECT: moz-fenix
|
||||||
|
artifacts:
|
||||||
|
- name: public
|
||||||
|
path: /builds/worker/artifacts
|
||||||
|
type: directory
|
||||||
|
run:
|
||||||
|
using: run-commands
|
||||||
|
use-caches: false
|
||||||
|
secrets:
|
||||||
|
- name: project/mobile/fenix/firebase
|
||||||
|
key: firebaseToken
|
||||||
|
path: .firebase_token.json
|
||||||
|
json: true
|
||||||
|
run-on-tasks-for: []
|
||||||
|
attributes:
|
||||||
|
build-type: debug
|
||||||
|
nightly: true
|
||||||
|
dependencies:
|
||||||
|
signing: signing-nightly
|
||||||
|
signing-android-test: signing-android-test-nightly
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
nightly-x86:
|
||||||
|
run:
|
||||||
|
commands:
|
||||||
|
- [wget, {artifact-reference: '<signing/public/build/x86/geckoNightly/target.apk>'}, '-O', app-x86.apk]
|
||||||
|
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/geckoNightly/target.apk>'}, '-O', android-test.apk]
|
||||||
|
- [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1']
|
||||||
|
treeherder:
|
||||||
|
symbol: nightly(startup-x86)
|
||||||
|
nightly-arm:
|
||||||
|
run:
|
||||||
|
commands:
|
||||||
|
- [wget, {artifact-reference: '<signing/public/build/armeabi-v7a/geckoNightly/target.apk>'}, '-O', app-arm.apk]
|
||||||
|
- [wget, {artifact-reference: '<signing-android-test/public/build/noarch/geckoNightly/target.apk>'}, '-O', android-test.apk]
|
||||||
|
- [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1']
|
||||||
|
treeherder:
|
||||||
|
symbol: nightly(startup-arm)
|
|
@ -45,6 +45,6 @@ jobs:
|
||||||
GOOGLE_PROJECT: moz-fenix
|
GOOGLE_PROJECT: moz-fenix
|
||||||
artifacts:
|
artifacts:
|
||||||
- name: public
|
- name: public
|
||||||
path: /build/fenix/results
|
path: /builds/worker/artifacts
|
||||||
type: directory
|
type: directory
|
||||||
worker-type: b-android
|
worker-type: b-android
|
||||||
|
|
Loading…
Reference in New Issue