diff --git a/automation/taskcluster/androidTest/flank-arm64-v8a.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml index 54ef5a4fb..f3548998a 100644 --- a/automation/taskcluster/androidTest/flank-arm64-v8a.yml +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -21,6 +21,10 @@ gcloud: # see: https://github.com/TestArmada/flank/issues/341 #results-history-name: tmp_parallel + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies noi reruns. + num-flaky-test-attempts: 2 + # test and app are the only required args app: /APP/PATH test: /TEST/PATH @@ -44,10 +48,6 @@ flank: # test shards - the amount of groups to split the test suite into # set to -1 to use one shard per test. max-test-shards: -1 - # repeat tests - the amount of times to run the tests. + # num-test-runs- the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 - - # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. - # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 3 + num-test-runs: 1 diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml new file mode 100644 index 000000000..0bb564273 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-armeabi-v7a-start-test.yml @@ -0,0 +1,54 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 2 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + + device: + - model: sailfish + version: 25 + - model: sailfish + version: 28 + +flank: + project: GOOGLE_PROJECT + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + diff --git a/automation/taskcluster/androidTest/flank-x86-start-test.yml b/automation/taskcluster/androidTest/flank-x86-start-test.yml new file mode 100644 index 000000000..755260af3 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-start-test.yml @@ -0,0 +1,54 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 2 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - class org.mozilla.fenix.ui.NavigationToolbarTest#visitURLTest + + device: + - model: Nexus6 + version: 25 + - model: Pixel2 + version: 28 + +flank: + project: GOOGLE_PROJECT + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index 16c44a42b..be3e92cc5 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -23,7 +23,7 @@ gcloud: # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. - num-flaky-test-attempts: 3 + num-flaky-test-attempts: 2 # test and app are the only required args app: /app/path @@ -51,7 +51,7 @@ flank: # test shards - the amount of groups to split the test suite into # set to -1 to use one shard per test. max-test-shards: -1 - # repeat tests - the amount of times to run the tests. + # num-test-runs: the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 + num-test-runs: 1 diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 6cdf49fb1..ae1fbbdbc 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -74,7 +74,10 @@ set +e if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then flank_template="${PATH_TEST}/flank-${device_type}.yml" - echo "device_type: ${device_type}" +elif [[ "${device_type}" == "x86-start-test" ]]; then + flank_template="${PATH_TEST}/flank-x86-start-test.yml" +elif [[ "${device_type}" == "arm-start-test" ]]; then + flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" else echo "NOT FOUND" exitcode=1 @@ -82,18 +85,32 @@ fi APK_APP="$(get_abs_filename $APK_APP)" APK_TEST="$(get_abs_filename $APK_TEST)" +echo "device_type: ${device_type}" echo "APK_APP: ${APK_APP}" echo "APK_TEST: ${APK_TEST}" # function to exit script with exit code from test run. # (Only 0 if all test executions passed) function failure_check() { + echo + echo if [[ $exitcode -ne 0 ]]; then - echo - echo echo "ERROR: UI test run failed, please check above URL" + else + echo "All UI test(s) have passed!" fi - exit $exitcode + + echo + echo "COPY ARTIFACTS" + echo + cp -r ./results /builds/worker/artifacts + + echo + echo "RESULTS" + echo + ls -la ./results + echo + echo } echo @@ -109,25 +126,5 @@ echo $JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT exitcode=$? failure_check -echo -echo -echo -echo "COPY ARTIFACTS" -echo -cp -r ./results ./test_artifacts -exitcode=$? -failure_check -echo -echo - -echo -echo "RESULTS" -echo -ls -la ./results -echo -echo - -echo "All UI test(s) have passed!" -echo -echo +exit $exitcode diff --git a/taskcluster/ci/startup-test/kind.yml b/taskcluster/ci/startup-test/kind.yml new file mode 100644 index 000000000..871c9c19d --- /dev/null +++ b/taskcluster/ci/startup-test/kind.yml @@ -0,0 +1,60 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +job-defaults: + description: Run a simple UI test to verify Nightly build can start + treeherder: + kind: test + platform: 'nightly-start-test/opt' + tier: 2 + worker-type: b-android + worker: + docker-image: {in-tree: ui-tests} + max-run-time: 7200 + env: + GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' + GOOGLE_PROJECT: moz-fenix + artifacts: + - name: public + path: /builds/worker/artifacts + type: directory + run: + using: run-commands + use-caches: false + secrets: + - name: project/mobile/fenix/firebase + key: firebaseToken + path: .firebase_token.json + json: true + run-on-tasks-for: [] + attributes: + build-type: debug + nightly: true + dependencies: + signing: signing-nightly + signing-android-test: signing-android-test-nightly + +jobs: + nightly-x86: + run: + commands: + - [wget, {artifact-reference: ''}, '-O', app-x86.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [automation/taskcluster/androidTest/ui-test.sh, x86-start-test, app-x86.apk, android-test.apk, '-1'] + treeherder: + symbol: nightly(startup-x86) + nightly-arm: + run: + commands: + - [wget, {artifact-reference: ''}, '-O', app-arm.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [automation/taskcluster/androidTest/ui-test.sh, arm-start-test, app-arm.apk, android-test.apk, '-1'] + treeherder: + symbol: nightly(startup-arm) diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index 7659402bf..60a76b090 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -45,6 +45,6 @@ jobs: GOOGLE_PROJECT: moz-fenix artifacts: - name: public - path: /build/fenix/results + path: /builds/worker/artifacts type: directory worker-type: b-android