diff --git a/automation/taskcluster/androidTest/flank-arm.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml similarity index 82% rename from automation/taskcluster/androidTest/flank-arm.yml rename to automation/taskcluster/androidTest/flank-arm64-v8a.yml index 0217414b2..4888c6df0 100644 --- a/automation/taskcluster/androidTest/flank-arm.yml +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -34,8 +34,6 @@ gcloud: performance-metrics: true device: - - model: shamu - version: 21 - model: sailfish version: 25 - model: sailfish @@ -49,8 +47,3 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 - # always run - these tests are inserted at the beginning of every shard - # useful if you need to grant permissions or login before other tests run - #test-targets-always-run: - #- class com.example.app.ExampleUiTest#testPasses - # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/flank-armabi-v7a.yml b/automation/taskcluster/androidTest/flank-armabi-v7a.yml new file mode 100644 index 000000000..784dbb0ef --- /dev/null +++ b/automation/taskcluster/androidTest/flank-armabi-v7a.yml @@ -0,0 +1,47 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # test and app are the only required args + app: /APP/PATH + test: /TEST/PATH + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + device: + - model: shamu + version: 21 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: -1 + # repeat tests - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/flank-x86-64.yml b/automation/taskcluster/androidTest/flank-x86-64.yml new file mode 100644 index 000000000..30595d419 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-64.yml @@ -0,0 +1,47 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + device: + - model: Nexus6 + version: 25 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: -1 + # repeat tests - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/flank-x86-tablet.yml b/automation/taskcluster/androidTest/flank-x86-tablet.yml index f44669f95..27b2b9a4a 100644 --- a/automation/taskcluster/androidTest/flank-x86-tablet.yml +++ b/automation/taskcluster/androidTest/flank-x86-tablet.yml @@ -47,8 +47,3 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 - # always run - these tests are inserted at the beginning of every shard - # useful if you need to grant permissions or login before other tests run - #test-targets-always-run: - #- class com.example.app.ExampleUiTest#testPasses - # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index 2e5c7c319..a8f4778a3 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -45,8 +45,3 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 - # always run - these tests are inserted at the beginning of every shard - # useful if you need to grant permissions or login before other tests run - #test-targets-always-run: - #- class com.example.app.ExampleUiTest#testPasses - # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index b65e9a68e..17a939b5b 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -47,9 +47,8 @@ fi JAVA_BIN="/usr/bin/java" PATH_TEST="./automation/taskcluster/androidTest" +PATH_APK="./app/build/outputs/apk/geckoNightly/debug" FLANK_BIN="/build/test-tools/flank.jar" -FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml" -FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml" echo echo "RETRIEVE SERVICE ACCT TOKEN" @@ -73,18 +72,30 @@ echo # and try to download the artifacts. We will exit with the actual error code later. set +e -if [[ "${device_type,,}" == "x86" ]] +APK_APP="${PATH_APK}/app-geckoNightly-${deviceType,,}-debug.apk" +if [[ "${device_type,,}" == "aarch64" ]] then - deviceType="X86" - flank_template="$FLANK_CONF_X86" + flank_template="${PATH_TEST}/flank-aarch64.yml" + APK_APP="${PATH_APK}/app-geckoNightly-arm64-v8a-debug.apk" +elif [[ "${device_type,,}" == "arm" ]] +then + flank_template="${PATH_TEST}/flank-arm.yml" + APK_APP="${PATH_APK}/app-geckoNightly-armeabi-v7a-debug.apk" +elif [[ "${device_type,,}" == "x86_64" ]] +then + flank_template="${PATH_TEST}/flank-x86-64.yml" + APK_APP="${PATH_APK}/app-geckoNightly-x86_64-debug.apk" +elif [[ "${device_type,,}" == "x86" ]] +then + flank_template="${PATH_TEST}/flank-x86.yml" + APK_APP="${PATH_APK}/app-geckoNightly-x86-debug.apk" else - deviceType="Arm" - flank_template="$FLANK_CONF_ARM" + echo "NOT FOUND" + exitcode=1 fi -APK_APP="./app/build/outputs/apk/${deviceType,,}/debug/app-${deviceType,,}-debug.apk" -APK_TEST="./app/build/outputs/apk/androidTest/${deviceType,,}/debug/app-${deviceType,,}-debug-androidTest.apk" - +APK_TEST="./app/build/outputs/apk/androidTest/debug/app-geckoNightly-debug-androidTest.apk" +ls -la ./app/build/outputs/apk/androidTest/debug # function to exit script with exit code from test run. # (Only 0 if all test executions passed) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 899ec30a6..164dc98ec 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -180,6 +180,7 @@ class TaskBuilder(object): ) test_commands = ( + 'automation/taskcluster/androidTest/ui-test.sh aarch64 -1', 'automation/taskcluster/androidTest/ui-test.sh arm -1', )