diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/HomeScreenTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/HomeScreenTest.kt index c0314ce8f..f0deb91e8 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/HomeScreenTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/HomeScreenTest.kt @@ -54,6 +54,7 @@ class HomeScreenTest { } @Test + @Ignore("Temp disable flakey test - see: https://github.com/mozilla-mobile/fenix/issues/5462") fun firstRunHomeScreenItemsTest() { homeScreen { verifyHomeScreen() diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/SearchTest.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/SearchTest.kt index 3b6a24651..e84141fbf 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/SearchTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/SearchTest.kt @@ -62,6 +62,7 @@ class SearchTest { } @Test + @Ignore("Temp disable flakey test - see: https://github.com/mozilla-mobile/fenix/issues/5462") fun shortcutSearchEngineSettingsTest() { homeScreen { }.openSearch { diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/FindInPageRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/FindInPageRobot.kt index f2008e56d..b0bb38fc6 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/FindInPageRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/FindInPageRobot.kt @@ -18,7 +18,7 @@ import androidx.test.uiautomator.By import androidx.test.uiautomator.UiDevice import androidx.test.uiautomator.Until import org.mozilla.fenix.R -import org.mozilla.fenix.helpers.TestAssetHelper.waitingTimeShort +import org.mozilla.fenix.helpers.TestAssetHelper.waitingTime import org.mozilla.fenix.helpers.click /** @@ -33,18 +33,18 @@ class FindInPageRobot { fun verifyFindInPageCloseButton() = assertFindInPageCloseButton()!! fun enterFindInPageQuery(expectedText: String) { - mDevice.wait(Until.findObject(By.res("find_in_page_query_text")), waitingTimeShort) + mDevice.wait(Until.findObject(By.res("find_in_page_query_text")), waitingTime) findInPageQuery().perform(clearText(), typeText(expectedText)) } fun verifyFindNextInPageResult(ratioCounter: String) { - mDevice.waitForIdle() + mDevice.wait(Until.findObject(By.text(ratioCounter)), waitingTime) findInPageResult().check(matches(withText((ratioCounter)))) findInPageNextButton().click() } fun verifyFindPrevInPageResult(ratioCounter: String) { - mDevice.waitForIdle() + mDevice.wait(Until.findObject(By.text(ratioCounter)), waitingTime) findInPageResult().check(matches(withText((ratioCounter)))) findInPagePrevButton().click() } diff --git a/automation/taskcluster/androidTest/flank-arm64-v8a.yml b/automation/taskcluster/androidTest/flank-arm64-v8a.yml index a571d584d..54ef5a4fb 100644 --- a/automation/taskcluster/androidTest/flank-arm64-v8a.yml +++ b/automation/taskcluster/androidTest/flank-arm64-v8a.yml @@ -47,3 +47,7 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 + + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 3 diff --git a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml b/automation/taskcluster/androidTest/flank-armeabi-v7a.yml index 784dbb0ef..ec0909cc7 100644 --- a/automation/taskcluster/androidTest/flank-armeabi-v7a.yml +++ b/automation/taskcluster/androidTest/flank-armeabi-v7a.yml @@ -45,3 +45,7 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 + + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 3 diff --git a/automation/taskcluster/androidTest/flank-x86-tablet.yml b/automation/taskcluster/androidTest/flank-x86-tablet.yml deleted file mode 100644 index 27b2b9a4a..000000000 --- a/automation/taskcluster/androidTest/flank-x86-tablet.yml +++ /dev/null @@ -1,49 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /APP/PATH - test: /TEST/PATH - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - device: - - model: Nexus9 - version: 21 - - model: Nexus9 - version: 22 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # repeat tests - the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml index a8f4778a3..04a3aa038 100644 --- a/automation/taskcluster/androidTest/flank-x86.yml +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -34,8 +34,10 @@ gcloud: performance-metrics: true device: - - model: Nexus7 - version: 21 + - model: Nexus9 + version: 25 + - model: Pixel2 + version: 28 flank: project: GOOGLE_PROJECT @@ -45,3 +47,7 @@ flank: # repeat tests - the amount of times to run the tests. # 1 runs the tests once. 10 runs all the tests 10x repeat-tests: 1 + + # The number of times a TestExecution should be re-attempted if one or more\nof its test cases fail for any reason. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 3 diff --git a/automation/taskcluster/androidTest/flank-x86_64.yml b/automation/taskcluster/androidTest/flank-x86_64.yml deleted file mode 100644 index 30595d419..000000000 --- a/automation/taskcluster/androidTest/flank-x86_64.yml +++ /dev/null @@ -1,47 +0,0 @@ -# gcloud args match the official gcloud cli -# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run -gcloud: - results-bucket: fenix_test_artifacts - record-video: true - - # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. - timeout: 30m - # will start test then close socket. no reports will be generated. - # to retrieve results later, use the "refresh" command - # reports will be generated from /results/matrix_ids.json - #async: true - # will start test then leave socket open. reports will be published - # to /results - # see: https://github.com/TestArmada/flank/issues/339 - async: false - - # results-history-name - # by default, set to app name - # declare results-history-name to create a separate dropdown menu in Firebase - # see: https://github.com/TestArmada/flank/issues/341 - #results-history-name: tmp_parallel - - # test and app are the only required args - app: /app/path - test: /test/path - - auto-google-login: true - use-orchestrator: true - environment-variables: - clearPackageData: true - directories-to-pull: - - /sdcard/screenshots - performance-metrics: true - - device: - - model: Nexus6 - version: 25 - -flank: - project: GOOGLE_PROJECT - # test shards - the amount of groups to split the test suite into - # set to -1 to use one shard per test. - max-test-shards: -1 - # repeat tests - the amount of times to run the tests. - # 1 runs the tests once. 10 runs all the tests 10x - repeat-tests: 1 diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 562919794..6f158a3be 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -82,6 +82,8 @@ else fi APK_TEST="./app/build/outputs/apk/androidTest/geckoNightly/debug/app-geckoNightly-debug-androidTest.apk" +echo "APK_PATH: ${APK_PATH}" +echo "APK_TEST: ${APK_TEST}" # function to exit script with exit code from test run. # (Only 0 if all test executions passed) diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index dcdbf2ecf..8079605aa 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -196,7 +196,7 @@ class TaskBuilder(object): ) test_commands = ( - 'automation/taskcluster/androidTest/ui-test.sh x86_64 -1', + 'automation/taskcluster/androidTest/ui-test.sh x86 -1', ) command = ' && '.join(