diff --git a/.cron.yml b/.cron.yml index 7dffdb220..0ee4b916a 100644 --- a/.cron.yml +++ b/.cron.yml @@ -32,3 +32,9 @@ jobs: treeherder-symbol: bump-ac target-tasks-method: bump_android_components when: [{hour: 14, minute: 0}] + - name: screenshots + job: + type: decision-task + treeherder-symbol: screenshots-D + target-tasks-method: screenshots + when: [{weekday: 'Monday', hour: 10, minute: 0}] diff --git a/app/src/androidTest/java/org/mozilla/fenix/screenshots/MenuScreenShotTest.kt b/app/src/androidTest/java/org/mozilla/fenix/screenshots/MenuScreenShotTest.kt index 663e96b01..c6e1e5951 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/screenshots/MenuScreenShotTest.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/screenshots/MenuScreenShotTest.kt @@ -92,9 +92,10 @@ class MenuScreenShotTest : ScreenshotTest() { Screengrab.screenshot("SettingsSubMenuDefaultBrowserRobot_settings-default-browser") mDevice.pressBack() - settingsTP() - Screengrab.screenshot("settings-enhanced-tp") - mDevice.pressBack() + // Disabled for Pixel 2 + // settingsTP() + // Screengrab.screenshot("settings-enhanced-tp") + // mDevice.pressBack() loginsAndPassword() Screengrab.screenshot("SettingsSubMenuLoginsAndPasswords-settings-logins-passwords") diff --git a/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml b/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml new file mode 100644 index 000000000..61adceda5 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-screenshots-tests.yml @@ -0,0 +1,55 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 20m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # The number of times a test execution should be re-attempted if one or more failures occur. + # The maximum number of reruns allowed is 10. Default is 0, which implies no reruns. + num-flaky-test-attempts: 1 + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + test-targets: + - package org.mozilla.fenix.screenshots + + device: + - model: Pixel2 + version: 28 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: 1 + # num-test-runs: the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + num-test-runs: 1 + diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 2b1812ea2..e45e9b8cc 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -80,6 +80,8 @@ elif [[ "${device_type}" == "x86-start-test" ]]; then flank_template="${PATH_TEST}/flank-x86-start-test.yml" elif [[ "${device_type}" == "arm-start-test" ]]; then flank_template="${PATH_TEST}/flank-armeabi-v7a-start-test.yml" +elif [[ "${device_type}" == "x86-screenshots-tests" ]]; then + flank_template="${PATH_TEST}/flank-x86-screenshots-tests.yml" else echo "FAILURE: flank config file not found!" exitcode=1 diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml index 4426d8989..66665abb2 100644 --- a/taskcluster/ci/ui-test/kind.yml +++ b/taskcluster/ci/ui-test/kind.yml @@ -11,44 +11,56 @@ transforms: job-defaults: attributes: + build-type: debug code-review: true retrigger: true + dependencies: + signing: signing-debug + signing-android-test: signing-android-test-debug + include-pull-request-number: true + run: + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + secrets: + - name: project/mobile/fenix/firebase + key: firebaseToken + path: .firebase_token.json + json: true + using: run-commands + use-caches: false + treeherder: + kind: test + platform: 'ui-test/opt' + tier: 2 + worker: + docker-image: {in-tree: ui-tests} + max-run-time: 7200 + env: + GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' + GOOGLE_PROJECT: moz-fenix + artifacts: + - name: public + path: /builds/worker/artifacts + type: directory + worker-type: b-android jobs: x86-debug: - attributes: - build-type: debug - dependencies: - signing: signing-debug - signing-android-test: signing-android-test-debug description: Test Fenix - include-pull-request-number: true run-on-tasks-for: [github-pull-request, github-push] run: commands: - - [wget, {artifact-reference: ''}, '-O', app.apk] - - [wget, {artifact-reference: ''}, '-O', android-test.apk] - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '50'] - secrets: - - name: project/mobile/fenix/firebase - key: firebaseToken - path: .firebase_token.json - json: true - using: run-commands - use-caches: false treeherder: - kind: test - platform: 'ui-test/opt' symbol: debug(ui-test-x86) - tier: 2 - worker: - docker-image: {in-tree: ui-tests} - max-run-time: 7200 - env: - GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' - GOOGLE_PROJECT: moz-fenix - artifacts: - - name: public - path: /builds/worker/artifacts - type: directory - worker-type: b-android + screenshots-x86: + attributes: + screenshots: true + description: Run UI screenshots tests to keep them up to date + run-on-tasks-for: [] + run: + commands: + - [automation/taskcluster/androidTest/ui-test.sh, x86-screenshots-tests, app.apk, android-test.apk, '-1'] + treeherder: + symbol: debug(screenshots-x86) diff --git a/taskcluster/fenix_taskgraph/target_tasks.py b/taskcluster/fenix_taskgraph/target_tasks.py index 9694d1b38..66242ebff 100644 --- a/taskcluster/fenix_taskgraph/target_tasks.py +++ b/taskcluster/fenix_taskgraph/target_tasks.py @@ -73,3 +73,13 @@ def target_tasks_bump_android_components(full_task_graph, parameters, graph_conf return task.attributes.get("bump-type", "") == "android-components" return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)] + + +@_target_task("screenshots") +def target_tasks_screnshots(full_task_graph, parameters, graph_config): + """Select the set of tasks required to generate screenshots on a real device.""" + + def filter(task, parameters): + return task.attributes.get("screenshots", False) + + return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]