From b39afe1548fd42ed8e3aaf2d9c1704839fd2402d Mon Sep 17 00:00:00 2001 From: Richard Pappalardo Date: Fri, 26 Jul 2019 08:08:01 -0700 Subject: [PATCH] Add taskcluster job for UI tests (#4088) --- .gitignore | 14 ++ app/build.gradle | 6 +- .../mozilla/fenix/ui/robots/SettingsRobot.kt | 4 +- .../taskcluster/androidTest/flank-arm.yml | 56 ++++++++ .../androidTest/flank-x86-tablet.yml | 54 ++++++++ .../taskcluster/androidTest/flank-x86.yml | 52 +++++++ automation/taskcluster/androidTest/ui-test.sh | 130 ++++++++++++++++++ automation/taskcluster/decision_task.py | 24 ++-- automation/taskcluster/helper/get-secret.py | 14 +- automation/taskcluster/lib/tasks.py | 51 ++++++- 10 files changed, 383 insertions(+), 22 deletions(-) create mode 100644 automation/taskcluster/androidTest/flank-arm.yml create mode 100644 automation/taskcluster/androidTest/flank-x86-tablet.yml create mode 100644 automation/taskcluster/androidTest/flank-x86.yml create mode 100755 automation/taskcluster/androidTest/ui-test.sh diff --git a/.gitignore b/.gitignore index 3817a69d3..2cd257f9c 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,10 @@ captures/ *.iml .idea/ +# Vim swap files +*.sw[op] + + # Keystore files # Uncomment the following lines if you do not want to check your keystore files in. #*.jks @@ -82,3 +86,13 @@ gen-external-apklibs # Python Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +venv/ + + +# UI test artifacts +.firebase_token* +results/ +test_artifacts/ +/build/test-tools/google-cloud-sdk/ +/build/test-tools/*.jar +/build/test-tools/*.gz diff --git a/app/build.gradle b/app/build.gradle index 2d8c6a0e8..bfff7a336 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -405,7 +405,11 @@ dependencies { androidTestImplementation Deps.mockwebserver testImplementation Deps.mozilla_support_test testImplementation Deps.androidx_junit - testImplementation Deps.robolectric + testImplementation (Deps.robolectric) { + exclude group: 'org.apache.maven' + } + + testImplementation 'org.apache.maven:maven-ant-tasks:2.1.3' implementation Deps.fragment_testing testImplementation Deps.places_forUnitTests diff --git a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt index 278cf740e..acdf7badb 100644 --- a/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt +++ b/app/src/androidTest/java/org/mozilla/fenix/ui/robots/SettingsRobot.kt @@ -42,12 +42,12 @@ class SettingsRobot { private fun assertSettingsView() { // verify that we are in the correct library view assertBasicsHeading() - assertAdvancedHeading() + assertPrivacyHeading() } private fun assertBasicsHeading() = onView(ViewMatchers.withText("Basics")) .check(matches(withEffectiveVisibility(Visibility.VISIBLE))) -private fun assertAdvancedHeading() = onView(ViewMatchers.withText("Advanced")) +private fun assertPrivacyHeading() = onView(ViewMatchers.withText("Privacy")) .check(matches(withEffectiveVisibility(Visibility.VISIBLE))) private fun goBackButton() = onView(CoreMatchers.allOf(withContentDescription("Navigate up"))) diff --git a/automation/taskcluster/androidTest/flank-arm.yml b/automation/taskcluster/androidTest/flank-arm.yml new file mode 100644 index 000000000..0217414b2 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-arm.yml @@ -0,0 +1,56 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # test and app are the only required args + app: /APP/PATH + test: /TEST/PATH + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + device: + - model: shamu + version: 21 + - model: sailfish + version: 25 + - model: sailfish + version: 28 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: -1 + # repeat tests - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + repeat-tests: 1 + # always run - these tests are inserted at the beginning of every shard + # useful if you need to grant permissions or login before other tests run + #test-targets-always-run: + #- class com.example.app.ExampleUiTest#testPasses + # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/flank-x86-tablet.yml b/automation/taskcluster/androidTest/flank-x86-tablet.yml new file mode 100644 index 000000000..f44669f95 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86-tablet.yml @@ -0,0 +1,54 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # test and app are the only required args + app: /APP/PATH + test: /TEST/PATH + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + device: + - model: Nexus9 + version: 21 + - model: Nexus9 + version: 22 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: -1 + # repeat tests - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + repeat-tests: 1 + # always run - these tests are inserted at the beginning of every shard + # useful if you need to grant permissions or login before other tests run + #test-targets-always-run: + #- class com.example.app.ExampleUiTest#testPasses + # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/flank-x86.yml b/automation/taskcluster/androidTest/flank-x86.yml new file mode 100644 index 000000000..2e5c7c319 --- /dev/null +++ b/automation/taskcluster/androidTest/flank-x86.yml @@ -0,0 +1,52 @@ +# gcloud args match the official gcloud cli +# https://cloud.google.com/sdk/gcloud/reference/firebase/test/android/run +gcloud: + results-bucket: fenix_test_artifacts + record-video: true + + # The maximum possible testing time is 30m on physical devices and 60m on virtual devices. + timeout: 30m + # will start test then close socket. no reports will be generated. + # to retrieve results later, use the "refresh" command + # reports will be generated from /results/matrix_ids.json + #async: true + # will start test then leave socket open. reports will be published + # to /results + # see: https://github.com/TestArmada/flank/issues/339 + async: false + + # results-history-name + # by default, set to app name + # declare results-history-name to create a separate dropdown menu in Firebase + # see: https://github.com/TestArmada/flank/issues/341 + #results-history-name: tmp_parallel + + # test and app are the only required args + app: /app/path + test: /test/path + + auto-google-login: true + use-orchestrator: true + environment-variables: + clearPackageData: true + directories-to-pull: + - /sdcard/screenshots + performance-metrics: true + + device: + - model: Nexus7 + version: 21 + +flank: + project: GOOGLE_PROJECT + # test shards - the amount of groups to split the test suite into + # set to -1 to use one shard per test. + max-test-shards: -1 + # repeat tests - the amount of times to run the tests. + # 1 runs the tests once. 10 runs all the tests 10x + repeat-tests: 1 + # always run - these tests are inserted at the beginning of every shard + # useful if you need to grant permissions or login before other tests run + #test-targets-always-run: + #- class com.example.app.ExampleUiTest#testPasses + # - class org.mozilla.focus.activty.SwitchContextTest#testPasses diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh new file mode 100755 index 000000000..b65e9a68e --- /dev/null +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +# This script does the following: +# 1. Retrieves gcloud service account token +# 2. Activates gcloud service account +# 3. Connects to google Firebase (using TestArmada's Flank tool) +# 4. Executes UI tests +# 5. Puts test artifacts into the test_artifacts folder + +# NOTE: +# Flank supports sharding across multiple devices at a time, but gcloud API +# only supports 1 defined APK per test run. + + +# If a command fails then do not proceed and fail this script too. +set -e + +######################### +# The command line help # +######################### +display_help() { + echo "Usage: $0 Build_Variant [Number_Shards...]" + echo + echo "Examples:" + echo "To run UI tests on ARM device shard (1 test / shard)" + echo "$ ui-test.sh arm -1" + echo + echo "To run UI tests on X86 device (on 3 shards)" + echo "$ ui-test.sh feature x86 3" + echo +} + +# Basic parameter check +if [[ $# -lt 1 ]]; then + echo "Error: please provide at least one build variant (arm|x86)" + display_help + exit 1 +fi + +device_type="$1" # arm | x86 +if [[ ! -z "$2" ]]; then + num_shards=$2 +fi + +JAVA_BIN="/usr/bin/java" +PATH_TEST="./automation/taskcluster/androidTest" +FLANK_BIN="/build/test-tools/flank.jar" +FLANK_CONF_ARM="${PATH_TEST}/flank-arm.yml" +FLANK_CONF_X86="${PATH_TEST}/flank-x86.yml" + +echo +echo "RETRIEVE SERVICE ACCT TOKEN" +echo +python automation/taskcluster/helper/get-secret.py --json -s project/mobile/fenix/firebase -k firebaseToken -f $GOOGLE_APPLICATION_CREDENTIALS +echo +echo + +echo +echo "ACTIVATE SERVICE ACCT" +echo +# this is where the Google Testcloud project ID is set +gcloud config set project "$GOOGLE_PROJECT" +echo + +gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS" +echo +echo + +# From now on disable exiting on error. If the tests fail we want to continue +# and try to download the artifacts. We will exit with the actual error code later. +set +e + +if [[ "${device_type,,}" == "x86" ]] +then + deviceType="X86" + flank_template="$FLANK_CONF_X86" +else + deviceType="Arm" + flank_template="$FLANK_CONF_ARM" +fi + +APK_APP="./app/build/outputs/apk/${deviceType,,}/debug/app-${deviceType,,}-debug.apk" +APK_TEST="./app/build/outputs/apk/androidTest/${deviceType,,}/debug/app-${deviceType,,}-debug-androidTest.apk" + + +# function to exit script with exit code from test run. +# (Only 0 if all test executions passed) +function failure_check() { + if [[ $exitcode -ne 0 ]]; then + echo + echo + echo "ERROR: UI test run failed, please check above URL" + fi + exit $exitcode +} + +echo +echo "EXECUTE TEST(S)" +echo +$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT +exitcode=$? +failure_check +echo +echo + +echo +echo "COPY ARTIFACTS" +echo +cp -r ./results ./test_artifacts +exitcode=$? +failure_check +echo +echo + +echo +echo "RESULTS" +echo +ls -la ./results +echo +echo + +echo "All UI test(s) have passed!" +echo +echo + + + diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 00938736b..768f2dd06 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -51,8 +51,8 @@ BUILDER = TaskBuilder( ) -def pr_or_push(is_push): - if not is_push and SKIP_TASKS_TRIGGER in PR_TITLE: +def pr(): + if SKIP_TASKS_TRIGGER in PR_TITLE: print("Pull request title contains", SKIP_TASKS_TRIGGER) print("Exit") return {} @@ -74,10 +74,18 @@ def pr_or_push(is_push): ): other_tasks[taskcluster.slugId()] = craft_function() - if is_push and SHORT_HEAD_BRANCH == 'master': + return (build_tasks, signing_tasks, other_tasks) + + +def push(): + all_tasks = pr() + other_tasks = all_tasks[-1] + other_tasks[taskcluster.slugId()] = BUILDER.craft_ui_tests_task() + + if SHORT_HEAD_BRANCH == 'master': other_tasks[taskcluster.slugId()] = BUILDER.craft_dependencies_task() - return (build_tasks, signing_tasks, other_tasks) + return all_tasks def raptor(is_staging): @@ -199,10 +207,10 @@ if __name__ == "__main__": command = result.command taskcluster_queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) - if command == 'pull-request': - ordered_groups_of_tasks = pr_or_push(False) - elif command == 'push': - ordered_groups_of_tasks = pr_or_push(True) + if command in ('pull-request'): + ordered_groups_of_tasks = pr() + elif command in ('push'): + ordered_groups_of_tasks = push() elif command == 'raptor': ordered_groups_of_tasks = raptor(result.staging) elif command == 'nightly': diff --git a/automation/taskcluster/helper/get-secret.py b/automation/taskcluster/helper/get-secret.py index bb55175d3..972c5fc2e 100644 --- a/automation/taskcluster/helper/get-secret.py +++ b/automation/taskcluster/helper/get-secret.py @@ -4,23 +4,20 @@ import argparse import base64 +import json import os import errno import taskcluster -def write_secret_to_file(path, data, key, base64decode=False, append=False, prefix=''): +def write_secret_to_file(path, data, key, base64decode=False, json_secret=False, append=False, prefix=''): path = os.path.join(os.path.dirname(__file__), '../../../' + path) - try: - os.makedirs(os.path.dirname(path)) - except OSError as error: - if error.errno != errno.EEXIST: - raise - with open(path, 'a' if append else 'w') as f: value = data['secret'][key] if base64decode: value = base64.b64decode(value) + if json_secret: + value = json.dumps(value) f.write(prefix + value) @@ -37,13 +34,14 @@ def main(): parser.add_argument('-k', dest='key', action="store", help='key of the secret') parser.add_argument('-f', dest="path", action="store", help='file to save secret to') parser.add_argument('--decode', dest="decode", action="store_true", default=False, help='base64 decode secret before saving to file') + parser.add_argument('--json', dest="json", action="store_true", default=False, help='serializes the secret to JSON format') parser.add_argument('--append', dest="append", action="store_true", default=False, help='append secret to existing file') parser.add_argument('--prefix', dest="prefix", action="store", default="", help='add prefix when writing secret to file') result = parser.parse_args() secret = fetch_secret_from_taskcluster(result.secret) - write_secret_to_file(result.path, secret, result.key, result.decode, result.append, result.prefix) + write_secret_to_file(result.path, secret, result.key, result.decode, result.json, result.append, result.prefix) if __name__ == "__main__": diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index f8518cbf7..3ab6b5fa1 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -15,6 +15,8 @@ DEFAULT_EXPIRES_IN = '1 year' DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk' _OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix' _DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task' +GOOGLE_PROJECT = "moz-fenix" +GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json' class TaskBuilder(object): @@ -161,6 +163,48 @@ class TaskBuilder(object): }, ) + + def craft_ui_tests_task(self): + artifacts = { + "public": { + "type": "directory", + "path": "/build/fenix/results", + "expires": taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)) + } + } + + env_vars = { + "GOOGLE_PROJECT": "moz-fenix", + "GOOGLE_APPLICATION_CREDENTIALS": ".firebase_token.json" + } + + gradle_commands = ( + './gradlew --no-daemon clean assembleArmDebug assembleArmDebugAndroidTest', + ) + + test_commands = ( + 'automation/taskcluster/androidTest/ui-test.sh arm -1', + ) + + command = ' && '.join( + cmd + for commands in (gradle_commands, test_commands) + for cmd in commands + if cmd + ) + + return self._craft_build_ish_task( + name='Fenix - UI test', + description='Execute Gradle tasks for UI tests', + command=command, + scopes=[ + 'secrets:get:project/mobile/fenix/firebase' + ], + artifacts=artifacts, + env_vars=env_vars, + ) + + def craft_detekt_task(self): return self._craft_clean_gradle_task( name='detekt', @@ -268,12 +312,13 @@ class TaskBuilder(object): def _craft_build_ish_task( self, name, description, command, dependencies=None, artifacts=None, scopes=None, - routes=None, treeherder=None + routes=None, treeherder=None, env_vars=None, ): dependencies = [] if dependencies is None else dependencies artifacts = {} if artifacts is None else artifacts scopes = [] if scopes is None else scopes routes = [] if routes is None else routes + env_vars = {} if env_vars is None else env_vars checkout_command = ' && '.join([ "export TERM=dumb", @@ -289,11 +334,11 @@ class TaskBuilder(object): features['chainOfTrust'] = True if any(scope.startswith('secrets:') for scope in scopes): features['taskclusterProxy'] = True - payload = { "features": features, + "env": env_vars, "maxRunTime": 7200, - "image": "mozillamobile/fenix:1.3", + "image": "mozillamobile/fenix:1.4", "command": [ "/bin/bash", "--login",