From de8bc2f8a512e44bd3a9219da7dcc1a94fff51be Mon Sep 17 00:00:00 2001 From: Johan Lorenzo Date: Thu, 7 Nov 2019 17:38:17 +0100 Subject: [PATCH] Split UI test --- app/build.gradle | 10 ++ automation/taskcluster/androidTest/ui-test.sh | 31 +++-- taskcluster/ci/build/kind.yml | 13 ++ taskcluster/ci/config.yml | 1 + taskcluster/ci/pr/kind.yml | 1 + taskcluster/ci/test/kind.yml | 28 ---- taskcluster/ci/ui-test/kind.yml | 50 +++++++ taskcluster/fenix_taskgraph/job.py | 128 +++++++++++++----- 8 files changed, 188 insertions(+), 74 deletions(-) create mode 100644 taskcluster/ci/ui-test/kind.yml diff --git a/app/build.gradle b/app/build.gradle index f1b83d1e2..72624c783 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -552,6 +552,16 @@ task printVariants { engine: it.productFlavors.find { it.dimension == 'engine' }.name, name: it.name, ]} + // AndroidTest is a special case not included above + variants.add([ + apks: [[ + abi: 'noarch', + fileName: 'app-geckoNightly-debug-androidTest.apk', + ]], + build_type: 'androidTest', + engine: 'geckoNightly', + name: 'androidTest', + ]) println 'variants: ' + groovy.json.JsonOutput.toJson(variants) } } diff --git a/automation/taskcluster/androidTest/ui-test.sh b/automation/taskcluster/androidTest/ui-test.sh index 05435ad5a..62e5c702f 100755 --- a/automation/taskcluster/androidTest/ui-test.sh +++ b/automation/taskcluster/androidTest/ui-test.sh @@ -33,6 +33,13 @@ display_help() { echo } +get_abs_filename() { + # $1 : relative filename + relative_filename="$1" + echo "$(cd "$(dirname "$relative_filename")" && pwd)/$(basename "$relative_filename")" +} + + # Basic parameter check if [[ $# -lt 1 ]]; then echo "Error: please provide at least one build variant (arm|x86)" @@ -41,23 +48,24 @@ if [[ $# -lt 1 ]]; then fi device_type="$1" # arm64-v8a | armeabi-v7a | x86_64 | x86 -if [[ ! -z "$2" ]]; then - num_shards=$2 +APK_APP="$2" +APK_TEST="$3" +if [[ ! -z "$4" ]]; then + num_shards=$4 fi JAVA_BIN="/usr/bin/java" PATH_TEST="./automation/taskcluster/androidTest" -PATH_APK="./app/build/outputs/apk/geckoNightly/debug" FLANK_BIN="/builds/worker/test-tools/flank.jar" echo echo "ACTIVATE SERVICE ACCT" echo # this is where the Google Testcloud project ID is set -gcloud config set project "$GOOGLE_PROJECT" +gcloud config set project "$GOOGLE_PROJECT" echo -gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS" +gcloud auth activate-service-account --key-file "$GOOGLE_APPLICATION_CREDENTIALS" echo echo @@ -66,7 +74,6 @@ echo set +e if [[ "${device_type}" =~ ^(arm64-v8a|armeabi-v7a|x86_64|x86)$ ]]; then - APK_APP="${PATH_APK}/app-geckoNightly-${device_type}-debug.apk" flank_template="${PATH_TEST}/flank-${device_type}.yml" echo "device_type: ${device_type}" else @@ -74,8 +81,9 @@ else exitcode=1 fi -APK_TEST="./app/build/outputs/apk/androidTest/geckoNightly/debug/app-geckoNightly-debug-androidTest.apk" -echo "APK_PATH: ${APK_PATH}" +APK_APP="$(get_abs_filename $APK_APP)" +APK_TEST="$(get_abs_filename $APK_TEST)" +echo "APK_APP: ${APK_APP}" echo "APK_TEST: ${APK_TEST}" # function to exit script with exit code from test run. @@ -92,7 +100,7 @@ function failure_check() { echo echo "FLANK VERSION" echo -$JAVA_BIN -jar $FLANK_BIN --version +$JAVA_BIN -jar $FLANK_BIN --version echo echo @@ -118,12 +126,9 @@ echo echo "RESULTS" echo ls -la ./results -echo +echo echo echo "All UI test(s) have passed!" echo echo - - - diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index ae6c6bee1..d506037dd 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -45,6 +45,19 @@ jobs: treeherder: symbol: debug(B) + android-test: + attributes: + code-review: true + run-on-tasks-for: [github-pull-request, github-push] + run: + geckoview-engine: geckoNightly + gradle-build-type: androidTest + apk-artifact-template: + # 2 differences here: "androidTest/" is added and "{gradle_build_type}" is forced to "debug" + path: '/builds/worker/checkouts/src/app/build/outputs/apk/androidTest/{geckoview_engine}/debug/{fileName}' + treeherder: + symbol: androidTest(B) + performance-test: run: geckoview-engine: geckoNightly diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index 3a9ebdeac..4000d84ef 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -2,6 +2,7 @@ trust-domain: mobile treeherder: group-names: + 'androidTest': 'Tasks related to the androidTest APK' 'beta': 'Nightly-related tasks' 'debug': 'Builds made for testing' 'forPerformanceTest': 'Builds made for Raptor and other performance tests' diff --git a/taskcluster/ci/pr/kind.yml b/taskcluster/ci/pr/kind.yml index 010854101..7286e34c8 100644 --- a/taskcluster/ci/pr/kind.yml +++ b/taskcluster/ci/pr/kind.yml @@ -8,6 +8,7 @@ kind-dependencies: - build - lint - test + - ui-test transforms: - taskgraph.transforms.code_review:transforms diff --git a/taskcluster/ci/test/kind.yml b/taskcluster/ci/test/kind.yml index c8f4eaa69..b7ca10caa 100644 --- a/taskcluster/ci/test/kind.yml +++ b/taskcluster/ci/test/kind.yml @@ -46,31 +46,3 @@ jobs: - name: public/reports/index.html path: /builds/worker/checkouts/src/app/build/reports/tests/testGeckoNightlyDebugUnitTest/index.html type: file - ui: - attributes: - build-type: debug - code-review: true - include-pull-request-number: true - run-on-tasks-for: [github-pull-request, github-push] - run: - # TODO Generate APKs in a build task instead - gradlew: ['clean', 'assembleDebug', 'assembleAndroidTest'] - post-gradlew: - - ['automation/taskcluster/androidTest/ui-test.sh', 'x86', '-1'] - secrets: - - name: project/mobile/fenix/firebase - key: firebaseToken - path: .firebase_token.json - json: true - treeherder: - symbol: ui - platform: 'ui-test/opt' - worker: - docker-image: {in-tree: ui-tests} - env: - GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' - GOOGLE_PROJECT: moz-fenix - artifacts: - - name: public - path: /build/fenix/results - type: directory diff --git a/taskcluster/ci/ui-test/kind.yml b/taskcluster/ci/ui-test/kind.yml new file mode 100644 index 000000000..0a7e08c87 --- /dev/null +++ b/taskcluster/ci/ui-test/kind.yml @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - fenix_taskgraph.transforms.test:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +jobs: + x86: + attributes: + build-type: debug + code-review: true + dependencies: + signing: signing-debug + signing-android-test: signing-android-test + description: Test Fenix + include-pull-request-number: true + run-on-tasks-for: [github-pull-request, github-push] + run: + commands: + - [wget, {artifact-reference: ''}, '-O', app.apk] + - [wget, {artifact-reference: ''}, '-O', android-test.apk] + - [automation/taskcluster/androidTest/ui-test.sh, x86, app.apk, android-test.apk, '-1'] + secrets: + - name: project/mobile/fenix/firebase + key: firebaseToken + path: .firebase_token.json + json: true + using: run-commands + use-caches: false + treeherder: + kind: test + platform: 'ui-test/opt' + symbol: debug(ui-test-x86) + tier: 2 + worker: + docker-image: {in-tree: ui-tests} + max-run-time: 7200 + env: + GOOGLE_APPLICATION_CREDENTIALS: '.firebase_token.json' + GOOGLE_PROJECT: moz-fenix + artifacts: + - name: public + path: /build/fenix/results + type: directory + worker-type: b-android diff --git a/taskcluster/fenix_taskgraph/job.py b/taskcluster/fenix_taskgraph/job.py index 1f1b21622..4169b1663 100644 --- a/taskcluster/fenix_taskgraph/job.py +++ b/taskcluster/fenix_taskgraph/job.py @@ -6,31 +6,51 @@ from __future__ import absolute_import, print_function, unicode_literals from taskgraph.transforms.job import run_job_using, configure_taskdesc_for_run from taskgraph.util import path -from taskgraph.util.schema import Schema +from taskgraph.util.schema import Schema, taskref_or_string from voluptuous import Required, Optional from six import text_type from pipes import quote as shell_quote -gradlew_schema = Schema( - { - Required("using"): "gradlew", - Optional("pre-gradlew"): [[text_type]], - Required("gradlew"): [text_type], - Optional("post-gradlew"): [[text_type]], - # Base work directory used to set up the task. - Required("workdir"): text_type, - Optional("use-caches"): bool, - Optional("secrets"): [ - { - Required("name"): text_type, - Required("path"): text_type, - Required("key"): text_type, - Optional("json"): bool, - } - ], - } -) +secret_schema = { + Required("name"): text_type, + Required("path"): text_type, + Required("key"): text_type, + Optional("json"): bool, +} + +gradlew_schema = Schema({ + Required("using"): "gradlew", + Optional("pre-gradlew"): [[text_type]], + Required("gradlew"): [text_type], + Optional("post-gradlew"): [[text_type]], + # Base work directory used to set up the task. + Required("workdir"): text_type, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], +}) + +run_commands_schema = Schema({ + Required("using"): "run-commands", + Required("commands"): [[taskref_or_string]], + Required("workdir"): text_type, + Optional("use-caches"): bool, + Optional("secrets"): [secret_schema], +}) + + +@run_job_using("docker-worker", "run-commands", schema=run_commands_schema) +def configure_run_commands_schema(config, job, taskdesc): + run = job["run"] + pre_commands = [ + _generate_secret_command(secret) for secret in run.get("secrets", []) + ] + all_commands = pre_commands + run.pop("commands", []) + + run["command"] = _convert_commands_to_string(all_commands) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) + configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) @run_job_using("docker-worker", "gradlew", schema=gradlew_schema) @@ -42,20 +62,13 @@ def configure_gradlew(config, job, taskdesc): {"ANDROID_SDK_ROOT": path.join(run["workdir"], "android-sdk-linux")} ) - # defer to the run_task implementation - run["command"] = _extract_command(run) - secrets = run.pop("secrets", []) - scopes = taskdesc.setdefault("scopes", []) - new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] - new_secret_scopes = list(set(new_secret_scopes)) # Scopes must not have any duplicates - scopes.extend(new_secret_scopes) - - run["cwd"] = "{checkout}" - run["using"] = "run-task" + run["command"] = _extract_gradlew_command(run) + _inject_secrets_scopes(run, taskdesc) + _set_run_task_attributes(job) configure_taskdesc_for_run(config, job, taskdesc, job["worker"]["implementation"]) -def _extract_command(run): +def _extract_gradlew_command(run): pre_gradle_commands = run.pop("pre-gradlew", []) pre_gradle_commands += [ _generate_secret_command(secret) for secret in run.get("secrets", []) @@ -65,8 +78,7 @@ def _extract_command(run): post_gradle_commands = run.pop("post-gradlew", []) commands = pre_gradle_commands + [gradle_command] + post_gradle_commands - shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in commands] - return " && ".join(shell_quoted_commands) + return _convert_commands_to_string(commands) def _generate_secret_command(secret): @@ -80,3 +92,53 @@ def _generate_secret_command(secret): secret_command.append("--json") return secret_command + + +def _convert_commands_to_string(commands): + should_artifact_reference = False + should_task_reference = False + + sanitized_commands = [] + for command in commands: + sanitized_parts = [] + for part in command: + if isinstance(part, dict): + if "artifact-reference" in part: + part_string = part["artifact-reference"] + should_artifact_reference = True + elif "task-reference" in part: + part_string = part["task-reference"] + should_task_reference = True + else: + raise ValueError('Unsupported dict: {}'.format(part)) + else: + part_string = part + + sanitized_parts.append(part_string) + sanitized_commands.append(sanitized_parts) + + shell_quoted_commands = [" ".join(map(shell_quote, command)) for command in sanitized_commands] + full_string_command = " && ".join(shell_quoted_commands) + + if should_artifact_reference and should_task_reference: + raise NotImplementedError('"arifact-reference" and "task-reference" cannot be both used') + elif should_artifact_reference: + return {"artifact-reference": full_string_command} + elif should_task_reference: + return {"task-reference": full_string_command} + else: + return full_string_command + + +def _inject_secrets_scopes(run, taskdesc): + secrets = run.pop("secrets", []) + scopes = taskdesc.setdefault("scopes", []) + new_secret_scopes = ["secrets:get:{}".format(secret["name"]) for secret in secrets] + new_secret_scopes = list(set(new_secret_scopes)) # Scopes must not have any duplicates + scopes.extend(new_secret_scopes) + + +def _set_run_task_attributes(job): + run = job["run"] + run["cwd"] = "{checkout}" + run["using"] = "run-task"