Fixes #6760 - Show UI test results in TC task (#10089)

automation/taskcluster/androidTest/parse-ui-test.py

@@ -0,0 +1,63 @@
+#!/usr/bin/python3
+
+from __future__ import print_function
+
+import sys
+import argparse
+from pathlib import Path
+import json
+import yaml
+
+def parse_args(cmdln_args):
+    parser = argparse.ArgumentParser(description="Parse UI test logs an results")
+    parser.add_argument(
+        "--output-md",
+        type=argparse.FileType("w", encoding="utf-8"),
+        help="Output markdown file.",
+        required=True,
+    )
+    parser.add_argument(
+        "--log",
+        type=argparse.FileType("r", encoding="utf-8"),
+        help="Log output of flank.",
+        required=True,
+    )
+    parser.add_argument(
+        "--results", type=Path, help="Directory containing flank results", required=True
+    )
+    parser.add_argument(
+        "--exit-code", type=int, help="Exit code of flank.", required=True
+    )
+    parser.add_argument("--device-type", help="Type of device ", required=True)
+    return parser.parse_args(args=cmdln_args)
+
+
+def extract_android_args(log):
+    return yaml.safe_load(log.split("AndroidArgs\n")[1].split("RunTests\n")[0])
+
+
+def main():
+    args = parse_args(sys.argv[1:])
+
+    log = args.log.read()
+    matrix_ids = json.loads(args.results.joinpath("matrix_ids.json").read_text())
+    #with args.results.joinpath("flank.yml") as f:
+    #    flank_config = yaml.safe_load(f)
+
+    android_args = extract_android_args(log)
+
+    print = args.output_md.write
+
+    print("# Devices\n")
+    print(yaml.safe_dump(android_args["gcloud"]["device"]))
+
+    print("# Results\n")
+    print("| matrix | result | logs |\n")
+    print("| --- | --- | --- |\n")
+    for matrix, matrix_result in matrix_ids.items():
+        print("| {matrixId} | {outcome} | [logs]({webLink}) |\n".format(**matrix_result))
+
+
+if __name__ == "__main__":
+    main()
+

automation/taskcluster/androidTest/ui-test.sh

@@ -56,6 +56,8 @@ fi
 JAVA_BIN="/usr/bin/java"
 PATH_TEST="./automation/taskcluster/androidTest"
 FLANK_BIN="/builds/worker/test-tools/flank.jar"
+ARTIFACT_DIR="/builds/worker/artifacts"
+RESULTS_DIR="${ARTIFACT_DIR}/results"
 
 echo
 echo "ACTIVATE SERVICE ACCT"
@@ -101,16 +103,23 @@ function failure_check() {
     fi
 
     echo
-    echo "COPY ARTIFACTS"
+    echo "RESULTS"
     echo
-    cp -r ./results /builds/worker/artifacts
+    ls -la "${RESULTS_DIR}" 
+
 
     echo
     echo "RESULTS"
     echo
-    ls -la ./results
-    echo
-    echo
+
+    mkdir -p /builds/worker/artifacts/github
+    chmod +x $PATH_TEST/parse-ui-test.py
+    $PATH_TEST/parse-ui-test.py \
+        --exit-code "${exitcode}" \
+        --log flank.log \
+        --results "${RESULTS_DIR}" \
+        --output-md "${ARTIFACT_DIR}/github/customCheckRunText.md" \
+	--device-type "${device_type}"
 }
 
 echo
@@ -123,7 +132,16 @@ echo
 echo
 echo "EXECUTE TEST(S)"
 echo
-$JAVA_BIN -jar $FLANK_BIN android run --config=$flank_template --max-test-shards=$num_shards --app=$APK_APP --test=$APK_TEST --project=$GOOGLE_PROJECT
+# Note that if --local-results-dir is "results", timestamped sub-directory will
+# contain the results. For any other value, the directory itself will have the results.
+$JAVA_BIN -jar $FLANK_BIN android run \
+	--config=$flank_template \
+	--max-test-shards=$num_shards \
+	--app=$APK_APP --test=$APK_TEST \
+	--local-result-dir="${RESULTS_DIR}" \
+	--project=$GOOGLE_PROJECT \
+	| tee flank.log
+
 exitcode=$?
 failure_check
 

taskcluster/docker/base/Dockerfile

@@ -39,6 +39,7 @@ RUN apt-get update -qq \
                           python \
                           python-pip \
                           python3 \
+                          python3-yaml \
                           locales \
                           unzip \
 			  mercurial \