diff --git a/automation/taskcluster/decision_task.py b/automation/taskcluster/decision_task.py index 60a696ab9..80e0a5001 100644 --- a/automation/taskcluster/decision_task.py +++ b/automation/taskcluster/decision_task.py @@ -92,9 +92,5 @@ def nightly_to_production_app(builder, is_staging, version_name): build_task = builder.craft_assemble_release_task( variant, 'nightly-legacy', is_staging, version_name) - # The signing and push-apk tasks are generated by taskgraph - tasks = [build_task] - if not is_staging: - tasks.append(builder.craft_upload_apk_nimbledroid_task(build_task['label'])) - - return tasks + # Nimbledroid, signing and push-apk tasks are generated by taskgraph + return [build_task] diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py index 3e27155dd..a77567a7b 100644 --- a/automation/taskcluster/lib/tasks.py +++ b/automation/taskcluster/lib/tasks.py @@ -228,27 +228,6 @@ class TaskBuilder(object): treeherder=treeherder, ) - def craft_upload_apk_nimbledroid_task(self, assemble_task_label): - # For GeckoView, upload nightly (it has release config) by default, all Release builds have WV - return self._craft_build_ish_task( - name="Upload Release APK to Nimbledroid", - description='Upload APKs to Nimbledroid for performance measurement and tracking.', - command=' && '.join([ - 'curl --location "{}//artifacts/public/build/armeabi-v7a/geckoNightly/target.apk" > target.apk'.format(_DEFAULT_TASK_URL), - 'python automation/taskcluster/upload_apk_nimbledroid.py', - ]), - treeherder={ - 'jobKind': 'test', - 'machine': { - 'platform': 'android-all', - }, - 'symbol': 'compare-locale', - 'tier': 2, - }, - scopes=["secrets:get:project/mobile/fenix/nimbledroid"], - dependencies={'build': assemble_task_label}, - ) - def craft_detekt_task(self): return self._craft_clean_gradle_task( name='detekt', diff --git a/automation/taskcluster/upload_apk_nimbledroid.py b/automation/taskcluster/upload_apk_nimbledroid.py index ff6f669d1..29f41d8d5 100644 --- a/automation/taskcluster/upload_apk_nimbledroid.py +++ b/automation/taskcluster/upload_apk_nimbledroid.py @@ -12,6 +12,7 @@ import requests import json import urllib2 import os +import sys url = "https://nimbledroid.com/api/v2/apks" @@ -32,15 +33,18 @@ def uploadApk(apk,key): print json.dumps(response.json(), indent=4) -def uploadNightlyFenixApk(key): - apk_url = 'https://index.taskcluster.net/v1/task/project.mobile.fenix.v2.nightly.latest/artifacts/public/build/armeabi-v7a/geckoNightly/target.apk' +def uploadNightlyFenixApk(apk_url, key): apk_data = urllib2.urlopen(apk_url).read() with open('./fenix_example_nd.apk', 'wb') as f: - f.write(apk_data) + f.write(apk_data) uploadApk({'apk' : open('fenix_example_nd.apk')},key) + # Get JSON data from taskcluster secrets service -secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) +secrets = taskcluster.Secrets({ + 'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'), +}) data = secrets.get('project/mobile/fenix/nimbledroid') # upload the nightly build to Nimbledroid -uploadNightlyFenixApk(data['secret']['api_key']) +apk_url = sys.argv[1] +uploadNightlyFenixApk(apk_url, data['secret']['api_key']) diff --git a/taskcluster/ci/config.yml b/taskcluster/ci/config.yml index f13cec876..8ea82e63a 100644 --- a/taskcluster/ci/config.yml +++ b/taskcluster/ci/config.yml @@ -26,7 +26,7 @@ workers: provisioner: aws-provisioner-v1 implementation: docker-worker os: linux - worker-type: 'mobile-{level}-b-ref-browser' + worker-type: 'mobile-{level}-b-fenix' images: provisioner: aws-provisioner-v1 implementation: docker-worker diff --git a/taskcluster/ci/docker-image/kind.yml b/taskcluster/ci/docker-image/kind.yml new file mode 100644 index 000000000..db2e9969b --- /dev/null +++ b/taskcluster/ci/docker-image/kind.yml @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- + +loader: taskgraph.loader.transform:loader + +transforms: + - taskgraph.transforms.docker_image:transforms + - taskgraph.transforms.cached_tasks:transforms + - taskgraph.transforms.task:transforms + +jobs: + base: + symbol: I(base) + nimbledroid: + parent: base + symbol: I(nimbledroid) diff --git a/taskcluster/ci/nimbledroid/kind.yml b/taskcluster/ci/nimbledroid/kind.yml new file mode 100644 index 000000000..408c2b2c0 --- /dev/null +++ b/taskcluster/ci/nimbledroid/kind.yml @@ -0,0 +1,43 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +--- +loader: taskgraph.loader.transform:loader + +transforms: + - fenix_taskgraph.nimbledroid:transforms + - taskgraph.transforms.job:transforms + - taskgraph.transforms.task:transforms + +kind-dependencies: + - old-decision + +job-defaults: + description: Upload APKs to Nimbledroid for performance measurement and tracking + worker-type: b-android + worker: + docker-image: {in-tree: nimbledroid} + max-run-time: 600 + run-on-tasks-for: [] + treeherder: + kind: test + tier: 2 + platform: android-all/opt + scopes: + - secrets:get:project/mobile/fenix/nimbledroid + run: + using: run-task + use-caches: false + command: + - python + - /builds/worker/checkouts/src/automation/taskcluster/upload_apk_nimbledroid.py + - {artifact-reference: ""} + +jobs: + nightly: + attributes: + nightly: true + dependencies: + build: Build FenixNightlyLegacy task # Comes from the old decision task + treeherder: + symbol: nimbledroid diff --git a/taskcluster/docker/base/Dockerfile b/taskcluster/docker/base/Dockerfile new file mode 100644 index 000000000..44f3587d8 --- /dev/null +++ b/taskcluster/docker/base/Dockerfile @@ -0,0 +1,66 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM ubuntu:18.04 + +MAINTAINER Tom Prince "mozilla@hocat.ca" + +# Add worker user +RUN mkdir /builds && \ + useradd -d /builds/worker -s /bin/bash -m worker && \ + chown worker:worker /builds/worker && \ + mkdir /builds/worker/artifacts && \ + chown worker:worker /builds/worker/artifacts + +WORKDIR /builds/worker/ + +#---------------------------------------------------------------------------------------------------------------------- +#-- Configuration ----------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +ENV LANG en_US.UTF-8 + +# Do not use fancy output on taskcluster +ENV TERM dumb + +ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false + +#---------------------------------------------------------------------------------------------------------------------- +#-- System ------------------------------------------------------------------------------------------------------------ +#---------------------------------------------------------------------------------------------------------------------- + +RUN apt-get update -qq \ + # We need to install tzdata before all of the other packages. Otherwise it will show an interactive dialog that + # we cannot navigate while building the Docker image. + && apt-get install -y tzdata \ + && apt-get install -y openjdk-8-jdk \ + wget \ + expect \ + git \ + curl \ + python \ + python-pip \ + python3 \ + locales \ + unzip \ + mercurial \ + && apt-get clean + +RUN pip install --upgrade pip + +RUN locale-gen en_US.UTF-8 + +# %include-run-task + +ENV SHELL=/bin/bash \ + HOME=/builds/worker \ + PATH=/builds/worker/.local/bin:$PATH + + +VOLUME /builds/worker/checkouts +VOLUME /builds/worker/.cache + + +# run-task expects to run as root +USER root diff --git a/taskcluster/docker/nimbledroid/Dockerfile b/taskcluster/docker/nimbledroid/Dockerfile new file mode 100644 index 000000000..aea2727d0 --- /dev/null +++ b/taskcluster/docker/nimbledroid/Dockerfile @@ -0,0 +1,6 @@ +# %ARG DOCKER_IMAGE_PARENT +FROM $DOCKER_IMAGE_PARENT + +MAINTAINER Kate Glazko + +RUN pip install taskcluster requests diff --git a/taskcluster/fenix_taskgraph/nimbledroid.py b/taskcluster/fenix_taskgraph/nimbledroid.py new file mode 100644 index 000000000..edf900cf9 --- /dev/null +++ b/taskcluster/fenix_taskgraph/nimbledroid.py @@ -0,0 +1,27 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +""" +Apply some defaults and minor modifications to the jobs defined in the nimbledroid +kind. This transform won't be necessary anymore once the full tasgraph migration is +done +""" + +from __future__ import absolute_import, print_function, unicode_literals + +import copy +import json + +from taskgraph.transforms.base import TransformSequence +from taskgraph.util.treeherder import inherit_treeherder_from_dep +from taskgraph.util.schema import resolve_keyed_by + +transforms = TransformSequence() + + +@transforms.add +def filter_out_non_nightly_legacy(config, tasks): + for dep_task in config.kind_dependencies_tasks: + if dep_task.label == 'Build FenixNightlyLegacy task': + for task in tasks: + yield task