diff --git a/.taskcluster.yml b/.taskcluster.yml new file mode 100644 index 000000000..0b831b6b6 --- /dev/null +++ b/.taskcluster.yml @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. +version: 1 +policy: + pullRequests: public +tasks: + - $if: 'tasks_for == "cron"' + then: + $let: + decision_task_id: {$eval: as_slugid("decision_task")} + expires_in: {$fromNow: '1 year'} + repository: ${event.repository.clone_url} + scheduler_id: focus-nightly-sched + is_mozilla_mobile_repo: + $eval: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix' + track: + $if: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix' + then: 'nightly' + else: 'staging-nightly' + in: + taskId: ${decision_task_id} + taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust + created: {$fromNow: ''} + deadline: {$fromNow: '2 hours'} + expires: ${expires_in} + schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust + provisionerId: aws-provisioner-v1 + workerType: gecko-focus # This workerType has ChainOfTrust enabled + requires: all-completed # Must be explicit because of Chain of Trust + priority: medium + retries: 5 + scopes: + $flatten: + - queue:scheduler-id:${scheduler_id} + - queue:create-task:highest:aws-provisioner-v1/gecko-focus + - project:mobile:fenix:releng:signing:format:autograph_fenix + - $if: is_mozilla_mobile_repo + then: + - queue:create-task:highest:scriptworker-prov-v1/mobile-signing-v1 + - queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-v1 + - project:mobile:fenix:releng:signing:cert:release-signing + - project:mobile:fenix:releng:googleplay:product:fenix + - queue:route:index.project.mobile.fenix.signed-nightly.* + else: + - queue:create-task:highest:scriptworker-prov-v1/mobile-signing-dep-v1 + - queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-dep-v1 + - project:mobile:fenix:releng:signing:cert:dep-signing + - project:mobile:fenix:releng:googleplay:product:fenix:dep + - queue:route:index.project.mobile.fenix.staging-signed-nightly.* + payload: + maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks + image: mozillamobile/fenix:1.3 + features: + taskclusterProxy: true + chainOfTrust: true + env: + TASK_ID: ${decision_task_id} + SCHEDULER_ID: ${scheduler_id} + MOBILE_HEAD_REPOSITORY: ${repository} + MOBILE_HEAD_BRANCH: ${event.release.target_commitish} + MOBILE_HEAD_REV: ${event.release.tag_name} + MOBILE_TRIGGERED_BY: ${event.sender.login} + command: + - /bin/bash + - --login + - -cx + - >- + cd .. + && git clone ${repository} repository + && cd repository + && pip install arrow + && python tools/taskcluster/decision_task_nightly.py \ + --track ${track} \ + --commit \ + --output /opt/repository/app/build/outputs/apk \ + --apk arm/release/app-arm-release-unsigned.apk \ + --apk x86/release/app-x86-release-unsigned.apk \ + --date ${now} + artifacts: + public/task-graph.json: + type: file + path: /opt/repository/task-graph.json + expires: ${expires_in} + public/actions.json: + type: file + path: /opt/repository/actions.json + expires: ${expires_in} + public/parameters.yml: + type: file + path: /opt/repository/parameters.yml + expires: ${expires_in} + extra: + cron: {$json: {$eval: 'cron'}} + tasks_for: ${tasks_for} + metadata: + name: Fenix Nightly Decision Task + description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id}) + owner: ${event.sender.login}@users.noreply.github.com + source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml diff --git a/automation/taskcluster/decision_task_nightly.py b/automation/taskcluster/decision_task_nightly.py new file mode 100644 index 000000000..14130a865 --- /dev/null +++ b/automation/taskcluster/decision_task_nightly.py @@ -0,0 +1,151 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +Decision task for nightly releases. +""" + +from __future__ import print_function + +import argparse +import arrow +import json +import lib.tasks +import os +import taskcluster + +TASK_ID = os.environ.get('TASK_ID') +SCHEDULER_ID = os.environ.get('SCHEDULER_ID') +GITHUB_HTTP_REPOSITORY = os.environ.get('MOBILE_HEAD_REPOSITORY') +HEAD_REV = os.environ.get('MOBILE_HEAD_REV') + +BUILDER = lib.tasks.TaskBuilder( + task_id=TASK_ID, + owner="android-components-team@mozilla.com", + source='{}/raw/{}/.taskcluster.yml'.format(GITHUB_HTTP_REPOSITORY, HEAD_REV), + scheduler_id=SCHEDULER_ID +) + + +def generate_build_task(apks): + artifacts = {'public/{}'.format(os.path.basename(apk)): { + "type": 'file', + "path": apk, + "expires": taskcluster.stringDate(taskcluster.fromNow('1 year')), + } for apk in apks} + + checkout = 'git clone {} repository && cd repository'.format(GITHUB_HTTP_REPOSITORY) + + return taskcluster.slugId(), BUILDER.build_task( + name="(Fenix) Build task", + description="Build Fenix from source code.", + command='cd .. && {} && ./gradlew --no-daemon clean test assembleRelease'.format(checkout), + features={ + "chainOfTrust": True, + "taskClusterProxy": True + }, + artifacts=artifacts + ) + + +def generate_signing_task(build_task_id, apks, date, is_staging): + artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks] + + signing_format = 'autograph_fenix' + index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly' + routes = [ + "index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(index_release, date.year, date.month, date.day), + "index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(index_release, date.year, date.month, date.day, HEAD_REV), + "index.project.mobile.fenix.{}.nightly.latest".format(index_release), + ] + scopes = [ + "project:mobile:fenix:releng:signing:format:{}".format(signing_format), + "project:mobile:fenix:releng:signing:cert:{}".format('dep-signing' if is_staging else 'release-signing') + ] + + return taskcluster.slugId(), BUILDER.build_signing_task( + build_task_id, + name="(Fenix) Signing task", + description="Sign release builds of Fenix", + apks=artifacts, + scopes=scopes, + routes=routes, + signing_format=signing_format, + is_staging=is_staging + ) + + +def generate_push_task(signing_task_id, apks, commit, is_staging): + artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks] + + return taskcluster.slugId(), BUILDER.build_push_task( + signing_task_id, + name="(Fenix) Push task", + description="Upload signed release builds of Fenix to Google Play", + apks=artifacts, + scopes=[ + "project:mobile:fenix:releng:googleplay:product:fenix{}".format(':dep' if is_staging else '') + ], + commit=commit, + is_staging=is_staging + ) + + +def populate_chain_of_trust_required_but_unused_files(): + # These files are needed to keep chainOfTrust happy. However, they have no need for Reference Browser + # at the moment. For more details, see: https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585 + + for file_name in ('actions.json', 'parameters.yml'): + with open(file_name, 'w') as f: + json.dump({}, f) + + +def nightly(apks, track, commit, date_string): + queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'}) + date = arrow.get(date_string) + is_staging = track == 'staging-nightly' + + task_graph = {} + + build_task_id, build_task = generate_build_task(apks) + lib.tasks.schedule_task(queue, build_task_id, build_task) + + task_graph[build_task_id] = {} + task_graph[build_task_id]['task'] = queue.task(build_task_id) + + sign_task_id, sign_task = generate_signing_task(build_task_id, apks, date, is_staging) + lib.tasks.schedule_task(queue, sign_task_id, sign_task) + + task_graph[sign_task_id] = {} + task_graph[sign_task_id]['task'] = queue.task(sign_task_id) + + push_task_id, push_task = generate_push_task(sign_task_id, apks, commit, is_staging) + lib.tasks.schedule_task(queue, push_task_id, push_task) + + task_graph[push_task_id] = {} + task_graph[push_task_id]['task'] = queue.task(push_task_id) + + print(json.dumps(task_graph, indent=4, separators=(',', ': '))) + + with open('task-graph.json', 'w') as f: + json.dump(task_graph, f) + + populate_chain_of_trust_required_but_unused_files() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description='Create a release pipeline (build, sign, publish) on taskcluster.') + + parser.add_argument('--track', dest="track", action="store", choices=['nightly', 'staging-nightly'], required=True) + parser.add_argument('--commit', dest="commit", action="store_true", help="commit the google play transaction") + parser.add_argument('--apk', dest="apks", metavar="path", action="append", help="Path to APKs to sign and upload", + required=True) + parser.add_argument('--output', dest="track", metavar="path", action="store", help="Path to the build output", + required=True) + parser.add_argument('--date', dest="date", action="store", help="ISO8601 timestamp for build") + + result = parser.parse_args() + apks = ["{}/{}".format(result.output, apk) for apk in result.apks] + nightly(apks, result.track, result.commit, result.date) diff --git a/automation/taskcluster/get-secret.py b/automation/taskcluster/get-secret.py deleted file mode 100644 index 1e8ca6314..000000000 --- a/automation/taskcluster/get-secret.py +++ /dev/null @@ -1,44 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import argparse -import base64 -import os -import taskcluster - - -def write_secret_to_file(path, data, key, base64decode=False): - path = os.path.join(os.path.dirname(__file__), '../../' + path) - with open(path, 'w') as f: - value = data['secret'][key] - if base64decode: - value = base64.b64decode(value) - f.write(value) - - -def fetch_secret_from_taskcluster(name): - secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) - return secrets.get(name) - - -def main(): - parser = argparse.ArgumentParser( - description='Fetch a taskcluster secret value and save it to a file.') - - parser.add_argument('-s', dest="secret", action="store", help="name of the secret") - parser.add_argument('-k', dest='key', action="store", help='key of the secret') - parser.add_argument('-f', dest="path", action="store", help='file to save secret to') - parser.add_argument( - '--decode', dest="decode", action="store_true", default=False, - help='base64 decode secret before saving to file' - ) - - result = parser.parse_args() - - secret = fetch_secret_from_taskcluster(result.secret) - write_secret_to_file(result.path, secret, result.key, result.decode) - - -if __name__ == "__main__": - main() diff --git a/automation/taskcluster/lib/__init__.py b/automation/taskcluster/lib/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/automation/taskcluster/lib/tasks.py b/automation/taskcluster/lib/tasks.py new file mode 100644 index 000000000..d6ff793cd --- /dev/null +++ b/automation/taskcluster/lib/tasks.py @@ -0,0 +1,152 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import datetime +import json +import os +import taskcluster + + +class TaskBuilder(object): + def __init__(self, task_id, repo_url, branch, commit, owner, source, scheduler_id): + self.task_id = task_id + self.repo_url = repo_url + self.branch = branch + self.commit = commit + self.owner = owner + self.source = source + self.scheduler_id = scheduler_id + + def build_task(self, name, description, command, artifacts, features): + created = datetime.datetime.now() + expires = taskcluster.fromNow('1 year') + deadline = taskcluster.fromNow('1 day') + + features = features.copy() + features.update({ + "taskclusterProxy": True + }) + + return { + "workerType": 'github-worker', + "taskGroupId": self.task_id, + "schedulerId": self.scheduler_id, + "expires": taskcluster.stringDate(expires), + "retries": 5, + "created": taskcluster.stringDate(created), + "tags": {}, + "priority": "lowest", + "deadline": taskcluster.stringDate(deadline), + "dependencies": [self.task_id], + "routes": [], + "scopes": [], + "requires": "all-completed", + "payload": { + "features": features, + "maxRunTime": 7200, + "image": "mozillamobile/fenix:1.3", + "command": [ + "/bin/bash", + "--login", + "-c", + command + ], + "artifacts": artifacts, + "deadline": taskcluster.stringDate(deadline) + }, + "provisionerId": "aws-provisioner-v1", + "metadata": { + "name": name, + "description": description, + "owner": self.owner, + "source": self.source + } + } + + def build_signing_task(self, build_task_id, name, description, signing_format, is_staging, apks, scopes, routes): + created = datetime.datetime.now() + expires = taskcluster.fromNow('1 year') + deadline = taskcluster.fromNow('1 day') + + return { + "workerType": 'mobile-signing-dep-v1' if is_staging else 'mobile-signing-v1', + "taskGroupId": self.task_id, + "schedulerId": self.scheduler_id, + "expires": taskcluster.stringDate(expires), + "retries": 5, + "created": taskcluster.stringDate(created), + "tags": {}, + "priority": "lowest", + "deadline": taskcluster.stringDate(deadline), + "dependencies": [self.task_id, build_task_id], + "routes": routes, + "scopes": scopes, + "requires": "all-completed", + "payload": { + "maxRunTime": 3600, + "upstreamArtifacts": [ + { + "paths": apks, + "formats": [signing_format], + "taskId": build_task_id, + "taskType": "build" + } + ] + }, + "provisionerId": "scriptworker-prov-v1", + "metadata": { + "name": name, + "description": description, + "owner": self.owner, + "source": self.source + } + } + + def build_push_task(self, signing_task_id, name, description, is_staging, apks, scopes, commit): + created = datetime.datetime.now() + expires = taskcluster.fromNow('1 year') + deadline = taskcluster.fromNow('1 day') + + return { + "workerType": 'mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1', + "taskGroupId": self.task_id, + "schedulerId": self.scheduler_id, + "expires": taskcluster.stringDate(expires), + "retries": 5, + "created": taskcluster.stringDate(created), + "tags": {}, + "priority": "lowest", + "deadline": taskcluster.stringDate(deadline), + "dependencies": [self.task_id, signing_task_id], + "routes": [], + "scopes": scopes, + "requires": "all-completed", + "payload": { + "commit": commit, + "google_play_track": 'nightly', + "upstreamArtifacts": [ + { + "paths": apks, + "taskId": signing_task_id, + "taskType": "signing" + } + ] + }, + "provisionerId": "scriptworker-prov-v1", + "metadata": { + "name": name, + "description": description, + "owner": "android-components-team@mozilla.com", + "source": "https://github.com/mozilla-mobile/fenix/tree/master/automation/taskcluster" + } + } + + +def schedule_task(queue, taskId, task): + print "TASK", taskId + print json.dumps(task, indent=4, separators=(',', ': ')) + + result = queue.createTask(taskId, task) + print "RESULT", taskId + print json.dumps(result) diff --git a/automation/taskcluster/sign-builds.py b/automation/taskcluster/sign-builds.py deleted file mode 100644 index 00f6d5484..000000000 --- a/automation/taskcluster/sign-builds.py +++ /dev/null @@ -1,89 +0,0 @@ -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -import argparse -import fnmatch -import os -import subprocess - - -def collect_apks(path, pattern): - matches = [] - for root, dirnames, filenames in os.walk(path): - for filename in fnmatch.filter(filenames, pattern): - matches.append(os.path.join(root, filename)) - return matches - - -def zipalign(path): - unsigned_apks = collect_apks(path, '*-unsigned.apk') - print("Found {apk_count} APK(s) to zipalign in {path}".format(apk_count=len(unsigned_apks), path=path)) - for apk in unsigned_apks: - print("Zipaligning", apk) - split = os.path.splitext(apk) - print(subprocess.check_output(["zipalign", "-f", "-v", "-p", "4", apk, split[0] + "-aligned" + split[1]])) - - -def sign(path, store, store_token, key_alias, key_token): - unsigned_apks = collect_apks(path, '*-aligned.apk') - print("Found {apk_count} APK(s) to sign in {path}".format(apk_count=len(unsigned_apks), path=path)) - - for apk in unsigned_apks: - print("Signing", apk) - print(subprocess.check_output([ - "apksigner", "sign", - "--ks", store, - "--ks-key-alias", key_alias, - "--ks-pass", "file:%s" % store_token, - "--key-pass", "file:%s" % key_token, - "-v", - "--out", apk.replace('unsigned', 'signed'), apk])) - - -def archive_result(path, archive): - if not os.path.exists(archive): - os.makedirs(archive) - - signed_apks = collect_apks(path, '*-signed-*.apk') - print("Found {apk_count} APK(s) to archive in {path}".format(apk_count=len(signed_apks), path=path)) - - for apk in signed_apks: - print("Verifying", apk) - print(subprocess.check_output(['apksigner', 'verify', apk])) - - destination = archive + "/" + os.path.basename(apk) - print("Archiving", apk) - print(" `->", destination) - os.rename(apk, destination) - - -def main(): - parser = argparse.ArgumentParser( - description='Zipaligns, signs and archives APKs') - parser.add_argument('--path', dest="path", action="store", help='Root path to search for APK files') - parser.add_argument('--zipalign', dest="zipalign", action="store_true", default=False, - help='Zipaligns APKs before signing') - parser.add_argument('--archive', metavar="PATH", dest="archive", action="store", default=False, - help='Path to save sign APKs to') - - parser.add_argument('--store', metavar="PATH", dest="store", action="store", help='Path to keystore') - parser.add_argument('--store-token', metavar="PATH", dest="store_token", action="store", - help='Path to keystore password file') - parser.add_argument('--key-alias', metavar="ALIAS", dest="key_alias", action="store", help='Key alias') - parser.add_argument('--key-token', metavar="PATH", dest="key_token", action="store", - help='Path to key password file') - - result = parser.parse_args() - - if result.zipalign: - zipalign(result.path) - - sign(result.path, result.store, result.store_token, result.key_alias, result.key_token) - - if result.archive: - archive_result(result.path, result.archive) - - -if __name__ == "__main__": - main()