Sets up nightly build automation
Mitchell Hentges
Emily Kager
parent
0f6528eeb9
commit
1d865471af
|
|
@ -0,0 +1,100 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
version: 1
|
||||
policy:
|
||||
pullRequests: public
|
||||
tasks:
|
||||
- $if: 'tasks_for == "cron"'
|
||||
then:
|
||||
$let:
|
||||
decision_task_id: {$eval: as_slugid("decision_task")}
|
||||
expires_in: {$fromNow: '1 year'}
|
||||
repository: ${event.repository.clone_url}
|
||||
scheduler_id: focus-nightly-sched
|
||||
is_mozilla_mobile_repo:
|
||||
$eval: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix'
|
||||
track:
|
||||
$if: event.repository.clone_url == 'https://github.com/mozilla-mobile/fenix'
|
||||
then: 'nightly'
|
||||
else: 'staging-nightly'
|
||||
in:
|
||||
taskId: ${decision_task_id}
|
||||
taskGroupId: ${decision_task_id} # Must be explicit because of Chain of Trust
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '2 hours'}
|
||||
expires: ${expires_in}
|
||||
schedulerId: ${scheduler_id} # Must be explicit because of Chain of Trust
|
||||
provisionerId: aws-provisioner-v1
|
||||
workerType: gecko-focus # This workerType has ChainOfTrust enabled
|
||||
requires: all-completed # Must be explicit because of Chain of Trust
|
||||
priority: medium
|
||||
retries: 5
|
||||
scopes:
|
||||
$flatten:
|
||||
- queue:scheduler-id:${scheduler_id}
|
||||
- queue:create-task:highest:aws-provisioner-v1/gecko-focus
|
||||
- project:mobile:fenix:releng:signing:format:autograph_fenix
|
||||
- $if: is_mozilla_mobile_repo
|
||||
then:
|
||||
- queue:create-task:highest:scriptworker-prov-v1/mobile-signing-v1
|
||||
- queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-v1
|
||||
- project:mobile:fenix:releng:signing:cert:release-signing
|
||||
- project:mobile:fenix:releng:googleplay:product:fenix
|
||||
- queue:route:index.project.mobile.fenix.signed-nightly.*
|
||||
else:
|
||||
- queue:create-task:highest:scriptworker-prov-v1/mobile-signing-dep-v1
|
||||
- queue:create-task:highest:scriptworker-prov-v1/mobile-pushapk-dep-v1
|
||||
- project:mobile:fenix:releng:signing:cert:dep-signing
|
||||
- project:mobile:fenix:releng:googleplay:product:fenix:dep
|
||||
- queue:route:index.project.mobile.fenix.staging-signed-nightly.*
|
||||
payload:
|
||||
maxRunTime: 600 # Decision should remain fast enough to schedule a handful of tasks
|
||||
image: mozillamobile/fenix:1.3
|
||||
features:
|
||||
taskclusterProxy: true
|
||||
chainOfTrust: true
|
||||
env:
|
||||
TASK_ID: ${decision_task_id}
|
||||
SCHEDULER_ID: ${scheduler_id}
|
||||
MOBILE_HEAD_REPOSITORY: ${repository}
|
||||
MOBILE_HEAD_BRANCH: ${event.release.target_commitish}
|
||||
MOBILE_HEAD_REV: ${event.release.tag_name}
|
||||
MOBILE_TRIGGERED_BY: ${event.sender.login}
|
||||
command:
|
||||
- /bin/bash
|
||||
- --login
|
||||
- -cx
|
||||
- >-
|
||||
cd ..
|
||||
&& git clone ${repository} repository
|
||||
&& cd repository
|
||||
&& pip install arrow
|
||||
&& python tools/taskcluster/decision_task_nightly.py \
|
||||
--track ${track} \
|
||||
--commit \
|
||||
--output /opt/repository/app/build/outputs/apk \
|
||||
--apk arm/release/app-arm-release-unsigned.apk \
|
||||
--apk x86/release/app-x86-release-unsigned.apk \
|
||||
--date ${now}
|
||||
artifacts:
|
||||
public/task-graph.json:
|
||||
type: file
|
||||
path: /opt/repository/task-graph.json
|
||||
expires: ${expires_in}
|
||||
public/actions.json:
|
||||
type: file
|
||||
path: /opt/repository/actions.json
|
||||
expires: ${expires_in}
|
||||
public/parameters.yml:
|
||||
type: file
|
||||
path: /opt/repository/parameters.yml
|
||||
expires: ${expires_in}
|
||||
extra:
|
||||
cron: {$json: {$eval: 'cron'}}
|
||||
tasks_for: ${tasks_for}
|
||||
metadata:
|
||||
name: Fenix Nightly Decision Task
|
||||
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
|
||||
owner: ${event.sender.login}@users.noreply.github.com
|
||||
source: ${repository}/raw/${event.release.tag_name}/.taskcluster.yml
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""
|
||||
Decision task for nightly releases.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import arrow
|
||||
import json
|
||||
import lib.tasks
|
||||
import os
|
||||
import taskcluster
|
||||
|
||||
TASK_ID = os.environ.get('TASK_ID')
|
||||
SCHEDULER_ID = os.environ.get('SCHEDULER_ID')
|
||||
GITHUB_HTTP_REPOSITORY = os.environ.get('MOBILE_HEAD_REPOSITORY')
|
||||
HEAD_REV = os.environ.get('MOBILE_HEAD_REV')
|
||||
|
||||
BUILDER = lib.tasks.TaskBuilder(
|
||||
task_id=TASK_ID,
|
||||
owner="android-components-team@mozilla.com",
|
||||
source='{}/raw/{}/.taskcluster.yml'.format(GITHUB_HTTP_REPOSITORY, HEAD_REV),
|
||||
scheduler_id=SCHEDULER_ID
|
||||
)
|
||||
|
||||
|
||||
def generate_build_task(apks):
|
||||
artifacts = {'public/{}'.format(os.path.basename(apk)): {
|
||||
"type": 'file',
|
||||
"path": apk,
|
||||
"expires": taskcluster.stringDate(taskcluster.fromNow('1 year')),
|
||||
} for apk in apks}
|
||||
|
||||
checkout = 'git clone {} repository && cd repository'.format(GITHUB_HTTP_REPOSITORY)
|
||||
|
||||
return taskcluster.slugId(), BUILDER.build_task(
|
||||
name="(Fenix) Build task",
|
||||
description="Build Fenix from source code.",
|
||||
command='cd .. && {} && ./gradlew --no-daemon clean test assembleRelease'.format(checkout),
|
||||
features={
|
||||
"chainOfTrust": True,
|
||||
"taskClusterProxy": True
|
||||
},
|
||||
artifacts=artifacts
|
||||
)
|
||||
|
||||
|
||||
def generate_signing_task(build_task_id, apks, date, is_staging):
|
||||
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
|
||||
|
||||
signing_format = 'autograph_fenix'
|
||||
index_release = 'staging-signed-nightly' if is_staging else 'signed-nightly'
|
||||
routes = [
|
||||
"index.project.mobile.fenix.{}.nightly.{}.{}.{}.latest".format(index_release, date.year, date.month, date.day),
|
||||
"index.project.mobile.fenix.{}.nightly.{}.{}.{}.revision.{}".format(index_release, date.year, date.month, date.day, HEAD_REV),
|
||||
"index.project.mobile.fenix.{}.nightly.latest".format(index_release),
|
||||
]
|
||||
scopes = [
|
||||
"project:mobile:fenix:releng:signing:format:{}".format(signing_format),
|
||||
"project:mobile:fenix:releng:signing:cert:{}".format('dep-signing' if is_staging else 'release-signing')
|
||||
]
|
||||
|
||||
return taskcluster.slugId(), BUILDER.build_signing_task(
|
||||
build_task_id,
|
||||
name="(Fenix) Signing task",
|
||||
description="Sign release builds of Fenix",
|
||||
apks=artifacts,
|
||||
scopes=scopes,
|
||||
routes=routes,
|
||||
signing_format=signing_format,
|
||||
is_staging=is_staging
|
||||
)
|
||||
|
||||
|
||||
def generate_push_task(signing_task_id, apks, commit, is_staging):
|
||||
artifacts = ["public/{}".format(os.path.basename(apk)) for apk in apks]
|
||||
|
||||
return taskcluster.slugId(), BUILDER.build_push_task(
|
||||
signing_task_id,
|
||||
name="(Fenix) Push task",
|
||||
description="Upload signed release builds of Fenix to Google Play",
|
||||
apks=artifacts,
|
||||
scopes=[
|
||||
"project:mobile:fenix:releng:googleplay:product:fenix{}".format(':dep' if is_staging else '')
|
||||
],
|
||||
commit=commit,
|
||||
is_staging=is_staging
|
||||
)
|
||||
|
||||
|
||||
def populate_chain_of_trust_required_but_unused_files():
|
||||
# These files are needed to keep chainOfTrust happy. However, they have no need for Reference Browser
|
||||
# at the moment. For more details, see: https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
|
||||
|
||||
for file_name in ('actions.json', 'parameters.yml'):
|
||||
with open(file_name, 'w') as f:
|
||||
json.dump({}, f)
|
||||
|
||||
|
||||
def nightly(apks, track, commit, date_string):
|
||||
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
|
||||
date = arrow.get(date_string)
|
||||
is_staging = track == 'staging-nightly'
|
||||
|
||||
task_graph = {}
|
||||
|
||||
build_task_id, build_task = generate_build_task(apks)
|
||||
lib.tasks.schedule_task(queue, build_task_id, build_task)
|
||||
|
||||
task_graph[build_task_id] = {}
|
||||
task_graph[build_task_id]['task'] = queue.task(build_task_id)
|
||||
|
||||
sign_task_id, sign_task = generate_signing_task(build_task_id, apks, date, is_staging)
|
||||
lib.tasks.schedule_task(queue, sign_task_id, sign_task)
|
||||
|
||||
task_graph[sign_task_id] = {}
|
||||
task_graph[sign_task_id]['task'] = queue.task(sign_task_id)
|
||||
|
||||
push_task_id, push_task = generate_push_task(sign_task_id, apks, commit, is_staging)
|
||||
lib.tasks.schedule_task(queue, push_task_id, push_task)
|
||||
|
||||
task_graph[push_task_id] = {}
|
||||
task_graph[push_task_id]['task'] = queue.task(push_task_id)
|
||||
|
||||
print(json.dumps(task_graph, indent=4, separators=(',', ': ')))
|
||||
|
||||
with open('task-graph.json', 'w') as f:
|
||||
json.dump(task_graph, f)
|
||||
|
||||
populate_chain_of_trust_required_but_unused_files()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Create a release pipeline (build, sign, publish) on taskcluster.')
|
||||
|
||||
parser.add_argument('--track', dest="track", action="store", choices=['nightly', 'staging-nightly'], required=True)
|
||||
parser.add_argument('--commit', dest="commit", action="store_true", help="commit the google play transaction")
|
||||
parser.add_argument('--apk', dest="apks", metavar="path", action="append", help="Path to APKs to sign and upload",
|
||||
required=True)
|
||||
parser.add_argument('--output', dest="track", metavar="path", action="store", help="Path to the build output",
|
||||
required=True)
|
||||
parser.add_argument('--date', dest="date", action="store", help="ISO8601 timestamp for build")
|
||||
|
||||
result = parser.parse_args()
|
||||
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
|
||||
nightly(apks, result.track, result.commit, result.date)
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
import taskcluster
|
||||
|
||||
|
||||
def write_secret_to_file(path, data, key, base64decode=False):
|
||||
path = os.path.join(os.path.dirname(__file__), '../../' + path)
|
||||
with open(path, 'w') as f:
|
||||
value = data['secret'][key]
|
||||
if base64decode:
|
||||
value = base64.b64decode(value)
|
||||
f.write(value)
|
||||
|
||||
|
||||
def fetch_secret_from_taskcluster(name):
|
||||
secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'})
|
||||
return secrets.get(name)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Fetch a taskcluster secret value and save it to a file.')
|
||||
|
||||
parser.add_argument('-s', dest="secret", action="store", help="name of the secret")
|
||||
parser.add_argument('-k', dest='key', action="store", help='key of the secret')
|
||||
parser.add_argument('-f', dest="path", action="store", help='file to save secret to')
|
||||
parser.add_argument(
|
||||
'--decode', dest="decode", action="store_true", default=False,
|
||||
help='base64 decode secret before saving to file'
|
||||
)
|
||||
|
||||
result = parser.parse_args()
|
||||
|
||||
secret = fetch_secret_from_taskcluster(result.secret)
|
||||
write_secret_to_file(result.path, secret, result.key, result.decode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,152 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import taskcluster
|
||||
|
||||
|
||||
class TaskBuilder(object):
|
||||
def __init__(self, task_id, repo_url, branch, commit, owner, source, scheduler_id):
|
||||
self.task_id = task_id
|
||||
self.repo_url = repo_url
|
||||
self.branch = branch
|
||||
self.commit = commit
|
||||
self.owner = owner
|
||||
self.source = source
|
||||
self.scheduler_id = scheduler_id
|
||||
|
||||
def build_task(self, name, description, command, artifacts, features):
|
||||
created = datetime.datetime.now()
|
||||
expires = taskcluster.fromNow('1 year')
|
||||
deadline = taskcluster.fromNow('1 day')
|
||||
|
||||
features = features.copy()
|
||||
features.update({
|
||||
"taskclusterProxy": True
|
||||
})
|
||||
|
||||
return {
|
||||
"workerType": 'github-worker',
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"tags": {},
|
||||
"priority": "lowest",
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"dependencies": [self.task_id],
|
||||
"routes": [],
|
||||
"scopes": [],
|
||||
"requires": "all-completed",
|
||||
"payload": {
|
||||
"features": features,
|
||||
"maxRunTime": 7200,
|
||||
"image": "mozillamobile/fenix:1.3",
|
||||
"command": [
|
||||
"/bin/bash",
|
||||
"--login",
|
||||
"-c",
|
||||
command
|
||||
],
|
||||
"artifacts": artifacts,
|
||||
"deadline": taskcluster.stringDate(deadline)
|
||||
},
|
||||
"provisionerId": "aws-provisioner-v1",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"owner": self.owner,
|
||||
"source": self.source
|
||||
}
|
||||
}
|
||||
|
||||
def build_signing_task(self, build_task_id, name, description, signing_format, is_staging, apks, scopes, routes):
|
||||
created = datetime.datetime.now()
|
||||
expires = taskcluster.fromNow('1 year')
|
||||
deadline = taskcluster.fromNow('1 day')
|
||||
|
||||
return {
|
||||
"workerType": 'mobile-signing-dep-v1' if is_staging else 'mobile-signing-v1',
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"tags": {},
|
||||
"priority": "lowest",
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"dependencies": [self.task_id, build_task_id],
|
||||
"routes": routes,
|
||||
"scopes": scopes,
|
||||
"requires": "all-completed",
|
||||
"payload": {
|
||||
"maxRunTime": 3600,
|
||||
"upstreamArtifacts": [
|
||||
{
|
||||
"paths": apks,
|
||||
"formats": [signing_format],
|
||||
"taskId": build_task_id,
|
||||
"taskType": "build"
|
||||
}
|
||||
]
|
||||
},
|
||||
"provisionerId": "scriptworker-prov-v1",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"owner": self.owner,
|
||||
"source": self.source
|
||||
}
|
||||
}
|
||||
|
||||
def build_push_task(self, signing_task_id, name, description, is_staging, apks, scopes, commit):
|
||||
created = datetime.datetime.now()
|
||||
expires = taskcluster.fromNow('1 year')
|
||||
deadline = taskcluster.fromNow('1 day')
|
||||
|
||||
return {
|
||||
"workerType": 'mobile-pushapk-dep-v1' if is_staging else 'mobile-pushapk-v1',
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"tags": {},
|
||||
"priority": "lowest",
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"dependencies": [self.task_id, signing_task_id],
|
||||
"routes": [],
|
||||
"scopes": scopes,
|
||||
"requires": "all-completed",
|
||||
"payload": {
|
||||
"commit": commit,
|
||||
"google_play_track": 'nightly',
|
||||
"upstreamArtifacts": [
|
||||
{
|
||||
"paths": apks,
|
||||
"taskId": signing_task_id,
|
||||
"taskType": "signing"
|
||||
}
|
||||
]
|
||||
},
|
||||
"provisionerId": "scriptworker-prov-v1",
|
||||
"metadata": {
|
||||
"name": name,
|
||||
"description": description,
|
||||
"owner": "android-components-team@mozilla.com",
|
||||
"source": "https://github.com/mozilla-mobile/fenix/tree/master/automation/taskcluster"
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def schedule_task(queue, taskId, task):
|
||||
print "TASK", taskId
|
||||
print json.dumps(task, indent=4, separators=(',', ': '))
|
||||
|
||||
result = queue.createTask(taskId, task)
|
||||
print "RESULT", taskId
|
||||
print json.dumps(result)
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import argparse
|
||||
import fnmatch
|
||||
import os
|
||||
import subprocess
|
||||
|
||||
|
||||
def collect_apks(path, pattern):
|
||||
matches = []
|
||||
for root, dirnames, filenames in os.walk(path):
|
||||
for filename in fnmatch.filter(filenames, pattern):
|
||||
matches.append(os.path.join(root, filename))
|
||||
return matches
|
||||
|
||||
|
||||
def zipalign(path):
|
||||
unsigned_apks = collect_apks(path, '*-unsigned.apk')
|
||||
print("Found {apk_count} APK(s) to zipalign in {path}".format(apk_count=len(unsigned_apks), path=path))
|
||||
for apk in unsigned_apks:
|
||||
print("Zipaligning", apk)
|
||||
split = os.path.splitext(apk)
|
||||
print(subprocess.check_output(["zipalign", "-f", "-v", "-p", "4", apk, split[0] + "-aligned" + split[1]]))
|
||||
|
||||
|
||||
def sign(path, store, store_token, key_alias, key_token):
|
||||
unsigned_apks = collect_apks(path, '*-aligned.apk')
|
||||
print("Found {apk_count} APK(s) to sign in {path}".format(apk_count=len(unsigned_apks), path=path))
|
||||
|
||||
for apk in unsigned_apks:
|
||||
print("Signing", apk)
|
||||
print(subprocess.check_output([
|
||||
"apksigner", "sign",
|
||||
"--ks", store,
|
||||
"--ks-key-alias", key_alias,
|
||||
"--ks-pass", "file:%s" % store_token,
|
||||
"--key-pass", "file:%s" % key_token,
|
||||
"-v",
|
||||
"--out", apk.replace('unsigned', 'signed'), apk]))
|
||||
|
||||
|
||||
def archive_result(path, archive):
|
||||
if not os.path.exists(archive):
|
||||
os.makedirs(archive)
|
||||
|
||||
signed_apks = collect_apks(path, '*-signed-*.apk')
|
||||
print("Found {apk_count} APK(s) to archive in {path}".format(apk_count=len(signed_apks), path=path))
|
||||
|
||||
for apk in signed_apks:
|
||||
print("Verifying", apk)
|
||||
print(subprocess.check_output(['apksigner', 'verify', apk]))
|
||||
|
||||
destination = archive + "/" + os.path.basename(apk)
|
||||
print("Archiving", apk)
|
||||
print(" `->", destination)
|
||||
os.rename(apk, destination)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Zipaligns, signs and archives APKs')
|
||||
parser.add_argument('--path', dest="path", action="store", help='Root path to search for APK files')
|
||||
parser.add_argument('--zipalign', dest="zipalign", action="store_true", default=False,
|
||||
help='Zipaligns APKs before signing')
|
||||
parser.add_argument('--archive', metavar="PATH", dest="archive", action="store", default=False,
|
||||
help='Path to save sign APKs to')
|
||||
|
||||
parser.add_argument('--store', metavar="PATH", dest="store", action="store", help='Path to keystore')
|
||||
parser.add_argument('--store-token', metavar="PATH", dest="store_token", action="store",
|
||||
help='Path to keystore password file')
|
||||
parser.add_argument('--key-alias', metavar="ALIAS", dest="key_alias", action="store", help='Key alias')
|
||||
parser.add_argument('--key-token', metavar="PATH", dest="key_token", action="store",
|
||||
help='Path to key password file')
|
||||
|
||||
result = parser.parse_args()
|
||||
|
||||
if result.zipalign:
|
||||
zipalign(result.path)
|
||||
|
||||
sign(result.path, result.store, result.store_token, result.key_alias, result.key_token)
|
||||
|
||||
if result.archive:
|
||||
archive_result(result.path, result.archive)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
Reference in New Issue