blallo
/
fenix
1
0
Fork 0
Code Releases Activity

Run build and tests on PRs and pushes

master
Johan Lorenzo 2019-03-27 15:23:03 +01:00 committed by Sebastian Kaspari
parent 89d0e9604a
commit 7f772404ce
5 changed files with 280 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
.taskcluster.yml
View File

@ -108,54 +108,104 @@ tasks:
taskclusterProxy: true
extra:
tasks_for: ${tasks_for}
treeherder:
machine:
platform: mobile-decision
metadata:
owner: ${user}@users.noreply.github.com
source: ${repository}/raw/${head_rev}/.taskcluster.yml
in:
- $if: 'tasks_for == "cron"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- $if: is_repo_trusted
then: assume:hook-id:project-mobile/fenix-nightly
else: assume:hook-id:project-mobile/fenix-nightly-staging
routes:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
features:
taskclusterProxy: true
chainOfTrust: true
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout ${head_rev}
&& python automation/taskcluster/decision_task.py \
release \
--nightly \
--track ${track} \
--commit \
--output /opt/fenix/app/build/outputs/apk \
--apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \
--apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \
--apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \
--date ${now}
artifacts:
public/task-graph.json:
type: file
path: /opt/fenix/task-graph.json
expires: ${expires_in}
public/actions.json:
type: file
path: /opt/fenix/actions.json
expires: ${expires_in}
public/parameters.yml:
type: file
path: /opt/fenix/parameters.yml
expires: ${expires_in}
extra:
cron: {$json: {$eval: 'cron'}}
metadata:
name: Fenix Nightly Decision Task
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
$if: 'tasks_for in ["github-pull-request", "github-push"]'
then:
$let:
pr_or_push_parameters:
payload:
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout ${head_rev}
&& python automation/taskcluster/decision_task.py pr-or-push
in:
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
then:
$let:
pull_request_title: ${event.pull_request.title}
pull_request_number: ${event.pull_request.number}
pull_request_url: ${event.pull_request.html_url}
in:
$mergeDeep:
- {$eval: 'default_task_definition'}
- {$eval: 'pr_or_push_parameters'}
- scopes:
- ${assume_scope_prefix}:pull-request
payload:
env:
GITHUB_PULL_TITLE: ${pull_request_title}
extra:
treeherder:
symbol: D-PR
metadata:
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- {$eval: 'pr_or_push_parameters'}
- scopes:
- ${assume_scope_prefix}:branch:${short_head_branch}
extra:
treeherder:
symbol: D
metadata:
name: Fenix - Decision task
description: Schedules the build and test tasks for Fenix.
else:
- $if: 'tasks_for == "cron"'
then:
$mergeDeep:
- {$eval: 'default_task_definition'}
- scopes:
- $if: is_repo_trusted
then: assume:hook-id:project-mobile/fenix-nightly
else: assume:hook-id:project-mobile/fenix-nightly-staging
routes:
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
payload:
features:
taskclusterProxy: true
chainOfTrust: true
command:
- >-
git fetch ${repository} ${head_branch}
&& git config advice.detachedHead false
&& git checkout ${head_rev}
&& python automation/taskcluster/decision_task.py \
release \
--nightly \
--track ${track} \
--commit \
--output /opt/fenix/app/build/outputs/apk \
--apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \
--apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \
--apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \
--date ${now}
artifacts:
public/task-graph.json:
type: file
path: /opt/fenix/task-graph.json
expires: ${expires_in}
public/actions.json:
type: file
path: /opt/fenix/actions.json
expires: ${expires_in}
public/parameters.yml:
type: file
path: /opt/fenix/parameters.yml
expires: ${expires_in}
extra:
cron: {$json: {$eval: 'cron'}}
metadata:
name: Fenix Nightly Decision Task
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})

12
app/build.gradle
View File

@ -333,6 +333,18 @@ if (project.hasProperty("raptor")) {
android.defaultConfig.manifestPlaceholders.isRaptorEnabled = "true"
}
// -------------------------------------------------------------------------------------------------
// Task for printing all build variants to build variants in parallel in automation
// -------------------------------------------------------------------------------------------------
task printBuildVariants {
doLast {
def buildVariants = android.applicationVariants.collect { variant ->
variant.name
}
println "variants: " + groovy.json.JsonOutput.toJson(buildVariants)
}
}
// Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we
// can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best.
apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle'

38
automation/taskcluster/decision_task.py
View File

@ -10,8 +10,10 @@ from __future__ import print_function
import argparse
import os
import sys
import taskcluster
from lib import build_variants
from lib.tasks import TaskBuilder, schedule_task_graph
from lib.util import (
populate_chain_of_trust_task_graph,
@ -38,6 +40,39 @@ BUILDER = TaskBuilder(
)
def pr_or_push():
if SKIP_TASKS_TRIGGER in PR_TITLE:
print("Pull request title contains", SKIP_TASKS_TRIGGER)
print("Exit")
exit(0)
print("Fetching build variants from gradle")
variants = build_variants.from_gradle()
if len(variants) == 0:
print("Could not get build variants from gradle")
sys.exit(2)
print("Got variants: {}".format(' '.join(variants)))
build_tasks = {}
other_tasks = {}
for variant in variants:
build_tasks[taskcluster.slugId()] = BUILDER.craft_assemble_task(variant)
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
for craft_function in (
BUILDER.craft_detekt_task,
BUILDER.craft_ktlint_task,
BUILDER.craft_lint_task,
BUILDER.craft_compare_locales_task,
):
other_tasks[taskcluster.slugId()] = craft_function()
return (build_tasks, other_tasks)
def nightly(apks, track, commit, date_string):
is_staging = track == 'staging-nightly'
@ -99,8 +134,7 @@ if __name__ == "__main__":
command = result.command
if command == 'pr-or-push':
# TODO
ordered_groups_of_tasks = {}
ordered_groups_of_tasks = pr_or_push()
elif command == 'release':
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
# nightly(apks, result.track, result.commit, result.date)

24
automation/taskcluster/lib/build_variants.py 100644
View File

@ -0,0 +1,24 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
from __future__ import print_function
import json
import subprocess
def from_gradle():
process = subprocess.Popen([
"./gradlew", "--no-daemon", "--quiet", "printBuildVariants"
], stdout=subprocess.PIPE)
(output, err) = process.communicate()
exit_code = process.wait()
if exit_code != 0:
print("Gradle command returned error: {}".format(exit_code))
variants_line = [line for line in output.split('\n') if line.startswith('variants: ')][0]
variants_json = variants_line.split(' ', 1)[1]
variants = json.loads(variants_json)
return variants

115
automation/taskcluster/lib/tasks.py
View File

@ -70,7 +70,7 @@ class TaskBuilder(object):
]
return self._craft_build_ish_task(
name='Fenix - Build task',
name='Build task',
description='Build Fenix from source code',
command=command,
scopes=[
@ -81,6 +81,60 @@ class TaskBuilder(object):
is_staging=is_staging,
)
def craft_assemble_task(self, variant):
return self._craft_gradle_clean_task(
name='assemble: {}'.format(variant),
description='Building and testing variant {}'.format(variant),
gradle_task='assemble{}'.format(variant.capitalize()),
artifacts=_craft_artifacts_from_variant(variant),
)
def craft_test_task(self, variant):
return self._craft_gradle_clean_task(
name='test: {}'.format(variant),
description='Building and testing variant {}'.format(variant),
gradle_task='test{}UnitTest'.format(variant.capitalize()),
)
def craft_detekt_task(self):
return self._craft_gradle_clean_task(
name='detekt',
description='Running detekt over all modules',
gradle_task='detekt'
)
def craft_ktlint_task(self):
return self._craft_gradle_clean_task(
name='ktlint',
description='Running ktlint over all modules',
gradle_task='ktlint'
)
def craft_lint_task(self):
return self._craft_gradle_clean_task(
name='lint',
description='Running ktlint over all modules',
gradle_task='lint'
)
def _craft_gradle_clean_task(self, name, description, gradle_task, artifacts=None):
return self._craft_build_ish_task(
name=name,
description=description,
command='./gradlew --no-daemon clean {}'.format(gradle_task),
artifacts=artifacts,
)
def craft_compare_locales_task(self):
return self._craft_build_ish_task(
name='compare-locales',
description='Validate strings.xml with compare-locales',
command=(
'pip install "compare-locales>=5.0.2,<6.0" && '
'compare-locales --validate l10n.toml .'
)
)
def _craft_build_ish_task(
self, name, description, command, dependencies=None, artifacts=None, scopes=None,
routes=None, is_staging=True
@ -155,7 +209,7 @@ class TaskBuilder(object):
"scopes": scopes,
"payload": payload,
"metadata": {
"name": name,
"name": "Fenix - {}".format(name),
"description": description,
"owner": self.owner,
"source": self.source,
@ -198,7 +252,7 @@ class TaskBuilder(object):
'dep-signing' if is_staging else 'release-signing'
)
],
name="Fenix - Signing task",
name="Signing task",
description="Sign release builds of Fenix",
payload=payload
)
@ -228,12 +282,65 @@ class TaskBuilder(object):
':dep' if is_staging else ''
)
],
name="Fenix - Push task",
name="Push task",
description="Upload signed release builds of Fenix to Google Play",
payload=payload
)
def _craft_artifacts_from_variant(variant):
return {
'public/target.apk': {
'type': 'file',
'path': _craft_apk_full_path_from_variant(variant),
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
}
}
def _craft_apk_full_path_from_variant(variant):
architecture, build_type = _get_architecture_and_build_type_from_variant(variant)
short_variant = variant[:-len(build_type)]
postfix = '-unsigned' if build_type == 'release' else ''
product = '{}{}'.format(product[0].lower(), product[1:])
return '/opt/fenix/app/build/outputs/apk/{short_variant}/{build_type}/app-{architecture}-{product}-{build_type}{postfix}.apk'.format( # noqa: E501
architecture=architecture,
build_type=build_type,
product=product,
short_variant=short_variant,
postfix=postfix
)
def _get_architecture_and_build_type_from_variant(variant):
variant = variant.lower()
architecture = None
if 'aarch64' in variant:
architecture = 'aarch64'
elif 'x86' in variant:
architecture = 'x86'
elif 'arm' in variant:
architecture = 'arm'
build_type = None
if variant.endswith('debug'):
build_type = 'debug'
elif variant.endswith('release'):
build_type = 'release'
if not architecture or not build_type:
raise ValueError(
'Unsupported variant "{}". Found architecture, build_type: {}'.format(
variant, (architecture, build_type)
)
)
return architecture, build_type
def schedule_task(queue, taskId, task):
print("TASK", taskId)
print(json.dumps(task, indent=4, separators=(',', ': ')))