Run build and tests on PRs and pushes
parent
89d0e9604a
commit
7f772404ce
144
.taskcluster.yml
144
.taskcluster.yml
|
@ -108,54 +108,104 @@ tasks:
|
||||||
taskclusterProxy: true
|
taskclusterProxy: true
|
||||||
extra:
|
extra:
|
||||||
tasks_for: ${tasks_for}
|
tasks_for: ${tasks_for}
|
||||||
|
treeherder:
|
||||||
|
machine:
|
||||||
|
platform: mobile-decision
|
||||||
metadata:
|
metadata:
|
||||||
owner: ${user}@users.noreply.github.com
|
owner: ${user}@users.noreply.github.com
|
||||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||||
in:
|
in:
|
||||||
- $if: 'tasks_for == "cron"'
|
$if: 'tasks_for in ["github-pull-request", "github-push"]'
|
||||||
then:
|
then:
|
||||||
$mergeDeep:
|
$let:
|
||||||
- {$eval: 'default_task_definition'}
|
pr_or_push_parameters:
|
||||||
- scopes:
|
payload:
|
||||||
- $if: is_repo_trusted
|
command:
|
||||||
then: assume:hook-id:project-mobile/fenix-nightly
|
- >-
|
||||||
else: assume:hook-id:project-mobile/fenix-nightly-staging
|
git fetch ${repository} ${head_branch}
|
||||||
routes:
|
&& git config advice.detachedHead false
|
||||||
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
|
&& git checkout ${head_rev}
|
||||||
payload:
|
&& python automation/taskcluster/decision_task.py pr-or-push
|
||||||
features:
|
in:
|
||||||
taskclusterProxy: true
|
- $if: 'tasks_for == "github-pull-request" && event["action"] in ["opened", "reopened", "synchronize"]'
|
||||||
chainOfTrust: true
|
then:
|
||||||
command:
|
$let:
|
||||||
- >-
|
pull_request_title: ${event.pull_request.title}
|
||||||
git fetch ${repository} ${head_branch}
|
pull_request_number: ${event.pull_request.number}
|
||||||
&& git config advice.detachedHead false
|
pull_request_url: ${event.pull_request.html_url}
|
||||||
&& git checkout ${head_rev}
|
in:
|
||||||
&& python automation/taskcluster/decision_task.py \
|
$mergeDeep:
|
||||||
release \
|
- {$eval: 'default_task_definition'}
|
||||||
--nightly \
|
- {$eval: 'pr_or_push_parameters'}
|
||||||
--track ${track} \
|
- scopes:
|
||||||
--commit \
|
- ${assume_scope_prefix}:pull-request
|
||||||
--output /opt/fenix/app/build/outputs/apk \
|
payload:
|
||||||
--apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \
|
env:
|
||||||
--apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \
|
GITHUB_PULL_TITLE: ${pull_request_title}
|
||||||
--apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \
|
extra:
|
||||||
--date ${now}
|
treeherder:
|
||||||
artifacts:
|
symbol: D-PR
|
||||||
public/task-graph.json:
|
metadata:
|
||||||
type: file
|
name: 'Fenix - Decision task (Pull Request #${pull_request_number})'
|
||||||
path: /opt/fenix/task-graph.json
|
description: 'Building and testing the Fenix - triggered by [#${pull_request_number}](${pull_request_url})'
|
||||||
expires: ${expires_in}
|
- $if: 'tasks_for == "github-push" && head_branch[:10] != "refs/tags/"'
|
||||||
public/actions.json:
|
then:
|
||||||
type: file
|
$mergeDeep:
|
||||||
path: /opt/fenix/actions.json
|
- {$eval: 'default_task_definition'}
|
||||||
expires: ${expires_in}
|
- {$eval: 'pr_or_push_parameters'}
|
||||||
public/parameters.yml:
|
- scopes:
|
||||||
type: file
|
- ${assume_scope_prefix}:branch:${short_head_branch}
|
||||||
path: /opt/fenix/parameters.yml
|
extra:
|
||||||
expires: ${expires_in}
|
treeherder:
|
||||||
extra:
|
symbol: D
|
||||||
cron: {$json: {$eval: 'cron'}}
|
metadata:
|
||||||
metadata:
|
name: Fenix - Decision task
|
||||||
name: Fenix Nightly Decision Task
|
description: Schedules the build and test tasks for Fenix.
|
||||||
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
|
else:
|
||||||
|
- $if: 'tasks_for == "cron"'
|
||||||
|
then:
|
||||||
|
$mergeDeep:
|
||||||
|
- {$eval: 'default_task_definition'}
|
||||||
|
- scopes:
|
||||||
|
- $if: is_repo_trusted
|
||||||
|
then: assume:hook-id:project-mobile/fenix-nightly
|
||||||
|
else: assume:hook-id:project-mobile/fenix-nightly-staging
|
||||||
|
routes:
|
||||||
|
- notify.email.fenix-eng-notifications@mozilla.com.on-failed
|
||||||
|
payload:
|
||||||
|
features:
|
||||||
|
taskclusterProxy: true
|
||||||
|
chainOfTrust: true
|
||||||
|
command:
|
||||||
|
- >-
|
||||||
|
git fetch ${repository} ${head_branch}
|
||||||
|
&& git config advice.detachedHead false
|
||||||
|
&& git checkout ${head_rev}
|
||||||
|
&& python automation/taskcluster/decision_task.py \
|
||||||
|
release \
|
||||||
|
--nightly \
|
||||||
|
--track ${track} \
|
||||||
|
--commit \
|
||||||
|
--output /opt/fenix/app/build/outputs/apk \
|
||||||
|
--apk armGreenfield/release/app-arm-greenfield-release-unsigned.apk \
|
||||||
|
--apk x86Greenfield/release/app-x86-greenfield-release-unsigned.apk \
|
||||||
|
--apk aarch64Greenfield/release/app-aarch64-greenfield-release-unsigned.apk \
|
||||||
|
--date ${now}
|
||||||
|
artifacts:
|
||||||
|
public/task-graph.json:
|
||||||
|
type: file
|
||||||
|
path: /opt/fenix/task-graph.json
|
||||||
|
expires: ${expires_in}
|
||||||
|
public/actions.json:
|
||||||
|
type: file
|
||||||
|
path: /opt/fenix/actions.json
|
||||||
|
expires: ${expires_in}
|
||||||
|
public/parameters.yml:
|
||||||
|
type: file
|
||||||
|
path: /opt/fenix/parameters.yml
|
||||||
|
expires: ${expires_in}
|
||||||
|
extra:
|
||||||
|
cron: {$json: {$eval: 'cron'}}
|
||||||
|
metadata:
|
||||||
|
name: Fenix Nightly Decision Task
|
||||||
|
description: Decision task scheduled by cron task [${cron.task_id}](https://tools.taskcluster.net/tasks/${cron.task_id})
|
||||||
|
|
|
@ -333,6 +333,18 @@ if (project.hasProperty("raptor")) {
|
||||||
android.defaultConfig.manifestPlaceholders.isRaptorEnabled = "true"
|
android.defaultConfig.manifestPlaceholders.isRaptorEnabled = "true"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// -------------------------------------------------------------------------------------------------
|
||||||
|
// Task for printing all build variants to build variants in parallel in automation
|
||||||
|
// -------------------------------------------------------------------------------------------------
|
||||||
|
task printBuildVariants {
|
||||||
|
doLast {
|
||||||
|
def buildVariants = android.applicationVariants.collect { variant ->
|
||||||
|
variant.name
|
||||||
|
}
|
||||||
|
println "variants: " + groovy.json.JsonOutput.toJson(buildVariants)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we
|
// Normally this should use the same version as the glean dependency. But since we are currently using AC snapshots we
|
||||||
// can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best.
|
// can't reference a git tag with a specific version here. So we are just using "master" and hoping for the best.
|
||||||
apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle'
|
apply from: 'https://github.com/mozilla-mobile/android-components/raw/master/components/service/glean/scripts/sdk_generator.gradle'
|
||||||
|
|
|
@ -10,8 +10,10 @@ from __future__ import print_function
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
import os
|
import os
|
||||||
|
import sys
|
||||||
import taskcluster
|
import taskcluster
|
||||||
|
|
||||||
|
from lib import build_variants
|
||||||
from lib.tasks import TaskBuilder, schedule_task_graph
|
from lib.tasks import TaskBuilder, schedule_task_graph
|
||||||
from lib.util import (
|
from lib.util import (
|
||||||
populate_chain_of_trust_task_graph,
|
populate_chain_of_trust_task_graph,
|
||||||
|
@ -38,6 +40,39 @@ BUILDER = TaskBuilder(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def pr_or_push():
|
||||||
|
if SKIP_TASKS_TRIGGER in PR_TITLE:
|
||||||
|
print("Pull request title contains", SKIP_TASKS_TRIGGER)
|
||||||
|
print("Exit")
|
||||||
|
exit(0)
|
||||||
|
|
||||||
|
print("Fetching build variants from gradle")
|
||||||
|
variants = build_variants.from_gradle()
|
||||||
|
|
||||||
|
if len(variants) == 0:
|
||||||
|
print("Could not get build variants from gradle")
|
||||||
|
sys.exit(2)
|
||||||
|
|
||||||
|
print("Got variants: {}".format(' '.join(variants)))
|
||||||
|
|
||||||
|
build_tasks = {}
|
||||||
|
other_tasks = {}
|
||||||
|
|
||||||
|
for variant in variants:
|
||||||
|
build_tasks[taskcluster.slugId()] = BUILDER.craft_assemble_task(variant)
|
||||||
|
build_tasks[taskcluster.slugId()] = BUILDER.craft_test_task(variant)
|
||||||
|
|
||||||
|
for craft_function in (
|
||||||
|
BUILDER.craft_detekt_task,
|
||||||
|
BUILDER.craft_ktlint_task,
|
||||||
|
BUILDER.craft_lint_task,
|
||||||
|
BUILDER.craft_compare_locales_task,
|
||||||
|
):
|
||||||
|
other_tasks[taskcluster.slugId()] = craft_function()
|
||||||
|
|
||||||
|
return (build_tasks, other_tasks)
|
||||||
|
|
||||||
|
|
||||||
def nightly(apks, track, commit, date_string):
|
def nightly(apks, track, commit, date_string):
|
||||||
is_staging = track == 'staging-nightly'
|
is_staging = track == 'staging-nightly'
|
||||||
|
|
||||||
|
@ -99,8 +134,7 @@ if __name__ == "__main__":
|
||||||
command = result.command
|
command = result.command
|
||||||
|
|
||||||
if command == 'pr-or-push':
|
if command == 'pr-or-push':
|
||||||
# TODO
|
ordered_groups_of_tasks = pr_or_push()
|
||||||
ordered_groups_of_tasks = {}
|
|
||||||
elif command == 'release':
|
elif command == 'release':
|
||||||
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
|
apks = ["{}/{}".format(result.output, apk) for apk in result.apks]
|
||||||
# nightly(apks, result.track, result.commit, result.date)
|
# nightly(apks, result.track, result.commit, result.date)
|
||||||
|
|
|
@ -0,0 +1,24 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import print_function
|
||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
|
||||||
|
|
||||||
|
def from_gradle():
|
||||||
|
process = subprocess.Popen([
|
||||||
|
"./gradlew", "--no-daemon", "--quiet", "printBuildVariants"
|
||||||
|
], stdout=subprocess.PIPE)
|
||||||
|
(output, err) = process.communicate()
|
||||||
|
exit_code = process.wait()
|
||||||
|
|
||||||
|
if exit_code != 0:
|
||||||
|
print("Gradle command returned error: {}".format(exit_code))
|
||||||
|
|
||||||
|
variants_line = [line for line in output.split('\n') if line.startswith('variants: ')][0]
|
||||||
|
variants_json = variants_line.split(' ', 1)[1]
|
||||||
|
variants = json.loads(variants_json)
|
||||||
|
|
||||||
|
return variants
|
|
@ -70,7 +70,7 @@ class TaskBuilder(object):
|
||||||
]
|
]
|
||||||
|
|
||||||
return self._craft_build_ish_task(
|
return self._craft_build_ish_task(
|
||||||
name='Fenix - Build task',
|
name='Build task',
|
||||||
description='Build Fenix from source code',
|
description='Build Fenix from source code',
|
||||||
command=command,
|
command=command,
|
||||||
scopes=[
|
scopes=[
|
||||||
|
@ -81,6 +81,60 @@ class TaskBuilder(object):
|
||||||
is_staging=is_staging,
|
is_staging=is_staging,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def craft_assemble_task(self, variant):
|
||||||
|
return self._craft_gradle_clean_task(
|
||||||
|
name='assemble: {}'.format(variant),
|
||||||
|
description='Building and testing variant {}'.format(variant),
|
||||||
|
gradle_task='assemble{}'.format(variant.capitalize()),
|
||||||
|
artifacts=_craft_artifacts_from_variant(variant),
|
||||||
|
)
|
||||||
|
|
||||||
|
def craft_test_task(self, variant):
|
||||||
|
return self._craft_gradle_clean_task(
|
||||||
|
name='test: {}'.format(variant),
|
||||||
|
description='Building and testing variant {}'.format(variant),
|
||||||
|
gradle_task='test{}UnitTest'.format(variant.capitalize()),
|
||||||
|
)
|
||||||
|
|
||||||
|
def craft_detekt_task(self):
|
||||||
|
return self._craft_gradle_clean_task(
|
||||||
|
name='detekt',
|
||||||
|
description='Running detekt over all modules',
|
||||||
|
gradle_task='detekt'
|
||||||
|
)
|
||||||
|
|
||||||
|
def craft_ktlint_task(self):
|
||||||
|
return self._craft_gradle_clean_task(
|
||||||
|
name='ktlint',
|
||||||
|
description='Running ktlint over all modules',
|
||||||
|
gradle_task='ktlint'
|
||||||
|
)
|
||||||
|
|
||||||
|
def craft_lint_task(self):
|
||||||
|
return self._craft_gradle_clean_task(
|
||||||
|
name='lint',
|
||||||
|
description='Running ktlint over all modules',
|
||||||
|
gradle_task='lint'
|
||||||
|
)
|
||||||
|
|
||||||
|
def _craft_gradle_clean_task(self, name, description, gradle_task, artifacts=None):
|
||||||
|
return self._craft_build_ish_task(
|
||||||
|
name=name,
|
||||||
|
description=description,
|
||||||
|
command='./gradlew --no-daemon clean {}'.format(gradle_task),
|
||||||
|
artifacts=artifacts,
|
||||||
|
)
|
||||||
|
|
||||||
|
def craft_compare_locales_task(self):
|
||||||
|
return self._craft_build_ish_task(
|
||||||
|
name='compare-locales',
|
||||||
|
description='Validate strings.xml with compare-locales',
|
||||||
|
command=(
|
||||||
|
'pip install "compare-locales>=5.0.2,<6.0" && '
|
||||||
|
'compare-locales --validate l10n.toml .'
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
def _craft_build_ish_task(
|
def _craft_build_ish_task(
|
||||||
self, name, description, command, dependencies=None, artifacts=None, scopes=None,
|
self, name, description, command, dependencies=None, artifacts=None, scopes=None,
|
||||||
routes=None, is_staging=True
|
routes=None, is_staging=True
|
||||||
|
@ -155,7 +209,7 @@ class TaskBuilder(object):
|
||||||
"scopes": scopes,
|
"scopes": scopes,
|
||||||
"payload": payload,
|
"payload": payload,
|
||||||
"metadata": {
|
"metadata": {
|
||||||
"name": name,
|
"name": "Fenix - {}".format(name),
|
||||||
"description": description,
|
"description": description,
|
||||||
"owner": self.owner,
|
"owner": self.owner,
|
||||||
"source": self.source,
|
"source": self.source,
|
||||||
|
@ -198,7 +252,7 @@ class TaskBuilder(object):
|
||||||
'dep-signing' if is_staging else 'release-signing'
|
'dep-signing' if is_staging else 'release-signing'
|
||||||
)
|
)
|
||||||
],
|
],
|
||||||
name="Fenix - Signing task",
|
name="Signing task",
|
||||||
description="Sign release builds of Fenix",
|
description="Sign release builds of Fenix",
|
||||||
payload=payload
|
payload=payload
|
||||||
)
|
)
|
||||||
|
@ -228,12 +282,65 @@ class TaskBuilder(object):
|
||||||
':dep' if is_staging else ''
|
':dep' if is_staging else ''
|
||||||
)
|
)
|
||||||
],
|
],
|
||||||
name="Fenix - Push task",
|
name="Push task",
|
||||||
description="Upload signed release builds of Fenix to Google Play",
|
description="Upload signed release builds of Fenix to Google Play",
|
||||||
payload=payload
|
payload=payload
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _craft_artifacts_from_variant(variant):
|
||||||
|
return {
|
||||||
|
'public/target.apk': {
|
||||||
|
'type': 'file',
|
||||||
|
'path': _craft_apk_full_path_from_variant(variant),
|
||||||
|
'expires': taskcluster.stringDate(taskcluster.fromNow(DEFAULT_EXPIRES_IN)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _craft_apk_full_path_from_variant(variant):
|
||||||
|
architecture, build_type = _get_architecture_and_build_type_from_variant(variant)
|
||||||
|
|
||||||
|
short_variant = variant[:-len(build_type)]
|
||||||
|
postfix = '-unsigned' if build_type == 'release' else ''
|
||||||
|
product = '{}{}'.format(product[0].lower(), product[1:])
|
||||||
|
|
||||||
|
return '/opt/fenix/app/build/outputs/apk/{short_variant}/{build_type}/app-{architecture}-{product}-{build_type}{postfix}.apk'.format( # noqa: E501
|
||||||
|
architecture=architecture,
|
||||||
|
build_type=build_type,
|
||||||
|
product=product,
|
||||||
|
short_variant=short_variant,
|
||||||
|
postfix=postfix
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _get_architecture_and_build_type_from_variant(variant):
|
||||||
|
variant = variant.lower()
|
||||||
|
|
||||||
|
architecture = None
|
||||||
|
if 'aarch64' in variant:
|
||||||
|
architecture = 'aarch64'
|
||||||
|
elif 'x86' in variant:
|
||||||
|
architecture = 'x86'
|
||||||
|
elif 'arm' in variant:
|
||||||
|
architecture = 'arm'
|
||||||
|
|
||||||
|
build_type = None
|
||||||
|
if variant.endswith('debug'):
|
||||||
|
build_type = 'debug'
|
||||||
|
elif variant.endswith('release'):
|
||||||
|
build_type = 'release'
|
||||||
|
|
||||||
|
if not architecture or not build_type:
|
||||||
|
raise ValueError(
|
||||||
|
'Unsupported variant "{}". Found architecture, build_type: {}'.format(
|
||||||
|
variant, (architecture, build_type)
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
return architecture, build_type
|
||||||
|
|
||||||
|
|
||||||
def schedule_task(queue, taskId, task):
|
def schedule_task(queue, taskId, task):
|
||||||
print("TASK", taskId)
|
print("TASK", taskId)
|
||||||
print(json.dumps(task, indent=4, separators=(',', ': ')))
|
print(json.dumps(task, indent=4, separators=(',', ': ')))
|
||||||
|
|
Loading…
Reference in New Issue