Bug 1580778 - Migrate build tasks to taskgraph and retire the old decision task (#5488)
parent
7c55aff1be
commit
6a8f7224dc
|
@ -8,7 +8,6 @@ jobs:
|
|||
job:
|
||||
type: decision-task
|
||||
treeherder-symbol: Nd
|
||||
# TODO change target method once first tasks are migrated off "old-decision"
|
||||
target-tasks-method: nightly
|
||||
when: [{hour: 6, minute: 0}]
|
||||
when: [{hour: 18, minute: 0}]
|
||||
|
@ -16,6 +15,5 @@ jobs:
|
|||
job:
|
||||
type: decision-task
|
||||
treeherder-symbol: raptor-D
|
||||
# TODO change target method once first tasks are migrated off "old-decision"
|
||||
target-tasks-method: raptor
|
||||
when: [{hour: 1, minute: 0}]
|
||||
|
|
|
@ -22,6 +22,7 @@ out/
|
|||
# Gradle files
|
||||
.gradle/
|
||||
build/
|
||||
!taskcluster/ci/build
|
||||
|
||||
# Local configuration file (sdk path, etc)
|
||||
local.properties
|
||||
|
|
|
@ -1,87 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
"""
|
||||
Decision task for nightly releases.
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import argparse
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
|
||||
import taskcluster
|
||||
|
||||
from lib.gradle import get_variant
|
||||
from lib.tasks import (
|
||||
fetch_mozharness_task_id,
|
||||
schedule_task_graph,
|
||||
TaskBuilder,
|
||||
)
|
||||
from lib.chain_of_trust import (
|
||||
populate_chain_of_trust_task_graph,
|
||||
populate_chain_of_trust_required_but_unused_files
|
||||
)
|
||||
|
||||
def pr(builder):
|
||||
tasks = []
|
||||
|
||||
variant = get_variant('debug', 'geckoNightly')
|
||||
tasks.append(builder.craft_assemble_pr_task(variant))
|
||||
tasks.append(builder.craft_test_pr_task(variant))
|
||||
|
||||
for task in tasks:
|
||||
task['attributes']['code-review'] = True
|
||||
|
||||
return tasks
|
||||
|
||||
def push(builder):
|
||||
# We want the same tasks on pushes than on PRs, for now.
|
||||
return pr(builder)
|
||||
|
||||
def raptor(builder, is_staging):
|
||||
mozharness_task_id = fetch_mozharness_task_id()
|
||||
gecko_revision = taskcluster.Queue({
|
||||
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
|
||||
}).task(mozharness_task_id)['payload']['env']['GECKO_HEAD_REV']
|
||||
|
||||
variant = get_variant('forPerformanceTest', 'geckoNightly')
|
||||
build_task = builder.craft_assemble_raptor_task(variant)
|
||||
|
||||
# Signing and Raptor tasks are generated in taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def release(builder, channel, engine, is_staging, version_name):
|
||||
variant = get_variant('fenix' + channel.capitalize(), engine)
|
||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
# The signing push-apk tasks are generated by taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def release_as_fennec(builder, is_staging, version_name):
|
||||
variant = get_variant('fennecProduction', 'geckoBeta')
|
||||
channel = 'fennec-production'
|
||||
|
||||
build_task = builder.craft_assemble_release_task(variant, channel, is_staging, version_name)
|
||||
|
||||
# The signing task is generated by taskgraph
|
||||
return [build_task]
|
||||
|
||||
|
||||
def nightly_to_production_app(builder, is_staging, version_name):
|
||||
# Since the Fenix nightly was launched, we've pushed it to the production app "org.mozilla.fenix" on the
|
||||
# "nightly" track. We're moving towards having each channel be published to its own app, but we need to
|
||||
# keep updating this "backwards-compatible" nightly for a while yet
|
||||
variant = get_variant('fenixNightlyLegacy', 'geckoNightly')
|
||||
taskcluster_apk_paths = variant.upstream_artifacts()
|
||||
|
||||
build_task = builder.craft_assemble_release_task(
|
||||
variant, 'nightly-legacy', is_staging, version_name)
|
||||
|
||||
# Nimbledroid, signing and push-apk tasks are generated by taskgraph
|
||||
return [build_task]
|
|
@ -1,20 +0,0 @@
|
|||
import json
|
||||
|
||||
|
||||
def populate_chain_of_trust_required_but_unused_files():
|
||||
# These files are needed to keep chainOfTrust happy. However, they have no
|
||||
# need for android-components, at the moment. For more details, see:
|
||||
# https://github.com/mozilla-releng/scriptworker/pull/209/files#r184180585
|
||||
|
||||
for file_names in ('actions.json', 'parameters.yml'):
|
||||
with open(file_names, 'w') as f:
|
||||
json.dump({}, f) # Yaml is a super-set of JSON.
|
||||
|
||||
|
||||
def populate_chain_of_trust_task_graph(full_task_graph):
|
||||
# full_task_graph must follow the format:
|
||||
# {
|
||||
# task_id: full_task_definition
|
||||
# }
|
||||
with open('task-graph.json', 'w') as f:
|
||||
json.dump(full_task_graph, f)
|
|
@ -2,360 +2,8 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import arrow
|
||||
import datetime
|
||||
import json
|
||||
import os
|
||||
import taskcluster
|
||||
|
||||
from ..lib.util import upper_case_first_letter, convert_camel_case_into_kebab_case, lower_case_first_letter
|
||||
|
||||
DEFAULT_EXPIRES_IN = '1 year'
|
||||
DEFAULT_APK_ARTIFACT_LOCATION = 'public/target.apk'
|
||||
_OFFICIAL_REPO_URL = 'https://github.com/mozilla-mobile/fenix'
|
||||
_DEFAULT_TASK_URL = 'https://queue.taskcluster.net/v1/task'
|
||||
GOOGLE_APPLICATION_CREDENTIALS = '.firebase_token.json'
|
||||
# Bug 1558456 - Stop tracking youtube-playback-test on motoG5 for >1080p cases
|
||||
ARM_RAPTOR_URL_PARAMS = [
|
||||
"exclude=1,2,9,10,17,18,21,22,26,28,30,32,39,40,47,"
|
||||
"48,55,56,63,64,71,72,79,80,83,84,89,90,95,96",
|
||||
]
|
||||
|
||||
|
||||
class TaskBuilder(object):
|
||||
def __init__(
|
||||
self,
|
||||
task_id,
|
||||
repo_url,
|
||||
git_ref,
|
||||
short_head_branch, commit, owner, source, scheduler_id, date_string,
|
||||
tasks_priority='lowest',
|
||||
trust_level=1
|
||||
):
|
||||
self.task_id = task_id
|
||||
self.repo_url = repo_url
|
||||
self.git_ref = git_ref
|
||||
self.short_head_branch = short_head_branch
|
||||
self.commit = commit
|
||||
self.owner = owner
|
||||
self.source = source
|
||||
self.scheduler_id = scheduler_id
|
||||
self.trust_level = trust_level
|
||||
self.tasks_priority = tasks_priority
|
||||
self.date = arrow.get(date_string, 'YYYYMMDDHHmmss')
|
||||
self.trust_level = trust_level
|
||||
|
||||
def craft_assemble_release_task(self, variant, channel, is_staging, version_name):
|
||||
if is_staging:
|
||||
secret_index = 'garbage/staging/project/mobile/fenix'
|
||||
else:
|
||||
secret_index = 'project/mobile/fenix/{}'.format(channel)
|
||||
|
||||
pre_gradle_commands = (
|
||||
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
|
||||
secret_index, key, target_file
|
||||
)
|
||||
for key, target_file in (
|
||||
('sentry_dsn', '.sentry_token'),
|
||||
('leanplum', '.leanplum_token'),
|
||||
('adjust', '.adjust_token'),
|
||||
('digital_asset_links', '.digital_asset_links_token'),
|
||||
('firebase', 'app/src/{}/res/values/firebase.xml'.format(variant.build_type)),
|
||||
)
|
||||
)
|
||||
|
||||
capitalized_build_type = upper_case_first_letter(variant.build_type)
|
||||
gradle_commands = (
|
||||
'./gradlew --no-daemon -PversionName="{}" clean test assemble{}'.format(
|
||||
version_name, capitalized_build_type),
|
||||
)
|
||||
|
||||
command = ' && '.join(
|
||||
cmd
|
||||
for commands in (pre_gradle_commands, gradle_commands)
|
||||
for cmd in commands
|
||||
if cmd
|
||||
)
|
||||
|
||||
routes = [] if is_staging else [
|
||||
"notify.email.fenix-eng-notifications@mozilla.com.on-failed"
|
||||
]
|
||||
|
||||
return self._craft_build_ish_task(
|
||||
name='Build {} task'.format(capitalized_build_type),
|
||||
description='Build Fenix {} from source code'.format(capitalized_build_type),
|
||||
command=command,
|
||||
scopes=[
|
||||
"secrets:get:{}".format(secret_index)
|
||||
],
|
||||
artifacts=variant.artifacts(),
|
||||
routes=routes,
|
||||
treeherder={
|
||||
'jobKind': 'build',
|
||||
'machine': {
|
||||
'platform': 'android-all',
|
||||
},
|
||||
'collection': {
|
||||
'opt': True,
|
||||
},
|
||||
'symbol': '{}-A'.format(variant.build_type),
|
||||
'tier': 1,
|
||||
},
|
||||
attributes={
|
||||
'apks': variant.upstream_artifacts_per_abi,
|
||||
}
|
||||
)
|
||||
|
||||
def craft_assemble_raptor_task(self, variant):
|
||||
command = ' && '.join((
|
||||
'echo "https://fake@sentry.prod.mozaws.net/368" > .sentry_token',
|
||||
'echo "--" > .adjust_token',
|
||||
'echo "-:-" > .leanplum_token',
|
||||
'touch .digital_asset_links_token',
|
||||
'./gradlew --no-daemon clean assemble{}'.format(variant.name),
|
||||
))
|
||||
|
||||
return self._craft_build_ish_task(
|
||||
name='assemble: {}'.format(variant.name),
|
||||
description='Building and testing variant {}'.format(variant.name),
|
||||
command=command,
|
||||
artifacts=variant.artifacts(),
|
||||
treeherder={
|
||||
'groupSymbol': variant.build_type,
|
||||
'jobKind': 'build',
|
||||
'machine': {
|
||||
'platform': 'android-all',
|
||||
},
|
||||
'symbol': 'A',
|
||||
'tier': 1,
|
||||
},
|
||||
attributes={
|
||||
'build-type': 'raptor',
|
||||
'apks': variant.upstream_artifacts_per_abi,
|
||||
},
|
||||
)
|
||||
|
||||
def craft_assemble_pr_task(self, variant):
|
||||
return self._craft_clean_gradle_task(
|
||||
name='assemble: {}'.format(variant.name),
|
||||
description='Building and testing variant {}'.format(variant.name),
|
||||
gradle_task='assemble{}'.format(variant.name),
|
||||
artifacts=variant.artifacts(),
|
||||
treeherder={
|
||||
'groupSymbol': variant.build_type,
|
||||
'jobKind': 'build',
|
||||
'machine': {
|
||||
'platform': 'android-all',
|
||||
},
|
||||
'symbol': 'A',
|
||||
'tier': 1,
|
||||
},
|
||||
)
|
||||
|
||||
def craft_test_pr_task(self, variant):
|
||||
# upload coverage only once, if the variant is arm64
|
||||
secret_index = 'project/mobile/fenix/public-tokens'
|
||||
pre_gradle_commands = (
|
||||
'python automation/taskcluster/helper/get-secret.py -s {} -k {} -f {}'.format(
|
||||
secret_index, key, target_file
|
||||
)
|
||||
for key, target_file in (
|
||||
('codecov', '.cc_token'),
|
||||
)
|
||||
)
|
||||
|
||||
gradle_commands = (
|
||||
'./gradlew --no-daemon clean -Pcoverage jacocoGeckoNightlyDebugTestReport',
|
||||
)
|
||||
|
||||
post_gradle_commands = (
|
||||
'automation/taskcluster/upload_coverage_report.sh',
|
||||
)
|
||||
|
||||
command = ' && '.join(
|
||||
cmd
|
||||
for commands in (pre_gradle_commands, gradle_commands, post_gradle_commands)
|
||||
for cmd in commands
|
||||
if cmd
|
||||
)
|
||||
|
||||
return self._craft_build_ish_task(
|
||||
name='test: {}'.format(variant.name),
|
||||
description='Building and testing variant {}'.format(variant.name),
|
||||
command=command,
|
||||
treeherder={
|
||||
'groupSymbol': variant.build_type,
|
||||
'jobKind': 'test',
|
||||
'machine': {
|
||||
'platform': 'android-all',
|
||||
},
|
||||
'symbol': 'T',
|
||||
'tier': 1,
|
||||
},
|
||||
scopes=[
|
||||
'secrets:get:{}'.format(secret_index)
|
||||
]
|
||||
)
|
||||
|
||||
def _craft_clean_gradle_task(
|
||||
self, name, description, gradle_task, artifacts=None, routes=None, treeherder=None, scopes=None
|
||||
):
|
||||
return self._craft_build_ish_task(
|
||||
name=name,
|
||||
description=description,
|
||||
command='./gradlew --no-daemon clean {}'.format(gradle_task),
|
||||
artifacts=artifacts,
|
||||
routes=routes,
|
||||
treeherder=treeherder,
|
||||
scopes=scopes,
|
||||
)
|
||||
|
||||
def _craft_build_ish_task(
|
||||
self, name, description, command, dependencies=None, artifacts=None,
|
||||
routes=None, treeherder=None, env_vars=None, scopes=None, attributes=None
|
||||
):
|
||||
artifacts = {} if artifacts is None else artifacts
|
||||
scopes = [] if scopes is None else scopes
|
||||
env_vars = {} if env_vars is None else env_vars
|
||||
|
||||
checkout_command = ' && '.join([
|
||||
"export TERM=dumb",
|
||||
"git fetch {} {}".format(self.repo_url, self.git_ref),
|
||||
"git config advice.detachedHead false",
|
||||
"git checkout FETCH_HEAD",
|
||||
])
|
||||
|
||||
command = '{} && {}'.format(checkout_command, command)
|
||||
|
||||
features = {}
|
||||
if artifacts:
|
||||
features['chainOfTrust'] = True
|
||||
if any(scope.startswith('secrets:') for scope in scopes):
|
||||
features['taskclusterProxy'] = True
|
||||
payload = {
|
||||
"features": features,
|
||||
"env": env_vars,
|
||||
"maxRunTime": 7200,
|
||||
"image": "mozillamobile/fenix:1.4",
|
||||
"command": [
|
||||
"/bin/bash",
|
||||
"--login",
|
||||
"-cx",
|
||||
# Some tasks like nimbledroid do have tasks references
|
||||
{'task-reference': command},
|
||||
],
|
||||
"artifacts": artifacts,
|
||||
}
|
||||
|
||||
return self._craft_default_task_definition(
|
||||
worker_type='mobile-{}-b-fenix'.format(self.trust_level),
|
||||
provisioner_id='aws-provisioner-v1',
|
||||
name=name,
|
||||
description=description,
|
||||
payload=payload,
|
||||
dependencies=dependencies,
|
||||
routes=routes,
|
||||
scopes=scopes,
|
||||
treeherder=treeherder,
|
||||
attributes=attributes,
|
||||
)
|
||||
|
||||
def _craft_default_task_definition(
|
||||
self,
|
||||
worker_type,
|
||||
provisioner_id,
|
||||
name,
|
||||
description,
|
||||
payload,
|
||||
dependencies=None,
|
||||
routes=None,
|
||||
scopes=None,
|
||||
treeherder=None,
|
||||
notify=None,
|
||||
attributes=None
|
||||
):
|
||||
dependencies = {} if dependencies is None else dependencies
|
||||
scopes = [] if scopes is None else scopes
|
||||
routes = [] if routes is None else routes
|
||||
treeherder = {} if treeherder is None else treeherder
|
||||
attributes = {} if attributes is None else attributes
|
||||
|
||||
created = datetime.datetime.now()
|
||||
deadline = taskcluster.fromNow('1 day')
|
||||
expires = taskcluster.fromNow(DEFAULT_EXPIRES_IN)
|
||||
|
||||
routes.append('checks')
|
||||
if self.trust_level == 3:
|
||||
routes.append('tc-treeherder.v2.fenix.{}'.format(self.commit))
|
||||
|
||||
extra = {
|
||||
"treeherder": treeherder,
|
||||
}
|
||||
if notify:
|
||||
extra['notify'] = notify
|
||||
|
||||
return {
|
||||
"attributes": attributes,
|
||||
"dependencies": dependencies,
|
||||
"label": name,
|
||||
"task": {
|
||||
"provisionerId": provisioner_id,
|
||||
"workerType": worker_type,
|
||||
"taskGroupId": self.task_id,
|
||||
"schedulerId": self.scheduler_id,
|
||||
"created": taskcluster.stringDate(created),
|
||||
"deadline": taskcluster.stringDate(deadline),
|
||||
"expires": taskcluster.stringDate(expires),
|
||||
"retries": 5,
|
||||
"tags": {},
|
||||
"priority": self.tasks_priority,
|
||||
"requires": "all-completed",
|
||||
"routes": routes,
|
||||
"scopes": scopes,
|
||||
"payload": payload,
|
||||
"extra": extra,
|
||||
"metadata": {
|
||||
"name": "Fenix - {}".format(name),
|
||||
"description": description,
|
||||
"owner": self.owner,
|
||||
"source": self.source,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def schedule_task(queue, taskId, task):
|
||||
print("TASK", taskId)
|
||||
print(json.dumps(task, indent=4, separators=(',', ': ')))
|
||||
|
||||
result = queue.createTask(taskId, task)
|
||||
print("RESULT", taskId)
|
||||
print(json.dumps(result))
|
||||
|
||||
|
||||
def schedule_task_graph(ordered_groups_of_tasks):
|
||||
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
|
||||
full_task_graph = {}
|
||||
|
||||
# TODO: Switch to async python to speed up submission
|
||||
for group_of_tasks in ordered_groups_of_tasks:
|
||||
for task_id, task_definition in group_of_tasks.items():
|
||||
schedule_task(queue, task_id, task_definition)
|
||||
|
||||
full_task_graph[task_id] = {
|
||||
# Some values of the task definition are automatically filled. Querying the task
|
||||
# allows to have the full definition. This is needed to make Chain of Trust happy
|
||||
'task': queue.task(task_id),
|
||||
}
|
||||
|
||||
return full_task_graph
|
||||
|
||||
|
||||
def fetch_mozharness_task_id():
|
||||
# We now want to use the latest available raptor
|
||||
raptor_index = 'gecko.v2.mozilla-central.nightly.latest.mobile.android-x86_64-opt'
|
||||
return taskcluster.Index({
|
||||
'rootUrl': os.environ.get('TASKCLUSTER_PROXY_URL', 'https://taskcluster.net'),
|
||||
}).findTask(raptor_index)['taskId']
|
||||
|
|
|
@ -1,15 +0,0 @@
|
|||
import re
|
||||
|
||||
|
||||
def convert_camel_case_into_kebab_case(string):
|
||||
# Inspired from https://stackoverflow.com/questions/1175208/elegant-python-function-to-convert-camelcase-to-snake-case # noqa: E501
|
||||
first_pass = re.sub('(.)([A-Z][a-z]+)', r'\1-\2', string)
|
||||
return re.sub('([a-z0-9])([A-Z])', r'\1-\2', first_pass).lower()
|
||||
|
||||
|
||||
def lower_case_first_letter(string):
|
||||
return '{}{}'.format(string[0].lower(), string[1:])
|
||||
|
||||
|
||||
def upper_case_first_letter(string):
|
||||
return string[0].upper() + string[1:]
|
|
@ -1,34 +0,0 @@
|
|||
import taskcluster
|
||||
|
||||
|
||||
class VariantApk:
|
||||
def __init__(self, build_type, abi, engine, file_name):
|
||||
self.abi = abi
|
||||
self.taskcluster_path = u'public/build/{}/{}/target.apk'.format(abi, engine)
|
||||
self.absolute_path = '/opt/fenix/app/build/outputs/apk/{}/{}/{}'.format(engine, build_type, file_name)
|
||||
|
||||
|
||||
class Variant:
|
||||
def __init__(self, name, build_type, apks):
|
||||
self.name = name
|
||||
self.build_type = build_type
|
||||
self._apks = apks
|
||||
|
||||
def get_apk(self, abi):
|
||||
return [apk for apk in self._apks if apk.abi == abi][0]
|
||||
|
||||
def artifacts(self):
|
||||
return {
|
||||
apk.taskcluster_path: {
|
||||
'type': 'file',
|
||||
'path': apk.absolute_path,
|
||||
'expires': taskcluster.stringDate(taskcluster.fromNow('1 year')),
|
||||
} for apk in self._apks
|
||||
}
|
||||
|
||||
def upstream_artifacts(self):
|
||||
return [apk.taskcluster_path for apk in self._apks]
|
||||
|
||||
@property
|
||||
def upstream_artifacts_per_abi(self):
|
||||
return {apk.abi: apk.taskcluster_path for apk in self._apks}
|
|
@ -1,105 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
import argparse
|
||||
import datetime
|
||||
import jsone
|
||||
import os
|
||||
import slugid
|
||||
import taskcluster
|
||||
import yaml
|
||||
|
||||
from git import Repo
|
||||
from lib.tasks import schedule_task
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(__file__), '../..')
|
||||
|
||||
|
||||
class InvalidGithubRepositoryError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def calculate_git_references(root):
|
||||
repo = Repo(root)
|
||||
remote = repo.remote()
|
||||
branch = repo.head.reference
|
||||
|
||||
if not remote.url.startswith('https://github.com'):
|
||||
raise InvalidGithubRepositoryError('expected remote to be a GitHub repository (accessed via HTTPs)')
|
||||
|
||||
html_url = remote.url[:-4] if remote.url.endswith('.git') else remote.url
|
||||
return html_url, str(branch), str(branch.commit)
|
||||
|
||||
|
||||
def make_decision_task(params):
|
||||
"""Generate a basic decision task, based on the root .taskcluster.yml"""
|
||||
with open(os.path.join(ROOT, '.taskcluster.yml'), 'rb') as f:
|
||||
taskcluster_yml = yaml.safe_load(f)
|
||||
|
||||
slugids = {}
|
||||
|
||||
def as_slugid(name):
|
||||
if name not in slugids:
|
||||
slugids[name] = slugid.nice()
|
||||
return slugids[name]
|
||||
|
||||
repository_parts = params['html_url'].split('/')
|
||||
repository_full_name = '/'.join((repository_parts[-2], repository_parts[-1]))
|
||||
|
||||
# provide a similar JSON-e context to what taskcluster-github provides
|
||||
context = {
|
||||
'tasks_for': 'cron',
|
||||
'cron': {
|
||||
'task_id': params['cron_task_id'],
|
||||
'name': params['name'],
|
||||
},
|
||||
'now': datetime.datetime.utcnow().isoformat()[:23] + 'Z',
|
||||
'as_slugid': as_slugid,
|
||||
'event': {
|
||||
'repository': {
|
||||
'html_url': params['html_url'],
|
||||
'full_name': repository_full_name,
|
||||
},
|
||||
'release': {
|
||||
'tag_name': params['head_rev'],
|
||||
'target_commitish': params['branch'],
|
||||
},
|
||||
'sender': {
|
||||
'login': 'TaskclusterHook',
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
rendered = jsone.render(taskcluster_yml, context)
|
||||
if len(rendered['tasks']) != 1:
|
||||
raise Exception('Expected .taskcluster.yml to only produce one cron task')
|
||||
task = rendered['tasks'][0]
|
||||
|
||||
task_id = task.pop('taskId')
|
||||
return task_id, task
|
||||
|
||||
|
||||
def schedule():
|
||||
parser = argparse.ArgumentParser(
|
||||
description='Creates and submit a graph of tasks on Taskcluster.'
|
||||
)
|
||||
|
||||
parser.add_argument('name', choices=['nightly', 'raptor'])
|
||||
result = parser.parse_args()
|
||||
queue = taskcluster.Queue({'baseUrl': 'http://taskcluster/queue/v1'})
|
||||
|
||||
html_url, branch, head_rev = calculate_git_references(ROOT)
|
||||
params = {
|
||||
'html_url': html_url,
|
||||
'head_rev': head_rev,
|
||||
'branch': branch,
|
||||
'cron_task_id': os.environ.get('CRON_TASK_ID', '<cron_task_id>'),
|
||||
'name': result.name,
|
||||
}
|
||||
decision_task_id, decision_task = make_decision_task(params)
|
||||
schedule_task(queue, decision_task_id, decision_task)
|
||||
print('All scheduled!')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
schedule()
|
|
@ -0,0 +1,108 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.transforms.build:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
job-defaults:
|
||||
description: Build Fenix from source code.
|
||||
treeherder:
|
||||
kind: build
|
||||
symbol: B
|
||||
platform: android-all/opt
|
||||
tier: 1
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: base}
|
||||
max-run-time: 7200
|
||||
chain-of-trust: true
|
||||
run:
|
||||
using: gradlew
|
||||
use-caches: false
|
||||
run-on-tasks-for: []
|
||||
# Builds generate multiple APKs with different ABIs. For each APK described
|
||||
# by `gradlew printVariant`, an artifact will be generated. `variant` and
|
||||
# the per-apk config from `printVariant` can be used as substitutions in
|
||||
# `name` and `path`.
|
||||
apk-artifact-template:
|
||||
type: file
|
||||
name: public/build/{abi}/{geckoview_engine}/target.apk
|
||||
path: '/builds/worker/checkouts/src/app/build/outputs/apk/{geckoview_engine}/{gradle_build_type}/{fileName}'
|
||||
|
||||
jobs:
|
||||
debug:
|
||||
attributes:
|
||||
code-review: true
|
||||
run-on-tasks-for: [github-pull-request, github-push]
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: debug
|
||||
treeherder:
|
||||
symbol: debug(B)
|
||||
|
||||
performance-test:
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: forPerformanceTest
|
||||
treeherder:
|
||||
symbol: forPerformanceTest(B)
|
||||
|
||||
nightly:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: fenixNightly
|
||||
treeherder:
|
||||
symbol: nightly(B)
|
||||
|
||||
nightly-legacy:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-nightly-version: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoNightly
|
||||
gradle-build-type: fenixNightlyLegacy
|
||||
treeherder:
|
||||
symbol: nightlyLegacy(B)
|
||||
|
||||
fennec-production:
|
||||
attributes:
|
||||
nightly: true
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fennecProduction
|
||||
treeherder:
|
||||
symbol: nightlyFennec(B)
|
||||
|
||||
beta:
|
||||
attributes:
|
||||
build-type: production
|
||||
release-type: beta
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fenixBeta
|
||||
run-on-tasks-for: [github-release]
|
||||
treeherder:
|
||||
symbol: beta(B)
|
||||
|
||||
production:
|
||||
attributes:
|
||||
release-type: production
|
||||
include-shippable-secrets: true
|
||||
run:
|
||||
geckoview-engine: geckoBeta
|
||||
gradle-build-type: fenixProduction
|
||||
run-on-tasks-for: [github-release]
|
||||
treeherder:
|
||||
symbol: production(B)
|
|
@ -2,6 +2,8 @@
|
|||
trust-domain: mobile
|
||||
treeherder:
|
||||
group-names:
|
||||
'beta': 'Nightly-related tasks'
|
||||
'debug': 'Builds made for testing'
|
||||
'forPerformanceTest': 'Builds made for Raptor and other performance tests'
|
||||
'I': 'Docker Image Builds'
|
||||
'nightly': 'Nightly-related tasks'
|
||||
|
|
|
@ -5,12 +5,11 @@
|
|||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.nimbledroid:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- old-decision
|
||||
- build
|
||||
|
||||
job-defaults:
|
||||
description: Upload APKs to Nimbledroid for performance measurement and tracking
|
||||
|
@ -37,7 +36,7 @@ jobs:
|
|||
nightly:
|
||||
attributes:
|
||||
nightly: true
|
||||
dependencies:
|
||||
build: Build FenixNightlyLegacy task # Comes from the old decision task
|
||||
treeherder:
|
||||
symbol: nimbledroid
|
||||
dependencies:
|
||||
build: build-nightly-legacy
|
||||
|
|
|
@ -5,7 +5,7 @@
|
|||
loader: taskgraph.loader.transform:loader
|
||||
|
||||
kind-dependencies:
|
||||
- old-decision
|
||||
- build
|
||||
- test
|
||||
|
||||
transforms:
|
||||
|
|
|
@ -5,7 +5,8 @@
|
|||
loader: fenix_taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.pushapk:transforms
|
||||
- fenix_taskgraph.transforms.single_dep:transforms
|
||||
- fenix_taskgraph.transforms.push_apk:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
|
@ -21,6 +22,21 @@ job-template:
|
|||
worker-type: push-apk
|
||||
worker:
|
||||
commit: true
|
||||
channel:
|
||||
by-build-type:
|
||||
nightly: nightly
|
||||
nightly-legacy: production
|
||||
production: production
|
||||
dep:
|
||||
by-level:
|
||||
'3': false
|
||||
default: true
|
||||
google-play-track:
|
||||
by-build-type:
|
||||
nightly-legacy: nightly
|
||||
default: ''
|
||||
product: fenix
|
||||
|
||||
treeherder:
|
||||
job-symbol: gp
|
||||
kind: build
|
||||
|
|
|
@ -1,14 +1,16 @@
|
|||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
transforms:
|
||||
- fenix_taskgraph.raptor:transforms
|
||||
- fenix_taskgraph.transforms.raptor:transforms
|
||||
- taskgraph.transforms.job:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- signing
|
||||
|
||||
only-for-build-types:
|
||||
- performance-test
|
||||
|
||||
only-for-abis:
|
||||
- armeabi-v7a
|
||||
- arm64-v8a
|
||||
|
@ -49,8 +51,6 @@ job-defaults:
|
|||
- name: public/test_info/
|
||||
path: workspace/build/blobber_upload_dir
|
||||
type: directory
|
||||
|
||||
|
||||
run:
|
||||
using: run-task
|
||||
checkout: false
|
||||
|
|
|
@ -2,101 +2,54 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: taskgraph.loader.transform:loader
|
||||
loader: fenix_taskgraph.loader.single_dep:loader
|
||||
|
||||
transforms:
|
||||
- fenix_taskgraph.signing:transforms
|
||||
- fenix_taskgraph.transforms.single_dep:transforms
|
||||
- fenix_taskgraph.transforms.signing:transforms
|
||||
- taskgraph.transforms.task:transforms
|
||||
|
||||
kind-dependencies:
|
||||
- old-decision
|
||||
- build
|
||||
|
||||
job-defaults:
|
||||
job-template:
|
||||
description: Sign Fenix
|
||||
worker-type:
|
||||
by-variant:
|
||||
by-build-type:
|
||||
(fennec-production|nightly|nightly-legacy|production):
|
||||
by-level:
|
||||
'3': signing
|
||||
default: dep-signing
|
||||
default: dep-signing
|
||||
worker:
|
||||
signing-type:
|
||||
by-build-type:
|
||||
fennec-production:
|
||||
by-level:
|
||||
'3': fennec-production-signing
|
||||
default: dep-signing
|
||||
nightly:
|
||||
by-level:
|
||||
'3': nightly-signing
|
||||
default: dep-signing
|
||||
nightly-legacy:
|
||||
by-level:
|
||||
'3': production-signing
|
||||
default: dep-signing
|
||||
performance-test: dep-signing
|
||||
production:
|
||||
by-level:
|
||||
'3': production-signing
|
||||
default: dep-signing
|
||||
default: dep-signing
|
||||
index:
|
||||
by-build-type:
|
||||
(fennec-production|nightly|performance-test|production):
|
||||
type: signing
|
||||
default: {}
|
||||
run-on-tasks-for: []
|
||||
treeherder:
|
||||
job-symbol: Bs
|
||||
kind: build
|
||||
tier: 2
|
||||
platform: android-all/opt
|
||||
|
||||
jobs:
|
||||
performance-test:
|
||||
attributes:
|
||||
build-type: performance-test
|
||||
index:
|
||||
type: signing
|
||||
worker:
|
||||
signing-type: dep-signing
|
||||
dependencies:
|
||||
build: 'assemble: geckoNightlyForPerformanceTest' # comes from the old-decision task
|
||||
treeherder:
|
||||
symbol: forPerformanceTest(s)
|
||||
|
||||
production:
|
||||
attributes:
|
||||
build-type: production
|
||||
index:
|
||||
type: signing
|
||||
worker:
|
||||
signing-type:
|
||||
by-level:
|
||||
'3': production-signing
|
||||
default: dep-signing
|
||||
dependencies:
|
||||
build: 'Build FenixProduction task' # comes from the old-decision task
|
||||
run-on-tasks-for: [github-release]
|
||||
treeherder:
|
||||
symbol: production(s)
|
||||
|
||||
fennec-production:
|
||||
attributes:
|
||||
build-type: fennec-production
|
||||
nightly: true
|
||||
index:
|
||||
type: signing
|
||||
worker:
|
||||
signing-type:
|
||||
by-level:
|
||||
'3': fennec-production-signing
|
||||
default: dep-signing
|
||||
dependencies:
|
||||
build: 'Build FennecProduction task' # comes from the old-decision task
|
||||
treeherder:
|
||||
symbol: nightlyFennec(s)
|
||||
|
||||
nightly:
|
||||
attributes:
|
||||
build-type: nightly
|
||||
nightly: true
|
||||
index:
|
||||
type: signing
|
||||
worker:
|
||||
signing-type:
|
||||
by-level:
|
||||
'3': nightly-signing
|
||||
default: dep-signing
|
||||
dependencies:
|
||||
build: 'Build FenixNightly task' # comes from the old-decision task
|
||||
treeherder:
|
||||
symbol: nightly(s)
|
||||
|
||||
nightly-legacy:
|
||||
attributes:
|
||||
build-type: nightly-legacy
|
||||
nightly: true
|
||||
worker:
|
||||
signing-type:
|
||||
by-level:
|
||||
'3': production-signing
|
||||
default: dep-signing
|
||||
dependencies:
|
||||
build: 'Build FenixNightlyLegacy task' # comes from the old-decision task
|
||||
treeherder:
|
||||
symbol: nightlyLegacy(s)
|
||||
tier: 2
|
||||
|
|
|
@ -15,12 +15,30 @@ job-defaults:
|
|||
tier: 2
|
||||
worker-type: b-android
|
||||
worker:
|
||||
docker-image: {in-tree: base}
|
||||
max-run-time: 7200
|
||||
run:
|
||||
using: gradlew
|
||||
use-caches: false
|
||||
|
||||
jobs:
|
||||
debug:
|
||||
attributes:
|
||||
build-type: debug
|
||||
geckoview-engine: geckoNightly
|
||||
code-review: true
|
||||
treeherder:
|
||||
platform: 'android-all/opt'
|
||||
symbol: debug(T)
|
||||
tier: 1
|
||||
run:
|
||||
gradlew: ['clean', '-Pcoverage', 'jacocoGeckoNightlyDebugTestReport']
|
||||
post-gradlew:
|
||||
- ['automation/taskcluster/upload_coverage_report.sh']
|
||||
secrets:
|
||||
- name: project/mobile/fenix/public-tokens
|
||||
key: codecov
|
||||
path: .cc_token
|
||||
ui:
|
||||
attributes:
|
||||
build-type: debug
|
||||
|
|
|
@ -19,12 +19,12 @@ WORKDIR /builds/worker/
|
|||
#-- Configuration -----------------------------------------------------------------------------------------------------
|
||||
#----------------------------------------------------------------------------------------------------------------------
|
||||
|
||||
ENV LANG en_US.UTF-8
|
||||
|
||||
# Do not use fancy output on taskcluster
|
||||
ENV TERM dumb
|
||||
|
||||
ENV GRADLE_OPTS -Xmx4096m -Dorg.gradle.daemon=false
|
||||
ENV ANDROID_SDK_VERSION='3859397' \
|
||||
ANDROID_SDK_ROOT='/builds/worker/android-sdk-linux' \
|
||||
GRADLE_OPTS='-Xmx4096m -Dorg.gradle.daemon=false' \
|
||||
LANG='en_US.UTF-8' \
|
||||
TERM='dumb' \
|
||||
SDK_ZIP_LOCATION="$HOME/sdk-tools-linux.zip"
|
||||
|
||||
#----------------------------------------------------------------------------------------------------------------------
|
||||
#-- System ------------------------------------------------------------------------------------------------------------
|
||||
|
@ -50,7 +50,14 @@ RUN apt-get update -qq \
|
|||
RUN pip install --upgrade pip
|
||||
RUN pip install taskcluster
|
||||
|
||||
RUN locale-gen en_US.UTF-8
|
||||
RUN locale-gen "$LANG"
|
||||
|
||||
RUN curl -o "$SDK_ZIP_LOCATION" "https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip" \
|
||||
&& unzip -d "$ANDROID_SDK_ROOT" "$SDK_ZIP_LOCATION" \
|
||||
&& rm "$SDK_ZIP_LOCATION" \
|
||||
&& yes | "${ANDROID_SDK_ROOT}/tools/bin/sdkmanager" --licenses \
|
||||
&& chown -R worker:worker "$ANDROID_SDK_ROOT"
|
||||
|
||||
|
||||
# %include-run-task
|
||||
|
||||
|
|
|
@ -2,23 +2,19 @@
|
|||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from ..lib.variant import Variant, VariantApk
|
||||
from taskgraph.util.memoize import memoize
|
||||
|
||||
|
||||
@memoize
|
||||
def get_variant(build_type, engine):
|
||||
print("Fetching variant information for build_type='{}', engine='{}'".format(build_type, engine))
|
||||
output = _run_gradle_process('printVariant', variantBuildType=build_type, variantEngine=engine)
|
||||
content = _extract_content_from_command_output(output, prefix='variant: ')
|
||||
raw_variant = json.loads(content)
|
||||
return Variant(
|
||||
raw_variant['name'],
|
||||
build_type,
|
||||
[VariantApk(build_type, raw_apk['abi'], engine, raw_apk['fileName']) for raw_apk in raw_variant['apks']]
|
||||
)
|
||||
return json.loads(content)
|
||||
|
||||
|
||||
def _run_gradle_process(gradle_command, **kwargs):
|
||||
|
@ -32,7 +28,7 @@ def _run_gradle_process(gradle_command, **kwargs):
|
|||
exit_code = process.wait()
|
||||
|
||||
if exit_code is not 0:
|
||||
print("Gradle command returned error: {}".format(exit_code))
|
||||
raise RuntimeError("Gradle command returned error: {}".format(exit_code))
|
||||
|
||||
return output
|
||||
|
|
@ -15,6 +15,7 @@ from pipes import quote as shell_quote
|
|||
gradlew_schema = Schema(
|
||||
{
|
||||
Required("using"): "gradlew",
|
||||
Optional("pre-gradlew"): [[text_type]],
|
||||
Required("gradlew"): [text_type],
|
||||
Optional("post-gradlew"): [[text_type]],
|
||||
# Base work directory used to set up the task.
|
||||
|
@ -53,7 +54,7 @@ def configure_gradlew(config, job, taskdesc):
|
|||
|
||||
|
||||
def _extract_command(run):
|
||||
pre_gradle_commands = [["taskcluster/scripts/install-sdk.sh"]]
|
||||
pre_gradle_commands = run.pop("pre-gradlew", [])
|
||||
pre_gradle_commands += [
|
||||
_generate_secret_command(secret) for secret in run.get("secrets", [])
|
||||
]
|
||||
|
|
|
@ -1,81 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
from __future__ import print_function, unicode_literals
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
current_dir = os.path.dirname(os.path.realpath(__file__))
|
||||
project_dir = os.path.realpath(os.path.join(current_dir, '..', '..', '..'))
|
||||
sys.path.append(project_dir)
|
||||
|
||||
from automation.taskcluster.decision_task import (
|
||||
pr,
|
||||
push,
|
||||
raptor,
|
||||
nightly_to_production_app,
|
||||
release,
|
||||
release_as_fennec,
|
||||
)
|
||||
from automation.taskcluster.lib.tasks import TaskBuilder
|
||||
|
||||
|
||||
def loader(kind, path, config, params, loaded_tasks):
|
||||
repo_url = params['head_repository']
|
||||
commit = params['head_rev']
|
||||
trust_level = int(params['level'])
|
||||
|
||||
builder = TaskBuilder(
|
||||
task_id=os.environ.get('TASK_ID'),
|
||||
repo_url=repo_url,
|
||||
git_ref=params['head_ref'],
|
||||
short_head_branch=params['head_ref'],
|
||||
commit=commit,
|
||||
owner=params['owner'],
|
||||
source='{}/raw/{}/.taskcluster.yml'.format(repo_url, commit),
|
||||
scheduler_id='mobile-level-{}'.format(trust_level),
|
||||
tasks_priority='highest', # TODO parametrize
|
||||
date_string=params['moz_build_date'],
|
||||
trust_level=trust_level,
|
||||
)
|
||||
|
||||
is_staging = trust_level != 3
|
||||
|
||||
tasks_for = params['tasks_for']
|
||||
if tasks_for == 'github-pull-request':
|
||||
ordered_groups_of_tasks = pr(builder)
|
||||
elif tasks_for == 'github-push':
|
||||
ordered_groups_of_tasks = push(builder)
|
||||
elif tasks_for == 'github-release':
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
beta_semver = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
|
||||
production_semver = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
|
||||
if beta_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'beta', 'geckoBeta', is_staging, version)
|
||||
elif production_semver.match(git_tag):
|
||||
ordered_groups_of_tasks = release(builder, 'production', 'geckoBeta', is_staging, version)
|
||||
else:
|
||||
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
|
||||
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
|
||||
elif tasks_for == 'cron':
|
||||
target_tasks_method = params['target_tasks_method']
|
||||
if target_tasks_method == 'raptor':
|
||||
ordered_groups_of_tasks = raptor(builder, is_staging)
|
||||
elif target_tasks_method == 'nightly':
|
||||
now = datetime.datetime.now().strftime('%y%m%d %H:%M')
|
||||
nightly_version = 'Nightly {}'.format(now)
|
||||
ordered_groups_of_tasks = release(builder, 'nightly', 'geckoNightly', is_staging, nightly_version) \
|
||||
+ nightly_to_production_app(builder, is_staging, nightly_version)
|
||||
ordered_groups_of_tasks += release_as_fennec(builder, is_staging, 'Signed-as-Fennec Nightly {}'.format(now))
|
||||
else:
|
||||
raise NotImplementedError('Unsupported task_name "{}"'.format(params))
|
||||
else:
|
||||
raise NotImplementedError('Unsupported tasks_for "{}"'.format(tasks_for))
|
||||
|
||||
for task in ordered_groups_of_tasks:
|
||||
yield task
|
|
@ -1,27 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the nimbledroid
|
||||
kind. This transform won't be necessary anymore once the full tasgraph migration is
|
||||
done
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import copy
|
||||
import json
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def filter_out_non_nightly_legacy(config, tasks):
|
||||
for dep_task in config.kind_dependencies_tasks:
|
||||
if dep_task.label == 'Build FenixNightlyLegacy task':
|
||||
for task in tasks:
|
||||
yield task
|
|
@ -1,75 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
_CHANNEL_PER_TASK_NAME = {
|
||||
'nightly': 'nightly',
|
||||
'nightly-legacy': 'production',
|
||||
'production': 'production',
|
||||
}
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_name_and_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["dependencies"] = {"signing": dep.label}
|
||||
task.setdefault("attributes", {}).update(dep.attributes.copy())
|
||||
task["name"] = _get_dependent_job_name_without_its_kind(dep)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
def _get_dependent_job_name_without_its_kind(dependent_job):
|
||||
return dependent_job.label[len(dependent_job.kind) + 1:]
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_treeherder_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["treeherder"] = inherit_treeherder_from_dep(task, dep)
|
||||
treeherder_group = dep.task["extra"]["treeherder"]["groupSymbol"]
|
||||
treeherder_symbol = join_symbol(treeherder_group, 'gp')
|
||||
task["treeherder"]["symbol"] = treeherder_symbol
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_worker_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
task_name = task["name"]
|
||||
|
||||
worker_definition = {}
|
||||
worker_definition["upstream-artifacts"] = [{
|
||||
"taskId": {"task-reference": "<signing>"},
|
||||
"taskType": "signing",
|
||||
"paths": dep.attributes["apks"].values(),
|
||||
}]
|
||||
worker_definition["dep"] = config.params["level"] != "3"
|
||||
worker_definition["channel"] = _CHANNEL_PER_TASK_NAME[task_name]
|
||||
|
||||
# Fenix production doesn't follow the rule {product}-{channel}
|
||||
worker_definition["certificate-alias"] = 'fenix' if task_name == 'production' else \
|
||||
"{}-{}".format(task["worker"]["product"], worker_definition["channel"])
|
||||
|
||||
if task_name == 'nightly-legacy':
|
||||
worker_definition["google-play-track"] = 'nightly'
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
|
@ -1,67 +0,0 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
# TODO remove this transform once build tasks are migrated to taskgraph
|
||||
@transforms.add
|
||||
def fetch_old_decision_dependency(config, tasks):
|
||||
for task in tasks:
|
||||
for dep_task in config.kind_dependencies_tasks:
|
||||
expected_signing_dep_label = task['dependencies']['build']
|
||||
if dep_task.label != expected_signing_dep_label:
|
||||
continue
|
||||
|
||||
task['primary-dependency'] = dep_task
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def define_signing_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
attributes = task.setdefault("attributes", {})
|
||||
upstream_attributes = dep.attributes.copy()
|
||||
upstream_attributes.update(attributes)
|
||||
task["attributes"] = upstream_attributes
|
||||
|
||||
task["attributes"]["signed"] = True
|
||||
if "run_on_tasks_for" in task["attributes"]:
|
||||
task["run-on-tasks-for"] = task["attributes"]["run_on_tasks_for"]
|
||||
|
||||
for key in ("worker-type", "worker.signing-type"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
variant=task["attributes"]["build-type"],
|
||||
level=config.params["level"],
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_signing_task(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
task["dependencies"] = {"build": dep.label}
|
||||
task["worker"]["upstream-artifacts"] = [
|
||||
{
|
||||
"taskId": {"task-reference": "<build>"},
|
||||
"taskType": "build",
|
||||
"paths": dep.attributes["apks"].values(),
|
||||
"formats": ["autograph_apk"],
|
||||
}
|
||||
]
|
||||
yield task
|
|
@ -4,22 +4,38 @@
|
|||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.target_tasks import _target_task, standard_filter
|
||||
import os
|
||||
import re
|
||||
|
||||
from taskgraph.target_tasks import _target_task, filter_for_tasks_for
|
||||
|
||||
|
||||
BETA_SEMVER = re.compile(r'^v\d+\.\d+\.\d+-beta\.\d+$')
|
||||
PRODUCTION_SEMVER = re.compile(r'^v\d+\.\d+\.\d+(-rc\.\d+)?$')
|
||||
|
||||
|
||||
# XXX We're overwritting the default target_task while all tasks are ported to taskgraph
|
||||
@_target_task('default')
|
||||
def target_tasks_default(full_task_graph, parameters, graph_config):
|
||||
"""Target the tasks which have indicated they should be run on this project
|
||||
via the `run_on_projects` attributes."""
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if _old_decision_filter(t, parameters)]
|
||||
|
||||
filter = filter_for_tasks_for
|
||||
if parameters["tasks_for"] == 'github-release':
|
||||
# TODO Move GIT_TAG as to a parameter
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
|
||||
def _old_decision_filter(task, parameters):
|
||||
if task.kind == 'old-decision':
|
||||
return True
|
||||
if BETA_SEMVER.match(git_tag):
|
||||
def filter(task, params):
|
||||
return task.attributes.get("release-type", "") == "beta"
|
||||
elif PRODUCTION_SEMVER.match(git_tag):
|
||||
def filter(task, params):
|
||||
return task.attributes.get("release-type", "") == "production"
|
||||
else:
|
||||
raise ValueError('Github tag must be in semver format and prefixed with a "v", '
|
||||
'e.g.: "v1.0.0-beta.0" (beta), "v1.0.0-rc.0" (production) or "v1.0.0" (production)')
|
||||
|
||||
return standard_filter(task, parameters)
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
||||
|
||||
|
||||
@_target_task("nightly")
|
||||
|
@ -27,20 +43,14 @@ def target_tasks_nightly(full_task_graph, parameters, graph_config):
|
|||
"""Select the set of tasks required for a nightly build."""
|
||||
|
||||
def filter(task, parameters):
|
||||
if task.attributes.get("nightly", False):
|
||||
return True
|
||||
|
||||
return _old_decision_filter(task, parameters)
|
||||
return task.attributes.get("nightly", False)
|
||||
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
||||
|
||||
|
||||
@_target_task('raptor')
|
||||
def target_tasks_raptor(full_task_graph, parameters, graph_config):
|
||||
def filter(t, params):
|
||||
if t.kind == 'raptor':
|
||||
return True
|
||||
|
||||
return _old_decision_filter(t, params)
|
||||
def filter(task, params):
|
||||
return task.kind == 'raptor'
|
||||
|
||||
return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
|
||||
|
|
|
@ -0,0 +1,128 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
import datetime
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from fenix_taskgraph.gradle import get_variant
|
||||
from fenix_taskgraph.util import upper_case_first_letter
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_variant_config(config, tasks):
|
||||
for task in tasks:
|
||||
attributes = task.setdefault("attributes", {})
|
||||
if not attributes.get("build-type"):
|
||||
attributes["build-type"] = task["name"]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_shippable_secrets(config, tasks):
|
||||
for task in tasks:
|
||||
secrets = task["run"].setdefault("secrets", [])
|
||||
|
||||
if task.pop("include-shippable-secrets", False) and config.params["level"] == "3":
|
||||
build_type = task["attributes"]["build-type"]
|
||||
secret_index = 'project/mobile/fenix/{}'.format(build_type)
|
||||
secrets.extend([{
|
||||
"key": key,
|
||||
"name": secret_index,
|
||||
"path": target_file,
|
||||
} for key, target_file in (
|
||||
('adjust', '.adjust_token'),
|
||||
('firebase', 'app/src/{}/res/values/firebase.xml'.format(build_type)),
|
||||
('digital_asset_links', '.digital_asset_links_token'),
|
||||
('leanplum', '.leanplum_token'),
|
||||
('sentry_dsn', '.sentry_token'),
|
||||
)])
|
||||
else:
|
||||
task["run"]["pre-gradlew"] = [[
|
||||
"echo", '"{}"'.format(fake_value), ">", target_file
|
||||
] for fake_value, target_file in (
|
||||
("--", ".adjust_token"),
|
||||
("", ".digital_asset_links_token"),
|
||||
("-:-", ".leanplum_token"),
|
||||
("https://fake@sentry.prod.mozaws.net/368", ".sentry_token"),
|
||||
)]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_gradle_command(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"]["gradle-build-type"]
|
||||
geckoview_engine = task["run"]["geckoview-engine"]
|
||||
variant_config = get_variant(gradle_build_type, geckoview_engine)
|
||||
|
||||
task["run"]["gradlew"] = [
|
||||
"clean",
|
||||
"assemble{}".format(upper_case_first_letter(variant_config["name"]))
|
||||
]
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_nightly_version(config, tasks):
|
||||
push_date_string = config.params["moz_build_date"]
|
||||
push_date_time = datetime.datetime.strptime(push_date_string, "%Y%m%d%H%M%S")
|
||||
formated_date_time = 'Nightly {}'.format(push_date_time.strftime('%y%m%d %H:%M'))
|
||||
|
||||
for task in tasks:
|
||||
if task.pop("include-nightly-version", False):
|
||||
task["run"]["gradlew"].append('-PversionName="{}"'.format(formated_date_time))
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_release_version(config, tasks):
|
||||
for task in tasks:
|
||||
if task.pop("include-release-version", False):
|
||||
# TODO Move GIT_TAG to a parameter
|
||||
git_tag = os.environ['GIT_TAG']
|
||||
version = git_tag[1:] # remove prefixed "v"
|
||||
|
||||
task["run"]["gradlew"].append('-PversionName="{}"'.format(version))
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def add_artifacts(config, tasks):
|
||||
for task in tasks:
|
||||
gradle_build_type = task["run"].pop("gradle-build-type")
|
||||
geckoview_engine = task["run"].pop("geckoview-engine")
|
||||
variant_config = get_variant(gradle_build_type, geckoview_engine)
|
||||
artifacts = task.setdefault("worker", {}).setdefault("artifacts", [])
|
||||
task["attributes"]["apks"] = apks = {}
|
||||
|
||||
if "apk-artifact-template" in task:
|
||||
artifact_template = task.pop("apk-artifact-template")
|
||||
for apk in variant_config["apks"]:
|
||||
apk_name = artifact_template["name"].format(
|
||||
geckoview_engine=geckoview_engine, **apk
|
||||
)
|
||||
artifacts.append({
|
||||
"type": artifact_template["type"],
|
||||
"name": apk_name,
|
||||
"path": artifact_template["path"].format(
|
||||
geckoview_engine=geckoview_engine,
|
||||
gradle_build_type=gradle_build_type,
|
||||
**apk
|
||||
),
|
||||
})
|
||||
apks[apk["abi"]] = apk_name
|
||||
|
||||
yield task
|
|
@ -0,0 +1,46 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def resolve_keys(config, tasks):
|
||||
for task in tasks:
|
||||
for key in ("worker.channel", "worker.dep", "worker.google-play-track"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
**{
|
||||
'build-type': task["attributes"]["build-type"],
|
||||
'level': config.params["level"],
|
||||
}
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_worker_definition(config, tasks):
|
||||
for task in tasks:
|
||||
worker_definition = {}
|
||||
worker_definition["certificate-alias"] = "{}-{}".format(
|
||||
task["worker"]["product"], task["worker"]["channel"]
|
||||
)
|
||||
# Fenix production doesn't follow the rule {product}-{channel}
|
||||
if task["attributes"]["build-type"] == "production":
|
||||
worker_definition["certificate-alias"] = "fenix"
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
|
@ -0,0 +1,46 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the jobs defined in the build
|
||||
kind.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.schema import resolve_keyed_by
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def resolve_keys(config, tasks):
|
||||
for task in tasks:
|
||||
for key in ("index", "worker-type", "worker.signing-type"):
|
||||
resolve_keyed_by(
|
||||
task,
|
||||
key,
|
||||
item_name=task["name"],
|
||||
**{
|
||||
'build-type': task["attributes"]["build-type"],
|
||||
'level': config.params["level"],
|
||||
}
|
||||
)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_signing_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
task["attributes"]["signed"] = True
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def set_signing_format(config, tasks):
|
||||
for task in tasks:
|
||||
for upstream_artifact in task["worker"]["upstream-artifacts"]:
|
||||
upstream_artifact["formats"] = ["autograph_apk"]
|
||||
yield task
|
|
@ -0,0 +1,62 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
"""
|
||||
Apply some defaults and minor modifications to the single_dep jobs.
|
||||
"""
|
||||
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
from taskgraph.transforms.base import TransformSequence
|
||||
from taskgraph.util.treeherder import inherit_treeherder_from_dep, join_symbol
|
||||
|
||||
|
||||
transforms = TransformSequence()
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_name_and_attributes(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
task["dependencies"] = {dep.kind: dep.label}
|
||||
copy_of_attributes = dep.attributes.copy()
|
||||
task.setdefault("attributes", copy_of_attributes)
|
||||
# run_on_tasks_for is set as an attribute later in the pipeline
|
||||
task.setdefault("run-on-tasks-for", copy_of_attributes['run_on_tasks_for'])
|
||||
task["name"] = _get_dependent_job_name_without_its_kind(dep)
|
||||
|
||||
yield task
|
||||
|
||||
|
||||
def _get_dependent_job_name_without_its_kind(dependent_job):
|
||||
return dependent_job.label[len(dependent_job.kind) + 1:]
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_upstream_artifacts(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task["primary-dependency"]
|
||||
|
||||
worker_definition = {}
|
||||
worker_definition["upstream-artifacts"] = [{
|
||||
"taskId": {"task-reference": "<{}>".format(dep.kind)},
|
||||
"taskType": dep.kind,
|
||||
"paths": sorted(dep.attributes["apks"].values()),
|
||||
}]
|
||||
|
||||
task["worker"].update(worker_definition)
|
||||
yield task
|
||||
|
||||
|
||||
@transforms.add
|
||||
def build_treeherder_definition(config, tasks):
|
||||
for task in tasks:
|
||||
dep = task.pop("primary-dependency")
|
||||
|
||||
task.setdefault("treeherder", {}).update(inherit_treeherder_from_dep(task, dep))
|
||||
job_group = dep.task["extra"]["treeherder"].get("groupSymbol", "?")
|
||||
job_symbol = task["treeherder"].pop("job-symbol")
|
||||
full_symbol = join_symbol(job_group, job_symbol)
|
||||
task["treeherder"]["symbol"] = full_symbol
|
||||
|
||||
yield task
|
|
@ -1,9 +1,11 @@
|
|||
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
---
|
||||
loader: fenix_taskgraph.loader.old_decision:loader
|
||||
|
||||
transforms: []
|
||||
from __future__ import absolute_import, print_function, unicode_literals
|
||||
|
||||
# XXX Everything is done in the loader until everything is migrated
|
||||
import re
|
||||
|
||||
|
||||
def upper_case_first_letter(string):
|
||||
return string[0].upper() + string[1:]
|
Loading…
Reference in New Issue