diff --git a/automation/taskcluster/l10n/filter-release-translations.py b/automation/taskcluster/l10n/filter-release-translations.py new file mode 100644 index 000000000..e1c3cdb8b --- /dev/null +++ b/automation/taskcluster/l10n/filter-release-translations.py @@ -0,0 +1,50 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +""" +This script takes the list of release locales defined in locales.py +and removes all non-release translations from the application's +resource folder. +This script is run before building a release version so that +those builds only contain locales we actually want to ship. +""" + +import os +import re +import shutil + +from locales import RELEASE_LOCALES + +LANGUAGE_REGEX = re.compile('^[a-z]{2,3}$') +LANGUAGE_REGION_REGEX = re.compile('^([a-z]{2})-r([A-Z]{2})$') + +# Get all resource directories that start with "values-" (and remove the "values-" prefix) +resources_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'app', 'src', 'main', 'res') +locale_dirs = [localeDir.replace('values-', '') for localeDir in os.listdir(resources_dir) + if os.path.isdir(os.path.join(resources_dir, localeDir)) + and localeDir.startswith('values-')] + +# Remove everything that doesn't look like it's a resource folder for a specific language (e.g. sw600dp) +locale_dirs = filter(lambda x: LANGUAGE_REGEX.match(x) or LANGUAGE_REGION_REGEX.match(x), locale_dirs) + +# Android prefixes regions with an "r" (de-rDE). Remove the prefix. +locale_dirs = [item.replace('-r','-') for item in locale_dirs] + +# Now determine the list of locales that are not in our release list +locales_to_remove = list(set(locale_dirs) - set(RELEASE_LOCALES)) + +print "RELEASE LOCALES:", ", ".join(RELEASE_LOCALES) +print "APP LOCALES:", ", ".join(locale_dirs) +print "REMOVE:", ", ".join(locales_to_remove) if len(locales_to_remove) > 0 else "-Nothing-" + +# Remove unneeded resource folders +for locale in locales_to_remove: + # Build resource folder names from locale: de -> values-de, de-DE -> vlaues-de-rDE + parts = locale.split("-") + folder = "values-" + (parts[0] if len(parts) == 1 else parts[0] + "-r" + parts[1]) + path = os.path.join(resources_dir, folder) + + print "* Removing: ", path + # This modifies source, which could cause problems if we ever start caching source + shutil.rmtree(path) \ No newline at end of file diff --git a/automation/taskcluster/l10n/locales.py b/automation/taskcluster/l10n/locales.py new file mode 100644 index 000000000..5e2e80fe1 --- /dev/null +++ b/automation/taskcluster/l10n/locales.py @@ -0,0 +1,59 @@ +#!/usr/bin/python + +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +from __future__ import absolute_import, print_function, unicode_literals + +import re + +OPEN_LOCALES = "locales = [" +CLOSE_LOCALES = "]" + +def trim_to_locale(str): + match = re.search('\s*"([a-z]+-?[A-Z]*)",\s*', str) + if not match: + raise Exception("Failed parsing locale found in l10n.toml: " + str) + return match.group(1) + + +# This file is a dumb parser that converts values from '/l10n.toml' to be easily consumed from +# Python. +# +# 'l10n.toml' has a very simple structure, and it is reasonable to believe that this (very basic) +# algorithm will continue to work as it is changed. +# +# Alternatives to custom parsing that were considered: +# - Using standard library module --- none exists to parse TOML +# - Importing a TOML parsing module --- introduces an additional build step, complexity, and +# security risk +# - Vendoring a TOML module --- large amount of code given the use case. Introduces a security +# risk +def get_release_locales(): + with open(r"l10n.toml") as f: + file = f.read().splitlines() + + locales_opened = False + locales_closed = False + + found_locales = [] + + for line in file: + if line == OPEN_LOCALES: + locales_opened = True + elif line == CLOSE_LOCALES: + locales_closed = True + break + elif locales_opened: + found_locales.append(trim_to_locale(line)) + + if locales_opened == False: + raise Exception("Could not find `locales` open in l10n.toml") + if locales_closed == False: + raise Exception("Could not find `locales` close in l10n.toml") + + return found_locales + + +RELEASE_LOCALES = get_release_locales() diff --git a/taskcluster/ci/build/kind.yml b/taskcluster/ci/build/kind.yml index 5cc328eed..c8c28f27c 100644 --- a/taskcluster/ci/build/kind.yml +++ b/taskcluster/ci/build/kind.yml @@ -100,6 +100,7 @@ jobs: release-type: beta include-release-version: true include-shippable-secrets: true + filter-incomplete-translations: true run: geckoview-engine: geckoBeta gradle-build-type: fenixBeta @@ -112,6 +113,7 @@ jobs: release-type: production include-release-version: true include-shippable-secrets: true + filter-incomplete-translations: true run: geckoview-engine: geckoBeta gradle-build-type: fenixProduction @@ -134,9 +136,11 @@ jobs: attributes: # TODO replace `nightly: true` by `release-type: beta` once the data migration # testing is over + # TODO when making the above change, uncomment `filter-incomplete-translations: true` nightly: true include-release-version: true include-shippable-secrets: true +# filter-incomplete-translations: true run: geckoview-engine: geckoBeta gradle-build-type: fennecBeta @@ -147,9 +151,11 @@ jobs: attributes: # TODO replace `nightly: true` by `release-type: production` once the data migration # testing is over + # TODO when making the above change, uncomment `filter-incomplete-translations: true` nightly: true include-release-version: true include-shippable-secrets: true +# filter-incomplete-translations: true run: geckoview-engine: geckoBeta gradle-build-type: fennecProduction diff --git a/taskcluster/fenix_taskgraph/transforms/build.py b/taskcluster/fenix_taskgraph/transforms/build.py index bd372e80b..1973761be 100644 --- a/taskcluster/fenix_taskgraph/transforms/build.py +++ b/taskcluster/fenix_taskgraph/transforms/build.py @@ -125,3 +125,13 @@ def add_artifacts(config, tasks): apks[apk["abi"]] = apk_name yield task + + +@transforms.add +def filter_incomplete_translation(config, tasks): + for task in tasks: + if task.pop("filter-incomplete-translations", False): + # filter-release-translations modifies source, which could cause problems if we ever start caching source + pre_gradlew = task["run"].setdefault("pre-gradlew", []) + pre_gradlew.append(["python", "automation/taskcluster/l10n/filter-release-translations.py"]) + yield task