3088 filter incomplete locales on release builds (#7581)
* For #3088: add translation pruning Py scripts from Focus The only change made was updating "Focus/Klar" to "Fenix" * For 3088: call translation pruning scripts during release builds * For 3088: fix filter-release-translations.py Moving it into Fenix introduced one additional layer of nesting, which was not accounted for in a relative path. 🤦 * For 3088: comment about the risks of modifying source * For 3088: updated locales.py to read from l10n.toml Also removed some previously unused properties from locales.py * For 3088: added error handling to script in locales.py * For 3088: updated locale parser per review commentsmaster
parent
904098387a
commit
93122b6128
|
@ -0,0 +1,50 @@
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
"""
|
||||||
|
This script takes the list of release locales defined in locales.py
|
||||||
|
and removes all non-release translations from the application's
|
||||||
|
resource folder.
|
||||||
|
This script is run before building a release version so that
|
||||||
|
those builds only contain locales we actually want to ship.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
|
||||||
|
from locales import RELEASE_LOCALES
|
||||||
|
|
||||||
|
LANGUAGE_REGEX = re.compile('^[a-z]{2,3}$')
|
||||||
|
LANGUAGE_REGION_REGEX = re.compile('^([a-z]{2})-r([A-Z]{2})$')
|
||||||
|
|
||||||
|
# Get all resource directories that start with "values-" (and remove the "values-" prefix)
|
||||||
|
resources_dir = os.path.join(os.path.dirname(__file__), '..', '..', '..', 'app', 'src', 'main', 'res')
|
||||||
|
locale_dirs = [localeDir.replace('values-', '') for localeDir in os.listdir(resources_dir)
|
||||||
|
if os.path.isdir(os.path.join(resources_dir, localeDir))
|
||||||
|
and localeDir.startswith('values-')]
|
||||||
|
|
||||||
|
# Remove everything that doesn't look like it's a resource folder for a specific language (e.g. sw600dp)
|
||||||
|
locale_dirs = filter(lambda x: LANGUAGE_REGEX.match(x) or LANGUAGE_REGION_REGEX.match(x), locale_dirs)
|
||||||
|
|
||||||
|
# Android prefixes regions with an "r" (de-rDE). Remove the prefix.
|
||||||
|
locale_dirs = [item.replace('-r','-') for item in locale_dirs]
|
||||||
|
|
||||||
|
# Now determine the list of locales that are not in our release list
|
||||||
|
locales_to_remove = list(set(locale_dirs) - set(RELEASE_LOCALES))
|
||||||
|
|
||||||
|
print "RELEASE LOCALES:", ", ".join(RELEASE_LOCALES)
|
||||||
|
print "APP LOCALES:", ", ".join(locale_dirs)
|
||||||
|
print "REMOVE:", ", ".join(locales_to_remove) if len(locales_to_remove) > 0 else "-Nothing-"
|
||||||
|
|
||||||
|
# Remove unneeded resource folders
|
||||||
|
for locale in locales_to_remove:
|
||||||
|
# Build resource folder names from locale: de -> values-de, de-DE -> vlaues-de-rDE
|
||||||
|
parts = locale.split("-")
|
||||||
|
folder = "values-" + (parts[0] if len(parts) == 1 else parts[0] + "-r" + parts[1])
|
||||||
|
path = os.path.join(resources_dir, folder)
|
||||||
|
|
||||||
|
print "* Removing: ", path
|
||||||
|
# This modifies source, which could cause problems if we ever start caching source
|
||||||
|
shutil.rmtree(path)
|
|
@ -0,0 +1,59 @@
|
||||||
|
#!/usr/bin/python
|
||||||
|
|
||||||
|
# This Source Code Form is subject to the terms of the Mozilla Public
|
||||||
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
||||||
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||||
|
|
||||||
|
from __future__ import absolute_import, print_function, unicode_literals
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
OPEN_LOCALES = "locales = ["
|
||||||
|
CLOSE_LOCALES = "]"
|
||||||
|
|
||||||
|
def trim_to_locale(str):
|
||||||
|
match = re.search('\s*"([a-z]+-?[A-Z]*)",\s*', str)
|
||||||
|
if not match:
|
||||||
|
raise Exception("Failed parsing locale found in l10n.toml: " + str)
|
||||||
|
return match.group(1)
|
||||||
|
|
||||||
|
|
||||||
|
# This file is a dumb parser that converts values from '/l10n.toml' to be easily consumed from
|
||||||
|
# Python.
|
||||||
|
#
|
||||||
|
# 'l10n.toml' has a very simple structure, and it is reasonable to believe that this (very basic)
|
||||||
|
# algorithm will continue to work as it is changed.
|
||||||
|
#
|
||||||
|
# Alternatives to custom parsing that were considered:
|
||||||
|
# - Using standard library module --- none exists to parse TOML
|
||||||
|
# - Importing a TOML parsing module --- introduces an additional build step, complexity, and
|
||||||
|
# security risk
|
||||||
|
# - Vendoring a TOML module --- large amount of code given the use case. Introduces a security
|
||||||
|
# risk
|
||||||
|
def get_release_locales():
|
||||||
|
with open(r"l10n.toml") as f:
|
||||||
|
file = f.read().splitlines()
|
||||||
|
|
||||||
|
locales_opened = False
|
||||||
|
locales_closed = False
|
||||||
|
|
||||||
|
found_locales = []
|
||||||
|
|
||||||
|
for line in file:
|
||||||
|
if line == OPEN_LOCALES:
|
||||||
|
locales_opened = True
|
||||||
|
elif line == CLOSE_LOCALES:
|
||||||
|
locales_closed = True
|
||||||
|
break
|
||||||
|
elif locales_opened:
|
||||||
|
found_locales.append(trim_to_locale(line))
|
||||||
|
|
||||||
|
if locales_opened == False:
|
||||||
|
raise Exception("Could not find `locales` open in l10n.toml")
|
||||||
|
if locales_closed == False:
|
||||||
|
raise Exception("Could not find `locales` close in l10n.toml")
|
||||||
|
|
||||||
|
return found_locales
|
||||||
|
|
||||||
|
|
||||||
|
RELEASE_LOCALES = get_release_locales()
|
|
@ -100,6 +100,7 @@ jobs:
|
||||||
release-type: beta
|
release-type: beta
|
||||||
include-release-version: true
|
include-release-version: true
|
||||||
include-shippable-secrets: true
|
include-shippable-secrets: true
|
||||||
|
filter-incomplete-translations: true
|
||||||
run:
|
run:
|
||||||
geckoview-engine: geckoBeta
|
geckoview-engine: geckoBeta
|
||||||
gradle-build-type: fenixBeta
|
gradle-build-type: fenixBeta
|
||||||
|
@ -112,6 +113,7 @@ jobs:
|
||||||
release-type: production
|
release-type: production
|
||||||
include-release-version: true
|
include-release-version: true
|
||||||
include-shippable-secrets: true
|
include-shippable-secrets: true
|
||||||
|
filter-incomplete-translations: true
|
||||||
run:
|
run:
|
||||||
geckoview-engine: geckoBeta
|
geckoview-engine: geckoBeta
|
||||||
gradle-build-type: fenixProduction
|
gradle-build-type: fenixProduction
|
||||||
|
@ -137,6 +139,7 @@ jobs:
|
||||||
nightly: true
|
nightly: true
|
||||||
include-release-version: true
|
include-release-version: true
|
||||||
include-shippable-secrets: true
|
include-shippable-secrets: true
|
||||||
|
filter-incomplete-translations: true
|
||||||
run:
|
run:
|
||||||
geckoview-engine: geckoBeta
|
geckoview-engine: geckoBeta
|
||||||
gradle-build-type: fennecBeta
|
gradle-build-type: fennecBeta
|
||||||
|
@ -150,6 +153,7 @@ jobs:
|
||||||
nightly: true
|
nightly: true
|
||||||
include-release-version: true
|
include-release-version: true
|
||||||
include-shippable-secrets: true
|
include-shippable-secrets: true
|
||||||
|
filter-incomplete-translations: true
|
||||||
run:
|
run:
|
||||||
geckoview-engine: geckoBeta
|
geckoview-engine: geckoBeta
|
||||||
gradle-build-type: fennecProduction
|
gradle-build-type: fennecProduction
|
||||||
|
|
|
@ -125,3 +125,12 @@ def add_artifacts(config, tasks):
|
||||||
apks[apk["abi"]] = apk_name
|
apks[apk["abi"]] = apk_name
|
||||||
|
|
||||||
yield task
|
yield task
|
||||||
|
|
||||||
|
|
||||||
|
@transforms.add
|
||||||
|
def filter_incomplete_translation(config, tasks):
|
||||||
|
for task in tasks:
|
||||||
|
if task.pop("filter-incomplete-translations", False):
|
||||||
|
# filter-release-translations modifies source, which could cause problems if we ever start caching source
|
||||||
|
task["run"]["pre-gradlew"].append(["python", "automation/taskcluster/l10n/filter-release-translations.py"])
|
||||||
|
yield task
|
||||||
|
|
Loading…
Reference in New Issue