From 393829e2aa91b506e366d5aa5433e56d63c55c1f Mon Sep 17 00:00:00 2001 From: Colin Lee Date: Mon, 7 Jan 2019 14:23:30 -0600 Subject: [PATCH] Re-add automation folder; support multiple arch builds --- .idea/misc.xml | 2 +- .idea/modules.xml | 1 + app/build.gradle | 54 +++++++++++++--- automation/docker/Dockerfile | 86 +++++++++++++++++++++++++ automation/gradle/versionCode.gradle | 44 +++++++++++++ automation/taskcluster/get-secret.py | 44 +++++++++++++ automation/taskcluster/sign-builds.py | 89 ++++++++++++++++++++++++++ buildSrc/src/main/java/Dependencies.kt | 27 ++++++++ 8 files changed, 338 insertions(+), 9 deletions(-) create mode 100644 automation/docker/Dockerfile create mode 100644 automation/gradle/versionCode.gradle create mode 100644 automation/taskcluster/get-secret.py create mode 100644 automation/taskcluster/sign-builds.py create mode 100644 buildSrc/src/main/java/Dependencies.kt diff --git a/.idea/misc.xml b/.idea/misc.xml index e0d5b93ff..b0c7b20c8 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -29,7 +29,7 @@ - + diff --git a/.idea/modules.xml b/.idea/modules.xml index a98bf6520..f8131b06d 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -3,6 +3,7 @@ + diff --git a/app/build.gradle b/app/build.gradle index 66b175b90..4793192dc 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -1,8 +1,7 @@ apply plugin: 'com.android.application' - apply plugin: 'kotlin-android' - apply plugin: 'kotlin-android-extensions' +apply from: "$project.rootDir/automation/gradle/versionCode.gradle" android { compileSdkVersion 28 @@ -20,14 +19,53 @@ android { proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' } } + + flavorDimensions "abi" + productFlavors { + // replace the libraries with 64-bit versions when they're ready + arm { dimension "abi" } + x86 { dimension "abi" } + } +} + +android.applicationVariants.all { variant -> + def buildType = variant.buildType.name + + if (buildType == "release" || buildType == "nightly") { + def versionCode = generatedVersionCode + + // The Google Play Store does not allow multiple APKs for the same app that all have the + // same version code. Therefore we need to have different version codes for our ARM and x86 + // builds. + + // Our generated version code now has a length of 9 (See tools/gradle/versionCode.gradle). + // Our x86 builds need a higher version code to avoid installing ARM builds on an x86 device + // with ARM compatibility mode. + + if (variant.flavorName.contains("X86")) { + versionCode = versionCode + 1 + }// else variant.flavorName.contains("Arm")) use generated version code + + variant.outputs.all { output -> + setVersionCodeOverride(versionCode) + } + } + + println("----------------------------------------------") + println("Build type: " + buildType) + println("Flavor: " + variant.flavorName) + println("Version code: " + variant.mergedFlavor.versionCode) } dependencies { implementation fileTree(dir: 'libs', include: ['*.jar']) - implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" - implementation 'com.android.support:appcompat-v7:28.0.0' - implementation 'com.android.support.constraint:constraint-layout:1.1.3' - testImplementation 'junit:junit:4.12' - androidTestImplementation 'com.android.support.test:runner:1.0.2' - androidTestImplementation 'com.android.support.test.espresso:espresso-core:3.0.2' + implementation Deps.kotlin_stdlib + implementation Deps.androidx_appcompat + implementation Deps.androidx_constraintlayout + testImplementation Deps.junit + androidTestImplementation Deps.tools_test_runner + androidTestImplementation Deps.tools_espresso_core + + armImplementation Deps.geckoview_nightly_arm + x86Implementation Deps.geckoview_nightly_x86 } diff --git a/automation/docker/Dockerfile b/automation/docker/Dockerfile new file mode 100644 index 000000000..dff84057b --- /dev/null +++ b/automation/docker/Dockerfile @@ -0,0 +1,86 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +FROM openjdk:8-alpine + +MAINTAINER Colin Lee "colinlee@mozilla.com" + +#---------------------------------------------------------------------------------------------------------------------- +#-- Configuration ----------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +ENV GLIBC_VERSION "2.27-r0" +ENV ANDROID_BUILD_TOOLS "27.0.3" +ENV ANDROID_SDK_VERSION "3859397" +ENV ANDROID_PLATFORM_VERSION "27" +ENV PROJECT_REPOSITORY "https://github.com/mozilla-mobile/fenix.git" + +#---------------------------------------------------------------------------------------------------------------------- +#-- System ------------------------------------------------------------------------------------------------------------ +#---------------------------------------------------------------------------------------------------------------------- + +RUN apk add --no-cache --virtual=.build-dependencies \ + bash \ + ca-certificates \ + curl \ + git \ + python \ + py-pip \ + unzip \ + wget + +RUN pip install --upgrade pip + +RUN pip install \ + taskcluster + +RUN wget https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub -O /etc/apk/keys/sgerrand.rsa.pub \ + && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-${GLIBC_VERSION}.apk -O /tmp/glibc.apk \ + && wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/${GLIBC_VERSION}/glibc-bin-${GLIBC_VERSION}.apk -O /tmp/glibc-bin.apk \ + && apk add --no-cache /tmp/glibc.apk /tmp/glibc-bin.apk \ + && rm -rf /tmp/* \ + && rm -rf /var/cache/apk/* + +#---------------------------------------------------------------------------------------------------------------------- +#-- Android ----------------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +RUN mkdir -p /build/android-sdk +WORKDIR /build + +ENV ANDROID_HOME /build/android-sdk +ENV ANDROID_SDK_HOME /build/android-sdk +ENV PATH ${PATH}:${ANDROID_SDK_HOME}/tools:${ANDROID_SDK_HOME}/tools/bin:${ANDROID_SDK_HOME}/platform-tools:/opt/tools:${ANDROID_SDK_HOME}/build-tools/${ANDROID_BUILD_TOOLS} + +RUN curl -L https://dl.google.com/android/repository/sdk-tools-linux-${ANDROID_SDK_VERSION}.zip > sdk.zip \ + && unzip sdk.zip -d ${ANDROID_SDK_HOME} \ + && rm sdk.zip + +RUN mkdir -p /build/android-sdk/.android/ +RUN touch /build/android-sdk/.android/repositories.cfg + +RUN yes | sdkmanager --licenses + +RUN sdkmanager --verbose "platform-tools" \ + "platforms;android-${ANDROID_PLATFORM_VERSION}" \ + "build-tools;${ANDROID_BUILD_TOOLS}" \ + "extras;android;m2repository" \ + "extras;google;m2repository" + +#---------------------------------------------------------------------------------------------------------------------- +#-- Project ----------------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +RUN git clone $PROJECT_REPOSITORY + +WORKDIR /build/fenix + +RUN ./gradlew --no-daemon assemble test lint detektCheck ktlint + +#---------------------------------------------------------------------------------------------------------------------- +#-- Addendum ---------------------------------------------------------------------------------------------------------- +#---------------------------------------------------------------------------------------------------------------------- + +# Alphine Linux creates a bash profile that overrides our PATH again. Let's fix that. +RUN echo "export PATH=$PATH" >> /etc/profile diff --git a/automation/gradle/versionCode.gradle b/automation/gradle/versionCode.gradle new file mode 100644 index 000000000..8ee994a60 --- /dev/null +++ b/automation/gradle/versionCode.gradle @@ -0,0 +1,44 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import java.text.SimpleDateFormat + +// This gradle scripts generates a "unique" version code for our release versions. +// +// The result of the version code depends on the timezone. We assume that this script will only be used +// for release versions and running on our build servers with a fixed timezone. +// +// The version code is composed like: yDDDHHmm +// * y = Double digit year, with 18 subtracted: 2018 -> 18 -> 0 +// * DDD = Day of the year, pad with zeros if needed: September 6th -> 249 +// * HH = Hour in day (00-23) +// * mm = Minute in hour +// +// For September 6th, 2018, 9:41 am this will generate the versionCode: 2490941 (0-249-09-41). +// +// Note that we only use this generated version code for builds we want to distribute. For local +// debug builds we use a fixed versionCode to not mess with the caching mechanism of the build +// system. + +ext { + def today = new Date() + + // We use the current year (double digit) and subtract 18. We first released Reference Browser in + // 2018 so this value will start counting at 0 and increment by one every year. + def year = String.valueOf((new SimpleDateFormat("yy").format(today) as int) - 18) + + // We use the day in the Year (e.g. 248) as opposed to month + day (0510) because it's one digit shorter. + // If needed we pad with zeros (e.g. 25 -> 025) + def day = String.format("%03d", (new SimpleDateFormat("D").format(today) as int)) + + // We append the hour in day (24h) and minute in hour (7:26 pm -> 1926). We do not append + // seconds. This assumes that we do not need to build multiple release(!) builds the same + // minute. + def time = new SimpleDateFormat("HHmm").format(today) + + generatedVersionCode = (year + day + time) as int + + println("Generated versionCode: $generatedVersionCode") + println() +} diff --git a/automation/taskcluster/get-secret.py b/automation/taskcluster/get-secret.py new file mode 100644 index 000000000..1e8ca6314 --- /dev/null +++ b/automation/taskcluster/get-secret.py @@ -0,0 +1,44 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import base64 +import os +import taskcluster + + +def write_secret_to_file(path, data, key, base64decode=False): + path = os.path.join(os.path.dirname(__file__), '../../' + path) + with open(path, 'w') as f: + value = data['secret'][key] + if base64decode: + value = base64.b64decode(value) + f.write(value) + + +def fetch_secret_from_taskcluster(name): + secrets = taskcluster.Secrets({'baseUrl': 'http://taskcluster/secrets/v1'}) + return secrets.get(name) + + +def main(): + parser = argparse.ArgumentParser( + description='Fetch a taskcluster secret value and save it to a file.') + + parser.add_argument('-s', dest="secret", action="store", help="name of the secret") + parser.add_argument('-k', dest='key', action="store", help='key of the secret') + parser.add_argument('-f', dest="path", action="store", help='file to save secret to') + parser.add_argument( + '--decode', dest="decode", action="store_true", default=False, + help='base64 decode secret before saving to file' + ) + + result = parser.parse_args() + + secret = fetch_secret_from_taskcluster(result.secret) + write_secret_to_file(result.path, secret, result.key, result.decode) + + +if __name__ == "__main__": + main() diff --git a/automation/taskcluster/sign-builds.py b/automation/taskcluster/sign-builds.py new file mode 100644 index 000000000..00f6d5484 --- /dev/null +++ b/automation/taskcluster/sign-builds.py @@ -0,0 +1,89 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +import argparse +import fnmatch +import os +import subprocess + + +def collect_apks(path, pattern): + matches = [] + for root, dirnames, filenames in os.walk(path): + for filename in fnmatch.filter(filenames, pattern): + matches.append(os.path.join(root, filename)) + return matches + + +def zipalign(path): + unsigned_apks = collect_apks(path, '*-unsigned.apk') + print("Found {apk_count} APK(s) to zipalign in {path}".format(apk_count=len(unsigned_apks), path=path)) + for apk in unsigned_apks: + print("Zipaligning", apk) + split = os.path.splitext(apk) + print(subprocess.check_output(["zipalign", "-f", "-v", "-p", "4", apk, split[0] + "-aligned" + split[1]])) + + +def sign(path, store, store_token, key_alias, key_token): + unsigned_apks = collect_apks(path, '*-aligned.apk') + print("Found {apk_count} APK(s) to sign in {path}".format(apk_count=len(unsigned_apks), path=path)) + + for apk in unsigned_apks: + print("Signing", apk) + print(subprocess.check_output([ + "apksigner", "sign", + "--ks", store, + "--ks-key-alias", key_alias, + "--ks-pass", "file:%s" % store_token, + "--key-pass", "file:%s" % key_token, + "-v", + "--out", apk.replace('unsigned', 'signed'), apk])) + + +def archive_result(path, archive): + if not os.path.exists(archive): + os.makedirs(archive) + + signed_apks = collect_apks(path, '*-signed-*.apk') + print("Found {apk_count} APK(s) to archive in {path}".format(apk_count=len(signed_apks), path=path)) + + for apk in signed_apks: + print("Verifying", apk) + print(subprocess.check_output(['apksigner', 'verify', apk])) + + destination = archive + "/" + os.path.basename(apk) + print("Archiving", apk) + print(" `->", destination) + os.rename(apk, destination) + + +def main(): + parser = argparse.ArgumentParser( + description='Zipaligns, signs and archives APKs') + parser.add_argument('--path', dest="path", action="store", help='Root path to search for APK files') + parser.add_argument('--zipalign', dest="zipalign", action="store_true", default=False, + help='Zipaligns APKs before signing') + parser.add_argument('--archive', metavar="PATH", dest="archive", action="store", default=False, + help='Path to save sign APKs to') + + parser.add_argument('--store', metavar="PATH", dest="store", action="store", help='Path to keystore') + parser.add_argument('--store-token', metavar="PATH", dest="store_token", action="store", + help='Path to keystore password file') + parser.add_argument('--key-alias', metavar="ALIAS", dest="key_alias", action="store", help='Key alias') + parser.add_argument('--key-token', metavar="PATH", dest="key_token", action="store", + help='Path to key password file') + + result = parser.parse_args() + + if result.zipalign: + zipalign(result.path) + + sign(result.path, result.store, result.store_token, result.key_alias, result.key_token) + + if result.archive: + archive_result(result.path, result.archive) + + +if __name__ == "__main__": + main() diff --git a/buildSrc/src/main/java/Dependencies.kt b/buildSrc/src/main/java/Dependencies.kt new file mode 100644 index 000000000..0b89e7786 --- /dev/null +++ b/buildSrc/src/main/java/Dependencies.kt @@ -0,0 +1,27 @@ +object Versions { + const val kotlin = "1.3.10" + const val coroutines = "1.0.1" + const val geckoNightly = "66.0.20181217093726" + + const val androidx_appcompat = "1.0.2" + const val androidx_constraintlayout = "1.1.3" + + const val junit = "4.12" + const val test_tools = "1.0.2" + const val espresso_core = "2.2.2" +} + +object Deps { + const val kotlin_stdlib = "org.jetbrains.kotlin:kotlin-stdlib-jdk7:${Versions.kotlin}" + const val kotlin_coroutines = "org.jetbrains.kotlinx:kotlinx-coroutines-android:${Versions.coroutines}" + + const val geckoview_nightly_arm = "org.mozilla.geckoview:geckoview-nightly-armeabi-v7a:${Versions.geckoNightly}" + const val geckoview_nightly_x86 = "org.mozilla.geckoview:geckoview-nightly-x86:${Versions.geckoNightly}" + + const val androidx_appcompat = "androidx.appcompat:appcompat:${Versions.androidx_appcompat}" + const val androidx_constraintlayout = "androidx.constraintlayout:constraintlayout:${Versions.androidx_constraintlayout}" + + const val junit = "junit:junit:${Versions.junit}" + const val tools_test_runner = "com.android.support.test:runner:${Versions.test_tools}" + const val tools_espresso_core = "com.android.support.test.espresso:espresso-core:${Versions.espresso_core}" +} \ No newline at end of file