From 0d824311956b2cde98ae44cb478bb69cabf45abb Mon Sep 17 00:00:00 2001 From: Alessio Placitelli Date: Fri, 3 May 2019 16:40:22 +0200 Subject: [PATCH] Generate the `activation` ping and send it with Glean (#1707) This fetches the Google Advertising ID, salts it and then applies hashing before sending a ping with it, at startup. Hashing and salting are used in order to prevent ourselves to correlate advertising IDs from the same user running different products we own off a single device. We will never send the client_id and the Google Advertising ID in the same ping. --- app/build.gradle | 2 + app/metrics.yaml | 36 +++- app/pings.yaml | 13 ++ .../components/metrics/ActivationPing.kt | 187 ++++++++++++++++++ .../components/metrics/GleanMetricsService.kt | 7 +- .../components/metrics/ActivationPingTest.kt | 127 ++++++++++++ buildSrc/src/main/java/Dependencies.kt | 4 + docs/activation.md | 36 ++++ docs/telemetry.md | 4 + 9 files changed, 413 insertions(+), 3 deletions(-) create mode 100644 app/pings.yaml create mode 100644 app/src/main/java/org/mozilla/fenix/components/metrics/ActivationPing.kt create mode 100644 app/src/test/java/org/mozilla/fenix/components/metrics/ActivationPingTest.kt create mode 100644 docs/activation.md diff --git a/app/build.gradle b/app/build.gradle index 8b74b2bda..3cd2c4848 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -321,6 +321,8 @@ dependencies { implementation Deps.adjust implementation Deps.installreferrer // Required by Adjust + implementation Deps.google_ads_id // Required for the Google Advertising ID + // androidTestImplementation Deps.tools_test_runner // androidTestImplementation Deps.tools_espresso_core diff --git a/app/metrics.yaml b/app/metrics.yaml index d79fcbf23..17030d194 100644 --- a/app/metrics.yaml +++ b/app/metrics.yaml @@ -599,4 +599,38 @@ preferences: - https://github.com/mozilla-mobile/fenix/pull/1896 notification_emails: - fenix-core@mozilla.com - expires: "2020-03-01" \ No newline at end of file + expires: "2020-03-01" + +activation: + identifier: + type: string + lifetime: ping + description: > + An hashed and salted version of the Google Advertising ID from the device. + This will never be sent in a ping that also contains the client_id. + send_in_pings: + - activation + bugs: + - 1538011 + - 1501822 + data_reviews: + - https://github.com/mozilla-mobile/fenix/pull/1707#issuecomment-486972209 + notification_emails: + - fenix-core@mozilla.com + expires: "2019-10-01" + + activation_id: + type: uuid + lifetime: user + description: > + An alternate identifier, not correlated with the client_id, generated once + and only sent with the activation ping. + send_in_pings: + - activation + bugs: + - 1538011 + data_reviews: + - https://github.com/mozilla-mobile/fenix/pull/1707#issuecomment-486972209 + notification_emails: + - fenix-core@mozilla.com + expires: "2019-10-01" \ No newline at end of file diff --git a/app/pings.yaml b/app/pings.yaml new file mode 100644 index 000000000..13c63c668 --- /dev/null +++ b/app/pings.yaml @@ -0,0 +1,13 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +$schema: moz://mozilla.org/schemas/glean/pings/1-0-0 + +activation: + description: > + This ping is intended to provide a measure of the activation of mobile products. + It's generated when Fenix starts, right after Glean is initialized. It doesn't + include the client_id, since it might be reporting an hashed version of the + Google Advertising ID. + include_client_id: false diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/ActivationPing.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/ActivationPing.kt new file mode 100644 index 000000000..c34d581d1 --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/ActivationPing.kt @@ -0,0 +1,187 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.components.metrics + +import android.content.Context +import android.content.SharedPreferences +import android.util.Base64 +import androidx.annotation.VisibleForTesting +import mozilla.components.support.base.log.logger.Logger +import org.mozilla.fenix.GleanMetrics.Activation +import org.mozilla.fenix.GleanMetrics.Pings +import com.google.android.gms.ads.identifier.AdvertisingIdClient +import com.google.android.gms.common.GooglePlayServicesNotAvailableException +import com.google.android.gms.common.GooglePlayServicesRepairableException +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.withContext +import kotlinx.coroutines.launch +import java.io.IOException +import java.security.NoSuchAlgorithmException +import java.security.spec.InvalidKeySpecException +import javax.crypto.SecretKeyFactory +import javax.crypto.spec.PBEKeySpec + +class ActivationPing(private val context: Context) { + companion object { + // The number of iterations to compute the hash. RFC 2898 suggests + // a minimum of 1000 iterations. + const val PBKDF2_ITERATIONS = 1000 + const val PBKDF2_KEY_LEN_BITS = 256 + } + + private val prefs: SharedPreferences by lazy { + context.getSharedPreferences( + "${this.javaClass.canonicalName}.prefs", Context.MODE_PRIVATE) + } + + /** + * Checks whether or not the activation ping was already + * triggered by the application. + * + * Note that this only tells us that Fenix triggered the + * ping and then delegated the transmission to Glean. We + * have no way to tell if it was actually sent or not. + * + * @return true if it was already triggered, false otherwise. + */ + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal fun wasAlreadyTriggered(): Boolean { + return prefs.getBoolean("ping_sent", false) + } + + /** + * Marks the "activation" ping as triggered by the application. + * This ensures the ping is not triggered again at the next app + * start. + */ + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal fun markAsTriggered() { + prefs.edit().putBoolean("ping_sent", true).apply() + } + + /** + * Query the Google Advertising API to get the Google Advertising ID. + * + * This is meant to be used off the main thread. The API will throw an + * exception and we will print a log message otherwise. + * + * @return a String containing the Google Advertising ID or null. + */ + @Suppress("TooGenericExceptionCaught") + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal fun getAdvertisingID(): String? { + return try { + AdvertisingIdClient.getAdvertisingIdInfo(context).id + } catch (e: GooglePlayServicesNotAvailableException) { + Logger.debug("ActivationPing - Google Play not installed on the device") + null + } catch (e: GooglePlayServicesRepairableException) { + Logger.debug("ActivationPing - recoverable error connecting to Google Play Services") + null + } catch (e: IllegalStateException) { + // This is unlikely to happen, as this should be running off the main thread. + Logger.debug("ActivationPing - AdvertisingIdClient must be called off the main thread") + null + } catch (e: IOException) { + Logger.debug("ActivationPing - unable to connect to Google Play Services") + null + } catch (e: NullPointerException) { + Logger.debug("ActivationPing - no Google Advertising ID available") + null + } + } + + /** + * Get the salt to use for hashing. This is a convenience + * function to help with unit tests. + */ + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal fun getHashingSalt(): String = "org.mozilla.fenix-salt" + + /** + * Produces an hashed version of the Google Advertising ID. + * We want users using more than one of our products to report a different + * ID in each of them. This function runs off the main thread and is CPU-bound. + * + * @return an hashed and salted Google Advertising ID or null if it was not possible + * to get the Google Advertising ID. + */ + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal suspend fun getHashedIdentifier(): String? = withContext(Dispatchers.Default) { + getAdvertisingID()?.let { unhashedID -> + // Add some salt to the ID, before hashing. For this specific use-case, it's ok + // to use the same salt value for all the hashes. We want hashes to be stable + // within a single product, but we don't want hashes to be the same across different + // products (e.g. Fennec vs Fenix). + val salt = getHashingSalt() + + // Apply hashing. + try { + // Note that we intentionally want to use slow hashing functions here in order + // to increase the cost of potentially repeatedly guess the original unhashed + // identifier. + val keySpec = PBEKeySpec( + unhashedID.toCharArray(), + salt.toByteArray(), + PBKDF2_ITERATIONS, + PBKDF2_KEY_LEN_BITS) + + val keyFactory = SecretKeyFactory.getInstance("PBKDF2WithHmacSHA1") + val hashedBytes = keyFactory.generateSecret(keySpec).encoded + Base64.encodeToString(hashedBytes, Base64.NO_WRAP) + } catch (e: java.lang.NullPointerException) { + Logger.error("ActivationPing - missing or wrong salt parameter") + null + } catch (e: IllegalArgumentException) { + Logger.error("ActivationPing - wrong parameter", e) + null + } catch (e: NoSuchAlgorithmException) { + Logger.error("ActivationPing - algorithm not available") + null + } catch (e: InvalidKeySpecException) { + Logger.error("ActivationPing - invalid key spec") + null + } + } + } + + /** + * Fills the metrics and triggers the 'activation' ping. + * This is a separate function to simplify unit-testing. + */ + @VisibleForTesting(otherwise = VisibleForTesting.PRIVATE) + internal fun triggerPing() { + // Generate the activation_id. + Activation.activationId.generateAndSet() + + CoroutineScope(Dispatchers.Default).launch { + val hashedId = getHashedIdentifier() + if (hashedId != null) { + Logger.info("ActivationPing - generating ping with the hashed id") + // We have a valid, hashed Google Advertising ID. + Activation.identifier.set(hashedId) + } + + Logger.info("ActivationPing - generating ping (has `identifier`: ${hashedId != null})") + Pings.activation.send() + markAsTriggered() + } + } + + /** + * Trigger sending the `activation` ping if it wasn't sent already. + * Then, mark it so that it doesn't get triggered next time Fenix + * starts. + */ + fun checkAndSend() { + if (wasAlreadyTriggered()) { + Logger.debug("ActivationPing - already generated") + return + } + + triggerPing() + } +} diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt index f58534033..bd0625cf4 100644 --- a/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt @@ -171,12 +171,15 @@ class GleanMetricsService(private val context: Context) : MetricsService { */ private lateinit var starter: Job + private val activationPing = ActivationPing(context) + override fun start() { if (initialized) return initialized = true starter = CoroutineScope(Dispatchers.Default).launch { + Glean.setUploadEnabled(true) Glean.initialize(context) Metrics.apply { @@ -195,9 +198,9 @@ class GleanMetricsService(private val context: Context) : MetricsService { code.set(defaultEngine.identifier) name.set(defaultEngine.name) submissionUrl.set(defaultEngine.buildSearchUrl("")) - - Glean.setUploadEnabled(true) } + + activationPing.checkAndSend() } } diff --git a/app/src/test/java/org/mozilla/fenix/components/metrics/ActivationPingTest.kt b/app/src/test/java/org/mozilla/fenix/components/metrics/ActivationPingTest.kt new file mode 100644 index 000000000..8c37c3187 --- /dev/null +++ b/app/src/test/java/org/mozilla/fenix/components/metrics/ActivationPingTest.kt @@ -0,0 +1,127 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.components.metrics + +import android.util.Base64 +import com.google.android.gms.ads.identifier.AdvertisingIdClient +import com.google.android.gms.common.GooglePlayServicesNotAvailableException +import com.google.android.gms.common.GooglePlayServicesRepairableException +import io.mockk.Runs +import io.mockk.every +import io.mockk.just +import io.mockk.mockk +import io.mockk.mockkStatic +import io.mockk.slot +import io.mockk.spyk +import io.mockk.verify +import org.junit.jupiter.api.Test + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertNull +import org.mockito.ArgumentMatchers.any +import org.mockito.ArgumentMatchers.anyString +import java.io.IOException + +internal class ActivationPingTest { + @Test + fun `getAdvertisingID() returns null if the API throws`() { + mockkStatic(AdvertisingIdClient::class) + + val exceptions = listOf( + GooglePlayServicesNotAvailableException(1), + GooglePlayServicesRepairableException(0, anyString(), any()), + IllegalStateException(), + IOException() + ) + + val ap = ActivationPing(mockk()) + exceptions.forEach { + every { + AdvertisingIdClient.getAdvertisingIdInfo(any()) + } throws it + + assertNull(ap.getAdvertisingID()) + } + } + + @Test + fun `getAdvertisingID() returns null if the API returns null info`() { + mockkStatic(AdvertisingIdClient::class) + every { AdvertisingIdClient.getAdvertisingIdInfo(any()) } returns null + + val ap = ActivationPing(mockk()) + assertNull(ap.getAdvertisingID()) + } + + @Test + fun `getAdvertisingID() returns a valid string if the API returns a valid ID`() { + val testId = "test-value-id" + + mockkStatic(AdvertisingIdClient::class) + every { + AdvertisingIdClient.getAdvertisingIdInfo(any()) + } returns AdvertisingIdClient.Info(testId, false) + + val ap = ActivationPing(mockk()) + assertEquals(testId, ap.getAdvertisingID()) + } + + @Test + fun `getHashedIdentifier() returns an hashed identifier`() { + val testId = "test-value-id" + val testPackageName = "org.mozilla-test.fenix" + val mockedHexReturn = "mocked-HEX" + + // Mock the Base64 to record the byte array that is passed in, + // which is the actual digest. We can't simply test the return value + // of |getHashedIdentifier| as these Android tests require us to mock + // Android-specific APIs. + mockkStatic(Base64::class) + val shaDigest = slot() + every { + Base64.encodeToString(capture(shaDigest), any()) + } returns mockedHexReturn + + // Get the hash identifier. + val mockAp = spyk(ActivationPing(mockk())) + every { mockAp.getAdvertisingID() } returns testId + every { mockAp.getHashingSalt() } returns testPackageName + runBlocking { + assertEquals(mockedHexReturn, mockAp.getHashedIdentifier()) + } + + // Check that the digest of the identifier matches with what we expect. + // Please note that in the real world, Base64.encodeToString would encode + // this to something much shorter, which we'd send with the ping. + val expectedDigestBytes = + "[52, -79, -84, 79, 101, 22, -82, -44, -44, -14, 21, 15, 48, 88, -94, -74, -8, 25, -72, -120, -37, 108, 47, 16, 2, -37, 126, 41, 102, -92, 103, 24]" + assertEquals(expectedDigestBytes, shaDigest.captured.contentToString()) + } + + @Test + fun `checkAndSend() triggers the ping if it wasn't marked as triggered`() { + val mockAp = spyk(ActivationPing(mockk()), recordPrivateCalls = true) + every { mockAp.wasAlreadyTriggered() } returns false + every { mockAp.markAsTriggered() } just Runs + + mockAp.checkAndSend() + + verify(exactly = 1) { mockAp.triggerPing() } + // Marking the ping as triggered happens in a co-routine off the main thread, + // so wait a bit for it. + verify(timeout = 5000, exactly = 1) { mockAp.markAsTriggered() } + } + + @Test + fun `checkAndSend() doesn't trigger the ping again if it was marked as triggered`() { + val mockAp = spyk(ActivationPing(mockk()), recordPrivateCalls = true) + every { mockAp.wasAlreadyTriggered() } returns true + + mockAp.checkAndSend() + + verify(exactly = 0) { mockAp.triggerPing() } + } +} diff --git a/buildSrc/src/main/java/Dependencies.kt b/buildSrc/src/main/java/Dependencies.kt index c6aafadd6..42a9923c3 100644 --- a/buildSrc/src/main/java/Dependencies.kt +++ b/buildSrc/src/main/java/Dependencies.kt @@ -45,6 +45,8 @@ private object Versions { const val tools_test_runner = "1.1.1" const val uiautomator = "2.1.3" const val test_tools = "1.0.2" + + const val google_ads_id_version = "16.0.0" } @Suppress("unused") @@ -164,4 +166,6 @@ object Deps { const val tools_test_rules = "com.android.support.test:rules:${Versions.tools_test_rules}" const val tools_test_runner = "com.android.support.test:runner:${Versions.tools_test_runner}" const val uiautomator = "com.android.support.test.uiautomator:uiautomator-v18:${Versions.uiautomator}" + + const val google_ads_id = "com.google.android.gms:play-services-ads-identifier:${Versions.google_ads_id_version}" } diff --git a/docs/activation.md b/docs/activation.md new file mode 100644 index 000000000..9c05c1755 --- /dev/null +++ b/docs/activation.md @@ -0,0 +1,36 @@ +# The `activation` ping + +## Description +This ping provides a measure of the activation of mobile products. + +## Scheduling +The `activation` ping is automatically sent at the very first startup, after Glean is initialized. +It is only sent once and only re-attempted a subsequent startups if it hasn't been sent yet. + +## Contents +This ping contains the following fields: + +| Field name | Type | Description | +|---|---|---| +| `identifier` | String | An hashed and salted version of the Google Advertising ID from the device. | +| `activation_id` | UUID | An alternate identifier, not correlated with the client_id, generated once and only sent with the activation ping. | + +The `activation` ping also includes the common [ping sections](https://github.com/mozilla-mobile/android-components/blob/master/components/service/glean/docs/pings/pings.md#ping-sections) +found in all pings, with the exclusion of the `client_id` (as defined by the [`pings.yaml`](../app/pings.yaml) file). + +## Example `activation` ping + +```json +{ + "ping_info": { }, + "client_info": { }, + "metrics": { + "string": { + "activation.identifier": "d+lnddDYN2ILBDGvhBIBHORRMrmVwTCp6rGLLFi8SMo=" + }, + "uuid": { + "activation.activation_id": "c0c40a5f-bd75-41ca-8097-9a38103de7fe" + } + } +} +``` diff --git a/docs/telemetry.md b/docs/telemetry.md index 3aacd030a..83c26306a 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -14,6 +14,10 @@ Fenix creates and tries to send a "baseline" ping. It is defined inside the [`me Fenix sends event pings that allows us to measure feature performance. These are defined inside the [`metrics.yaml`](https://github.com/mozilla-mobile/fenix/blob/master/app/metrics.yaml) file. +## Activation + +Fenix sends an activation ping once, at startup. Further documentation can be found in the [`activation` ping](activation.md) docs. + ## Leanplum Events | Event | Leanplum Key | extras |