From 0c2d22f595ca7e320c4e2a2774f508bc78ca8d2c Mon Sep 17 00:00:00 2001 From: Mihai Branescu Date: Thu, 23 Apr 2020 23:12:13 +0300 Subject: [PATCH] For #6557 - add in_content telemetry class --- .../java/org/mozilla/fenix/components/Core.kt | 15 +- .../telemetry/incontent/InContentTelemetry.kt | 148 ++++++++++++++++++ .../telemetry/incontent/TrackKeyInfo.kt | 19 +++ 3 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/InContentTelemetry.kt create mode 100644 app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/TrackKeyInfo.kt diff --git a/app/src/main/java/org/mozilla/fenix/components/Core.kt b/app/src/main/java/org/mozilla/fenix/components/Core.kt index 20e03b93b..aac21e9b2 100644 --- a/app/src/main/java/org/mozilla/fenix/components/Core.kt +++ b/app/src/main/java/org/mozilla/fenix/components/Core.kt @@ -42,10 +42,11 @@ import org.mozilla.fenix.AppRequestInterceptor import org.mozilla.fenix.Config import org.mozilla.fenix.HomeActivity import org.mozilla.fenix.R -import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry import org.mozilla.fenix.ext.components import org.mozilla.fenix.ext.settings import org.mozilla.fenix.media.MediaService +import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry +import org.mozilla.fenix.search.telemetry.incontent.InContentTelemetry import org.mozilla.fenix.utils.Mockable import java.util.concurrent.TimeUnit @@ -125,12 +126,14 @@ class Core(private val context: Context) { */ val sessionManager by lazy { SessionManager(engine, store).also { sessionManager -> - // Install the "icons" WebExtension to automatically load icons for every visited website. icons.install(engine, store) // Install the "ads" WebExtension to get the links in an partner page. - ads.install(engine, store) + adsTelemetry.install(engine, store) + + // Install the "cookies" WebExtension and tracks user interaction with SERPs. + searchTelemetry.install(engine, store) // Show an ongoing notification when recording devices (camera, microphone) are used by web content RecordingDevicesNotificationFeature(context, sessionManager) @@ -175,10 +178,14 @@ class Core(private val context: Context) { BrowserIcons(context, client) } - val ads by lazy { + val adsTelemetry by lazy { AdsTelemetry(context.components.analytics.metrics) } + val searchTelemetry by lazy { + InContentTelemetry(context.components.analytics.metrics) + } + /** * Shortcut component for managing shortcuts on the device home screen. */ diff --git a/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/InContentTelemetry.kt b/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/InContentTelemetry.kt new file mode 100644 index 000000000..5571ca8ef --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/InContentTelemetry.kt @@ -0,0 +1,148 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.search.telemetry.incontent + +import android.net.Uri +import androidx.annotation.VisibleForTesting +import androidx.core.net.toUri +import mozilla.components.browser.state.store.BrowserStore +import mozilla.components.concept.engine.Engine +import org.json.JSONObject +import org.mozilla.fenix.components.metrics.Event +import org.mozilla.fenix.components.metrics.MetricController +import org.mozilla.fenix.search.telemetry.BaseSearchTelemetry +import org.mozilla.fenix.search.telemetry.ExtensionInfo +import org.mozilla.fenix.search.telemetry.SearchProviderModel + +class InContentTelemetry(private val metrics: MetricController) : BaseSearchTelemetry() { + + override fun install(engine: Engine, store: BrowserStore) { + val info = ExtensionInfo( + id = COOKIES_EXTENSION_ID, + resourceUrl = COOKIES_EXTENSION_RESOURCE_URL, + messageId = COOKIES_MESSAGE_ID + ) + installWebExtension(engine, store, info) + } + + override fun processMessage(message: JSONObject) { + val cookies = getMessageList( + message, + COOKIES_MESSAGE_DOCUMENT_URLS_KEY + ) + trackPartnerUrlTypeMetric(message.getString(COOKIES_MESSAGE_SESSION_URL_KEY), cookies) + } + + @VisibleForTesting + internal fun trackPartnerUrlTypeMetric(url: String, cookies: List) { + val provider = getProviderForUrl(url) + var trackKey: TrackKeyInfo? = null + + provider?.let { + val uri = url.toUri() + val paramSet = uri.queryParameterNames + if (!paramSet.contains(provider.queryParam)) { + return + } + var code: String? = null + + if (provider.codeParam.isNotEmpty()) { + code = uri.getQueryParameter(provider.codeParam) + // Try cookies first because Bing has followOnCookies and valid code, but no + // followOnParams => would tracks organic instead of sap-follow-on + if (provider.followOnCookies.isNotEmpty()) { + // Checks if engine contains a valid follow-on cookie, otherwise return default + trackKey = getTrackKeyFromCookies(provider, uri, cookies, code) + } + + // For Bing if it didn't have a valid cookie and for all the other search engines + if (resultNotComputedFromCookies(trackKey) && hasValidCode(code, provider)) { + val type = getSapType(provider.followOnParams, paramSet) + trackKey = TrackKeyInfo(provider.name, type, code) + } + } + + // Go default if no codeParam was found + if (trackKey == null) { + trackKey = TrackKeyInfo(provider.name, SEARCH_TYPE_ORGANIC, code) + } + + trackKey?.let { + metrics.track(Event.SearchInContent(it.createTrackKey())) + } + } + } + + private fun resultNotComputedFromCookies(trackKey: TrackKeyInfo?): Boolean = + trackKey == null || trackKey.type == SEARCH_TYPE_ORGANIC + + private fun hasValidCode(code: String?, provider: SearchProviderModel): Boolean = + code != null && provider.codePrefixes.any { prefix -> code.startsWith(prefix) } + + private fun getSapType(followOnParams: List, paramSet: Set): String { + return if (followOnParams.any { param -> paramSet.contains(param) }) { + SEARCH_TYPE_SAP_FOLLOW_ON + } else { + SEARCH_TYPE_SAP + } + } + + private fun getTrackKeyFromCookies( + provider: SearchProviderModel, + uri: Uri, + cookies: List, + code: String? + ): TrackKeyInfo { + // Especially Bing requires lots of extra work related to cookies. + for (followOnCookie in provider.followOnCookies) { + val eCode = uri.getQueryParameter(followOnCookie.extraCodeParam) + if (eCode == null || !followOnCookie.extraCodePrefixes.any { prefix -> + eCode.startsWith(prefix) + }) { + continue + } + + // If this cookie is present, it's probably an SAP follow-on. + // This might be an organic follow-on in the same session, but there + // is no way to tell the difference. + for (cookie in cookies) { + if (cookie.getString("name") != followOnCookie.name) { + continue + } + val valueList = cookie.getString("value") + .split("=") + .map { item -> item.trim() } + + if (valueList.size == 2 && valueList[0] == followOnCookie.codeParam && + followOnCookie.codePrefixes.any { prefix -> + valueList[1].startsWith( + prefix + ) + } + ) { + return TrackKeyInfo(provider.name, SEARCH_TYPE_SAP_FOLLOW_ON, valueList[1]) + } + } + } + return TrackKeyInfo(provider.name, SEARCH_TYPE_ORGANIC, code) + } + + companion object { + @VisibleForTesting + internal const val COOKIES_EXTENSION_ID = "BrowserCookiesExtension" + @VisibleForTesting + internal const val COOKIES_EXTENSION_RESOURCE_URL = + "resource://android/assets/extensions/cookies/" + @VisibleForTesting + internal const val COOKIES_MESSAGE_SESSION_URL_KEY = "url" + @VisibleForTesting + internal const val COOKIES_MESSAGE_DOCUMENT_URLS_KEY = "cookies" + private const val COOKIES_MESSAGE_ID = "BrowserCookiesMessage" + + private const val SEARCH_TYPE_ORGANIC = "organic" + private const val SEARCH_TYPE_SAP = "sap" + private const val SEARCH_TYPE_SAP_FOLLOW_ON = "sap-follow-on" + } +} diff --git a/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/TrackKeyInfo.kt b/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/TrackKeyInfo.kt new file mode 100644 index 000000000..73473943f --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/search/telemetry/incontent/TrackKeyInfo.kt @@ -0,0 +1,19 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.search.telemetry.incontent + +import java.util.Locale + +internal data class TrackKeyInfo( + var providerName: String, + var type: String, + var code: String? +) { + fun createTrackKey(): String { + return "${providerName.toLowerCase(Locale.ROOT)}.in-content" + + ".${type.toLowerCase(Locale.ROOT)}" + + ".${code?.toLowerCase(Locale.ROOT) ?: "none"}" + } +}