From 28496fecc0e36871b5e3d695022b18ec63fc82d7 Mon Sep 17 00:00:00 2001 From: Mihai Branescu Date: Thu, 23 Apr 2020 23:11:22 +0300 Subject: [PATCH] For #6557 - split base extension functionality from ads --- .../fenix/browser/UriOpenedObserver.kt | 4 +- .../search/telemetry/BaseSearchTelemetry.kt | 157 ++++++++++++++++++ .../fenix/search/telemetry/ExtensionInfo.kt | 11 ++ .../search/telemetry/ads/AdsTelemetry.kt | 154 ++--------------- 4 files changed, 185 insertions(+), 141 deletions(-) create mode 100644 app/src/main/java/org/mozilla/fenix/search/telemetry/BaseSearchTelemetry.kt create mode 100644 app/src/main/java/org/mozilla/fenix/search/telemetry/ExtensionInfo.kt diff --git a/app/src/main/java/org/mozilla/fenix/browser/UriOpenedObserver.kt b/app/src/main/java/org/mozilla/fenix/browser/UriOpenedObserver.kt index 35eaf7e35..fb48418a7 100644 --- a/app/src/main/java/org/mozilla/fenix/browser/UriOpenedObserver.kt +++ b/app/src/main/java/org/mozilla/fenix/browser/UriOpenedObserver.kt @@ -9,10 +9,10 @@ import androidx.fragment.app.FragmentActivity import androidx.lifecycle.LifecycleOwner import mozilla.components.browser.session.Session import mozilla.components.browser.session.SessionManager -import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry import org.mozilla.fenix.components.metrics.MetricController import org.mozilla.fenix.ext.components import org.mozilla.fenix.ext.metrics +import org.mozilla.fenix.search.telemetry.ads.AdsTelemetry class UriOpenedObserver( private val owner: LifecycleOwner, @@ -25,7 +25,7 @@ class UriOpenedObserver( activity, activity.components.core.sessionManager, activity.metrics, - activity.components.core.ads + activity.components.core.adsTelemetry ) @VisibleForTesting diff --git a/app/src/main/java/org/mozilla/fenix/search/telemetry/BaseSearchTelemetry.kt b/app/src/main/java/org/mozilla/fenix/search/telemetry/BaseSearchTelemetry.kt new file mode 100644 index 000000000..f5025f92e --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/search/telemetry/BaseSearchTelemetry.kt @@ -0,0 +1,157 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.search.telemetry + +import androidx.annotation.VisibleForTesting +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.map +import mozilla.components.browser.state.state.BrowserState +import mozilla.components.browser.state.store.BrowserStore +import mozilla.components.concept.engine.Engine +import mozilla.components.concept.engine.EngineSession +import mozilla.components.concept.engine.webextension.MessageHandler +import mozilla.components.concept.engine.webextension.WebExtension +import mozilla.components.lib.state.ext.flowScoped +import mozilla.components.support.base.log.logger.Logger +import mozilla.components.support.ktx.android.org.json.toList +import mozilla.components.support.ktx.kotlinx.coroutines.flow.filterChanged +import org.json.JSONObject + +abstract class BaseSearchTelemetry { + + @VisibleForTesting + internal val providerList = listOf( + SearchProviderModel( + name = "google", + regexp = "^https:\\/\\/www\\.google\\.(?:.+)\\/search", + queryParam = "q", + codeParam = "client", + codePrefixes = listOf("firefox"), + followOnParams = listOf("oq", "ved", "ei"), + extraAdServersRegexps = listOf("^https?:\\/\\/www\\.google(?:adservices)?\\.com\\/(?:pagead\\/)?aclk") + ), + SearchProviderModel( + name = "duckduckgo", + regexp = "^https:\\/\\/duckduckgo\\.com\\/", + queryParam = "q", + codeParam = "t", + codePrefixes = listOf("f"), + extraAdServersRegexps = listOf( + "^https:\\/\\/duckduckgo.com\\/y\\.js", + "^https:\\/\\/www\\.amazon\\.(?:[a-z.]{2,24}).*(?:tag=duckduckgo-)" + ) + ), + SearchProviderModel( + name = "yahoo", + regexp = "^https:\\/\\/(?:.*)search\\.yahoo\\.com\\/search", + queryParam = "p" + ), + SearchProviderModel( + name = "baidu", + regexp = "^https:\\/\\/www\\.baidu\\.com\\/from=844b\\/(?:s|baidu)", + queryParam = "wd", + codeParam = "tn", + codePrefixes = listOf("34046034_", "monline_"), + followOnParams = listOf("oq") + ), + SearchProviderModel( + name = "bing", + regexp = "^https:\\/\\/www\\.bing\\.com\\/search", + queryParam = "q", + codeParam = "pc", + codePrefixes = listOf("MOZ", "MZ"), + followOnCookies = listOf( + SearchProviderCookie( + extraCodeParam = "form", + extraCodePrefixes = listOf("QBRE"), + host = "www.bing.com", + name = "SRCHS", + codeParam = "PC", + codePrefixes = listOf("MOZ", "MZ") + ) + ), + extraAdServersRegexps = listOf( + "^https:\\/\\/www\\.bing\\.com\\/acli?c?k", + "^https:\\/\\/www\\.bing\\.com\\/fd\\/ls\\/GLinkPingPost\\.aspx.*acli?c?k" + ) + ) + ) + + abstract fun install(engine: Engine, store: BrowserStore) + + internal fun getProviderForUrl(url: String): SearchProviderModel? { + for (provider in providerList) { + if (Regex(provider.regexp).containsMatchIn(url)) { + return provider + } + } + return null + } + + protected fun installWebExtension( + engine: Engine, + store: BrowserStore, + extensionInfo: ExtensionInfo + ) { + engine.installWebExtension( + id = extensionInfo.id, + url = extensionInfo.resourceUrl, + allowContentMessaging = true, + onSuccess = { extension -> + store.flowScoped { flow -> subscribeToUpdates(flow, extension, extensionInfo) } + }, + onError = { _, throwable -> + Logger.error("Could not install ${extensionInfo.id} extension", throwable) + }) + } + + private suspend fun subscribeToUpdates( + flow: Flow, + extension: WebExtension, + extensionInfo: ExtensionInfo + ) { + // Whenever we see a new EngineSession in the store then we register our content message + // handler if it has not been added yet. + flow.map { it.tabs } + .filterChanged { it.engineState.engineSession } + .collect { state -> + val engineSession = state.engineState.engineSession ?: return@collect + + if (extension.hasContentMessageHandler(engineSession, extensionInfo.messageId)) { + return@collect + } + extension.registerContentMessageHandler( + engineSession, + extensionInfo.messageId, + SearchTelemetryMessageHandler() + ) + } + } + + protected fun getMessageList(message: JSONObject, key: String): List { + return message.getJSONArray(key).toList() + } + + /** + * This method is used to process any valid json message coming from a web-extension + */ + protected abstract fun processMessage(message: JSONObject) + + internal inner class SearchTelemetryMessageHandler : MessageHandler { + + override fun onMessage(message: Any, source: EngineSession?): Any? { + if (message is JSONObject) { + processMessage(message) + } else { + throw IllegalStateException("Received unexpected message: $message") + } + + // Needs to return something that is not null and not Unit: + // https://github.com/mozilla-mobile/android-components/issues/2969 + return "" + } + } +} diff --git a/app/src/main/java/org/mozilla/fenix/search/telemetry/ExtensionInfo.kt b/app/src/main/java/org/mozilla/fenix/search/telemetry/ExtensionInfo.kt new file mode 100644 index 000000000..dd01b1515 --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/search/telemetry/ExtensionInfo.kt @@ -0,0 +1,11 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.fenix.search.telemetry + +data class ExtensionInfo( + internal val id: String, + internal val resourceUrl: String, + internal val messageId: String +) diff --git a/app/src/main/java/org/mozilla/fenix/search/telemetry/ads/AdsTelemetry.kt b/app/src/main/java/org/mozilla/fenix/search/telemetry/ads/AdsTelemetry.kt index 27bef0bbe..848046c37 100644 --- a/app/src/main/java/org/mozilla/fenix/search/telemetry/ads/AdsTelemetry.kt +++ b/app/src/main/java/org/mozilla/fenix/search/telemetry/ads/AdsTelemetry.kt @@ -5,104 +5,27 @@ package org.mozilla.fenix.search.telemetry.ads import androidx.annotation.VisibleForTesting -import kotlinx.coroutines.flow.Flow -import kotlinx.coroutines.flow.collect -import kotlinx.coroutines.flow.map -import mozilla.components.browser.state.state.BrowserState import mozilla.components.browser.state.store.BrowserStore import mozilla.components.concept.engine.Engine -import mozilla.components.concept.engine.EngineSession -import mozilla.components.concept.engine.webextension.MessageHandler -import mozilla.components.concept.engine.webextension.WebExtension -import mozilla.components.lib.state.ext.flowScoped -import mozilla.components.support.base.log.logger.Logger -import mozilla.components.support.ktx.android.org.json.toList -import mozilla.components.support.ktx.kotlinx.coroutines.flow.filterChanged import org.json.JSONObject import org.mozilla.fenix.components.metrics.Event import org.mozilla.fenix.components.metrics.MetricController import org.mozilla.fenix.ext.containsAds -import org.mozilla.fenix.search.telemetry.SearchProviderCookie -import org.mozilla.fenix.search.telemetry.SearchProviderModel +import org.mozilla.fenix.search.telemetry.BaseSearchTelemetry +import org.mozilla.fenix.search.telemetry.ExtensionInfo -class AdsTelemetry(private val metrics: MetricController) { +class AdsTelemetry(private val metrics: MetricController) : BaseSearchTelemetry() { - @VisibleForTesting - internal val providerList = listOf( - SearchProviderModel( - name = "google", - regexp = "^https:\\/\\/www\\.google\\.(?:.+)\\/search", - queryParam = "q", - codeParam = "client", - codePrefixes = listOf("firefox"), - followOnParams = listOf("oq", "ved", "ei"), - extraAdServersRegexps = listOf("^https?:\\/\\/www\\.google(?:adservices)?\\.com\\/(?:pagead\\/)?aclk") - ), - SearchProviderModel( - name = "duckduckgo", - regexp = "^https:\\/\\/duckduckgo\\.com\\/", - queryParam = "q", - codeParam = "t", - codePrefixes = listOf("ff"), - followOnParams = listOf("oq", "ved", "ei"), - extraAdServersRegexps = listOf( - "^https:\\/\\/duckduckgo.com\\/y\\.js", - "^https:\\/\\/www\\.amazon\\.(?:[a-z.]{2,24}).*(?:tag=duckduckgo-)" - ) - ), - SearchProviderModel( - name = "yahoo", - regexp = "^https:\\/\\/(?:.*)search\\.yahoo\\.com\\/search", - queryParam = "p" - ), - SearchProviderModel( - name = "baidu", - regexp = "^https:\\/\\/www\\.baidu\\.com\\/from=844b\\/(?:s|baidu)", - queryParam = "wd", - codeParam = "tn", - codePrefixes = listOf("34046034_", "monline_"), - followOnParams = listOf("oq") - ), - SearchProviderModel( - name = "bing", - regexp = "^https:\\/\\/www\\.bing\\.com\\/search", - queryParam = "q", - codeParam = "pc", - codePrefixes = listOf("MOZ", "MZ"), - followOnParams = listOf("oq", "ved", "ei"), - followOnCookies = listOf( - SearchProviderCookie( - extraCodeParam = "form", - extraCodePrefixes = listOf("QBRE"), - host = "www.bing.com", - name = "SRCHS", - codeParam = "PC", - codePrefixes = listOf("MOZ", "MZ") - ) - ), - extraAdServersRegexps = listOf( - "^https:\\/\\/www\\.bing\\.com\\/acli?c?k", - "^https:\\/\\/www\\.bing\\.com\\/fd\\/ls\\/GLinkPingPost\\.aspx.*acli?c?k" - ) - ) - ) - - fun install( + override fun install( engine: Engine, store: BrowserStore ) { - engine.installWebExtension( + val info = ExtensionInfo( id = ADS_EXTENSION_ID, - url = ADS_EXTENSION_RESOURCE_URL, - allowContentMessaging = true, - onSuccess = { extension -> - Logger.debug("Installed ads extension") - - store.flowScoped { flow -> subscribeToUpdates(flow, extension) } - }, - onError = { _, throwable -> - Logger.error("Could not install ads extension", throwable) - }) + resourceUrl = ADS_EXTENSION_RESOURCE_URL, + messageId = ADS_MESSAGE_ID + ) + installWebExtension(engine, store, info) } fun trackAdClickedMetric(sessionUrl: String?, urlPath: List) { @@ -117,61 +40,14 @@ class AdsTelemetry(private val metrics: MetricController) { } } - @VisibleForTesting - internal fun getProviderForUrl(url: String): SearchProviderModel? { - for (provider in providerList) { - if (Regex(provider.regexp).containsMatchIn(url)) { - return provider + override fun processMessage(message: JSONObject) { + val urls = getMessageList(message, ADS_MESSAGE_DOCUMENT_URLS_KEY) + val provider = getProviderForUrl(message.getString(ADS_MESSAGE_SESSION_URL_KEY)) + provider?.let { + if (it.containsAds(urls)) { + metrics.track(Event.SearchWithAds(it.name)) } } - return null - } - - private suspend fun subscribeToUpdates(flow: Flow, extension: WebExtension) { - // Whenever we see a new EngineSession in the store then we register our content message - // handler if it has not been added yet. - flow.map { it.tabs } - .filterChanged { it.engineState.engineSession } - .collect { state -> - val engineSession = state.engineState.engineSession ?: return@collect - - if (extension.hasContentMessageHandler(engineSession, ADS_MESSAGE_ID)) { - return@collect - } - - val messageHandler = AdsTelemetryContentMessageHandler() - extension.registerContentMessageHandler( - engineSession, - ADS_MESSAGE_ID, - messageHandler - ) - } - } - - @VisibleForTesting - internal inner class AdsTelemetryContentMessageHandler : MessageHandler { - - override fun onMessage(message: Any, source: EngineSession?): Any? { - if (message is JSONObject) { - val urls = getDocumentUrlList(message) - val provider = getProviderForUrl(message.getString(ADS_MESSAGE_SESSION_URL_KEY)) - provider?.let { - if (it.containsAds(urls)) { - metrics.track(Event.SearchWithAds(it.name)) - } - } - } else { - throw IllegalStateException("Received unexpected message: $message") - } - - // Needs to return something that is not null and not Unit: - // https://github.com/mozilla-mobile/android-components/issues/2969 - return "" - } - - private fun getDocumentUrlList(message: JSONObject): List { - return message.getJSONArray(ADS_MESSAGE_DOCUMENT_URLS_KEY).toList() - } } companion object {