From 93ca1f6d9f7811ab9cfa5d335feb9bdfc8c6a802 Mon Sep 17 00:00:00 2001 From: Mihai Branescu Date: Wed, 8 Apr 2020 15:51:07 +0300 Subject: [PATCH] For #6558 - added handling of web extension + sending metric for has_ads --- app/metrics.yaml | 28 +++ .../org/mozilla/fenix/ads/AdsTelemetry.kt | 161 ++++++++++++++++++ .../mozilla/fenix/ads/SearchProviderCookie.kt | 10 ++ .../mozilla/fenix/ads/SearchProviderModel.kt | 12 ++ .../java/org/mozilla/fenix/components/Core.kt | 10 ++ .../components/metrics/GleanMetricsService.kt | 11 ++ .../fenix/components/metrics/Metrics.kt | 10 ++ .../main/java/org/mozilla/fenix/ext/Ads.kt | 25 +++ .../java/org/mozilla/fenix/ext/JsonArray.kt | 12 ++ docs/metrics.md | 2 + 10 files changed, 281 insertions(+) create mode 100644 app/src/main/java/org/mozilla/fenix/ads/AdsTelemetry.kt create mode 100644 app/src/main/java/org/mozilla/fenix/ads/SearchProviderCookie.kt create mode 100644 app/src/main/java/org/mozilla/fenix/ads/SearchProviderModel.kt create mode 100644 app/src/main/java/org/mozilla/fenix/ext/Ads.kt create mode 100644 app/src/main/java/org/mozilla/fenix/ext/JsonArray.kt diff --git a/app/metrics.yaml b/app/metrics.yaml index a67df021e..66fb1678e 100644 --- a/app/metrics.yaml +++ b/app/metrics.yaml @@ -2001,6 +2001,34 @@ installation: - fenix-core@mozilla.com expires: "2020-09-01" +browser.search: + with_ads: + type: labeled_counter + description: > + Records counts of SERP pages with adverts displayed. The key format is ‘’. + send_in_pings: + - metrics + bugs: + - https://github.com/mozilla-mobile/fenix/issues/6558 + data_reviews: + - https://github.com/mozilla-mobile/fenix/issues/6558 + notification_emails: + - fenix-core@mozilla.com + expires: "2020-09-01" + ad_clicks: + type: labeled_counter + description: > + Records clicks of adverts on SERP pages. The key format is ‘’. + send_in_pings: + - metrics + bugs: + - https://github.com/mozilla-mobile/fenix/issues/6558 + data_reviews: + - https://github.com/mozilla-mobile/fenix/issues/6558 + notification_emails: + - fenix-core@mozilla.com + expires: "2020-09-01" + addons: open_addons_in_settings: type: event diff --git a/app/src/main/java/org/mozilla/fenix/ads/AdsTelemetry.kt b/app/src/main/java/org/mozilla/fenix/ads/AdsTelemetry.kt new file mode 100644 index 000000000..f2bd9f4cb --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/ads/AdsTelemetry.kt @@ -0,0 +1,161 @@ +package org.mozilla.fenix.ads + +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.collect +import kotlinx.coroutines.flow.map +import mozilla.components.browser.state.state.BrowserState +import mozilla.components.browser.state.store.BrowserStore +import mozilla.components.concept.engine.Engine +import mozilla.components.concept.engine.EngineSession +import mozilla.components.concept.engine.webextension.MessageHandler +import mozilla.components.concept.engine.webextension.WebExtension +import mozilla.components.lib.state.ext.flowScoped +import mozilla.components.support.base.log.logger.Logger +import mozilla.components.support.ktx.kotlinx.coroutines.flow.filterChanged +import org.json.JSONObject +import org.mozilla.fenix.components.metrics.Event +import org.mozilla.fenix.components.metrics.MetricController +import org.mozilla.fenix.ext.containsAds +import org.mozilla.fenix.ext.toList + +class AdsTelemetry(private val metrics: MetricController) { + + private val providerList = listOf( + SearchProviderModel( + name = "google", + regexp = "^https:\\/\\/www\\.google\\.(?:.+)\\/search", + queryParam = "q", + codeParam = "client", + codePrefixes = listOf("firefox"), + followOnParams = listOf("oq", "ved", "ei"), + extraAdServersRegexps = listOf("^https?:\\/\\/www\\.google(?:adservices)?\\.com\\/(?:pagead\\/)?aclk") + ), + SearchProviderModel( + name = "duckduckgo", + regexp = "^https:\\/\\/duckduckgo\\.com\\/", + queryParam = "q", + codeParam = "t", + codePrefixes = listOf("ff"), + followOnParams = listOf("oq", "ved", "ei"), + extraAdServersRegexps = listOf( + "^https:\\/\\/duckduckgo.com\\/y\\.js" + ) + ), + SearchProviderModel( + name = "yahoo", + regexp = "^https:\\/\\/(?:.*)search\\.yahoo\\.com\\/search", + queryParam = "p" + ), + SearchProviderModel( + name = "baidu", + regexp = "^https:\\/\\/www\\.baidu\\.com\\/(?:s|baidu)", + queryParam = "wd", + codeParam = "tn", + codePrefixes = listOf("34046034_", "monline_"), + followOnParams = listOf("oq") + ), + SearchProviderModel( + name = "bing", + regexp = "^https:\\/\\/www\\.bing\\.com\\/search", + queryParam = "q", + codeParam = "pc", + codePrefixes = listOf("MOZ", "MZ"), + followOnParams = listOf("oq", "ved", "ei"), + followOnCookies = listOf( + SearchProviderCookie( + extraCodeParam = "form", + extraCodePrefixes = listOf("QBRE"), + host = "www.bing.com", + name = "SRCHS", + codeParam = "PC", + codePrefixes = listOf("MOZ", "MZ") + ) + ), + extraAdServersRegexps = listOf( + "^https:\\/\\/www\\.bing\\.com\\/acli?c?k", + "^https:\\/\\/www\\.bing\\.com\\/fd\\/ls\\/GLinkPingPost\\.aspx.*acli?c?k" + ) + ) + ) + + fun install( + engine: Engine, + store: BrowserStore + ) { + engine.installWebExtension( + id = ADS_EXTENSION_ID, + url = ADS_EXTENSION_RESOURCE_URL, + allowContentMessaging = true, + onSuccess = { extension -> + Logger.debug("Installed ads extension") + + store.flowScoped { flow -> subscribeToUpdates(flow, extension) } + }, + onError = { _, throwable -> + Logger.error("Could not install ads extension", throwable) + }) + } + + private fun getProviderForUrl(url: String): SearchProviderModel? { + for (provider in providerList) { + if (Regex(provider.regexp).containsMatchIn(url)) { + return provider + } + } + return null + } + + private suspend fun subscribeToUpdates(flow: Flow, extension: WebExtension) { + // Whenever we see a new EngineSession in the store then we register our content message + // handler if it has not been added yet. + flow.map { it.tabs } + .filterChanged { it.engineState.engineSession } + .collect { state -> + val engineSession = state.engineState.engineSession ?: return@collect + + if (extension.hasContentMessageHandler(engineSession, ADS_MESSAGE_ID)) { + return@collect + } + + val messageHandler = AdsTelemetryContentMessageHandler() + extension.registerContentMessageHandler( + engineSession, + ADS_MESSAGE_ID, + messageHandler + ) + } + } + + private inner class AdsTelemetryContentMessageHandler : MessageHandler { + + override fun onMessage(message: Any, source: EngineSession?): Any? { + if (message is JSONObject) { + val urls = getDocumentUrlList(message) + val provider = getProviderForUrl(message.getString(ADS_MESSAGE_SESSION_URL_KEY)) + provider?.let { + if (it.containsAds(urls)) { + metrics.track(Event.SearchWithAds(it.name)) + } + } + } else { + throw IllegalStateException("Received unexpected message: $message") + } + + // Needs to return something that is not null and not Unit: + // https://github.com/mozilla-mobile/android-components/issues/2969 + return "" + } + + private fun getDocumentUrlList(message: JSONObject): List { + return message.getJSONArray(ADS_MESSAGE_DOCUMENT_URLS_KEY).toList() + } + } + + companion object { + private const val ADS_EXTENSION_ID = "mozacBrowserAds" + private const val ADS_EXTENSION_RESOURCE_URL = "resource://android/assets/extensions/ads/" + private const val ADS_MESSAGE_ID = "MozacBrowserAds" + private const val ADS_MESSAGE_SESSION_URL_KEY = "url" + private const val ADS_MESSAGE_DOCUMENT_URLS_KEY = "urls" + } +} diff --git a/app/src/main/java/org/mozilla/fenix/ads/SearchProviderCookie.kt b/app/src/main/java/org/mozilla/fenix/ads/SearchProviderCookie.kt new file mode 100644 index 000000000..47615af9c --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/ads/SearchProviderCookie.kt @@ -0,0 +1,10 @@ +package org.mozilla.fenix.ads + +data class SearchProviderCookie( + val extraCodeParam: String, + val extraCodePrefixes: List, + val host: String, + val name: String, + val codeParam: String, + val codePrefixes: List +) diff --git a/app/src/main/java/org/mozilla/fenix/ads/SearchProviderModel.kt b/app/src/main/java/org/mozilla/fenix/ads/SearchProviderModel.kt new file mode 100644 index 000000000..e105e014a --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/ads/SearchProviderModel.kt @@ -0,0 +1,12 @@ +package org.mozilla.fenix.ads + +data class SearchProviderModel( + val name: String, + val regexp: String, + val queryParam: String, + val codeParam: String = "", + val codePrefixes: List = ArrayList(), + val followOnParams: List = ArrayList(), + val extraAdServersRegexps: List = ArrayList(), + val followOnCookies: List = ArrayList() +) diff --git a/app/src/main/java/org/mozilla/fenix/components/Core.kt b/app/src/main/java/org/mozilla/fenix/components/Core.kt index 62d47209e..13ff02d1c 100644 --- a/app/src/main/java/org/mozilla/fenix/components/Core.kt +++ b/app/src/main/java/org/mozilla/fenix/components/Core.kt @@ -42,6 +42,8 @@ import org.mozilla.fenix.AppRequestInterceptor import org.mozilla.fenix.Config import org.mozilla.fenix.HomeActivity import org.mozilla.fenix.R +import org.mozilla.fenix.ads.AdsTelemetry +import org.mozilla.fenix.ext.components import org.mozilla.fenix.ext.settings import org.mozilla.fenix.media.MediaService import org.mozilla.fenix.utils.Mockable @@ -123,9 +125,13 @@ class Core(private val context: Context) { */ val sessionManager by lazy { SessionManager(engine, store).also { sessionManager -> + // Install the "icons" WebExtension to automatically load icons for every visited website. icons.install(engine, store) + // Install the "ads" WebExtension to get the links in an partner page. + ads.install(engine, store) + // Show an ongoing notification when recording devices (camera, microphone) are used by web content RecordingDevicesNotificationFeature(context, sessionManager) .enable() @@ -169,6 +175,10 @@ class Core(private val context: Context) { BrowserIcons(context, client) } + val ads by lazy { + AdsTelemetry(context.components.analytics.metrics) + } + /** * Shortcut component for managing shortcuts on the device home screen. */ diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt index 73fc35806..16842e710 100644 --- a/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt @@ -12,6 +12,7 @@ import org.mozilla.fenix.GleanMetrics.AboutPage import org.mozilla.fenix.GleanMetrics.Addons import org.mozilla.fenix.GleanMetrics.AppTheme import org.mozilla.fenix.GleanMetrics.BookmarksManagement +import org.mozilla.fenix.GleanMetrics.BrowserSearch import org.mozilla.fenix.GleanMetrics.Collections import org.mozilla.fenix.GleanMetrics.ContextMenu import org.mozilla.fenix.GleanMetrics.CrashReporter @@ -106,6 +107,16 @@ private val Event.wrapper: EventWrapper<*>? }, { Events.performedSearchKeys.valueOf(it) } ) + is Event.SearchWithAds -> EventWrapper( + { + BrowserSearch.withAds[label].add(1) + } + ) + is Event.SearchAdClicked -> EventWrapper( + { + BrowserSearch.adClicks[label].add(1) + } + ) is Event.SearchShortcutSelected -> EventWrapper( { SearchShortcuts.selected.record(it) }, { SearchShortcuts.selectedKeys.valueOf(it) } diff --git a/app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt b/app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt index e8618536b..725d475a4 100644 --- a/app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt +++ b/app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt @@ -365,6 +365,16 @@ sealed class Event { get() = mapOf(AppTheme.darkThemeSelectedKeys.source to source.name) } + data class SearchWithAds(val providerName: String) : Event() { + val label: String + get() = providerName + } + + data class SearchAdClicked(val providerName: String) : Event() { + val label: String + get() = providerName + } + class ContextMenuItemTapped private constructor(val item: String) : Event() { override val extras: Map? get() = mapOf(ContextMenu.itemTappedKeys.named to item) diff --git a/app/src/main/java/org/mozilla/fenix/ext/Ads.kt b/app/src/main/java/org/mozilla/fenix/ext/Ads.kt new file mode 100644 index 000000000..d27e25e79 --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/ext/Ads.kt @@ -0,0 +1,25 @@ +package org.mozilla.fenix.ext + +import org.mozilla.fenix.ads.SearchProviderModel + +fun SearchProviderModel.containsAds(urlList: List): Boolean { + return urlList.containsAds(this.extraAdServersRegexps) +} + +private fun String.isAd(adRegexps: List): Boolean { + for (adsRegex in adRegexps) { + if (Regex(adsRegex).containsMatchIn(this)) { + return true + } + } + return false +} + +private fun List.containsAds(adRegexps: List): Boolean { + for (url in this) { + if (url.isAd(adRegexps)) { + return true + } + } + return false +} diff --git a/app/src/main/java/org/mozilla/fenix/ext/JsonArray.kt b/app/src/main/java/org/mozilla/fenix/ext/JsonArray.kt new file mode 100644 index 000000000..ab49469e6 --- /dev/null +++ b/app/src/main/java/org/mozilla/fenix/ext/JsonArray.kt @@ -0,0 +1,12 @@ +package org.mozilla.fenix.ext + +import org.json.JSONArray + +@Suppress("UNCHECKED_CAST") +fun JSONArray.toList(): List { + val result = ArrayList() + for (i in 0 until length()) { + result.add(get(i) as T) + } + return result +} diff --git a/docs/metrics.md b/docs/metrics.md index 685e37ba0..982c37201 100644 --- a/docs/metrics.md +++ b/docs/metrics.md @@ -46,6 +46,8 @@ The following metrics are added to the ping: | Name | Type | Description | Data reviews | Extras | Expiration | | --- | --- | --- | --- | --- | --- | +| browser.search.ad_clicks |[labeled_counter](https://mozilla.github.io/glean/book/user/metrics/labeled_counters.html) |Records clicks of adverts on SERP pages. The key format is ‘’. |[1](https://github.com/mozilla-mobile/fenix/issues/6558)||2020-09-01 | +| browser.search.with_ads |[labeled_counter](https://mozilla.github.io/glean/book/user/metrics/labeled_counters.html) |Records counts of SERP pages with adverts displayed. The key format is ‘’. |[1](https://github.com/mozilla-mobile/fenix/issues/6558)||2020-09-01 | | events.total_uri_count |[counter](https://mozilla.github.io/glean/book/user/metrics/counter.html) |A counter of URIs visited by the user in the current session, including page reloads. This does not include background page requests and URIs from embedded pages or private browsing. |[1](https://github.com/mozilla-mobile/fenix/pull/1785), [2](https://github.com/mozilla-mobile/fenix/pull/8314)||2020-09-01 | | metrics.search_count |[labeled_counter](https://mozilla.github.io/glean/book/user/metrics/labeled_counters.html) |The labels for this counter are `.`. If the search engine is bundled with Fenix `search-engine-name` will be the name of the search engine. If it's a custom search engine (defined: https://github.com/mozilla-mobile/fenix/issues/1607) the value will be `custom`. `source` will be: `action`, `suggestion`, `widget` or `shortcut` (depending on the source from which the search started). Also added the `other` option for the source but it should never enter on this case. |[1](https://github.com/mozilla-mobile/fenix/pull/1677), [2](https://github.com/mozilla-mobile/fenix/pull/5216), [3](https://github.com/mozilla-mobile/fenix/pull/7310)||2020-09-01 |