For #6558 - added handling of web extension + sending metric for has_ads

app/metrics.yaml

@@ -2001,6 +2001,34 @@ installation:
       - fenix-core@mozilla.com
     expires: "2020-09-01"
 
+browser.search:
+    with_ads:
+      type: labeled_counter
+      description: >
+        Records counts of SERP pages with adverts displayed. The key format is ‘<provider-name>’.
+      send_in_pings:
+        - metrics
+      bugs:
+        - https://github.com/mozilla-mobile/fenix/issues/6558
+      data_reviews:
+        - https://github.com/mozilla-mobile/fenix/issues/6558
+      notification_emails:
+        - fenix-core@mozilla.com
+      expires: "2020-09-01"
+    ad_clicks:
+      type: labeled_counter
+      description: >
+        Records clicks of adverts on SERP pages. The key format is ‘<provider-name>’.
+      send_in_pings:
+        - metrics
+      bugs:
+        - https://github.com/mozilla-mobile/fenix/issues/6558
+      data_reviews:
+        - https://github.com/mozilla-mobile/fenix/issues/6558
+      notification_emails:
+        - fenix-core@mozilla.com
+      expires: "2020-09-01"
+
 addons:
   open_addons_in_settings:
     type: event

app/src/main/java/org/mozilla/fenix/ads/AdsTelemetry.kt

@@ -0,0 +1,161 @@
+package org.mozilla.fenix.ads
+
+import kotlinx.coroutines.flow.Flow
+import kotlinx.coroutines.flow.collect
+import kotlinx.coroutines.flow.map
+import mozilla.components.browser.state.state.BrowserState
+import mozilla.components.browser.state.store.BrowserStore
+import mozilla.components.concept.engine.Engine
+import mozilla.components.concept.engine.EngineSession
+import mozilla.components.concept.engine.webextension.MessageHandler
+import mozilla.components.concept.engine.webextension.WebExtension
+import mozilla.components.lib.state.ext.flowScoped
+import mozilla.components.support.base.log.logger.Logger
+import mozilla.components.support.ktx.kotlinx.coroutines.flow.filterChanged
+import org.json.JSONObject
+import org.mozilla.fenix.components.metrics.Event
+import org.mozilla.fenix.components.metrics.MetricController
+import org.mozilla.fenix.ext.containsAds
+import org.mozilla.fenix.ext.toList
+
+class AdsTelemetry(private val metrics: MetricController) {
+
+    private val providerList = listOf(
+        SearchProviderModel(
+            name = "google",
+            regexp = "^https:\\/\\/www\\.google\\.(?:.+)\\/search",
+            queryParam = "q",
+            codeParam = "client",
+            codePrefixes = listOf("firefox"),
+            followOnParams = listOf("oq", "ved", "ei"),
+            extraAdServersRegexps = listOf("^https?:\\/\\/www\\.google(?:adservices)?\\.com\\/(?:pagead\\/)?aclk")
+        ),
+        SearchProviderModel(
+            name = "duckduckgo",
+            regexp = "^https:\\/\\/duckduckgo\\.com\\/",
+            queryParam = "q",
+            codeParam = "t",
+            codePrefixes = listOf("ff"),
+            followOnParams = listOf("oq", "ved", "ei"),
+            extraAdServersRegexps = listOf(
+                "^https:\\/\\/duckduckgo.com\\/y\\.js"
+            )
+        ),
+        SearchProviderModel(
+            name = "yahoo",
+            regexp = "^https:\\/\\/(?:.*)search\\.yahoo\\.com\\/search",
+            queryParam = "p"
+        ),
+        SearchProviderModel(
+            name = "baidu",
+            regexp = "^https:\\/\\/www\\.baidu\\.com\\/(?:s|baidu)",
+            queryParam = "wd",
+            codeParam = "tn",
+            codePrefixes = listOf("34046034_", "monline_"),
+            followOnParams = listOf("oq")
+        ),
+        SearchProviderModel(
+            name = "bing",
+            regexp = "^https:\\/\\/www\\.bing\\.com\\/search",
+            queryParam = "q",
+            codeParam = "pc",
+            codePrefixes = listOf("MOZ", "MZ"),
+            followOnParams = listOf("oq", "ved", "ei"),
+            followOnCookies = listOf(
+                SearchProviderCookie(
+                    extraCodeParam = "form",
+                    extraCodePrefixes = listOf("QBRE"),
+                    host = "www.bing.com",
+                    name = "SRCHS",
+                    codeParam = "PC",
+                    codePrefixes = listOf("MOZ", "MZ")
+                )
+            ),
+            extraAdServersRegexps = listOf(
+                "^https:\\/\\/www\\.bing\\.com\\/acli?c?k",
+                "^https:\\/\\/www\\.bing\\.com\\/fd\\/ls\\/GLinkPingPost\\.aspx.*acli?c?k"
+            )
+        )
+    )
+
+    fun install(
+        engine: Engine,
+        store: BrowserStore
+    ) {
+        engine.installWebExtension(
+            id = ADS_EXTENSION_ID,
+            url = ADS_EXTENSION_RESOURCE_URL,
+            allowContentMessaging = true,
+            onSuccess = { extension ->
+                Logger.debug("Installed ads extension")
+
+                store.flowScoped { flow -> subscribeToUpdates(flow, extension) }
+            },
+            onError = { _, throwable ->
+                Logger.error("Could not install ads extension", throwable)
+            })
+    }
+
+    private fun getProviderForUrl(url: String): SearchProviderModel? {
+        for (provider in providerList) {
+            if (Regex(provider.regexp).containsMatchIn(url)) {
+                return provider
+            }
+        }
+        return null
+    }
+
+    private suspend fun subscribeToUpdates(flow: Flow<BrowserState>, extension: WebExtension) {
+        // Whenever we see a new EngineSession in the store then we register our content message
+        // handler if it has not been added yet.
+        flow.map { it.tabs }
+            .filterChanged { it.engineState.engineSession }
+            .collect { state ->
+                val engineSession = state.engineState.engineSession ?: return@collect
+
+                if (extension.hasContentMessageHandler(engineSession, ADS_MESSAGE_ID)) {
+                    return@collect
+                }
+
+                val messageHandler = AdsTelemetryContentMessageHandler()
+                extension.registerContentMessageHandler(
+                    engineSession,
+                    ADS_MESSAGE_ID,
+                    messageHandler
+                )
+            }
+    }
+
+    private inner class AdsTelemetryContentMessageHandler : MessageHandler {
+
+        override fun onMessage(message: Any, source: EngineSession?): Any? {
+            if (message is JSONObject) {
+                val urls = getDocumentUrlList(message)
+                val provider = getProviderForUrl(message.getString(ADS_MESSAGE_SESSION_URL_KEY))
+                provider?.let {
+                    if (it.containsAds(urls)) {
+                        metrics.track(Event.SearchWithAds(it.name))
+                    }
+                }
+            } else {
+                throw IllegalStateException("Received unexpected message: $message")
+            }
+
+            // Needs to return something that is not null and not Unit:
+            // https://github.com/mozilla-mobile/android-components/issues/2969
+            return ""
+        }
+
+        private fun getDocumentUrlList(message: JSONObject): List<String> {
+            return message.getJSONArray(ADS_MESSAGE_DOCUMENT_URLS_KEY).toList()
+        }
+    }
+
+    companion object {
+        private const val ADS_EXTENSION_ID = "mozacBrowserAds"
+        private const val ADS_EXTENSION_RESOURCE_URL = "resource://android/assets/extensions/ads/"
+        private const val ADS_MESSAGE_ID = "MozacBrowserAds"
+        private const val ADS_MESSAGE_SESSION_URL_KEY = "url"
+        private const val ADS_MESSAGE_DOCUMENT_URLS_KEY = "urls"
+    }
+}

app/src/main/java/org/mozilla/fenix/ads/SearchProviderCookie.kt

@@ -0,0 +1,10 @@
+package org.mozilla.fenix.ads
+
+data class SearchProviderCookie(
+    val extraCodeParam: String,
+    val extraCodePrefixes: List<String>,
+    val host: String,
+    val name: String,
+    val codeParam: String,
+    val codePrefixes: List<String>
+)

app/src/main/java/org/mozilla/fenix/ads/SearchProviderModel.kt

@@ -0,0 +1,12 @@
+package org.mozilla.fenix.ads
+
+data class SearchProviderModel(
+    val name: String,
+    val regexp: String,
+    val queryParam: String,
+    val codeParam: String = "",
+    val codePrefixes: List<String> = ArrayList(),
+    val followOnParams: List<String> = ArrayList(),
+    val extraAdServersRegexps: List<String> = ArrayList(),
+    val followOnCookies: List<SearchProviderCookie> = ArrayList()
+)

app/src/main/java/org/mozilla/fenix/components/Core.kt

@@ -42,6 +42,8 @@ import org.mozilla.fenix.AppRequestInterceptor
 import org.mozilla.fenix.Config
 import org.mozilla.fenix.HomeActivity
 import org.mozilla.fenix.R
+import org.mozilla.fenix.ads.AdsTelemetry
+import org.mozilla.fenix.ext.components
 import org.mozilla.fenix.ext.settings
 import org.mozilla.fenix.media.MediaService
 import org.mozilla.fenix.utils.Mockable
@@ -123,9 +125,13 @@ class Core(private val context: Context) {
      */
     val sessionManager by lazy {
         SessionManager(engine, store).also { sessionManager ->
+
             // Install the "icons" WebExtension to automatically load icons for every visited website.
             icons.install(engine, store)
 
+            // Install the "ads" WebExtension to get the links in an partner page.
+            ads.install(engine, store)
+
             // Show an ongoing notification when recording devices (camera, microphone) are used by web content
             RecordingDevicesNotificationFeature(context, sessionManager)
                 .enable()
@@ -169,6 +175,10 @@ class Core(private val context: Context) {
         BrowserIcons(context, client)
     }
 
+    val ads by lazy {
+        AdsTelemetry(context.components.analytics.metrics)
+    }
+
     /**
      * Shortcut component for managing shortcuts on the device home screen.
      */

app/src/main/java/org/mozilla/fenix/components/metrics/GleanMetricsService.kt

@@ -12,6 +12,7 @@ import org.mozilla.fenix.GleanMetrics.AboutPage
 import org.mozilla.fenix.GleanMetrics.Addons
 import org.mozilla.fenix.GleanMetrics.AppTheme
 import org.mozilla.fenix.GleanMetrics.BookmarksManagement
+import org.mozilla.fenix.GleanMetrics.BrowserSearch
 import org.mozilla.fenix.GleanMetrics.Collections
 import org.mozilla.fenix.GleanMetrics.ContextMenu
 import org.mozilla.fenix.GleanMetrics.CrashReporter
@@ -106,6 +107,16 @@ private val Event.wrapper: EventWrapper<*>?
             },
             { Events.performedSearchKeys.valueOf(it) }
         )
+        is Event.SearchWithAds -> EventWrapper<NoExtraKeys>(
+            {
+                BrowserSearch.withAds[label].add(1)
+            }
+        )
+        is Event.SearchAdClicked -> EventWrapper<NoExtraKeys>(
+            {
+                BrowserSearch.adClicks[label].add(1)
+            }
+        )
         is Event.SearchShortcutSelected -> EventWrapper(
             { SearchShortcuts.selected.record(it) },
             { SearchShortcuts.selectedKeys.valueOf(it) }

app/src/main/java/org/mozilla/fenix/components/metrics/Metrics.kt

@@ -365,6 +365,16 @@ sealed class Event {
             get() = mapOf(AppTheme.darkThemeSelectedKeys.source to source.name)
     }
 
+    data class SearchWithAds(val providerName: String) : Event() {
+        val label: String
+            get() = providerName
+    }
+
+    data class SearchAdClicked(val providerName: String) : Event() {
+        val label: String
+            get() = providerName
+    }
+
     class ContextMenuItemTapped private constructor(val item: String) : Event() {
         override val extras: Map<ContextMenu.itemTappedKeys, String>?
             get() = mapOf(ContextMenu.itemTappedKeys.named to item)

app/src/main/java/org/mozilla/fenix/ext/Ads.kt

@@ -0,0 +1,25 @@
+package org.mozilla.fenix.ext
+
+import org.mozilla.fenix.ads.SearchProviderModel
+
+fun SearchProviderModel.containsAds(urlList: List<String>): Boolean {
+    return urlList.containsAds(this.extraAdServersRegexps)
+}
+
+private fun String.isAd(adRegexps: List<String>): Boolean {
+    for (adsRegex in adRegexps) {
+        if (Regex(adsRegex).containsMatchIn(this)) {
+            return true
+        }
+    }
+    return false
+}
+
+private fun List<String>.containsAds(adRegexps: List<String>): Boolean {
+    for (url in this) {
+        if (url.isAd(adRegexps)) {
+            return true
+        }
+    }
+    return false
+}

app/src/main/java/org/mozilla/fenix/ext/JsonArray.kt

@@ -0,0 +1,12 @@
+package org.mozilla.fenix.ext
+
+import org.json.JSONArray
+
+@Suppress("UNCHECKED_CAST")
+fun <T> JSONArray.toList(): List<T> {
+    val result = ArrayList<T>()
+    for (i in 0 until length()) {
+        result.add(get(i) as T)
+    }
+    return result
+}