Bug 1836248 - Integrate the extended stripOnShare feature into the URLQueryStringStripper. r=pbz,anti-tracking-reviewers

Differential Revision: https://phabricator.services.mozilla.com/D181462

dom/chrome-webidl/StripOnShareRule.webidl

@@ -0,0 +1,15 @@
+/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/.
+ */
+
+/* This dictionary is used for the strip on share feature.
+ * It stores a list of query parameters and a list of sites
+ * for which the query parameters are stripped
+ */
+[GenerateInitFromJSON]
+dictionary StripRule {
+  sequence<DOMString> queryParams = [];
+  sequence<DOMString> topLevelSites = [];
+};

dom/chrome-webidl/moz.build

@@ -79,6 +79,7 @@ WEBIDL_FILES = [
     "PrecompiledScript.webidl",
     "PromiseDebugging.webidl",
     "SessionStoreUtils.webidl",
+    "StripOnShareRule.webidl",
     "StructuredCloneHolder.webidl",
     "TelemetryStopwatch.webidl",
     "UserInteraction.webidl",

toolkit/components/antitracking/URLQueryStringStripper.cpp

@@ -17,6 +17,7 @@
 #include "nsIURIMutator.h"
 #include "nsUnicharUtils.h"
 #include "nsURLHelper.h"
+#include "mozilla/dom/StripOnShareRuleBinding.h"
 
 namespace {
 
@@ -79,8 +80,66 @@ URLQueryStringStripper::StripForCopyOrShare(nsIURI* aURI,
   if (!StaticPrefs::privacy_query_stripping_strip_on_share_enabled()) {
     return NS_ERROR_NOT_AVAILABLE;
   }
-  uint32_t numStripped;
-  return StripQueryString(aURI, strippedURI, &numStripped);
+  NS_ENSURE_ARG_POINTER(aURI);
+  NS_ENSURE_ARG_POINTER(strippedURI);
+  int aStripCount = 0;
+
+  nsAutoCString query;
+  nsresult rv = aURI->GetQuery(query);
+  NS_ENSURE_SUCCESS(rv, rv);
+  // We don't need to do anything if there is no query string.
+  if (query.IsEmpty()) {
+    return NS_OK;
+  }
+  nsAutoCString host;
+  rv = aURI->GetHost(host);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  URLParams params;
+
+  URLParams::Parse(query, [&](nsString&& name, nsString&& value) {
+    nsAutoString lowerCaseName;
+    ToLowerCase(name, lowerCaseName);
+    // Look through the global rules.
+    dom::StripRule globalRule;
+    bool keyExists = mStripOnShareMap.Get("*"_ns, &globalRule);
+    // There should always be a global rule.
+    MOZ_ASSERT(keyExists);
+    for (const auto& param : globalRule.mQueryParams) {
+      if (param == lowerCaseName) {
+        aStripCount++;
+        return true;
+      }
+    }
+
+    // Check for site specific rules.
+    dom::StripRule siteSpecificRule;
+    keyExists = mStripOnShareMap.Get(host, &siteSpecificRule);
+    if (keyExists) {
+      for (const auto& param : siteSpecificRule.mQueryParams) {
+        if (param == lowerCaseName) {
+          aStripCount++;
+          return true;
+        }
+      }
+    }
+
+    params.Append(name, value);
+    return true;
+  });
+
+  if (!aStripCount) {
+    return NS_OK;
+  }
+
+  nsAutoString newQuery;
+  params.Serialize(newQuery, false);
+
+  Unused << NS_MutateURI(aURI)
+                .SetQuery(NS_ConvertUTF16toUTF8(newQuery))
+                .Finalize(strippedURI);
+
+  return NS_OK;
 }
 
 NS_IMETHODIMP
@@ -128,32 +187,83 @@ void URLQueryStringStripper::OnPrefChange(const char* aPref, void* aData) {
 }
 
 nsresult URLQueryStringStripper::Init() {
+  nsresult rv;
   if (mIsInitialized) {
+    rv = gQueryStringStripper->ManageObservers();
+    NS_ENSURE_SUCCESS(rv, rv);
     return NS_OK;
   }
   mIsInitialized = true;
 
   mListService = do_GetService("@mozilla.org/query-stripping-list-service;1");
   NS_ENSURE_TRUE(mListService, NS_ERROR_FAILURE);
+  rv = gQueryStringStripper->ManageObservers();
+  NS_ENSURE_SUCCESS(rv, rv);
+  return NS_OK;
+}
 
-  return mListService->RegisterAndRunObserver(gQueryStringStripper);
+// (Un)registers a QPS/Strip-on-share observer according to the QPS prefs states
+// and the strip-on-share pref state. This is called whenever one of the three
+// prefs changes, to ensure that we are not observing one of the lists although
+// the corresponding feature is not turned on.
+nsresult URLQueryStringStripper::ManageObservers() {
+  MOZ_ASSERT(mListService);
+  nsresult rv;
+  // Register QPS observer.
+  // We are not listening to QPS but the feature is on, register a listener.
+  if (!mObservingQPS) {
+    if (StaticPrefs::privacy_query_stripping_enabled() ||
+        StaticPrefs::privacy_query_stripping_enabled_pbmode()) {
+      rv = mListService->RegisterAndRunObserver(gQueryStringStripper);
+      NS_ENSURE_SUCCESS(rv, rv);
+      mObservingQPS = true;
+    }
+  } else {
+    // Unregister QPS observer.
+    // We are listening to QPS but the feature is off, unregister.
+    if (!StaticPrefs::privacy_query_stripping_enabled() &&
+        !StaticPrefs::privacy_query_stripping_enabled_pbmode()) {
+      // Clean up QPS lists.
+      mList.Clear();
+      mAllowList.Clear();
+      rv = mListService->UnregisterObserver(this);
+      NS_ENSURE_SUCCESS(rv, rv);
+      mObservingQPS = false;
+    }
+  }
+
+  // Register Strip on Share observer.
+  // We are not listening to strip-on-share but the feature is on, register an
+  // Observer.
+  if (!mObservingStripOnShare) {
+    if (StaticPrefs::privacy_query_stripping_strip_on_share_enabled()) {
+      rv = mListService->RegisterAndRunObserverStripOnShare(
+          gQueryStringStripper);
+      NS_ENSURE_SUCCESS(rv, rv);
+      mObservingStripOnShare = true;
+    }
+  } else {
+    // Unregister Strip on Share observer.
+    // We are listening to strip-on-share but the feature is off, unregister.
+    if (!StaticPrefs::privacy_query_stripping_strip_on_share_enabled()) {
+      // Clean up strip-on-share list
+      mStripOnShareMap.Clear();
+      rv = mListService->UnregisterStripOnShareObserver(this);
+      NS_ENSURE_SUCCESS(rv, rv);
+      mObservingStripOnShare = false;
+    }
+  }
+  return NS_OK;
 }
 
 nsresult URLQueryStringStripper::Shutdown() {
   if (!mIsInitialized) {
     return NS_OK;
   }
+  nsresult rv = gQueryStringStripper->ManageObservers();
+  NS_ENSURE_SUCCESS(rv, rv);
   mIsInitialized = false;
-
-  mList.Clear();
-  mAllowList.Clear();
-
-  MOZ_ASSERT(mListService);
-  mListService = do_GetService("@mozilla.org/query-stripping-list-service;1");
-
-  mListService->UnregisterObserver(this);
   mListService = nullptr;
-
   return NS_OK;
 }
 
@@ -256,6 +366,22 @@ URLQueryStringStripper::OnQueryStrippingListUpdate(
   return NS_OK;
 }
 
+NS_IMETHODIMP
+URLQueryStringStripper::OnStripOnShareUpdate(const nsTArray<nsString>& aArgs,
+                                             JSContext* aCx) {
+  for (const auto& ruleString : aArgs) {
+    dom::StripRule rule;
+    if (NS_WARN_IF(!rule.Init(ruleString))) {
+      // Skipping malformed rules
+      continue;
+    }
+    for (const auto& topLevelSite : rule.mTopLevelSites) {
+      mStripOnShareMap.InsertOrUpdate(NS_ConvertUTF16toUTF8(topLevelSite),
+                                      rule);
+    }
+  }
+  return NS_OK;
+}
 // static
 NS_IMETHODIMP
 URLQueryStringStripper::TestGetStripList(nsACString& aStripList) {

toolkit/components/antitracking/URLQueryStringStripper.h

@@ -10,9 +10,10 @@
 #include "nsIURLQueryStringStripper.h"
 #include "nsIURLQueryStrippingListService.h"
 #include "nsIObserver.h"
-
+#include "mozilla/dom/StripOnShareRuleBinding.h"
 #include "nsStringFwd.h"
 #include "nsTHashSet.h"
+#include "nsTHashMap.h"
 
 class nsIURI;
 
@@ -35,6 +36,7 @@ class URLQueryStringStripper final : public nsIObserver,
   ~URLQueryStringStripper() = default;
 
   static void OnPrefChange(const char* aPref, void* aData);
+  nsresult ManageObservers();
 
   [[nodiscard]] nsresult Init();
   [[nodiscard]] nsresult Shutdown();
@@ -50,7 +52,12 @@ class URLQueryStringStripper final : public nsIObserver,
   nsTHashSet<nsString> mList;
   nsTHashSet<nsCString> mAllowList;
   nsCOMPtr<nsIURLQueryStrippingListService> mListService;
+  nsTHashMap<nsCString, dom::StripRule> mStripOnShareMap;
   bool mIsInitialized;
+  // Indicates whether or not we currently have registered an observer
+  // for the QPS/strip-on-share list updates
+  bool mObservingQPS = false;
+  bool mObservingStripOnShare = false;
 };
 
 }  // namespace mozilla

toolkit/components/antitracking/URLQueryStrippingListService.sys.mjs

@@ -23,6 +23,22 @@ XPCOMUtils.defineLazyGetter(lazy, "logger", () => {
   });
 });
 
+// Lazy getter for the strip-on-share strip list.
+XPCOMUtils.defineLazyGetter(lazy, "StripOnShareList", async () => {
+  let response = await fetch(
+    "chrome://global/content/antitracking/StripOnShare.json"
+  );
+  if (!response.ok) {
+    lazy.logger.error(
+      "Error fetching strip-on-share strip list" + response.status
+    );
+    throw new Error(
+      "Error fetching strip-on-share strip list" + response.status
+    );
+  }
+  return response.json();
+});
+
 export class URLQueryStrippingListService {
   classId = Components.ID("{afff16f0-3fd2-4153-9ccd-c6d9abd879e4}");
   QueryInterface = ChromeUtils.generateQI(["nsIURLQueryStrippingListService"]);
@@ -37,6 +53,8 @@ export class URLQueryStrippingListService {
   constructor() {
     lazy.logger.debug("constructor");
     this.observers = new Set();
+    this.stripOnShareObservers = new Set();
+    this.stripOnShareParams = null;
     this.prefStripList = new Set();
     this.prefAllowList = new Set();
     this.remoteStripList = new Set();
@@ -145,6 +163,9 @@ export class URLQueryStrippingListService {
     Services.prefs.removeObserver(PREF_ALLOW_LIST_NAME, this);
   }
 
+  get hasObservers() {
+    return !this.observers.size && !this.stripOnShareObservers.size;
+  }
   _onRemoteSettingsUpdate(entries) {
     this.remoteStripList.clear();
     this.remoteAllowList.clear();
@@ -192,6 +213,7 @@ export class URLQueryStrippingListService {
     }
 
     this._notifyObservers();
+    this._notifyStripOnShareObservers();
   }
 
   _getListFromSharedData() {
@@ -231,6 +253,44 @@ export class URLQueryStrippingListService {
     }
   }
 
+  async _notifyStripOnShareObservers(observer) {
+    this.stripOnShareParams = await lazy.StripOnShareList;
+    if (!this.stripOnShareParams) {
+      lazy.logger.error("StripOnShare list is undefined");
+      return;
+    }
+    // Add the qps params to the global rules of the strip-on-share list.
+    let qpsParams = [...this.prefStripList, ...this.remoteStripList].map(
+      param => param.toLowerCase()
+    );
+    this.stripOnShareParams.global.queryParams.push(...qpsParams);
+    // Getting rid of duplicates.
+    this.stripOnShareParams.global.queryParams = [
+      ...new Set(this.stripOnShareParams.global.queryParams),
+    ];
+    // Build an array of StripOnShareRules.
+    let rules = Object.values(this.stripOnShareParams);
+    let stringifiedRules = [];
+    // We need to stringify the rules so later we can initialise WebIDL dictionaries from them.
+    // The dictionaries init call needs stringified json.
+    rules.forEach(rule => {
+      stringifiedRules.push(JSON.stringify(rule));
+    });
+
+    let observers = observer ? new Set([observer]) : this.stripOnShareObservers;
+
+    if (observers.size) {
+      lazy.logger.debug("_notifyStripOnShareObservers", {
+        observerCount: observers.size,
+        runObserverAfterRegister: observer != null,
+        stringifiedRules,
+      });
+    }
+    for (let obs of observers) {
+      obs.onStripOnShareUpdate(stringifiedRules);
+    }
+  }
+
   async registerAndRunObserver(observer) {
     lazy.logger.debug("registerAndRunObserver", {
       isInitialized: this.#isInitialized,
@@ -242,10 +302,30 @@ export class URLQueryStrippingListService {
     this._notifyObservers(observer);
   }
 
+  async registerAndRunObserverStripOnShare(observer) {
+    lazy.logger.debug("registerAndRunObserverStripOnShare", {
+      isInitialized: this.#isInitialized,
+      pendingInit: this.#pendingInit,
+    });
+
+    await this.#init();
+    this.stripOnShareObservers.add(observer);
+    await this._notifyStripOnShareObservers(observer);
+  }
+
   async unregisterObserver(observer) {
     this.observers.delete(observer);
 
-    if (!this.observers.size) {
+    if (this.hasObservers) {
+      lazy.logger.debug("Last observer unregistered, shutting down...");
+      await this.#shutdown();
+    }
+  }
+
+  async unregisterStripOnShareObserver(observer) {
+    this.stripOnShareObservers.delete(observer);
+
+    if (this.hasObservers) {
       lazy.logger.debug("Last observer unregistered, shutting down...");
       await this.#shutdown();
     }

toolkit/components/antitracking/data/StripOnShare.json

@@ -0,0 +1,97 @@
+{
+  "global": {
+    "queryParams": [
+      "utm_ad",
+      "utm_affiliate",
+      "utm_brand",
+      "utm_campaign",
+      "utm_campaign",
+      "utm_campaignid",
+      "utm_channel",
+      "utm_cid",
+      "utm_content",
+      "utm_content",
+      "utm_creative",
+      "utm_emcid",
+      "utm_emmid",
+      "utm_id",
+      "utm_id_",
+      "utm_keyword",
+      "utm_medium",
+      "utm_medium",
+      "utm_name",
+      "utm_place",
+      "utm_product",
+      "utm_pubreferrer",
+      "utm_reader",
+      "utm_referrer",
+      "utm_serial",
+      "utm_session",
+      "utm_siteid",
+      "utm_social",
+      "utm_social-type",
+      "utm_source",
+      "utm_source",
+      "utm_supplier",
+      "utm_swu",
+      "utm_term",
+      "utm_umguk",
+      "utm_userid",
+      "utm_viz_id",
+      "vero_conv",
+      "ymid",
+      "var",
+      "s_cid",
+      "hsa_grp",
+      "hsa_cam",
+      "hsa_src",
+      "hsa_ad",
+      "hsa_acc",
+      "hsa_kw",
+      "hsa_tgt",
+      "hsa_ver",
+      "hsa_la",
+      "hsa_ol",
+      "hsa_net",
+      "hsa_mt"
+    ],
+    "topLevelSites": ["*"]
+  },
+  "twitter": {
+    "queryParams": ["ref_src", "ref_url"],
+    "topLevelSites": ["www.twitter.com"]
+  },
+  "instagram": {
+    "queryParams": ["igshid", "ig_rid"],
+    "topLevelSites": ["www.instagram.com"]
+  },
+  "amazon": {
+    "queryParams": [
+      "keywords",
+      "pd_rd_r",
+      "pd_rd_w",
+      "pd_rd_wg",
+      "pf_rd_r",
+      "pf_rd_p",
+      "sr",
+      "content-id"
+    ],
+    "topLevelSites": [
+      "www.amazon.com",
+      "www.amazon.de",
+      "www.amazon.nl",
+      "www.amazon.fr",
+      "www.amazon.co.jp",
+      "www.amazon.in",
+      "www.amazon.es",
+      "www.amazon.ac",
+      "www.amazon.cn",
+      "www.amazon.eg",
+      "www.amazon.in",
+      "www.amazon.co.uk",
+      "www.amazon.it",
+      "www.amazon.pl",
+      "www.amazon.sg"
+    ]
+  }
+}

toolkit/components/antitracking/jar.mn

@@ -0,0 +1,12 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+toolkit.jar:
+  # This is the strip list that contains the query parameters stripped by the strip-on-share feature
+  # (In addition to this list, the strip-on-share feature also strips the query params from the qps list).
+  # The strip list has the format:
+  # domain1: {queryParams: [param1, param2, ..], topLevelSites: [www.site.de, www.site.com,...]}, domain2: {...}
+  # This list will be consumed from the nsIQueryStrippingListService and
+  # later be dispatched to the nsIURLQueryStringStripper in a further processed form.
+  content/global/antitracking/StripOnShare.json           (data/StripOnShare.json)

toolkit/components/antitracking/moz.build

@@ -7,6 +7,8 @@
 with Files("**"):
     BUG_COMPONENT = ("Core", "Privacy: Anti-Tracking")
 
+JAR_MANIFESTS += ["jar.mn"]
+
 XPIDL_SOURCES += [
     "nsIContentBlockingAllowList.idl",
     "nsIPartitioningExceptionListService.idl",

toolkit/components/antitracking/nsIURLQueryStrippingListService.idl

@@ -7,7 +7,7 @@
 /**
  * Observer for query stripping list updates.
  */
-[scriptable, function, uuid(ef56ae12-b1bb-43e6-b1d8-16459cb98dfd)]
+[scriptable, uuid(ef56ae12-b1bb-43e6-b1d8-16459cb98dfd)]
 interface nsIURLQueryStrippingListObserver : nsISupports
 {
   /**
@@ -23,6 +23,18 @@ interface nsIURLQueryStrippingListObserver : nsISupports
    *        stripping.
    */
   void onQueryStrippingListUpdate(in AString aStripList, in ACString aAllowList);
+
+  /**
+   * Called by nsIQueryStrippingListService when the list of query stripping
+   * parameters for strip-on-share feature is updated and when the observer is first registered.
+   *
+   * @param aStripRules
+   *        An Array of stringified strip rules.
+   *        A stringified rule has the form of:
+   *        "'queryParams': ['param1', 'param2', ...], 'topLevelSites': ['www.site.com', 'www.site.de', ...]"
+   */
+  [implicit_jscontext]
+  void onStripOnShareUpdate(in Array<AString> aStripRules);
 };
 
 /**
@@ -45,6 +57,20 @@ interface nsIURLQueryStrippingListService : nsISupports
    */
   void registerAndRunObserver(in nsIURLQueryStrippingListObserver aObserver);
 
+
+  /**
+   * Register a new observer to strip-on-share stripping list updates
+   * (this is the strip-on-share list combined with the QPS list).
+   * When the observer is registered it is called immediately once. Afterwards it will be called
+   * when there is an remote settings update to the QPS strip list.
+   *
+   * @param aObserver
+   *        An nsIURLQueryStrippingListObserver object or function that
+   *        will receive updates to the strip list and the allow list. Will be
+   *        called immediately with the current list value.
+   */
+  void registerAndRunObserverStripOnShare(in nsIURLQueryStrippingListObserver aObserver);
+
   /**
    * Unregister an observer.
    *
@@ -53,6 +79,14 @@ interface nsIURLQueryStrippingListService : nsISupports
    */
   void unregisterObserver(in nsIURLQueryStrippingListObserver aObserver);
 
+  /**
+   * Unregister an observer for strip-on-share.
+   *
+   * @param aObserver
+   *        The nsIURLQueryStrippingListObserver object to unregister.
+   */
+  void unregisterStripOnShareObserver(in nsIURLQueryStrippingListObserver aObserver);
+
   /**
    * Clear all Lists.
    *

toolkit/components/antitracking/test/browser/browser_urlQueryStrippingListService.js

@@ -25,6 +25,18 @@ const TEST_THIRD_PARTY_URI = TEST_DOMAIN_2 + TEST_PATH + "empty.html";
 // URLQueryStrippingListService. We need to use the event here so that the same
 // observer can be called multiple times.
 class UpdateEvent extends EventTarget {}
+
+// The Observer that is registered needs to implement onQueryStrippingListUpdate
+// like a nsIURLQueryStrippingListObserver does
+class ListObserver {
+  updateEvent = new UpdateEvent();
+
+  onQueryStrippingListUpdate(stripList, allowList) {
+    let event = new CustomEvent("update", { detail: { stripList, allowList } });
+    this.updateEvent.dispatchEvent(event);
+  }
+}
+
 function waitForEvent(element, eventName) {
   return BrowserTestUtils.waitForEvent(element, eventName).then(e => e.detail);
 }
@@ -100,12 +112,8 @@ add_task(async function testPrefSettings() {
   });
 
   // Test if the observer been called when adding to the service.
-  let updateEvent = new UpdateEvent();
-  let obs = (stripList, allowList) => {
-    let event = new CustomEvent("update", { detail: { stripList, allowList } });
-    updateEvent.dispatchEvent(event);
-  };
-  let promise = waitForEvent(updateEvent, "update");
+  let obs = new ListObserver();
+  let promise = waitForEvent(obs.updateEvent, "update");
   urlQueryStrippingListService.registerAndRunObserver(obs);
   let lists = await promise;
   is(lists.stripList, "", "No strip list at the beginning.");
@@ -116,7 +124,7 @@ add_task(async function testPrefSettings() {
   await check("pref_query2=456", "pref_query2=456");
 
   // Set pref for strip list
-  promise = waitForEvent(updateEvent, "update");
+  promise = waitForEvent(obs.updateEvent, "update");
   await SpecialPowers.pushPrefEnv({
     set: [["privacy.query_stripping.strip_list", "pref_query1 pref_query2"]],
   });
@@ -134,7 +142,7 @@ add_task(async function testPrefSettings() {
   await check("pref_query2=456", "");
 
   // Set the pref for allow list.
-  promise = waitForEvent(updateEvent, "update");
+  promise = waitForEvent(obs.updateEvent, "update");
   await SpecialPowers.pushPrefEnv({
     set: [["privacy.query_stripping.allow_list", "example.net"]],
   });
@@ -173,12 +181,8 @@ add_task(async function testRemoteSettings() {
   await db.importChanges({}, Date.now(), []);
 
   // Test if the observer been called when adding to the service.
-  let updateEvent = new UpdateEvent();
-  let obs = (stripList, allowList) => {
-    let event = new CustomEvent("update", { detail: { stripList, allowList } });
-    updateEvent.dispatchEvent(event);
-  };
-  let promise = waitForEvent(updateEvent, "update");
+  let obs = new ListObserver();
+  let promise = waitForEvent(obs.updateEvent, "update");
   urlQueryStrippingListService.registerAndRunObserver(obs);
   let lists = await promise;
   is(lists.stripList, "", "No strip list at the beginning.");
@@ -189,7 +193,7 @@ add_task(async function testRemoteSettings() {
   await check("remote_query2=456", "remote_query2=456");
 
   // Set record for strip list.
-  promise = waitForEvent(updateEvent, "update");
+  promise = waitForEvent(obs.updateEvent, "update");
   await RemoteSettings(COLLECTION_NAME).emit("sync", {
     data: {
       current: [
@@ -216,7 +220,7 @@ add_task(async function testRemoteSettings() {
   await check("remote_query2=456", "");
 
   // Set record for strip list and allow list.
-  promise = waitForEvent(updateEvent, "update");
+  promise = waitForEvent(obs.updateEvent, "update");
   await RemoteSettings(COLLECTION_NAME).emit("sync", {
     data: {
       current: [