Bug 1271313 - Measure the number of total URIs and unique domains visited in a "session fragment". r=gijs, data-review=rweiss

MozReview-Commit-ID: 4sYvZTcSM3u

browser/modules/BrowserUsageTelemetry.jsm

@@ -10,18 +10,26 @@ this.EXPORTED_SYMBOLS = ["BrowserUsageTelemetry"];
 const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
 
 Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils",
+                                  "resource://gre/modules/PrivateBrowsingUtils.jsm");
+
+// The upper bound for the count of the visited unique domain names.
+const MAX_UNIQUE_VISITED_DOMAINS = 100;
 
 // Observed topic names.
 const WINDOWS_RESTORED_TOPIC = "sessionstore-windows-restored";
 const TELEMETRY_SUBSESSIONSPLIT_TOPIC = "internal-telemetry-after-subsession-split";
 const DOMWINDOW_OPENED_TOPIC = "domwindowopened";
-const DOMWINDOW_CLOSED_TOPIC = "domwindowclosed";
 
 // Probe names.
 const MAX_TAB_COUNT_SCALAR_NAME = "browser.engagement.max_concurrent_tab_count";
 const MAX_WINDOW_COUNT_SCALAR_NAME = "browser.engagement.max_concurrent_window_count";
 const TAB_OPEN_EVENT_COUNT_SCALAR_NAME = "browser.engagement.tab_open_event_count";
 const WINDOW_OPEN_EVENT_COUNT_SCALAR_NAME = "browser.engagement.window_open_event_count";
+const UNIQUE_DOMAINS_COUNT_SCALAR_NAME = "browser.engagement.unique_domains_count";
+const TOTAL_URI_COUNT_SCALAR_NAME = "browser.engagement.total_uri_count";
 
 function getOpenTabsAndWinsCounts() {
   let tabCount = 0;
@@ -37,6 +45,59 @@ function getOpenTabsAndWinsCounts() {
   return { tabCount, winCount };
 }
 
+let URICountListener = {
+  // A set containing the visited domains, see bug 1271310.
+  _domainSet: new Set(),
+
+  onLocationChange(browser, webProgress, request, uri, flags) {
+    // Don't count this URI if it's an error page.
+    if (flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_ERROR_PAGE) {
+      return;
+    }
+
+    // We only care about top level loads.
+    if (!webProgress.isTopLevel) {
+      return;
+    }
+
+    // Only consider http(s) schemas.
+    if (!uri.schemeIs("http") && !uri.schemeIs("https")) {
+      return;
+    }
+
+    // Update the URI counts.
+    Services.telemetry.scalarAdd(TOTAL_URI_COUNT_SCALAR_NAME, 1);
+
+    // We only want to count the unique domains up to MAX_UNIQUE_VISITED_DOMAINS.
+    if (this._domainSet.size == MAX_UNIQUE_VISITED_DOMAINS) {
+      return;
+    }
+
+    // Unique domains should be aggregated by (eTLD + 1): x.test.com and y.test.com
+    // are counted once as test.com.
+    try {
+      // Even if only considering http(s) URIs, |getBaseDomain| could still throw
+      // due to the URI containing invalid characters or the domain actually being
+      // an ipv4 or ipv6 address.
+      this._domainSet.add(Services.eTLD.getBaseDomain(uri));
+    } catch (e) {
+      return;
+    }
+
+    Services.telemetry.scalarSet(UNIQUE_DOMAINS_COUNT_SCALAR_NAME, this._domainSet.size);
+  },
+
+  /**
+   * Reset the counts. This should be called when breaking a session in Telemetry.
+   */
+  reset() {
+    this._domainSet.clear();
+  },
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIWebProgressListener,
+                                         Ci.nsISupportsWeakReference]),
+};
+
 let BrowserUsageTelemetry = {
   init() {
     Services.obs.addObserver(this, WINDOWS_RESTORED_TOPIC, false);
@@ -52,11 +113,13 @@ let BrowserUsageTelemetry = {
     const counts = getOpenTabsAndWinsCounts();
     Services.telemetry.scalarSetMaximum(MAX_TAB_COUNT_SCALAR_NAME, counts.tabCount);
     Services.telemetry.scalarSetMaximum(MAX_WINDOW_COUNT_SCALAR_NAME, counts.winCount);
+
+    // Reset the URI counter.
+    URICountListener.reset();
   },
 
   uninit() {
     Services.obs.removeObserver(this, DOMWINDOW_OPENED_TOPIC, false);
-    Services.obs.removeObserver(this, DOMWINDOW_CLOSED_TOPIC, false);
     Services.obs.removeObserver(this, TELEMETRY_SUBSESSIONSPLIT_TOPIC, false);
     Services.obs.removeObserver(this, WINDOWS_RESTORED_TOPIC, false);
   },
@@ -69,9 +132,6 @@ let BrowserUsageTelemetry = {
       case DOMWINDOW_OPENED_TOPIC:
         this._onWindowOpen(subject);
         break;
-      case DOMWINDOW_CLOSED_TOPIC:
-        this._unregisterWindow(subject);
-        break;
       case TELEMETRY_SUBSESSIONSPLIT_TOPIC:
         this.afterSubsessionSplit();
         break;
@@ -83,6 +143,9 @@ let BrowserUsageTelemetry = {
       case "TabOpen":
         this._onTabOpen();
         break;
+      case "unload":
+        this._unregisterWindow(event.target);
+        break;
     }
   },
 
@@ -94,7 +157,6 @@ let BrowserUsageTelemetry = {
   _setupAfterRestore() {
     // Make sure to catch new chrome windows and subsession splits.
     Services.obs.addObserver(this, DOMWINDOW_OPENED_TOPIC, false);
-    Services.obs.addObserver(this, DOMWINDOW_CLOSED_TOPIC, false);
     Services.obs.addObserver(this, TELEMETRY_SUBSESSIONSPLIT_TOPIC, false);
 
     // Attach the tabopen handlers to the existing Windows.
@@ -113,20 +175,28 @@ let BrowserUsageTelemetry = {
    * Adds listeners to a single chrome window.
    */
   _registerWindow(win) {
+    win.addEventListener("unload", this);
     win.addEventListener("TabOpen", this, true);
+
+    // Don't include URI and domain counts when in private mode.
+    if (PrivateBrowsingUtils.isWindowPrivate(win)) {
+      return;
+    }
+    win.gBrowser.addTabsProgressListener(URICountListener);
   },
 
   /**
    * Removes listeners from a single chrome window.
    */
   _unregisterWindow(win) {
-    // Ignore non-browser windows.
+    win.removeEventListener("unload", this);
-    if (!(win instanceof Ci.nsIDOMWindow) ||
+    win.removeEventListener("TabOpen", this, true);
-        win.document.documentElement.getAttribute("windowtype") != "navigator:browser") {
+
+    // Don't include URI and domain counts when in private mode.
+    if (PrivateBrowsingUtils.isWindowPrivate(win.defaultView)) {
       return;
     }
-
+    win.defaultView.gBrowser.removeTabsProgressListener(URICountListener);
-    win.removeEventListener("TabOpen", this, true);
   },
 
   /**

browser/modules/test/browser_UsageTelemetry.js

@@ -4,9 +4,46 @@ const MAX_CONCURRENT_TABS = "browser.engagement.max_concurrent_tab_count";
 const TAB_EVENT_COUNT = "browser.engagement.tab_open_event_count";
 const MAX_CONCURRENT_WINDOWS = "browser.engagement.max_concurrent_window_count";
 const WINDOW_OPEN_COUNT = "browser.engagement.window_open_event_count";
+const TOTAL_URI_COUNT = "browser.engagement.total_uri_count";
+const UNIQUE_DOMAINS_COUNT = "browser.engagement.unique_domains_count";
 
 const TELEMETRY_SUBSESSION_TOPIC = "internal-telemetry-after-subsession-split";
 
+/**
+ * Waits for the web progress listener associated with this tab to fire an
+ * onLocationChange for a non-error page.
+ *
+ * @param {xul:browser} browser
+ *        A xul:browser.
+ *
+ * @return {Promise}
+ * @resolves When navigating to a non-error page.
+ */
+function browserLocationChanged(browser) {
+  return new Promise(resolve => {
+    let wpl = {
+      onStateChange() {},
+      onSecurityChange() {},
+      onStatusChange() {},
+      onLocationChange(aWebProgress, aRequest, aURI, aFlags) {
+        if (!(aFlags & Ci.nsIWebProgressListener.LOCATION_CHANGE_ERROR_PAGE)) {
+          browser.webProgress.removeProgressListener(filter);
+          filter.removeProgressListener(wpl);
+          resolve();
+        };
+      },
+      QueryInterface: XPCOMUtils.generateQI([
+        Ci.nsIWebProgressListener,
+        Ci.nsIWebProgressListener2,
+      ]),
+    };
+    const filter = Cc["@mozilla.org/appshell/component/browser-status-filter;1"]
+                     .createInstance(Ci.nsIWebProgress);
+    filter.addProgressListener(wpl, Ci.nsIWebProgress.NOTIFY_ALL);
+    browser.webProgress.addProgressListener(filter, Ci.nsIWebProgress.NOTIFY_ALL);
+  });
+};
+
 /**
  * An helper that checks the value of a scalar if it's expected to be > 0,
  * otherwise makes sure that the scalar it's not reported.
@@ -22,7 +59,7 @@ let checkScalar = (scalars, scalarName, value, msg) => {
 /**
  * Get a snapshot of the scalars and check them against the provided values.
  */
-let checkScalars = (maxTabs, tabOpenCount, maxWindows, windowsOpenCount) => {
+let checkScalars = (maxTabs, tabOpenCount, maxWindows, windowsOpenCount, totalURIs, domainCount) => {
   const scalars =
     Services.telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
 
@@ -35,6 +72,10 @@ let checkScalars = (maxTabs, tabOpenCount, maxWindows, windowsOpenCount) => {
               "The maximum window count must match the expected value.");
   checkScalar(scalars, WINDOW_OPEN_COUNT, windowsOpenCount,
               "The number of window open event count must match the expected value.");
+  checkScalar(scalars, TOTAL_URI_COUNT, totalURIs,
+              "The total URI count must match the expected value.");
+  checkScalar(scalars, UNIQUE_DOMAINS_COUNT, domainCount,
+              "The unique domains count must match the expected value.");
 };
 
 add_task(function* test_tabsAndWindows() {
@@ -51,14 +92,14 @@ add_task(function* test_tabsAndWindows() {
   openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(gBrowser, "about:blank"));
   expectedTabOpenCount = 1;
   expectedMaxTabs = 2;
-  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount);
+  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount, 0, 0);
 
   // Add two new tabs in the same window.
   openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(gBrowser, "about:blank"));
   openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(gBrowser, "about:blank"));
   expectedTabOpenCount += 2;
   expectedMaxTabs += 2;
-  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount);
+  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount, 0, 0);
 
   // Add a new window and then some tabs in it. An empty new windows counts as a tab.
   let win = yield BrowserTestUtils.openNewBrowserWindow();
@@ -73,7 +114,7 @@ add_task(function* test_tabsAndWindows() {
 
   // Remove a tab from the first window, the max shouldn't change.
   yield BrowserTestUtils.removeTab(openedTabs.pop());
-  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount);
+  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount, 0, 0);
 
   // Remove all the extra windows and tabs.
   for (let tab of openedTabs) {
@@ -82,7 +123,7 @@ add_task(function* test_tabsAndWindows() {
   yield BrowserTestUtils.closeWindow(win);
 
   // Make sure all the scalars still have the expected values.
-  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount);
+  checkScalars(expectedMaxTabs, expectedTabOpenCount, expectedMaxWins, expectedWinOpenCount, 0, 0);
 });
 
 add_task(function* test_subsessionSplit() {
@@ -94,10 +135,11 @@ add_task(function* test_subsessionSplit() {
   let openedTabs = [];
   openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(win.gBrowser, "about:blank"));
   openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(win.gBrowser, "about:blank"));
-  openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(win.gBrowser, "about:blank"));
+  openedTabs.push(yield BrowserTestUtils.openNewForegroundTab(win.gBrowser, "http://www.example.com"));
 
   // Check that the scalars have the right values.
-  checkScalars(5 /*maxTabs*/, 4 /*tabOpen*/, 2 /*maxWins*/, 1 /*winOpen*/);
+  checkScalars(5 /*maxTabs*/, 4 /*tabOpen*/, 2 /*maxWins*/, 1 /*winOpen*/,
+               1 /* toalURIs */, 1 /* uniqueDomains */);
 
   // Remove a tab.
   yield BrowserTestUtils.removeTab(openedTabs.pop());
@@ -111,7 +153,8 @@ add_task(function* test_subsessionSplit() {
   // After a subsession split, only the MAX_CONCURRENT_* scalars must be available
   // and have the correct value. No tabs or windows were opened so other scalars
   // must not be reported.
-  checkScalars(4 /*maxTabs*/, 0 /*tabOpen*/, 2 /*maxWins*/, 0 /*winOpen*/);
+  checkScalars(4 /*maxTabs*/, 0 /*tabOpen*/, 2 /*maxWins*/, 0 /*winOpen*/,
+               0 /* toalURIs */, 0 /* uniqueDomains */);
 
   // Remove all the extra windows and tabs.
   for (let tab of openedTabs) {
@@ -120,6 +163,86 @@ add_task(function* test_subsessionSplit() {
   yield BrowserTestUtils.closeWindow(win);
 });
 
+add_task(function* test_URIAndDomainCounts() {
+  // Let's reset the counts.
+  Services.telemetry.clearScalars();
+
+  let checkCounts = (URICount, domainCount) => {
+    // Get a snapshot of the scalars and then clear them.
+    const scalars =
+      Services.telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
+    checkScalar(scalars, TOTAL_URI_COUNT, URICount,
+                "The URI scalar must contain the expected value.");
+    checkScalar(scalars, UNIQUE_DOMAINS_COUNT, domainCount,
+                "The unique domains scalar must contain the expected value.");
+  };
+
+  // Check that about:blank doesn't get counted in the URI total.
+  let firstTab = yield BrowserTestUtils.openNewForegroundTab(gBrowser, "about:blank");
+  checkCounts(0, 0);
+
+  // Open a different page and check the counts.
+  yield BrowserTestUtils.loadURI(firstTab.linkedBrowser, "http://example.com/");
+  yield BrowserTestUtils.browserLoaded(firstTab.linkedBrowser);
+  checkCounts(1, 1);
+
+  // Activating a different tab must not increase the URI count.
+  let secondTab = yield BrowserTestUtils.openNewForegroundTab(gBrowser, "about:blank");
+  yield BrowserTestUtils.switchTab(gBrowser, firstTab);
+  checkCounts(1, 1);
+  yield BrowserTestUtils.removeTab(secondTab);
+
+  // Open a new window and set the tab to a new address.
+  let newWin = yield BrowserTestUtils.openNewBrowserWindow();
+  yield BrowserTestUtils.loadURI(newWin.gBrowser.selectedBrowser, "http://example.com/");
+  yield BrowserTestUtils.browserLoaded(newWin.gBrowser.selectedBrowser);
+  checkCounts(2, 1);
+
+  // We should not count AJAX requests.
+  const XHR_URL = "http://example.com/r";
+  yield ContentTask.spawn(newWin.gBrowser.selectedBrowser, XHR_URL, function(url) {
+    return new Promise(resolve => {
+      var xhr = new content.window.XMLHttpRequest();
+      xhr.open("GET", url);
+      xhr.onload = () => resolve();
+      xhr.send();
+    });
+  });
+  checkCounts(2, 1);
+
+  // Check that we're counting page fragments.
+  let loadingStopped = browserLocationChanged(newWin.gBrowser.selectedBrowser);
+  yield BrowserTestUtils.loadURI(newWin.gBrowser.selectedBrowser, "http://example.com/#2");
+  yield loadingStopped;
+  checkCounts(3, 1);
+
+  // Check test.domain.com and some.domain.com are only counted once unique.
+  yield BrowserTestUtils.loadURI(newWin.gBrowser.selectedBrowser, "http://test1.example.com/");
+  yield BrowserTestUtils.browserLoaded(newWin.gBrowser.selectedBrowser);
+  checkCounts(4, 1);
+
+  // Make sure that the unique domains counter is incrementing for a different domain.
+  yield BrowserTestUtils.loadURI(newWin.gBrowser.selectedBrowser, "https://example.org/");
+  yield BrowserTestUtils.browserLoaded(newWin.gBrowser.selectedBrowser);
+  checkCounts(5, 2);
+
+  // Check that we only account for top level loads (e.g. we don't count URIs from
+  // embedded iframes).
+  yield ContentTask.spawn(newWin.gBrowser.selectedBrowser, null, function* () {
+    let doc = content.document;
+    let iframe = doc.createElement("iframe");
+    let promiseIframeLoaded = ContentTaskUtils.waitForEvent(iframe, "load", false);
+    iframe.src = "https://example.org/test";
+    doc.body.insertBefore(iframe, doc.body.firstChild);
+    yield promiseIframeLoaded;
+  });
+  checkCounts(5, 2);
+
+  // Clean up.
+  yield BrowserTestUtils.removeTab(firstTab);
+  yield BrowserTestUtils.closeWindow(newWin);
+});
+
 add_task(function* test_privateMode() {
   // Let's reset the counts.
   Services.telemetry.clearScalars();
@@ -133,6 +256,8 @@ add_task(function* test_privateMode() {
   const scalars =
     Services.telemetry.snapshotScalars(Ci.nsITelemetry.DATASET_RELEASE_CHANNEL_OPTIN);
 
+  ok(!(TOTAL_URI_COUNT in scalars), "We should not track URIs in private mode.");
+  ok(!(UNIQUE_DOMAINS_COUNT in scalars), "We should not track unique domains in private mode.");
   is(scalars[TAB_EVENT_COUNT], 1, "The number of open tab event count must match the expected value.");
   is(scalars[MAX_CONCURRENT_TABS], 2, "The maximum tab count must match the expected value.");
   is(scalars[WINDOW_OPEN_COUNT], 1, "The number of window open event count must match the expected value.");

toolkit/components/telemetry/Scalars.yaml

@@ -121,3 +121,31 @@ browser.engagement:
     notification_emails:
       - rweiss@mozilla.com
     release_channel_collection: opt-out
+
+  total_uri_count:
+    bug_numbers:
+      - 1271313
+    description: >
+      The count of the total non-unique http(s) URIs visited in a subsession, including
+      page reloads, after the session has been restored. This does not include background
+      page requests and URIs from embedded pages or private browsing.
+    expires: "55"
+    kind: uint
+    notification_emails:
+      - rweiss@mozilla.com
+    release_channel_collection: opt-out
+
+  unique_domains_count:
+    bug_numbers:
+      - 1271310
+    description: >
+      The count of the unique domains visited in a subsession, after the session
+      has been restored. Subdomains under eTLD are aggregated after the first level
+      (i.e. test.example.com and other.example.com are only counted once).
+      This does not include background page requests and domains from embedded pages
+      or private browsing. The count is limited to 100 unique domains.
+    expires: "55"
+    kind: uint
+    notification_emails:
+      - rweiss@mozilla.com
+    release_channel_collection: opt-out