Bug 1889111 - Instrument Firefox Desktop UI Interactions with events r=TravisLong,kcochrane,Gijs

Assumption: Browser Usage Telemetry (ideally) records only and all interesting
interactions with Firefox Desktop's UI, and preserving syntax and semantics
when instrumenting using events is valuable.

Value this provides over existing keyed scalars:
* Order of operations (did three tabs open and then three tabs close, or did
  a single tab open-close three times?)
* Flow control (several atomic interactions combine to a user task. flow_id
  grouping allows us to see that easily in analysis. e.g. Open a tab, open
  prefs, privacy prefs, change a setting.)
* Glean

This is aiming for prototype quality and a prototype lifetime, to see if it's
worth investing more than just a week or two into.

Differential Revision: https://phabricator.services.mozilla.com/D207908

browser/base/content/browser-allTabsMenu.js

@@ -6,6 +6,7 @@
 /* eslint-env mozilla/browser-window */
 
 ChromeUtils.defineESModuleGetters(this, {
+  BrowserUsageTelemetry: "resource:///modules/BrowserUsageTelemetry.sys.mjs",
   TabsPanel: "resource:///modules/TabsList.sys.mjs",
 });
 
@@ -149,6 +150,10 @@ var gTabsPanel = {
         entrypoint,
         1
       );
+      BrowserUsageTelemetry.recordInteractionEvent(
+        entrypoint,
+        "all-tabs-panel-entrypoint"
+      );
       PanelUI.showSubView(
         this.kElements.allTabsView,
         this.allTabsButton,

browser/modules/BrowserUsageTelemetry.sys.mjs

@@ -160,6 +160,11 @@ const PLACES_OPEN_COMMANDS = [
   "placesCmd_open:tab",
 ];
 
+// How long of a delay between events means the start of a new flow?
+// Used by Browser UI Interaction event instrumentation.
+// Default: 5min.
+const FLOW_IDLE_TIME = 5 * 60 * 1000;
+
 function telemetryId(widgetId, obscureAddons = true) {
   // Add-on IDs need to be obscured.
   function addonId(id) {
@@ -872,6 +877,7 @@ export let BrowserUsageTelemetry = {
     let source = this._getWidgetContainer(node);
 
     if (item && source) {
+      this.recordInteractionEvent(item, source);
       let scalar = `browser.ui.interaction.${source.replace(/-/g, "_")}`;
       Services.telemetry.keyedScalarAdd(scalar, telemetryId(item), 1);
       if (SET_USAGECOUNT_PREF_BUTTONS.includes(item)) {
@@ -889,6 +895,7 @@ export let BrowserUsageTelemetry = {
         node.closest("menupopup")?.triggerNode
       );
       if (triggerContainer) {
+        this.recordInteractionEvent(item, contextMenu);
         let scalar = `browser.ui.interaction.${contextMenu.replace(/-/g, "_")}`;
         Services.telemetry.keyedScalarAdd(
           scalar,
@@ -899,6 +906,34 @@ export let BrowserUsageTelemetry = {
     }
   },
 
+  _flowId: null,
+  _flowIdTS: 0,
+
+  recordInteractionEvent(widgetId, source) {
+    // A note on clocks. Cu.now() is monotonic, but its behaviour across
+    // computer sleeps is different per platform.
+    // We're okay with this for flows because we're looking at idle times
+    // on the order of minutes and within the same machine, so the weirdest
+    // thing we may expect is a flow that accidentally continues across a
+    // sleep. Until we have evidence that this is common, we're in the clear.
+    if (!this._flowId || this._flowIdTS + FLOW_IDLE_TIME < Cu.now()) {
+      // We submit the ping full o' events on every new flow,
+      // including at startup.
+      GleanPings.prototypeNoCodeEvents.submit();
+      // We use a GUID here because we need to identify events in a flow
+      // out of all events from all flows across all clients.
+      this._flowId = Services.uuid.generateUUID();
+    }
+    this._flowIdTS = Cu.now();
+
+    const extra = {
+      source,
+      widgetId: telemetryId(widgetId),
+      flowId: this._flowId,
+    };
+    Glean.browserUsage.interaction.record(extra);
+  },
+
   /**
    * Listens for UI interactions in the window.
    */

browser/modules/metrics.yaml

@@ -118,3 +118,49 @@ performance.interaction:
       - mconley@mozilla.com
       - perf-telemetry-alerts@mozilla.com
     expires: never
+
+browser.usage:
+  interaction:
+    type: event
+    description: >
+      The user interacted with something in the Firefox Desktop frontend.
+      Could be via mouse or keyboard, could be a command or a UI element.
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1889111
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1889111
+    expires: 132
+    data_sensitivity: [interaction]
+    notification_emails:
+      - chutten@mozilla.com
+    extra_keys:
+      flow_id:
+        type: string
+        description: >
+          An UUIDv4 used to group interaction events together under the
+          assumption that they're part of the same user activity.
+          See BrowserUsageTelemetry's FLOW_IDLE_TIME for details.
+      source:
+        type: string
+        description: >
+          The source of the interaction. Usually a UI section
+          (like `bookmarks_bar` or `content_context`), but can also be an input
+          method (like `keyboard`).
+          The full list of supported `source`s can be found in
+          `BrowserUsageTelemetry`'s `BROWSER_UI_CONTAINER_IDS. Plus `keyboard`
+          and panes from `about:preferences` listed in `PREFERENCES_PANES`.
+          See `_getWidgetContainer` for details.
+      widget_id:
+        type: string
+        description: >
+          The item interacted with.
+          Usually the `id` of the DOM Node that the user used,
+          sometimes the `id` of the parent or ancestor Node instead.
+          This node is then conjugated by obscuring any addon id in it
+          (turning it to the string `addonX` where `X` is a number stable
+          within a browsing session) and then replacing any underscore with a
+          hyphen.
+          See `BrowserUsageTelemetry#_getWidgetID` and `telemetryId`.
+          e.g. "Browser:Reload", "key-newNavigatorTab", "PanelUI-Bookmarks".
+    send_in_pings:
+      - prototype-no-code-events

browser/modules/pings.yaml

@@ -0,0 +1,22 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+---
+$schema: moz://mozilla.org/schemas/glean/pings/2-0-0
+
+prototype-no-code-events:
+  description: |
+    **Prototype-only ping not for general use!**
+    Transport for no-code Firefox Desktop frontend instrumentation,
+    should mostly contain no-code events in browser.ui.* categories.
+    Submitted whenever the next flow of events begins (including startup).
+  include_client_id: true
+  bugs:
+    - https://bugzilla.mozilla.org/show_bug.cgi?id=1889111
+  data_reviews:
+    - https://bugzilla.mozilla.org/show_bug.cgi?id=1889111
+  notification_emails:
+    - chutten@mozilla.com
+    - tlong@mozilla.com
+  enabled: false # To be enabled by Server Knobs for selected populations.

browser/modules/test/browser/browser_UsageTelemetry_interaction.js

@@ -33,6 +33,8 @@ const AREAS = [
 // keys in the scalars. Also runs keyed scalar checks against non-area types
 // passed in through expectedOther.
 function assertInteractionScalars(expectedAreas, expectedOther = {}) {
+  // Every time this checks Scalars, it clears them. So clear FOG too.
+  Services.fog.testResetFOG();
   let processScalars =
     Services.telemetry.getSnapshotForKeyedScalars("main", true)?.parent ?? {};
 
@@ -83,6 +85,7 @@ add_task(async function toolbarButtons() {
     });
 
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     let newTab = await BrowserTestUtils.openNewForegroundTab(gBrowser);
     let tabClose = BrowserTestUtils.waitForTabClosing(newTab);
@@ -164,6 +167,22 @@ add_task(async function toolbarButtons() {
 
     click(customButton);
 
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "stop-reload-button"],
+        ["nav-bar", "back-button"],
+        ["nav-bar", "back-button"],
+        ["all-tabs-panel-entrypoint", "alltabs-button"],
+        ["tabs-bar", "alltabs-button"],
+        ["tabs-bar", "tab-close-button"],
+        ["bookmarks-bar", "bookmark-item"],
+        ["nav-bar", "12foo"],
+      ],
+      events
+    );
     assertInteractionScalars(
       {
         nav_bar: {
@@ -192,6 +211,7 @@ add_task(async function toolbarButtons() {
 add_task(async function contextMenu() {
   await BrowserTestUtils.withNewTab("https://example.com", async browser => {
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     let tab = gBrowser.getTabForBrowser(browser);
     let context = elem("tabContextMenu");
@@ -207,6 +227,16 @@ add_task(async function contextMenu() {
     context.activateItem(document.getElementById("context_toggleMuteTab"));
     await hidden;
 
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["tabs-context", "context-toggleMuteTab"],
+        ["tabs-context-entrypoint", "context-toggleMuteTab"],
+      ],
+      events
+    );
     assertInteractionScalars({
       tabs_context: {
         "context-toggleMuteTab": 1,
@@ -233,6 +263,16 @@ add_task(async function contextMenu() {
     );
     await hidden;
 
+    events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["tabs-context", "toolbar-context-selectAllTabs"],
+        ["tabs-context-entrypoint", "toolbar-context-selectAllTabs"],
+      ],
+      events
+    );
     assertInteractionScalars({
       tabs_context: {
         "toolbar-context-selectAllTabs": 1,
@@ -318,6 +358,7 @@ add_task(async function contextMenu_entrypoints() {
 add_task(async function appMenu() {
   await BrowserTestUtils.withNewTab("https://example.com", async () => {
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     let shown = BrowserTestUtils.waitForEvent(
       elem("appMenu-popup"),
@@ -339,9 +380,21 @@ add_task(async function appMenu() {
       nav_bar: {
         "PanelUI-menu-button": 1,
       },
-      app_menu: {},
+      app_menu: {
+        [findButtonID]: 1,
+      },
     };
-    expectedScalars.app_menu[findButtonID] = 1;
+
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "PanelUI-menu-button"],
+        ["app-menu", findButtonID],
+      ],
+      events
+    );
 
     assertInteractionScalars(expectedScalars);
   });
@@ -350,6 +403,7 @@ add_task(async function appMenu() {
 add_task(async function devtools() {
   await BrowserTestUtils.withNewTab("https://example.com", async () => {
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     let shown = BrowserTestUtils.waitForEvent(
       elem("appMenu-popup"),
@@ -381,6 +435,17 @@ add_task(async function devtools() {
     BrowserTestUtils.removeTab(tab);
 
     // Note that item ID's have '_' converted to '-'.
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "PanelUI-menu-button"],
+        ["app-menu", "appMenu-more-button2"],
+        ["app-menu", "key-viewSource"],
+      ],
+      events
+    );
     assertInteractionScalars({
       nav_bar: {
         "PanelUI-menu-button": 1,
@@ -398,6 +463,7 @@ add_task(async function webextension() {
 
   await BrowserTestUtils.withNewTab("https://example.com", async browser => {
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     function background() {
       browser.commands.onCommand.addListener(() => {
@@ -470,6 +536,11 @@ add_task(async function webextension() {
     // As the first add-on interacted with this should show up as `addon0`.
 
     click("random_addon_example_com-browser-action");
+    let events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["nav-bar", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       nav_bar: {
         addon0: 1,
@@ -482,6 +553,11 @@ add_task(async function webextension() {
     );
 
     click("pageAction-urlbar-random_addon_example_com");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["pageaction-urlbar", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       pageaction_urlbar: {
         addon0: 1,
@@ -490,6 +566,11 @@ add_task(async function webextension() {
 
     EventUtils.synthesizeKey("j", { altKey: true, shiftKey: true });
     await extension.awaitMessage("oncommand");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["keyboard", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       keyboard: {
         addon0: 1,
@@ -498,6 +579,11 @@ add_task(async function webextension() {
 
     EventUtils.synthesizeKey("q", { altKey: true, shiftKey: true });
     await extension.awaitMessage("sidebar-opened");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["keyboard", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       keyboard: {
         addon0: 1,
@@ -537,6 +623,11 @@ add_task(async function webextension() {
     // A second extension should be `addon1`.
 
     click("random_addon2_example_com-browser-action");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["nav-bar", "addon1"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       nav_bar: {
         addon1: 1,
@@ -549,6 +640,11 @@ add_task(async function webextension() {
     );
 
     click("pageAction-urlbar-random_addon2_example_com");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["pageaction-urlbar", "addon1"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       pageaction_urlbar: {
         addon1: 1,
@@ -557,6 +653,11 @@ add_task(async function webextension() {
 
     EventUtils.synthesizeKey("9", { altKey: true, shiftKey: true });
     await extension2.awaitMessage("oncommand");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["keyboard", "addon1"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       keyboard: {
         addon1: 1,
@@ -565,6 +666,11 @@ add_task(async function webextension() {
 
     // The first should have retained its ID.
     click("random_addon_example_com-browser-action");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["nav-bar", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       nav_bar: {
         addon0: 1,
@@ -573,6 +679,11 @@ add_task(async function webextension() {
 
     EventUtils.synthesizeKey("j", { altKey: true, shiftKey: true });
     await extension.awaitMessage("oncommand");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["keyboard", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       keyboard: {
         addon0: 1,
@@ -580,6 +691,11 @@ add_task(async function webextension() {
     });
 
     click("pageAction-urlbar-random_addon_example_com");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["pageaction-urlbar", "addon0"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       pageaction_urlbar: {
         addon0: 1,
@@ -595,6 +711,14 @@ add_task(async function webextension() {
     // The second should retain its ID.
     click("random_addon2_example_com-browser-action");
     click("random_addon2_example_com-browser-action");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [
+        ["nav-bar", "addon1"],
+        ["nav-bar", "addon1"],
+      ],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       nav_bar: {
         addon1: 2,
@@ -602,6 +726,11 @@ add_task(async function webextension() {
     });
 
     click("pageAction-urlbar-random_addon2_example_com");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["pageaction-urlbar", "addon1"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       pageaction_urlbar: {
         addon1: 1,
@@ -610,6 +739,11 @@ add_task(async function webextension() {
 
     EventUtils.synthesizeKey("9", { altKey: true, shiftKey: true });
     await extension2.awaitMessage("oncommand");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["keyboard", "addon1"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       keyboard: {
         addon1: 1,
@@ -643,6 +777,11 @@ add_task(async function webextension() {
     await shown;
 
     click("random_addon3_example_com-browser-action");
+    events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["unified-extensions-area", "addon2"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       unified_extensions_area: {
         addon2: 1,
@@ -669,6 +808,7 @@ add_task(async function mainMenu() {
 
   await BrowserTestUtils.withNewTab("https://example.com", async () => {
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     CustomizableUI.setToolbarVisibility("toolbar-menubar", true);
 
@@ -686,6 +826,11 @@ add_task(async function mainMenu() {
     click("menu_selectAll");
     await hidden;
 
+    let events = Glean.browserUsage.interaction.testGetValue();
+    Assert.deepEqual(
+      [["menu-bar", "menu-selectAll"]],
+      events.map(e => [e.extra.source, e.extra.widget_id])
+    );
     assertInteractionScalars({
       menu_bar: {
         // Note that the _ is replaced with - for telemetry identifiers.
@@ -706,6 +851,7 @@ add_task(async function preferences() {
     await finalPrefPaneLoaded;
 
     Services.telemetry.getSnapshotForKeyedScalars("main", true);
+    Services.fog.testResetFOG();
 
     await BrowserTestUtils.synthesizeMouseAtCenter(
       "#browserRestoreSession",
@@ -742,6 +888,16 @@ add_task(async function preferences() {
     await onLearnMoreOpened;
     gBrowser.removeCurrentTab();
 
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["preferences_paneGeneral", "browserRestoreSession"],
+        ["preferences_panePrivacy", "contentBlockingLearnMore"],
+      ],
+      events
+    );
     assertInteractionScalars({
       preferences_paneGeneral: {
         browserRestoreSession: 1,
@@ -806,6 +962,17 @@ async function history_appMenu(useContextClick) {
 
       app_menu: { "history-item": 1, "appMenu-history-button": 1 },
     };
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "PanelUI-menu-button"],
+        ["app-menu", "appMenu-history-button"],
+        ["app-menu", "history-item"],
+      ],
+      events
+    );
     assertInteractionScalars(expectedScalars);
   });
 }
@@ -852,6 +1019,17 @@ async function bookmarks_appMenu(useContextClick) {
 
       app_menu: { "bookmark-item": 1, "appMenu-bookmarks-button": 1 },
     };
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "PanelUI-menu-button"],
+        ["app-menu", "appMenu-bookmarks-button"],
+        ["app-menu", "bookmark-item"],
+      ],
+      events
+    );
     assertInteractionScalars(expectedScalars);
   });
 }
@@ -893,6 +1071,17 @@ async function bookmarks_library_navbar(useContextClick) {
         "appMenu-library-bookmarks-button": 1,
       },
     };
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "library-button"],
+        ["nav-bar", "appMenu-library-bookmarks-button"],
+        ["nav-bar", "bookmark-item"],
+      ],
+      events
+    );
     assertInteractionScalars(expectedScalars);
   });
 
@@ -940,6 +1129,17 @@ async function history_library_navbar(useContextClick) {
         "appMenu-library-history-button": 1,
       },
     };
+    let events = Glean.browserUsage.interaction
+      .testGetValue()
+      .map(e => [e.extra.source, e.extra.widget_id]);
+    Assert.deepEqual(
+      [
+        ["nav-bar", "library-button"],
+        ["nav-bar", "appMenu-library-history-button"],
+        ["nav-bar", "history-item"],
+      ],
+      events
+    );
     assertInteractionScalars(expectedScalars);
   });
 

toolkit/components/glean/metrics_index.py

@@ -130,6 +130,7 @@ firefox_desktop_pings = [
     "browser/components/pocket/pings.yaml",
     "browser/components/search/pings.yaml",
     "browser/components/urlbar/pings.yaml",
+    "browser/modules/pings.yaml",
     "toolkit/components/crashes/pings.yaml",
     "toolkit/components/resistfingerprinting/pings.yaml",
     "toolkit/components/telemetry/pings.yaml",