Bug 1795471 - Add metrics to understand why users do not use background update, r=nalexander

Add metrics to help understand how many users are not using background
update on Windows.

Differential Revision: https://phabricator.services.mozilla.com/D168754

browser/components/metrics.yaml

@@ -38,3 +38,43 @@ browser.launched_to_handle:
           launched to complete.
         type: string
     telemetry_mirror: BrowserLaunched_to_handle_SystemNotification_Toast
+
+background_update:
+  reasons_to_not_update:
+    type: string_list
+    description: >
+      Records which error was causing the background updater to fail.
+      This list supercedes the `background-update.reason` in
+      `mozapps/update/metrics.yaml`
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1795471
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1795471
+    data_sensitivity:
+      - technical
+    notification_emails:
+      - install-update@mozilla.com
+    expires: never
+    send_in_pings:
+      - background-update
+      - metrics
+    lifetime: application
+
+  time_last_update_scheduled:
+    type: datetime
+    time_unit: day
+    description: >
+      Last time the background update was triggered.
+    bugs:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1795471
+    data_reviews:
+      - https://bugzilla.mozilla.org/show_bug.cgi?id=1795471
+    data_sensitivity:
+      - interaction
+    notification_emails:
+      - install-update@mozilla.com
+    expires: never
+    send_in_pings:
+      - background-update
+      - metrics
+    lifetime: application

toolkit/mozapps/update/BackgroundUpdate.jsm

@@ -194,6 +194,8 @@ var BackgroundUpdate = {
       reasons.push(this.REASON.MANUAL_UPDATE_ONLY);
     }
 
+    this._recordGleanMetrics(reasons);
+
     return reasons;
   },
 
@@ -263,6 +265,8 @@ var BackgroundUpdate = {
       reasons.push(this.REASON.SERVICE_REGISTRY_KEY_MISSING);
     }
 
+    this._recordGleanMetrics(reasons);
+
     return reasons;
   },
 
@@ -390,6 +394,9 @@ var BackgroundUpdate = {
       `${SLUG}: checking eligibility before scheduling background update task`
     );
 
+    // datetime with an empty parameter records 'now'
+    Glean.backgroundUpdate.timeLastUpdateScheduled.set();
+
     let previousEnabled;
     let successfullyReadPrevious;
     try {
@@ -774,6 +781,35 @@ var BackgroundUpdate = {
 
     return defaultProfileTargetingSnapshot;
   },
+
+  /**
+   * Local helper function to record all reasons why the background updater is
+   * not used with Glean. This function will only track the first 20 reasons.
+   * It is also fault tolerant and will only display debug messages if the
+   * metric cannot be recorded for any reason.
+   *
+   * @param {array of strings} [reasons]
+   *        a list of BackgroundUpdate.REASON values (=> string)
+   */
+  async _recordGleanMetrics(reasons) {
+    // Record Glean metrics with all the reasons why the update was impossible.
+    for (const [key, value] of Object.entries(this.REASON)) {
+      if (reasons.includes(value)) {
+        try {
+          // `testGetValue` throws `NS_ERROR_LOSS_OF_SIGNIFICANT_DATA` in case
+          // of `InvalidOverflow` and other outstanding errors.
+          Glean.backgroundUpdate.reasonsToNotUpdate.testGetValue();
+          Glean.backgroundUpdate.reasonsToNotUpdate.add(key);
+        } catch (e) {
+          // Debug print an error message and break the loop to avoid Glean
+          // messages on the console would otherwise be caused by the add().
+          lazy.log.debug("Error recording reasonsToNotUpdate");
+          console.log("Error recording reasonsToNotUpdate");
+          break;
+        }
+      }
+    }
+  },
 };
 
 BackgroundUpdate.REASON = {

toolkit/mozapps/update/tests/unit_background_update/head.js

@@ -16,3 +16,43 @@ const do_backgroundtask = BackgroundTasksTestUtils.do_backgroundtask.bind(
 const setupProfileService = BackgroundTasksTestUtils.setupProfileService.bind(
   BackgroundTasksTestUtils
 );
+
+// Helper function to register a callback to catch a Glean ping before its
+// submission. The function returns all string_list items, but not as they
+// appear in the ping itself, but as full text representation, which is the
+// value of the corresponding field. This makes the test more unique, because
+// the values often contain chars, which are not allowed in glean metric labels
+//
+// @returns: an array which contains all glean metrics, but as full text
+//           representation from the BackgroundUpdate.REASON object => its
+//           values, see description for further details.
+//
+async function checkGleanPing() {
+  let retval = ["EMPTY"];
+  let ping_submitted = false;
+
+  const { maybeSubmitBackgroundUpdatePing } = ChromeUtils.import(
+    "resource://gre/modules/backgroundtasks/BackgroundTask_backgroundupdate.jsm"
+  );
+  const { BackgroundUpdate } = ChromeUtils.import(
+    "resource://gre/modules/BackgroundUpdate.jsm"
+  );
+
+  GleanPings.backgroundUpdate.testBeforeNextSubmit(_ => {
+    ping_submitted = true;
+    retval = Glean.backgroundUpdate.reasonsToNotUpdate.testGetValue().map(v => {
+      return BackgroundUpdate.REASON[v];
+    });
+    Assert.ok(Array.isArray(retval));
+    return retval;
+  });
+  await maybeSubmitBackgroundUpdatePing();
+  Assert.ok(ping_submitted, "Glean ping successfully submitted");
+
+  // The metric has `lifetime: application` set, but when testing we do not
+  // want to keep the results around and avoid, that one test can influence
+  // another. That is why we clear this string_list.
+  Glean.backgroundUpdate.reasonsToNotUpdate.set([]);
+
+  return retval;
+}

toolkit/mozapps/update/tests/unit_background_update/test_backgroundupdate_reason.js

@@ -0,0 +1,66 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
+ * vim: sw=2 ts=2 sts=2 et
+ * this source code form is subject to the terms of the mozilla public license,
+ * v. 2.0. if a copy of the mpl was not distributed with this file, you can
+ * obtain one at http://mozilla.org/mpl/2.0/. */
+
+"use strict";
+
+const { BackgroundUpdate } = ChromeUtils.import(
+  "resource://gre/modules/BackgroundUpdate.jsm"
+);
+
+// These tests use per-installation prefs, and those are a shared resource, so
+// they require some non-trivial setup.
+setupTestCommon(null);
+standardInit();
+
+add_setup(function test_setup() {
+  // FOG needs a profile directory to put its data in.
+  do_get_profile();
+
+  // We need to initialize it once, otherwise operations will be stuck in the
+  // pre-init queue.
+  Services.fog.initializeFOG();
+});
+
+// Because we want to use the keys from REASON as strings and send these with
+// Glean, we have to make sure, that they meet the requirements for `String
+// Lists` and are not too long.
+add_task(async function test_reasons_length() {
+  for (const key of Object.keys(BackgroundUpdate.REASON)) {
+    Glean.backgroundUpdate.reasonsToNotUpdate.add(key);
+    // No exception means success.
+    Assert.ok(
+      Array.isArray(Glean.backgroundUpdate.reasonsToNotUpdate.testGetValue()),
+      "Glean allows the name of the reason to be '" + key + "'"
+    );
+  }
+});
+
+// The string list in Glean can overflow and has a hard limit of 20 entries.
+// This test toggles a switch to reach this limit and fails if this causes an
+// exception, because we want to avoid that statistical data collection can have
+// an negative impact on the success rate of background updates.
+add_task(async function test_reasons_overflow() {
+  let prev = await UpdateUtils.getAppUpdateAutoEnabled();
+  try {
+    for (let i = 1; i <= 21; i++) {
+      await UpdateUtils.setAppUpdateAutoEnabled(false);
+      await BackgroundUpdate._reasonsToNotUpdateInstallation();
+      await UpdateUtils.setAppUpdateAutoEnabled(true);
+      await BackgroundUpdate._reasonsToNotUpdateInstallation();
+      Assert.ok(true, "Overflow test successful for run #" + i);
+    }
+  } finally {
+    ok(true, "resetting AppUpdateAutoEnabled to " + prev);
+    await UpdateUtils.setAppUpdateAutoEnabled(prev);
+  }
+});
+
+add_task(() => {
+  // `setupTestCommon()` calls `do_test_pending()`; this calls
+  // `do_test_finish()`.  The `add_task` schedules this to run after all the
+  // other tests have completed.
+  doTestFinish();
+});

toolkit/mozapps/update/tests/unit_background_update/test_backgroundupdate_reason_schedule.js

@@ -34,6 +34,16 @@ AddonTestUtils.createAppInfo(
   "42"
 );
 
+add_setup(function test_setup() {
+  // FOG needs a profile directory to put its data in.
+  do_get_profile();
+
+  // We need to initialize it once, otherwise operations will be stuck in the pre-init queue.
+  Services.fog.initializeFOG();
+
+  setupProfileService();
+});
+
 add_task(
   {
     skip_if: () => !AppConstants.MOZ_BACKGROUNDTASKS,
@@ -89,6 +99,11 @@ add_task(
       result.includes(REASON.LANGPACK_INSTALLED),
       "Reasons include LANGPACK_INSTALLED"
     );
+    result = await checkGleanPing();
+    Assert.ok(
+      result.includes(REASON.LANGPACK_INSTALLED),
+      "Recognizes a language pack is installed."
+    );
 
     // Now turn off langpack updating.
     Services.prefs.setBoolPref("app.update.langpack.enabled", false);
@@ -98,6 +113,11 @@ add_task(
       !result.includes(REASON.LANGPACK_INSTALLED),
       "Reasons does not include LANGPACK_INSTALLED"
     );
+    result = await checkGleanPing();
+    Assert.ok(
+      !result.includes(REASON.LANGPACK_INSTALLED),
+      "No Glean metric when no language pack is installed."
+    );
   }
 );
 

toolkit/mozapps/update/tests/unit_background_update/test_backgroundupdate_reason_update.js

@@ -47,16 +47,31 @@ async function setupPolicyEngineWithJson(json, customSchema) {
   return EnterprisePolicyTesting.setupPolicyEngineWithJson(json, customSchema);
 }
 
+add_setup(function test_setup() {
+  // FOG needs a profile directory to put its data in.
+  do_get_profile();
+
+  // We need to initialize it once, otherwise operations will be stuck in the pre-init queue.
+  Services.fog.initializeFOG();
+
+  setupProfileService();
+});
+
 add_task(async function test_reasons_update_no_app_update_auto() {
   let prev = await UpdateUtils.getAppUpdateAutoEnabled();
   try {
     await UpdateUtils.setAppUpdateAutoEnabled(false);
     let result = await reasons();
     Assert.ok(result.includes(REASON.NO_APP_UPDATE_AUTO));
+    result = await checkGleanPing();
+    Assert.ok(result.includes(REASON.NO_APP_UPDATE_AUTO));
 
     await UpdateUtils.setAppUpdateAutoEnabled(true);
     result = await reasons();
     Assert.ok(!result.includes(REASON.NO_APP_UPDATE_AUTO));
+
+    result = await checkGleanPing();
+    Assert.ok(!result.includes(REASON.NO_APP_UPDATE_AUTO));
   } finally {
     await UpdateUtils.setAppUpdateAutoEnabled(prev);
   }
@@ -73,6 +88,8 @@ add_task(async function test_reasons_update_no_app_update_background_enabled() {
     );
     let result = await reasons();
     Assert.ok(result.includes(REASON.NO_APP_UPDATE_BACKGROUND_ENABLED));
+    result = await checkGleanPing();
+    Assert.ok(result.includes(REASON.NO_APP_UPDATE_BACKGROUND_ENABLED));
 
     await UpdateUtils.writeUpdateConfigSetting(
       "app.update.background.enabled",
@@ -80,6 +97,8 @@ add_task(async function test_reasons_update_no_app_update_background_enabled() {
     );
     result = await reasons();
     Assert.ok(!result.includes(REASON.NO_APP_UPDATE_BACKGROUND_ENABLED));
+    result = await checkGleanPing();
+    Assert.ok(!result.includes(REASON.NO_APP_UPDATE_BACKGROUND_ENABLED));
   } finally {
     await UpdateUtils.writeUpdateConfigSetting(
       "app.update.background.enabled",
@@ -101,6 +120,8 @@ add_task(async function test_reasons_update_cannot_usually_check() {
       .get(() => false);
     result = await reasons();
     Assert.ok(result.includes(REASON.CANNOT_USUALLY_CHECK));
+    result = await checkGleanPing();
+    Assert.ok(result.includes(REASON.CANNOT_USUALLY_CHECK));
   } finally {
     sandbox.restore();
   }
@@ -121,6 +142,10 @@ add_task(async function test_reasons_update_can_usually_stage_or_appl() {
     Assert.ok(
       !result.includes(REASON.CANNOT_USUALLY_STAGE_AND_CANNOT_USUALLY_APPLY)
     );
+    result = await checkGleanPing();
+    Assert.ok(
+      !result.includes(REASON.CANNOT_USUALLY_STAGE_AND_CANNOT_USUALLY_APPLY)
+    );
 
     sandbox
       .stub(UpdateService.prototype, "canUsuallyStageUpdates")
@@ -132,6 +157,10 @@ add_task(async function test_reasons_update_can_usually_stage_or_appl() {
     Assert.ok(
       result.includes(REASON.CANNOT_USUALLY_STAGE_AND_CANNOT_USUALLY_APPLY)
     );
+    result = await checkGleanPing();
+    Assert.ok(
+      result.includes(REASON.CANNOT_USUALLY_STAGE_AND_CANNOT_USUALLY_APPLY)
+    );
   } finally {
     sandbox.restore();
   }
@@ -159,10 +188,17 @@ add_task(
       Services.prefs.setBoolPref("app.update.BITS.enabled", false);
       let result = await reasons();
       Assert.ok(result.includes(REASON.WINDOWS_CANNOT_USUALLY_USE_BITS));
+      result = await checkGleanPing();
+      Assert.ok(
+        result.includes(REASON.WINDOWS_CANNOT_USUALLY_USE_BITS),
+        "result : " + result.join("', '") + "']"
+      );
 
       Services.prefs.setBoolPref("app.update.BITS.enabled", true);
       result = await reasons();
       Assert.ok(!result.includes(REASON.WINDOWS_CANNOT_USUALLY_USE_BITS));
+      result = await checkGleanPing();
+      Assert.ok(!result.includes(REASON.WINDOWS_CANNOT_USUALLY_USE_BITS));
     } finally {
       sandbox.restore();
       Services.prefs.setBoolPref("app.update.BITS.enabled", prev);
@@ -184,11 +220,15 @@ add_task(async function test_reasons_update_manual_update_only() {
 
   let result = await reasons();
   Assert.ok(result.includes(REASON.MANUAL_UPDATE_ONLY));
+  result = await checkGleanPing();
+  Assert.ok(result.includes(REASON.MANUAL_UPDATE_ONLY));
 
   await setupPolicyEngineWithJson({});
 
   result = await reasons();
   Assert.ok(!result.includes(REASON.MANUAL_UPDATE_ONLY));
+  result = await checkGleanPing();
+  Assert.ok(!result.includes(REASON.MANUAL_UPDATE_ONLY));
 });
 
 add_task(() => {

toolkit/mozapps/update/tests/unit_background_update/xpcshell.ini

@@ -19,5 +19,6 @@ skip-if =
   os == "win" && os_version == "6.1" # Skip on Azure - frequent failure
 
 [test_backgroundupdate_glean.js]
+[test_backgroundupdate_reason.js]
 [test_backgroundupdate_reason_update.js]
 [test_backgroundupdate_reason_schedule.js]