Bug 1907783 - Use the new clear on shutdown branch to prevent losing pending shutdown items on unexpected shutdown. r=pbz,places-reviewers,Standard8

Differential Revision: https://phabricator.services.mozilla.com/D216616

browser/components/places/tests/unit/test_clearHistory_shutdown_v2.js

@@ -0,0 +1,181 @@
+/* -*- indent-tabs-mode: nil; js-indent-level: 2 -*- */
+/* vim:set ts=2 sw=2 sts=2 et: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * Tests that requesting clear history at shutdown will really clear history.
+ */
+
+const URIS = [
+  "http://a.example1.com/",
+  "http://b.example1.com/",
+  "http://b.example2.com/",
+  "http://c.example3.com/",
+];
+
+const FTP_URL = "ftp://localhost/clearHistoryOnShutdown/";
+
+const { Sanitizer } = ChromeUtils.importESModule(
+  "resource:///modules/Sanitizer.sys.mjs"
+);
+
+// Send the profile-after-change notification to the form history component to ensure
+// that it has been initialized.
+var formHistoryStartup = Cc[
+  "@mozilla.org/satchel/form-history-startup;1"
+].getService(Ci.nsIObserver);
+formHistoryStartup.observe(null, "profile-after-change", null);
+ChromeUtils.defineESModuleGetters(this, {
+  FormHistory: "resource://gre/modules/FormHistory.sys.mjs",
+});
+
+var timeInMicroseconds = Date.now() * 1000;
+
+add_task(async function test_execute() {
+  info("Initialize browserglue before Places");
+
+  // Avoid default bookmarks import.
+  let glue = Cc["@mozilla.org/browser/browserglue;1"].getService(
+    Ci.nsIObserver
+  );
+  glue.observe(null, "initial-migration-will-import-default-bookmarks", null);
+  Sanitizer.onStartup();
+
+  Services.prefs.setBoolPref(Sanitizer.PREF_SHUTDOWN_BRANCH + "cache", true);
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage",
+    true
+  );
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "historyFormDataAndDownloads",
+    true
+  );
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage",
+    true
+  );
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "siteSettings",
+    true
+  );
+
+  Services.prefs.setBoolPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN, true);
+
+  info("Add visits.");
+  for (let aUrl of URIS) {
+    await PlacesTestUtils.addVisits({
+      uri: uri(aUrl),
+      visitDate: timeInMicroseconds++,
+      transition: PlacesUtils.history.TRANSITION_TYPED,
+    });
+  }
+  info("Add cache.");
+  await storeCache(FTP_URL, "testData");
+  info("Add form history.");
+  await addFormHistory();
+  Assert.equal(await getFormHistoryCount(), 1, "Added form history");
+
+  info("Simulate and wait shutdown.");
+  await shutdownPlaces();
+
+  Assert.equal(await getFormHistoryCount(), 0, "Form history cleared");
+
+  let stmt = DBConn(true).createStatement(
+    "SELECT id FROM moz_places WHERE url = :page_url "
+  );
+
+  try {
+    URIS.forEach(function (aUrl) {
+      stmt.params.page_url = aUrl;
+      Assert.ok(!stmt.executeStep());
+      stmt.reset();
+    });
+  } finally {
+    stmt.finalize();
+  }
+
+  info("Check cache");
+  // Check cache.
+  await checkCache(FTP_URL);
+});
+
+function addFormHistory() {
+  let now = Date.now() * 1000;
+  return FormHistory.update({
+    op: "add",
+    fieldname: "testfield",
+    value: "test",
+    timesUsed: 1,
+    firstUsed: now,
+    lastUsed: now,
+  });
+}
+
+async function getFormHistoryCount() {
+  return FormHistory.count({ fieldname: "testfield" });
+}
+
+function storeCache(aURL, aContent) {
+  let cache = Services.cache2;
+  let storage = cache.diskCacheStorage(Services.loadContextInfo.default);
+
+  return new Promise(resolve => {
+    let storeCacheListener = {
+      onCacheEntryCheck() {
+        return Ci.nsICacheEntryOpenCallback.ENTRY_WANTED;
+      },
+
+      onCacheEntryAvailable(entry, isnew, status) {
+        Assert.equal(status, Cr.NS_OK);
+
+        entry.setMetaDataElement("servertype", "0");
+        var os = entry.openOutputStream(0, -1);
+
+        var written = os.write(aContent, aContent.length);
+        if (written != aContent.length) {
+          do_throw(
+            "os.write has not written all data!\n" +
+              "  Expected: " +
+              written +
+              "\n" +
+              "  Actual: " +
+              aContent.length +
+              "\n"
+          );
+        }
+        os.close();
+        resolve();
+      },
+    };
+
+    storage.asyncOpenURI(
+      Services.io.newURI(aURL),
+      "",
+      Ci.nsICacheStorage.OPEN_NORMALLY,
+      storeCacheListener
+    );
+  });
+}
+
+function checkCache(aURL) {
+  let cache = Services.cache2;
+  let storage = cache.diskCacheStorage(Services.loadContextInfo.default);
+
+  return new Promise(resolve => {
+    let checkCacheListener = {
+      onCacheEntryAvailable(entry, isnew, status) {
+        Assert.equal(status, Cr.NS_ERROR_CACHE_KEY_NOT_FOUND);
+        resolve();
+      },
+    };
+
+    storage.asyncOpenURI(
+      Services.io.newURI(aURL),
+      "",
+      Ci.nsICacheStorage.OPEN_READONLY,
+      checkCacheListener
+    );
+  });
+}

browser/components/places/tests/unit/xpcshell.toml

@@ -32,6 +32,10 @@ support-files = [
 ["test_browserGlue_restore.js"]
 
 ["test_clearHistory_shutdown.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=true"]
+
+["test_clearHistory_shutdown_v2.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=false"]
 
 ["test_interactions_blocklist.js"]
 

browser/modules/Sanitizer.sys.mjs

@@ -58,8 +58,15 @@ export var Sanitizer = {
    * Pref branches to fetch sanitization options from.
    */
   PREF_CPD_BRANCH: "privacy.cpd.",
-  PREF_SHUTDOWN_BRANCH: "privacy.clearOnShutdown.",
-  PREF_SHUTDOWN_V2_BRANCH: "privacy.clearOnShutdown_v2.",
+  /*
+   * We need to choose between two branches for shutdown since there are separate prefs for the new
+   * clear history dialog
+   */
+  get PREF_SHUTDOWN_BRANCH() {
+    return lazy.useOldClearHistoryDialog
+      ? "privacy.clearOnShutdown."
+      : "privacy.clearOnShutdown_v2.";
+  },
 
   /**
    * The fallback timestamp used when no argument is given to
@@ -1119,10 +1126,10 @@ async function sanitizeOnShutdown(progress) {
   if (Sanitizer.shouldSanitizeOnShutdown) {
     // Need to sanitize upon shutdown
     progress.advancement = "shutdown-cleaner";
-    let shutdownBranch = lazy.useOldClearHistoryDialog
-      ? Sanitizer.PREF_SHUTDOWN_BRANCH
-      : Sanitizer.PREF_SHUTDOWN_V2_BRANCH;
-    let itemsToClear = getItemsToClearFromPrefBranch(shutdownBranch);
+
+    let itemsToClear = getItemsToClearFromPrefBranch(
+      Sanitizer.PREF_SHUTDOWN_BRANCH
+    );
     await Sanitizer.sanitize(itemsToClear, { progress });
 
     // We didn't crash during shutdown sanitization, so annotate it to avoid

browser/modules/test/unit/test_Sanitizer_interrupted_v2.js

@@ -0,0 +1,144 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+do_get_profile();
+
+// Test that interrupted sanitizations are properly tracked.
+
+add_task(async function () {
+  const { Sanitizer } = ChromeUtils.importESModule(
+    "resource:///modules/Sanitizer.sys.mjs"
+  );
+
+  Services.prefs.setBoolPref(Sanitizer.PREF_NEWTAB_SEGREGATION, false);
+
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN);
+    Services.prefs.clearUserPref(
+      Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage"
+    );
+    Services.prefs.clearUserPref(Sanitizer.PREF_NEWTAB_SEGREGATION);
+  });
+  Services.prefs.setBoolPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN, true);
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage",
+    true
+  );
+
+  await Sanitizer.onStartup();
+  Assert.ok(Sanitizer.shouldSanitizeOnShutdown, "Should sanitize on shutdown");
+
+  let pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    1,
+    "Should have 1 pending sanitization"
+  );
+  Assert.equal(
+    pendingSanitizations[0].id,
+    "shutdown",
+    "Should be the shutdown sanitization"
+  );
+  Assert.ok(
+    pendingSanitizations[0].itemsToClear.includes("cookiesAndStorage"),
+    "Pref has been setup"
+  );
+  Assert.ok(
+    !pendingSanitizations[0].options.isShutdown,
+    "Shutdown option is not present"
+  );
+
+  // Check the preference listeners.
+  Services.prefs.setBoolPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN, false);
+  pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    0,
+    "Should not have pending sanitizations"
+  );
+  Assert.ok(
+    !Sanitizer.shouldSanitizeOnShutdown,
+    "Should not sanitize on shutdown"
+  );
+  Services.prefs.setBoolPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN, true);
+  pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    1,
+    "Should have 1 pending sanitization"
+  );
+  Assert.equal(
+    pendingSanitizations[0].id,
+    "shutdown",
+    "Should be the shutdown sanitization"
+  );
+
+  Assert.ok(
+    pendingSanitizations[0].itemsToClear.includes("cookiesAndStorage"),
+    "Pending sanitizations should include cookiesAndStorage"
+  );
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage",
+    false
+  );
+  pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    1,
+    "Should have 1 pending sanitization"
+  );
+  Assert.ok(
+    !pendingSanitizations[0].itemsToClear.includes("cookiesAndStorage"),
+    "Pending sanitizations should have been updated"
+  );
+
+  // Check a sanitization properly rebuilds the pref.
+  await Sanitizer.sanitize(["cookiesAndStorage"]);
+  pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    1,
+    "Should have 1 pending sanitization"
+  );
+  Assert.equal(
+    pendingSanitizations[0].id,
+    "shutdown",
+    "Should be the shutdown sanitization"
+  );
+
+  // Startup should run the pending one and setup a new shutdown sanitization.
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "cookiesAndStorage",
+    false
+  );
+  await Sanitizer.onStartup();
+  pendingSanitizations = JSON.parse(
+    Services.prefs.getStringPref(Sanitizer.PREF_PENDING_SANITIZATIONS, "[]")
+  );
+  Assert.equal(
+    pendingSanitizations.length,
+    1,
+    "Should have 1 pending sanitization"
+  );
+  Assert.equal(
+    pendingSanitizations[0].id,
+    "shutdown",
+    "Should be the shutdown sanitization"
+  );
+  Assert.ok(
+    !pendingSanitizations[0].itemsToClear.includes("cookiesAndStorage"),
+    "Pref has been setup"
+  );
+});

browser/modules/test/unit/xpcshell.toml

@@ -28,6 +28,10 @@ run-if = ["os == 'win'"] # Test of a Windows-specific feature
 run-if = ["os == 'win'"] # Test of a Windows-specific feature
 
 ["test_Sanitizer_interrupted.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=true"]
+
+["test_Sanitizer_interrupted_v2.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=false"]
 
 ["test_SiteDataManager.js"]
 

security/manager/ssl/tests/unit/test_sss_sanitizeOnShutdown_v2.js

@@ -0,0 +1,59 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// The purpose of this test is to ensure that Firefox sanitizes site security
+// service data on shutdown if configured to do so.
+
+ChromeUtils.defineESModuleGetters(this, {
+  Sanitizer: "resource:///modules/Sanitizer.sys.mjs",
+  TestUtils: "resource://testing-common/TestUtils.sys.mjs",
+});
+
+Sanitizer.onStartup();
+
+// This helps us away from test timed out. If service worker manager(swm) hasn't
+// been initilaized before profile-change-teardown, this test would fail due to
+// the shutdown blocker added by swm. Normally, swm should be initialized before
+// that and the similar crash signatures are fixed. So, assume this cannot
+// happen in the real world and initilaize swm here as a workaround.
+Cc["@mozilla.org/serviceworkers/manager;1"].getService(
+  Ci.nsIServiceWorkerManager
+);
+
+add_task(async function run_test() {
+  do_get_profile();
+  let SSService = Cc["@mozilla.org/ssservice;1"].getService(
+    Ci.nsISiteSecurityService
+  );
+  let header = "max-age=50000";
+  SSService.processHeader(Services.io.newURI("https://example.com"), header);
+  await TestUtils.waitForCondition(() => {
+    let stateFileContents = get_data_storage_contents(SSS_STATE_FILE_NAME);
+    return stateFileContents
+      ? stateFileContents.includes("example.com")
+      : false;
+  });
+
+  // Configure Firefox to clear this data on shutdown.
+  Services.prefs.setBoolPref(
+    Sanitizer.PREF_SHUTDOWN_BRANCH + "siteSettings",
+    true
+  );
+  Services.prefs.setBoolPref(Sanitizer.PREF_SANITIZE_ON_SHUTDOWN, true);
+
+  // Simulate shutdown.
+  Services.startup.advanceShutdownPhase(
+    Services.startup.SHUTDOWN_PHASE_APPSHUTDOWNTEARDOWN
+  );
+  Services.startup.advanceShutdownPhase(
+    Services.startup.SHUTDOWN_PHASE_APPSHUTDOWN
+  );
+
+  await TestUtils.waitForCondition(() => {
+    let stateFile = do_get_profile();
+    stateFile.append(SSS_STATE_FILE_NAME);
+    return !stateFile.exists();
+  });
+});

security/manager/ssl/tests/unit/xpcshell.toml

@@ -337,6 +337,17 @@ skip-if = ["condprof"]  # Bug 1769154 - as designed
 ["test_sss_resetState.js"]
 
 ["test_sss_sanitizeOnShutdown.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=true"]
+firefox-appdir = "browser"
+# Sanitization works differently on Android - this doesn't apply.
+# browser/modules/Sanitizer.sys.mjs used by the test isn't available in Thunderbird.
+skip-if = [
+  "os == 'android'",
+  "appname == 'thunderbird'"
+]
+
+["test_sss_sanitizeOnShutdown_v2.js"]
+prefs = ["privacy.sanitize.useOldClearHistoryDialog=false"]
 firefox-appdir = "browser"
 # Sanitization works differently on Android - this doesn't apply.
 # browser/modules/Sanitizer.sys.mjs used by the test isn't available in Thunderbird.