Bug 1717068 - Use newer blocklist dump if available r=Gijs

Differential Revision: https://phabricator.services.mozilla.com/D118739

services/settings/test/unit/xpcshell.ini

@@ -9,6 +9,7 @@ skip-if = appname == "thunderbird" # Bug 1662758 - these tests don't pass with a
 [test_attachments_downloader.js]
 support-files = test_attachments_downloader/**
 [test_remote_settings.js]
+[test_remote_settings_dump_lastmodified.js]
 [test_remote_settings_offline.js]
 [test_remote_settings_poll.js]
 [test_remote_settings_worker.js]

toolkit/mozapps/extensions/Blocklist.jsm

@@ -1008,7 +1008,7 @@ this.ExtensionBlocklistMLBF = {
         this._stashes = null;
         return;
       }
-      let records = await this._client.get();
+      let records = await this._client.get({ loadDumpIfNewer: true });
       if (isUpdateReplaced()) {
         return;
       }

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/head.js

@@ -7,7 +7,8 @@ const { ComponentUtils } = ChromeUtils.import(
 
 const MLBF_RECORD = {
   id: "A blocklist entry that refers to a MLBF file",
-  last_modified: 1,
+  // Higher than any last_modified in addons-bloomfilters.json:
+  last_modified: Date.now(),
   attachment: {
     size: 32,
     hash: "6af648a5d6ce6dbee99b0aab1780d24d204977a6606ad670d5372ef22fac1052",

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js

@@ -38,14 +38,18 @@ async function sha256(arrayBuffer) {
   return Array.from(new Uint8Array(hash), toHex).join("");
 }
 
-add_task(async function verify_dump_first_run() {
+// A list of { inputRecord, downloadPromise }:
+// - inputRecord is the record that was used for looking up the MLBF.
+// - downloadPromise is the result of trying to download it.
+const observed = [];
+
+add_task(async function setup() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
+  ExtensionBlocklistMLBF.ensureInitialized();
 
   // Tapping into the internals of ExtensionBlocklistMLBF._fetchMLBF to observe
   // MLBF request details.
-  const observed = [];
 
-  ExtensionBlocklistMLBF.ensureInitialized();
   // Despite being called "download", this does not actually access the network
   // when there is a valid dump.
   const originalImpl = ExtensionBlocklistMLBF._client.attachments.download;
@@ -55,6 +59,10 @@ add_task(async function verify_dump_first_run() {
     return downloadPromise;
   };
 
+  await promiseStartupManager();
+});
+
+async function verifyBlocklistWorksWithDump() {
   Assert.equal(
     await Blocklist.getAddonBlocklistState(blockedAddon),
     Ci.nsIBlocklistService.STATE_BLOCKED,
@@ -65,10 +73,13 @@ add_task(async function verify_dump_first_run() {
     Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
     "A known non-blocked add-on should not be blocked"
   );
+}
 
+add_task(async function verify_dump_first_run() {
+  await verifyBlocklistWorksWithDump();
   Assert.equal(observed.length, 1, "expected number of MLBF download requests");
 
-  const { inputRecord, downloadPromise } = observed[0];
+  const { inputRecord, downloadPromise } = observed.pop();
 
   Assert.ok(inputRecord, "addons-bloomfilters collection dump exists");
 
@@ -94,3 +105,52 @@ add_task(async function verify_dump_first_run() {
     "The content of the attachment should actually matches the record"
   );
 });
+
+add_task(async function use_dump_fallback_when_collection_is_out_of_sync() {
+  await AddonTestUtils.loadBlocklistRawData({
+    // last_modified higher than any value in addons-bloomfilters.json.
+    extensionsMLBF: [{ last_modified: Date.now() }],
+  });
+  Assert.equal(observed.length, 1, "Expected new download on update");
+
+  const { inputRecord, downloadPromise } = observed.pop();
+  Assert.equal(inputRecord, null, "No MLBF record found");
+
+  const downloadResult = await downloadPromise;
+  Assert.equal(
+    downloadResult._source,
+    "dump_fallback",
+    "should have used fallback despite the absence of a MLBF record"
+  );
+
+  await verifyBlocklistWorksWithDump();
+  Assert.equal(observed.length, 0, "Blocklist uses cached result");
+});
+
+// Verifies that the dump would supersede local data. This can happen after an
+// application upgrade, where the local database contains outdated records from
+// a previous application version.
+add_task(async function verify_dump_supersedes_old_dump() {
+  // Delete in-memory value; otherwise the cached record from the previous test
+  // task would be re-used and nothing would be downloaded.
+  delete ExtensionBlocklistMLBF._mlbfData;
+
+  await AddonTestUtils.loadBlocklistRawData({
+    // last_modified lower than any value in addons-bloomfilters.json.
+    extensionsMLBF: [{ last_modified: 1 }],
+  });
+  Assert.equal(observed.length, 1, "Expected new download on update");
+
+  const { inputRecord, downloadPromise } = observed.pop();
+  Assert.ok(inputRecord, "should have read from addons-bloomfilters dump");
+
+  const downloadResult = await downloadPromise;
+  Assert.equal(
+    downloadResult._source,
+    "dump_match",
+    "Should have replaced outdated collection records with dump"
+  );
+
+  await verifyBlocklistWorksWithDump();
+  Assert.equal(observed.length, 0, "Blocklist uses cached result");
+});

toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_telemetry.js

@@ -115,7 +115,13 @@ add_task(async function test_without_stashes() {
 // Test what happens when the collection was inadvertently emptied,
 // but still with a cached mlbf from before.
 add_task(async function test_without_collection_but_cache() {
-  await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: [] });
+  await AddonTestUtils.loadBlocklistRawData({
+    // Insert a dummy record with a value of last_modified which is higher than
+    // any value of last_modified in addons-bloomfilters.json, to prevent the
+    // blocklist implementation from automatically falling back to the packaged
+    // JSON dump.
+    extensionsMLBF: [{ last_modified: Date.now() }],
+  });
   assertTelemetryScalars({
     "blocklist.mlbf_enabled": true,
     "blocklist.mlbf_source": "cache_fallback",