Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-28 07:13:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out 8 changesets (bug 1620621) for XPCshell failures in xpcshell/rs-blocklist/test_blocklist_mlbf_fetch.js

Browse Source 
Backed out changeset 06fccc75c09e (bug 1620621)
Backed out changeset 8b4e286967c0 (bug 1620621)
Backed out changeset cfde27a748fd (bug 1620621)
Backed out changeset 7abf836343be (bug 1620621)
Backed out changeset 1a28d1de8f76 (bug 1620621)
Backed out changeset 90c08438be0a (bug 1620621)
Backed out changeset 723a3b4e7bbf (bug 1620621)
Backed out changeset bbc991f09d5d (bug 1620621)
This commit is contained in:
Dorel Luca 2020-04-30 05:44:28 +03:00
parent 257084d3fa
commit 58186c88dc
43 changed files with 11 additions and 1856 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.eslintrc.js
View File

@ -932,9 +932,6 @@ module.exports = {
"toolkit/mozapps/extensions/test/browser/browser_gmpProvider.js",
"toolkit/mozapps/extensions/test/xpcshell/head_addons.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_clients.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_fetch.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_update.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_regexp_split.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_targetapp_filter.js",
"toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_telemetry.js",

13
Cargo.lock generated
View File

@ -485,18 +485,6 @@ dependencies = [
"ppv-lite86",
]
[[package]]
name = "cascade_bloom_filter"
version = "0.1.0"
dependencies = [
"nserror",
"nsstring",
"rental",
"rust_cascade",
"thin-vec",
"xpcom",
]
[[package]]
name = "cast"
version = "0.2.2"
@ -1824,7 +1812,6 @@ dependencies = [
"authenticator",
"bitsdownload",
"bookmark_sync",
"cascade_bloom_filter",
"cert_storage",
"chardetng_c",
"cose-c",

6
modules/libpref/init/all.js
View File

@ -2329,15 +2329,12 @@ pref("extensions.abuseReport.amoDetailsURL", "https://services.addons.mozilla.or
// Blocklist preferences
pref("extensions.blocklist.enabled", true);
pref("extensions.blocklist.useMLBF", false);
pref("extensions.blocklist.useMLBF.stashes", false);
// Required blocklist freshness for OneCRL OCSP bypass (default is 30 hours)
// Note that this needs to exceed the interval at which we update OneCRL data,
// configured in services.settings.poll_interval .
pref("security.onecrl.maximum_staleness_in_seconds", 108000);
pref("extensions.blocklist.detailsURL", "https://blocked.cdn.mozilla.net/");
pref("extensions.blocklist.itemURL", "https://blocked.cdn.mozilla.net/%blockID%.html");
pref("extensions.blocklist.addonItemURL", "https://addons.mozilla.org/%LOCALE%/%APP%/blocked-addon/%addonID%/%addonVersion%/");
// Controls what level the blocklist switches from warning about items to forcibly
// blocking them.
pref("extensions.blocklist.level", 2);
@ -2346,9 +2343,6 @@ pref("services.blocklist.bucket", "blocklists");
pref("services.blocklist.addons.collection", "addons");
pref("services.blocklist.addons.checked", 0);
pref("services.blocklist.addons.signer", "remote-settings.content-signature.mozilla.org");
pref("services.blocklist.addons-mlbf.collection", "addons-bloomfilters");
pref("services.blocklist.addons-mlbf.checked", 0);
pref("services.blocklist.addons-mlbf.signer", "remote-settings.content-signature.mozilla.org");
pref("services.blocklist.plugins.collection", "plugins");
pref("services.blocklist.plugins.checked", 0);
pref("services.blocklist.plugins.signer", "remote-settings.content-signature.mozilla.org");

14
services/common/docs/RemoteSettings.rst
View File

@ -152,25 +152,11 @@ The provided helper will:
This will allow us to package attachments as part of a Firefox release (see `Bug 1542177 <https://bugzilla.mozilla.org/show_bug.cgi?id=1542177>`_)
and return them to calling code as ``resource://`` from within a package archive.
.. note::
By default, the ``download()`` method is prone to leaving extraneous files in the profile directory
(see `Bug 1634127 <https://bugzilla.mozilla.org/show_bug.cgi?id=1634127>`_).
Pass the ``useCache`` option to use an IndexedDB-based cache, and unlock the following features:
The ``fallbackToCache`` option allows callers to fall back to the cached file and record, if the requested record's attachment fails to download.
This enables callers to always have a valid pair of attachment and record,
provided that the attachment has been retrieved at least once.
The ``fallbackToDump`` option activates a fallback to a dump that has been
packaged with the client, when other ways to load the attachment have failed.
.. note::
A ``downloadAsBytes()`` method returning an ``ArrayBuffer`` is also available, if writing the attachment into the user profile is not necessary.
.. _services/initial-data:
Initial data

197
services/settings/Attachments.jsm
View File

@ -44,173 +44,6 @@ class Downloader {
this._cdnURL = null;
}
/**
* @returns {Object} An object with async "get", "set" and "delete" methods.
* The keys are strings, the values may be any object that
* can be stored in IndexedDB (including Blob).
*/
get cacheImpl() {
throw new Error("This Downloader does not support caching");
}
/**
* Download attachment and return the result together with the record.
* If the requested record cannot be downloaded and fallbacks are enabled, the
* returned attachment may have a different record than the input record.
*
* @param {Object} record A Remote Settings entry with attachment.
* If omitted, the attachmentId option must be set.
* @param {Object} options Some download options.
* @param {Number} options.retries Number of times download should be retried (default: `3`)
* @param {Number} options.checkHash Check content integrity (default: `true`)
* @param {string} options.attachmentId The attachment identifier to use for
* caching and accessing the attachment.
* (default: record.id)
* @param {Boolean} options.useCache Whether to use a cache to read and store
* the attachment. (default: false)
* @param {Boolean} options.fallbackToCache Return the cached attachment when the
* input record cannot be fetched.
* (default: false)
* @param {Boolean} options.fallbackToDump Use the remote settings dump as a
* potential source of the attachment.
* (default: false)
* @throws {Downloader.DownloadError} if the file could not be fetched.
* @throws {Downloader.BadContentError} if the downloaded content integrity is not valid.
* @returns {Object} An object with two properties:
* buffer: ArrayBuffer with the file content.
* record: Record associated with the bytes.
* _source: identifies the source of the result. Used for testing.
*/
async download(record, options) {
let {
retries,
checkHash,
attachmentId = record?.id,
useCache = false,
fallbackToCache = false,
fallbackToDump = false,
} = options || {};
if (!useCache) {
// For backwards compatibility.
// WARNING: Its return type is different from what's documented.
// See downloadToDisk's documentation.
return this.downloadToDisk(record, options);
}
if (!this.cacheImpl) {
throw new Error("useCache is true but there is no cacheImpl!");
}
if (!attachmentId) {
// Check for pre-condition. This should not happen, but it is explicitly
// checked to avoid mixing up attachments, which could be dangerous.
throw new Error("download() was called without attachmentId or recordID");
}
let buffer, cachedRecord;
if (useCache) {
try {
let cached = await this.cacheImpl.get(attachmentId);
if (cached) {
cachedRecord = cached.record;
buffer = await cached.blob.arrayBuffer();
}
} catch (e) {
Cu.reportError(e);
}
}
if (buffer && record) {
const { size, hash } = cachedRecord.attachment;
if (
record.attachment.size === size &&
record.attachment.hash === hash &&
(await RemoteSettingsWorker.checkContentHash(buffer, size, hash))
) {
// Best case: File already downloaded and still up to date.
return { buffer, record: cachedRecord, _source: "cache_match" };
}
}
let errorIfAllFails;
// No cached attachment available. Check if an attachment is available in
// the dump that is packaged with the client.
let dumpInfo;
if (fallbackToDump && record) {
try {
dumpInfo = await this._readAttachmentDump(attachmentId);
// Check if there is a match. If there is no match, we will fall through
// to the next case (downloading from the network). We may still use the
// dump at the end of the function as the ultimate fallback.
if (record.attachment.hash === dumpInfo.record.attachment.hash) {
return {
buffer: await dumpInfo.readBuffer(),
record: dumpInfo.record,
_source: "dump_match",
};
}
} catch (e) {
fallbackToDump = false;
errorIfAllFails = e;
}
}
// There is no local version that matches the requested record.
// Try to download the attachment specified in record.
if (record && record.attachment) {
try {
const newBuffer = await this.downloadAsBytes(record, {
retries,
checkHash,
});
const blob = new Blob([newBuffer]);
if (useCache) {
// Caching is optional, don't wait for the cache before returning.
this.cacheImpl
.set(attachmentId, { record, blob })
.catch(e => Cu.reportError(e));
}
return { buffer: newBuffer, record, _source: "remote_match" };
} catch (e) {
// No network, corrupted content, etc.
errorIfAllFails = e;
}
}
// Unable to find an attachment that matches the record. Consider falling
// back to local versions, even if their attachment hash do not match the
// one from the requested record.
if (buffer && fallbackToCache) {
const { size, hash } = cachedRecord.attachment;
if (await RemoteSettingsWorker.checkContentHash(buffer, size, hash)) {
return { buffer, record: cachedRecord, _source: "cache_fallback" };
}
}
// Unable to find a valid attachment, fall back to the packaged dump.
if (fallbackToDump) {
try {
dumpInfo = dumpInfo || (await this._readAttachmentDump(attachmentId));
return {
buffer: await dumpInfo.readBuffer(),
record: dumpInfo.record,
_source: "dump_fallback",
};
} catch (e) {
errorIfAllFails = e;
}
}
if (errorIfAllFails) {
throw errorIfAllFails;
}
throw new Downloader.DownloadError(attachmentId);
}
/**
* Download the record attachment into the local profile directory
* and return a file:// URL that points to the local path.
@ -224,7 +57,7 @@ class Downloader {
* @throws {Downloader.BadContentError} if the downloaded file integrity is not valid.
* @returns {String} the absolute file path to the downloaded attachment.
*/
async downloadToDisk(record, options = {}) {
async download(record, options = {}) {
const { retries = 3 } = options;
const {
attachment: { filename, size, hash },
@ -328,10 +161,6 @@ class Downloader {
await this._rmDirs();
}
async deleteCached(attachmentId) {
return this.cacheImpl.delete(attachmentId);
}
async _baseAttachmentsURL() {
if (!this._cdnURL) {
const server = Services.prefs.getCharPref("services.settings.server");
@ -358,30 +187,6 @@ class Downloader {
return resp.arrayBuffer();
}
async _readAttachmentDump(attachmentId) {
async function fetchResource(resourceUrl) {
try {
return await fetch(resourceUrl);
} catch (e) {
throw new Downloader.DownloadError(resourceUrl);
}
}
const resourceUrlPrefix =
Downloader._RESOURCE_BASE_URL + "/" + this.folders.join("/") + "/";
const recordUrl = `${resourceUrlPrefix}${attachmentId}.meta.json`;
const attachmentUrl = `${resourceUrlPrefix}${attachmentId}`;
const record = await (await fetchResource(recordUrl)).json();
return {
record,
async readBuffer() {
return (await fetchResource(attachmentUrl)).arrayBuffer();
},
};
}
// Separate variable to allow tests to override this.
static _RESOURCE_BASE_URL = "resource://app/defaults";
async _makeDirs() {
const dirPath = OS.Path.join(
OS.Constants.Path.localProfileDir,

44
services/settings/Database.jsm
View File

@ -19,7 +19,7 @@ var EXPORTED_SYMBOLS = ["Database"];
// IndexedDB name.
const DB_NAME = "remote-settings";
const DB_VERSION = 3;
const DB_VERSION = 2;
/**
* Wrap IndexedDB errors to catch them more easily.
@ -239,42 +239,6 @@ class Database {
}
}
async getAttachment(attachmentId) {
let entry = null;
try {
await executeIDB(
"attachments",
store => {
store.get([this.identifier, attachmentId]).onsuccess = e => {
entry = e.target.result;
};
},
{ mode: "readonly" }
);
} catch (e) {
throw new IndexedDBError(e, "getAttachment()", this.identifier);
}
return entry ? entry.attachment : null;
}
async saveAttachment(attachmentId, attachment) {
try {
await executeIDB(
"attachments",
store => {
if (attachment) {
store.put({ cid: this.identifier, attachmentId, attachment });
} else {
store.delete([this.identifier, attachmentId]);
}
},
{ desc: "saveAttachment(" + attachmentId + ") in " + this.identifier }
);
} catch (e) {
throw new IndexedDBError(e, "saveAttachment()", this.identifier);
}
}
async clear() {
try {
await this.saveLastModified(null);
@ -402,12 +366,6 @@ async function openIDB(callback) {
keyPath: "cid",
});
}
if (event.oldVersion < 3) {
// Attachment store
db.createObjectStore("attachments", {
keyPath: ["cid", "attachmentId"],
});
}
};
request.onerror = event =>
reject(new IndexedDBError(event.target.error, "open()"));

16
services/settings/RemoteSettingsClient.jsm
View File

@ -145,22 +145,6 @@ class AttachmentDownloader extends Downloader {
this._client = client;
}
get cacheImpl() {
const cacheImpl = {
get: async attachmentId => {
return this._client.db.getAttachment(attachmentId);
},
set: async (attachmentId, attachment) => {
return this._client.db.saveAttachment(attachmentId, attachment);
},
delete: async attachmentId => {
return this._client.db.saveAttachment(attachmentId, null);
},
};
Object.defineProperty(this, "cacheImpl", { value: cacheImpl });
return cacheImpl;
}
/**
* Download attachment and report Telemetry on failure.
*

2
services/settings/RemoteSettingsWorker.js
View File

@ -17,7 +17,7 @@ importScripts(
);
const IDB_NAME = "remote-settings";
const IDB_VERSION = 3;
const IDB_VERSION = 2;
const IDB_RECORDS_STORE = "records";
const IDB_TIMESTAMPS_STORE = "timestamps";

1
services/settings/dumps/blocklists/addons-bloomfilters.json
View File

@ -1 +0,0 @@
{"data":[{"schema":1588016498560,"attachment":{"hash":"164992bd106fd2c4cb039f8c1b2581f1d5f2c8ecc1635a2aa69efd10fd2dd7fd","size":65411,"filename":"filter.bin","location":"staging/addons-bloomfilters/1db2b4c3-3608-4657-a66c-18a26e16d2d4.bin","mimetype":"application/octet-stream"},"key_format":"{guid}:{version}","attachment_type":"bloomfilter-base","generation_time":1588098908496,"id":"7b10f7bb-aa73-4733-933b-f03f3cabd5f6","last_modified":1588099019245}]}

BIN
services/settings/dumps/blocklists/addons-bloomfilters/addons-mlbf.bin
View File

Binary file not shown.

1
services/settings/dumps/blocklists/addons-bloomfilters/addons-mlbf.bin.meta.json
View File

@ -1 +0,0 @@
{"schema":1588016498560,"attachment":{"hash":"164992bd106fd2c4cb039f8c1b2581f1d5f2c8ecc1635a2aa69efd10fd2dd7fd","size":65411,"filename":"filter.bin","location":"staging/addons-bloomfilters/1db2b4c3-3608-4657-a66c-18a26e16d2d4.bin","mimetype":"application/octet-stream"},"key_format":"{guid}:{version}","attachment_type":"bloomfilter-base","generation_time":1588098908496,"id":"7b10f7bb-aa73-4733-933b-f03f3cabd5f6","last_modified":1588099019245}

8
services/settings/dumps/blocklists/moz.build
View File

@ -7,15 +7,9 @@
with Files('**'):
BUG_COMPONENT = ('Toolkit', 'Blocklist Implementation')
FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons-bloomfilters.json',
'addons.json',
FINAL_TARGET_FILES.defaults.settings.blocklists += ['addons.json',
'gfx.json',
'plugins.json']
FINAL_TARGET_FILES.defaults.settings.blocklists['addons-bloomfilters'] += [
'addons-bloomfilters/addons-mlbf.bin',
'addons-bloomfilters/addons-mlbf.bin.meta.json'
]
if CONFIG['MOZ_BUILD_APP'] == 'browser':
DIST_SUBDIR = 'browser'

172
services/settings/test/unit/test_attachments_downloader.js
View File

@ -27,16 +27,6 @@ const RECORD = {
},
};
const RECORD_OF_DUMP = {
id: "filename-of-dump.txt",
attachment: {
filename: "filename-of-dump.txt",
hash: "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
size: 25,
},
some_key: "some metadata",
};
let downloader;
let server;
@ -306,165 +296,3 @@ add_task(async function test_downloader_reports_offline_error() {
});
});
add_task(clear_state);
add_task(async function test_download_cached() {
const client = RemoteSettings("main", "some-collection");
const attachmentId = "dummy filename";
const badRecord = {
attachment: {
...RECORD.attachment,
hash: "non-matching hash",
location: "non-existing-location-should-fail.bin",
},
};
async function downloadWithCache(record, options) {
options = { ...options, useCache: true };
return client.attachments.download(record, options);
}
function checkInfo(downloadResult, expectedSource, msg) {
Assert.deepEqual(
downloadResult.record,
RECORD,
`${msg} : expected identical record`
);
// Simple check: assume that content is identical if the size matches.
Assert.equal(
downloadResult.buffer.byteLength,
RECORD.attachment.size,
`${msg} : expected buffer`
);
Assert.equal(
downloadResult._source,
expectedSource,
`${msg} : expected source of the result`
);
}
await Assert.rejects(
downloadWithCache(null, { attachmentId }),
/DownloadError: Could not download dummy filename/,
"Download without record or cache should fail."
);
// Populate cache.
const info1 = await downloadWithCache(RECORD, { attachmentId });
checkInfo(info1, "remote_match", "first time download");
await Assert.rejects(
downloadWithCache(null, { attachmentId }),
/DownloadError: Could not download dummy filename/,
"Download without record still fails even if there is a cache."
);
await Assert.rejects(
downloadWithCache(badRecord, { attachmentId }),
/DownloadError: Could not download .*non-existing-location-should-fail.bin/,
"Download with non-matching record still fails even if there is a cache."
);
// Download from cache.
const info2 = await downloadWithCache(RECORD, { attachmentId });
checkInfo(info2, "cache_match", "download matching record from cache");
const info3 = await downloadWithCache(RECORD, {
attachmentId,
fallbackToCache: true,
});
checkInfo(info3, "cache_match", "fallbackToCache accepts matching record");
const info4 = await downloadWithCache(null, {
attachmentId,
fallbackToCache: true,
});
checkInfo(info4, "cache_fallback", "fallbackToCache accepts null record");
const info5 = await downloadWithCache(badRecord, {
attachmentId,
fallbackToCache: true,
});
checkInfo(info5, "cache_fallback", "fallbackToCache ignores bad record");
// Bye bye cache.
await client.attachments.deleteCached(attachmentId);
await Assert.rejects(
downloadWithCache(null, { attachmentId, fallbackToCache: true }),
/DownloadError: Could not download dummy filename/,
"Download without cache should fail again."
);
await Assert.rejects(
downloadWithCache(badRecord, { attachmentId, fallbackToCache: true }),
/DownloadError: Could not download .*non-existing-location-should-fail.bin/,
"Download should fail to fall back to a download of a non-existing record"
);
});
add_task(clear_state);
add_task(async function test_download_from_dump() {
const bucketNamePref = "services.testing.custom-bucket-name-in-this-test";
Services.prefs.setCharPref(bucketNamePref, "dump-bucket");
const client = RemoteSettings("dump-collection", { bucketNamePref });
// Temporarily replace the resource:-URL with another resource:-URL.
const orig_RESOURCE_BASE_URL = Downloader._RESOURCE_BASE_URL;
Downloader._RESOURCE_BASE_URL = "resource://rs-downloader-test";
const resProto = Services.io
.getProtocolHandler("resource")
.QueryInterface(Ci.nsIResProtocolHandler);
resProto.setSubstitution(
"rs-downloader-test",
Services.io.newFileURI(do_get_file("test_attachments_downloader"))
);
function checkInfo(result, expectedSource) {
Assert.equal(
new TextDecoder().decode(new Uint8Array(result.buffer)),
"This would be a RS dump.\n",
"expected content from dump"
);
Assert.deepEqual(result.record, RECORD_OF_DUMP, "expected record for dump");
Assert.equal(result._source, expectedSource, "expected source of dump");
}
// If record matches, should happen before network request.
const dump1 = await client.attachments.download(RECORD_OF_DUMP, {
// Note: attachmentId not set, so should fall back to record.id.
useCache: true,
fallbackToDump: true,
});
checkInfo(dump1, "dump_match");
// If no record given, should try network first, but then fall back to dump.
const dump2 = await client.attachments.download(null, {
attachmentId: RECORD_OF_DUMP.id,
useCache: true,
fallbackToDump: true,
});
checkInfo(dump2, "dump_fallback");
await Assert.rejects(
client.attachments.download(null, {
attachmentId: "filename-without-meta.txt",
useCache: true,
fallbackToDump: true,
}),
/Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-meta\.txt/,
"Cannot download dump that lacks a .meta.json file"
);
await Assert.rejects(
client.attachments.download(null, {
attachmentId: "filename-without-content.txt",
useCache: true,
fallbackToDump: true,
}),
/Could not download resource:\/\/rs-downloader-test\/settings\/dump-bucket\/dump-collection\/filename-without-content\.txt/,
"Cannot download dump that is missing, despite the existing .meta.json"
);
// Restore, just in case.
Downloader._RESOURCE_BASE_URL = orig_RESOURCE_BASE_URL;
resProto.setSubstitution("rs-downloader-test", null);
});
// Not really needed because the last test doesn't modify the main collection,
// but added for consistency with other tests tasks around here.
add_task(clear_state);

1
services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt
View File

@ -1 +0,0 @@
This would be a RS dump.

9
services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-of-dump.txt.meta.json
View File

@ -1,9 +0,0 @@
{
"id": "filename-of-dump.txt",
"attachment": {
"filename": "filename-of-dump.txt",
"hash": "4c46ef7e4f1951d210fe54c21e07c09bab265fd122580083ed1d6121547a8c6b",
"size": 25
},
"some_key": "some metadata"
}

9
services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-content.meta.json
View File

@ -1,9 +0,0 @@
{
"fyi": "This .meta.json file describes an attachment, but that attachment is missing.",
"attachment": {
"filename": "filename-without-content.txt",
"hash": "...",
"size": "..."
}
}

1
services/settings/test/unit/test_attachments_downloader/settings/dump-bucket/dump-collection/filename-without-meta.txt
View File

@ -1 +0,0 @@
The filename-without-meta.txt.meta.json file is missing.

2
services/settings/test/unit/xpcshell.ini
View File

@ -9,8 +9,6 @@ support-files =
support-files = test_attachments_downloader/**
[test_remote_settings.js]
[test_remote_settings_poll.js]
skip-if = true # bug 1634203
[test_remote_settings_worker.js]
skip-if = true # bug 1634203
[test_remote_settings_jexl_filters.js]
[test_remote_settings_signatures.js]

12
toolkit/components/cascade_bloom_filter/Cargo.toml
View File

@ -1,12 +0,0 @@
[package]
name = "cascade_bloom_filter"
version = "0.1.0"
authors = ["Rob Wu <rob@robwu.nl>"]
[dependencies]
nserror = { path = "../../../xpcom/rust/nserror" }
nsstring = { path = "../../../xpcom/rust/nsstring" }
rental = "0.5.5"
rust_cascade = "0.6.0"
thin-vec = { version = "0.1.0", features = ["gecko-ffi"] }
xpcom = { path = "../../../xpcom/rust/xpcom" }

25
toolkit/components/cascade_bloom_filter/CascadeFilter.cpp
View File

@ -1,25 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsCOMPtr.h"
#include "CascadeFilter.h"
namespace {
extern "C" {
// Implemented in Rust.
void cascade_filter_construct(nsICascadeFilter** aResult);
}
} // namespace
namespace mozilla {
already_AddRefed<nsICascadeFilter> ConstructCascadeFilter() {
nsCOMPtr<nsICascadeFilter> filter;
cascade_filter_construct(getter_AddRefs(filter));
return filter.forget();
}
} // namespace mozilla

17
toolkit/components/cascade_bloom_filter/CascadeFilter.h
View File

@ -1,17 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#ifndef CASCADE_BLOOM_FILTER_CASCADE_FILTER_H_
#define CASCADE_BLOOM_FILTER_CASCADE_FILTER_H_
#include "nsICascadeFilter.h"
#include "nsCOMPtr.h"
namespace mozilla {
already_AddRefed<nsICascadeFilter> ConstructCascadeFilter();
} // namespace mozilla
#endif // CASCADE_BLOOM_FILTER_CASCADE_FILTER_H_

15
toolkit/components/cascade_bloom_filter/components.conf
View File

@ -1,15 +0,0 @@
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
Classes = [
{
'cid': '{c8d0b0b3-17f8-458b-9264-7b67b288fe79}',
'contract_ids': ['@mozilla.org/cascade-filter;1'],
'type': 'nsICascadeFilter',
'headers': ['mozilla/CascadeFilter.h'],
'constructor': 'mozilla::ConstructCascadeFilter',
},
]

30
toolkit/components/cascade_bloom_filter/moz.build
View File

@ -1,30 +0,0 @@
# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
# vim: set filetype=python:
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
with Files('**'):
BUG_COMPONENT = ('Toolkit', 'Blocklist Implementation')
XPCSHELL_TESTS_MANIFESTS += ['test/xpcshell/xpcshell.ini']
XPIDL_SOURCES += [
'nsICascadeFilter.idl',
]
XPIDL_MODULE = 'cascade_bindings'
EXPORTS.mozilla += [
"CascadeFilter.h"
]
SOURCES += [
"CascadeFilter.cpp"
]
XPCOM_MANIFESTS += [
'components.conf',
]
FINAL_LIBRARY = 'xul'

28
toolkit/components/cascade_bloom_filter/nsICascadeFilter.idl
View File

@ -1,28 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
#include "nsISupports.idl"
/**
* A consumer of a filter cascade, i.e. a cascaded bloom filter as generated by
* https://github.com/mozilla/filter-cascade
*/
[scriptable, uuid(c8d0b0b3-17f8-458b-9264-7b67b288fe79)]
interface nsICascadeFilter : nsISupports {
/**
* Initialize with the data that represents the filter cascade.
* This method can be called repeatedly.
*
* @throws NS_ERROR_INVALID_ARG if the input is malformed.
*/
void setFilterData(in Array<octet> data);
/**
* Check whether a given key is a member of the filter cascade.
* The result can only be relied upon if the key was known at the time of the
* filter generation. If the key is unknown, the method may incorrectly
* return true (due to the probabilistic nature of bloom filters).
*/
boolean has(in ACString key);
};

75
toolkit/components/cascade_bloom_filter/src/lib.rs
View File

@ -1,75 +0,0 @@
/* This Source Code Form is subject to the terms of the Mozilla Public
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
extern crate nserror;
extern crate nsstring;
#[macro_use]
extern crate rental;
extern crate rust_cascade;
extern crate thin_vec;
#[macro_use]
extern crate xpcom;
use nserror::{nsresult, NS_ERROR_INVALID_ARG, NS_ERROR_NOT_INITIALIZED, NS_OK};
use nsstring::nsACString;
use rust_cascade::Cascade;
use std::cell::RefCell;
use thin_vec::ThinVec;
use xpcom::interfaces::nsICascadeFilter;
use xpcom::{xpcom_method, RefPtr};
// Cascade does not take ownership of the data, so we must own the data in order to pass its
// reference to Cascade.
rental! {
mod rentals {
use super::Cascade;
#[rental]
pub struct CascadeWithOwnedData {
owndata: Box<[u8]>,
cascade: Box<Cascade<'owndata>>,
}
}
}
#[derive(xpcom)]
#[xpimplements(nsICascadeFilter)]
#[refcnt = "nonatomic"]
pub struct InitCascadeFilter {
filter: RefCell<Option<rentals::CascadeWithOwnedData>>,
}
impl CascadeFilter {
fn new() -> RefPtr<CascadeFilter> {
CascadeFilter::allocate(InitCascadeFilter {
filter: RefCell::new(None),
})
}
xpcom_method!(set_filter_data => SetFilterData(data: *const ThinVec<u8>));
fn set_filter_data(&self, data: &ThinVec<u8>) -> Result<(), nsresult> {
let filter = rentals::CascadeWithOwnedData::try_new_or_drop(
Vec::from(data.as_slice()).into_boxed_slice(),
|data| Cascade::from_bytes(data).unwrap_or(None).ok_or(NS_ERROR_INVALID_ARG)
)?;
self.filter.borrow_mut().replace(filter);
Ok(())
}
xpcom_method!(has => Has(key: *const nsACString) -> bool);
fn has(&self, key: &nsACString) -> Result<bool, nsresult> {
match self.filter.borrow().as_ref() {
None => Err(NS_ERROR_NOT_INITIALIZED),
Some(filter) => Ok(filter.rent(|cascade| cascade.has(&*key))),
}
}
}
#[no_mangle]
pub unsafe extern "C" fn cascade_filter_construct(result: &mut *const nsICascadeFilter) {
let inst: RefPtr<CascadeFilter> = CascadeFilter::new();
*result = inst.coerce::<nsICascadeFilter>();
std::mem::forget(inst);
}

51
toolkit/components/cascade_bloom_filter/test/xpcshell/test_cascade_bindings.js
View File

@ -1,51 +0,0 @@
"use strict";
const CASCADE_CID = "@mozilla.org/cascade-filter;1";
const CASCADE_IID = Ci.nsICascadeFilter;
const CascadeFilter = Components.Constructor(CASCADE_CID, CASCADE_IID);
add_task(function CascadeFilter_uninitialized() {
let filter = new CascadeFilter();
Assert.throws(
() => filter.has(""),
e => e.result === Cr.NS_ERROR_NOT_INITIALIZED,
"Cannot use has() if the filter is not initialized"
);
});
add_task(function CascadeFilter_with_setFilterData() {
let filter = new CascadeFilter();
Assert.throws(
() => filter.setFilterData(),
e => e.result === Cr.NS_ERROR_XPC_NOT_ENOUGH_ARGS,
"setFilterData without parameters should throw"
);
Assert.throws(
() => filter.setFilterData(null),
e => e.result === Cr.NS_ERROR_XPC_CANT_CONVERT_PRIMITIVE_TO_ARRAY,
"setFilterData with null parameter is invalid"
);
Assert.throws(
() => filter.setFilterData(new Uint8Array()),
e => e.result === Cr.NS_ERROR_INVALID_ARG,
"setFilterData with empty array is invalid"
);
// Test data based on rust_cascade's unit tests (bloom_v1_test_from_bytes),
// with two bytes in front to have a valid format.
const TEST_DATA = [1, 0, 1, 9, 0, 0, 0, 1, 0, 0, 0, 1, 0x41, 0];
Assert.throws(
() => filter.setFilterData(new Uint8Array(TEST_DATA.slice(1))),
e => e.result === Cr.NS_ERROR_INVALID_ARG,
"setFilterData with invalid data (missing head) is invalid"
);
Assert.throws(
() => filter.setFilterData(new Uint8Array(TEST_DATA.slice(0, -1))),
e => e.result === Cr.NS_ERROR_INVALID_ARG,
"setFilterData with invalid data (missing tail) is invalid"
);
filter.setFilterData(new Uint8Array(TEST_DATA));
Assert.equal(filter.has("this"), true, "has(this) should be true");
Assert.equal(filter.has("that"), true, "has(that) should be true");
Assert.equal(filter.has("other"), false, "has(other) should be false");
});

1
toolkit/components/cascade_bloom_filter/test/xpcshell/xpcshell.ini
View File

@ -1 +0,0 @@
[test_cascade_bindings.js]

1
toolkit/components/moz.build
View File

@ -23,7 +23,6 @@ DIRS += [
'backgroundhangmonitor',
'bitsdownload',
'browser',
'cascade_bloom_filter',
'certviewer',
'cleardata',
'clearsitedata',

1
toolkit/library/rust/shared/Cargo.toml
View File

@ -34,7 +34,6 @@ log = {version = "0.4", features = ["release_max_level_info"]}
env_logger = {version = "0.6", default-features = false} # disable `regex` to reduce code size
cose-c = { version = "0.1.5" }
jsrust_shared = { path = "../../../../js/src/rust/shared" }
cascade_bloom_filter = { path = "../../../components/cascade_bloom_filter" }
cert_storage = { path = "../../../../security/manager/ssl/cert_storage", optional = true }
bitsdownload = { path = "../../../components/bitsdownload", optional = true }
storage = { path = "../../../../storage/rust" }

1
toolkit/library/rust/shared/lib.rs
View File

@ -15,7 +15,6 @@ extern crate authenticator;
extern crate bitsdownload;
#[cfg(feature = "moz_places")]
extern crate bookmark_sync;
extern crate cascade_bloom_filter;
#[cfg(feature = "new_cert_storage")]
extern crate cert_storage;
extern crate chardetng_c;

368
toolkit/mozapps/extensions/Blocklist.jsm
View File

@ -38,12 +38,6 @@ ChromeUtils.defineModuleGetter(
"resource://services-settings/remote-settings.js"
);
const CascadeFilter = Components.Constructor(
"@mozilla.org/cascade-filter;1",
"nsICascadeFilter",
"setFilterData"
);
// The whole ID should be surrounded by literal ().
// IDs may contain alphanumerics, _, -, {}, @ and a literal '.'
// They may also contain backslashes (needed to escape the {} and dot)
@ -133,12 +127,9 @@ function doesAddonEntryMatch(matches, addonProps) {
const TOOLKIT_ID = "toolkit@mozilla.org";
const PREF_BLOCKLIST_ITEM_URL = "extensions.blocklist.itemURL";
const PREF_BLOCKLIST_ADDONITEM_URL = "extensions.blocklist.addonItemURL";
const PREF_BLOCKLIST_ENABLED = "extensions.blocklist.enabled";
const PREF_BLOCKLIST_LEVEL = "extensions.blocklist.level";
const PREF_BLOCKLIST_SUPPRESSUI = "extensions.blocklist.suppressUI";
const PREF_BLOCKLIST_USE_MLBF = "extensions.blocklist.useMLBF";
const PREF_BLOCKLIST_USE_MLBF_STASHES = "extensions.blocklist.useMLBF.stashes";
const PREF_EM_LOGGING_ENABLED = "extensions.logging.enabled";
const URI_BLOCKLIST_DIALOG =
"chrome://mozapps/content/extensions/blocklist.xhtml";
@ -160,17 +151,10 @@ const PREF_BLOCKLIST_PLUGINS_COLLECTION =
const PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS =
"services.blocklist.plugins.checked";
const PREF_BLOCKLIST_PLUGINS_SIGNER = "services.blocklist.plugins.signer";
// Blocklist v2 - legacy JSON format.
const PREF_BLOCKLIST_ADDONS_COLLECTION = "services.blocklist.addons.collection";
const PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS =
"services.blocklist.addons.checked";
const PREF_BLOCKLIST_ADDONS_SIGNER = "services.blocklist.addons.signer";
// Blocklist v3 - MLBF format.
const PREF_BLOCKLIST_ADDONS3_COLLECTION =
"services.blocklist.addons-mlbf.collection";
const PREF_BLOCKLIST_ADDONS3_CHECKED_SECONDS =
"services.blocklist.addons-mlbf.checked";
const PREF_BLOCKLIST_ADDONS3_SIGNER = "services.blocklist.addons-mlbf.signer";
const BlocklistTelemetry = {
/**
@ -1076,9 +1060,6 @@ this.PluginBlocklistRS = {
* "last_modified": 1480349215672,
* }
*
* This is a legacy format, and implements deprecated operations (bug 1620580).
* ExtensionBlocklistMLBF supersedes this implementation.
*
* Note: we assign to the global to allow tests to reach the object directly.
*/
this.ExtensionBlocklistRS = {
@ -1160,15 +1141,6 @@ this.ExtensionBlocklistRS = {
shutdown() {
if (this._client) {
this._client.off("sync", this._onUpdate);
this._didShutdown = true;
}
},
// Called when the blocklist implementation is changed via a pref.
undoShutdown() {
if (this._didShutdown) {
this._client.on("sync", this._onUpdate);
this._didShutdown = false;
}
},
@ -1314,304 +1286,6 @@ this.ExtensionBlocklistRS = {
},
};
/**
* The extensions blocklist implementation, the third version.
*
* The current blocklist is represented by a multi-level bloom filter (MLBF)
* (aka "Cascade Bloom Filter") that works like a set, i.e. supports a has()
* operation, except it is probabilistic. The MLBF is 100% accurate for known
* entries and unreliable for unknown entries. When the backend generates the
* MLBF, all known add-ons are recorded, including their block state. Unknown
* add-ons are identified by their signature date being newer than the MLBF's
* generation time, and they are considered to not be blocked.
*
* Legacy blocklists used to distinguish between "soft block" and "hard block",
* but the current blocklist only supports one type of block ("hard block").
* After checking the blocklist states, any previous "soft blocked" addons will
* either be (hard) blocked or unblocked based on the blocklist.
*
* The MLBF is attached to a RemoteSettings record, as follows:
*
* {
* "generation_time": 1585692000000,
* "attachment": { ... RemoteSettings attachment ... }
* "attachment_type": "bloomfilter-base",
* }
*
* To update the blocklist, a replacement MLBF is published:
*
* {
* "generation_time": 1585692000000,
* "attachment": { ... RemoteSettings attachment ... }
* "attachment_type": "bloomfilter-full",
* }
*
* The collection can also contain stashes:
*
* {
* "stash_time": 1585692000001,
* "stash": {
* "blocked": [ "addonid:1.0", ... ],
* "unblocked": [ "addonid:1.0", ... ]
* }
*
* Stashes can be used to update the blocklist without forcing the whole MLBF
* to be downloaded again. These stashes are applied on top of the base MLBF.
* The use of stashes is currently optional, and toggled via the
* extensions.blocklist.useMLBF.stashes preference (true = use stashes).
*
* Note: we assign to the global to allow tests to reach the object directly.
*/
this.ExtensionBlocklistMLBF = {
RS_ATTACHMENT_ID: "addons-mlbf.bin",
async _fetchMLBF(record) {
// |record| may be unset. In that case, the MLBF dump is used instead
// (provided that the client has been built with it included).
let hash = record?.attachment.hash;
if (this._mlbfData && hash && this._mlbfData.cascadeHash === hash) {
// Not changed, let's re-use it.
return this._mlbfData;
}
const {
buffer,
record: actualRecord,
} = await this._client.attachments.download(record, {
attachmentId: this.RS_ATTACHMENT_ID,
useCache: true,
fallbackToCache: true,
fallbackToDump: true,
});
return {
cascadeHash: actualRecord.attachment.hash,
cascadeFilter: new CascadeFilter(new Uint8Array(buffer)),
// Note: generation_time is semantically distinct from last_modified.
// generation_time is compared with the signing date of the add-on, so it
// should be in sync with the signing service's clock.
// In contrast, last_modified does not have such strong requirements.
generationTime: actualRecord.generation_time,
};
},
async _updateMLBF(forceUpdate = false) {
// The update process consists of fetching the collection, followed by
// potentially multiple network requests. As long as the collection has not
// been changed, repeated update requests can be coalesced. But when the
// collection has been updated, all pending update requests should await the
// new update request instead of the previous one.
if (!forceUpdate && this._updatePromise) {
return this._updatePromise;
}
const isUpdateReplaced = () => this._updatePromise != updatePromise;
const updatePromise = (async () => {
if (!gBlocklistEnabled) {
this._mlbfData = null;
this._stashes = null;
return;
}
let records = await this._client.get();
if (isUpdateReplaced()) {
return;
}
let mlbfRecords = records
.filter(r => r.attachment)
// Newest attachments first.
.sort((a, b) => b.generation_time - a.generation_time);
let mlbfRecord;
if (this.stashesEnabled) {
mlbfRecord = mlbfRecords.find(
r => r.attachment_type == "bloomfilter-base"
);
this._stashes = records
.filter(({ stash }) => {
return (
// Exclude non-stashes, e.g. MLBF attachments.
stash &&
// Sanity check for type.
Array.isArray(stash.blocked) &&
Array.isArray(stash.unblocked)
);
})
// Sort by stash time - newest first.
.sort((a, b) => b.stash_time - a.stash_time)
.map(({ stash }) => ({
blocked: new Set(stash.blocked),
unblocked: new Set(stash.unblocked),
}));
} else {
mlbfRecord = mlbfRecords.find(
r =>
r.attachment_type == "bloomfilter-full" ||
r.attachment_type == "bloomfilter-base"
);
this._stashes = null;
}
let mlbf = await this._fetchMLBF(mlbfRecord);
// When a MLBF dump is packaged with the browser, mlbf will always be
// non-null at this point.
if (isUpdateReplaced()) {
return;
}
this._mlbfData = mlbf;
})()
.catch(e => {
Cu.reportError(e);
})
.then(() => {
if (!isUpdateReplaced()) {
this._updatePromise = null;
}
return this._updatePromise;
});
this._updatePromise = updatePromise;
return updatePromise;
},
ensureInitialized() {
if (!gBlocklistEnabled || this._initialized) {
return;
}
this._initialized = true;
this._client = RemoteSettings(
Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS3_COLLECTION),
{
bucketNamePref: PREF_BLOCKLIST_BUCKET,
lastCheckTimePref: PREF_BLOCKLIST_ADDONS3_CHECKED_SECONDS,
signerName: Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS3_SIGNER),
}
);
this._onUpdate = this._onUpdate.bind(this);
this._client.on("sync", this._onUpdate);
this.stashesEnabled = Services.prefs.getBoolPref(
PREF_BLOCKLIST_USE_MLBF_STASHES,
false
);
},
shutdown() {
if (this._client) {
this._client.off("sync", this._onUpdate);
this._didShutdown = true;
}
},
// Called when the blocklist implementation is changed via a pref.
undoShutdown() {
if (this._didShutdown) {
this._client.on("sync", this._onUpdate);
this._didShutdown = false;
}
},
async _onUpdate() {
this.ensureInitialized();
await this._updateMLBF(true);
// Check add-ons from XPIProvider.
const types = ["extension", "theme", "locale", "dictionary"];
let addons = await AddonManager.getAddonsByTypes(types);
for (let addon of addons) {
let oldState = addon.blocklistState;
await addon.updateBlocklistState(false);
let state = addon.blocklistState;
LOG(
"Blocklist state for " +
addon.id +
" changed from " +
oldState +
" to " +
state
);
// We don't want to re-warn about add-ons
if (state == oldState) {
continue;
}
// Ensure that softDisabled is false if the add-on is not soft blocked
// (by a previous implementation of the blocklist).
if (state != Ci.nsIBlocklistService.STATE_SOFTBLOCKED) {
addon.softDisabled = false;
}
}
AddonManagerPrivate.updateAddonAppDisabledStates();
},
async getState(addon) {
let state = await this.getEntry(addon);
return state ? state.state : Ci.nsIBlocklistService.STATE_NOT_BLOCKED;
},
async getEntry(addon) {
if (!this._mlbfData) {
this.ensureInitialized();
await this._updateMLBF(false);
}
let blockKey = addon.id + ":" + addon.version;
if (this._stashes) {
// Stashes are ordered by newest first.
for (let stash of this._stashes) {
// blocked and unblocked do not have overlapping entries.
if (stash.blocked.has(blockKey)) {
return this._createBlockEntry(addon);
}
if (stash.unblocked.has(blockKey)) {
return null;
}
}
}
if (!addon.signedState) {
// The MLBF does not apply to unsigned add-ons.
return null;
}
if (!this._mlbfData) {
// This could happen in theory in any of the following cases:
// - the blocklist is disabled.
// - The RemoteSettings backend served a malformed MLBF.
// - The RemoteSettings backend is unreachable, and this client was built
// without including a dump of the MLBF.
//
// ... in other words, this shouldn't happen in practice.
return null;
}
let { cascadeFilter, generationTime } = this._mlbfData;
if (!cascadeFilter.has(blockKey)) {
// Add-on not blocked or unknown.
return null;
}
// Add-on blocked, or unknown add-on inadvertently labeled as blocked.
if (addon.signedDate > generationTime) {
// The bloom filter only reports 100% accurate results for known add-ons.
// Since the add-on was unknown when the bloom filter was generated, the
// block decision is incorrect and should be treated as unblocked.
return null;
}
return this._createBlockEntry(addon);
},
_createBlockEntry(addon) {
return {
state: Ci.nsIBlocklistService.STATE_BLOCKED,
url: this.createBlocklistURL(addon.id, addon.version),
};
},
createBlocklistURL(id, version) {
let url = Services.urlFormatter.formatURLPref(PREF_BLOCKLIST_ADDONITEM_URL);
return url.replace(/%addonID%/g, id).replace(/%addonVersion%/g, version);
},
};
const EXTENSION_BLOCK_FILTERS = [
"id",
"name",
@ -1704,7 +1378,6 @@ let Blocklist = {
Services.prefs.getIntPref(PREF_BLOCKLIST_LEVEL, DEFAULT_LEVEL),
MAX_BLOCK_LEVEL
);
this._chooseExtensionBlocklistImplementationFromPref();
Services.prefs.addObserver("extensions.blocklist.", this);
Services.prefs.addObserver(PREF_EM_LOGGING_ENABLED, this);
@ -1725,7 +1398,7 @@ let Blocklist = {
shutdown() {
GfxBlocklistRS.shutdown();
PluginBlocklistRS.shutdown();
this.ExtensionBlocklist.shutdown();
ExtensionBlocklistRS.shutdown();
Services.obs.removeObserver(this, "xpcom-shutdown");
Services.prefs.removeObserver("extensions.blocklist.", this);
@ -1759,27 +1432,6 @@ let Blocklist = {
);
this._blocklistUpdated();
break;
case PREF_BLOCKLIST_USE_MLBF:
let oldImpl = this.ExtensionBlocklist;
this._chooseExtensionBlocklistImplementationFromPref();
if (oldImpl._initialized) {
oldImpl.shutdown();
this.ExtensionBlocklist.undoShutdown();
this.ExtensionBlocklist._onUpdate();
} // else neither has been initialized yet. Wait for it to happen.
break;
case PREF_BLOCKLIST_USE_MLBF_STASHES:
ExtensionBlocklistMLBF.stashesEnabled = Services.prefs.getBoolPref(
PREF_BLOCKLIST_USE_MLBF_STASHES,
false
);
if (
ExtensionBlocklistMLBF._initialized &&
!ExtensionBlocklistMLBF._didShutdown
) {
ExtensionBlocklistMLBF._onUpdate();
}
break;
}
break;
}
@ -1788,7 +1440,7 @@ let Blocklist = {
loadBlocklistAsync() {
// Need to ensure we notify gfx of new stuff.
GfxBlocklistRS.checkForEntries();
this.ExtensionBlocklist.ensureInitialized();
ExtensionBlocklistRS.ensureInitialized();
PluginBlocklistRS.ensureInitialized();
},
@ -1801,25 +1453,15 @@ let Blocklist = {
},
getAddonBlocklistState(addon, appVersion, toolkitVersion) {
// NOTE: appVersion/toolkitVersion are only used by ExtensionBlocklistRS.
return this.ExtensionBlocklist.getState(addon, appVersion, toolkitVersion);
return ExtensionBlocklistRS.getState(addon, appVersion, toolkitVersion);
},
getAddonBlocklistEntry(addon, appVersion, toolkitVersion) {
// NOTE: appVersion/toolkitVersion are only used by ExtensionBlocklistRS.
return this.ExtensionBlocklist.getEntry(addon, appVersion, toolkitVersion);
},
_chooseExtensionBlocklistImplementationFromPref() {
if (Services.prefs.getBoolPref(PREF_BLOCKLIST_USE_MLBF, false)) {
this.ExtensionBlocklist = ExtensionBlocklistMLBF;
} else {
this.ExtensionBlocklist = ExtensionBlocklistRS;
}
return ExtensionBlocklistRS.getEntry(addon, appVersion, toolkitVersion);
},
_blocklistUpdated() {
this.ExtensionBlocklist._onUpdate();
ExtensionBlocklistRS._onUpdate();
PluginBlocklistRS._onUpdate();
},
};

1
toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
View File

@ -891,7 +891,6 @@ var AddonTestUtils = {
);
const blocklistMapping = {
extensions: bsPass.ExtensionBlocklistRS,
extensionsMLBF: bsPass.ExtensionBlocklistMLBF,
plugins: bsPass.PluginBlocklistRS,
};

13
toolkit/mozapps/extensions/internal/XPIDatabase.jsm
View File

@ -146,7 +146,6 @@ const PROP_JSON_FIELDS = [
"targetApplications",
"targetPlatforms",
"signedState",
"signedDate",
"seen",
"dependencies",
"incognito",
@ -1364,7 +1363,7 @@ function defineAddonWrapperProperty(name, getter) {
});
});
["installDate", "updateDate", "signedDate"].forEach(function(aProp) {
["installDate", "updateDate"].forEach(function(aProp) {
defineAddonWrapperProperty(aProp, function() {
let addon = addonFor(this);
if (addon[aProp]) {
@ -2985,13 +2984,9 @@ this.XPIDatabaseReconcile = {
let checkSigning =
aOldAddon.signedState === undefined && SIGNED_TYPES.has(aOldAddon.type);
// signedDate must be set if signedState is set.
let signedDateMissing =
aOldAddon.signedDate === undefined &&
(aOldAddon.signedState || checkSigning);
let manifest = null;
if (checkSigning || aReloadMetadata || signedDateMissing) {
if (checkSigning || aReloadMetadata) {
try {
manifest = XPIInstall.syncLoadManifest(aAddonState, aLocation);
} catch (err) {
@ -3008,10 +3003,6 @@ this.XPIDatabaseReconcile = {
aOldAddon.signedState = manifest.signedState;
}
if (signedDateMissing) {
aOldAddon.signedDate = manifest.signedDate;
}
// May be updating from a version of the app that didn't support all the
// properties of the currently-installed add-ons.
if (aReloadMetadata) {

1
toolkit/mozapps/extensions/internal/XPIInstall.jsm
View File

@ -678,7 +678,6 @@ var loadManifest = async function(aPackage, aLocation, aOldAddon) {
let { signedState, cert } = await aPackage.verifySignedState(addon);
addon.signedState = signedState;
addon.signedDate = cert?.validity?.notBefore / 1000 || null;
if (!addon.isPrivileged) {
addon.hidden = false;
}

3
toolkit/mozapps/extensions/internal/XPIProvider.jsm
View File

@ -460,7 +460,6 @@ const JSON_FIELDS = Object.freeze([
"rootURI",
"runInSafeMode",
"signedState",
"signedDate",
"startupData",
"telemetryKey",
"type",
@ -552,7 +551,6 @@ class XPIState {
rootURI: this.rootURI,
runInSafeMode: this.runInSafeMode,
signedState: this.signedState,
signedDate: this.signedDate,
telemetryKey: this.telemetryKey,
version: this.version,
};
@ -641,7 +639,6 @@ class XPIState {
this.dependencies = aDBAddon.dependencies;
this.runInSafeMode = canRunInSafeMode(aDBAddon);
this.signedState = aDBAddon.signedState;
this.signedDate = aDBAddon.signedDate;
this.file = aDBAddon._sourceBundle;
this.rootURI = aDBAddon.rootURI;

BIN
toolkit/mozapps/extensions/test/xpcshell/data/mlbf-blocked1-unblocked2.bin
View File

Binary file not shown.

12
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/head.js
View File

@ -1,14 +1,2 @@
// Appease eslint.
/* import-globals-from ../head_addons.js */
const MLBF_RECORD = {
id: "A blocklist entry that refers to a MLBF file",
last_modified: 1,
attachment: {
size: 32,
hash: "6af648a5d6ce6dbee99b0aab1780d24d204977a6606ad670d5372ef22fac1052",
filename: "does-not-matter.bin",
},
attachment_type: "bloomfilter-base",
generation_time: 1577833200000,
};

169
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf.js
View File

@ -1,169 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* https://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
const { ExtensionBlocklistMLBF } = ChromeUtils.import(
"resource://gre/modules/Blocklist.jsm",
null
);
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
AddonTestUtils.useRealCertChecks = true;
// A real, signed XPI for use in the test.
const SIGNED_ADDON_XPI_FILE = do_get_file("../data/webext-implicit-id.xpi");
const SIGNED_ADDON_ID = "webext_implicit_id@tests.mozilla.org";
const SIGNED_ADDON_VERSION = "1.0";
const SIGNED_ADDON_KEY = `${SIGNED_ADDON_ID}:${SIGNED_ADDON_VERSION}`;
const SIGNED_ADDON_SIGN_TIME = 1459980789000; // notBefore of certificate.
function mockMLBF({ blocked = [], notblocked = [], generationTime }) {
// Mock _fetchMLBF to be able to have a deterministic cascade filter.
ExtensionBlocklistMLBF._fetchMLBF = async () => {
return {
cascadeFilter: {
has(blockKey) {
if (blocked.includes(blockKey)) {
return true;
}
if (notblocked.includes(blockKey)) {
return false;
}
throw new Error(`Block entry must explicitly be listed: ${blockKey}`);
},
},
generationTime,
};
};
}
add_task(async function setup() {
await promiseStartupManager();
mockMLBF({});
await AddonTestUtils.loadBlocklistRawData({
extensionsMLBF: [MLBF_RECORD],
});
});
// Checks: Initially unblocked, then blocked, then unblocked again.
add_task(async function signed_xpi_initially_unblocked() {
mockMLBF({
blocked: [],
notblocked: [SIGNED_ADDON_KEY],
generationTime: SIGNED_ADDON_SIGN_TIME + 1,
});
await ExtensionBlocklistMLBF._onUpdate();
await promiseInstallFile(SIGNED_ADDON_XPI_FILE);
let addon = await promiseAddonByID(SIGNED_ADDON_ID);
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_NOT_BLOCKED);
mockMLBF({
blocked: [SIGNED_ADDON_KEY],
notblocked: [],
generationTime: SIGNED_ADDON_SIGN_TIME + 1,
});
await ExtensionBlocklistMLBF._onUpdate();
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_BLOCKED);
Assert.deepEqual(
await Blocklist.getAddonBlocklistEntry(addon),
{
state: Ci.nsIBlocklistService.STATE_BLOCKED,
url:
"https://addons.mozilla.org/en-US/xpcshell/blocked-addon/webext_implicit_id@tests.mozilla.org/1.0/",
},
"Blocked addon should have blocked entry"
);
mockMLBF({
blocked: [SIGNED_ADDON_KEY],
notblocked: [],
// MLBF generationTime is older, so "blocked" entry should not apply.
generationTime: SIGNED_ADDON_SIGN_TIME - 1,
});
await ExtensionBlocklistMLBF._onUpdate();
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_NOT_BLOCKED);
await addon.uninstall();
});
// Checks: Initially blocked on install, then unblocked.
add_task(async function signed_xpi_blocked_on_install() {
mockMLBF({
blocked: [SIGNED_ADDON_KEY],
notblocked: [],
generationTime: SIGNED_ADDON_SIGN_TIME + 1,
});
await ExtensionBlocklistMLBF._onUpdate();
await promiseInstallFile(SIGNED_ADDON_XPI_FILE);
let addon = await promiseAddonByID(SIGNED_ADDON_ID);
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_BLOCKED);
Assert.ok(addon.appDisabled, "Blocked add-on is disabled on install");
mockMLBF({
blocked: [],
notblocked: [SIGNED_ADDON_KEY],
generationTime: SIGNED_ADDON_SIGN_TIME - 1,
});
await ExtensionBlocklistMLBF._onUpdate();
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_NOT_BLOCKED);
Assert.ok(!addon.appDisabled, "Re-enabled after unblock");
await addon.uninstall();
});
// An unsigned add-on cannot be blocked.
add_task(async function unsigned_not_blocked() {
const UNSIGNED_ADDON_ID = "not-signed@tests.mozilla.org";
const UNSIGNED_ADDON_VERSION = "1.0";
const UNSIGNED_ADDON_KEY = `${UNSIGNED_ADDON_ID}:${UNSIGNED_ADDON_VERSION}`;
mockMLBF({
blocked: [UNSIGNED_ADDON_KEY],
notblocked: [],
generationTime: SIGNED_ADDON_SIGN_TIME + 1,
});
await ExtensionBlocklistMLBF._onUpdate();
let unsignedAddonFile = createTempWebExtensionFile({
manifest: {
version: UNSIGNED_ADDON_VERSION,
applications: { gecko: { id: UNSIGNED_ADDON_ID } },
},
});
// Unsigned add-ons can generally only be loaded as a temporary install.
let [addon] = await Promise.all([
AddonManager.installTemporaryAddon(unsignedAddonFile),
promiseWebExtensionStartup(UNSIGNED_ADDON_ID),
]);
Assert.equal(addon.signedState, AddonManager.SIGNEDSTATE_MISSING);
Assert.equal(addon.blocklistState, Ci.nsIBlocklistService.STATE_NOT_BLOCKED);
Assert.equal(
await Blocklist.getAddonBlocklistState(addon),
Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
"Unsigned temporary add-on is not blocked"
);
await addon.uninstall();
});
// To make sure that unsigned_not_blocked did not trivially pass, we also check
// that add-ons can actually be blocked when installed as a temporary add-on.
add_task(async function signed_temporary() {
mockMLBF({
blocked: [SIGNED_ADDON_KEY],
notblocked: [],
generationTime: SIGNED_ADDON_SIGN_TIME + 1,
});
await ExtensionBlocklistMLBF._onUpdate();
await Assert.rejects(
AddonManager.installTemporaryAddon(SIGNED_ADDON_XPI_FILE),
/Add-on webext_implicit_id@tests.mozilla.org is not compatible with application version/,
"Blocklisted add-on cannot be installed"
);
});

96
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_dump.js
View File

@ -1,96 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* https://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
/**
* @fileOverview Verifies that the MLBF dump of the addons blocklist is
* correctly registered.
*/
Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
const { ExtensionBlocklist: ExtensionBlocklistMLBF } = Blocklist;
// A known blocked version from bug 1626602.
const blockedAddon = {
id: "{6f62927a-e380-401a-8c9e-c485b7d87f0d}",
version: "9.2.0",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
// The following date is the date of the first checked in MLBF. Any MLBF
// generated in the future should be generated after this date, to be useful.
signedDate: 1588098908496, // 2020-04-28 (dummy date)
};
// A known add-on that is not blocked, as of writing. It is likely not going
// to be blocked because it does not have any executable code.
const nonBlockedAddon = {
id: "disable-ctrl-q-and-cmd-q@robwu.nl",
version: "1",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
signedDate: 1482430349000, // 2016-12-22 (actual signing time).
};
async function sha256(arrayBuffer) {
Cu.importGlobalProperties(["crypto"]);
let hash = await crypto.subtle.digest("SHA-256", arrayBuffer);
const toHex = b => b.toString(16).padStart(2, "0");
return Array.from(new Uint8Array(hash), toHex).join("");
}
add_task(async function verify_dump_first_run() {
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
// Tapping into the internals of ExtensionBlocklistMLBF._fetchMLBF to observe
// MLBF request details.
const observed = [];
ExtensionBlocklistMLBF.ensureInitialized();
// Despite being called "download", this does not actually access the network
// when there is a valid dump.
const originalImpl = ExtensionBlocklistMLBF._client.attachments.download;
ExtensionBlocklistMLBF._client.attachments.download = function(record) {
let downloadPromise = originalImpl.apply(this, arguments);
observed.push({ inputRecord: record, downloadPromise });
return downloadPromise;
};
Assert.equal(
await Blocklist.getAddonBlocklistState(blockedAddon),
Ci.nsIBlocklistService.STATE_BLOCKED,
"A add-on that is known to be on the blocklist should be blocked"
);
Assert.equal(
await Blocklist.getAddonBlocklistState(nonBlockedAddon),
Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
"A known non-blocked add-on should not be blocked"
);
Assert.equal(observed.length, 1, "expected number of MLBF download requests");
const { inputRecord, downloadPromise } = observed[0];
Assert.ok(inputRecord, "addons-bloomfilters collection dump exists");
const downloadResult = await downloadPromise;
// Verify that the "download" result really originates from the local dump.
// "dump_match" means that the record exists in the collection and that an
// attachment was found.
//
// If this fails:
// - "dump_fallback" means that the MLBF attachment is out of sync with the
// collection data.
// - undefined could mean that the implementation of Attachments.jsm changed.
Assert.equal(
downloadResult._source,
"dump_match",
"MLBF attachment should match the RemoteSettings collection"
);
Assert.equal(
await sha256(downloadResult.buffer),
inputRecord.attachment.hash,
"The content of the attachment should actually matches the record"
);
});

173
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_fetch.js
View File

@ -1,173 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* https://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
/**
* @fileOverview Tests the MLBF and RemoteSettings synchronization logic.
*/
Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
const { Downloader } = ChromeUtils.import(
"resource://services-settings/Attachments.jsm"
);
const { ExtensionBlocklistMLBF } = ChromeUtils.import(
"resource://gre/modules/Blocklist.jsm",
null
);
// This test needs to interact with the RemoteSettings client.
ExtensionBlocklistMLBF.ensureInitialized();
add_task(async function fetch_invalid_mlbf_record() {
let invalidRecord = {
attachment: { size: 1, hash: "definitely not valid" },
generation_time: 1,
};
// _fetchMLBF(invalidRecord) may succeed if there is a MLBF dump packaged with
// the application. This test intentionally hides the actual path to get
// deterministic results. To check whether the dump is correctly registered,
// run test_blocklist_mlbf_dump.js
// Forget about the packaged attachment.
Downloader._RESOURCE_BASE_URL = "invalid://bogus";
await Assert.rejects(
ExtensionBlocklistMLBF._fetchMLBF(invalidRecord),
/NetworkError/,
"record not found when there is no packaged MLBF"
);
});
// Other tests can mock _testMLBF, so let's verify that it works as expected.
add_task(async function fetch_valid_mlbf() {
const url = Services.io.newFileURI(
do_get_file("../data/mlbf-blocked1-unblocked2.bin")
).spec;
Cu.importGlobalProperties(["fetch"]);
const blob = await (await fetch(url)).blob();
await ExtensionBlocklistMLBF._client.db.saveAttachment(
ExtensionBlocklistMLBF.RS_ATTACHMENT_ID,
{ record: JSON.parse(JSON.stringify(MLBF_RECORD)), blob }
);
const result = await ExtensionBlocklistMLBF._fetchMLBF(MLBF_RECORD);
Assert.equal(result.cascadeHash, MLBF_RECORD.attachment.hash, "hash OK");
Assert.equal(result.generationTime, MLBF_RECORD.generation_time, "time OK");
Assert.ok(result.cascadeFilter.has("@blocked:1"), "item blocked");
Assert.ok(!result.cascadeFilter.has("@unblocked:2"), "item not blocked");
const result2 = await ExtensionBlocklistMLBF._fetchMLBF({
attachment: { size: 1, hash: "invalid" },
generation_time: Date.now(),
});
Assert.equal(
result2.cascadeHash,
MLBF_RECORD.attachment.hash,
"The cached MLBF should be used when the attachment is invalid"
);
// The attachment is kept in the database for use by the next test task.
});
// Test that results of the public API are consistent with the MLBF file.
add_task(async function public_api_uses_mlbf() {
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
await promiseStartupManager();
const blockedAddon = {
id: "@blocked",
version: "1",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
signedDate: 0, // a date in the past, before MLBF's generationTime.
};
const nonBlockedAddon = {
id: "@unblocked",
version: "2",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
signedDate: 0, // a date in the past, before MLBF's generationTime.
};
await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: [MLBF_RECORD] });
Assert.deepEqual(
await Blocklist.getAddonBlocklistEntry(blockedAddon),
{
state: Ci.nsIBlocklistService.STATE_BLOCKED,
url:
"https://addons.mozilla.org/en-US/xpcshell/blocked-addon/@blocked/1/",
},
"Blocked addon should have blocked entry"
);
Assert.deepEqual(
await Blocklist.getAddonBlocklistEntry(nonBlockedAddon),
null,
"Non-blocked addon should not be blocked"
);
Assert.equal(
await Blocklist.getAddonBlocklistState(blockedAddon),
Ci.nsIBlocklistService.STATE_BLOCKED,
"Blocked entry should have blocked state"
);
Assert.equal(
await Blocklist.getAddonBlocklistState(nonBlockedAddon),
Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
"Non-blocked entry should have unblocked state"
);
// Note: Blocklist collection and attachment carries over to the next test.
});
// Checks the remaining cases of database corruption that haven't been handled
// before.
add_task(async function handle_database_corruption() {
const blockedAddon = {
id: "@blocked",
version: "1",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
signedDate: 0, // a date in the past, before MLBF's generationTime.
};
async function checkBlocklistWorks() {
Assert.equal(
await Blocklist.getAddonBlocklistState(blockedAddon),
Ci.nsIBlocklistService.STATE_BLOCKED,
"Add-on should be blocked by the blocklist"
);
}
// In the fetch_invalid_mlbf_record we checked that a cached / packaged MLBF
// attachment is used as a fallback when the record is invalid. Here we also
// check that there is a fallback when there is no record at all.
await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: [] });
// When the collection is empty, the last known MLBF should be used anyway.
await checkBlocklistWorks();
// Now we also remove the cached file...
await ExtensionBlocklistMLBF._client.db.saveAttachment(
ExtensionBlocklistMLBF.RS_ATTACHMENT_ID,
null
);
// Deleting the file shouldn't cause issues because the MLBF is loaded once
// and then kept in memory.
await checkBlocklistWorks();
// Force an update while we don't have any blocklist data nor cache.
await ExtensionBlocklistMLBF._onUpdate();
// As a fallback, continue to use the in-memory version of the blocklist.
await checkBlocklistWorks();
// Memory gone, e.g. after a browser restart.
delete ExtensionBlocklistMLBF._mlbfData;
Assert.equal(
await Blocklist.getAddonBlocklistState(blockedAddon),
Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
"Blocklist can't work if all blocklist data is gone"
);
});

192
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_stashes.js
View File

@ -1,192 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* https://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
Services.prefs.setBoolPref("extensions.blocklist.useMLBF", true);
Services.prefs.setBoolPref("extensions.blocklist.useMLBF.stashes", true);
createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
const ExtensionBlocklistMLBF = Blocklist.ExtensionBlocklist;
const MLBF_LOAD_ATTEMPTS = [];
ExtensionBlocklistMLBF._fetchMLBF = async record => {
MLBF_LOAD_ATTEMPTS.push(record);
return {
generationTime: 0,
cascadeFilter: {
has(blockKey) {
if (blockKey === "@onlyblockedbymlbf:1") {
return true;
}
throw new Error("bloom filter should not be used in this test");
},
},
};
};
async function toggleStashPref(val) {
Assert.ok(!ExtensionBlocklistMLBF._updatePromise, "no pending update");
Services.prefs.setBoolPref("extensions.blocklist.useMLBF.stashes", val);
// A pref observer should trigger an update.
Assert.ok(ExtensionBlocklistMLBF._updatePromise, "update pending");
await Blocklist.ExtensionBlocklist._updatePromise;
}
async function checkBlockState(addonId, version, expectBlocked) {
let addon = {
id: addonId,
version,
// Note: signedState and signedDate are missing, so the MLBF does not apply
// and we will effectively only test stashing.
};
let state = await Blocklist.getAddonBlocklistState(addon);
if (expectBlocked) {
Assert.equal(state, Ci.nsIBlocklistService.STATE_BLOCKED);
} else {
Assert.equal(state, Ci.nsIBlocklistService.STATE_NOT_BLOCKED);
}
}
add_task(async function setup() {
await promiseStartupManager();
});
// Tests that add-ons can be blocked / unblocked via the stash.
add_task(async function basic_stash() {
await AddonTestUtils.loadBlocklistRawData({
extensionsMLBF: [
{
stash_time: 0,
stash: {
blocked: ["@blocked:1"],
unblocked: ["@notblocked:2"],
},
},
],
});
await checkBlockState("@blocked", "1", true);
await checkBlockState("@notblocked", "2", false);
// Not in stash (but unsigned, so shouldn't reach MLBF):
await checkBlockState("@blocked", "2", false);
Assert.equal(
await Blocklist.getAddonBlocklistState({
id: "@onlyblockedbymlbf",
version: "1",
signedState: 2, // = AddonManager.SIGNEDSTATE_SIGNED.
signedDate: 0, // = the MLBF's generationTime.
}),
Ci.nsIBlocklistService.STATE_BLOCKED,
"falls through to MLBF if entry is not found in stash"
);
Assert.deepEqual(MLBF_LOAD_ATTEMPTS, [null], "MLBF attachment not found");
});
// Tests that invalid stash entries are ignored.
add_task(async function invalid_stashes() {
await AddonTestUtils.loadBlocklistRawData({
extensionsMLBF: [
{},
{ stash: null },
{ stash: 1 },
{ stash: {} },
{ stash: { blocked: ["@broken:1", "@okid:1"] } },
{ stash: { unblocked: ["@broken:2"] } },
// The only correct entry:
{ stash: { blocked: ["@okid:2"], unblocked: ["@okid:1"] } },
{ stash: { blocked: ["@broken:1", "@okid:1"] } },
{ stash: { unblocked: ["@broken:2", "@okid:2"] } },
],
});
// The valid stash entry should be applied:
await checkBlockState("@okid", "1", false);
await checkBlockState("@okid", "2", true);
// Entries from invalid stashes should be ignored:
await checkBlockState("@broken", "1", false);
await checkBlockState("@broken", "2", false);
});
// Blocklist stashes should be processed in the reverse chronological order.
add_task(async function stash_time_order() {
await AddonTestUtils.loadBlocklistRawData({
extensionsMLBF: [
// "@a:1" and "@a:2" are blocked at time 1, but unblocked later.
{ stash_time: 2, stash: { blocked: [], unblocked: ["@a:1"] } },
{ stash_time: 1, stash: { blocked: ["@a:1", "@a:2"], unblocked: [] } },
{ stash_time: 3, stash: { blocked: [], unblocked: ["@a:2"] } },
// "@b:1" and "@b:2" are unblocked at time 4, but blocked later.
{ stash_time: 5, stash: { blocked: ["@b:1"], unblocked: [] } },
{ stash_time: 4, stash: { blocked: [], unblocked: ["@b:1", "@b:2"] } },
{ stash_time: 6, stash: { blocked: ["@b:2"], unblocked: [] } },
],
});
await checkBlockState("@a", "1", false);
await checkBlockState("@a", "2", false);
await checkBlockState("@b", "1", true);
await checkBlockState("@b", "2", true);
});
// Tests that the correct records+attachment are chosen depending on the pref.
add_task(async function mlbf_attachment_type_and_stash_is_correct() {
MLBF_LOAD_ATTEMPTS.length = 0;
const records = [
{ stash_time: 0, stash: { blocked: ["@blocked:1"], unblocked: [] } },
{ attachment_type: "bloomfilter-base", attachment: {}, generation_time: 0 },
{ attachment_type: "bloomfilter-full", attachment: {}, generation_time: 1 },
];
await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: records });
// Check that the pref works.
await checkBlockState("@blocked", "1", true);
await toggleStashPref(false);
await checkBlockState("@blocked", "1", false);
await toggleStashPref(true);
await checkBlockState("@blocked", "1", true);
Assert.deepEqual(
MLBF_LOAD_ATTEMPTS.map(r => r?.attachment_type),
[
// Initial load with pref true
"bloomfilter-base",
// Pref off.
"bloomfilter-full",
// Pref on again.
"bloomfilter-base",
],
"Expected attempts to load MLBF as part of update"
);
});
// When stashes are disabled, "bloomfilter-full" may be used (as seen in the
// previous test, mlbf_attachment_type_and_stash_is_correct). With stashes
// enabled, "bloomfilter-full" should be ignored, however.
add_task(async function mlbf_bloomfilter_full_ignored() {
MLBF_LOAD_ATTEMPTS.length = 0;
await AddonTestUtils.loadBlocklistRawData({
extensionsMLBF: [{ attachment_type: "bloomfilter-full", attachment: {} }],
});
// When stashes are enabled, only bloomfilter-base records should be used.
// Since there are no such records, we shouldn't find anything.
Assert.deepEqual(MLBF_LOAD_ATTEMPTS, [null], "no matching MLBFs found");
});
// Tests that the most recent MLBF is downloaded.
add_task(async function mlbf_generation_time_recent() {
MLBF_LOAD_ATTEMPTS.length = 0;
const records = [
{ attachment_type: "bloomfilter-base", attachment: {}, generation_time: 2 },
{ attachment_type: "bloomfilter-base", attachment: {}, generation_time: 3 },
{ attachment_type: "bloomfilter-base", attachment: {}, generation_time: 1 },
];
await AddonTestUtils.loadBlocklistRawData({ extensionsMLBF: records });
Assert.equal(
MLBF_LOAD_ATTEMPTS[0].generation_time,
3,
"expected to load most recent MLBF"
);
});

77
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/test_blocklist_mlbf_update.js
View File

@ -1,77 +0,0 @@
/* Any copyright is dedicated to the Public Domain.
* https://creativecommons.org/publicdomain/zero/1.0/ */
"use strict";
/**
* @fileOverview Checks that the MLBF updating logic works reasonably.
*/
const { ExtensionBlocklistMLBF } = ChromeUtils.import(
"resource://gre/modules/Blocklist.jsm",
null
);
// This test needs to interact with the RemoteSettings client.
ExtensionBlocklistMLBF.ensureInitialized();
// Multiple internal calls to update should be coalesced and end up with the
// MLBF attachment from the last update call.
add_task(async function collapse_multiple_pending_update_requests() {
const observed = [];
// The first step of starting an update is to read from the RemoteSettings
// collection. When a non-forced update is requested while another update is
// pending, the non-forced update should return/await the previous call
// instead of starting a new read/fetch from the RemoteSettings collection.
// Add a spy to the RemoteSettings client, so we can verify that the number
// of RemoteSettings accesses matches with what we expect.
const originalClientGet = ExtensionBlocklistMLBF._client.get;
const spyClientGet = (tag, returnValue) => {
ExtensionBlocklistMLBF._client.get = async function() {
// Record the method call.
observed.push(tag);
// Clone a valid record and tag it so we can identify it below.
let dummyRecord = JSON.parse(JSON.stringify(MLBF_RECORD));
dummyRecord.tagged = tag;
return [dummyRecord];
};
};
// Another significant part of updating is fetching the MLBF attachment.
// Add a spy too, so we can check which attachment is being requested.
const originalFetchMLBF = ExtensionBlocklistMLBF._fetchMLBF;
ExtensionBlocklistMLBF._fetchMLBF = async function(record) {
observed.push(`fetchMLBF:${record.tagged}`);
throw new Error(`Deliberately ignoring call to MLBF:${record.tagged}`);
};
spyClientGet("initial"); // Very first call = read RS.
let update1 = ExtensionBlocklistMLBF._updateMLBF(false);
spyClientGet("unexpected update2"); // Non-forced update = reuse update1.
let update2 = ExtensionBlocklistMLBF._updateMLBF(false);
spyClientGet("forced1"); // forceUpdate=true = supersede previous update.
let forcedUpdate1 = ExtensionBlocklistMLBF._updateMLBF(true);
spyClientGet("forced2"); // forceUpdate=true = supersede previous update.
let forcedUpdate2 = ExtensionBlocklistMLBF._updateMLBF(true);
let res = await Promise.all([update1, update2, forcedUpdate1, forcedUpdate2]);
Assert.equal(observed.length, 4, "expected number of observed events");
Assert.equal(observed[0], "initial", "First update should request records");
Assert.equal(observed[1], "forced1", "Forced update supersedes initial");
Assert.equal(observed[2], "forced2", "Forced update supersedes forced1");
// We call the _updateMLBF methods immediately after each other. Every update
// request starts with an asynchronous operation (looking up the RS records),
// so the implementation should return early for all update requests except
// for the last one. So we should only observe a fetch for the last request.
Assert.equal(observed[3], "fetchMLBF:forced2", "expected fetch result");
// All update requests should end up with the same result.
Assert.equal(res[0], res[1], "update1 == update2");
Assert.equal(res[1], res[2], "update2 == forcedUpdate1");
Assert.equal(res[2], res[3], "forcedUpdate1 == forcedUpdate2");
ExtensionBlocklistMLBF._client.get = originalClientGet;
ExtensionBlocklistMLBF._fetchMLBF = originalFetchMLBF;
});

6
toolkit/mozapps/extensions/test/xpcshell/rs-blocklist/xpcshell.ini
View File

@ -16,12 +16,6 @@ tags = remote-settings
[test_blocklist_metadata_filters.js]
# Bug 676992: test consistently hangs on Android
skip-if = os == "android"
[test_blocklist_mlbf.js]
[test_blocklist_mlbf_dump.js]
skip-if = os == "android" # addons-bloomfilters not yet listed in mobile/android/installer/package-manifest.in
[test_blocklist_mlbf_fetch.js]
[test_blocklist_mlbf_stashes.js]
[test_blocklist_mlbf_update.js]
[test_blocklist_osabi.js]
# Bug 676992: test consistently hangs on Android
skip-if = os == "android"