Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-24 13:21:05 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Backed out 2 changesets (bug 1725660, bug 1719560) for causing build bustages. CLOSED TREE

Browse Source 
Backed out changeset 6e11829eec98 (bug 1719560)
Backed out changeset 66571899587f (bug 1725660)
This commit is contained in:
Butkovits Atila 2022-04-27 22:59:47 +03:00
parent 00d62a96a1
commit 04a3fefa59
34 changed files with 161 additions and 616 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
services/common/tests/unit/xpcshell.ini
View File

@ -10,7 +10,7 @@ support-files =
[test_kinto.js]
tags = blocklist
[test_storage_adapter.js]
tags = remote-settings blocklist
tags = remote-settingsblocklist
[test_storage_adapter_shutdown.js]
tags = remote-settings blocklist

25
services/settings/RemoteSettingsClient.jsm
View File

@ -381,10 +381,6 @@ class RemoteSettingsClient extends EventEmitter {
);
await this.sync({ loadDump: false });
}
// Return `true` to indicate we don't need to `verifySignature`,
// since a trusted dump was loaded or a signature verification
// happened during synchronization.
return true;
})();
} else {
console.debug(`${this.identifier} Awaiting existing import.`);
@ -402,14 +398,7 @@ class RemoteSettingsClient extends EventEmitter {
);
if (!this._importingPromise) {
// As part of importing, any existing data is wiped.
this._importingPromise = (async () => {
const importedFromDump = await this._importJSONDump();
// Return `true` to skip signature verification if a dump was found.
// The dump can be missing if a collection is listed in the timestamps summary file,
// because its dump is present in the source tree, but the dump was not
// included in the `package-manifest.in` file. (eg. android, thunderbird)
return importedFromDump >= 0;
})();
this._importingPromise = this._importJSONDump();
} else {
console.debug(`${this.identifier} Awaiting existing import.`);
}
@ -418,11 +407,10 @@ class RemoteSettingsClient extends EventEmitter {
if (this._importingPromise) {
try {
if (await this._importingPromise) {
// No need to verify signature, because either we've just loaded a trusted
// dump (here or in a parallel call), or it was verified during sync.
verifySignature = false;
}
await this._importingPromise;
// No need to verify signature, because either we've just load a trusted
// dump (here or in a parallel call), or it was verified during sync.
verifySignature = false;
} catch (e) {
// Report error, but continue because there could have been data
// loaded from a parrallel call.
@ -828,6 +816,9 @@ class RemoteSettingsClient extends EventEmitter {
this.bucketName,
this.collectionName
);
console.info(
`${this.identifier} imported ${result} records from JSON dump`
);
if (gTimingEnabled) {
const end = Cu.now() * 1000;
PerformanceCounters.storeExecutionTime(

17
services/settings/RemoteSettingsWorker.js
View File

@ -72,7 +72,7 @@ const Agent = {
* @returns {int} Number of records loaded from dump or -1 if no dump found.
*/
async importJSONDump(bucket, collection) {
const { data: records, timestamp } = await SharedUtils.loadJSONDump(
const { data: records } = await SharedUtils.loadJSONDump(
bucket,
collection
);
@ -83,7 +83,7 @@ const Agent = {
if (gShutdown) {
throw new Error("Can't import when we've started shutting down.");
}
await importDumpIDB(bucket, collection, records, timestamp);
await importDumpIDB(bucket, collection, records);
return records.length;
},
@ -121,8 +121,8 @@ const Agent = {
}
},
_test_only_import(bucket, collection, records, timestamp) {
return importDumpIDB(bucket, collection, records, timestamp);
_test_only_import(bucket, collection, records) {
return importDumpIDB(bucket, collection, records);
},
};
@ -153,9 +153,8 @@ let gPendingTransactions = new Set();
* @param {String} bucket
* @param {String} collection
* @param {Array<Object>} records
* @param {Number} timestamp
*/
async function importDumpIDB(bucket, collection, records, timestamp) {
async function importDumpIDB(bucket, collection, records) {
// Open the DB. It will exist since if we are running this, it means
// we already tried to read the timestamp in `remote-settings.js`
const db = await IDBHelpers.openIDB(false /* do not allow upgrades */);
@ -173,7 +172,11 @@ async function importDumpIDB(bucket, collection, records, timestamp) {
records.forEach(item => {
item._cid = cid;
});
// Store the collection timestamp.
// Store the highest timestamp as the collection timestamp (or zero if dump is empty).
const timestamp =
records.length === 0
? 0
: Math.max(...records.map(record => record.last_modified));
let { transaction, promise } = IDBHelpers.executeIDB(
db,
[IDB_RECORDS_STORE, IDB_TIMESTAMPS_STORE],

2
services/settings/SharedUtils.jsm
View File

@ -50,7 +50,7 @@ var SharedUtils = {
response = await fetch(fileURI);
} catch (e) {
// Return null if file is missing.
return { data: null, timestamp: null };
return { data: null };
}
// Will throw if JSON is invalid.
return response.json();

28
services/settings/Utils.jsm
View File

@ -11,11 +11,11 @@ const { XPCOMUtils } = ChromeUtils.import(
const { ServiceRequest } = ChromeUtils.import(
"resource://gre/modules/ServiceRequest.jsm"
);
XPCOMUtils.defineLazyModuleGetters(this, {
SharedUtils: "resource://services-settings/SharedUtils.jsm",
AppConstants: "resource://gre/modules/AppConstants.jsm",
});
ChromeUtils.defineModuleGetter(
this,
"AppConstants",
"resource://gre/modules/AppConstants.jsm"
);
XPCOMUtils.defineLazyServiceGetter(
this,
@ -202,6 +202,7 @@ var Utils = {
*/
async hasLocalData(client) {
const timestamp = await client.db.getLastModified();
// Note: timestamp will be 0 if empty JSON dump is loaded.
return timestamp !== null;
},
@ -254,12 +255,17 @@ var Utils = {
const identifier = `${bucket}/${collection}`;
let lastModified = this._dumpStats[identifier];
if (lastModified === undefined) {
const { timestamp: dumpTimestamp } = await SharedUtils.loadJSONDump(
bucket,
collection
);
// Client recognize -1 as missing dump.
lastModified = dumpTimestamp ?? -1;
try {
let res = await fetch(
`resource://app/defaults/settings/${bucket}/${collection}.json`
);
let records = (await res.json()).data;
// Records in dumps are sorted by last_modified, newest first.
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
lastModified = records[0]?.last_modified || 0;
} catch (e) {
lastModified = -1;
}
this._dumpStats[identifier] = lastModified;
}
return lastModified;

3
services/settings/dumps/blocklists/addons-bloomfilters.json
View File

@ -674,6 +674,5 @@
"id": "7120faf8-4b30-4f60-8f63-4d7ddfc6daef",
"last_modified": 1639658383523
}
],
"timestamp": 1650371906091
]
}

3
services/settings/dumps/blocklists/addons.json
View File

@ -21956,6 +21956,5 @@
"id": "aad4545f-8f9d-dd53-2aa8-e8945cad6185",
"last_modified": 1480349192987
}
],
"timestamp": 1604940558744
]
}

3
services/settings/dumps/blocklists/gfx.json
View File

@ -1139,6 +1139,5 @@
"id": "0afc5fe9-1f81-7fde-2424-0cb4d933c173",
"last_modified": 1480349134090
}
],
"timestamp": 1643818378440
]
}

3
services/settings/dumps/blocklists/plugins.json
View File

@ -3510,6 +3510,5 @@
"id": "38797744-cb92-1a49-4c81-2a900691fdba",
"last_modified": 1480349146047
}
],
"timestamp": 1603126502200
]
}

65
services/settings/dumps/gen_last_modified.py
View File

@ -4,8 +4,6 @@
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import json
import glob
import os
import buildconfig
import mozpack.path as mozpath
@ -15,21 +13,23 @@ def get_last_modified(full_path_to_remote_settings_dump_file):
"""
Get the last_modified for the given file name.
- File must exist
- Must be a JSON dictionary with a data list and a timestamp,
e.g. `{"data": [], "timestamp": 42}`
- Must be a JSON dictionary with a data list, e.g. `{"data": []}`
- Every element in `data` should contain a "last_modified" key.
- The first element must have the highest "last_modified" value.
"""
with open(full_path_to_remote_settings_dump_file, "r") as f:
changeset = json.load(f)
records = json.load(f)["data"]
assert isinstance(records, list)
records = changeset["data"]
assert isinstance(records, list)
last_modified = changeset.get("timestamp")
assert isinstance(
last_modified, int
), f"{full_path_to_remote_settings_dump_file} is missing the timestamp. See Bug 1725660"
# Various RemoteSettings client code default to 0 when the set of
# records is empty (-1 is reserved for failures / non-existing files).
last_modified = 0
if records:
# Records in dumps are sorted by last_modified, newest first:
# https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304 # NOQA: E501
last_modified = records[0]["last_modified"]
assert isinstance(last_modified, int)
return last_modified
@ -54,30 +54,31 @@ def main(output):
"mobile/android",
"comm/mail",
"comm/suite",
), (
"Cannot determine location of Remote Settings "
f"dumps for platform {buildconfig.substs['MOZ_BUILD_APP']}"
)
dumps_locations = []
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
dumps_locations += ["services/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android":
dumps_locations += ["services/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":
dumps_locations += ["mozilla/services/settings/dumps/"]
dumps_locations += ["mail/app/settings/dumps/"]
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/suite":
dumps_locations += ["mozilla/services/settings/dumps/"]
remotesettings_dumps = {}
for dumps_location in dumps_locations:
dumps_root_dir = mozpath.join(buildconfig.topsrcdir, dumps_location)
for path in glob.iglob(mozpath.join(dumps_root_dir, "*/*.json")):
folder, filename = os.path.split(path)
bucket = os.path.basename(folder)
collection, _ = os.path.splitext(filename)
remotesettings_dumps[f"{bucket}/{collection}"] = path
# For simplicity, let's hardcode the path of the RemoteSettings dumps whose
# last_modified date is looked up.
# TODO bug 1719560: Replace hardcoded values with something more generic.
if buildconfig.substs["MOZ_BUILD_APP"] != "mobile/android":
# Until bug 1639050 is resolved, the dump isn't packaged with Android.
remotesettings_dumps["blocklists/addons-bloomfilters"] = mozpath.join(
buildconfig.topsrcdir,
"services/settings/dumps/blocklists/addons-bloomfilters.json",
)
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
# This is only packaged with browser.
remotesettings_dumps["main/search-config"] = mozpath.join(
buildconfig.topsrcdir,
"services/settings/dumps/main/search-config.json",
)
if buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":
# This is only packaged with Thunderbird.
remotesettings_dumps["main/search-config"] = mozpath.join(
buildconfig.topsrcdir,
"comm/mail/app/settings/dumps/thunderbird/search-config.json",
)
output_dict = {}
input_files = set()

3
services/settings/dumps/main/anti-tracking-url-decoration.json
View File

@ -6,6 +6,5 @@
"id": "60e82333-914d-4cfa-95b1-5f034b5a704b",
"last_modified": 1564511755134
}
],
"timestamp": 1564511755134
]
}

3
services/settings/dumps/main/devtools-compatibility-browsers.json
View File

@ -342,6 +342,5 @@
"id": "0299f77d-0498-481a-9896-498095a7e301",
"last_modified": 1645448267445
}
],
"timestamp": 1648570256624
]
}

5
services/settings/dumps/main/example.json
View File

@ -1,4 +1,3 @@
{
"data": [],
"timestamp": 1337
}
"data": []
}

3
services/settings/dumps/main/hijack-blocklists.json
View File

@ -54,6 +54,5 @@
"id": "homepage-urls",
"last_modified": 1605801189250
}
],
"timestamp": 1605801189258
]
}

3
services/settings/dumps/main/language-dictionaries.json
View File

@ -642,6 +642,5 @@
"id": "af",
"last_modified": 1539698843264
}
],
"timestamp": 1569410800356
]
}

3
services/settings/dumps/main/password-recipes.json
View File

@ -129,6 +129,5 @@
"id": "00814138-7ede-4f56-8953-b6d1c99d5f26",
"last_modified": 1600889016416
}
],
"timestamp": 1642005109349
]
}

303
services/settings/dumps/main/password-rules.json
View File

@ -1,299 +1,5 @@
{
"data": [
{
"Domain": "ae.com",
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit;",
"id": "1b76634d-c4dc-467f-8b1d-458c0753faf1",
"last_modified": 1650898337595
},
{
"Domain": "app.digio.in",
"password-rules": "minlength: 8; maxlength: 15;",
"id": "9199eb80-2ea2-4efc-974a-13646cd9e2bf",
"last_modified": 1650898337590
},
{
"Domain": "app.parkmobile.io",
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit; required: [!@#$%^&];",
"id": "267bc941-86ee-4558-9d72-3f875ff0ac95",
"last_modified": 1650898337584
},
{
"Domain": "areariservata.bancaetica.it",
"password-rules": "minlength: 8; maxlength: 10; required: lower; required: upper; required: digit; required: [!#&*+/=@_];",
"id": "1316e054-4002-41b3-bb25-c2af275f1b48",
"last_modified": 1650898337579
},
{
"Domain": "bcassessment.ca",
"password-rules": "minlength: 8; maxlength: 14;",
"id": "2b72ecba-f8e4-4cc4-9c2c-01ad7b8eff60",
"last_modified": 1650898337573
},
{
"Domain": "belkin.com",
"password-rules": "minlength: 8; required: lower, upper; required: digit; required: [$!@~_,%&];",
"id": "89e880c3-2742-4802-bd33-ea506d3b7020",
"last_modified": 1650898337568
},
{
"Domain": "bilibili.com",
"password-rules": "maxlength: 16;",
"id": "3867a12f-b3e8-4343-94eb-6ddefba5b1a5",
"last_modified": 1650898337562
},
{
"Domain": "billerweb.com",
"password-rules": "minlength: 8; max-consecutive: 2; required: digit; required: upper,lower;",
"id": "6d89d12e-2ea9-4077-9dbc-c2566ccca370",
"last_modified": 1650898337557
},
{
"Domain": "bochk.com",
"password-rules": "minlength: 8; maxlength: 12; max-consecutive: 3; required: lower; required: upper; required: digit; allowed: [#$%&()*+,.:;<=>?@_];",
"id": "2a133fcd-fa66-473c-a09b-07014d0d3ccd",
"last_modified": 1650898337551
},
{
"Domain": "collectivehealth.com",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit;",
"id": "b86c63d8-60e4-4c8a-965d-56af269f6b76",
"last_modified": 1650898337546
},
{
"Domain": "dan.org",
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit; required: [!@$%^&*];",
"id": "b7ca591b-c242-4520-a849-ccd74a90076a",
"last_modified": 1650898337540
},
{
"Domain": "dbs.com.hk",
"password-rules": "minlength: 8; maxlength: 30; required: lower; required: upper; required: digit;",
"id": "fe2f51c2-8ccb-4ee4-97cb-73288337f6ab",
"last_modified": 1650898337535
},
{
"Domain": "equifax.com",
"password-rules": "minlength: 8; maxlength: 20; required: lower; required: upper; required: digit; required: [!$*+@];",
"id": "ce20ebcc-b423-45f5-9643-f4651c55a133",
"last_modified": 1650898337530
},
{
"Domain": "fidelity.com",
"password-rules": "minlength: 6; maxlength: 20; required: lower; allowed: upper,digit,[!$%'()+,./:;=?@^_|~];",
"id": "51bcb63b-7da1-4dd5-a024-e395d32e3c36",
"last_modified": 1650898337524
},
{
"Domain": "flysas.com",
"password-rules": "minlength: 8; maxlength: 14; required: lower; required: upper; required: digit; required: [-~!@#$%^&_+=`|(){}[:\"'<>,.?]];",
"id": "c7b74d79-d787-4575-bfb4-92b531f422d9",
"last_modified": 1650898337519
},
{
"Domain": "gamestop.com",
"password-rules": "minlength: 8; maxlength: 225; required: lower; required: upper; required: digit; required: [!@#$%];",
"id": "ef9def1f-a244-4cc1-8937-f42865ce62c7",
"last_modified": 1650898337513
},
{
"Domain": "hangseng.com",
"password-rules": "minlength: 8; maxlength: 30; required: lower; required: upper; required: digit;",
"id": "bb13b29e-8fa0-4d68-b400-e546834a30a7",
"last_modified": 1650898337508
},
{
"Domain": "hkbea.com",
"password-rules": "minlength: 8; maxlength: 12; required: lower; required: upper; required: digit;",
"id": "a771074d-6e15-42aa-811a-731930d173b5",
"last_modified": 1650898337502
},
{
"Domain": "hkexpress.com",
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: upper; required: digit; required: special;",
"id": "94abbd2d-2201-43f1-b3c8-33900f207b4c",
"last_modified": 1650898337497
},
{
"Domain": "hsbc.com.hk",
"password-rules": "minlength: 6; maxlength: 30; required: lower; required: upper; required: digit; allowed: ['.@_];",
"id": "b657a5f0-268c-4acd-85fe-5085bcdb380c",
"last_modified": 1650898337492
},
{
"Domain": "kfc.ca",
"password-rules": "minlength: 6; maxlength: 15; required: lower; required: upper; required: digit; required: [!@#$%&?*];",
"id": "ca2e08a6-cc00-47a4-8517-737552dfedab",
"last_modified": 1650898337486
},
{
"Domain": "launtel.net.au",
"password-rules": "minlength: 8; required: digit; required: digit; allowed: lower, upper;",
"id": "eede7e13-6f0b-408a-bf1d-292130631834",
"last_modified": 1650898337480
},
{
"Domain": "lg.com",
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit; allowed: [-!#$%&'()*+,.:;=?@[^_{|}~]];",
"id": "fb2b3324-dd2d-4518-a309-af50f685689a",
"last_modified": 1650898337475
},
{
"Domain": "lloydsbank.co.uk",
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: digit; allowed: upper;",
"id": "56265f6f-6348-4900-8abf-07c4ab2e1c72",
"last_modified": 1650898337470
},
{
"Domain": "loyalty.accor.com",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!#$%&=@];",
"id": "403e9c7d-1dbc-44f4-a825-81aa4127afca",
"last_modified": 1650898337464
},
{
"Domain": "lsacsso.b2clogin.com",
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit, [-!#$%&*?@^_];",
"id": "369eb957-3b3f-4377-ac76-52026dfffbfa",
"last_modified": 1650898337458
},
{
"Domain": "medicare.gov",
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit; required: [@!$%^*()];",
"id": "27a56617-f363-4455-a60c-8582af732012",
"last_modified": 1650898337453
},
{
"Domain": "mpv.tickets.com",
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: upper; required: digit;",
"id": "9deab323-bc0f-4433-bbc3-e4d26dbfa7e0",
"last_modified": 1650898337448
},
{
"Domain": "my.konami.net",
"password-rules": "minlength: 8; maxlength: 32; required: lower; required: upper; required: digit;",
"id": "6a62b70b-6412-4ff4-945a-168dfc38b634",
"last_modified": 1650898337443
},
{
"Domain": "order.wendys.com",
"password-rules": "minlength: 6; maxlength: 20; required: lower; required: upper; required: digit; allowed: [!#$%&()*+/=?^_{}];",
"id": "57882459-1180-4faf-a8dd-c7475bcfeb0f",
"last_modified": 1650898337437
},
{
"Domain": "preferredhotels.com",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!#$%&()*+@^_];",
"id": "4290b8b6-dd7c-431b-a4cb-36e661f72423",
"last_modified": 1650898337432
},
{
"Domain": "premier.ticketek.com.au",
"password-rules": "minlength: 6; maxlength: 16;",
"id": "26b1c942-2d06-496b-984b-65044ad2f1ab",
"last_modified": 1650898337427
},
{
"Domain": "premierinn.com",
"password-rules": "minlength: 8; required: upper; required: digit; allowed: lower;",
"id": "f6663626-8861-4bfc-a704-5ff1e23e9e22",
"last_modified": 1650898337421
},
{
"Domain": "prestocard.ca",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit,[!\"#$%&'()*+,<>?@];",
"id": "ed580dc6-16af-4ec5-8ae2-b40fe99f0260",
"last_modified": 1650898337416
},
{
"Domain": "questdiagnostics.com",
"password-rules": "minlength: 8; maxlength: 30; required: upper, lower; required: digit, [!#$%&()*+<>?@^_~];",
"id": "a331ccb9-a2b8-44dd-bc35-0878eb51b4fe",
"last_modified": 1650898337411
},
{
"Domain": "renaud-bray.com",
"password-rules": "minlength: 8; maxlength: 38; allowed: upper,lower,digit;",
"id": "210a8aea-1089-487d-bcaa-3c937374f650",
"last_modified": 1650898337405
},
{
"Domain": "rogers.com",
"password-rules": "minlength: 8; required: lower, upper; required: digit; required: [!@#$];",
"id": "a13b20d6-0b68-4a05-866a-e8a14adee208",
"last_modified": 1650898337400
},
{
"Domain": "ruc.dk",
"password-rules": "minlength: 6; maxlength: 8; required: lower, upper; required: [-!#%&(){}*+;%/<=>?_];",
"id": "85d4c3e2-7f8c-480d-a20e-c55aee12a2e7",
"last_modified": 1650898337394
},
{
"Domain": "runescape.com",
"password-rules": "minlength: 5; maxlength: 20; required: lower; required: upper; required: digit;",
"id": "e14ecd5a-951f-4c27-bf45-0ee3fb830032",
"last_modified": 1650898337389
},
{
"Domain": "salslimo.com",
"password-rules": "minlength: 8; maxlength: 50; required: upper; required: lower; required: digit; required: [!@#$&*];",
"id": "afa90771-c472-48cc-96d3-21c770bb958c",
"last_modified": 1650898337384
},
{
"Domain": "secure.snnow.ca",
"password-rules": "minlength: 7; maxlength: 16; required: digit; allowed: lower, upper;",
"id": "f6a3e835-190e-4043-a070-943d821f708e",
"last_modified": 1650898337378
},
{
"Domain": "tix.soundrink.com",
"password-rules": "minlength: 6; maxlength: 16;",
"id": "0cb09a67-566e-492c-a4d7-af473e964472",
"last_modified": 1650898337373
},
{
"Domain": "training.confluent.io",
"password-rules": "minlength: 6; maxlength: 16; required: lower; required: upper; required: digit; allowed: [!#$%*@^_~];",
"id": "d00c2ee9-6f05-495c-9833-bae526019ddc",
"last_modified": 1650898337368
},
{
"Domain": "usps.com",
"password-rules": "minlength: 8; maxlength: 50; max-consecutive: 2; required: lower; required: upper; required: digit; allowed: [-!\"#&'()+,./?@];",
"id": "caacf36d-df98-43ab-8d5c-d7d40d79d696",
"last_modified": 1650898337362
},
{
"Domain": "visabenefits-auth.axa-assistance.us",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!\"#$%&()*,.:<>?@^{|}];",
"id": "3afe232d-ff3d-4e3d-b6bd-da947dac7884",
"last_modified": 1650898337357
},
{
"Domain": "wegmans.com",
"password-rules": "minlength: 8; required: digit; required: upper,lower; required: [!#$%&*+=?@^];",
"id": "d7036bd9-5301-43a1-b427-25a876ff4c07",
"last_modified": 1650898337352
},
{
"Domain": "xvoucher.com",
"password-rules": "minlength: 11; required: upper; required: digit; required: [!@#$%&_];",
"id": "85cc3546-8a5c-4400-bd01-c0386897a442",
"last_modified": 1650898337346
},
{
"Domain": "zara.com",
"password-rules": "minlength: 8; required: lower; required: upper; required: digit;",
"id": "4496846f-b282-4965-97f8-97c2b814881a",
"last_modified": 1650898337341
},
{
"Domain": "americanexpress.com",
"password-rules": "minlength: 8; maxlength: 20; max-consecutive: 4; required: lower, upper; required: digit; allowed: [%&_?#=];",
"id": "986919ff-d801-45f6-ab75-af40e0c9e0ad",
"last_modified": 1650898337335
},
{
"Domain": "163.com",
"password-rules": "minlength: 6; maxlength: 16;",
@ -354,6 +60,12 @@
"id": "46f95801-6c08-404b-99f6-ac36b347e45b",
"last_modified": 1624479577522
},
{
"Domain": "americanexpress.com",
"password-rules": "minlength: 8; maxlength: 20; max-consecutive: 4; required: lower, upper; required: digit; allowed: [-%&_?#=];",
"id": "986919ff-d801-45f6-ab75-af40e0c9e0ad",
"last_modified": 1624479577517
},
{
"Domain": "anatel.gov.br",
"password-rules": "minlength: 6; maxlength: 15; allowed: lower, upper, digit;",
@ -1560,6 +1272,5 @@
"id": "5fe248fb-3e63-4300-8f67-3e6006af0e87",
"last_modified": 1624479576629
}
],
"timestamp": 1650898337595
]
}

3
services/settings/dumps/main/search-config.json
View File

@ -2907,6 +2907,5 @@
"id": "43559b11-05dc-4750-b131-afdbd9d25905",
"last_modified": 1583937832210
}
],
"timestamp": 1648132005528
]
}

3
services/settings/dumps/main/search-default-override-allowlist.json
View File

@ -12,6 +12,5 @@
"id": "c5de2b3f-da15-4bc5-b36f-9555043b5ef3",
"last_modified": 1595254618540
}
],
"timestamp": 1595254618540
]
}

3
services/settings/dumps/main/search-telemetry-v2.json
View File

@ -149,6 +149,5 @@
"id": "19c434a3-d173-4871-9743-290ac92a3f6a",
"last_modified": 1643136933989
}
],
"timestamp": 1647619735693
]
}

3
services/settings/dumps/main/sites-classification.json
View File

@ -458,6 +458,5 @@
"id": "48ec9db9-0de1-49aa-9b82-1a2372dce31e",
"last_modified": 1539907283705
}
],
"timestamp": 1544035467383
]
}

3
services/settings/dumps/main/top-sites.json
View File

@ -396,6 +396,5 @@
"id": "89e3da9e-1d84-4c24-beab-d0692fc7cc45",
"last_modified": 1599146777345
}
],
"timestamp": 1647020600359
]
}

3
services/settings/dumps/main/url-classifier-skip-urls.json
View File

@ -1,4 +1,3 @@
{
"data": [],
"timestamp": 1606870304609
"data": []
}

204
services/settings/dumps/main/websites-with-shared-credential-backends.json
View File

@ -2,23 +2,16 @@
"data": [
{
"relatedRealms": [
[
"3docean.net",
"audiojungle.net",
"codecanyon.net",
"envato.com",
"graphicriver.net",
"photodune.net",
"placeit.net",
"themeforest.net",
"tutsplus.com",
"videohive.net"
],
[
"aa.com",
"americanairlines.com",
"americanairlines.jp"
],
[
"airfrance.com",
"klm.com",
"flyingblue.com"
],
[
"aetna.com",
"banneraetna.myplanportal.com"
@ -78,33 +71,6 @@
"airbnb.com",
"airbnb.co.ve"
],
[
"airfrance.com",
"klm.com",
"flyingblue.com"
],
[
"airnewzealand.co.nz",
"airnewzealand.com",
"airnewzealand.com.au"
],
[
"albertsons.com",
"acmemarkets.com",
"carrsqc.com",
"jewelosco.com",
"pavilions.com",
"randalls.com",
"safeway.com",
"shaws.com",
"starmarket.com",
"tomthumb.com",
"vons.com"
],
[
"alibaba.com",
"aliexpress.com"
],
[
"alltrails.com",
"alltrails.io"
@ -127,8 +93,7 @@
"amazon.sa",
"amazon.sg",
"amazon.se",
"amazon.pl",
"ring.com"
"amazon.pl"
],
[
"amcrestcloud.com",
@ -152,26 +117,14 @@
"apple.com",
"icloud.com"
],
[
"atlassian.com",
"trello.com"
],
[
"att.com",
"att.net"
],
[
"audi.com",
"audiusa.com"
],
[
"bahn.de",
"bahn.com"
],
[
"battle.net",
"blizzard.com"
],
[
"beachbodyondemand.com",
"teambeachbody.com"
@ -195,21 +148,13 @@
"whistlerblackcomb.com"
],
[
"boingo.com",
"boingohotspot.com"
],
[
"bol.com",
"kobo.com"
"battle.net",
"blizzard.com"
],
[
"boudinbakery.com",
"boudincatering.com"
],
[
"braze.com",
"braze.eu"
],
[
"capitalone.com",
"capitalone360.com"
@ -243,12 +188,8 @@
"dinersclubus.com"
],
[
"discordapp.com",
"discord.com"
],
[
"discordmerch.com",
"discord.store"
"discord.com",
"discordapp.com"
],
[
"dish.com",
@ -263,15 +204,6 @@
"dropbox.com",
"getdropbox.com"
],
[
"e.leclerc",
"auto.leclerc",
"location.leclerc",
"chezmoi.lelcerc",
"traiteur.lelcerc",
"mesrecettes.lelcerc",
"leclercdrive.fr"
],
[
"eater.com",
"polygon.com",
@ -302,10 +234,6 @@
"ebay.pl",
"ebay.vn"
],
[
"eurosport.no",
"eurosportplayer.com"
],
[
"eventbrite.at",
"eventbrite.be",
@ -338,14 +266,6 @@
"fandangonow.com",
"fandango.com"
],
[
"fidelity.com",
"fidelityinvestments.com"
],
[
"flyblade.com",
"blade.com"
],
[
"fnac.com",
"fnacspectacles.com"
@ -367,20 +287,6 @@
"gogoair.com",
"gogoinflight.com"
],
[
"hbo.com",
"hbomax.com",
"hbonow.com"
],
[
"igen.fr",
"watchgeneration.fr",
"macg.co"
],
[
"ikonpass.com",
"skilynx.com"
],
[
"intuit.com",
"mint.com"
@ -413,12 +319,6 @@
"lookmark.io",
"lookmark.link"
],
[
"lrz.de",
"mwn.de",
"mytum.de",
"tum.de"
],
[
"lufthansa.com",
"miles-and-more.com"
@ -436,8 +336,7 @@
"microsoftonline.com",
"office.com",
"skype.com",
"onenote.com",
"hotmail.com"
"onenote.com"
],
[
"minecraft.net",
@ -447,20 +346,6 @@
"mytotalconnectcomfort.com",
"tccna.honeywell.com"
],
[
"myuhc.com",
"uhc.com",
"optum.com",
"optumrx.com"
],
[
"neatorama.com",
"neatoshop.com"
],
[
"newyorker.com",
"vanityfair.com"
],
[
"nintendolife.com",
"purexbox.com",
@ -473,17 +358,12 @@
"nsn.com"
],
[
"nordvpn.com",
"nordpass.com",
"nordaccount.com"
"norwegianreward.com",
"norwegian.com"
],
[
"norwegian.com",
"norwegianreward.com"
],
[
"olo.com",
"olo.express"
"optumrx.com",
"optum.com"
],
[
"pinterest.com",
@ -523,26 +403,10 @@
"protonmail.com",
"protonvpn.com"
],
[
"qnap.com",
"myqnapcloud.com"
],
[
"questdiagnostics.com",
"care360.com"
],
[
"rocketaccount.com",
"rocketmortgage.com"
],
[
"scholarshare529.com",
"secureaccountview.com"
],
[
"scoutingevent.com",
"campreservation.com"
],
[
"scribbr.com",
"scribbr.de",
@ -567,9 +431,7 @@
[
"shopdisney.com",
"go.com",
"disneyplus.com",
"espn.com",
"disneystore.com"
"disneyplus.com"
],
[
"slcl.overdrive.com",
@ -622,6 +484,10 @@
"tp-link.com",
"tplinkcloud.com"
],
[
"uhc.com",
"myuhc.com"
],
[
"umsystem.edu",
"mst.edu",
@ -634,11 +500,6 @@
"unitedwifi.com"
],
[
"uspowerboating.com",
"ussailing.org"
],
[
"verizon.com",
"verizonwireless.com",
"vzw.com"
],
@ -650,14 +511,14 @@
"perigold.com",
"birchlane.com"
],
[
"wellsfargo.com",
"wellsfargoadvisors.com"
],
[
"wiimmfi.de",
"wii-homebrew.com"
],
[
"wellsfargo.com",
"wellsfargoadvisors.com"
],
[
"wikipedia.org",
"mediawiki.org",
@ -697,23 +558,10 @@
[
"wsj.com",
"dowjones.com"
],
[
"www.vistaprint.ca",
"account.vistaprint.com"
],
[
"yahoo.com",
"flickr.com"
],
[
"zixmail.net",
"zixmessagecenter.com"
]
],
"id": "8c3d4151-8e68-4bb3-a3fd-babf4aba2cdc",
"last_modified": 1650898092205
"last_modified": 1624479523629
}
],
"timestamp": 1650898092205
]
}

3
services/settings/dumps/security-state/intermediates.json
View File

@ -23652,6 +23652,5 @@
"id": "586b6f01-c7f7-4a5e-ba71-e7dca3e77add",
"last_modified": 1559865863642
}
],
"timestamp": 1650877037744
]
}

3
services/settings/dumps/security-state/onecrl.json
View File

@ -21780,6 +21780,5 @@
"id": "ae8bec3c-3b92-822e-53f1-68394cbb1758",
"last_modified": 1480349158647
}
],
"timestamp": 1648145180742
]
}

3
services/settings/static-dumps/main/doh-config.json
View File

@ -10,6 +10,5 @@
"id": "global",
"last_modified": 1621943462970
}
],
"timestamp": 1621943462970
]
}

3
services/settings/static-dumps/main/doh-providers.json
View File

@ -18,6 +18,5 @@
"id": "cloudflare-global",
"last_modified": 1621943542615
}
],
"timestamp": 1621943542621
]
}

43
services/settings/test/unit/test_remote_settings_dump_lastmodified.js
View File

@ -1,21 +1,27 @@
"use strict";
const { AppConstants } = ChromeUtils.import(
"resource://gre/modules/AppConstants.jsm"
);
const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm");
Cu.importGlobalProperties(["fetch"]);
async function getLocalDumpLastModified(bucket, collection) {
let res;
try {
res = await fetch(
`resource://app/defaults/settings/${bucket}/${collection}.json`
);
} catch (e) {
return -1;
let res = await fetch(
`resource://app/defaults/settings/${bucket}/${collection}.json`
);
let records = (await res.json()).data;
if (records.some(r => r.last_modified > records[0].last_modified)) {
// The dump importer should ensure that the newest record is at the front:
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
ok(false, `${bucket}/${collection} - newest record should be in the front`);
}
const { timestamp } = await res.json();
ok(timestamp >= 0, `${bucket}/${collection} dump has timestamp`);
return timestamp;
return records.reduce(
(max, { last_modified }) => Math.max(last_modified, max),
0
);
}
add_task(async function lastModified_of_non_existing_dump() {
@ -32,24 +38,23 @@ add_task(async function lastModified_of_non_existing_dump() {
});
add_task(async function lastModified_summary_is_correct() {
if (AppConstants.platform == "android") {
// TODO bug 1719560: When implemented, remove this condition.
equal(JSON.stringify(Utils._dumpStats), "{}", "No dumps on Android yet");
return;
}
ok(Object.keys(Utils._dumpStats).length > 0, "Contains summary of dumps");
let checked = 0;
for (let [identifier, lastModified] of Object.entries(Utils._dumpStats)) {
let [bucket, collection] = identifier.split("/");
let actual = await getLocalDumpLastModified(bucket, collection);
if (actual < 0) {
info(`${identifier} has no dump, skip.`);
continue;
}
info(`Checking correctness of ${identifier}`);
let [bucket, collection] = identifier.split("/");
equal(
await Utils.getLocalDumpLastModified(bucket, collection),
lastModified,
`Expected last_modified value for ${identifier}`
);
let actual = await getLocalDumpLastModified(bucket, collection);
equal(lastModified, actual, `last_modified should match collection`);
checked++;
}
ok(checked > 0, "At least one dump was packaged and checked.");
});

11
services/settings/test/unit/test_remote_settings_offline.js
View File

@ -26,10 +26,12 @@ add_task(async function setup() {
bucketName: TEST_BUCKET,
});
const dump = await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION);
DUMP_RECORDS = dump.data;
DUMP_LAST_MODIFIED = dump.timestamp;
DUMP_RECORDS = (await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION))
.data;
DUMP_LAST_MODIFIED = DUMP_RECORDS.reduce(
(max, { last_modified }) => Math.max(last_modified, max),
-Infinity
);
// Dumps are fetched via the following, which sorts the records, newest first.
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
equal(
@ -44,7 +46,6 @@ async function importData(records) {
TEST_BUCKET,
TEST_COLLECTION,
records,
records[0]?.last_modified || 0,
]);
}

2
services/settings/test/unit/test_shutdown_handling.js
View File

@ -108,7 +108,7 @@ add_task(async function test_shutdown_worker() {
let records = [{}];
let importPromise = RemoteSettingsWorker._execute(
"_test_only_import",
["main", "language-dictionaries", records, 0],
["main", "language-dictionaries", records],
{ mustComplete: true }
);
let stringifyPromise = RemoteSettingsWorker.canonicalStringify(

10
taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh
View File

@ -290,9 +290,9 @@ function compare_remote_settings_files {
# 1. List remote settings collections from server.
echo "INFO: fetch remote settings list from server"
${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\
# 2. For each entry ${bucket, collection, last_modified}
while IFS="/" read -r bucket collection last_modified; do
${JQ} -r '.data[] | .bucket+"/"+.collection' |\
# 2. For each entry ${bucket, collection}
while IFS="/" read -r bucket collection; do
# 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
@ -307,10 +307,10 @@ function compare_remote_settings_files {
fi
# 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}"
remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}"
${WGET} -qO- "$remote_records_url" | ${JQ} . > "${local_location_output}"
# 5. Download attachments if needed.
if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then

1
toolkit/components/search/tests/xpcshell/searchconfigs/test_selector_db_out_of_date.js
View File

@ -35,7 +35,6 @@ add_task(async function test_selector_db_out_of_date() {
last_modified: 1606227264000,
},
],
1606227264000,
]);
// Now load the configuration and check we get what we expect.

2
toolkit/components/search/tests/xpcshell/searchconfigs/xpcshell.ini
View File

@ -4,7 +4,7 @@ head = head_searchconfig.js
dupe-manifest =
support-files =
../../../../../../browser/locales/all-locales
tags=searchconfig remote-settings
tags=searchconfig
# These are extensive tests, we don't need to run them on asan/tsan.
# They are also skipped for mobile and Thunderbird as these are specifically
# testing the Firefox config at the moment.