mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-11-24 13:21:05 +00:00
Backed out 2 changesets (bug 1719560, bug 1725660) for causing build bustages CLOSED TREE
Backed out changeset 63407a7e0e90 (bug 1719560) Backed out changeset 3c455478efb6 (bug 1725660)
This commit is contained in:
Noemi Erli
parent
ae79166df6
commit
24d64734ac
@ -10,7 +10,7 @@ support-files =
|
||||
[test_kinto.js]
|
||||
tags = blocklist
|
||||
[test_storage_adapter.js]
|
||||
tags = remote-settings blocklist
|
||||
tags = remote-settingsblocklist
|
||||
[test_storage_adapter_shutdown.js]
|
||||
tags = remote-settings blocklist
|
||||
|
||||
|
||||
@ -381,10 +381,6 @@ class RemoteSettingsClient extends EventEmitter {
|
||||
);
|
||||
await this.sync({ loadDump: false });
|
||||
}
|
||||
// Return `true` to indicate we don't need to `verifySignature`,
|
||||
// since a trusted dump was loaded or a signature verification
|
||||
// happened during synchronization.
|
||||
return true;
|
||||
})();
|
||||
} else {
|
||||
console.debug(`${this.identifier} Awaiting existing import.`);
|
||||
@ -402,14 +398,7 @@ class RemoteSettingsClient extends EventEmitter {
|
||||
);
|
||||
if (!this._importingPromise) {
|
||||
// As part of importing, any existing data is wiped.
|
||||
this._importingPromise = (async () => {
|
||||
const importedFromDump = await this._importJSONDump();
|
||||
// Return `true` to skip signature verification if a dump was found.
|
||||
// The dump can be missing if a collection is listed in the timestamps summary file,
|
||||
// because its dump is present in the source tree, but the dump was not
|
||||
// included in the `package-manifest.in` file. (eg. android, thunderbird)
|
||||
return importedFromDump >= 0;
|
||||
})();
|
||||
this._importingPromise = this._importJSONDump();
|
||||
} else {
|
||||
console.debug(`${this.identifier} Awaiting existing import.`);
|
||||
}
|
||||
@ -418,11 +407,10 @@ class RemoteSettingsClient extends EventEmitter {
|
||||
|
||||
if (this._importingPromise) {
|
||||
try {
|
||||
if (await this._importingPromise) {
|
||||
// No need to verify signature, because either we've just loaded a trusted
|
||||
// dump (here or in a parallel call), or it was verified during sync.
|
||||
verifySignature = false;
|
||||
}
|
||||
await this._importingPromise;
|
||||
// No need to verify signature, because either we've just load a trusted
|
||||
// dump (here or in a parallel call), or it was verified during sync.
|
||||
verifySignature = false;
|
||||
} catch (e) {
|
||||
// Report error, but continue because there could have been data
|
||||
// loaded from a parrallel call.
|
||||
@ -828,6 +816,9 @@ class RemoteSettingsClient extends EventEmitter {
|
||||
this.bucketName,
|
||||
this.collectionName
|
||||
);
|
||||
console.info(
|
||||
`${this.identifier} imported ${result} records from JSON dump`
|
||||
);
|
||||
if (gTimingEnabled) {
|
||||
const end = Cu.now() * 1000;
|
||||
PerformanceCounters.storeExecutionTime(
|
||||
|
||||
@ -72,7 +72,7 @@ const Agent = {
|
||||
* @returns {int} Number of records loaded from dump or -1 if no dump found.
|
||||
*/
|
||||
async importJSONDump(bucket, collection) {
|
||||
const { data: records, timestamp } = await SharedUtils.loadJSONDump(
|
||||
const { data: records } = await SharedUtils.loadJSONDump(
|
||||
bucket,
|
||||
collection
|
||||
);
|
||||
@ -83,7 +83,7 @@ const Agent = {
|
||||
if (gShutdown) {
|
||||
throw new Error("Can't import when we've started shutting down.");
|
||||
}
|
||||
await importDumpIDB(bucket, collection, records, timestamp);
|
||||
await importDumpIDB(bucket, collection, records);
|
||||
return records.length;
|
||||
},
|
||||
|
||||
@ -121,8 +121,8 @@ const Agent = {
|
||||
}
|
||||
},
|
||||
|
||||
_test_only_import(bucket, collection, records, timestamp) {
|
||||
return importDumpIDB(bucket, collection, records, timestamp);
|
||||
_test_only_import(bucket, collection, records) {
|
||||
return importDumpIDB(bucket, collection, records);
|
||||
},
|
||||
};
|
||||
|
||||
@ -153,9 +153,8 @@ let gPendingTransactions = new Set();
|
||||
* @param {String} bucket
|
||||
* @param {String} collection
|
||||
* @param {Array<Object>} records
|
||||
* @param {Number} timestamp
|
||||
*/
|
||||
async function importDumpIDB(bucket, collection, records, timestamp) {
|
||||
async function importDumpIDB(bucket, collection, records) {
|
||||
// Open the DB. It will exist since if we are running this, it means
|
||||
// we already tried to read the timestamp in `remote-settings.js`
|
||||
const db = await IDBHelpers.openIDB(false /* do not allow upgrades */);
|
||||
@ -173,7 +172,11 @@ async function importDumpIDB(bucket, collection, records, timestamp) {
|
||||
records.forEach(item => {
|
||||
item._cid = cid;
|
||||
});
|
||||
// Store the collection timestamp.
|
||||
// Store the highest timestamp as the collection timestamp (or zero if dump is empty).
|
||||
const timestamp =
|
||||
records.length === 0
|
||||
? 0
|
||||
: Math.max(...records.map(record => record.last_modified));
|
||||
let { transaction, promise } = IDBHelpers.executeIDB(
|
||||
db,
|
||||
[IDB_RECORDS_STORE, IDB_TIMESTAMPS_STORE],
|
||||
|
||||
@ -50,7 +50,7 @@ var SharedUtils = {
|
||||
response = await fetch(fileURI);
|
||||
} catch (e) {
|
||||
// Return null if file is missing.
|
||||
return { data: null, timestamp: null };
|
||||
return { data: null };
|
||||
}
|
||||
// Will throw if JSON is invalid.
|
||||
return response.json();
|
||||
|
||||
@ -11,11 +11,11 @@ const { XPCOMUtils } = ChromeUtils.import(
|
||||
const { ServiceRequest } = ChromeUtils.import(
|
||||
"resource://gre/modules/ServiceRequest.jsm"
|
||||
);
|
||||
|
||||
XPCOMUtils.defineLazyModuleGetters(this, {
|
||||
SharedUtils: "resource://services-settings/SharedUtils.jsm",
|
||||
AppConstants: "resource://gre/modules/AppConstants.jsm",
|
||||
});
|
||||
ChromeUtils.defineModuleGetter(
|
||||
this,
|
||||
"AppConstants",
|
||||
"resource://gre/modules/AppConstants.jsm"
|
||||
);
|
||||
|
||||
XPCOMUtils.defineLazyServiceGetter(
|
||||
this,
|
||||
@ -202,6 +202,7 @@ var Utils = {
|
||||
*/
|
||||
async hasLocalData(client) {
|
||||
const timestamp = await client.db.getLastModified();
|
||||
// Note: timestamp will be 0 if empty JSON dump is loaded.
|
||||
return timestamp !== null;
|
||||
},
|
||||
|
||||
@ -254,12 +255,17 @@ var Utils = {
|
||||
const identifier = `${bucket}/${collection}`;
|
||||
let lastModified = this._dumpStats[identifier];
|
||||
if (lastModified === undefined) {
|
||||
const { timestamp: dumpTimestamp } = await SharedUtils.loadJSONDump(
|
||||
bucket,
|
||||
collection
|
||||
);
|
||||
// Client recognize -1 as missing dump.
|
||||
lastModified = dumpTimestamp ?? -1;
|
||||
try {
|
||||
let res = await fetch(
|
||||
`resource://app/defaults/settings/${bucket}/${collection}.json`
|
||||
);
|
||||
let records = (await res.json()).data;
|
||||
// Records in dumps are sorted by last_modified, newest first.
|
||||
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
|
||||
lastModified = records[0]?.last_modified || 0;
|
||||
} catch (e) {
|
||||
lastModified = -1;
|
||||
}
|
||||
this._dumpStats[identifier] = lastModified;
|
||||
}
|
||||
return lastModified;
|
||||
|
||||
@ -674,6 +674,5 @@
|
||||
"id": "7120faf8-4b30-4f60-8f63-4d7ddfc6daef",
|
||||
"last_modified": 1639658383523
|
||||
}
|
||||
],
|
||||
"timestamp": 1650371906091
|
||||
]
|
||||
}
|
||||
|
||||
@ -21956,6 +21956,5 @@
|
||||
"id": "aad4545f-8f9d-dd53-2aa8-e8945cad6185",
|
||||
"last_modified": 1480349192987
|
||||
}
|
||||
],
|
||||
"timestamp": 1604940558744
|
||||
]
|
||||
}
|
||||
|
||||
@ -1139,6 +1139,5 @@
|
||||
"id": "0afc5fe9-1f81-7fde-2424-0cb4d933c173",
|
||||
"last_modified": 1480349134090
|
||||
}
|
||||
],
|
||||
"timestamp": 1643818378440
|
||||
]
|
||||
}
|
||||
|
||||
@ -3510,6 +3510,5 @@
|
||||
"id": "38797744-cb92-1a49-4c81-2a900691fdba",
|
||||
"last_modified": 1480349146047
|
||||
}
|
||||
],
|
||||
"timestamp": 1603126502200
|
||||
]
|
||||
}
|
||||
|
||||
@ -4,8 +4,6 @@
|
||||
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
import json
|
||||
import glob
|
||||
import os
|
||||
|
||||
import buildconfig
|
||||
import mozpack.path as mozpath
|
||||
@ -15,21 +13,23 @@ def get_last_modified(full_path_to_remote_settings_dump_file):
|
||||
"""
|
||||
Get the last_modified for the given file name.
|
||||
- File must exist
|
||||
- Must be a JSON dictionary with a data list and a timestamp,
|
||||
e.g. `{"data": [], "timestamp": 42}`
|
||||
- Must be a JSON dictionary with a data list, e.g. `{"data": []}`
|
||||
- Every element in `data` should contain a "last_modified" key.
|
||||
- The first element must have the highest "last_modified" value.
|
||||
"""
|
||||
with open(full_path_to_remote_settings_dump_file, "r") as f:
|
||||
changeset = json.load(f)
|
||||
records = json.load(f)["data"]
|
||||
assert isinstance(records, list)
|
||||
|
||||
records = changeset["data"]
|
||||
assert isinstance(records, list)
|
||||
last_modified = changeset.get("timestamp")
|
||||
assert isinstance(
|
||||
last_modified, int
|
||||
), f"{full_path_to_remote_settings_dump_file} is missing the timestamp. See Bug 1725660"
|
||||
# Various RemoteSettings client code default to 0 when the set of
|
||||
# records is empty (-1 is reserved for failures / non-existing files).
|
||||
last_modified = 0
|
||||
if records:
|
||||
# Records in dumps are sorted by last_modified, newest first:
|
||||
# https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304 # NOQA: E501
|
||||
last_modified = records[0]["last_modified"]
|
||||
|
||||
assert isinstance(last_modified, int)
|
||||
return last_modified
|
||||
|
||||
|
||||
@ -54,30 +54,31 @@ def main(output):
|
||||
"mobile/android",
|
||||
"comm/mail",
|
||||
"comm/suite",
|
||||
), (
|
||||
"Cannot determine location of Remote Settings "
|
||||
f"dumps for platform {buildconfig.substs['MOZ_BUILD_APP']}"
|
||||
)
|
||||
|
||||
dumps_locations = []
|
||||
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
|
||||
dumps_locations += ["services/settings/dumps/"]
|
||||
elif buildconfig.substs["MOZ_BUILD_APP"] == "mobile/android":
|
||||
dumps_locations += ["services/settings/dumps/"]
|
||||
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":
|
||||
dumps_locations += ["mozilla/services/settings/dumps/"]
|
||||
dumps_locations += ["mail/app/settings/dumps/"]
|
||||
elif buildconfig.substs["MOZ_BUILD_APP"] == "comm/suite":
|
||||
dumps_locations += ["mozilla/services/settings/dumps/"]
|
||||
|
||||
remotesettings_dumps = {}
|
||||
for dumps_location in dumps_locations:
|
||||
dumps_root_dir = mozpath.join(buildconfig.topsrcdir, *dumps_location.split("/"))
|
||||
for path in glob.iglob(mozpath.join(dumps_root_dir, "*", "*.json")):
|
||||
folder, filename = os.path.split(path)
|
||||
bucket = os.path.basename(folder)
|
||||
collection, _ = os.path.splitext(filename)
|
||||
remotesettings_dumps[f"{bucket}/{collection}"] = path
|
||||
|
||||
# For simplicity, let's hardcode the path of the RemoteSettings dumps whose
|
||||
# last_modified date is looked up.
|
||||
# TODO bug 1719560: Replace hardcoded values with something more generic.
|
||||
if buildconfig.substs["MOZ_BUILD_APP"] != "mobile/android":
|
||||
# Until bug 1639050 is resolved, the dump isn't packaged with Android.
|
||||
remotesettings_dumps["blocklists/addons-bloomfilters"] = mozpath.join(
|
||||
buildconfig.topsrcdir,
|
||||
"services/settings/dumps/blocklists/addons-bloomfilters.json",
|
||||
)
|
||||
if buildconfig.substs["MOZ_BUILD_APP"] == "browser":
|
||||
# This is only packaged with browser.
|
||||
remotesettings_dumps["main/search-config"] = mozpath.join(
|
||||
buildconfig.topsrcdir,
|
||||
"services/settings/dumps/main/search-config.json",
|
||||
)
|
||||
if buildconfig.substs["MOZ_BUILD_APP"] == "comm/mail":
|
||||
# This is only packaged with Thunderbird.
|
||||
remotesettings_dumps["main/search-config"] = mozpath.join(
|
||||
buildconfig.topsrcdir,
|
||||
"comm/mail/app/settings/dumps/thunderbird/search-config.json",
|
||||
)
|
||||
|
||||
output_dict = {}
|
||||
input_files = set()
|
||||
|
||||
@ -6,6 +6,5 @@
|
||||
"id": "60e82333-914d-4cfa-95b1-5f034b5a704b",
|
||||
"last_modified": 1564511755134
|
||||
}
|
||||
],
|
||||
"timestamp": 1564511755134
|
||||
]
|
||||
}
|
||||
|
||||
@ -342,6 +342,5 @@
|
||||
"id": "0299f77d-0498-481a-9896-498095a7e301",
|
||||
"last_modified": 1645448267445
|
||||
}
|
||||
],
|
||||
"timestamp": 1648570256624
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
{
|
||||
"data": [],
|
||||
"timestamp": 1337
|
||||
}
|
||||
"data": []
|
||||
}
|
||||
|
||||
@ -54,6 +54,5 @@
|
||||
"id": "homepage-urls",
|
||||
"last_modified": 1605801189250
|
||||
}
|
||||
],
|
||||
"timestamp": 1605801189258
|
||||
]
|
||||
}
|
||||
|
||||
@ -642,6 +642,5 @@
|
||||
"id": "af",
|
||||
"last_modified": 1539698843264
|
||||
}
|
||||
],
|
||||
"timestamp": 1569410800356
|
||||
]
|
||||
}
|
||||
|
||||
@ -129,6 +129,5 @@
|
||||
"id": "00814138-7ede-4f56-8953-b6d1c99d5f26",
|
||||
"last_modified": 1600889016416
|
||||
}
|
||||
],
|
||||
"timestamp": 1642005109349
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,299 +1,5 @@
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"Domain": "ae.com",
|
||||
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit;",
|
||||
"id": "1b76634d-c4dc-467f-8b1d-458c0753faf1",
|
||||
"last_modified": 1650898337595
|
||||
},
|
||||
{
|
||||
"Domain": "app.digio.in",
|
||||
"password-rules": "minlength: 8; maxlength: 15;",
|
||||
"id": "9199eb80-2ea2-4efc-974a-13646cd9e2bf",
|
||||
"last_modified": 1650898337590
|
||||
},
|
||||
{
|
||||
"Domain": "app.parkmobile.io",
|
||||
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit; required: [!@#$%^&];",
|
||||
"id": "267bc941-86ee-4558-9d72-3f875ff0ac95",
|
||||
"last_modified": 1650898337584
|
||||
},
|
||||
{
|
||||
"Domain": "areariservata.bancaetica.it",
|
||||
"password-rules": "minlength: 8; maxlength: 10; required: lower; required: upper; required: digit; required: [!#&*+/=@_];",
|
||||
"id": "1316e054-4002-41b3-bb25-c2af275f1b48",
|
||||
"last_modified": 1650898337579
|
||||
},
|
||||
{
|
||||
"Domain": "bcassessment.ca",
|
||||
"password-rules": "minlength: 8; maxlength: 14;",
|
||||
"id": "2b72ecba-f8e4-4cc4-9c2c-01ad7b8eff60",
|
||||
"last_modified": 1650898337573
|
||||
},
|
||||
{
|
||||
"Domain": "belkin.com",
|
||||
"password-rules": "minlength: 8; required: lower, upper; required: digit; required: [$!@~_,%&];",
|
||||
"id": "89e880c3-2742-4802-bd33-ea506d3b7020",
|
||||
"last_modified": 1650898337568
|
||||
},
|
||||
{
|
||||
"Domain": "bilibili.com",
|
||||
"password-rules": "maxlength: 16;",
|
||||
"id": "3867a12f-b3e8-4343-94eb-6ddefba5b1a5",
|
||||
"last_modified": 1650898337562
|
||||
},
|
||||
{
|
||||
"Domain": "billerweb.com",
|
||||
"password-rules": "minlength: 8; max-consecutive: 2; required: digit; required: upper,lower;",
|
||||
"id": "6d89d12e-2ea9-4077-9dbc-c2566ccca370",
|
||||
"last_modified": 1650898337557
|
||||
},
|
||||
{
|
||||
"Domain": "bochk.com",
|
||||
"password-rules": "minlength: 8; maxlength: 12; max-consecutive: 3; required: lower; required: upper; required: digit; allowed: [#$%&()*+,.:;<=>?@_];",
|
||||
"id": "2a133fcd-fa66-473c-a09b-07014d0d3ccd",
|
||||
"last_modified": 1650898337551
|
||||
},
|
||||
{
|
||||
"Domain": "collectivehealth.com",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit;",
|
||||
"id": "b86c63d8-60e4-4c8a-965d-56af269f6b76",
|
||||
"last_modified": 1650898337546
|
||||
},
|
||||
{
|
||||
"Domain": "dan.org",
|
||||
"password-rules": "minlength: 8; maxlength: 25; required: lower; required: upper; required: digit; required: [!@$%^&*];",
|
||||
"id": "b7ca591b-c242-4520-a849-ccd74a90076a",
|
||||
"last_modified": 1650898337540
|
||||
},
|
||||
{
|
||||
"Domain": "dbs.com.hk",
|
||||
"password-rules": "minlength: 8; maxlength: 30; required: lower; required: upper; required: digit;",
|
||||
"id": "fe2f51c2-8ccb-4ee4-97cb-73288337f6ab",
|
||||
"last_modified": 1650898337535
|
||||
},
|
||||
{
|
||||
"Domain": "equifax.com",
|
||||
"password-rules": "minlength: 8; maxlength: 20; required: lower; required: upper; required: digit; required: [!$*+@];",
|
||||
"id": "ce20ebcc-b423-45f5-9643-f4651c55a133",
|
||||
"last_modified": 1650898337530
|
||||
},
|
||||
{
|
||||
"Domain": "fidelity.com",
|
||||
"password-rules": "minlength: 6; maxlength: 20; required: lower; allowed: upper,digit,[!$%'()+,./:;=?@^_|~];",
|
||||
"id": "51bcb63b-7da1-4dd5-a024-e395d32e3c36",
|
||||
"last_modified": 1650898337524
|
||||
},
|
||||
{
|
||||
"Domain": "flysas.com",
|
||||
"password-rules": "minlength: 8; maxlength: 14; required: lower; required: upper; required: digit; required: [-~!@#$%^&_+=`|(){}[:\"'<>,.?]];",
|
||||
"id": "c7b74d79-d787-4575-bfb4-92b531f422d9",
|
||||
"last_modified": 1650898337519
|
||||
},
|
||||
{
|
||||
"Domain": "gamestop.com",
|
||||
"password-rules": "minlength: 8; maxlength: 225; required: lower; required: upper; required: digit; required: [!@#$%];",
|
||||
"id": "ef9def1f-a244-4cc1-8937-f42865ce62c7",
|
||||
"last_modified": 1650898337513
|
||||
},
|
||||
{
|
||||
"Domain": "hangseng.com",
|
||||
"password-rules": "minlength: 8; maxlength: 30; required: lower; required: upper; required: digit;",
|
||||
"id": "bb13b29e-8fa0-4d68-b400-e546834a30a7",
|
||||
"last_modified": 1650898337508
|
||||
},
|
||||
{
|
||||
"Domain": "hkbea.com",
|
||||
"password-rules": "minlength: 8; maxlength: 12; required: lower; required: upper; required: digit;",
|
||||
"id": "a771074d-6e15-42aa-811a-731930d173b5",
|
||||
"last_modified": 1650898337502
|
||||
},
|
||||
{
|
||||
"Domain": "hkexpress.com",
|
||||
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: upper; required: digit; required: special;",
|
||||
"id": "94abbd2d-2201-43f1-b3c8-33900f207b4c",
|
||||
"last_modified": 1650898337497
|
||||
},
|
||||
{
|
||||
"Domain": "hsbc.com.hk",
|
||||
"password-rules": "minlength: 6; maxlength: 30; required: lower; required: upper; required: digit; allowed: ['.@_];",
|
||||
"id": "b657a5f0-268c-4acd-85fe-5085bcdb380c",
|
||||
"last_modified": 1650898337492
|
||||
},
|
||||
{
|
||||
"Domain": "kfc.ca",
|
||||
"password-rules": "minlength: 6; maxlength: 15; required: lower; required: upper; required: digit; required: [!@#$%&?*];",
|
||||
"id": "ca2e08a6-cc00-47a4-8517-737552dfedab",
|
||||
"last_modified": 1650898337486
|
||||
},
|
||||
{
|
||||
"Domain": "launtel.net.au",
|
||||
"password-rules": "minlength: 8; required: digit; required: digit; allowed: lower, upper;",
|
||||
"id": "eede7e13-6f0b-408a-bf1d-292130631834",
|
||||
"last_modified": 1650898337480
|
||||
},
|
||||
{
|
||||
"Domain": "lg.com",
|
||||
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit; allowed: [-!#$%&'()*+,.:;=?@[^_{|}~]];",
|
||||
"id": "fb2b3324-dd2d-4518-a309-af50f685689a",
|
||||
"last_modified": 1650898337475
|
||||
},
|
||||
{
|
||||
"Domain": "lloydsbank.co.uk",
|
||||
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: digit; allowed: upper;",
|
||||
"id": "56265f6f-6348-4900-8abf-07c4ab2e1c72",
|
||||
"last_modified": 1650898337470
|
||||
},
|
||||
{
|
||||
"Domain": "loyalty.accor.com",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!#$%&=@];",
|
||||
"id": "403e9c7d-1dbc-44f4-a825-81aa4127afca",
|
||||
"last_modified": 1650898337464
|
||||
},
|
||||
{
|
||||
"Domain": "lsacsso.b2clogin.com",
|
||||
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit, [-!#$%&*?@^_];",
|
||||
"id": "369eb957-3b3f-4377-ac76-52026dfffbfa",
|
||||
"last_modified": 1650898337458
|
||||
},
|
||||
{
|
||||
"Domain": "medicare.gov",
|
||||
"password-rules": "minlength: 8; maxlength: 16; required: lower; required: upper; required: digit; required: [@!$%^*()];",
|
||||
"id": "27a56617-f363-4455-a60c-8582af732012",
|
||||
"last_modified": 1650898337453
|
||||
},
|
||||
{
|
||||
"Domain": "mpv.tickets.com",
|
||||
"password-rules": "minlength: 8; maxlength: 15; required: lower; required: upper; required: digit;",
|
||||
"id": "9deab323-bc0f-4433-bbc3-e4d26dbfa7e0",
|
||||
"last_modified": 1650898337448
|
||||
},
|
||||
{
|
||||
"Domain": "my.konami.net",
|
||||
"password-rules": "minlength: 8; maxlength: 32; required: lower; required: upper; required: digit;",
|
||||
"id": "6a62b70b-6412-4ff4-945a-168dfc38b634",
|
||||
"last_modified": 1650898337443
|
||||
},
|
||||
{
|
||||
"Domain": "order.wendys.com",
|
||||
"password-rules": "minlength: 6; maxlength: 20; required: lower; required: upper; required: digit; allowed: [!#$%&()*+/=?^_{}];",
|
||||
"id": "57882459-1180-4faf-a8dd-c7475bcfeb0f",
|
||||
"last_modified": 1650898337437
|
||||
},
|
||||
{
|
||||
"Domain": "preferredhotels.com",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!#$%&()*+@^_];",
|
||||
"id": "4290b8b6-dd7c-431b-a4cb-36e661f72423",
|
||||
"last_modified": 1650898337432
|
||||
},
|
||||
{
|
||||
"Domain": "premier.ticketek.com.au",
|
||||
"password-rules": "minlength: 6; maxlength: 16;",
|
||||
"id": "26b1c942-2d06-496b-984b-65044ad2f1ab",
|
||||
"last_modified": 1650898337427
|
||||
},
|
||||
{
|
||||
"Domain": "premierinn.com",
|
||||
"password-rules": "minlength: 8; required: upper; required: digit; allowed: lower;",
|
||||
"id": "f6663626-8861-4bfc-a704-5ff1e23e9e22",
|
||||
"last_modified": 1650898337421
|
||||
},
|
||||
{
|
||||
"Domain": "prestocard.ca",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit,[!\"#$%&'()*+,<>?@];",
|
||||
"id": "ed580dc6-16af-4ec5-8ae2-b40fe99f0260",
|
||||
"last_modified": 1650898337416
|
||||
},
|
||||
{
|
||||
"Domain": "questdiagnostics.com",
|
||||
"password-rules": "minlength: 8; maxlength: 30; required: upper, lower; required: digit, [!#$%&()*+<>?@^_~];",
|
||||
"id": "a331ccb9-a2b8-44dd-bc35-0878eb51b4fe",
|
||||
"last_modified": 1650898337411
|
||||
},
|
||||
{
|
||||
"Domain": "renaud-bray.com",
|
||||
"password-rules": "minlength: 8; maxlength: 38; allowed: upper,lower,digit;",
|
||||
"id": "210a8aea-1089-487d-bcaa-3c937374f650",
|
||||
"last_modified": 1650898337405
|
||||
},
|
||||
{
|
||||
"Domain": "rogers.com",
|
||||
"password-rules": "minlength: 8; required: lower, upper; required: digit; required: [!@#$];",
|
||||
"id": "a13b20d6-0b68-4a05-866a-e8a14adee208",
|
||||
"last_modified": 1650898337400
|
||||
},
|
||||
{
|
||||
"Domain": "ruc.dk",
|
||||
"password-rules": "minlength: 6; maxlength: 8; required: lower, upper; required: [-!#%&(){}*+;%/<=>?_];",
|
||||
"id": "85d4c3e2-7f8c-480d-a20e-c55aee12a2e7",
|
||||
"last_modified": 1650898337394
|
||||
},
|
||||
{
|
||||
"Domain": "runescape.com",
|
||||
"password-rules": "minlength: 5; maxlength: 20; required: lower; required: upper; required: digit;",
|
||||
"id": "e14ecd5a-951f-4c27-bf45-0ee3fb830032",
|
||||
"last_modified": 1650898337389
|
||||
},
|
||||
{
|
||||
"Domain": "salslimo.com",
|
||||
"password-rules": "minlength: 8; maxlength: 50; required: upper; required: lower; required: digit; required: [!@#$&*];",
|
||||
"id": "afa90771-c472-48cc-96d3-21c770bb958c",
|
||||
"last_modified": 1650898337384
|
||||
},
|
||||
{
|
||||
"Domain": "secure.snnow.ca",
|
||||
"password-rules": "minlength: 7; maxlength: 16; required: digit; allowed: lower, upper;",
|
||||
"id": "f6a3e835-190e-4043-a070-943d821f708e",
|
||||
"last_modified": 1650898337378
|
||||
},
|
||||
{
|
||||
"Domain": "tix.soundrink.com",
|
||||
"password-rules": "minlength: 6; maxlength: 16;",
|
||||
"id": "0cb09a67-566e-492c-a4d7-af473e964472",
|
||||
"last_modified": 1650898337373
|
||||
},
|
||||
{
|
||||
"Domain": "training.confluent.io",
|
||||
"password-rules": "minlength: 6; maxlength: 16; required: lower; required: upper; required: digit; allowed: [!#$%*@^_~];",
|
||||
"id": "d00c2ee9-6f05-495c-9833-bae526019ddc",
|
||||
"last_modified": 1650898337368
|
||||
},
|
||||
{
|
||||
"Domain": "usps.com",
|
||||
"password-rules": "minlength: 8; maxlength: 50; max-consecutive: 2; required: lower; required: upper; required: digit; allowed: [-!\"#&'()+,./?@];",
|
||||
"id": "caacf36d-df98-43ab-8d5c-d7d40d79d696",
|
||||
"last_modified": 1650898337362
|
||||
},
|
||||
{
|
||||
"Domain": "visabenefits-auth.axa-assistance.us",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit; required: [!\"#$%&()*,.:<>?@^{|}];",
|
||||
"id": "3afe232d-ff3d-4e3d-b6bd-da947dac7884",
|
||||
"last_modified": 1650898337357
|
||||
},
|
||||
{
|
||||
"Domain": "wegmans.com",
|
||||
"password-rules": "minlength: 8; required: digit; required: upper,lower; required: [!#$%&*+=?@^];",
|
||||
"id": "d7036bd9-5301-43a1-b427-25a876ff4c07",
|
||||
"last_modified": 1650898337352
|
||||
},
|
||||
{
|
||||
"Domain": "xvoucher.com",
|
||||
"password-rules": "minlength: 11; required: upper; required: digit; required: [!@#$%&_];",
|
||||
"id": "85cc3546-8a5c-4400-bd01-c0386897a442",
|
||||
"last_modified": 1650898337346
|
||||
},
|
||||
{
|
||||
"Domain": "zara.com",
|
||||
"password-rules": "minlength: 8; required: lower; required: upper; required: digit;",
|
||||
"id": "4496846f-b282-4965-97f8-97c2b814881a",
|
||||
"last_modified": 1650898337341
|
||||
},
|
||||
{
|
||||
"Domain": "americanexpress.com",
|
||||
"password-rules": "minlength: 8; maxlength: 20; max-consecutive: 4; required: lower, upper; required: digit; allowed: [%&_?#=];",
|
||||
"id": "986919ff-d801-45f6-ab75-af40e0c9e0ad",
|
||||
"last_modified": 1650898337335
|
||||
},
|
||||
{
|
||||
"Domain": "163.com",
|
||||
"password-rules": "minlength: 6; maxlength: 16;",
|
||||
@ -354,6 +60,12 @@
|
||||
"id": "46f95801-6c08-404b-99f6-ac36b347e45b",
|
||||
"last_modified": 1624479577522
|
||||
},
|
||||
{
|
||||
"Domain": "americanexpress.com",
|
||||
"password-rules": "minlength: 8; maxlength: 20; max-consecutive: 4; required: lower, upper; required: digit; allowed: [-%&_?#=];",
|
||||
"id": "986919ff-d801-45f6-ab75-af40e0c9e0ad",
|
||||
"last_modified": 1624479577517
|
||||
},
|
||||
{
|
||||
"Domain": "anatel.gov.br",
|
||||
"password-rules": "minlength: 6; maxlength: 15; allowed: lower, upper, digit;",
|
||||
@ -1560,6 +1272,5 @@
|
||||
"id": "5fe248fb-3e63-4300-8f67-3e6006af0e87",
|
||||
"last_modified": 1624479576629
|
||||
}
|
||||
],
|
||||
"timestamp": 1650898337595
|
||||
]
|
||||
}
|
||||
|
||||
@ -2907,6 +2907,5 @@
|
||||
"id": "43559b11-05dc-4750-b131-afdbd9d25905",
|
||||
"last_modified": 1583937832210
|
||||
}
|
||||
],
|
||||
"timestamp": 1648132005528
|
||||
]
|
||||
}
|
||||
|
||||
@ -12,6 +12,5 @@
|
||||
"id": "c5de2b3f-da15-4bc5-b36f-9555043b5ef3",
|
||||
"last_modified": 1595254618540
|
||||
}
|
||||
],
|
||||
"timestamp": 1595254618540
|
||||
]
|
||||
}
|
||||
|
||||
@ -149,6 +149,5 @@
|
||||
"id": "19c434a3-d173-4871-9743-290ac92a3f6a",
|
||||
"last_modified": 1643136933989
|
||||
}
|
||||
],
|
||||
"timestamp": 1647619735693
|
||||
]
|
||||
}
|
||||
|
||||
@ -458,6 +458,5 @@
|
||||
"id": "48ec9db9-0de1-49aa-9b82-1a2372dce31e",
|
||||
"last_modified": 1539907283705
|
||||
}
|
||||
],
|
||||
"timestamp": 1544035467383
|
||||
]
|
||||
}
|
||||
|
||||
@ -396,6 +396,5 @@
|
||||
"id": "89e3da9e-1d84-4c24-beab-d0692fc7cc45",
|
||||
"last_modified": 1599146777345
|
||||
}
|
||||
],
|
||||
"timestamp": 1647020600359
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,4 +1,3 @@
|
||||
{
|
||||
"data": [],
|
||||
"timestamp": 1606870304609
|
||||
"data": []
|
||||
}
|
||||
|
||||
@ -2,23 +2,16 @@
|
||||
"data": [
|
||||
{
|
||||
"relatedRealms": [
|
||||
[
|
||||
"3docean.net",
|
||||
"audiojungle.net",
|
||||
"codecanyon.net",
|
||||
"envato.com",
|
||||
"graphicriver.net",
|
||||
"photodune.net",
|
||||
"placeit.net",
|
||||
"themeforest.net",
|
||||
"tutsplus.com",
|
||||
"videohive.net"
|
||||
],
|
||||
[
|
||||
"aa.com",
|
||||
"americanairlines.com",
|
||||
"americanairlines.jp"
|
||||
],
|
||||
[
|
||||
"airfrance.com",
|
||||
"klm.com",
|
||||
"flyingblue.com"
|
||||
],
|
||||
[
|
||||
"aetna.com",
|
||||
"banneraetna.myplanportal.com"
|
||||
@ -78,33 +71,6 @@
|
||||
"airbnb.com",
|
||||
"airbnb.co.ve"
|
||||
],
|
||||
[
|
||||
"airfrance.com",
|
||||
"klm.com",
|
||||
"flyingblue.com"
|
||||
],
|
||||
[
|
||||
"airnewzealand.co.nz",
|
||||
"airnewzealand.com",
|
||||
"airnewzealand.com.au"
|
||||
],
|
||||
[
|
||||
"albertsons.com",
|
||||
"acmemarkets.com",
|
||||
"carrsqc.com",
|
||||
"jewelosco.com",
|
||||
"pavilions.com",
|
||||
"randalls.com",
|
||||
"safeway.com",
|
||||
"shaws.com",
|
||||
"starmarket.com",
|
||||
"tomthumb.com",
|
||||
"vons.com"
|
||||
],
|
||||
[
|
||||
"alibaba.com",
|
||||
"aliexpress.com"
|
||||
],
|
||||
[
|
||||
"alltrails.com",
|
||||
"alltrails.io"
|
||||
@ -127,8 +93,7 @@
|
||||
"amazon.sa",
|
||||
"amazon.sg",
|
||||
"amazon.se",
|
||||
"amazon.pl",
|
||||
"ring.com"
|
||||
"amazon.pl"
|
||||
],
|
||||
[
|
||||
"amcrestcloud.com",
|
||||
@ -152,26 +117,14 @@
|
||||
"apple.com",
|
||||
"icloud.com"
|
||||
],
|
||||
[
|
||||
"atlassian.com",
|
||||
"trello.com"
|
||||
],
|
||||
[
|
||||
"att.com",
|
||||
"att.net"
|
||||
],
|
||||
[
|
||||
"audi.com",
|
||||
"audiusa.com"
|
||||
],
|
||||
[
|
||||
"bahn.de",
|
||||
"bahn.com"
|
||||
],
|
||||
[
|
||||
"battle.net",
|
||||
"blizzard.com"
|
||||
],
|
||||
[
|
||||
"beachbodyondemand.com",
|
||||
"teambeachbody.com"
|
||||
@ -195,21 +148,13 @@
|
||||
"whistlerblackcomb.com"
|
||||
],
|
||||
[
|
||||
"boingo.com",
|
||||
"boingohotspot.com"
|
||||
],
|
||||
[
|
||||
"bol.com",
|
||||
"kobo.com"
|
||||
"battle.net",
|
||||
"blizzard.com"
|
||||
],
|
||||
[
|
||||
"boudinbakery.com",
|
||||
"boudincatering.com"
|
||||
],
|
||||
[
|
||||
"braze.com",
|
||||
"braze.eu"
|
||||
],
|
||||
[
|
||||
"capitalone.com",
|
||||
"capitalone360.com"
|
||||
@ -243,12 +188,8 @@
|
||||
"dinersclubus.com"
|
||||
],
|
||||
[
|
||||
"discordapp.com",
|
||||
"discord.com"
|
||||
],
|
||||
[
|
||||
"discordmerch.com",
|
||||
"discord.store"
|
||||
"discord.com",
|
||||
"discordapp.com"
|
||||
],
|
||||
[
|
||||
"dish.com",
|
||||
@ -263,15 +204,6 @@
|
||||
"dropbox.com",
|
||||
"getdropbox.com"
|
||||
],
|
||||
[
|
||||
"e.leclerc",
|
||||
"auto.leclerc",
|
||||
"location.leclerc",
|
||||
"chezmoi.lelcerc",
|
||||
"traiteur.lelcerc",
|
||||
"mesrecettes.lelcerc",
|
||||
"leclercdrive.fr"
|
||||
],
|
||||
[
|
||||
"eater.com",
|
||||
"polygon.com",
|
||||
@ -302,10 +234,6 @@
|
||||
"ebay.pl",
|
||||
"ebay.vn"
|
||||
],
|
||||
[
|
||||
"eurosport.no",
|
||||
"eurosportplayer.com"
|
||||
],
|
||||
[
|
||||
"eventbrite.at",
|
||||
"eventbrite.be",
|
||||
@ -338,14 +266,6 @@
|
||||
"fandangonow.com",
|
||||
"fandango.com"
|
||||
],
|
||||
[
|
||||
"fidelity.com",
|
||||
"fidelityinvestments.com"
|
||||
],
|
||||
[
|
||||
"flyblade.com",
|
||||
"blade.com"
|
||||
],
|
||||
[
|
||||
"fnac.com",
|
||||
"fnacspectacles.com"
|
||||
@ -367,20 +287,6 @@
|
||||
"gogoair.com",
|
||||
"gogoinflight.com"
|
||||
],
|
||||
[
|
||||
"hbo.com",
|
||||
"hbomax.com",
|
||||
"hbonow.com"
|
||||
],
|
||||
[
|
||||
"igen.fr",
|
||||
"watchgeneration.fr",
|
||||
"macg.co"
|
||||
],
|
||||
[
|
||||
"ikonpass.com",
|
||||
"skilynx.com"
|
||||
],
|
||||
[
|
||||
"intuit.com",
|
||||
"mint.com"
|
||||
@ -413,12 +319,6 @@
|
||||
"lookmark.io",
|
||||
"lookmark.link"
|
||||
],
|
||||
[
|
||||
"lrz.de",
|
||||
"mwn.de",
|
||||
"mytum.de",
|
||||
"tum.de"
|
||||
],
|
||||
[
|
||||
"lufthansa.com",
|
||||
"miles-and-more.com"
|
||||
@ -436,8 +336,7 @@
|
||||
"microsoftonline.com",
|
||||
"office.com",
|
||||
"skype.com",
|
||||
"onenote.com",
|
||||
"hotmail.com"
|
||||
"onenote.com"
|
||||
],
|
||||
[
|
||||
"minecraft.net",
|
||||
@ -447,20 +346,6 @@
|
||||
"mytotalconnectcomfort.com",
|
||||
"tccna.honeywell.com"
|
||||
],
|
||||
[
|
||||
"myuhc.com",
|
||||
"uhc.com",
|
||||
"optum.com",
|
||||
"optumrx.com"
|
||||
],
|
||||
[
|
||||
"neatorama.com",
|
||||
"neatoshop.com"
|
||||
],
|
||||
[
|
||||
"newyorker.com",
|
||||
"vanityfair.com"
|
||||
],
|
||||
[
|
||||
"nintendolife.com",
|
||||
"purexbox.com",
|
||||
@ -473,17 +358,12 @@
|
||||
"nsn.com"
|
||||
],
|
||||
[
|
||||
"nordvpn.com",
|
||||
"nordpass.com",
|
||||
"nordaccount.com"
|
||||
"norwegianreward.com",
|
||||
"norwegian.com"
|
||||
],
|
||||
[
|
||||
"norwegian.com",
|
||||
"norwegianreward.com"
|
||||
],
|
||||
[
|
||||
"olo.com",
|
||||
"olo.express"
|
||||
"optumrx.com",
|
||||
"optum.com"
|
||||
],
|
||||
[
|
||||
"pinterest.com",
|
||||
@ -523,26 +403,10 @@
|
||||
"protonmail.com",
|
||||
"protonvpn.com"
|
||||
],
|
||||
[
|
||||
"qnap.com",
|
||||
"myqnapcloud.com"
|
||||
],
|
||||
[
|
||||
"questdiagnostics.com",
|
||||
"care360.com"
|
||||
],
|
||||
[
|
||||
"rocketaccount.com",
|
||||
"rocketmortgage.com"
|
||||
],
|
||||
[
|
||||
"scholarshare529.com",
|
||||
"secureaccountview.com"
|
||||
],
|
||||
[
|
||||
"scoutingevent.com",
|
||||
"campreservation.com"
|
||||
],
|
||||
[
|
||||
"scribbr.com",
|
||||
"scribbr.de",
|
||||
@ -567,9 +431,7 @@
|
||||
[
|
||||
"shopdisney.com",
|
||||
"go.com",
|
||||
"disneyplus.com",
|
||||
"espn.com",
|
||||
"disneystore.com"
|
||||
"disneyplus.com"
|
||||
],
|
||||
[
|
||||
"slcl.overdrive.com",
|
||||
@ -622,6 +484,10 @@
|
||||
"tp-link.com",
|
||||
"tplinkcloud.com"
|
||||
],
|
||||
[
|
||||
"uhc.com",
|
||||
"myuhc.com"
|
||||
],
|
||||
[
|
||||
"umsystem.edu",
|
||||
"mst.edu",
|
||||
@ -634,11 +500,6 @@
|
||||
"unitedwifi.com"
|
||||
],
|
||||
[
|
||||
"uspowerboating.com",
|
||||
"ussailing.org"
|
||||
],
|
||||
[
|
||||
"verizon.com",
|
||||
"verizonwireless.com",
|
||||
"vzw.com"
|
||||
],
|
||||
@ -650,14 +511,14 @@
|
||||
"perigold.com",
|
||||
"birchlane.com"
|
||||
],
|
||||
[
|
||||
"wellsfargo.com",
|
||||
"wellsfargoadvisors.com"
|
||||
],
|
||||
[
|
||||
"wiimmfi.de",
|
||||
"wii-homebrew.com"
|
||||
],
|
||||
[
|
||||
"wellsfargo.com",
|
||||
"wellsfargoadvisors.com"
|
||||
],
|
||||
[
|
||||
"wikipedia.org",
|
||||
"mediawiki.org",
|
||||
@ -697,23 +558,10 @@
|
||||
[
|
||||
"wsj.com",
|
||||
"dowjones.com"
|
||||
],
|
||||
[
|
||||
"www.vistaprint.ca",
|
||||
"account.vistaprint.com"
|
||||
],
|
||||
[
|
||||
"yahoo.com",
|
||||
"flickr.com"
|
||||
],
|
||||
[
|
||||
"zixmail.net",
|
||||
"zixmessagecenter.com"
|
||||
]
|
||||
],
|
||||
"id": "8c3d4151-8e68-4bb3-a3fd-babf4aba2cdc",
|
||||
"last_modified": 1650898092205
|
||||
"last_modified": 1624479523629
|
||||
}
|
||||
],
|
||||
"timestamp": 1650898092205
|
||||
]
|
||||
}
|
||||
|
||||
@ -23652,6 +23652,5 @@
|
||||
"id": "586b6f01-c7f7-4a5e-ba71-e7dca3e77add",
|
||||
"last_modified": 1559865863642
|
||||
}
|
||||
],
|
||||
"timestamp": 1650877037744
|
||||
]
|
||||
}
|
||||
|
||||
@ -21780,6 +21780,5 @@
|
||||
"id": "ae8bec3c-3b92-822e-53f1-68394cbb1758",
|
||||
"last_modified": 1480349158647
|
||||
}
|
||||
],
|
||||
"timestamp": 1648145180742
|
||||
]
|
||||
}
|
||||
|
||||
@ -10,6 +10,5 @@
|
||||
"id": "global",
|
||||
"last_modified": 1621943462970
|
||||
}
|
||||
],
|
||||
"timestamp": 1621943462970
|
||||
]
|
||||
}
|
||||
|
||||
@ -18,6 +18,5 @@
|
||||
"id": "cloudflare-global",
|
||||
"last_modified": 1621943542615
|
||||
}
|
||||
],
|
||||
"timestamp": 1621943542621
|
||||
]
|
||||
}
|
||||
|
||||
@ -1,21 +1,27 @@
|
||||
"use strict";
|
||||
|
||||
const { AppConstants } = ChromeUtils.import(
|
||||
"resource://gre/modules/AppConstants.jsm"
|
||||
);
|
||||
const { Utils } = ChromeUtils.import("resource://services-settings/Utils.jsm");
|
||||
|
||||
Cu.importGlobalProperties(["fetch"]);
|
||||
|
||||
async function getLocalDumpLastModified(bucket, collection) {
|
||||
let res;
|
||||
try {
|
||||
res = await fetch(
|
||||
`resource://app/defaults/settings/${bucket}/${collection}.json`
|
||||
);
|
||||
} catch (e) {
|
||||
return -1;
|
||||
let res = await fetch(
|
||||
`resource://app/defaults/settings/${bucket}/${collection}.json`
|
||||
);
|
||||
let records = (await res.json()).data;
|
||||
|
||||
if (records.some(r => r.last_modified > records[0].last_modified)) {
|
||||
// The dump importer should ensure that the newest record is at the front:
|
||||
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
|
||||
ok(false, `${bucket}/${collection} - newest record should be in the front`);
|
||||
}
|
||||
const { timestamp } = await res.json();
|
||||
ok(timestamp >= 0, `${bucket}/${collection} dump has timestamp`);
|
||||
return timestamp;
|
||||
return records.reduce(
|
||||
(max, { last_modified }) => Math.max(last_modified, max),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
add_task(async function lastModified_of_non_existing_dump() {
|
||||
@ -32,24 +38,23 @@ add_task(async function lastModified_of_non_existing_dump() {
|
||||
});
|
||||
|
||||
add_task(async function lastModified_summary_is_correct() {
|
||||
if (AppConstants.platform == "android") {
|
||||
// TODO bug 1719560: When implemented, remove this condition.
|
||||
equal(JSON.stringify(Utils._dumpStats), "{}", "No dumps on Android yet");
|
||||
return;
|
||||
}
|
||||
ok(Object.keys(Utils._dumpStats).length > 0, "Contains summary of dumps");
|
||||
|
||||
let checked = 0;
|
||||
for (let [identifier, lastModified] of Object.entries(Utils._dumpStats)) {
|
||||
let [bucket, collection] = identifier.split("/");
|
||||
let actual = await getLocalDumpLastModified(bucket, collection);
|
||||
if (actual < 0) {
|
||||
info(`${identifier} has no dump, skip.`);
|
||||
continue;
|
||||
}
|
||||
info(`Checking correctness of ${identifier}`);
|
||||
let [bucket, collection] = identifier.split("/");
|
||||
equal(
|
||||
await Utils.getLocalDumpLastModified(bucket, collection),
|
||||
lastModified,
|
||||
`Expected last_modified value for ${identifier}`
|
||||
);
|
||||
|
||||
let actual = await getLocalDumpLastModified(bucket, collection);
|
||||
equal(lastModified, actual, `last_modified should match collection`);
|
||||
checked++;
|
||||
}
|
||||
ok(checked > 0, "At least one dump was packaged and checked.");
|
||||
});
|
||||
|
||||
@ -26,10 +26,12 @@ add_task(async function setup() {
|
||||
bucketName: TEST_BUCKET,
|
||||
});
|
||||
|
||||
const dump = await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION);
|
||||
DUMP_RECORDS = dump.data;
|
||||
DUMP_LAST_MODIFIED = dump.timestamp;
|
||||
|
||||
DUMP_RECORDS = (await SharedUtils.loadJSONDump(TEST_BUCKET, TEST_COLLECTION))
|
||||
.data;
|
||||
DUMP_LAST_MODIFIED = DUMP_RECORDS.reduce(
|
||||
(max, { last_modified }) => Math.max(last_modified, max),
|
||||
-Infinity
|
||||
);
|
||||
// Dumps are fetched via the following, which sorts the records, newest first.
|
||||
// https://searchfox.org/mozilla-central/rev/5b3444ad300e244b5af4214212e22bd9e4b7088a/taskcluster/docker/periodic-updates/scripts/periodic_file_updates.sh#304
|
||||
equal(
|
||||
@ -44,7 +46,6 @@ async function importData(records) {
|
||||
TEST_BUCKET,
|
||||
TEST_COLLECTION,
|
||||
records,
|
||||
records[0]?.last_modified || 0,
|
||||
]);
|
||||
}
|
||||
|
||||
|
||||
@ -108,7 +108,7 @@ add_task(async function test_shutdown_worker() {
|
||||
let records = [{}];
|
||||
let importPromise = RemoteSettingsWorker._execute(
|
||||
"_test_only_import",
|
||||
["main", "language-dictionaries", records, 0],
|
||||
["main", "language-dictionaries", records],
|
||||
{ mustComplete: true }
|
||||
);
|
||||
let stringifyPromise = RemoteSettingsWorker.canonicalStringify(
|
||||
|
||||
@ -290,9 +290,9 @@ function compare_remote_settings_files {
|
||||
# 1. List remote settings collections from server.
|
||||
echo "INFO: fetch remote settings list from server"
|
||||
${WGET} -qO- "${REMOTE_SETTINGS_SERVER}/buckets/monitor/collections/changes/records" |\
|
||||
${JQ} -r '.data[] | .bucket+"/"+.collection+"/"+(.last_modified|tostring)' |\
|
||||
# 2. For each entry ${bucket, collection, last_modified}
|
||||
while IFS="/" read -r bucket collection last_modified; do
|
||||
${JQ} -r '.data[] | .bucket+"/"+.collection' |\
|
||||
# 2. For each entry ${bucket, collection}
|
||||
while IFS="/" read -r bucket collection; do
|
||||
|
||||
# 3. Download the dump from HG into REMOTE_SETTINGS_INPUT folder
|
||||
hg_dump_url="${HGREPO}/raw-file/default${REMOTE_SETTINGS_DIR}/${bucket}/${collection}.json"
|
||||
@ -307,10 +307,10 @@ function compare_remote_settings_files {
|
||||
fi
|
||||
|
||||
# 4. Download server version into REMOTE_SETTINGS_OUTPUT folder
|
||||
remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/changeset?_expected=${last_modified}"
|
||||
remote_records_url="$REMOTE_SETTINGS_SERVER/buckets/${bucket}/collections/${collection}/records"
|
||||
local_location_output="$REMOTE_SETTINGS_OUTPUT/${bucket}/${collection}.json"
|
||||
mkdir -p "$REMOTE_SETTINGS_OUTPUT/${bucket}"
|
||||
${WGET} -qO- "$remote_records_url" | ${JQ} '{"data": .changes, "timestamp": .timestamp}' > "${local_location_output}"
|
||||
${WGET} -qO- "$remote_records_url" | ${JQ} . > "${local_location_output}"
|
||||
|
||||
# 5. Download attachments if needed.
|
||||
if [ "${bucket}" = "blocklists" ] && [ "${collection}" = "addons-bloomfilters" ]; then
|
||||
|
||||
@ -35,7 +35,6 @@ add_task(async function test_selector_db_out_of_date() {
|
||||
last_modified: 1606227264000,
|
||||
},
|
||||
],
|
||||
1606227264000,
|
||||
]);
|
||||
|
||||
// Now load the configuration and check we get what we expect.
|
||||
|
||||
@ -4,7 +4,7 @@ head = head_searchconfig.js
|
||||
dupe-manifest =
|
||||
support-files =
|
||||
../../../../../../browser/locales/all-locales
|
||||
tags=searchconfig remote-settings
|
||||
tags=searchconfig
|
||||
# These are extensive tests, we don't need to run them on asan/tsan.
|
||||
# They are also skipped for mobile and Thunderbird as these are specifically
|
||||
# testing the Firefox config at the moment.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user