Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-28 07:13:20 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1450998 - Improve API and docs for Remote Settings r=mgoodwin

Browse Source 
MozReview-Commit-ID: EszfBy3xNP9

--HG--
extra : rebase_source : c7027e100d1c47ebb4e3b81e0c74fce30240cb42
This commit is contained in:
Mathieu Leplatre 2018-03-29 14:38:16 -07:00
parent 297ca3383c
commit 2dc7664a19
12 changed files with 305 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
modules/libpref/init/all.js
View File

@ -2679,6 +2679,8 @@ pref("security.allow_chrome_frames_inside_content", false);
// Services security settings
pref("services.settings.server", "https://firefox.settings.services.mozilla.com/v1");
pref("services.settings.changes.path", "/buckets/monitor/collections/changes/records");
pref("services.settings.default_bucket", "main");
pref("services.settings.default_signer", "");
// Blocklist preferences
pref("extensions.blocklist.enabled", true);

3
services/common/blocklist-clients.js
View File

@ -143,18 +143,21 @@ function initialize() {
AddonBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PREF_BLOCKLIST_ADDONS_COLLECTION), {
bucketName: Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET),
lastCheckTimePref: PREF_BLOCKLIST_ADDONS_CHECKED_SECONDS,
signerName: "", // disabled
});
AddonBlocklistClient.on("change", updateJSONBlocklist.bind(null, AddonBlocklistClient));
PluginBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PREF_BLOCKLIST_PLUGINS_COLLECTION), {
bucketName: Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET),
lastCheckTimePref: PREF_BLOCKLIST_PLUGINS_CHECKED_SECONDS,
signerName: "", // disabled
});
PluginBlocklistClient.on("change", updateJSONBlocklist.bind(null, PluginBlocklistClient));
GfxBlocklistClient = RemoteSettings(Services.prefs.getCharPref(PREF_BLOCKLIST_GFX_COLLECTION), {
bucketName: Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET),
lastCheckTimePref: PREF_BLOCKLIST_GFX_CHECKED_SECONDS,
signerName: "", // disabled
});
GfxBlocklistClient.on("change", updateJSONBlocklist.bind(null, GfxBlocklistClient));

111
services/common/docs/RemoteSettings.rst Normal file
View File

@ -0,0 +1,111 @@
.. _services/remotesettings:
===============
Remote Settings
===============
The `remote-settings.js <https://dxr.mozilla.org/mozilla-central/source/services/common/remote-settings.js>`_ module offers the ability to fetch remote settings that are kept in sync with Mozilla servers.
Usage
=====
The `get()` method returns the list of entries for a specific key. Each entry can have arbitrary attributes, and can only be modified on the server.
.. code-block:: js
const { RemoteSettings } = ChromeUtils.import("resource://services-common/remote-settings.js", {});
const data = await RemoteSettings("a-key").get();
/*
data == [
{label: "Yahoo", enabled: true, weight: 10, id: "d0782d8d", last_modified: 1522764475905},
{label: "Google", enabled: true, weight: 20, id: "8883955f", last_modified: 1521539068414},
{label: "Ecosia", enabled: false, weight: 5, id: "337c865d", last_modified: 1520527480321},
]
*/
for(const entry of data) {
// Do something with entry...
// await InternalAPI.load(entry.id, entry.label, entry.weight);
});
.. note::
The ``id`` and ``last_modified`` (timestamp) attributes are assigned by the server.
Options
-------
The list can optionally be filtered or ordered:
.. code-block:: js
const subset = await RemoteSettings("a-key").get({
filters: {
"enabled": true,
},
order: "-weight"
});
Events
------
The ``change`` event allows to be notified when the remote settings are changed. The event ``data`` attribute contains the whole new list of settings.
.. code-block:: js
RemoteSettings("a-key").on("change", event => {
const { data } = event;
for(const entry of data) {
// Do something with entry...
// await InternalAPI.reload(entry.id, entry.label, entry.weight);
}
});
.. note::
Currently, the update of remote settings is triggered by the `nsBlocklistService <https://dxr.mozilla.org/mozilla-central/source/toolkit/mozapps/extensions/nsBlocklistService.js>`_ (~ every 24H).
File attachments
----------------
When an entry has a file attached to it, it has an ``attachment`` attribute, which contains the file related information (url, hash, size, mimetype, etc.). Remote files are not downloaded automatically.
.. code-block:: js
const data = await RemoteSettings("a-key").get();
data.filter(d => d.attachment)
.forEach(async ({ attachment: { url, filename, size } }) => {
if (size < OS.freeDiskSpace) {
await downloadLocally(url, filename);
}
});
Uptake Telemetry
================
Some :ref:`uptake telemetry <telemetry/collection/uptake>` is collected in order to monitor how remote settings are propagated.
It is submitted to a single :ref:`keyed histogram <histogram-type-keyed>` whose id is ``UPTAKE_REMOTE_CONTENT_RESULT_1`` and the keys are prefixed with ``main/`` (eg. ``main/a-key`` in the above example).
Create new remote settings
==========================
Staff members can create new kinds of remote settings, following `this documentation <mana docs>`_.
It basically consists in:
#. Choosing a key (eg. ``search-providers``)
#. Assigning collaborators to editors and reviewers groups
#. (*optional*) Define a JSONSchema to validate entries
#. (*optional*) Allow attachments on entries
And once done:
#. Create, modify or delete entries and let reviewers approve the changes
#. Wait for Firefox to pick-up the changes for your settings key
.. _mana docs: https://mana.mozilla.org/wiki/pages/viewpage.action?pageId=66655528

10
services/common/docs/index.rst Normal file
View File

@ -0,0 +1,10 @@
========
Services
========
This is the nascent documentation of the Firefox services.
.. toctree::
:maxdepth: 1
RemoteSettings

1
services/common/moz.build
View File

@ -42,3 +42,4 @@ JS_PREFERENCE_FILES += [
'services-common.js',
]
SPHINX_TREES['services'] = 'docs'

257
services/common/remote-settings.js
View File

@ -4,7 +4,7 @@
"use strict";
var EXPORTED_SYMBOLS = ["RemoteSettings", "pollChanges"];
var EXPORTED_SYMBOLS = ["RemoteSettings"];
ChromeUtils.import("resource://gre/modules/Services.jsm");
ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
@ -23,6 +23,8 @@ ChromeUtils.defineModuleGetter(this, "UptakeTelemetry",
"resource://services-common/uptake-telemetry.js");
const PREF_SETTINGS_SERVER = "services.settings.server";
const PREF_SETTINGS_DEFAULT_BUCKET = "services.settings.default_bucket";
const PREF_SETTINGS_DEFAULT_SIGNER = "services.settings.default_signer";
const PREF_SETTINGS_VERIFY_SIGNATURE = "services.settings.verify_signature";
const PREF_SETTINGS_SERVER_BACKOFF = "services.settings.server.backoff";
const PREF_SETTINGS_CHANGES_PATH = "services.settings.changes.path";
@ -41,20 +43,6 @@ const INVALID_SIGNATURE = "Invalid content/signature";
// filename, even though it isn't descriptive of who is using it.
const KINTO_STORAGE_PATH = "kinto.sqlite";
const gRemoteSettingsClients = new Map();
// Get or instantiate a remote settings client.
function RemoteSettings(collectionName, options) {
const { bucketName } = options;
const key = `${bucketName}/${collectionName}`;
if (!gRemoteSettingsClients.has(key)) {
const c = new RemoteSettingsClient(collectionName, options);
gRemoteSettingsClients.set(key, c);
}
return gRemoteSettingsClients.get(key);
}
function mergeChanges(collection, localRecords, changes) {
const records = {};
@ -203,6 +191,24 @@ class RemoteSettingsClient {
}
}
/**
* Lists settings.
*
* @param {Object} options The options object.
* @param {Object} options.filters Filter the results (default: `{}`).
* @param {Object} options.order The order to apply (default: `-last_modified`).
* @return {Promise}
*/
async get(options = {}) {
// In Bug 1451031, we will do some jexl filtering to limit the list items
// whose target is matched.
const { filters, order } = options;
return this.openCollection(async c => {
const { data } = await c.list({ filters, order });
return data;
});
}
/**
* Synchronize from Kinto server, if necessary.
*
@ -262,7 +268,7 @@ class RemoteSettingsClient {
try {
// Server changes have priority during synchronization.
const strategy = Kinto.syncStrategy.SERVER_WINS;
const {ok} = await collection.sync({remote, strategy});
const { ok } = await collection.sync({remote, strategy});
if (!ok) {
// Some synchronization conflicts occured.
reportStatus = UptakeTelemetry.STATUS.CONFLICT_ERROR;
@ -304,7 +310,7 @@ class RemoteSettingsClient {
}
}
// Read local collection of records.
const {data} = await collection.list();
const { data } = await collection.list();
// Handle the obtained records (ie. apply locally).
try {
@ -397,106 +403,141 @@ class RemoteSettingsClient {
* @param {Date} serverTime the current date return by server.
*/
_updateLastCheck(serverTime) {
if (!this.lastCheckTimePref) {
// If not set (default), it is not necessary to store the last check timestamp.
return;
}
// Storing the last check time is mainly a matter of retro-compatibility with
// the blocklists clients.
const checkedServerTimeInSeconds = Math.round(serverTime / 1000);
Services.prefs.setIntPref(this.lastCheckTimePref, checkedServerTimeInSeconds);
}
}
// This is called by the ping mechanism.
// returns a promise that rejects if something goes wrong
async function pollChanges() {
// Check if the server backoff time is elapsed.
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
const backoffReleaseTime = Services.prefs.getCharPref(PREF_SETTINGS_SERVER_BACKOFF);
const remainingMilliseconds = parseInt(backoffReleaseTime, 10) - Date.now();
if (remainingMilliseconds > 0) {
// Backoff time has not elapsed yet.
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY,
UptakeTelemetry.STATUS.BACKOFF);
throw new Error(`Server is asking clients to back off; retry in ${Math.ceil(remainingMilliseconds / 1000)}s.`);
} else {
Services.prefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
function remoteSettingsFunction() {
const _clients = new Map();
// If not explicitly specified, use the default bucket name and signer.
const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
const remoteSettings = function(collectionName, options) {
// Get or instantiate a remote settings client.
const rsOptions = {
bucketName: mainBucket,
signerName: defaultSigner,
...options
};
const { bucketName } = rsOptions;
const key = `${bucketName}/${collectionName}`;
if (!_clients.has(key)) {
const c = new RemoteSettingsClient(collectionName, rsOptions);
_clients.set(key, c);
}
}
return _clients.get(key);
};
// Right now, we only use the collection name and the last modified info
const kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
const changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
let lastEtag;
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
lastEtag = Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
}
let pollResult;
try {
pollResult = await fetchLatestChanges(changesEndpoint, lastEtag);
} catch (e) {
// Report polling error to Uptake Telemetry.
let report;
if (/Server/.test(e.message)) {
report = UptakeTelemetry.STATUS.SERVER_ERROR;
} else if (/NetworkError/.test(e.message)) {
report = UptakeTelemetry.STATUS.NETWORK_ERROR;
} else {
report = UptakeTelemetry.STATUS.UNKNOWN_ERROR;
}
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY, report);
// No need to go further.
throw new Error(`Polling for changes failed: ${e.message}.`);
}
const {serverTimeMillis, changes, currentEtag, backoffSeconds} = pollResult;
// Report polling success to Uptake Telemetry.
const report = changes.length == 0 ? UptakeTelemetry.STATUS.UP_TO_DATE
: UptakeTelemetry.STATUS.SUCCESS;
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY, report);
// Check if the server asked the clients to back off (for next poll).
if (backoffSeconds) {
const backoffReleaseTime = Date.now() + backoffSeconds * 1000;
Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, backoffReleaseTime);
}
// Record new update time and the difference between local and server time.
// Negative clockDifference means local time is behind server time
// by the absolute of that value in seconds (positive means it's ahead)
const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
// Iterate through the collections version info and initiate a synchronization
// on the related remote settings client.
let firstError;
for (const change of changes) {
const {bucket, collection, last_modified: lastModified} = change;
const key = `${bucket}/${collection}`;
if (!gRemoteSettingsClients.has(key)) {
continue;
}
const client = gRemoteSettingsClients.get(key);
if (client.bucketName != bucket) {
continue;
}
try {
await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
} catch (e) {
if (!firstError) {
firstError = e;
// This is called by the ping mechanism.
// returns a promise that rejects if something goes wrong
remoteSettings.pollChanges = async () => {
// Check if the server backoff time is elapsed.
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
const backoffReleaseTime = Services.prefs.getCharPref(PREF_SETTINGS_SERVER_BACKOFF);
const remainingMilliseconds = parseInt(backoffReleaseTime, 10) - Date.now();
if (remainingMilliseconds > 0) {
// Backoff time has not elapsed yet.
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY,
UptakeTelemetry.STATUS.BACKOFF);
throw new Error(`Server is asking clients to back off; retry in ${Math.ceil(remainingMilliseconds / 1000)}s.`);
} else {
Services.prefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
}
}
}
if (firstError) {
// cause the promise to reject by throwing the first observed error
throw firstError;
}
// Save current Etag for next poll.
if (currentEtag) {
Services.prefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
}
// Right now, we only use the collection name and the last modified info
const kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
const changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
Services.obs.notifyObservers(null, "remote-settings-changes-polled");
let lastEtag;
if (Services.prefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
lastEtag = Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
}
let pollResult;
try {
pollResult = await fetchLatestChanges(changesEndpoint, lastEtag);
} catch (e) {
// Report polling error to Uptake Telemetry.
let report;
if (/Server/.test(e.message)) {
report = UptakeTelemetry.STATUS.SERVER_ERROR;
} else if (/NetworkError/.test(e.message)) {
report = UptakeTelemetry.STATUS.NETWORK_ERROR;
} else {
report = UptakeTelemetry.STATUS.UNKNOWN_ERROR;
}
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY, report);
// No need to go further.
throw new Error(`Polling for changes failed: ${e.message}.`);
}
const {serverTimeMillis, changes, currentEtag, backoffSeconds} = pollResult;
// Report polling success to Uptake Telemetry.
const report = changes.length == 0 ? UptakeTelemetry.STATUS.UP_TO_DATE
: UptakeTelemetry.STATUS.SUCCESS;
UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY, report);
// Check if the server asked the clients to back off (for next poll).
if (backoffSeconds) {
const backoffReleaseTime = Date.now() + backoffSeconds * 1000;
Services.prefs.setCharPref(PREF_SETTINGS_SERVER_BACKOFF, backoffReleaseTime);
}
// Record new update time and the difference between local and server time.
// Negative clockDifference means local time is behind server time
// by the absolute of that value in seconds (positive means it's ahead)
const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
// Iterate through the collections version info and initiate a synchronization
// on the related remote settings client.
let firstError;
for (const change of changes) {
const {bucket, collection, last_modified: lastModified} = change;
const key = `${bucket}/${collection}`;
if (!_clients.has(key)) {
continue;
}
const client = _clients.get(key);
if (client.bucketName != bucket) {
continue;
}
try {
await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
} catch (e) {
if (!firstError) {
firstError = e;
}
}
}
if (firstError) {
// cause the promise to reject by throwing the first observed error
throw firstError;
}
// Save current Etag for next poll.
if (currentEtag) {
Services.prefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
}
Services.obs.notifyObservers(null, "remote-settings-changes-polled");
};
return remoteSettings;
}
var RemoteSettings = remoteSettingsFunction();

32
services/common/tests/unit/test_blocklist_certificates.js
View File

@ -53,13 +53,11 @@ add_task(async function test_something() {
// Test an empty db populates
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
await OneCRLBlocklistClient.openCollection(async (collection) => {
// Open the collection, verify it's been populated:
const list = await collection.list();
// We know there will be initial values from the JSON dump.
// (at least as many as in the dump shipped when this test was written).
Assert.ok(list.data.length >= 363);
});
// Open the collection, verify it's been populated:
const list = await OneCRLBlocklistClient.get();
// We know there will be initial values from the JSON dump.
// (at least as many as in the dump shipped when this test was written).
Assert.ok(list.length >= 363);
// No sync will be intented if maybeSync() is up-to-date.
Services.prefs.clearUserPref("services.settings.server");
@ -83,22 +81,18 @@ add_task(async function test_something() {
await OneCRLBlocklistClient.maybeSync(2000, Date.now());
await OneCRLBlocklistClient.openCollection(async (collection) => {
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
const list = await collection.list();
Assert.equal(list.data.length, 1);
});
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
const before = await OneCRLBlocklistClient.get();
Assert.equal(before.length, 1);
// Test the db is updated when we call again with a later lastModified value
await OneCRLBlocklistClient.maybeSync(4000, Date.now());
await OneCRLBlocklistClient.openCollection(async (collection) => {
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
const list = await collection.list();
Assert.equal(list.data.length, 3);
});
// Open the collection, verify it's been updated:
// Our test data now has two records; both should be in the local collection
const after = await OneCRLBlocklistClient.get();
Assert.equal(after.length, 3);
// Try to maybeSync with the current lastModified value - no connection
// should be attempted.

14
services/common/tests/unit/test_blocklist_clients.js
View File

@ -103,11 +103,9 @@ add_task(async function test_initial_dump_is_loaded_as_synced_when_collection_is
// Test an empty db populates, but don't reach server (specified timestamp <= dump).
await client.maybeSync(1, Date.now());
// Open the collection, verify the loaded data has status to synced:
await client.openCollection(async (collection) => {
const list = await collection.list();
equal(list.data[0]._status, "synced");
});
// Verify the loaded data has status to synced:
const list = await client.get();
equal(list[0]._status, "synced");
}
});
add_task(clear_state);
@ -119,10 +117,8 @@ add_task(async function test_records_obtained_from_server_are_stored_in_db() {
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
await client.openCollection(async (collection) => {
const list = await collection.list();
equal(list.data.length, 1);
});
const list = await client.get();
equal(list.length, 1);
}
});
add_task(clear_state);

12
services/common/tests/unit/test_blocklist_pinning.js
View File

@ -84,10 +84,8 @@ add_task(async function test_something() {
// Open the collection, verify it's been populated:
// Our test data has a single record; it should be in the local collection
await PinningPreloadClient.openCollection(async (collection) => {
const list = await collection.list();
Assert.equal(list.data.length, 1);
});
const before = await PinningPreloadClient.get();
Assert.equal(before.length, 1);
// check that a pin exists for one.example.com
ok(sss.isSecureURI(sss.HEADER_HPKP,
@ -98,10 +96,8 @@ add_task(async function test_something() {
// Open the collection, verify it's been updated:
// Our data now has four new records; all should be in the local collection
await PinningPreloadClient.openCollection(async (collection) => {
const list = await collection.list();
Assert.equal(list.data.length, 5);
});
const after = await PinningPreloadClient.get();
Assert.equal(after.length, 5);
// check that a pin exists for two.example.com and three.example.com
ok(sss.isSecureURI(sss.HEADER_HPKP,

8
services/common/tests/unit/test_blocklist_signatures.js
View File

@ -51,11 +51,9 @@ function getCertChain() {
}
async function checkRecordCount(client, count) {
await client.openCollection(async (collection) => {
// Check we have the expected number of records
const records = await collection.list();
Assert.equal(count, records.data.length);
});
// Check we have the expected number of records
const records = await client.get();
Assert.equal(count, records.length);
}
// Check to ensure maybeSync is called with correct values when a changes

4
services/common/tests/unit/test_remote_settings_poll.js
View File

@ -1,6 +1,6 @@
ChromeUtils.import("resource://testing-common/httpd.js");
const { UptakeTelemetry } = ChromeUtils.import("resource://services-common/uptake-telemetry.js", {});
const RemoteSettings = ChromeUtils.import("resource://services-common/remote-settings.js", {});
const { RemoteSettings } = ChromeUtils.import("resource://services-common/remote-settings.js", {});
var server;
@ -60,7 +60,7 @@ add_task(async function test_check_maybeSync() {
// ensure we get the maybeSync call
// add a test kinto client that will respond to lastModified information
// for a collection called 'test-collection'
const c = RemoteSettings.RemoteSettings("test-collection", {
const c = RemoteSettings("test-collection", {
bucketName: "test-bucket",
});
c.maybeSync = () => {};

3
toolkit/mozapps/extensions/nsBlocklistService.js
View File

@ -36,7 +36,8 @@ XPCOMUtils.defineLazyGetter(this, "RemoteSettings", function() {
// Instantiate blocklist clients.
BlocklistClients.initialize();
// Import RemoteSettings for ``pollChanges()``
return ChromeUtils.import("resource://services-common/remote-settings.js", {});
const { RemoteSettings } = ChromeUtils.import("resource://services-common/remote-settings.js", {});
return RemoteSettings;
});
const TOOLKIT_ID = "toolkit@mozilla.org";