Bug 1253740 - Introduce extension-storage engine with a sanity test, r=markh

Note that this "enables" the engine using a pref, even though it might
not be ready yet, so that the tests can pass.

MozReview-Commit-ID: AZ0TVERiQDU

--HG--
extra : rebase_source : e8518187e3a4f404bad193ce26f6c523ec06abe0
extra : intermediate-source : b052cf501ce8a838706f63f46eb6262b63ac5560
extra : source : 183547f4dbbedc9ee3399b6a474016d0e89a12e8

browser/app/profile/firefox.js

@@ -1043,7 +1043,7 @@ pref("browser.taskbar.lists.refreshInSeconds", 120);
 #endif
 
 // The sync engines to use.
-pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons");
+pref("services.sync.registerEngines", "Bookmarks,Form,History,Password,Prefs,Tab,Addons,ExtensionStorage");
 // Preferences to be synced by default
 pref("services.sync.prefs.sync.accessibility.blockautorefresh", true);
 pref("services.sync.prefs.sync.accessibility.browsewithcaret", true);

services/sync/modules/engines/extension-storage.js

@@ -0,0 +1,103 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+                                  "resource://gre/modules/ExtensionStorageSync.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+  SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+  _trackerObj: ExtensionStorageTracker,
+  // we don't need these since we implement our own sync logic
+  _storeObj: undefined,
+  _recordObj: undefined,
+
+  syncPriority: 10,
+
+  _sync: function () {
+    return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+  },
+
+  get enabled() {
+    // By default, we sync extension storage if we sync addons. This
+    // lets us simplify the UX since users probably don't consider
+    // "extension preferences" a separate category of syncing.
+    // However, we also respect engine.extension-storage.force, which
+    // can be set to true or false, if a power user wants to customize
+    // the behavior despite the lack of UI.
+    const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+    if (forced !== undefined) {
+      return forced;
+    }
+    return Svc.Prefs.get("engine.addons", false);
+  },
+};
+
+function ExtensionStorageTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+  __proto__: Tracker.prototype,
+
+  startTracking: function () {
+    Svc.Obs.add("ext.storage.sync-changed", this);
+  },
+
+  stopTracking: function () {
+    Svc.Obs.remove("ext.storage.sync-changed", this);
+  },
+
+  observe: function (subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+
+    if (this.ignoreAll) {
+      return;
+    }
+
+    if (topic !== "ext.storage.sync-changed") {
+      return;
+    }
+
+    // Single adds, removes and changes are not so important on their
+    // own, so let's just increment score a bit.
+    this.score += SCORE_INCREMENT_MEDIUM;
+  },
+
+  // Override a bunch of methods which don't do anything for us.
+  // This is a performance hack.
+  saveChangedIDs: function() {
+  },
+  loadChangedIDs: function() {
+  },
+  ignoreID: function() {
+  },
+  unignoreID: function() {
+  },
+  addChangedID: function() {
+  },
+  removeChangedID: function() {
+  },
+  clearChangedIDs: function() {
+  },
+};

services/sync/modules/service.js

@@ -44,6 +44,7 @@ const ENGINE_MODULES = {
   Password: "passwords.js",
   Prefs: "prefs.js",
   Tab: "tabs.js",
+  ExtensionStorage: "extension-storage.js",
 };
 
 const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,

services/sync/modules/telemetry.js

@@ -51,7 +51,7 @@ const PING_FORMAT_VERSION = 1;
 
 // The set of engines we record telemetry for - any other engines are ignored.
 const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
-                         "passwords", "prefs", "tabs"]);
+                         "passwords", "prefs", "tabs", "extension-storage"]);
 
 // A regex we can use to replace the profile dir in error messages. We use a
 // regexp so we can simply replace all case-insensitive occurences.

services/sync/moz.build

@@ -52,6 +52,7 @@ EXTRA_JS_MODULES['services-sync'].engines += [
     'modules/engines/addons.js',
     'modules/engines/bookmarks.js',
     'modules/engines/clients.js',
+    'modules/engines/extension-storage.js',
     'modules/engines/forms.js',
     'modules/engines/history.js',
     'modules/engines/passwords.js',

services/sync/services-sync.js

@@ -31,6 +31,7 @@ pref("services.sync.engine.passwords", true);
 pref("services.sync.engine.prefs", true);
 pref("services.sync.engine.tabs", true);
 pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|chrome://weave/.*|wyciwyg:.*|file:.*|blob:.*)$");
+pref("services.sync.engine.extension-storage", true);
 
 pref("services.sync.jpake.serverURL", "https://setup.services.mozilla.com/");
 pref("services.sync.jpake.pollInterval", 1000);
@@ -68,6 +69,7 @@ pref("services.sync.log.logger.engine.passwords", "Debug");
 pref("services.sync.log.logger.engine.prefs", "Debug");
 pref("services.sync.log.logger.engine.tabs", "Debug");
 pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.extension-storage", "Debug");
 pref("services.sync.log.logger.engine.apps", "Debug");
 pref("services.sync.log.logger.identity", "Debug");
 pref("services.sync.log.logger.userapi", "Debug");

services/sync/tests/unit/head_helpers.js

@@ -76,6 +76,24 @@ function loadAddonTestFunctions() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 }
 
+function webExtensionsTestPath(path) {
+  if (path[0] != "/") {
+    throw Error("Path must begin with '/': " + path);
+  }
+
+  return "../../../../toolkit/components/extensions/test/xpcshell" + path;
+}
+
+/**
+ * Loads the WebExtension test functions by importing its test file.
+ */
+function loadWebExtensionTestFunctions() {
+  const path = webExtensionsTestPath("/head_sync.js");
+  let file = do_get_file(path);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec, gGlobalScope);
+}
+
 function getAddonInstall(name) {
   let f = do_get_file(ExtensionsTestPath("/addons/" + name + ".xpi"));
   let cb = Async.makeSyncCallback();

services/sync/tests/unit/test_extension_storage_engine.js

@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+function mock(options) {
+  let calls = [];
+  let ret = function() {
+    calls.push(arguments);
+    return options.returns;
+  }
+  Object.setPrototypeOf(ret, {
+    __proto__: Function.prototype,
+    get calls() {
+      return calls;
+    }
+  });
+  return ret;
+}
+
+add_task(function* test_calling_sync_calls__sync() {
+  let oldSync = ExtensionStorageEngine.prototype._sync;
+  let syncMock = ExtensionStorageEngine.prototype._sync = mock({returns: true});
+  try {
+    // I wanted to call the main sync entry point for the entire
+    // package, but that fails because it tries to sync ClientEngine
+    // first, which fails.
+    yield engine.sync();
+  } finally {
+    ExtensionStorageEngine.prototype._sync = oldSync;
+  }
+  equal(syncMock.calls.length, 1);
+});
+
+add_task(function* test_calling_sync_calls_ext_storage_sync() {
+  const extension = {id: "my-extension"};
+  let oldSync = ExtensionStorageSync.syncAll;
+  let syncMock = ExtensionStorageSync.syncAll = mock({returns: Promise.resolve()});
+  try {
+    yield* withSyncContext(function* (context) {
+      // Set something so that everyone knows that we're using storage.sync
+      yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+
+      yield engine._sync();
+    });
+  } finally {
+    ExtensionStorageSync.syncAll = oldSync;
+  }
+  do_check_true(syncMock.calls.length >= 1);
+});

services/sync/tests/unit/test_extension_storage_tracker.js

@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/extension-storage.js");
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/ExtensionStorageSync.jsm");
+
+Service.engineManager.register(ExtensionStorageEngine);
+const engine = Service.engineManager.get("extension-storage");
+do_get_profile();   // so we can use FxAccounts
+loadWebExtensionTestFunctions();
+
+add_task(function* test_changing_extension_storage_changes_score() {
+  const tracker = engine._tracker;
+  const extension = {id: "my-extension-id"};
+  Svc.Obs.notify("weave:engine:start-tracking");
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.set(extension, {"a": "b"}, context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  tracker.resetScore();
+  yield* withSyncContext(function*(context) {
+    yield ExtensionStorageSync.remove(extension, "a", context);
+  });
+  do_check_eq(tracker.score, SCORE_INCREMENT_MEDIUM);
+
+  Svc.Obs.notify("weave:engine:stop-tracking");
+});
+
+function run_test() {
+  run_next_test();
+}

services/sync/tests/unit/test_load_modules.js

@@ -9,6 +9,7 @@ const modules = [
   "engines/addons.js",
   "engines/bookmarks.js",
   "engines/clients.js",
+  "engines/extension-storage.js",
   "engines/forms.js",
   "engines/history.js",
   "engines/passwords.js",

services/sync/tests/unit/xpcshell.ini

@@ -15,6 +15,7 @@ support-files =
   systemaddon-search.xml
   !/services/common/tests/unit/head_helpers.js
   !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js
+  !/toolkit/components/extensions/test/xpcshell/head_sync.js
 
 # The manifest is roughly ordered from low-level to high-level. When making
 # systemic sweeping changes, this makes it easier to identify errors closer to
@@ -162,6 +163,8 @@ skip-if = debug
 [test_bookmark_validator.js]
 [test_clients_engine.js]
 [test_clients_escape.js]
+[test_extension_storage_engine.js]
+[test_extension_storage_tracker.js]
 [test_forms_store.js]
 [test_forms_tracker.js]
 # Too many intermittent "ASSERTION: thread pool wasn't shutdown: '!mPool'" (bug 804479)

toolkit/components/extensions/ExtensionStorageSync.jsm

@@ -25,6 +25,8 @@ XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorage",
                                   "resource://gre/modules/ExtensionStorage.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "loadKinto",
                                   "resource://services-common/kinto-offline-client.js");
+XPCOMUtils.defineLazyModuleGetter(this, "Observers",
+                                  "resource://services-common/observers.js");
 XPCOMUtils.defineLazyModuleGetter(this, "Task",
                                   "resource://gre/modules/Task.jsm");
 XPCOMUtils.defineLazyPreferenceGetter(this, "prefPermitsStorageSync",
@@ -327,6 +329,7 @@ this.ExtensionStorageSync = {
   },
 
   notifyListeners(extension, changes) {
+    Observers.notify("ext.storage.sync-changed");
     let listeners = this.listeners.get(extension) || new Set();
     if (listeners) {
       for (let listener of listeners) {

toolkit/components/extensions/test/xpcshell/head_sync.js

@@ -0,0 +1,67 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+/* exported withSyncContext */
+
+Components.utils.import("resource://gre/modules/Services.jsm", this);
+Components.utils.import("resource://gre/modules/ExtensionUtils.jsm", this);
+
+var {
+  BaseContext,
+} = ExtensionUtils;
+
+class Context extends BaseContext {
+  constructor(principal) {
+    super();
+    Object.defineProperty(this, "principal", {
+      value: principal,
+      configurable: true,
+    });
+    this.sandbox = Components.utils.Sandbox(principal, {wantXrays: false});
+    this.extension = {id: "test@web.extension"};
+  }
+
+  get cloneScope() {
+    return this.sandbox;
+  }
+}
+
+/**
+ * Call the given function with a newly-constructed context.
+ * Unload the context on the way out.
+ *
+ * @param {function} f    the function to call
+ */
+function* withContext(f) {
+  const ssm = Services.scriptSecurityManager;
+  const PRINCIPAL1 = ssm.createCodebasePrincipalFromOrigin("http://www.example.org");
+  const context = new Context(PRINCIPAL1);
+  try {
+    yield* f(context);
+  } finally {
+    yield context.unload();
+  }
+}
+
+/**
+ * Like withContext(), but also turn on the "storage.sync" pref for
+ * the duration of the function.
+ * Calls to this function can be replaced with calls to withContext
+ * once the pref becomes on by default.
+ *
+ * @param {function} f    the function to call
+ */
+function* withSyncContext(f) {
+  const STORAGE_SYNC_PREF = "webextensions.storage.sync.enabled";
+  let prefs = Services.prefs;
+
+  try {
+    prefs.setBoolPref(STORAGE_SYNC_PREF, true);
+    yield* withContext(f);
+  } finally {
+    prefs.clearUserPref(STORAGE_SYNC_PREF);
+  }
+}

toolkit/components/extensions/test/xpcshell/xpcshell.ini

@@ -4,7 +4,7 @@ tail =
 firefox-appdir = browser
 skip-if = appname == "thunderbird"
 support-files =
-  data/**
+  data/** head_sync.js
 tags = webextensions
 
 [test_csp_custom_policies.js]