From 835f7407b3ceade3e1600ba73ba31984342f350f Mon Sep 17 00:00:00 2001 From: Cosmin Sabou Date: Sun, 3 Jun 2018 16:10:23 +0300 Subject: [PATCH] Backed out 10 changesets (bug 1406181) as per developers request. Backed out changeset 06461ddb2699 (bug 1406181) Backed out changeset fd61d9faedf0 (bug 1406181) Backed out changeset b52c2fb70ae1 (bug 1406181) Backed out changeset 4f387b4a76a9 (bug 1406181) Backed out changeset db783c96c076 (bug 1406181) Backed out changeset 62e9126ecd0d (bug 1406181) Backed out changeset d34810cab822 (bug 1406181) Backed out changeset 3241c2dfb296 (bug 1406181) Backed out changeset 912a2eaf4d26 (bug 1406181) Backed out changeset fdac47b8ef20 (bug 1406181) --- modules/libpref/init/all.js | 3 - .../ContextualIdentityService.jsm | 75 +-- .../tests/unit/test_corruptedFile.js | 37 +- .../tests/unit/test_migratedFile.js | 93 ---- .../tests/unit/xpcshell.ini | 2 - toolkit/components/extensions/Extension.jsm | 25 +- .../extensions/ExtensionStorage.jsm | 84 +--- .../extensions/ExtensionStorageIDB.jsm | 460 ------------------ .../extensions/child/ext-storage.js | 228 ++++----- toolkit/components/extensions/moz.build | 1 - .../extensions/parent/ext-storage.js | 52 +- .../mochitest/test_ext_storage_cleanup.html | 244 ++++------ .../extensions/test/xpcshell/head.js | 32 +- .../test/xpcshell/test_ext_storage.js | 207 ++++---- .../test/xpcshell/test_ext_storage_content.js | 32 +- .../test_ext_storage_idb_data_migration.js | 151 ------ .../test/xpcshell/test_ext_storage_tab.js | 14 +- .../xpcshell/test_ext_storage_telemetry.js | 14 +- .../test/xpcshell/xpcshell-common.ini | 1 - toolkit/modules/IndexedDB.jsm | 86 +--- 20 files changed, 330 insertions(+), 1511 deletions(-) delete mode 100644 toolkit/components/contextualidentity/tests/unit/test_migratedFile.js delete mode 100644 toolkit/components/extensions/ExtensionStorageIDB.jsm delete mode 100644 toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js index 1c038ae0f5c8..83e95ccaa584 100644 --- a/modules/libpref/init/all.js +++ b/modules/libpref/init/all.js @@ -4945,9 +4945,6 @@ pref("extensions.webextensions.tabhide.enabled", true); pref("extensions.webextensions.background-delayed-startup", false); -// Whether or not the installed extensions should be migrated to the storage.local IndexedDB backend. -pref("extensions.webextensions.ExtensionStorageIDB.enabled", false); - // Report Site Issue button pref("extensions.webcompat-reporter.newIssueEndpoint", "https://webcompat.com/issues/new"); #if defined(MOZ_DEV_EDITION) || defined(NIGHTLY_BUILD) diff --git a/toolkit/components/contextualidentity/ContextualIdentityService.jsm b/toolkit/components/contextualidentity/ContextualIdentityService.jsm index 7a9699502e09..aa9451463b58 100644 --- a/toolkit/components/contextualidentity/ContextualIdentityService.jsm +++ b/toolkit/components/contextualidentity/ContextualIdentityService.jsm @@ -100,19 +100,7 @@ _ContextualIdentityService.prototype = { icon: "", color: "", name: "userContextIdInternal.thumbnail", - accessKey: "", - }, - // This userContextId is used by ExtensionStorageIDB.jsm to create an IndexedDB database - // opened with the extension principal but not directly accessible to the extension code - // (do not change the userContextId assigned here, otherwise the installed extensions will - // not be able to access the data previously stored with the browser.storage.local API). - { userContextId: -10, - public: false, - icon: "", - color: "", - name: "userContextIdInternal.webextStorageLocal", - accessKey: "", - }, + accessKey: "" }, ], _identities: null, @@ -160,13 +148,10 @@ _ContextualIdentityService.prototype = { resetDefault() { this._identities = []; - - this._lastUserContextId = this._defaultIdentities.map( - identity => identity.userContextId).sort().pop(); - // Clone the array - for (let identity of this._defaultIdentities) { - this._identities.push(Object.assign({}, identity)); + this._lastUserContextId = this._defaultIdentities.length; + for (let i = 0; i < this._lastUserContextId; i++) { + this._identities.push(Object.assign({}, this._defaultIdentities[i])); } this._openedIdentities = new Set(); @@ -311,12 +296,7 @@ _ContextualIdentityService.prototype = { saveNeeded = true; } - if (data.version == 3) { - data = this.migrate3to4(data); - saveNeeded = true; - } - - if (data.version != 4) { + if (data.version != 3) { dump("ERROR - ContextualIdentityService - Unknown version found in " + this._path + "\n"); this.loadError(null); return; @@ -365,13 +345,6 @@ _ContextualIdentityService.prototype = { return Cu.cloneInto(this._identities.find(info => !info.public && info.name == name), {}); }, - // getDefaultPrivateIdentity is similar to getPrivateIdentity but it only looks in the - // default identities (e.g. it is used in the data migration methods to retrieve a new default - // private identity and add it to the containers data stored on file). - getDefaultPrivateIdentity(name) { - return Cu.cloneInto(this._defaultIdentities.find(info => !info.public && info.name == name), {}); - }, - getPublicIdentityFromId(userContextId) { this.ensureDataReady(); return Cu.cloneInto(this._identities.find(info => info.userContextId == userContextId && @@ -435,11 +408,6 @@ _ContextualIdentityService.prototype = { notifyAllContainersCleared() { for (let identity of this._identities) { - // Don't clear the data related to private identities (e.g. the one used internally - // for the thumbnails and the one used for the storage.local IndexedDB backend). - if (!identity.public) { - continue; - } Services.obs.notifyObservers(null, "clear-origin-attributes-data", JSON.stringify({ userContextId: identity.userContextId })); } @@ -492,11 +460,8 @@ _ContextualIdentityService.prototype = { }, deleteContainerData() { - // Compute the range of userContextId to clear (and exclude 0 which is reserved - // to the default firefox identity). let minUserContextId = 1; let maxUserContextId = minUserContextId; - const enumerator = Services.cookies.enumerator; while (enumerator.hasMoreElements()) { const cookie = enumerator.getNext().QueryInterface(Ci.nsICookie); @@ -505,18 +470,9 @@ _ContextualIdentityService.prototype = { } } - // Collect the userContextId related to the identities that should not be cleared - // (the ones marked as `public = false`). - const keepDataContextIds = this._identities - .filter(identity => !identity.public) - .map(identity => identity.userContextId); - for (let i = minUserContextId; i <= maxUserContextId; ++i) { - // Skip any userContextIds that should not be cleared. - if (!keepDataContextIds.includes(i)) { - Services.obs.notifyObservers(null, "clear-origin-attributes-data", - JSON.stringify({ userContextId: i })); - } + Services.obs.notifyObservers(null, "clear-origin-attributes-data", + JSON.stringify({ userContextId: i })); } }, @@ -527,23 +483,6 @@ _ContextualIdentityService.prototype = { return data; }, - - migrate3to4(data) { - // Migrating from 3 to 4 is: - // - adding the reserver userContextId used by the webextension storage.local API - // - add the keepData property to all the existent identities - // - increasing the version id. - // - // This migration was needed for Bug 1406181. See bug 1406181 for rationale. - const webextStorageLocalIdentity = this.getDefaultPrivateIdentity( - "userContextIdInternal.webextStorageLocal"); - - data.identities.push(webextStorageLocalIdentity); - - data.version = 4; - - return data; - }, }; let path = OS.Path.join(OS.Constants.Path.profileDir, "containers.json"); diff --git a/toolkit/components/contextualidentity/tests/unit/test_corruptedFile.js b/toolkit/components/contextualidentity/tests/unit/test_corruptedFile.js index d3be15c75a29..962c935dcb04 100644 --- a/toolkit/components/contextualidentity/tests/unit/test_corruptedFile.js +++ b/toolkit/components/contextualidentity/tests/unit/test_corruptedFile.js @@ -19,9 +19,10 @@ const COOKIE = { isHttpOnly: false, isSession: true, expiry: 2145934800, + originAttributes: { userContextId: 1 }, }; -function createCookie(userContextId) { +function createCookie() { Services.cookies.add(COOKIE.host, COOKIE.path, COOKIE.name, @@ -30,15 +31,15 @@ function createCookie(userContextId) { COOKIE.isHttpOnly, COOKIE.isSession, COOKIE.expiry, - {userContextId}); + COOKIE.originAttributes); } -function hasCookie(userContextId) { +function hasCookie() { let found = false; - let enumerator = Services.cookies.getCookiesFromHost(BASE_URL, {userContextId}); + let enumerator = Services.cookies.getCookiesFromHost(BASE_URL, COOKIE.originAttributes); while (enumerator.hasMoreElements()) { let cookie = enumerator.getNext().QueryInterface(Ci.nsICookie); - if (cookie.originAttributes.userContextId == userContextId) { + if (cookie.originAttributes.userContextId == COOKIE.originAttributes.userContextId) { found = true; break; } @@ -48,17 +49,8 @@ function hasCookie(userContextId) { // Correpted file should delete all. add_task(async function corruptedFile() { - const thumbnailPrivateId = ContextualIdentityService._defaultIdentities.filter( - identity => identity.name === "userContextIdInternal.thumbnail").pop().userContextId; - - // Create a cookie in the userContextId 1. - createCookie(1); - - // Create a cookie in the thumbnail private userContextId. - createCookie(thumbnailPrivateId); - - ok(hasCookie(1), "We have the new cookie in a public identity!"); - ok(hasCookie(thumbnailPrivateId), "We have the new cookie in the thumbnail private identity!"); + createCookie(); + ok(hasCookie(), "We have the new cookie!"); // Let's create a corrupted file. await OS.File.writeAtomic(TEST_STORE_FILE_PATH, "{ vers", @@ -67,17 +59,8 @@ add_task(async function corruptedFile() { let cis = ContextualIdentityService.createNewInstanceForTesting(TEST_STORE_FILE_PATH); ok(!!cis, "We have our instance of ContextualIdentityService"); - equal(cis.getPublicIdentities().length, 4, "We should have the default public identities"); - - const privThumbnailIdentity = cis.getPrivateIdentity("userContextIdInternal.thumbnail"); - equal(privThumbnailIdentity && privThumbnailIdentity.userContextId, thumbnailPrivateId, - "We should have the default thumbnail private identity"); + equal(cis.getPublicIdentities().length, 4, "We should have the default identities"); // Cookie is gone! - ok(!hasCookie(1), "We should not have the new cookie in the userContextId 1!"); - - // The data stored in the non-public userContextId (e.g. thumbnails private identity) - // should have not be cleared. - ok(hasCookie(thumbnailPrivateId), - "We should have the new cookie in the thumbnail private userContextId!"); + ok(!hasCookie(), "We should not have the new cookie!"); }); diff --git a/toolkit/components/contextualidentity/tests/unit/test_migratedFile.js b/toolkit/components/contextualidentity/tests/unit/test_migratedFile.js deleted file mode 100644 index 3b9cd642bf08..000000000000 --- a/toolkit/components/contextualidentity/tests/unit/test_migratedFile.js +++ /dev/null @@ -1,93 +0,0 @@ -"use strict"; - -const profileDir = do_get_profile(); - -ChromeUtils.import("resource://gre/modules/ContextualIdentityService.jsm"); -ChromeUtils.import("resource://gre/modules/Services.jsm"); -ChromeUtils.import("resource://gre/modules/osfile.jsm"); - -const TEST_STORE_FILE_PATH = OS.Path.join(profileDir.path, "test-containers.json"); - -// Test the containers JSON file migrations. -add_task(async function migratedFile() { - // Let's create a file that has to be migrated. - const oldFileData = { - version: 2, - lastUserContextId: 6, - identities: [ - { userContextId: 1, - public: true, - icon: "fingerprint", - color: "blue", - l10nID: "userContextPersonal.label", - accessKey: "userContextPersonal.accesskey", - telemetryId: 1, - }, - { userContextId: 2, - public: true, - icon: "briefcase", - color: "orange", - l10nID: "userContextWork.label", - accessKey: "userContextWork.accesskey", - telemetryId: 2, - }, - { userContextId: 3, - public: true, - icon: "dollar", - color: "green", - l10nID: "userContextBanking.label", - accessKey: "userContextBanking.accesskey", - telemetryId: 3, - }, - { userContextId: 4, - public: true, - icon: "cart", - color: "pink", - l10nID: "userContextShopping.label", - accessKey: "userContextShopping.accesskey", - telemetryId: 4, - }, - { userContextId: 5, - public: false, - icon: "", - color: "", - name: "userContextIdInternal.thumbnail", - accessKey: "", - }, - { userContextId: 6, - public: true, - icon: "cart", - color: "ping", - name: "Custom user-created identity", - }, - ] - }; - - await OS.File.writeAtomic(TEST_STORE_FILE_PATH, JSON.stringify(oldFileData), - { tmpPath: TEST_STORE_FILE_PATH + ".tmp" }); - - let cis = ContextualIdentityService.createNewInstanceForTesting(TEST_STORE_FILE_PATH); - ok(!!cis, "We have our instance of ContextualIdentityService"); - - // Check that the custom user-created identity exists. - - const expectedPublicLength = oldFileData.identities.filter( - identity => identity.public).length; - const publicIdentities = cis.getPublicIdentities(); - const oldLastIdentity = oldFileData.identities[oldFileData.identities.length - 1]; - const customUserCreatedIdentity = publicIdentities.filter( - identity => identity.name === oldLastIdentity.name).pop(); - - equal(publicIdentities.length, expectedPublicLength, - "We should have the expected number of public identities"); - ok(!!customUserCreatedIdentity, "Got the custom user-created identity"); - - // Check that the reserved userContextIdInternal.webextStorageLocal identity exists. - - const webextStorageLocalPrivateId = ContextualIdentityService._defaultIdentities.filter( - identity => identity.name === "userContextIdInternal.webextStorageLocal").pop().userContextId; - - const privWebExtStorageLocal = cis.getPrivateIdentity("userContextIdInternal.webextStorageLocal"); - equal(privWebExtStorageLocal && privWebExtStorageLocal.userContextId, webextStorageLocalPrivateId, - "We should have the default userContextIdInternal.webextStorageLocal private identity"); -}); diff --git a/toolkit/components/contextualidentity/tests/unit/xpcshell.ini b/toolkit/components/contextualidentity/tests/unit/xpcshell.ini index 400314f3cd7a..b5febc5b367d 100644 --- a/toolkit/components/contextualidentity/tests/unit/xpcshell.ini +++ b/toolkit/components/contextualidentity/tests/unit/xpcshell.ini @@ -5,5 +5,3 @@ firefox-appdir = browser skip-if = appname == "thunderbird" [test_corruptedFile.js] skip-if = appname == "thunderbird" -[test_migratedFile.js] -skip-if = appname == "thunderbird" diff --git a/toolkit/components/extensions/Extension.jsm b/toolkit/components/extensions/Extension.jsm index 119e1686f654..85d26c34d892 100644 --- a/toolkit/components/extensions/Extension.jsm +++ b/toolkit/components/extensions/Extension.jsm @@ -41,7 +41,6 @@ XPCOMUtils.defineLazyModuleGetters(this, { AddonSettings: "resource://gre/modules/addons/AddonSettings.jsm", AppConstants: "resource://gre/modules/AppConstants.jsm", AsyncShutdown: "resource://gre/modules/AsyncShutdown.jsm", - ContextualIdentityService: "resource://gre/modules/ContextualIdentityService.jsm", ExtensionCommon: "resource://gre/modules/ExtensionCommon.jsm", ExtensionPermissions: "resource://gre/modules/ExtensionPermissions.jsm", ExtensionStorage: "resource://gre/modules/ExtensionStorage.jsm", @@ -104,13 +103,6 @@ XPCOMUtils.defineLazyGetter(this, "console", ExtensionUtils.getConsole); XPCOMUtils.defineLazyGetter(this, "LocaleData", () => ExtensionCommon.LocaleData); -// The userContextID reserved for the extension storage (its purpose is ensuring that the IndexedDB -// storage used by the browser.storage.local API is not directly accessible from the extension code). -XPCOMUtils.defineLazyGetter(this, "WEBEXT_STORAGE_USER_CONTEXT_ID", () => { - return ContextualIdentityService.getDefaultPrivateIdentity( - "userContextIdInternal.webextStorageLocal").userContextId; -}); - /** * Classify an individual permission from a webextension manifest * as a host/origin permission, an api permission, or a regular permission. @@ -225,10 +217,10 @@ var UninstallObserver = { } if (!Services.prefs.getBoolPref(LEAVE_STORAGE_PREF, false)) { - // Clear browser.storage.local backends. + // Clear browser.local.storage AsyncShutdown.profileChangeTeardown.addBlocker( - `Clear Extension Storage ${addon.id} (File Backend)`, - ExtensionStorage.clear(addon.id, {shouldNotifyListeners: false})); + `Clear Extension Storage ${addon.id}`, + ExtensionStorage.clear(addon.id)); // Clear any IndexedDB storage created by the extension let baseURI = Services.io.newURI(`moz-extension://${uuid}/`); @@ -236,12 +228,6 @@ var UninstallObserver = { baseURI, {}); Services.qms.clearStoragesForPrincipal(principal); - // Clear any storage.local data stored in the IDBBackend. - let storagePrincipal = Services.scriptSecurityManager.createCodebasePrincipal(baseURI, { - userContextId: WEBEXT_STORAGE_USER_CONTEXT_ID, - }); - Services.qms.clearStoragesForPrincipal(storagePrincipal); - // Clear localStorage created by the extension let storage = Services.domStorageManager.getStorage(null, principal); if (storage) { @@ -1285,7 +1271,6 @@ class Extension extends ExtensionData { this.baseURL = this.getURL(""); this.baseURI = Services.io.newURI(this.baseURL).QueryInterface(Ci.nsIURL); this.principal = this.createPrincipal(); - this.views = new Set(); this._backgroundPageFrameLoader = null; @@ -1407,8 +1392,8 @@ class Extension extends ExtensionData { this.emit("test-harness-message", ...args); } - createPrincipal(uri = this.baseURI, originAttributes = {}) { - return Services.scriptSecurityManager.createCodebasePrincipal(uri, originAttributes); + createPrincipal(uri = this.baseURI) { + return Services.scriptSecurityManager.createCodebasePrincipal(uri, {}); } // Checks that the given URL is a child of our baseURI. diff --git a/toolkit/components/extensions/ExtensionStorage.jsm b/toolkit/components/extensions/ExtensionStorage.jsm index c2c54925ad40..beb3e536146a 100644 --- a/toolkit/components/extensions/ExtensionStorage.jsm +++ b/toolkit/components/extensions/ExtensionStorage.jsm @@ -129,18 +129,6 @@ var ExtensionStorage = { return promise; }, - /** - * Clear the cached jsonFilePromise for a given extensionId - * (used by ExtensionStorageIDB to free the jsonFile once the data migration - * has been completed). - * - * @param {string} extensionId - * The ID of the extension for which to return a file. - */ - clearCachedFile(extensionId) { - this.jsonFilePromises.delete(extensionId); - }, - /** * Sanitizes the given value, and returns a JSON-compatible * representation of it, based on the privileges of the given global. @@ -251,32 +239,22 @@ var ExtensionStorage = { * * @param {string} extensionId * The ID of the extension for which to clear storage. - * @param {object} options - * @param {boolean} [options.shouldNotifyListeners = true] - * Whether or not collect and send the changes to the listeners, - * used when the extension data is being cleared on uninstall. * @returns {Promise} */ - async clear(extensionId, {shouldNotifyListeners = true} = {}) { + async clear(extensionId) { let jsonFile = await this.getFile(extensionId); let changed = false; let changes = {}; for (let [prop, oldValue] of jsonFile.data.entries()) { - if (shouldNotifyListeners) { - changes[prop] = {oldValue: serialize(oldValue)}; - } - + changes[prop] = {oldValue: serialize(oldValue)}; jsonFile.data.delete(prop); changed = true; } if (changed) { - if (shouldNotifyListeners) { - this.notifyListeners(extensionId, changes); - } - + this.notifyListeners(extensionId, changes); jsonFile.saveSoon(); } return null; @@ -367,62 +345,6 @@ var ExtensionStorage = { this.jsonFilePromises.clear(); } }, - - // Serializes an arbitrary value into a StructuredCloneHolder, if appropriate. - serialize, - - /** - * Serializes the given storage items for transporting between processes. - * - * @param {BaseContext} context - * The context to use for the created StructuredCloneHolder - * objects. - * @param {Array|object} items - * The items to serialize. If an object is provided, its - * values are serialized to StructuredCloneHolder objects. - * Otherwise, it is returned as-is. - * @returns {Array|object} - */ - serializeForContext(context, items) { - if (items && typeof items === "object" && !Array.isArray(items)) { - let result = {}; - for (let [key, value] of Object.entries(items)) { - try { - result[key] = new StructuredCloneHolder(value, context.cloneScope); - } catch (e) { - throw new ExtensionUtils.ExtensionError(String(e)); - } - } - return result; - } - return items; - }, - - /** - * Deserializes the given storage items into the given extension context. - * - * @param {BaseContext} context - * The context to use to deserialize the StructuredCloneHolder objects. - * @param {object} items - * The items to deserialize. Any property of the object which - * is a StructuredCloneHolder instance is deserialized into - * the extension scope. Any other object is cloned into the - * extension scope directly. - * @returns {object} - */ - deserializeForContext(context, items) { - let result = new context.cloneScope.Object(); - for (let [key, value] of Object.entries(items)) { - if (value && typeof value === "object" && - Cu.getClassName(value, true) === "StructuredCloneHolder") { - value = value.deserialize(context.cloneScope); - } else { - value = Cu.cloneInto(value, context.cloneScope); - } - result[key] = value; - } - return result; - }, }; XPCOMUtils.defineLazyGetter( diff --git a/toolkit/components/extensions/ExtensionStorageIDB.jsm b/toolkit/components/extensions/ExtensionStorageIDB.jsm deleted file mode 100644 index 8f44846780dd..000000000000 --- a/toolkit/components/extensions/ExtensionStorageIDB.jsm +++ /dev/null @@ -1,460 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -this.EXPORTED_SYMBOLS = ["ExtensionStorageIDB"]; - -ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); -ChromeUtils.import("resource://gre/modules/IndexedDB.jsm"); - -XPCOMUtils.defineLazyModuleGetters(this, { - ContextualIdentityService: "resource://gre/modules/ContextualIdentityService.jsm", - ExtensionStorage: "resource://gre/modules/ExtensionStorage.jsm", - ExtensionUtils: "resource://gre/modules/ExtensionUtils.jsm", - Services: "resource://gre/modules/Services.jsm", - OS: "resource://gre/modules/osfile.jsm", -}); - -// The userContextID reserved for the extension storage (its purpose is ensuring that the IndexedDB -// storage used by the browser.storage.local API is not directly accessible from the extension code). -XPCOMUtils.defineLazyGetter(this, "WEBEXT_STORAGE_USER_CONTEXT_ID", () => { - return ContextualIdentityService.getDefaultPrivateIdentity( - "userContextIdInternal.webextStorageLocal").userContextId; -}); - -const IDB_NAME = "webExtensions-storage-local"; -const IDB_DATA_STORENAME = "storage-local-data"; -const IDB_VERSION = 1; - -// Whether or not the installed extensions should be migrated to the storage.local IndexedDB backend. -const BACKEND_ENABLED_PREF = "extensions.webextensions.ExtensionStorageIDB.enabled"; - -class ExtensionStorageLocalIDB extends IndexedDB { - onupgradeneeded(event) { - if (event.oldVersion < 1) { - this.createObjectStore(IDB_DATA_STORENAME); - } - } - - static openForPrincipal(storagePrincipal) { - // The db is opened using an extension principal isolated in a reserved user context id. - return super.openForPrincipal(storagePrincipal, IDB_NAME, IDB_VERSION); - } - - async isEmpty() { - const cursor = await this.objectStore(IDB_DATA_STORENAME, "readonly").openKeyCursor(); - return cursor.done; - } - - /** - * Asynchronously sets the values of the given storage items. - * - * @param {object} items - * The storage items to set. For each property in the object, - * the storage value for that property is set to its value in - * said object. Any values which are StructuredCloneHolder - * instances are deserialized before being stored. - * @param {object} options - * @param {function} options.serialize - * Set to a function which will be used to serialize the values into - * a StructuredCloneHolder object (if appropriate) and being sent - * across the processes (it is also used to detect data cloning errors - * and raise an appropriate error to the caller). - * - * @returns {Promise} - * Return a promise which resolves to the computed "changes" object - * or null. - */ - async set(items, {serialize} = {}) { - const changes = {}; - let changed = false; - - // Explicitly create a transaction, so that we can explicitly abort it - // as soon as one of the put requests fails. - const transaction = this.transaction(IDB_DATA_STORENAME, "readwrite"); - const objectStore = transaction.objectStore(IDB_DATA_STORENAME, "readwrite"); - - for (let key of Object.keys(items)) { - try { - let oldValue = await objectStore.get(key); - - await objectStore.put(items[key], key); - - changes[key] = { - oldValue: oldValue && serialize ? serialize(oldValue) : oldValue, - newValue: serialize ? serialize(items[key]) : items[key], - }; - changed = true; - } catch (err) { - transaction.abort(); - - // Ensure that the error we throw is converted into an ExtensionError - // (e.g. DataCloneError instances raised from the internal IndexedDB - // operation have to be converted to be accessible to the extension code). - throw new ExtensionUtils.ExtensionError(String(err)); - } - } - - return changed ? changes : null; - } - - /** - * Asynchronously retrieves the values for the given storage items. - * - * @param {Array|object|null} [keysOrItems] - * The storage items to get. If an array, the value of each key - * in the array is returned. If null, the values of all items - * are returned. If an object, the value for each key in the - * object is returned, or that key's value if the item is not - * set. - * @returns {Promise} - * An object which has a property for each requested key, - * containing that key's value as stored in the IndexedDB - * storage. - */ - async get(keysOrItems) { - let keys; - let defaultValues; - - if (Array.isArray(keysOrItems)) { - keys = keysOrItems; - } else if (keysOrItems && typeof(keysOrItems) === "object") { - keys = Object.keys(keysOrItems); - defaultValues = keysOrItems; - } - - const result = {}; - - // Retrieve all the stored data using a cursor when browser.storage.local.get() - // has been called with no keys. - if (keys == null) { - const cursor = await this.objectStore(IDB_DATA_STORENAME, "readonly").openCursor(); - while (!cursor.done) { - result[cursor.key] = cursor.value; - await cursor.continue(); - } - } else { - const objectStore = this.objectStore(IDB_DATA_STORENAME); - for (let key of keys) { - const storedValue = await objectStore.get(key); - if (storedValue === undefined) { - if (defaultValues && defaultValues[key] !== undefined) { - result[key] = defaultValues[key]; - } - } else { - result[key] = storedValue; - } - } - } - - return result; - } - - /** - * Asynchronously removes the given storage items. - * - * @param {string|Array} keys - * A string key of a list of storage items keys to remove. - * @returns {Promise} - * Returns an object which contains applied changes. - */ - async remove(keys) { - // Ensure that keys is an array of strings. - keys = [].concat(keys); - - if (keys.length === 0) { - // Early exit if there is nothing to remove. - return null; - } - - const changes = {}; - let changed = false; - - const objectStore = this.objectStore(IDB_DATA_STORENAME, "readwrite"); - - for (let key of keys) { - let oldValue = await objectStore.get(key); - changes[key] = {oldValue}; - - if (oldValue) { - changed = true; - } - - await objectStore.delete(key); - } - - return changed ? changes : null; - } - - /** - * Asynchronously clears all storage entries. - * - * @returns {Promise} - * Returns an object which contains applied changes. - */ - async clear() { - const changes = {}; - let changed = false; - - const objectStore = this.objectStore(IDB_DATA_STORENAME, "readwrite"); - - const cursor = await objectStore.openCursor(); - while (!cursor.done) { - changes[cursor.key] = {oldValue: cursor.value}; - changed = true; - await cursor.continue(); - } - - await objectStore.clear(); - - return changed ? changes : null; - } -} - -/** - * Migrate the data stored in the JSONFile backend to the IDB Backend. - * - * Returns a promise which is resolved once the data migration has been - * completed and the new IDB backend can be enabled. - * Rejects if the data has been read successfully from the JSONFile backend - * but it failed to be saved in the new IDB backend. - * - * This method is called only from the main process (where the file - * can be opened). - * - * @param {Extension} extension - * The extension to migrate to the new IDB backend. - * @param {nsIPrincipal} storagePrincipal - * The "internally reserved" extension storagePrincipal to be used to create - * the ExtensionStorageLocalIDB instance. - */ -async function migrateJSONFileData(extension, storagePrincipal) { - let oldStoragePath; - let oldStorageExists; - let idbConn; - let hasEmptyIDB; - let oldDataRead = false; - let migrated = false; - - try { - idbConn = await ExtensionStorageLocalIDB.openForPrincipal(storagePrincipal); - hasEmptyIDB = await idbConn.isEmpty(); - - if (!hasEmptyIDB) { - // If the IDB backend is enabled and there is data already stored in the IDB backend, - // there is no "going back": any data that has not been migrated will be still on disk - // but it is not going to be migrated anymore, it could be eventually used to allow - // a user to manually retrieve the old data file). - return; - } - - // Migrate any data stored in the JSONFile backend (if any), and remove the old data file - // if the migration has been completed successfully. - oldStoragePath = ExtensionStorage.getStorageFile(extension.id); - oldStorageExists = await OS.File.exists(oldStoragePath); - - if (oldStorageExists) { - Services.console.logStringMessage( - `Migrating storage.local data for ${extension.policy.debugName}...`); - - const jsonFile = await ExtensionStorage.getFile(extension.id); - const data = {}; - for (let [key, value] of jsonFile.data.entries()) { - data[key] = value; - } - oldDataRead = true; - await idbConn.set(data); - migrated = true; - Services.console.logStringMessage( - `storage.local data successfully migrated to IDB Backend for ${extension.policy.debugName}.`); - } - } catch (err) { - extension.logWarning(`Error on migrating storage.local data: ${err.message}::${err.stack}`); - if (oldDataRead) { - // If the data has been read successfully and it has been failed to be stored - // into the IndexedDB backend, then clear any partially stored data and reject - // the data migration promise explicitly (which would prevent the new backend - // from being enabled for this session). - Services.qms.clearStoragesForPrincipal(storagePrincipal); - throw err; - } - } finally { - // Clear the jsonFilePromise cached by the ExtensionStorage (so that the file - // can be immediatelly removed when we call OS.File.remove). - ExtensionStorage.clearCachedFile(extension.id); - } - - // If the IDB backend has been enabled, try to remove the old storage.local data file, - // but keep using the selected backend even if it fails to be removed. - if (oldStorageExists && migrated) { - try { - await OS.File.remove(oldStoragePath); - } catch (err) { - extension.logWarning(err.message); - } - } -} - -/** - * This ExtensionStorage class implements a backend for the storage.local API which - * uses IndexedDB to store the data. - */ -this.ExtensionStorageIDB = { - BACKEND_ENABLED_PREF, - - // Map> - listeners: new Map(), - - // Keep track if the IDB backend has been selected or not for a running extension - // (the selected backend should never change while the extension is running, even if the - // related preference has been changed in the meantime): - // - // WeakMap Promise - selectedBackendPromises: new WeakMap(), - - init() { - XPCOMUtils.defineLazyPreferenceGetter(this, "isBackendEnabled", BACKEND_ENABLED_PREF, false); - }, - - getStoragePrincipal(extension) { - return extension.createPrincipal(extension.baseURI, { - userContextId: WEBEXT_STORAGE_USER_CONTEXT_ID, - }); - }, - - /** - * Select the preferred backend and return a promise which is resolved once the - * selected backend is ready to be used (e.g. if the extension is switching from - * the old JSONFile storage to the new IDB backend, any previously stored data will - * be migrated to the backend before the promise is resolved). - * - * This method is called from both the main and child (content or extension) processes: - * - an extension child context will call this method lazily, when the browser.storage.local - * is being used for the first time, and it will result into asking the main process - * to call the same method in the main process - * - on the main process side, it will check if the new IDB backend can be used (and if it can, - * it will migrate any existing data into the new backend, which needs to happen in the - * main process where the file can directly be accessed) - * - * The result will be cached while the extension is still running, and so an extension - * child context is going to ask the main process only once per child process, and on the - * main process side the backend selection and data migration will happen only once. - * - * @param {BaseContext} context - * The extension context that is selecting the storage backend. - * - * @returns {Promise} - * Returns a promise which resolves to an object which provides a - * `backendEnabled` boolean property, and if it is true the extension should use - * the IDB backend and the object also includes a `storagePrincipal` property - * of type nsIPrincipal, otherwise `backendEnabled` will be false when the - * extension should use the old JSONFile backend (e.g. because the IDB backend has - * not been enabled from the preference). - */ - selectBackend(context) { - const {extension} = context; - - if (!this.selectedBackendPromises.has(extension)) { - let promise; - - if (context.childManager) { - // Ask the parent process if the new backend is enabled for the - // running extension. - promise = context.childManager.callParentAsyncFunction( - "storage.local.IDBBackend.selectBackend", [] - ).then(result => { - if (!result.backendEnabled) { - return {backendEnabled: false}; - } - - return { - ...result, - // In the child process, we need to deserialize the storagePrincipal - // from the StructuredCloneHolder used to send it across the processes. - storagePrincipal: result.storagePrincipal.deserialize(this), - }; - }); - } else { - // If migrating to the IDB backend is not enabled by the preference, then we - // don't need to migrate any data and the new backend is not enabled. - if (!this.isBackendEnabled) { - return Promise.resolve({backendEnabled: false}); - } - - // In the main process, lazily create a storagePrincipal isolated in a - // reserved user context id (its purpose is ensuring that the IndexedDB storage used - // by the browser.storage.local API is not directly accessible from the extension code). - const storagePrincipal = this.getStoragePrincipal(extension); - - // Serialize the nsIPrincipal object into a StructuredCloneHolder related to the privileged - // js global, ready to be sent to the child processes. - const serializedPrincipal = new StructuredCloneHolder(storagePrincipal, this); - - promise = migrateJSONFileData(extension, storagePrincipal).then(() => { - return {backendEnabled: true, storagePrincipal: serializedPrincipal}; - }).catch(err => { - // If the data migration promise is rejected, the old data has been read - // successfully from the old JSONFile backend but it failed to be saved - // into the IndexedDB backend (which is likely unrelated to the kind of - // data stored and more likely a general issue with the IndexedDB backend) - // In this case we keep the JSONFile backend enabled for this session - // and we will retry to migrate to the IDB Backend the next time the - // extension is being started. - // TODO Bug 1465129: This should be a very unlikely scenario, some telemetry - // data about it may be useful. - extension.logWarning("JSONFile backend is being kept enabled by an unexpected " + - `IDBBackend failure: ${err.message}::${err.stack}`); - return {backendEnabled: false}; - }); - } - - this.selectedBackendPromises.set(extension, promise); - } - - return this.selectedBackendPromises.get(extension); - }, - - /** - * Open a connection to the IDB storage.local db for a given extension. - * given extension. - * - * @param {nsIPrincipal} storagePrincipal - * The "internally reserved" extension storagePrincipal to be used to create - * the ExtensionStorageLocalIDB instance. - * - * @returns {Promise} - * Return a promise which resolves to the opened IDB connection. - */ - open(storagePrincipal) { - return ExtensionStorageLocalIDB.openForPrincipal(storagePrincipal); - }, - - addOnChangedListener(extensionId, listener) { - let listeners = this.listeners.get(extensionId) || new Set(); - listeners.add(listener); - this.listeners.set(extensionId, listeners); - }, - - removeOnChangedListener(extensionId, listener) { - let listeners = this.listeners.get(extensionId); - listeners.delete(listener); - }, - - notifyListeners(extensionId, changes) { - let listeners = this.listeners.get(extensionId); - if (listeners) { - for (let listener of listeners) { - listener(changes); - } - } - }, - - hasListeners(extensionId) { - let listeners = this.listeners.get(extensionId); - return listeners && listeners.size > 0; - }, -}; - -ExtensionStorageIDB.init(); diff --git a/toolkit/components/extensions/child/ext-storage.js b/toolkit/components/extensions/child/ext-storage.js index ecebd2ac9152..53f4a621ce34 100644 --- a/toolkit/components/extensions/child/ext-storage.js +++ b/toolkit/components/extensions/child/ext-storage.js @@ -2,117 +2,66 @@ ChromeUtils.defineModuleGetter(this, "ExtensionStorage", "resource://gre/modules/ExtensionStorage.jsm"); -ChromeUtils.defineModuleGetter(this, "ExtensionStorageIDB", - "resource://gre/modules/ExtensionStorageIDB.jsm"); ChromeUtils.defineModuleGetter(this, "TelemetryStopwatch", "resource://gre/modules/TelemetryStopwatch.jsm"); +var { + ExtensionError, +} = ExtensionUtils; + const storageGetHistogram = "WEBEXT_STORAGE_LOCAL_GET_MS"; const storageSetHistogram = "WEBEXT_STORAGE_LOCAL_SET_MS"; -// Wrap a storage operation in a TelemetryStopWatch. -async function measureOp(histogram, fn) { - const stopwatchKey = {}; - TelemetryStopwatch.start(histogram, stopwatchKey); - try { - let result = await fn(); - TelemetryStopwatch.finish(histogram, stopwatchKey); - return result; - } catch (err) { - TelemetryStopwatch.cancel(histogram, stopwatchKey); - throw err; - } -} - this.storage = class extends ExtensionAPI { - getLocalFileBackend(context, {deserialize, serialize}) { - return { - get(keys) { - return measureOp(storageGetHistogram, () => { - return context.childManager.callParentAsyncFunction( - "storage.local.JSONFileBackend.get", - [serialize(keys)]).then(deserialize); - }); - }, - set(items) { - return measureOp(storageSetHistogram, () => { - return context.childManager.callParentAsyncFunction( - "storage.local.JSONFileBackend.set", [serialize(items)]); - }); - }, - remove(keys) { - return context.childManager.callParentAsyncFunction( - "storage.local.JSONFileBackend.remove", [serialize(keys)]); - }, - clear() { - return context.childManager.callParentAsyncFunction( - "storage.local.JSONFileBackend.clear", []); - }, - }; - } - - getLocalIDBBackend(context, {hasParentListeners, serialize, storagePrincipal}) { - const dbPromise = ExtensionStorageIDB.open(storagePrincipal); - - return { - get(keys) { - return measureOp(storageGetHistogram, async () => { - const db = await dbPromise; - return db.get(keys); - }); - }, - set(items) { - return measureOp(storageSetHistogram, async () => { - const db = await dbPromise; - const changes = await db.set(items, { - serialize: ExtensionStorage.serialize, - }); - - if (!changes) { - return; - } - - const hasListeners = await hasParentListeners(); - if (hasListeners) { - await context.childManager.callParentAsyncFunction( - "storage.local.IDBBackend.fireOnChanged", [changes]); - } - }); - }, - async remove(keys) { - const db = await dbPromise; - const changes = await db.remove(keys); - - if (!changes) { - return; - } - - const hasListeners = await hasParentListeners(); - if (hasListeners) { - await context.childManager.callParentAsyncFunction( - "storage.local.IDBBackend.fireOnChanged", [changes]); - } - }, - async clear() { - const db = await dbPromise; - const changes = await db.clear(context.extension); - - if (!changes) { - return; - } - - const hasListeners = await hasParentListeners(); - if (hasListeners) { - await context.childManager.callParentAsyncFunction( - "storage.local.IDBBackend.fireOnChanged", [changes]); - } - }, - }; - } - getAPI(context) { - const serialize = ExtensionStorage.serializeForContext.bind(null, context); - const deserialize = ExtensionStorage.deserializeForContext.bind(null, context); + /** + * Serializes the given storage items for transporting to the parent + * process. + * + * @param {Array|object} items + * The items to serialize. If an object is provided, its + * values are serialized to StructuredCloneHolder objects. + * Otherwise, it is returned as-is. + * @returns {Array|object} + */ + function serialize(items) { + if (items && typeof items === "object" && !Array.isArray(items)) { + let result = {}; + for (let [key, value] of Object.entries(items)) { + try { + result[key] = new StructuredCloneHolder(value, context.cloneScope); + } catch (e) { + throw new ExtensionError(String(e)); + } + } + return result; + } + return items; + } + + /** + * Deserializes the given storage items from the parent process into + * the extension context. + * + * @param {object} items + * The items to deserialize. Any property of the object which + * is a StructuredCloneHolder instance is deserialized into + * the extension scope. Any other object is cloned into the + * extension scope directly. + * @returns {object} + */ + function deserialize(items) { + let result = new context.cloneScope.Object(); + for (let [key, value] of Object.entries(items)) { + if (value && typeof value === "object" && Cu.getClassName(value, true) === "StructuredCloneHolder") { + value = value.deserialize(context.cloneScope); + } else { + value = Cu.cloneInto(value, context.cloneScope); + } + result[key] = value; + } + return result; + } function sanitize(items) { // The schema validator already takes care of arrays (which are only allowed @@ -134,56 +83,47 @@ this.storage = class extends ExtensionAPI { return sanitized; } - // Detect the actual storage.local enabled backend for the extension (as soon as the - // storage.local API has been accessed for the first time). - let promiseStorageLocalBackend; - const getStorageLocalBackend = async () => { - const { - backendEnabled, - storagePrincipal, - } = await ExtensionStorageIDB.selectBackend(context); - - if (!backendEnabled) { - return this.getLocalFileBackend(context, {deserialize, serialize}); - } - - return this.getLocalIDBBackend(context, { - storagePrincipal, - hasParentListeners() { - // We spare a good amount of memory if there are no listeners around - // (e.g. because they have never been subscribed or they have been removed - // in the meantime). - return context.childManager.callParentAsyncFunction( - "storage.local.IDBBackend.hasListeners", []); - }, - serialize, - }); - }; - - // Generate the backend-agnostic local API wrapped methods. - const local = {}; - for (let method of ["get", "set", "remove", "clear"]) { - local[method] = async function(...args) { - if (!promiseStorageLocalBackend) { - promiseStorageLocalBackend = getStorageLocalBackend(); - } - const backend = await promiseStorageLocalBackend; - return backend[method](...args); - }; - } - return { storage: { - local, + local: { + get: async function(keys) { + const stopwatchKey = {}; + TelemetryStopwatch.start(storageGetHistogram, stopwatchKey); + try { + let result = await context.childManager.callParentAsyncFunction("storage.local.get", [ + serialize(keys), + ]).then(deserialize); + TelemetryStopwatch.finish(storageGetHistogram, stopwatchKey); + return result; + } catch (e) { + TelemetryStopwatch.cancel(storageGetHistogram, stopwatchKey); + throw e; + } + }, + set: async function(items) { + const stopwatchKey = {}; + TelemetryStopwatch.start(storageSetHistogram, stopwatchKey); + try { + let result = await context.childManager.callParentAsyncFunction("storage.local.set", [ + serialize(items), + ]); + TelemetryStopwatch.finish(storageSetHistogram, stopwatchKey); + return result; + } catch (e) { + TelemetryStopwatch.cancel(storageSetHistogram, stopwatchKey); + throw e; + } + }, + }, sync: { - get(keys) { + get: function(keys) { keys = sanitize(keys); return context.childManager.callParentAsyncFunction("storage.sync.get", [ keys, ]); }, - set(items) { + set: function(items) { items = sanitize(items); return context.childManager.callParentAsyncFunction("storage.sync.set", [ items, diff --git a/toolkit/components/extensions/moz.build b/toolkit/components/extensions/moz.build index bdc366be7fb1..2b0c1a29d367 100644 --- a/toolkit/components/extensions/moz.build +++ b/toolkit/components/extensions/moz.build @@ -20,7 +20,6 @@ EXTRA_JS_MODULES += [ 'ExtensionPreferencesManager.jsm', 'ExtensionSettingsStore.jsm', 'ExtensionStorage.jsm', - 'ExtensionStorageIDB.jsm', 'ExtensionStorageSync.jsm', 'ExtensionUtils.jsm', 'FindContent.jsm', diff --git a/toolkit/components/extensions/parent/ext-storage.js b/toolkit/components/extensions/parent/ext-storage.js index e7162a5d64df..63baca5b4643 100644 --- a/toolkit/components/extensions/parent/ext-storage.js +++ b/toolkit/components/extensions/parent/ext-storage.js @@ -3,7 +3,6 @@ XPCOMUtils.defineLazyModuleGetters(this, { AddonManagerPrivate: "resource://gre/modules/AddonManager.jsm", ExtensionStorage: "resource://gre/modules/ExtensionStorage.jsm", - ExtensionStorageIDB: "resource://gre/modules/ExtensionStorageIDB.jsm", extensionStorageSync: "resource://gre/modules/ExtensionStorageSync.jsm", NativeManifests: "resource://gre/modules/NativeManifests.jsm", }); @@ -36,58 +35,37 @@ const lookupManagedStorage = async (extensionId, context) => { this.storage = class extends ExtensionAPI { getAPI(context) { let {extension} = context; - return { storage: { local: { - // Private storage.local JSONFile backend methods (used internally by the child - // ext-storage.js module). - JSONFileBackend: { - get(spec) { - return ExtensionStorage.get(extension.id, spec); - }, - set(items) { - return ExtensionStorage.set(extension.id, items); - }, - remove(keys) { - return ExtensionStorage.remove(extension.id, keys); - }, - clear() { - return ExtensionStorage.clear(extension.id); - }, + get: function(spec) { + return ExtensionStorage.get(extension.id, spec); }, - // Private storage.local IDB backend methods (used internally by the child ext-storage.js - // module). - IDBBackend: { - selectBackend() { - return ExtensionStorageIDB.selectBackend(context); - }, - hasListeners() { - return ExtensionStorageIDB.hasListeners(extension.id); - }, - fireOnChanged(changes) { - ExtensionStorageIDB.notifyListeners(extension.id, changes); - }, - onceDataMigrated() { - return ExtensionStorageIDB.onceDataMigrated(context); - }, + set: function(items) { + return ExtensionStorage.set(extension.id, items); + }, + remove: function(keys) { + return ExtensionStorage.remove(extension.id, keys); + }, + clear: function() { + return ExtensionStorage.clear(extension.id); }, }, sync: { - get(spec) { + get: function(spec) { enforceNoTemporaryAddon(extension.id); return extensionStorageSync.get(extension, spec, context); }, - set(items) { + set: function(items) { enforceNoTemporaryAddon(extension.id); return extensionStorageSync.set(extension, items, context); }, - remove(keys) { + remove: function(keys) { enforceNoTemporaryAddon(extension.id); return extensionStorageSync.remove(extension, keys, context); }, - clear() { + clear: function() { enforceNoTemporaryAddon(extension.id); return extensionStorageSync.clear(extension, context); }, @@ -123,11 +101,9 @@ this.storage = class extends ExtensionAPI { }; ExtensionStorage.addOnChangedListener(extension.id, listenerLocal); - ExtensionStorageIDB.addOnChangedListener(extension.id, listenerLocal); extensionStorageSync.addOnChangedListener(extension, listenerSync, context); return () => { ExtensionStorage.removeOnChangedListener(extension.id, listenerLocal); - ExtensionStorageIDB.removeOnChangedListener(extension.id, listenerLocal); extensionStorageSync.removeOnChangedListener(extension, listenerSync); }; }, diff --git a/toolkit/components/extensions/test/mochitest/test_ext_storage_cleanup.html b/toolkit/components/extensions/test/mochitest/test_ext_storage_cleanup.html index 69a40a9a21f1..6dcf345ff690 100644 --- a/toolkit/components/extensions/test/mochitest/test_ext_storage_cleanup.html +++ b/toolkit/components/extensions/test/mochitest/test_ext_storage_cleanup.html @@ -13,116 +13,94 @@ diff --git a/toolkit/components/extensions/test/xpcshell/head.js b/toolkit/components/extensions/test/xpcshell/head.js index b69ba982a2ff..9452ea7c2ab0 100644 --- a/toolkit/components/extensions/test/xpcshell/head.js +++ b/toolkit/components/extensions/test/xpcshell/head.js @@ -1,7 +1,6 @@ "use strict"; -/* exported createHttpServer, promiseConsoleOutput, cleanupDir, clearCache, testEnv - runWithPrefs */ +/* exported createHttpServer, promiseConsoleOutput, cleanupDir, clearCache, testEnv */ ChromeUtils.import("resource://gre/modules/AppConstants.jsm"); ChromeUtils.import("resource://gre/modules/Services.jsm"); @@ -111,32 +110,3 @@ function cleanupDir(dir) { tryToRemoveDir(); }); } - -// Run a test with the specified preferences and then clear them -// right after the test function run (whether it passes or fails). -async function runWithPrefs(prefsToSet, testFn) { - try { - for (let [pref, value] of prefsToSet) { - info(`Setting pref "${pref}": ${value}`); - switch (typeof(value)) { - case "boolean": - Services.prefs.setBoolPref(pref, value); - break; - case "number": - Services.prefs.setIntPref(pref, value); - break; - case "string": - Services.prefs.setStringPref(pref, value); - break; - default: - throw new Error("runWithPrefs doesn't support this pref type yet"); - } - } - await testFn(); - } finally { - for (let [prefName] of prefsToSet) { - info(`Clearing pref "${prefName}"`); - Services.prefs.clearUserPref(prefName); - } - } -} diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage.js index f922d17c8b9f..33bda79c7ee5 100644 --- a/toolkit/components/extensions/test/xpcshell/test_ext_storage.js +++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage.js @@ -2,10 +2,8 @@ /* vim: set sts=2 sw=2 et tw=80: */ "use strict"; -ChromeUtils.defineModuleGetter(this, "ExtensionStorageIDB", - "resource://gre/modules/ExtensionStorageIDB.jsm"); - const STORAGE_SYNC_PREF = "webextensions.storage.sync.enabled"; +ChromeUtils.import("resource://gre/modules/Preferences.jsm"); add_task(async function setup() { await ExtensionTestUtils.startAddonManager(); @@ -106,84 +104,81 @@ add_task(async function test_single_initialization() { } }); -add_task(function test_config_flag_needed() { - async function testFn() { - function background() { - let promises = []; - let apiTests = [ - {method: "get", args: ["foo"]}, - {method: "set", args: [{foo: "bar"}]}, - {method: "remove", args: ["foo"]}, - {method: "clear", args: []}, - ]; - apiTests.forEach(testDef => { - promises.push(browser.test.assertRejects( - browser.storage.sync[testDef.method](...testDef.args), - "Please set webextensions.storage.sync.enabled to true in about:config", - `storage.sync.${testDef.method} is behind a flag`)); - }); +add_task(async function test_config_flag_needed() { + function background() { + let promises = []; + let apiTests = [ + {method: "get", args: ["foo"]}, + {method: "set", args: [{foo: "bar"}]}, + {method: "remove", args: ["foo"]}, + {method: "clear", args: []}, + ]; + apiTests.forEach(testDef => { + promises.push(browser.test.assertRejects( + browser.storage.sync[testDef.method](...testDef.args), + "Please set webextensions.storage.sync.enabled to true in about:config", + `storage.sync.${testDef.method} is behind a flag`)); + }); - Promise.all(promises).then(() => browser.test.notifyPass("flag needed")); + Promise.all(promises).then(() => browser.test.notifyPass("flag needed")); + } + + Preferences.set(STORAGE_SYNC_PREF, false); + ok(!Preferences.get(STORAGE_SYNC_PREF)); + let extension = ExtensionTestUtils.loadExtension({ + manifest: { + permissions: ["storage"], + }, + background: `(${background})(${checkGetImpl})`, + }); + + await extension.startup(); + await extension.awaitFinish("flag needed"); + await extension.unload(); + Preferences.reset(STORAGE_SYNC_PREF); +}); + +add_task(async function test_reloading_extensions_works() { + // Just some random extension ID that we can re-use + const extensionId = "my-extension-id@1"; + + function loadExtension() { + function background() { + browser.storage.sync.set({"a": "b"}).then(() => { + browser.test.notifyPass("set-works"); + }); } - ok(!Services.prefs.getBoolPref(STORAGE_SYNC_PREF, false), - "The `${STORAGE_SYNC_PREF}` should be set to false"); - - let extension = ExtensionTestUtils.loadExtension({ + return ExtensionTestUtils.loadExtension({ manifest: { permissions: ["storage"], }, - background: `(${background})(${checkGetImpl})`, - }); - - await extension.startup(); - await extension.awaitFinish("flag needed"); - await extension.unload(); + background: `(${background})()`, + }, extensionId); } - return runWithPrefs([[STORAGE_SYNC_PREF, false]], testFn); + Preferences.set(STORAGE_SYNC_PREF, true); + + let extension1 = loadExtension(); + + await extension1.startup(); + await extension1.awaitFinish("set-works"); + await extension1.unload(); + + let extension2 = loadExtension(); + + await extension2.startup(); + await extension2.awaitFinish("set-works"); + await extension2.unload(); + + Preferences.reset(STORAGE_SYNC_PREF); }); -add_task(function test_reloading_extensions_works() { - async function testFn() { - // Just some random extension ID that we can re-use - const extensionId = "my-extension-id@1"; - - function loadExtension() { - function background() { - browser.storage.sync.set({"a": "b"}).then(() => { - browser.test.notifyPass("set-works"); - }); - } - - return ExtensionTestUtils.loadExtension({ - manifest: { - permissions: ["storage"], - }, - background: `(${background})()`, - }, extensionId); - } - - ok(Services.prefs.getBoolPref(STORAGE_SYNC_PREF, false), - "The `${STORAGE_SYNC_PREF}` should be set to true"); - - let extension1 = loadExtension(); - - await extension1.startup(); - await extension1.awaitFinish("set-works"); - await extension1.unload(); - - let extension2 = loadExtension(); - - await extension2.startup(); - await extension2.awaitFinish("set-works"); - await extension2.unload(); - } - - return runWithPrefs([[STORAGE_SYNC_PREF, true]], testFn); +registerCleanupFunction(() => { + Preferences.reset(STORAGE_SYNC_PREF); }); -async function test_background_page_storage(testAreaName) { +add_task(async function test_backgroundScript() { async function backgroundScript(checkGet) { let globalChanges, gResolve; function clearGlobalChanges() { @@ -373,59 +368,49 @@ async function test_background_page_storage(testAreaName) { }, }; + Preferences.set(STORAGE_SYNC_PREF, true); + let extension = ExtensionTestUtils.loadExtension(extensionData); await extension.startup(); await extension.awaitMessage("ready"); - extension.sendMessage(`test-${testAreaName}`); + extension.sendMessage("test-local"); await extension.awaitMessage("test-finished"); + extension.sendMessage("test-sync"); + await extension.awaitMessage("test-finished"); + + Preferences.reset(STORAGE_SYNC_PREF); await extension.unload(); -} - -add_task(function test_storage_local_file_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, false]], - () => test_background_page_storage("local")); }); -add_task(function test_storage_local_idb_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]], - () => test_background_page_storage("local")); -}); +add_task(async function test_storage_requires_real_id() { + async function backgroundScript() { + const EXCEPTION_MESSAGE = + "The storage API is not available with a temporary addon ID. " + + "Please add an explicit addon ID to your manifest. " + + "For more information see https://bugzil.la/1323228."; -add_task(function test_storage_sync() { - return runWithPrefs([[STORAGE_SYNC_PREF, true]], - () => test_background_page_storage("sync")); -}); + await browser.test.assertRejects(browser.storage.sync.set({"foo": "bar"}), + EXCEPTION_MESSAGE); -add_task(function test_storage_sync_requires_real_id() { - async function testFn() { - async function backgroundScript() { - const EXCEPTION_MESSAGE = - "The storage API is not available with a temporary addon ID. " + - "Please add an explicit addon ID to your manifest. " + - "For more information see https://bugzil.la/1323228."; - - await browser.test.assertRejects(browser.storage.sync.set({"foo": "bar"}), - EXCEPTION_MESSAGE); - - browser.test.notifyPass("exception correct"); - } - - let extensionData = { - background: `(${backgroundScript})(${checkGetImpl})`, - manifest: { - permissions: ["storage"], - }, - useAddonManager: "temporary", - }; - - let extension = ExtensionTestUtils.loadExtension(extensionData); - await extension.startup(); - await extension.awaitFinish("exception correct"); - - await extension.unload(); + browser.test.notifyPass("exception correct"); } - return runWithPrefs([[STORAGE_SYNC_PREF, true]], testFn); + let extensionData = { + background: `(${backgroundScript})(${checkGetImpl})`, + manifest: { + permissions: ["storage"], + }, + useAddonManager: "temporary", + }; + + Preferences.set(STORAGE_SYNC_PREF, true); + + let extension = ExtensionTestUtils.loadExtension(extensionData); + await extension.startup(); + await extension.awaitFinish("exception correct"); + + Preferences.reset(STORAGE_SYNC_PREF); + await extension.unload(); }); diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage_content.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage_content.js index 45910c7ad8b2..56565cefeed3 100644 --- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_content.js +++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_content.js @@ -1,7 +1,5 @@ "use strict"; -ChromeUtils.import("resource://gre/modules/ExtensionStorageIDB.jsm"); - PromiseTestUtils.whitelistRejectionsGlobally(/WebExtension context not found/); const server = createHttpServer({hosts: ["example.com"]}); @@ -237,7 +235,11 @@ let extensionData = { }, }; -async function test_contentscript_storage(storageType) { +add_task(async function test_contentscript() { + await ExtensionTestUtils.startAddonManager(); + Services.prefs.setBoolPref(STORAGE_SYNC_PREF, true); + + let contentPage = await ExtensionTestUtils.loadContentPage( "http://example.com/data/file_sample.html"); @@ -245,28 +247,12 @@ async function test_contentscript_storage(storageType) { await extension.startup(); await extension.awaitMessage("ready"); - extension.sendMessage(`test-${storageType}`); + extension.sendMessage("test-local"); + await extension.awaitMessage("test-finished"); + + extension.sendMessage("test-sync"); await extension.awaitMessage("test-finished"); await extension.unload(); await contentPage.close(); -} - -add_task(async function setup() { - await ExtensionTestUtils.startAddonManager(); -}); - -add_task(async function test_contentscript_storage_sync() { - return runWithPrefs([[STORAGE_SYNC_PREF, true]], - () => test_contentscript_storage("sync")); -}); - -add_task(async function test_contentscript_storage_local_file_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, false]], - () => test_contentscript_storage("local")); -}); - -add_task(async function test_contentscript_storage_local_idb_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]], - () => test_contentscript_storage("local")); }); diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js deleted file mode 100644 index 87fa8e9b43f0..000000000000 --- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_idb_data_migration.js +++ /dev/null @@ -1,151 +0,0 @@ -/* -*- Mode: indent-tabs-mode: nil; js-indent-level: 2 -*- */ -/* vim: set sts=2 sw=2 et tw=80: */ -"use strict"; - -// This test file verifies various scenarios related to the data migration -// from the JSONFile backend to the IDB backend. - -ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm"); - -XPCOMUtils.defineLazyModuleGetters(this, { - ExtensionStorage: "resource://gre/modules/ExtensionStorage.jsm", - ExtensionStorageIDB: "resource://gre/modules/ExtensionStorageIDB.jsm", - OS: "resource://gre/modules/osfile.jsm", -}); - -async function createExtensionJSONFileWithData(extensionId, data) { - await ExtensionStorage.set(extensionId, data); - const jsonFile = await ExtensionStorage.getFile(extensionId); - await jsonFile._save(); - const oldStorageFilename = ExtensionStorage.getStorageFile(extensionId); - equal(await OS.File.exists(oldStorageFilename), true, "The old json file has been created"); - - return {jsonFile, oldStorageFilename}; -} - -add_task(async function setup() { - Services.prefs.setBoolPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF, true); -}); - -// Test that the old data is migrated successfully to the new storage backend -// and that the original JSONFile is being removed. -add_task(async function test_storage_local_data_migration() { - const EXTENSION_ID = "extension-to-be-migrated@mozilla.org"; - - const data = { - "test_key_string": "test_value1", - "test_key_number": 1000, - "test_nested_data": { - "nested_key": true, - }, - }; - - // Store some fake data in the storage.local file backend before starting the extension. - const {oldStorageFilename} = await createExtensionJSONFileWithData(EXTENSION_ID, data); - - async function background() { - const storedData = await browser.storage.local.get(); - - browser.test.assertEq("test_value1", storedData.test_key_string, - "Got the expected data after the storage.local data migration"); - browser.test.assertEq(1000, storedData.test_key_number, - "Got the expected data after the storage.local data migration"); - browser.test.assertEq(true, storedData.test_nested_data.nested_key, - "Got the expected data after the storage.local data migration"); - - browser.test.sendMessage("storage-local-data-migrated"); - } - - let extension = ExtensionTestUtils.loadExtension({ - manifest: { - permissions: ["storage"], - applications: { - gecko: { - id: EXTENSION_ID, - }, - }, - }, - background, - }); - - await extension.startup(); - - await extension.awaitMessage("storage-local-data-migrated"); - - const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal(extension.extension); - - const idbConn = await ExtensionStorageIDB.open(storagePrincipal); - - equal(await idbConn.isEmpty(extension.extension), false, - "Data stored in the ExtensionStorageIDB backend as expected"); - - equal(await OS.File.exists(oldStorageFilename), false, - "The old json storage file should have been removed"); - - await extension.unload(); -}); - -// Test that if the old JSONFile data file is corrupted and the old data -// can't be successfully migrated to the new storage backend, then: -// - the new storage backend for that extension is still initialized and enabled -// - any new data is being stored in the new backend -// - the old file is being renamed (with the `.corrupted` suffix that JSONFile.jsm -// adds when it fails to load the data file) and still available on disk. -add_task(async function test_storage_local_corrupted_data_migration() { - const EXTENSION_ID = "extension-corrupted-data-migration@mozilla.org"; - - const invalidData = `{"test_key_string": "test_value1"`; - const oldStorageFilename = ExtensionStorage.getStorageFile(EXTENSION_ID); - - const profileDir = OS.Constants.Path.profileDir; - await OS.File.makeDir(OS.Path.join(profileDir, "browser-extension-data", EXTENSION_ID), - {from: profileDir, ignoreExisting: true}); - - // Write the json file with some invalid data. - await OS.File.writeAtomic(oldStorageFilename, invalidData, {flush: true}); - equal(await OS.File.read(oldStorageFilename, {encoding: "utf-8"}), - invalidData, "The old json file has been overwritten with invalid data"); - - async function background() { - const storedData = await browser.storage.local.get(); - - browser.test.assertEq(Object.keys(storedData).length, 0, - "No data should be found found on invalid data migration"); - - await browser.storage.local.set({"test_key_string_on_IDBBackend": "expected-value"}); - - browser.test.sendMessage("storage-local-data-migrated-and-set"); - } - - let extension = ExtensionTestUtils.loadExtension({ - manifest: { - permissions: ["storage"], - applications: { - gecko: { - id: EXTENSION_ID, - }, - }, - }, - background, - }); - - await extension.startup(); - - await extension.awaitMessage("storage-local-data-migrated-and-set"); - - const storagePrincipal = ExtensionStorageIDB.getStoragePrincipal(extension.extension); - - const idbConn = await ExtensionStorageIDB.open(storagePrincipal); - - equal(await idbConn.isEmpty(extension.extension), false, - "Data stored in the ExtensionStorageIDB backend as expected"); - - equal(await OS.File.exists(`${oldStorageFilename}.corrupt`), true, - "The old json storage should still be available if failed to be read"); - - await extension.unload(); -}); - -add_task(function test_storage_local_data_migration_clear_pref() { - Services.prefs.clearUserPref(ExtensionStorageIDB.BACKEND_ENABLED_PREF); -}); diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js index 25c8ab8e2396..a9f46f15c100 100644 --- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js +++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_tab.js @@ -1,8 +1,6 @@ "use strict"; -ChromeUtils.import("resource://gre/modules/ExtensionStorageIDB.jsm"); - -async function test_multiple_pages() { +add_task(async function test_multiple_pages() { let extension = ExtensionTestUtils.loadExtension({ async background() { function awaitMessage(expectedMsg, api = "test") { @@ -89,14 +87,4 @@ async function test_multiple_pages() { await extension.startup(); await extension.awaitFinish("storage-multiple"); await extension.unload(); -} - -add_task(async function test_storage_local_file_backend_from_tab() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, false]], - test_multiple_pages); -}); - -add_task(async function test_storage_local_idb_backend_from_tab() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]], - test_multiple_pages); }); diff --git a/toolkit/components/extensions/test/xpcshell/test_ext_storage_telemetry.js b/toolkit/components/extensions/test/xpcshell/test_ext_storage_telemetry.js index adf808c5946a..cce0499c4378 100644 --- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_telemetry.js +++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_telemetry.js @@ -2,13 +2,11 @@ /* vim: set sts=2 sw=2 et tw=80: */ "use strict"; -ChromeUtils.import("resource://gre/modules/ExtensionStorageIDB.jsm"); - const HISTOGRAM_IDS = [ "WEBEXT_STORAGE_LOCAL_SET_MS", "WEBEXT_STORAGE_LOCAL_GET_MS", ]; -async function test_telemetry_background() { +add_task(async function test_telemetry_background() { const server = createHttpServer(); server.registerDirectory("/data/", do_get_file("data")); @@ -103,14 +101,4 @@ async function test_telemetry_background() { } await extension1.unload(); -} - -add_task(function test_telemetry_background_file_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, false]], - test_telemetry_background); -}); - -add_task(function test_telemetry_background_idb_backend() { - return runWithPrefs([[ExtensionStorageIDB.BACKEND_ENABLED_PREF, true]], - test_telemetry_background); }); diff --git a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini index bccbc89c9b95..0a016ce3c5e3 100644 --- a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini +++ b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini @@ -95,7 +95,6 @@ skip-if = true # bug 1315829 skip-if = os == "android" [test_ext_startup_perf.js] [test_ext_storage.js] -[test_ext_storage_idb_data_migration.js] [test_ext_storage_content.js] [test_ext_storage_managed.js] skip-if = os == "android" diff --git a/toolkit/modules/IndexedDB.jsm b/toolkit/modules/IndexedDB.jsm index 9114de6651b6..215b8bc63055 100644 --- a/toolkit/modules/IndexedDB.jsm +++ b/toolkit/modules/IndexedDB.jsm @@ -123,52 +123,20 @@ function forwardMethods(cls, target, methods) { } class Cursor { - constructor(cursorRequest, source) { - this.cursorRequest = cursorRequest; + constructor(cursor, source) { + this.cursor = cursor; this.source = source; - this.cursor = null; - } - - get done() { - return !this.cursor; - } - - // This method is used internally to wait the cursor's IDBRequest to have been - // completed and the internal cursor has been updated (used when we initially - // create the cursor from Cursed.openCursor/openKeyCursor, and in the method - // of this class defined by defineCursorUpdateMethods). - async awaitRequest() { - this.cursor = await wrapRequest(this.cursorRequest); - return this; } } -/** - * Define the Cursor class methods that update the cursor (continue, continuePrimaryKey - * and advance) as async functions that call the related IDBCursor methods and - * await the cursor's IDBRequest to be completed. - * - * @param {function} cls - * The class constructor for which to define the cursor update methods. - * @param {Array} methods - * A list of "cursor update" method names to define. - */ -function defineCursorUpdateMethods(cls, methods) { - for (let method of methods) { - cls.prototype[method] = async function(...args) { - const promise = this.awaitRequest(); - this.cursor[method](...args); - await promise; - }; - } -} - -defineCursorUpdateMethods(Cursor, ["advance", "continue", "continuePrimaryKey"]); - forwardGetters(Cursor, "cursor", ["direction", "key", "primaryKey"]); + wrapMethods(Cursor, "cursor", ["delete", "update"]); +forwardMethods(Cursor, "cursor", + ["advance", "continue", "continuePrimaryKey"]); + class CursorWithValue extends Cursor {} forwardGetters(CursorWithValue, "cursor", ["value"]); @@ -179,13 +147,15 @@ class Cursed { } openCursor(...args) { - const cursor = new CursorWithValue(this.cursed.openCursor(...args), this); - return cursor.awaitRequest(); + return wrapRequest(this.cursed.openCursor(...args)).then(cursor => { + return new CursorWithValue(cursor, this); + }); } openKeyCursor(...args) { - const cursor = new Cursor(this.cursed.openKeyCursor(...args), this); - return cursor.awaitRequest(); + return wrapRequest(this.cursed.openKeyCursor(...args)).then(cursor => { + return new Cursor(cursor, this); + }); } } @@ -282,37 +252,7 @@ class IndexedDB { */ static open(dbName, options, onupgradeneeded = null) { let request = indexedDB.open(dbName, options); - return this._wrapOpenRequest(request, onupgradeneeded); - } - /** - * Opens the database for a given principal and with the given name, returns - * a Promise which resolves to an IndexedDB instance when the operation completes. - * - * @param {nsIPrincipal} principal - * The principal to open the database for. - * @param {string} dbName - * The name of the database to open. - * @param {object} options - * The options with which to open the database. - * @param {integer} options.version - * The schema version with which the database needs to be opened. If - * the database does not exist, or its current schema version does - * not match, the `onupgradeneeded` function will be called. - * @param {function} [onupgradeneeded] - * A function which will be called with an IndexedDB object as its - * first parameter when the database needs to be created, or its - * schema needs to be upgraded. If this function is not provided, the - * {@link #onupgradeneeded} method will be called instead. - * - * @returns {Promise} - */ - static openForPrincipal(principal, dbName, options, onupgradeneeded = null) { - const request = indexedDB.openForPrincipal(principal, dbName, options); - return this._wrapOpenRequest(request, onupgradeneeded); - } - - static _wrapOpenRequest(request, onupgradeneeded = null) { request.onupgradeneeded = event => { let db = new this(request.result); if (onupgradeneeded) { @@ -322,7 +262,7 @@ class IndexedDB { } }; - return wrapRequest(request).then(db => new this(db)); + return wrapRequest(request).then(db => new IndexedDB(db)); } constructor(db) {