Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-11-30 00:01:50 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Bug 1210296 part 1 - Remove most event loop spinning from Sync. r=kitcambridge,markh,tcsc

Browse Source 
MozReview-Commit-ID: 9RpgSgXykWt

--HG--
extra : rebase_source : 834df5b9b38a8332885a6c488f64215b550cad33
This commit is contained in:
Edouard Oger 2017-06-05 18:50:07 -04:00
parent 63938a224e
commit 0943c36687
26 changed files with 928 additions and 1025 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
browser/base/content/browser-sync.js
View File

@ -283,7 +283,9 @@ var gSync = {
},
sendTabToDevice(url, clientId, title) {
Weave.Service.clientsEngine.sendURIToClientForDisplay(url, clientId, title);
Weave.Service.clientsEngine.sendURIToClientForDisplay(url, clientId, title).catch(e => {
console.error("Could not send tab to device", e);
});
},
populateSendTabToDevicesMenu(devicesPopup, url, title, createDeviceNodeFn) {
@ -441,7 +443,6 @@ var gSync = {
if (state.status == UIState.STATUS_SIGNED_IN) {
setTimeout(() => Weave.Service.errorHandler.syncAndReportErrors(), 0);
}
Services.obs.notifyObservers(null, "cloudsync:user-sync");
},
openPrefs(entryPoint = "syncbutton", origin = undefined) {

30
services/common/async.js
View File

@ -117,7 +117,7 @@ this.Async = {
// Watch for app-quit notification to stop any sync calls
Services.obs.addObserver(function onQuitApplication() {
Services.obs.removeObserver(onQuitApplication, "quit-application");
Async.checkAppReady = function() {
Async.checkAppReady = Async.promiseYield = function() {
let exception = Components.Exception("App. Quitting", Cr.NS_ERROR_ABORT);
exception.appIsShuttingDown = true;
throw exception;
@ -231,4 +231,32 @@ this.Async = {
});
return cb.wait();
},
/**
* A "tight loop" of promises can still lock up the browser for some time.
* Periodically waiting for a promise returned by this function will solve
* that.
* You should probably not use this method directly and instead use jankYielder
* below.
* Some reference here:
* - https://gist.github.com/jesstelford/bbb30b983bddaa6e5fef2eb867d37678
* - https://bugzilla.mozilla.org/show_bug.cgi?id=1094248
*/
promiseYield() {
return new Promise(resolve => {
Services.tm.currentThread.dispatch(resolve, Ci.nsIThread.DISPATCH_NORMAL);
});
},
// Returns a method that yields every X calls.
// Common case is calling the returned method every iteration in a loop.
jankYielder(yieldEvery = 50) {
let iterations = 0;
return async () => {
Async.checkAppReady(); // Let it throw!
if (++iterations % yieldEvery === 0) {
await Async.promiseYield();
}
}
}
};

28
services/sync/modules/SyncedTabs.jsm
View File

@ -141,14 +141,14 @@ let SyncedTabsInternal = {
return result;
},
syncTabs(force) {
async syncTabs(force) {
if (!force) {
// Don't bother refetching tabs if we already did so recently
let lastFetch = Preferences.get("services.sync.lastTabFetch", 0);
let now = Math.floor(Date.now() / 1000);
if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL) {
log.info("_refetchTabs was done recently, do not doing it again");
return Promise.resolve(false);
return false;
}
}
@ -156,23 +156,17 @@ let SyncedTabsInternal = {
// of a login failure.
if (Weave.Status.checkSetup() == Weave.CLIENT_NOT_CONFIGURED) {
log.info("Sync client is not configured, so not attempting a tab sync");
return Promise.resolve(false);
return false;
}
// Ask Sync to just do the tabs engine if it can.
// Sync is currently synchronous, so do it after an event-loop spin to help
// keep the UI responsive.
return new Promise((resolve, reject) => {
Services.tm.dispatchToMainThread(() => {
try {
log.info("Doing a tab sync.");
Weave.Service.sync(["tabs"]);
resolve(true);
} catch (ex) {
log.error("Sync failed", ex);
reject(ex);
}
});
});
try {
log.info("Doing a tab sync.");
await Weave.Service.sync(["tabs"]);
return true;
} catch (ex) {
log.error("Sync failed", ex);
throw ex;
}
},
observe(subject, topic, data) {

220
services/sync/modules/addonsreconciler.js
View File

@ -58,14 +58,14 @@ this.EXPORTED_SYMBOLS = ["AddonsReconciler", "CHANGE_INSTALLED",
* The usage pattern for instances of this class is:
*
* let reconciler = new AddonsReconciler();
* reconciler.loadState(null, function(error) { ... });
* await reconciler.ensureStateLoaded();
*
* // At this point, your instance should be ready to use.
*
* When you are finished with the instance, please call:
*
* reconciler.stopListening();
* reconciler.saveState(...);
* await reconciler.saveState(...);
*
* There are 2 classes of listeners in the AddonManager: AddonListener and
* InstallListener. This class is a listener for both (member functions just
@ -124,13 +124,6 @@ AddonsReconciler.prototype = {
/** Flag indicating whether we are listening to AddonManager events. */
_listening: false,
/**
* Whether state has been loaded from a file.
*
* State is loaded on demand if an operation requires it.
*/
_stateLoaded: false,
/**
* Define this as false if the reconciler should not persist state
* to disk when handling events.
@ -171,10 +164,16 @@ AddonsReconciler.prototype = {
* Returns an object mapping add-on IDs to objects containing metadata.
*/
get addons() {
this._ensureStateLoaded();
return this._addons;
},
async ensureStateLoaded() {
if (!this._promiseStateLoaded) {
this._promiseStateLoaded = this.loadState();
}
return this._promiseStateLoaded;
},
/**
* Load reconciler state from a file.
*
@ -184,68 +183,48 @@ AddonsReconciler.prototype = {
* If the file does not exist or there was an error parsing the file, the
* state will be transparently defined as empty.
*
* @param path
* @param file
* Path to load. ".json" is appended automatically. If not defined,
* a default path will be consulted.
* @param callback
* Callback to be executed upon file load. The callback receives a
* truthy error argument signifying whether an error occurred and a
* boolean indicating whether data was loaded.
*/
loadState: function loadState(path, callback) {
let file = path || DEFAULT_STATE_FILE;
Utils.jsonLoad(file, this, function(json) {
this._addons = {};
this._changes = [];
async loadState(file = DEFAULT_STATE_FILE) {
let json = await Utils.jsonLoad(file, this);
this._addons = {};
this._changes = [];
if (!json) {
this._log.debug("No data seen in loaded file: " + file);
if (callback) {
callback(null, false);
}
if (!json) {
this._log.debug("No data seen in loaded file: " + file);
return false;
}
return;
}
let version = json.version;
if (!version || version != 1) {
this._log.error("Could not load JSON file because version not " +
"supported: " + version);
return false;
}
let version = json.version;
if (!version || version != 1) {
this._log.error("Could not load JSON file because version not " +
"supported: " + version);
if (callback) {
callback(null, false);
}
this._addons = json.addons;
for (let id in this._addons) {
let record = this._addons[id];
record.modified = new Date(record.modified);
}
return;
}
for (let [time, change, id] of json.changes) {
this._changes.push([new Date(time), change, id]);
}
this._addons = json.addons;
for (let id in this._addons) {
let record = this._addons[id];
record.modified = new Date(record.modified);
}
for (let [time, change, id] of json.changes) {
this._changes.push([new Date(time), change, id]);
}
if (callback) {
callback(null, true);
}
});
return true;
},
/**
* Saves the current state to a file in the local profile.
*
* @param path
* @param file
* String path in profile to save to. If not defined, the default
* will be used.
* @param callback
* Function to be invoked on save completion. No parameters will be
* passed to callback.
*/
saveState: function saveState(path, callback) {
let file = path || DEFAULT_STATE_FILE;
async saveState(file = DEFAULT_STATE_FILE) {
let state = {version: 1, addons: {}, changes: []};
for (let [id, record] of Object.entries(this._addons)) {
@ -264,7 +243,7 @@ AddonsReconciler.prototype = {
}
this._log.info("Saving reconciler state to file: " + file);
Utils.jsonSave(file, this, state, callback);
await Utils.jsonSave(file, this, state);
},
/**
@ -341,66 +320,60 @@ AddonsReconciler.prototype = {
/**
* Refreshes the global state of add-ons by querying the AddonManager.
*/
refreshGlobalState: function refreshGlobalState(callback) {
async refreshGlobalState() {
this._log.info("Refreshing global state from AddonManager.");
this._ensureStateLoaded();
let installs;
let addons = await AddonManager.getAllAddons();
AddonManager.getAllAddons(addons => {
let ids = {};
let ids = {};
for (let addon of addons) {
ids[addon.id] = true;
this.rectifyStateFromAddon(addon);
for (let addon of addons) {
ids[addon.id] = true;
this.rectifyStateFromAddon(addon);
}
// Look for locally-defined add-ons that no longer exist and update their
// record.
for (let [id, addon] of Object.entries(this._addons)) {
if (id in ids) {
continue;
}
// Look for locally-defined add-ons that no longer exist and update their
// record.
for (let [id, addon] of Object.entries(this._addons)) {
if (id in ids) {
continue;
}
// If the id isn't in ids, it means that the add-on has been deleted or
// the add-on is in the process of being installed. We detect the
// latter by seeing if an AddonInstall is found for this add-on.
// If the id isn't in ids, it means that the add-on has been deleted or
// the add-on is in the process of being installed. We detect the
// latter by seeing if an AddonInstall is found for this add-on.
if (!installs) {
installs = await AddonManager.getAllInstalls();
}
if (!installs) {
let cb = Async.makeSyncCallback();
AddonManager.getAllInstalls(cb);
installs = Async.waitForSyncCallback(cb);
}
let installFound = false;
for (let install of installs) {
if (install.addon && install.addon.id == id &&
install.state == AddonManager.STATE_INSTALLED) {
let installFound = false;
for (let install of installs) {
if (install.addon && install.addon.id == id &&
install.state == AddonManager.STATE_INSTALLED) {
installFound = true;
break;
}
}
if (installFound) {
continue;
}
if (addon.installed) {
addon.installed = false;
this._log.debug("Adding change because add-on not present in " +
"Add-on Manager: " + id);
this._addChange(new Date(), CHANGE_UNINSTALLED, addon);
installFound = true;
break;
}
}
// See note for _shouldPersist.
if (this._shouldPersist) {
this.saveState(null, callback);
} else {
callback();
if (installFound) {
continue;
}
});
if (addon.installed) {
addon.installed = false;
this._log.debug("Adding change because add-on not present in " +
"Add-on Manager: " + id);
this._addChange(new Date(), CHANGE_UNINSTALLED, addon);
}
}
// See note for _shouldPersist.
if (this._shouldPersist) {
await this.saveState();
}
},
/**
@ -415,9 +388,8 @@ AddonsReconciler.prototype = {
* @param addon
* Addon instance being updated.
*/
rectifyStateFromAddon: function rectifyStateFromAddon(addon) {
rectifyStateFromAddon(addon) {
this._log.debug(`Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`);
this._ensureStateLoaded();
let id = addon.id;
let enabled = !addon.userDisabled;
@ -504,9 +476,7 @@ AddonsReconciler.prototype = {
* change_type - One of CHANGE_* constants.
* id - ID of add-on that changed.
*/
getChangesSinceDate: function getChangesSinceDate(date) {
this._ensureStateLoaded();
getChangesSinceDate(date) {
let length = this._changes.length;
for (let i = 0; i < length; i++) {
if (this._changes[i][0] >= date) {
@ -523,9 +493,7 @@ AddonsReconciler.prototype = {
* @param date
* Entries older than this Date will be removed.
*/
pruneChangesBeforeDate: function pruneChangesBeforeDate(date) {
this._ensureStateLoaded();
pruneChangesBeforeDate(date) {
this._changes = this._changes.filter(function test_age(change) {
return change[0] >= date;
});
@ -533,10 +501,8 @@ AddonsReconciler.prototype = {
/**
* Obtains the set of all known Sync GUIDs for add-ons.
*
* @return Object with guids as keys and values of true.
*/
getAllSyncGUIDs: function getAllSyncGUIDs() {
getAllSyncGUIDs() {
let result = {};
for (let id in this.addons) {
result[id] = true;
@ -552,9 +518,8 @@ AddonsReconciler.prototype = {
*
* @param guid
* Sync GUID of add-on to retrieve.
* @return Object on success on null on failure.
*/
getAddonStateFromSyncGUID: function getAddonStateFromSyncGUID(guid) {
getAddonStateFromSyncGUID(guid) {
for (let id in this.addons) {
let addon = this.addons[id];
if (addon.guid == guid) {
@ -565,27 +530,10 @@ AddonsReconciler.prototype = {
return null;
},
/**
* Ensures that state is loaded before continuing.
*
* This is called internally by anything that accesses the internal data
* structures. It effectively just-in-time loads serialized state.
*/
_ensureStateLoaded: function _ensureStateLoaded() {
if (this._stateLoaded) {
return;
}
let cb = Async.makeSpinningCallback();
this.loadState(null, cb);
cb.wait();
this._stateLoaded = true;
},
/**
* Handler that is invoked as part of the AddonManager listeners.
*/
_handleListener: function _handlerListener(action, addon, requiresRestart) {
_handleListener(action, addon, requiresRestart) {
// Since this is called as an observer, we explicitly trap errors and
// log them to ourselves so we don't see errors reported elsewhere.
try {
@ -629,9 +577,7 @@ AddonsReconciler.prototype = {
// See note for _shouldPersist.
if (this._shouldPersist) {
let cb = Async.makeSpinningCallback();
this.saveState(null, cb);
cb.wait();
Async.promiseSpinningly(this.saveState());
}
} catch (ex) {
this._log.warn("Exception", ex);

59
services/sync/modules/addonutils.js
View File

@ -153,43 +153,42 @@ AddonUtilsInternal.prototype = {
},
/**
* Uninstalls the Addon instance and invoke a callback when it is done.
* Uninstalls the addon instance.
*
* @param addon
* Addon instance to uninstall.
* @param cb
* Function to be invoked when uninstall has finished. It receives a
* truthy value signifying error and the add-on which was uninstalled.
*/
uninstallAddon: function uninstallAddon(addon, cb) {
let listener = {
onUninstalling(uninstalling, needsRestart) {
if (addon.id != uninstalling.id) {
return;
}
async uninstallAddon(addon) {
return new Promise(res => {
let listener = {
onUninstalling(uninstalling, needsRestart) {
if (addon.id != uninstalling.id) {
return;
}
// We assume restartless add-ons will send the onUninstalled event
// soon.
if (!needsRestart) {
return;
}
// We assume restartless add-ons will send the onUninstalled event
// soon.
if (!needsRestart) {
return;
}
// For non-restartless add-ons, we issue the callback on uninstalling
// because we will likely never see the uninstalled event.
AddonManager.removeAddonListener(listener);
cb(null, addon);
},
onUninstalled(uninstalled) {
if (addon.id != uninstalled.id) {
return;
}
// For non-restartless add-ons, we issue the callback on uninstalling
// because we will likely never see the uninstalled event.
AddonManager.removeAddonListener(listener);
res(addon);
},
onUninstalled(uninstalled) {
if (addon.id != uninstalled.id) {
return;
}
AddonManager.removeAddonListener(listener);
cb(null, addon);
}
};
AddonManager.addAddonListener(listener);
addon.uninstall();
AddonManager.removeAddonListener(listener);
res(addon);
}
};
AddonManager.addAddonListener(listener);
addon.uninstall();
});
},
/**

43
services/sync/modules/bookmark_repair.js
View File

@ -198,7 +198,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
the specified validation information.
Returns true if a repair was started and false otherwise.
*/
startRepairs(validationInfo, flowID) {
async startRepairs(validationInfo, flowID) {
if (this._currentState != STATE.NOT_REPAIRING) {
log.info(`Can't start a repair - repair with ID ${this._flowID} is already in progress`);
return false;
@ -238,7 +238,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
Returns true if we could continue the repair - even if the state didn't
actually move. Returns false if we aren't actually repairing.
*/
continueRepairs(response = null) {
async continueRepairs(response = null) {
// Note that "ABORTED" and "FINISHED" should never be current when this
// function returns - this function resets to NOT_REPAIRING in those cases.
if (this._currentState == STATE.NOT_REPAIRING) {
@ -253,7 +253,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
state = this._currentState;
log.info("continueRepairs starting with state", state);
try {
newState = this._continueRepairs(state, response);
newState = await this._continueRepairs(state, response);
log.info("continueRepairs has next state", newState);
} catch (ex) {
if (!(ex instanceof AbortRepairError)) {
@ -294,7 +294,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
return true;
}
_continueRepairs(state, response = null) {
async _continueRepairs(state, response = null) {
if (this.anyClientsRepairing(this._flowID)) {
throw new AbortRepairError("other clients repairing");
}
@ -326,7 +326,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
this.service.recordTelemetryEvent("repair", "abandon", "missing", extra);
break;
}
if (this._isCommandPending(clientID, flowID)) {
if ((await this._isCommandPending(clientID, flowID))) {
// So the command we previously sent is still queued for the client
// (ie, that client is yet to have synced). Let's see if we should
// give up on that client.
@ -356,7 +356,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
if (state == STATE.SENT_REQUEST) {
log.info(`previous request to client ${clientID} was removed - trying a second time`);
state = STATE.SENT_SECOND_REQUEST;
this._writeRequest(clientID);
await this._writeRequest(clientID);
} else {
// this was the second time around, so give up on this client
log.info(`previous 2 requests to client ${clientID} were removed - need a new client`);
@ -373,7 +373,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
}
this._addToPreviousRemoteClients(this._currentRemoteClient);
this._currentRemoteClient = newClientID;
this._writeRequest(newClientID);
await this._writeRequest(newClientID);
state = STATE.SENT_REQUEST;
break;
@ -434,7 +434,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
/* Issue a repair request to a specific client.
*/
_writeRequest(clientID) {
async _writeRequest(clientID) {
log.trace("writing repair request to client", clientID);
let ids = this._currentIDs;
if (!ids) {
@ -449,7 +449,7 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
ids,
flowID,
}
this.service.clientsEngine.sendCommand("repairRequest", [request], clientID, { flowID });
await this.service.clientsEngine.sendCommand("repairRequest", [request], clientID, { flowID });
this.prefs.set(PREF.REPAIR_WHEN, Math.floor(this._now()));
// record telemetry about this
let extra = {
@ -486,11 +486,12 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
/* Is our command still in the "commands" queue for the specific client?
*/
_isCommandPending(clientID, flowID) {
async _isCommandPending(clientID, flowID) {
// getClientCommands() is poorly named - it's only outgoing commands
// from us we have yet to write. For our purposes, we want to check
// them and commands previously written (which is in .commands)
let commands = [...this.service.clientsEngine.getClientCommands(clientID),
let clientCommands = await this.service.clientsEngine.getClientCommands(clientID);
let commands = [...clientCommands,
...this.service.clientsEngine.remoteClient(clientID).commands || []];
for (let command of commands) {
if (command.command != "repairRequest" || command.args.length != 1) {
@ -559,8 +560,8 @@ class BookmarkRepairRequestor extends CollectionRepairRequestor {
class BookmarkRepairResponder extends CollectionRepairResponder {
async repair(request, rawCommand) {
if (request.request != "upload") {
this._abortRepair(request, rawCommand,
`Don't understand request type '${request.request}'`);
await this._abortRepair(request, rawCommand,
`Don't understand request type '${request.request}'`);
return;
}
@ -605,7 +606,7 @@ class BookmarkRepairResponder extends CollectionRepairResponder {
this.service.recordTelemetryEvent("repairResponse", "uploading", undefined, eventExtra);
} else {
// We were unable to help with the repair, so report that we are done.
this._finishRepair();
await this._finishRepair();
}
} catch (ex) {
if (Async.isShutdownException(ex)) {
@ -616,7 +617,7 @@ class BookmarkRepairResponder extends CollectionRepairResponder {
// on, but we record the failure reason in telemetry.
log.error("Failed to respond to the repair request", ex);
this._currentState.failureReason = SyncTelemetry.transformError(ex);
this._finishRepair();
await this._finishRepair();
}
}
@ -690,10 +691,10 @@ class BookmarkRepairResponder extends CollectionRepairResponder {
}
Svc.Obs.remove("weave:engine:sync:uploaded", this.onUploaded, this);
log.debug(`bookmarks engine has uploaded stuff - creating a repair response`);
this._finishRepair();
Async.promiseSpinningly(this._finishRepair());
}
_finishRepair() {
async _finishRepair() {
let clientsEngine = this.service.clientsEngine;
let flowID = this._currentState.request.flowID;
let response = {
@ -704,9 +705,9 @@ class BookmarkRepairResponder extends CollectionRepairResponder {
ids: this._currentState.ids,
}
let clientID = this._currentState.request.requestor;
clientsEngine.sendCommand("repairResponse", [response], clientID, { flowID });
await clientsEngine.sendCommand("repairResponse", [response], clientID, { flowID });
// and nuke the request from our client.
clientsEngine.removeLocalCommand(this._currentState.rawCommand);
await clientsEngine.removeLocalCommand(this._currentState.rawCommand);
let eventExtra = {
flowID,
numIDs: response.ids.length.toString(),
@ -722,9 +723,9 @@ class BookmarkRepairResponder extends CollectionRepairResponder {
this._currentState = null;
}
_abortRepair(request, rawCommand, why) {
async _abortRepair(request, rawCommand, why) {
log.warn(`aborting repair request: ${why}`);
this.service.clientsEngine.removeLocalCommand(rawCommand);
await this.service.clientsEngine.removeLocalCommand(rawCommand);
// record telemetry for this.
let eventExtra = {
flowID: request.flowID,

16
services/sync/modules/bookmark_validator.js
View File

@ -7,10 +7,12 @@
const { interfaces: Ci, utils: Cu } = Components;
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/Timer.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-common/utils.js");
XPCOMUtils.defineLazyModuleGetter(this, "Async",
"resource://services-common/async.js");
XPCOMUtils.defineLazyModuleGetter(this, "PlacesUtils",
"resource://gre/modules/PlacesUtils.jsm");
@ -291,11 +293,9 @@ class BookmarkValidator {
let records = [];
let recordsByGuid = new Map();
let syncedRoots = SYNCED_ROOTS;
let yieldCounter = 0;
let maybeYield = Async.jankYielder();
async function traverse(treeNode, synced) {
if (++yieldCounter % 50 === 0) {
await new Promise(resolve => setTimeout(resolve, 50));
}
await maybeYield();
if (!synced) {
synced = syncedRoots.includes(treeNode.guid);
} else if (isNodeIgnored(treeNode)) {
@ -406,8 +406,9 @@ class BookmarkValidator {
let resultRecords = [];
let yieldCounter = 0;
let maybeYield = Async.jankYielder();
for (let record of serverRecords) {
await maybeYield();
if (!record.id) {
++problemData.missingIDs;
continue;
@ -445,9 +446,6 @@ class BookmarkValidator {
return PlacesSyncUtils.bookmarks.guidToSyncId(childID);
});
}
if (++yieldCounter % 50 === 0) {
await new Promise(resolve => setTimeout(resolve, 50));
}
}
for (let deletedId of deletedItemIds) {

6
services/sync/modules/browserid_identity.js
View File

@ -282,9 +282,9 @@ this.BrowserIDManager.prototype = {
// (which is signaled by `this.whenReadyToAuthenticate.promise` resolving).
this.whenReadyToAuthenticate.promise.then(() => {
Services.obs.notifyObservers(null, "weave:service:setup-complete");
return new Promise(resolve => { Weave.Utils.nextTick(resolve, null); })
return Async.promiseYield();
}).then(() => {
Weave.Service.sync();
return Weave.Service.sync();
}).catch(e => {
this._log.warn("Failed to trigger setup complete notification", e);
});
@ -292,7 +292,7 @@ this.BrowserIDManager.prototype = {
} break;
case fxAccountsCommon.ONLOGOUT_NOTIFICATION:
Weave.Service.startOver();
Async.promiseSpinningly(Weave.Service.startOver());
// startOver will cause this instance to be thrown away, so there's
// nothing else to do.
break;

4
services/sync/modules/collection_repair.js
View File

@ -92,7 +92,7 @@ class CollectionRepairRequestor {
reported in telemetry.
*/
startRepairs(validationInfo, flowID) {
async startRepairs(validationInfo, flowID) {
throw new Error("not implemented");
}
@ -106,7 +106,7 @@ class CollectionRepairRequestor {
by a remote repair responder.
*/
continueRepairs(responseInfo = null) {
async continueRepairs(responseInfo = null) {
throw new Error("not implemented");
}
}

29
services/sync/modules/collection_validator.js
View File

@ -5,6 +5,10 @@
"use strict";
const Cu = Components.utils;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyModuleGetter(this, "Async",
"resource://services-common/async.js");
this.EXPORTED_SYMBOLS = ["CollectionValidator", "CollectionProblemData"];
@ -101,7 +105,7 @@ class CollectionValidator {
// Turn the server item into something that can be easily compared with the client
// items.
normalizeServerItem(item) {
async normalizeServerItem(item) {
return item;
}
@ -136,15 +140,24 @@ class CollectionValidator {
// clientRecords: Normalized client records
// records: Normalized server records,
// deletedRecords: Array of ids that were marked as deleted by the server.
compareClientWithServer(clientItems, serverItems) {
clientItems = clientItems.map(item => this.normalizeClientItem(item));
serverItems = serverItems.map(item => this.normalizeServerItem(item));
async compareClientWithServer(clientItems, serverItems) {
let maybeYield = Async.jankYielder();
const clientRecords = [];
for (let item of clientItems) {
await maybeYield();
clientRecords.push(this.normalizeClientItem(item));
}
const serverRecords = [];
for (let item of serverItems) {
await maybeYield();
serverRecords.push((await this.normalizeServerItem(item)));
}
let problems = this.emptyProblemData();
let seenServer = new Map();
let serverDeleted = new Set();
let allRecords = new Map();
for (let record of serverItems) {
for (let record of serverRecords) {
let id = record[this.idProp];
if (!id) {
++problems.missingIDs;
@ -165,7 +178,7 @@ class CollectionValidator {
}
let seenClient = new Map();
for (let record of clientItems) {
for (let record of clientRecords) {
let id = record[this.idProp];
record.shouldSync = this.syncedByClient(record);
seenClient.set(id, record);
@ -203,8 +216,8 @@ class CollectionValidator {
}
return {
problemData: problems,
clientRecords: clientItems,
records: serverItems,
clientRecords,
records: serverRecords,
deletedRecords: [...serverDeleted]
};
}

12
services/sync/modules/doctor.js
View File

@ -72,7 +72,7 @@ this.Doctor = {
try {
for (let [collection, requestor] of Object.entries(this._getAllRepairRequestors())) {
try {
let advanced = requestor.continueRepairs();
let advanced = await requestor.continueRepairs();
log.info(`${collection} reparier ${advanced ? "advanced" : "did not advance"}.`);
} catch (ex) {
if (Async.isShutdownException(ex)) {
@ -137,7 +137,7 @@ this.Doctor = {
if (Object.values(engineInfos).filter(i => i.maxRecords != -1).length != 0) {
// at least some of the engines have maxRecord restrictions which require
// us to ask the server for the counts.
let countInfo = this._fetchCollectionCounts();
let countInfo = await this._fetchCollectionCounts();
for (let [engineName, recordCount] of Object.entries(countInfo)) {
if (engineName in engineInfos) {
engineInfos[engineName].recordCount = recordCount;
@ -193,7 +193,7 @@ this.Doctor = {
}
},
_maybeCure(engine, validationResults, flowID) {
async _maybeCure(engine, validationResults, flowID) {
if (!this._shouldRepair(engine)) {
log.info(`Skipping repair of ${engine.name} - disabled via preferences`);
return;
@ -206,7 +206,7 @@ this.Doctor = {
return; // TODO: It would be nice if we could request a validation to be
// done on next sync.
}
didStart = requestor.startRepairs(validationResults, flowID);
didStart = await requestor.startRepairs(validationResults, flowID);
}
log.info(`${didStart ? "did" : "didn't"} start a repair of ${engine.name} with flowID ${flowID}`);
},
@ -216,10 +216,10 @@ this.Doctor = {
},
// mainly for mocking.
_fetchCollectionCounts() {
async _fetchCollectionCounts() {
let collectionCountsURL = Service.userBaseURL + "info/collection_counts";
try {
let infoResp = Service._fetchInfo(collectionCountsURL);
let infoResp = await Service._fetchInfo(collectionCountsURL);
if (!infoResp.success) {
log.error("Can't fetch collection counts: request to info/collection_counts responded with "
+ infoResp.status);

308
services/sync/modules/engines.js
View File

@ -14,7 +14,6 @@ this.EXPORTED_SYMBOLS = [
var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Services.jsm");
Cu.import("resource://gre/modules/JSONFile.jsm");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/async.js");
@ -310,12 +309,6 @@ this.Store = function Store(name, engine) {
}
Store.prototype = {
_sleep: function _sleep(delay) {
let cb = Async.makeSyncCallback();
this._timer.initWithCallback(cb, delay, Ci.nsITimer.TYPE_ONE_SHOT);
Async.waitForSyncCallback(cb);
},
/**
* Apply multiple incoming records against the store.
*
@ -331,11 +324,13 @@ Store.prototype = {
* @param records Array of records to apply
* @return Array of record IDs which did not apply cleanly
*/
applyIncomingBatch(records) {
async applyIncomingBatch(records) {
let failed = [];
let maybeYield = Async.jankYielder();
for (let record of records) {
await maybeYield();
try {
this.applyIncoming(record);
await this.applyIncoming(record);
} catch (ex) {
if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
// This kind of exception should have a 'cause' attribute, which is an
@ -347,7 +342,6 @@ Store.prototype = {
throw ex;
}
this._log.warn("Failed to apply incoming record " + record.id, ex);
this.engine._noteApplyFailure();
failed.push(record.id);
}
}
@ -367,13 +361,13 @@ Store.prototype = {
* @param record
* Record to apply
*/
applyIncoming(record) {
async applyIncoming(record) {
if (record.deleted)
this.remove(record);
else if (!this.itemExists(record.id))
this.create(record);
await this.remove(record);
else if (!(await this.itemExists(record.id)))
await this.create(record);
else
this.update(record);
await this.update(record);
},
// override these in derived objects
@ -387,7 +381,7 @@ Store.prototype = {
* @param record
* The store record to create an item from
*/
create(record) {
async create(record) {
throw "override create in a subclass";
},
@ -400,7 +394,7 @@ Store.prototype = {
* @param record
* The store record to delete an item from
*/
remove(record) {
async remove(record) {
throw "override remove in a subclass";
},
@ -413,7 +407,7 @@ Store.prototype = {
* @param record
* The record to use to update an item from
*/
update(record) {
async update(record) {
throw "override update in a subclass";
},
@ -427,7 +421,7 @@ Store.prototype = {
* string record ID
* @return boolean indicating whether record exists locally
*/
itemExists(id) {
async itemExists(id) {
throw "override itemExists in a subclass";
},
@ -445,7 +439,7 @@ Store.prototype = {
* constructor for the newly-created record.
* @return record type for this engine
*/
createRecord(id, collection) {
async createRecord(id, collection) {
throw "override createRecord in a subclass";
},
@ -457,7 +451,7 @@ Store.prototype = {
* @param newID
* string new record ID
*/
changeItemID(oldID, newID) {
async changeItemID(oldID, newID) {
throw "override changeItemID in a subclass";
},
@ -467,7 +461,7 @@ Store.prototype = {
* @return Object with ID strings as keys and values of true. The values
* are ignored.
*/
getAllIDs() {
async getAllIDs() {
throw "override getAllIDs in a subclass";
},
@ -481,7 +475,7 @@ Store.prototype = {
* can be thought of as clearing out all state and restoring the "new
* browser" state.
*/
wipe() {
async wipe() {
throw "override wipe in a subclass";
}
};
@ -601,9 +595,11 @@ EngineManager.prototype = {
* Engine object used to get an instance of the engine
* @return The engine object if anything failed
*/
register(engineObject) {
async register(engineObject) {
if (Array.isArray(engineObject)) {
engineObject.map(this.register, this);
for (const e of engineObject) {
await this.register(e);
}
return;
}
@ -613,6 +609,9 @@ EngineManager.prototype = {
if (name in this._engines) {
this._log.error("Engine '" + name + "' is already registered!");
} else {
if (engine.initialize) {
await engine.initialize();
}
this._engines[name] = engine;
}
} catch (ex) {
@ -661,7 +660,7 @@ this.Engine = function Engine(name, service) {
this._modified = this.emptyChangeset();
this._tracker; // initialize tracker to load previously changed IDs
this._log.debug("Engine initialized");
this._log.debug("Engine constructed");
}
Engine.prototype = {
// _storeObj, and _trackerObj should to be overridden in subclasses
@ -711,40 +710,40 @@ Engine.prototype = {
return tracker;
},
sync() {
async sync() {
if (!this.enabled) {
return;
return false;
}
if (!this._sync) {
throw "engine does not implement _sync method";
}
this._notify("sync", this.name, this._sync)();
return this._notify("sync", this.name, this._sync)();
},
/**
* Get rid of any local meta-data.
*/
resetClient() {
async resetClient() {
if (!this._resetClient) {
throw "engine does not implement _resetClient method";
}
this._notify("reset-client", this.name, this._resetClient)();
return this._notify("reset-client", this.name, this._resetClient)();
},
_wipeClient() {
this.resetClient();
async _wipeClient() {
await this.resetClient();
this._log.debug("Deleting all local data");
this._tracker.ignoreAll = true;
this._store.wipe();
await this._store.wipe();
this._tracker.ignoreAll = false;
this._tracker.clearChangedIDs();
},
wipeClient() {
this._notify("wipe-client", this.name, this._wipeClient)();
async wipeClient() {
return this._notify("wipe-client", this.name, this._wipeClient)();
},
/**
@ -764,8 +763,8 @@ Engine.prototype = {
this.SyncEngine = function SyncEngine(name, service) {
Engine.call(this, name || "SyncEngine", service);
this.loadToFetch();
this.loadPreviousFailed();
// Async initializations can be made in the initialize() method.
// The set of records needing a weak reupload.
// The difference between this and a "normal" reupload is that these records
// are only tracked in memory, and if the reupload attempt fails (shutdown,
@ -818,6 +817,12 @@ SyncEngine.prototype = {
// How many records to process in a single batch.
applyIncomingBatchSize: DEFAULT_STORE_BATCH_SIZE,
async initialize() {
await this.loadToFetch();
await this.loadPreviousFailed();
this._log.debug("SyncEngine initialized", this.name);
},
get storageURL() {
return this.service.storageURL;
},
@ -866,60 +871,55 @@ SyncEngine.prototype = {
return this._toFetch;
},
set toFetch(val) {
let cb = (error) => {
if (error) {
this._log.error("Failed to read JSON records to fetch", error);
}
}
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._toFetch) {
return;
}
this._toFetch = val;
Utils.namedTimer(function() {
Utils.jsonSave("toFetch/" + this.name, this, val, cb);
try {
Async.promiseSpinningly(Utils.jsonSave("toFetch/" + this.name, this, val));
} catch (error) {
this._log.error("Failed to read JSON records to fetch", error);
}
}, 0, this, "_toFetchDelay");
},
loadToFetch() {
async loadToFetch() {
// Initialize to empty if there's no file.
this._toFetch = [];
Utils.jsonLoad("toFetch/" + this.name, this, function(toFetch) {
if (toFetch) {
this._toFetch = toFetch;
}
});
let toFetch = await Utils.jsonLoad("toFetch/" + this.name, this);
if (toFetch) {
this._toFetch = toFetch;
}
},
get previousFailed() {
return this._previousFailed;
},
set previousFailed(val) {
let cb = (error) => {
if (error) {
this._log.error("Failed to set previousFailed", error);
} else {
this._log.debug("Successfully wrote previousFailed.");
}
}
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._previousFailed) {
return;
}
this._previousFailed = val;
Utils.namedTimer(function() {
Utils.jsonSave("failed/" + this.name, this, val, cb);
Utils.jsonSave("failed/" + this.name, this, val).then(() => {
this._log.debug("Successfully wrote previousFailed.");
})
.catch((error) => {
this._log.error("Failed to set previousFailed", error);
});
}, 0, this, "_previousFailedDelay");
},
loadPreviousFailed() {
async loadPreviousFailed() {
// Initialize to empty if there's no file
this._previousFailed = [];
Utils.jsonLoad("failed/" + this.name, this, function(previousFailed) {
if (previousFailed) {
this._previousFailed = previousFailed;
}
});
let previousFailed = await Utils.jsonLoad("failed/" + this.name, this);
if (previousFailed) {
this._previousFailed = previousFailed;
}
},
/*
@ -945,13 +945,13 @@ SyncEngine.prototype = {
* Returns a changeset for this sync. Engine implementations can override this
* method to bypass the tracker for certain or all changed items.
*/
getChangedIDs() {
async getChangedIDs() {
return this._tracker.changedIDs;
},
// Create a new record using the store and add in metadata.
_createRecord(id) {
let record = this._store.createRecord(id, this.name);
async _createRecord(id) {
let record = await this._store.createRecord(id, this.name);
record.id = id;
record.collection = this.name;
return record;
@ -967,10 +967,10 @@ SyncEngine.prototype = {
},
// Any setup that needs to happen at the beginning of each sync.
_syncStartup() {
async _syncStartup() {
// Determine if we need to wipe on outdated versions
let metaGlobal = Async.promiseSpinningly(this.service.recordManager.get(this.metaURL));
let metaGlobal = await this.service.recordManager.get(this.metaURL);
let engines = metaGlobal.payload.engines || {};
let engineData = engines[this.name] || {};
@ -1001,13 +1001,13 @@ SyncEngine.prototype = {
// Changes to syncID mean we'll need to upload everything
this._log.debug("Engine syncIDs: " + [engineData.syncID, this.syncID]);
this.syncID = engineData.syncID;
this._resetClient();
await this._resetClient();
}
// Delete any existing data and reupload on bad version or missing meta.
// No crypto component here...? We could regenerate per-collection keys...
if (needsWipe) {
this.wipeServer();
await this.wipeServer();
}
// Save objects that need to be uploaded in this._modified. We also save
@ -1019,10 +1019,10 @@ SyncEngine.prototype = {
this.lastSyncLocal = Date.now();
let initialChanges;
if (this.lastSync) {
initialChanges = this.pullNewChanges();
initialChanges = await this.pullNewChanges();
} else {
this._log.debug("First sync, uploading all items");
initialChanges = this.pullAllChanges();
initialChanges = await this.pullAllChanges();
}
this._modified.replace(initialChanges);
// Clear the tracker now. If the sync fails we'll add the ones we failed
@ -1050,7 +1050,7 @@ SyncEngine.prototype = {
* In the most awful and untestable way possible.
* This now accepts something that makes testing vaguely less impossible.
*/
_processIncoming(newitems) {
async _processIncoming(newitems) {
this._log.trace("Downloading & applying server changes");
// Figure out how many total items to fetch this sync; do less on mobile.
@ -1086,10 +1086,10 @@ SyncEngine.prototype = {
// cease.
let aborting = undefined;
function doApplyBatch() {
async function doApplyBatch() {
this._tracker.ignoreAll = true;
try {
failed = failed.concat(this._store.applyIncomingBatch(applyBatch));
failed = failed.concat((await this._store.applyIncomingBatch(applyBatch)));
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
@ -1103,10 +1103,10 @@ SyncEngine.prototype = {
applyBatch = [];
}
function doApplyBatchAndPersistFailed() {
async function doApplyBatchAndPersistFailed() {
// Apply remaining batch.
if (applyBatch.length) {
doApplyBatch.call(this);
await doApplyBatch.call(this);
}
// Persist failed items so we refetch them.
if (failed.length) {
@ -1123,7 +1123,7 @@ SyncEngine.prototype = {
// called for every incoming record.
let self = this;
let recordHandler = function(item) {
let recordHandler = async function(item) {
if (aborting) {
return;
}
@ -1145,7 +1145,7 @@ SyncEngine.prototype = {
if (!Utils.isHMACMismatch(ex)) {
throw ex;
}
let strategy = self.handleHMACMismatch(item, true);
let strategy = await self.handleHMACMismatch(item, true);
if (strategy == SyncEngine.kRecoveryStrategy.retry) {
// You only get one retry.
try {
@ -1158,7 +1158,7 @@ SyncEngine.prototype = {
if (!Utils.isHMACMismatch(ex)) {
throw ex;
}
strategy = self.handleHMACMismatch(item, false);
strategy = await self.handleHMACMismatch(item, false);
}
}
@ -1171,7 +1171,6 @@ SyncEngine.prototype = {
// Fall through to error case.
case SyncEngine.kRecoveryStrategy.error:
self._log.warn("Error decrypting record", ex);
self._noteApplyFailure();
failed.push(item.id);
return;
case SyncEngine.kRecoveryStrategy.ignore:
@ -1185,7 +1184,6 @@ SyncEngine.prototype = {
throw ex;
}
self._log.warn("Error decrypting record", ex);
self._noteApplyFailure();
failed.push(item.id);
return;
}
@ -1198,16 +1196,14 @@ SyncEngine.prototype = {
let shouldApply;
try {
shouldApply = self._reconcile(item);
shouldApply = await self._reconcile(item);
} catch (ex) {
if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
self._log.warn("Reconciliation failed: aborting incoming processing.");
self._noteApplyFailure();
failed.push(item.id);
aborting = ex.cause;
} else if (!Async.isShutdownException(ex)) {
self._log.warn("Failed to reconcile incoming record " + item.id, ex);
self._noteApplyFailure();
failed.push(item.id);
return;
} else {
@ -1224,23 +1220,24 @@ SyncEngine.prototype = {
}
if (applyBatch.length == self.applyIncomingBatchSize) {
doApplyBatch.call(self);
await doApplyBatch.call(self);
}
self._store._sleep(0);
};
// Only bother getting data from the server if there's new things
if (this.lastModified == null || this.lastModified > this.lastSync) {
let { response, records } = Async.promiseSpinningly(newitems.getBatched());
let { response, records } = await newitems.getBatched();
if (!response.success) {
response.failureCode = ENGINE_DOWNLOAD_FAIL;
throw response;
}
let maybeYield = Async.jankYielder();
for (let record of records) {
recordHandler(record);
await maybeYield();
await recordHandler(record);
}
doApplyBatchAndPersistFailed.call(this);
await doApplyBatchAndPersistFailed.call(this);
if (aborting) {
throw aborting;
@ -1258,7 +1255,7 @@ SyncEngine.prototype = {
// index: Orders by the sortindex descending (highest weight first).
guidColl.sort = "index";
let guids = Async.promiseSpinningly(guidColl.get());
let guids = await guidColl.get();
if (!guids.success)
throw guids;
@ -1289,16 +1286,18 @@ SyncEngine.prototype = {
newitems.newer = 0;
newitems.ids = fetchBatch.slice(0, batchSize);
let resp = Async.promiseSpinningly(newitems.get());
let resp = await newitems.get();
if (!resp.success) {
resp.failureCode = ENGINE_DOWNLOAD_FAIL;
throw resp;
}
let maybeYield = Async.jankYielder();
for (let json of resp.obj) {
await maybeYield();
let record = new this._recordObj();
record.deserialize(json);
recordHandler(record);
await recordHandler(record);
}
// This batch was successfully applied. Not using
@ -1322,12 +1321,11 @@ SyncEngine.prototype = {
}
// Apply remaining items.
doApplyBatchAndPersistFailed.call(this);
await doApplyBatchAndPersistFailed.call(this);
count.newFailed = this.previousFailed.reduce((count, engine) => {
if (failedInPreviousSync.indexOf(engine) == -1) {
count++;
this._noteApplyNewFailure();
}
return count;
}, 0);
@ -1348,21 +1346,13 @@ SyncEngine.prototype = {
return false;
},
_noteApplyFailure() {
// here would be a good place to record telemetry...
},
_noteApplyNewFailure() {
// here would be a good place to record telemetry...
},
/**
* Find a GUID of an item that is a duplicate of the incoming item but happens
* to have a different GUID
*
* @return GUID of the similar item; falsy otherwise
*/
_findDupe(item) {
async _findDupe(item) {
// By default, assume there's no dupe items for the engine
},
@ -1379,7 +1369,7 @@ SyncEngine.prototype = {
// record to the server -- any extra work that's needed as part of this
// process should be done at this point (such as mark the record's parent
// for reuploading in the case of bookmarks).
_shouldReviveRemotelyDeletedRecord(remoteItem) {
async _shouldReviveRemotelyDeletedRecord(remoteItem) {
return true;
},
@ -1396,7 +1386,7 @@ SyncEngine.prototype = {
this._delete.ids.push(id);
},
_switchItemToDupe(localDupeGUID, incomingItem) {
async _switchItemToDupe(localDupeGUID, incomingItem) {
// The local, duplicate ID is always deleted on the server.
this._deleteId(localDupeGUID);
@ -1405,7 +1395,7 @@ SyncEngine.prototype = {
// contract were stronger, this could be changed.
this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " +
incomingItem.id);
this._store.changeItemID(localDupeGUID, incomingItem.id);
return this._store.changeItemID(localDupeGUID, incomingItem.id);
},
/**
@ -1418,7 +1408,7 @@ SyncEngine.prototype = {
* @return boolean
* Truthy if incoming record should be applied. False if not.
*/
_reconcile(item) {
async _reconcile(item) {
if (this._log.level <= Log.Level.Trace) {
this._log.trace("Incoming: " + item);
}
@ -1426,7 +1416,7 @@ SyncEngine.prototype = {
// We start reconciling by collecting a bunch of state. We do this here
// because some state may change during the course of this function and we
// need to operate on the original values.
let existsLocally = this._store.itemExists(item.id);
let existsLocally = await this._store.itemExists(item.id);
let locallyModified = this._modified.has(item.id);
// TODO Handle clock drift better. Tracked in bug 721181.
@ -1469,7 +1459,7 @@ SyncEngine.prototype = {
}
// If the local record is newer, we defer to individual engines for
// how to handle this. By default, we revive the record.
let willRevive = this._shouldReviveRemotelyDeletedRecord(item);
let willRevive = await this._shouldReviveRemotelyDeletedRecord(item);
this._log.trace("Local record is newer -- reviving? " + willRevive);
return !willRevive;
@ -1484,7 +1474,7 @@ SyncEngine.prototype = {
// refresh the metadata collected above. See bug 710448 for the history
// of this logic.
if (!existsLocally) {
let localDupeGUID = this._findDupe(item);
let localDupeGUID = await this._findDupe(item);
if (localDupeGUID) {
this._log.trace("Local item " + localDupeGUID + " is a duplicate for " +
"incoming item " + item.id);
@ -1492,7 +1482,7 @@ SyncEngine.prototype = {
// The current API contract does not mandate that the ID returned by
// _findDupe() actually exists. Therefore, we have to perform this
// check.
existsLocally = this._store.itemExists(localDupeGUID);
existsLocally = await this._store.itemExists(localDupeGUID);
// If the local item was modified, we carry its metadata forward so
// appropriate reconciling can be performed.
@ -1508,7 +1498,7 @@ SyncEngine.prototype = {
}
// Tell the engine to do whatever it needs to switch the items.
this._switchItemToDupe(localDupeGUID, item);
await this._switchItemToDupe(localDupeGUID, item);
this._log.debug("Local item after duplication: age=" + localAge +
"; modified=" + locallyModified + "; exists=" +
@ -1553,7 +1543,7 @@ SyncEngine.prototype = {
// want to defer this logic is because it would avoid a redundant and
// possibly expensive dip into the storage layer to query item state.
// This should get addressed in the async rewrite, so we ignore it for now.
let localRecord = this._createRecord(item.id);
let localRecord = await this._createRecord(item.id);
let recordsEqual = Utils.deepEquals(item.cleartext,
localRecord.cleartext);
@ -1589,7 +1579,7 @@ SyncEngine.prototype = {
},
// Upload outgoing records.
_uploadOutgoing() {
async _uploadOutgoing() {
this._log.trace("Uploading local changes to server.");
// collection we'll upload
@ -1604,7 +1594,7 @@ SyncEngine.prototype = {
let failed = [];
let successful = [];
let handleResponse = (resp, batchOngoing = false) => {
let handleResponse = async (resp, batchOngoing = false) => {
// Note: We don't want to update this.lastSync, or this._modified until
// the batch is complete, however we want to remember success/failure
// indicators for when that happens.
@ -1639,7 +1629,7 @@ SyncEngine.prototype = {
this._modified.delete(id);
}
this._onRecordsWritten(successful, failed);
await this._onRecordsWritten(successful, failed);
// clear for next batch
failed.length = 0;
@ -1652,7 +1642,7 @@ SyncEngine.prototype = {
let out;
let ok = false;
try {
out = this._createRecord(id);
out = await this._createRecord(id);
if (this._log.level <= Log.Level.Trace)
this._log.trace("Outgoing: " + out);
@ -1675,7 +1665,7 @@ SyncEngine.prototype = {
}
this._needWeakReupload.delete(id);
if (ok) {
let { enqueued, error } = postQueue.enqueue(out);
let { enqueued, error } = await postQueue.enqueue(out);
if (!enqueued) {
++counts.failed;
if (!this.allowSkippedRecord) {
@ -1685,14 +1675,14 @@ SyncEngine.prototype = {
this._log.warn(`Failed to enqueue record "${id}" (skipping)`, error);
}
}
this._store._sleep(0);
await Async.promiseYield();
}
postQueue.flush(true);
await postQueue.flush(true);
}
if (this._needWeakReupload.size) {
try {
const { sent, failed } = this._weakReupload(up);
const { sent, failed } = await this._weakReupload(up);
counts.sent += sent;
counts.failed += failed;
} catch (e) {
@ -1707,7 +1697,7 @@ SyncEngine.prototype = {
}
},
_weakReupload(collection) {
async _weakReupload(collection) {
const counts = { sent: 0, failed: 0 };
let pendingSent = 0;
let postQueue = collection.newPostQueue(this._log, this.lastSync, (resp, batchOngoing = false) => {
@ -1723,7 +1713,7 @@ SyncEngine.prototype = {
}
});
let pendingWeakReupload = this.buildWeakReuploadMap(this._needWeakReupload);
let pendingWeakReupload = await this.buildWeakReuploadMap(this._needWeakReupload);
for (let [id, encodedRecord] of pendingWeakReupload) {
try {
this._log.trace("Outgoing (weak)", encodedRecord);
@ -1740,34 +1730,34 @@ SyncEngine.prototype = {
// `enqueued` is only false if the specific item failed to enqueue, but
// other items should be/are fine. For example, enqueued will be false if
// it is larger than the max post or WBO size.
let { enqueued } = postQueue.enqueue(encodedRecord);
let { enqueued } = await postQueue.enqueue(encodedRecord);
if (!enqueued) {
++counts.failed;
} else {
++pendingSent;
}
this._store._sleep(0);
await Async.promiseYield();
}
postQueue.flush(true);
await postQueue.flush(true);
return counts;
},
_onRecordsWritten(succeeded, failed) {
async _onRecordsWritten(succeeded, failed) {
// Implement this method to take specific actions against successfully
// uploaded records and failed records.
},
// Any cleanup necessary.
// Save the current snapshot so as to calculate changes at next sync
_syncFinish() {
async _syncFinish() {
this._log.trace("Finishing up sync");
this._tracker.resetScore();
let doDelete = Utils.bind2(this, function(key, val) {
let doDelete = async (key, val) => {
let coll = new Collection(this.engineURL, this._recordObj, this.service);
coll[key] = val;
Async.promiseSpinningly(coll.delete());
});
await coll.delete();
};
for (let [key, val] of Object.entries(this._delete)) {
// Remove the key for future uses
@ -1775,18 +1765,18 @@ SyncEngine.prototype = {
// Send a simple delete for the property
if (key != "ids" || val.length <= 100)
doDelete(key, val);
await doDelete(key, val);
else {
// For many ids, split into chunks of at most 100
while (val.length > 0) {
doDelete(key, val.slice(0, 100));
await doDelete(key, val.slice(0, 100));
val = val.slice(100);
}
}
}
},
_syncCleanup() {
async _syncCleanup() {
this._needWeakReupload.clear();
if (!this._modified) {
return;
@ -1794,26 +1784,26 @@ SyncEngine.prototype = {
try {
// Mark failed WBOs as changed again so they are reuploaded next time.
this.trackRemainingChanges();
await this.trackRemainingChanges();
} finally {
this._modified.clear();
}
},
_sync() {
async _sync() {
try {
this._syncStartup();
await this._syncStartup();
Observers.notify("weave:engine:sync:status", "process-incoming");
this._processIncoming();
await this._processIncoming();
Observers.notify("weave:engine:sync:status", "upload-outgoing");
this._uploadOutgoing();
this._syncFinish();
await this._uploadOutgoing();
await this._syncFinish();
} finally {
this._syncCleanup();
await this._syncCleanup();
}
},
canDecrypt() {
async canDecrypt() {
// Report failure even if there's nothing to decrypt
let canDecrypt = false;
@ -1828,7 +1818,7 @@ SyncEngine.prototype = {
// Any failure fetching/decrypting will just result in false
try {
this._log.trace("Trying to decrypt a record from the server..");
let json = Async.promiseSpinningly(test.get()).obj[0];
let json = (await test.get()).obj[0];
let record = new this._recordObj();
record.deserialize(json);
record.decrypt(key);
@ -1843,18 +1833,18 @@ SyncEngine.prototype = {
return canDecrypt;
},
_resetClient() {
async _resetClient() {
this.resetLastSync();
this.previousFailed = [];
this.toFetch = [];
},
wipeServer() {
let response = Async.promiseSpinningly(this.service.resource(this.engineURL).delete());
async wipeServer() {
let response = await this.service.resource(this.engineURL).delete();
if (response.status != 200 && response.status != 404) {
throw response;
}
this._resetClient();
await this._resetClient();
},
async removeClientData() {
@ -1878,9 +1868,9 @@ SyncEngine.prototype = {
*
* All return values will be part of the kRecoveryStrategy enumeration.
*/
handleHMACMismatch(item, mayRetry) {
async handleHMACMismatch(item, mayRetry) {
// By default we either try again, or bail out noisily.
return (this.service.handleHMACEvent() && mayRetry) ?
return ((await this.service.handleHMACEvent()) && mayRetry) ?
SyncEngine.kRecoveryStrategy.retry :
SyncEngine.kRecoveryStrategy.error;
},
@ -1895,9 +1885,10 @@ SyncEngine.prototype = {
*
* @return A `Changeset` object.
*/
pullAllChanges() {
async pullAllChanges() {
let changes = {};
for (let id in this._store.getAllIDs()) {
let ids = await this._store.getAllIDs();
for (let id in ids) {
changes[id] = 0;
}
return changes;
@ -1910,7 +1901,7 @@ SyncEngine.prototype = {
*
* @return A `Changeset` object.
*/
pullNewChanges() {
async pullNewChanges() {
return this.getChangedIDs();
},
@ -1919,7 +1910,7 @@ SyncEngine.prototype = {
* items that failed to upload. This method is called at the end of each sync.
*
*/
trackRemainingChanges() {
async trackRemainingChanges() {
for (let [id, change] of this._modified.entries()) {
this._tracker.addChangedID(id, change);
}
@ -1931,11 +1922,14 @@ SyncEngine.prototype = {
* shouldn't upload as part of a weak reupload (items that have changed,
* for example).
*/
buildWeakReuploadMap(idSet) {
async buildWeakReuploadMap(idSet) {
let result = new Map();
let maybeYield = Async.jankYielder();
for (let id of idSet) {
await maybeYield();
try {
result.set(id, this._createRecord(id));
let record = await this._createRecord(id);
result.set(id, record);
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;

117
services/sync/modules/engines/addons.js
View File

@ -128,10 +128,15 @@ AddonsEngine.prototype = {
_reconciler: null,
async initialize() {
await SyncEngine.prototype.initialize.call(this);
await this._reconciler.ensureStateLoaded();
},
/**
* Override parent method to find add-ons by their public ID, not Sync GUID.
*/
_findDupe: function _findDupe(item) {
async _findDupe(item) {
let id = item.addonID;
// The reconciler should have been updated at the top of the sync, so we
@ -153,7 +158,7 @@ AddonsEngine.prototype = {
* Override getChangedIDs to pull in tracker changes plus changes from the
* reconciler log.
*/
getChangedIDs: function getChangedIDs() {
async getChangedIDs() {
let changes = {};
for (let [id, modified] of Object.entries(this._tracker.changedIDs)) {
changes[id] = modified;
@ -201,13 +206,12 @@ AddonsEngine.prototype = {
* are complicated and we force a full refresh, just in case the listeners
* missed something.
*/
_syncStartup: function _syncStartup() {
async _syncStartup() {
// We refresh state before calling parent because syncStartup in the parent
// looks for changed IDs, which is dependent on add-on state being up to
// date.
this._refreshReconcilerState();
SyncEngine.prototype._syncStartup.call(this);
await this._refreshReconcilerState();
return SyncEngine.prototype._syncStartup.call(this);
},
/**
@ -218,24 +222,21 @@ AddonsEngine.prototype = {
* changes (thousands) for it to slow things down significantly. This is
* highly unlikely to occur. Still, we exercise defense just in case.
*/
_syncCleanup: function _syncCleanup() {
async _syncCleanup() {
let ms = 1000 * this.lastSync - PRUNE_ADDON_CHANGES_THRESHOLD;
this._reconciler.pruneChangesBeforeDate(new Date(ms));
SyncEngine.prototype._syncCleanup.call(this);
return SyncEngine.prototype._syncCleanup.call(this);
},
/**
* Helper function to ensure reconciler is up to date.
*
* This will synchronously load the reconciler's state from the file
* This will load the reconciler's state from the file
* system (if needed) and refresh the state of the reconciler.
*/
_refreshReconcilerState: function _refreshReconcilerState() {
async _refreshReconcilerState() {
this._log.debug("Refreshing reconciler state");
let cb = Async.makeSpinningCallback();
this._reconciler.refreshGlobalState(cb);
cb.wait();
return this._reconciler.refreshGlobalState();
},
isAddonSyncable(addon, ignoreRepoCheck) {
@ -267,7 +268,7 @@ AddonsStore.prototype = {
/**
* Override applyIncoming to filter out records we can't handle.
*/
applyIncoming: function applyIncoming(record) {
async applyIncoming(record) {
// The fields we look at aren't present when the record is deleted.
if (!record.deleted) {
// Ignore records not belonging to our application ID because that is the
@ -295,14 +296,14 @@ AddonsStore.prototype = {
return;
}
Store.prototype.applyIncoming.call(this, record);
await Store.prototype.applyIncoming.call(this, record);
},
/**
* Provides core Store API to create/install an add-on from a record.
*/
create: function create(record) {
async create(record) {
let cb = Async.makeSpinningCallback();
AddonUtils.installAddons([{
id: record.addonID,
@ -342,9 +343,9 @@ AddonsStore.prototype = {
/**
* Provides core Store API to remove/uninstall an add-on from a record.
*/
remove: function remove(record) {
async remove(record) {
// If this is called, the payload is empty, so we have to find by GUID.
let addon = this.getAddonByGUID(record.id);
let addon = await this.getAddonByGUID(record.id);
if (!addon) {
// We don't throw because if the add-on could not be found then we assume
// it has already been uninstalled and there is nothing for this function
@ -353,16 +354,14 @@ AddonsStore.prototype = {
}
this._log.info("Uninstalling add-on: " + addon.id);
let cb = Async.makeSpinningCallback();
AddonUtils.uninstallAddon(addon, cb);
cb.wait();
await AddonUtils.uninstallAddon(addon);
},
/**
* Provides core Store API to update an add-on from a record.
*/
update: function update(record) {
let addon = this.getAddonByID(record.addonID);
async update(record) {
let addon = await this.getAddonByID(record.addonID);
// update() is called if !this.itemExists. And, since itemExists consults
// the reconciler only, we need to take care of some corner cases.
@ -391,7 +390,7 @@ AddonsStore.prototype = {
/**
* Provide core Store API to determine if a record exists.
*/
itemExists: function itemExists(guid) {
async itemExists(guid) {
let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
return !!addon;
@ -407,7 +406,7 @@ AddonsStore.prototype = {
*
* @return AddonRecord instance
*/
createRecord: function createRecord(guid, collection) {
async createRecord(guid, collection) {
let record = new AddonRecord(collection, guid);
record.applicationID = Services.appinfo.ID;
@ -436,18 +435,16 @@ AddonsStore.prototype = {
*
* This implements a core API of the store.
*/
changeItemID: function changeItemID(oldID, newID) {
async changeItemID(oldID, newID) {
// We always update the GUID in the reconciler because it will be
// referenced later in the sync process.
let state = this.reconciler.getAddonStateFromSyncGUID(oldID);
if (state) {
state.guid = newID;
let cb = Async.makeSpinningCallback();
this.reconciler.saveState(null, cb);
cb.wait();
await this.reconciler.saveState();
}
let addon = this.getAddonByGUID(oldID);
let addon = await this.getAddonByGUID(oldID);
if (!addon) {
this._log.debug("Cannot change item ID (" + oldID + ") in Add-on " +
"Manager because old add-on not present: " + oldID);
@ -462,7 +459,7 @@ AddonsStore.prototype = {
*
* This implements a core Store API.
*/
getAllIDs: function getAllIDs() {
async getAllIDs() {
let ids = {};
let addons = this.reconciler.addons;
@ -482,15 +479,16 @@ AddonsStore.prototype = {
* This uninstalls all syncable addons from the application. In case of
* error, it logs the error and keeps trying with other add-ons.
*/
wipe: function wipe() {
async wipe() {
this._log.info("Processing wipe.");
this.engine._refreshReconcilerState();
await this.engine._refreshReconcilerState();
// We only wipe syncable add-ons. Wipe is a Sync feature not a security
// feature.
for (let guid in this.getAllIDs()) {
let addon = this.getAddonByGUID(guid);
let ids = await this.getAllIDs();
for (let guid in ids) {
let addon = await this.getAddonByGUID(guid);
if (!addon) {
this._log.debug("Ignoring add-on because it couldn't be obtained: " +
guid);
@ -498,7 +496,7 @@ AddonsStore.prototype = {
}
this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
Utils.catch.call(this, () => addon.uninstall())();
await Utils.catch.call(this, () => addon.uninstall())();
}
},
@ -507,29 +505,25 @@ AddonsStore.prototype = {
***************************************************************************/
/**
* Synchronously obtain an add-on from its public ID.
* Obtain an add-on from its public ID.
*
* @param id
* Add-on ID
* @return Addon or undefined if not found
*/
getAddonByID: function getAddonByID(id) {
let cb = Async.makeSyncCallback();
AddonManager.getAddonByID(id, cb);
return Async.waitForSyncCallback(cb);
async getAddonByID(id) {
return AddonManager.getAddonByID(id);
},
/**
* Synchronously obtain an add-on from its Sync GUID.
* Obtain an add-on from its Sync GUID.
*
* @param guid
* Add-on Sync GUID
* @return DBAddonInternal or null
*/
getAddonByGUID: function getAddonByGUID(guid) {
let cb = Async.makeSyncCallback();
AddonManager.getAddonBySyncGUID(guid, cb);
return Async.waitForSyncCallback(cb);
async getAddonByGUID(guid) {
return AddonManager.getAddonBySyncGUID(guid);
},
/**
@ -757,20 +751,17 @@ class AddonValidator extends CollectionValidator {
this.engine = engine;
}
getClientItems() {
return Promise.all([
AddonManager.getAllAddons(),
AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"]),
]).then(([installed, addonsWithPendingOperation]) => {
// Addons pending install won't be in the first list, but addons pending
// uninstall/enable/disable will be in both lists.
let all = new Map(installed.map(addon => [addon.id, addon]));
for (let addon of addonsWithPendingOperation) {
all.set(addon.id, addon);
}
// Convert to an array since Map.prototype.values returns an iterable
return [...all.values()];
});
async getClientItems() {
const installed = await AddonManager.getAllAddons();
const addonsWithPendingOperation = await AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"]);
// Addons pending install won't be in the first list, but addons pending
// uninstall/enable/disable will be in both lists.
let all = new Map(installed.map(addon => [addon.id, addon]));
for (let addon of addonsWithPendingOperation) {
all.set(addon.id, addon);
}
// Convert to an array since Map.prototype.values returns an iterable
return [...all.values()];
}
normalizeClientItem(item) {
@ -790,8 +781,8 @@ class AddonValidator extends CollectionValidator {
};
}
normalizeServerItem(item) {
let guid = this.engine._findDupe(item);
async normalizeServerItem(item) {
let guid = await this.engine._findDupe(item);
if (guid) {
item.id = guid;
}

195
services/sync/modules/engines/bookmarks.js
View File

@ -293,12 +293,9 @@ BookmarksEngine.prototype = {
return new BookmarksChangeset();
},
_guidMapFailed: false,
_buildGUIDMap: function _buildGUIDMap() {
let store = this._store;
async _buildGUIDMap() {
let guidMap = {};
let tree = Async.promiseSpinningly(PlacesUtils.promiseBookmarksTree(""));
let tree = await PlacesUtils.promiseBookmarksTree("");
function* walkBookmarksTree(tree, parent = null) {
if (tree) {
@ -308,7 +305,6 @@ BookmarksEngine.prototype = {
}
if (tree.children) {
for (let child of tree.children) {
store._sleep(0); // avoid jank while looping.
yield* walkBookmarksTree(child, tree);
}
}
@ -323,7 +319,9 @@ BookmarksEngine.prototype = {
}
}
let maybeYield = Async.jankYielder();
for (let [node, parent] of walkBookmarksRoots(tree)) {
await maybeYield();
let {guid, type: placeType} = node;
guid = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
let key;
@ -371,7 +369,7 @@ BookmarksEngine.prototype = {
},
// Helper function to get a dupe GUID for an item.
_mapDupe: function _mapDupe(item) {
async _mapDupe(item) {
// Figure out if we have something to key with.
let key;
let altKey;
@ -402,7 +400,7 @@ BookmarksEngine.prototype = {
// Figure out if we have a map to use!
// This will throw in some circumstances. That's fine.
let guidMap = this._guidMap;
let guidMap = await this.getGuidMap();
// Give the GUID if we have the matching pair.
let parentName = item.parentName || "";
@ -433,52 +431,43 @@ BookmarksEngine.prototype = {
return undefined;
},
_syncStartup: function _syncStart() {
SyncEngine.prototype._syncStartup.call(this);
async _syncStartup() {
await SyncEngine.prototype._syncStartup.call(this);
let cb = Async.makeSpinningCallback();
(async () => {
try {
// For first-syncs, make a backup for the user to restore
if (this.lastSync == 0) {
this._log.debug("Bookmarks backup starting.");
await PlacesBackups.create(null, true);
this._log.debug("Bookmarks backup done.");
}
})().then(
cb, ex => {
// Failure to create a backup is somewhat bad, but probably not bad
// enough to prevent syncing of bookmarks - so just log the error and
// continue.
this._log.warn("Error while backing up bookmarks, but continuing with sync", ex);
cb();
}
);
cb.wait();
this.__defineGetter__("_guidMap", function() {
// Create a mapping of folder titles and separator positions to GUID.
// We do this lazily so that we don't do any work unless we reconcile
// incoming items.
let guidMap;
try {
guidMap = this._buildGUIDMap();
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Error while building GUID map, skipping all other incoming items", ex);
throw {code: Engine.prototype.eEngineAbortApplyIncoming,
cause: ex};
}
delete this._guidMap;
return this._guidMap = guidMap;
});
} catch (ex) {
// Failure to create a backup is somewhat bad, but probably not bad
// enough to prevent syncing of bookmarks - so just log the error and
// continue.
this._log.warn("Error while backing up bookmarks, but continuing with sync", ex);
}
this._store._childrenToOrder = {};
this._store.clearPendingDeletions();
},
async getGuidMap() {
if (this._guidMap) {
return this._guidMap;
}
try {
return this._guidMap = await this._buildGUIDMap();
} catch (ex) {
if (Async.isShutdownException(ex)) {
throw ex;
}
this._log.warn("Error while building GUID map, skipping all other incoming items", ex);
throw {code: Engine.prototype.eEngineAbortApplyIncoming,
cause: ex};
}
},
async _deletePending() {
// Delete pending items -- See the comment above BookmarkStore's deletePending
let newlyModified = await this._store.deletePending();
@ -488,7 +477,7 @@ BookmarksEngine.prototype = {
}
},
_shouldReviveRemotelyDeletedRecord(item) {
async _shouldReviveRemotelyDeletedRecord(item) {
let modifiedTimestamp = this._modified.getModifiedTimestamp(item.id);
if (!modifiedTimestamp) {
// We only expect this to be called with items locally modified, so
@ -501,7 +490,7 @@ BookmarksEngine.prototype = {
// we use `touch` to mark the parent of this record for uploading next sync, in order
// to ensure its children array is accurate. If `touch` returns new change records,
// we revive the item and insert the changes into the current changeset.
let newChanges = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.touch(item.id));
let newChanges = await PlacesSyncUtils.bookmarks.touch(item.id);
if (newChanges) {
this._modified.insert(newChanges);
return true;
@ -509,11 +498,11 @@ BookmarksEngine.prototype = {
return false;
},
_processIncoming(newitems) {
async _processIncoming(newitems) {
try {
SyncEngine.prototype._processIncoming.call(this, newitems);
await SyncEngine.prototype._processIncoming.call(this, newitems);
} finally {
Async.promiseSpinningly(this._postProcessIncoming());
await this._postProcessIncoming();
}
},
@ -531,25 +520,25 @@ BookmarksEngine.prototype = {
this._store._childrenToOrder = {};
},
_syncFinish: function _syncFinish() {
SyncEngine.prototype._syncFinish.call(this);
async _syncFinish() {
await SyncEngine.prototype._syncFinish.call(this);
this._tracker._ensureMobileQuery();
},
_syncCleanup: function _syncCleanup() {
SyncEngine.prototype._syncCleanup.call(this);
async _syncCleanup() {
await SyncEngine.prototype._syncCleanup.call(this);
delete this._guidMap;
},
_createRecord: function _createRecord(id) {
async _createRecord(id) {
if (this._modified.isTombstone(id)) {
// If we already know a changed item is a tombstone, just create the
// record without dipping into Places.
return this._createTombstone(id);
}
// Create the record as usual, but mark it as having dupes if necessary.
let record = SyncEngine.prototype._createRecord.call(this, id);
let entry = this._mapDupe(record);
let record = await SyncEngine.prototype._createRecord.call(this, id);
let entry = await this._mapDupe(record);
if (entry != null && entry.hasDupe) {
record.hasDupe = true;
}
@ -562,7 +551,7 @@ BookmarksEngine.prototype = {
return record;
},
buildWeakReuploadMap(idSet) {
async buildWeakReuploadMap(idSet) {
// We want to avoid uploading records which have changed, since that could
// cause an inconsistent state on the server.
//
@ -576,20 +565,20 @@ BookmarksEngine.prototype = {
// building the weakReuploadMap (which is where the calls to createRecord()
// occur) as an optimization, and once after for correctness, to handle the
// unlikely case that a record was modified while we were building the map.
let initialChanges = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.getChangedIds());
let initialChanges = await PlacesSyncUtils.bookmarks.getChangedIds();
for (let changed of initialChanges) {
idSet.delete(changed);
}
let map = SyncEngine.prototype.buildWeakReuploadMap.call(this, idSet);
let changes = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.getChangedIds());
let map = await SyncEngine.prototype.buildWeakReuploadMap.call(this, idSet);
let changes = await PlacesSyncUtils.bookmarks.getChangedIds();
for (let id of changes) {
map.delete(id);
}
return map;
},
_findDupe: function _findDupe(item) {
async _findDupe(item) {
this._log.trace("Finding dupe for " + item.id +
" (already duped: " + item.hasDupe + ").");
@ -598,40 +587,41 @@ BookmarksEngine.prototype = {
this._log.trace(item.id + " already a dupe: not finding one.");
return null;
}
let mapped = this._mapDupe(item);
let mapped = await this._mapDupe(item);
this._log.debug(item.id + " mapped to " + mapped);
// We must return a string, not an object, and the entries in the GUIDMap
// are created via "new String()" making them an object.
return mapped ? mapped.toString() : mapped;
},
pullAllChanges() {
async pullAllChanges() {
return this.pullNewChanges();
},
pullNewChanges() {
return Async.promiseSpinningly(this._tracker.promiseChangedIDs());
async pullNewChanges() {
return this._tracker.promiseChangedIDs();
},
trackRemainingChanges() {
async trackRemainingChanges() {
let changes = this._modified.changes;
Async.promiseSpinningly(PlacesSyncUtils.bookmarks.pushChanges(changes));
await PlacesSyncUtils.bookmarks.pushChanges(changes);
},
_deleteId(id) {
this._noteDeletedId(id);
},
_resetClient() {
SyncEngine.prototype._resetClient.call(this);
Async.promiseSpinningly(PlacesSyncUtils.bookmarks.reset());
async _resetClient() {
await SyncEngine.prototype._resetClient.call(this);
await PlacesSyncUtils.bookmarks.reset();
},
// Called when _findDupe returns a dupe item and the engine has decided to
// switch the existing item to the new incoming item.
_switchItemToDupe(localDupeGUID, incomingItem) {
let newChanges = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.dedupe(
localDupeGUID, incomingItem.id, incomingItem.parentid));
async _switchItemToDupe(localDupeGUID, incomingItem) {
let newChanges = await PlacesSyncUtils.bookmarks.dedupe(localDupeGUID,
incomingItem.id,
incomingItem.parentid);
this._modified.insert(newChanges);
},
@ -677,11 +667,11 @@ function BookmarksStore(name, engine) {
BookmarksStore.prototype = {
__proto__: Store.prototype,
itemExists: function BStore_itemExists(id) {
return this.idForGUID(id) > 0;
async itemExists(id) {
return (await this.idForGUID(id)) > 0;
},
applyIncoming: function BStore_applyIncoming(record) {
async applyIncoming(record) {
this._log.debug("Applying record " + record.id);
let isSpecial = PlacesSyncUtils.bookmarks.ROOTS.includes(record.id);
@ -692,7 +682,7 @@ BookmarksStore.prototype = {
}
// Don't bother with pre and post-processing for deletions.
Store.prototype.applyIncoming.call(this, record);
await Store.prototype.applyIncoming.call(this, record);
return;
}
@ -719,19 +709,19 @@ BookmarksStore.prototype = {
this._log.debug("Remote parent is " + parentGUID);
// Do the normal processing of incoming records
Store.prototype.applyIncoming.call(this, record);
await Store.prototype.applyIncoming.call(this, record);
if (record.type == "folder" && record.children) {
this._childrenToOrder[record.id] = record.children;
}
},
create: function BStore_create(record) {
async create(record) {
let info = record.toSyncBookmark();
// This can throw if we're inserting an invalid or incomplete bookmark.
// That's fine; the exception will be caught by `applyIncomingBatch`
// without aborting further processing.
let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.insert(info));
let item = await PlacesSyncUtils.bookmarks.insert(info);
if (item) {
this._log.trace(`Created ${item.kind} ${item.syncId} under ${
item.parentSyncId}`, item);
@ -741,14 +731,14 @@ BookmarksStore.prototype = {
}
},
remove: function BStore_remove(record) {
async remove(record) {
this._log.trace(`Buffering removal of item "${record.id}".`);
this._itemsToDelete.add(record.id);
},
update: function BStore_update(record) {
async update(record) {
let info = record.toSyncBookmark();
let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.update(info));
let item = await PlacesSyncUtils.bookmarks.update(info);
if (item) {
this._log.trace(`Updated ${item.kind} ${item.syncId} under ${
item.parentSyncId}`, item);
@ -812,8 +802,8 @@ BookmarksStore.prototype = {
},
// Create a record starting from the weave id (places guid)
createRecord: function createRecord(id, collection) {
let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.fetch(id));
async createRecord(id, collection) {
let item = await PlacesSyncUtils.bookmarks.fetch(id);
if (!item) { // deleted item
let record = new PlacesItem(collection, id);
record.deleted = true;
@ -828,26 +818,29 @@ BookmarksStore.prototype = {
let record = new recordObj(collection, id);
record.fromSyncBookmark(item);
record.sortindex = this._calculateIndex(record);
record.sortindex = await this._calculateIndex(record);
return record;
},
GUIDForId: function GUIDForId(id) {
let guid = Async.promiseSpinningly(PlacesUtils.promiseItemGuid(id));
async GUIDForId(id) {
let guid = await PlacesUtils.promiseItemGuid(id);
return PlacesSyncUtils.bookmarks.guidToSyncId(guid);
},
idForGUID: function idForGUID(guid) {
async idForGUID(guid) {
// guid might be a String object rather than a string.
guid = PlacesSyncUtils.bookmarks.syncIdToGuid(guid.toString());
return Async.promiseSpinningly(PlacesUtils.promiseItemId(guid).catch(
ex => -1));
try {
return await PlacesUtils.promiseItemId(guid);
} catch (ex) {
return -1;
}
},
_calculateIndex: function _calculateIndex(record) {
async _calculateIndex(record) {
// Ensure folders have a very high sort index so they're not synced last.
if (record.type == "folder")
return FOLDER_SORTINDEX;
@ -860,7 +853,7 @@ BookmarksStore.prototype = {
// Add in the bookmark's frecency if we have something.
if (record.bmkUri != null) {
let frecency = Async.promiseSpinningly(PlacesSyncUtils.history.fetchURLFrecency(record.bmkUri));
let frecency = await PlacesSyncUtils.history.fetchURLFrecency(record.bmkUri);
if (frecency != -1)
index += frecency;
}
@ -868,13 +861,11 @@ BookmarksStore.prototype = {
return index;
},
wipe: function BStore_wipe() {
async wipe() {
this.clearPendingDeletions();
Async.promiseSpinningly((async () => {
// Save a backup before clearing out all bookmarks.
await PlacesBackups.create(null, true);
await PlacesSyncUtils.bookmarks.wipe();
})());
// Save a backup before clearing out all bookmarks.
await PlacesBackups.create(null, true);
await PlacesSyncUtils.bookmarks.wipe();
}
};
@ -1007,10 +998,12 @@ BookmarksTracker.prototype = {
this._log.debug("Tracking all items on successful import.");
this._log.debug("Restore succeeded: wiping server and other clients.");
this.engine.service.resetClient([this.name]);
this.engine.service.wipeServer([this.name]);
this.engine.service.clientsEngine.sendCommand("wipeEngine", [this.name],
null, { reason: "bookmark-restore" });
Async.promiseSpinningly((async () => {
await this.engine.service.resetClient([this.name]);
await this.engine.service.wipeServer([this.name]);
await this.engine.service.clientsEngine.sendCommand("wipeEngine", [this.name],
null, { reason: "bookmark-restore" });
})());
break;
case "bookmarks-restore-failed":
this._log.debug("Tracking all items on failed import.");

198
services/sync/modules/engines/clients.js
View File

@ -225,60 +225,57 @@ ClientEngine.prototype = {
return false;
},
_readCommands() {
let cb = Async.makeSpinningCallback();
Utils.jsonLoad("commands", this, commands => cb(null, commands));
return cb.wait() || {};
async _readCommands() {
let commands = await Utils.jsonLoad("commands", this);
return commands || {};
},
/**
* Low level function, do not use directly (use _addClientCommand instead).
*/
_saveCommands(commands) {
let cb = Async.makeSpinningCallback();
Utils.jsonSave("commands", this, commands, error => {
if (error) {
this._log.error("Failed to save JSON outgoing commands", error);
}
cb();
});
cb.wait();
async _saveCommands(commands) {
try {
await Utils.jsonSave("commands", this, commands);
} catch (error) {
this._log.error("Failed to save JSON outgoing commands", error);
}
},
_prepareCommandsForUpload() {
let cb = Async.makeSpinningCallback();
Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors
.then(() => {
Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands));
});
return cb.wait() || {};
async _prepareCommandsForUpload() {
try {
await Utils.jsonMove("commands", "commands-syncing", this)
} catch (e) {
// Ignore errors
}
let commands = await Utils.jsonLoad("commands-syncing", this);
return commands || {};
},
_deleteUploadedCommands() {
async _deleteUploadedCommands() {
delete this._currentlySyncingCommands;
Async.promiseSpinningly(
Utils.jsonRemove("commands-syncing", this).catch(err => {
this._log.error("Failed to delete syncing-commands file", err);
})
);
try {
await Utils.jsonRemove("commands-syncing", this);
} catch (err) {
this._log.error("Failed to delete syncing-commands file", err);
}
},
// Gets commands for a client we are yet to write to the server. Doesn't
// include commands for that client which are already on the server.
// We should rename this!
getClientCommands(clientId) {
const allCommands = this._readCommands();
async getClientCommands(clientId) {
const allCommands = await this._readCommands();
return allCommands[clientId] || [];
},
removeLocalCommand(command) {
async removeLocalCommand(command) {
// the implementation of this engine is such that adding a command to
// the local client is how commands are deleted! ¯\_(ツ)_/¯
this._addClientCommand(this.localID, command);
await this._addClientCommand(this.localID, command);
},
_addClientCommand(clientId, command) {
const localCommands = this._readCommands();
async _addClientCommand(clientId, command) {
const localCommands = await this._readCommands();
const localClientCommands = localCommands[clientId] || [];
const remoteClient = this._store._remoteClients[clientId];
let remoteClientCommands = []
@ -290,19 +287,19 @@ ClientEngine.prototype = {
return false;
}
localCommands[clientId] = localClientCommands.concat(command);
this._saveCommands(localCommands);
await this._saveCommands(localCommands);
return true;
},
_removeClientCommands(clientId) {
const allCommands = this._readCommands();
async _removeClientCommands(clientId) {
const allCommands = await this._readCommands();
delete allCommands[clientId];
this._saveCommands(allCommands);
await this._saveCommands(allCommands);
},
updateKnownStaleClients() {
async updateKnownStaleClients() {
this._log.debug("Updating the known stale clients");
this._refreshKnownStaleClients();
await this._refreshKnownStaleClients();
for (let client of Object.values(this._store._remoteClients)) {
if (client.fxaDeviceId && this._knownStaleFxADeviceIds.includes(client.fxaDeviceId)) {
this._log.info(`Hiding stale client ${client.id} - in known stale clients list`);
@ -313,14 +310,15 @@ ClientEngine.prototype = {
// We assume that clients not present in the FxA Device Manager list have been
// disconnected and so are stale
_refreshKnownStaleClients() {
async _refreshKnownStaleClients() {
this._log.debug("Refreshing the known stale clients list");
let localClients = Object.values(this._store._remoteClients)
.filter(client => client.fxaDeviceId) // iOS client records don't have fxaDeviceId
.map(client => client.fxaDeviceId);
let fxaClients;
try {
fxaClients = Async.promiseSpinningly(this.fxAccounts.getDeviceList()).map(device => device.id);
let deviceList = await this.fxAccounts.getDeviceList();
fxaClients = deviceList.map(device => device.id);
} catch (ex) {
this._log.error("Could not retrieve the FxA device list", ex);
this._knownStaleFxADeviceIds = [];
@ -329,26 +327,26 @@ ClientEngine.prototype = {
this._knownStaleFxADeviceIds = Utils.arraySub(localClients, fxaClients);
},
_syncStartup() {
async _syncStartup() {
this.isFirstSync = !this.lastRecordUpload;
// Reupload new client record periodically.
if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
this._tracker.addChangedID(this.localID);
this.lastRecordUpload = Date.now() / 1000;
}
SyncEngine.prototype._syncStartup.call(this);
return SyncEngine.prototype._syncStartup.call(this);
},
_processIncoming() {
async _processIncoming() {
// Fetch all records from the server.
this.lastSync = 0;
this._incomingClients = {};
try {
SyncEngine.prototype._processIncoming.call(this);
await SyncEngine.prototype._processIncoming.call(this);
// Refresh the known stale clients list at startup and when we receive
// "device connected/disconnected" push notifications.
if (!this._knownStaleFxADeviceIds) {
this._refreshKnownStaleClients();
await this._refreshKnownStaleClients();
}
// Since clients are synced unconditionally, any records in the local store
// that don't exist on the server must be for disconnected clients. Remove
@ -358,7 +356,7 @@ ClientEngine.prototype = {
for (let id in this._store._remoteClients) {
if (!this._incomingClients[id]) {
this._log.info(`Removing local state for deleted client ${id}`);
this._removeRemoteClient(id);
await this._removeRemoteClient(id);
}
}
// Bug 1264498: Mobile clients don't remove themselves from the clients
@ -391,8 +389,8 @@ ClientEngine.prototype = {
}
},
_uploadOutgoing() {
this._currentlySyncingCommands = this._prepareCommandsForUpload();
async _uploadOutgoing() {
this._currentlySyncingCommands = await this._prepareCommandsForUpload();
const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands);
for (let clientId of clientWithPendingCommands) {
if (this._store._remoteClients[clientId] || this.localID == clientId) {
@ -400,7 +398,7 @@ ClientEngine.prototype = {
}
}
let updatedIDs = this._modified.ids();
SyncEngine.prototype._uploadOutgoing.call(this);
await SyncEngine.prototype._uploadOutgoing.call(this);
// Record the response time as the server time for each item we uploaded.
for (let id of updatedIDs) {
if (id != this.localID) {
@ -409,7 +407,7 @@ ClientEngine.prototype = {
}
},
_onRecordsWritten(succeeded, failed) {
async _onRecordsWritten(succeeded, failed) {
// Reconcile the status of the local records with what we just wrote on the
// server
for (let id of succeeded) {
@ -430,7 +428,7 @@ ClientEngine.prototype = {
continue;
}
// fixup the client record, so our copy of _remoteClients matches what we uploaded.
this._store._remoteClients[id] = this._store.createRecord(id);
this._store._remoteClients[id] = await this._store.createRecord(id);
// we could do better and pass the reference to the record we just uploaded,
// but this will do for now
}
@ -442,10 +440,10 @@ ClientEngine.prototype = {
if (!commandChanges) {
continue;
}
this._addClientCommand(id, commandChanges);
await this._addClientCommand(id, commandChanges);
}
this._deleteUploadedCommands();
await this._deleteUploadedCommands();
// Notify other devices that their own client collection changed
const idsToNotify = succeeded.reduce((acc, id) => {
@ -480,7 +478,7 @@ ClientEngine.prototype = {
}
},
_syncFinish() {
async _syncFinish() {
// Record histograms for our device types, and also write them to a pref
// so non-histogram telemetry (eg, UITelemetry) and the sync scheduler
// has easy access to them, and so they are accurate even before we've
@ -504,15 +502,15 @@ ClientEngine.prototype = {
Services.telemetry.getHistogramById(hid).add(count);
Svc.Prefs.set(prefName, count);
}
SyncEngine.prototype._syncFinish.call(this);
return SyncEngine.prototype._syncFinish.call(this);
},
_reconcile: function _reconcile(item) {
async _reconcile(item) {
// Every incoming record is reconciled, so we use this to track the
// contents of the collection on the server.
this._incomingClients[item.id] = item.modified;
if (!this._store.itemExists(item.id)) {
if (!(await this._store.itemExists(item.id))) {
return true;
}
// Clients are synced unconditionally, so we'll always have new records.
@ -521,25 +519,30 @@ ClientEngine.prototype = {
// work around this by updating the record during reconciliation, and
// returning false to indicate that the record doesn't need to be applied
// later.
this._store.update(item);
await this._store.update(item);
return false;
},
// Treat reset the same as wiping for locally cached clients
_resetClient() {
this._wipeClient();
async _resetClient() {
await this._wipeClient();
},
_wipeClient: function _wipeClient() {
SyncEngine.prototype._resetClient.call(this);
async _wipeClient() {
await SyncEngine.prototype._resetClient.call(this);
this._knownStaleFxADeviceIds = null;
delete this.localCommands;
this._store.wipe();
const logRemoveError = err => this._log.warn("Could not delete json file", err);
Async.promiseSpinningly(
Utils.jsonRemove("commands", this).catch(logRemoveError)
.then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError))
);
await this._store.wipe();
try {
await Utils.jsonRemove("commands", this);
} catch (err) {
this._log.warn("Could not delete commands.json", err);
}
try {
await Utils.jsonRemove("commands-syncing", this)
} catch (err) {
this._log.warn("Could not delete commands-syncing.json", err);
}
},
async removeClientData() {
@ -548,10 +551,10 @@ ClientEngine.prototype = {
},
// Override the default behavior to delete bad records from the server.
handleHMACMismatch: function handleHMACMismatch(item, mayRetry) {
async handleHMACMismatch(item, mayRetry) {
this._log.debug("Handling HMAC mismatch for " + item.id);
let base = SyncEngine.prototype.handleHMACMismatch.call(this, item, mayRetry);
let base = await SyncEngine.prototype.handleHMACMismatch.call(this, item, mayRetry);
if (base != SyncEngine.kRecoveryStrategy.error)
return base;
@ -586,7 +589,7 @@ ClientEngine.prototype = {
* @param args Array of arguments/data for command
* @param clientId Client to send command to
*/
_sendCommandToClient(command, args, clientId, telemetryExtra) {
async _sendCommandToClient(command, args, clientId, telemetryExtra) {
this._log.trace("Sending " + command + " to " + clientId);
let client = this._store._remoteClients[clientId];
@ -605,7 +608,7 @@ ClientEngine.prototype = {
flowID: telemetryExtra.flowID,
};
if (this._addClientCommand(clientId, action)) {
if ((await this._addClientCommand(clientId, action))) {
this._log.trace(`Client ${clientId} got a new action`, [command, args]);
this._tracker.addChangedID(clientId);
try {
@ -623,13 +626,13 @@ ClientEngine.prototype = {
*
* @return false to abort sync
*/
processIncomingCommands: function processIncomingCommands() {
return this._notify("clients:process-commands", "", function() {
async processIncomingCommands() {
return this._notify("clients:process-commands", "", async function() {
if (!this.localCommands) {
return true;
}
const clearedCommands = this._readCommands()[this.localID];
const clearedCommands = await this._readCommands()[this.localID];
const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command));
let didRemoveCommand = false;
let URIsToDisplay = [];
@ -648,13 +651,13 @@ ClientEngine.prototype = {
engines = null;
// Fallthrough
case "resetEngine":
this.service.resetClient(engines);
await this.service.resetClient(engines);
break;
case "wipeAll":
engines = null;
// Fallthrough
case "wipeEngine":
this.service.wipeClient(engines);
await this.service.wipeClient(engines);
break;
case "logout":
this.service.logout();
@ -672,7 +675,7 @@ ClientEngine.prototype = {
this._log.warn("repairResponse for unknown collection", response);
break;
}
if (!requestor.continueRepairs(response)) {
if (!(await requestor.continueRepairs(response))) {
this._log.warn("repairResponse couldn't continue the repair", response);
}
break;
@ -686,7 +689,7 @@ ClientEngine.prototype = {
break;
}
try {
if (Async.promiseSpinningly(responder.repair(request, rawCommand))) {
if ((await responder.repair(request, rawCommand))) {
// We've started a repair - once that collection has synced it
// will write a "response" command and arrange for this repair
// request to be removed from the local command list - if we
@ -715,7 +718,7 @@ ClientEngine.prototype = {
}
// Add the command to the "cleared" commands list
if (shouldRemoveCommand) {
this.removeLocalCommand(rawCommand);
await this.removeLocalCommand(rawCommand);
didRemoveCommand = true;
}
}
@ -737,6 +740,7 @@ ClientEngine.prototype = {
* Calling this does not actually sync the command data to the server. If the
* client already has the command/args pair, it won't receive a duplicate
* command.
* This method is async since it writes the command to a file.
*
* @param command
* Command to invoke on remote clients
@ -749,7 +753,7 @@ ClientEngine.prototype = {
* A unique identifier used to track success for this operation across
* devices.
*/
sendCommand(command, args, clientId = null, telemetryExtra = {}) {
async sendCommand(command, args, clientId = null, telemetryExtra = {}) {
let commandData = this._commands[command];
// Don't send commands that we don't know about.
if (!commandData) {
@ -769,11 +773,11 @@ ClientEngine.prototype = {
}
if (clientId) {
this._sendCommandToClient(command, args, clientId, telemetryExtra);
await this._sendCommandToClient(command, args, clientId, telemetryExtra);
} else {
for (let [id, record] of Object.entries(this._store._remoteClients)) {
if (!record.stale) {
this._sendCommandToClient(command, args, id, telemetryExtra);
await this._sendCommandToClient(command, args, id, telemetryExtra);
}
}
}
@ -796,10 +800,10 @@ ClientEngine.prototype = {
* @param title
* Title of the page being sent.
*/
sendURIToClientForDisplay: function sendURIToClientForDisplay(uri, clientId, title) {
async sendURIToClientForDisplay(uri, clientId, title) {
this._log.info("Sending URI to client: " + uri + " -> " +
clientId + " (" + title + ")");
this.sendCommand("displayURI", [uri, this.localID, title], clientId);
await this.sendCommand("displayURI", [uri, this.localID, title], clientId);
this._tracker.score += SCORE_INCREMENT_XLARGE;
},
@ -831,10 +835,10 @@ ClientEngine.prototype = {
Svc.Obs.notify("weave:engine:clients:display-uris", uris);
},
_removeRemoteClient(id) {
async _removeRemoteClient(id) {
delete this._store._remoteClients[id];
this._tracker.removeChangedID(id);
this._removeClientCommands(id);
await this._removeClientCommands(id);
this._modified.delete(id);
},
};
@ -847,11 +851,11 @@ ClientStore.prototype = {
_remoteClients: {},
create(record) {
this.update(record);
async create(record) {
await this.update(record);
},
update: function update(record) {
async update(record) {
if (record.id == this.engine.localID) {
// Only grab commands from the server; local name/type always wins
this.engine.localCommands = record.commands;
@ -860,7 +864,7 @@ ClientStore.prototype = {
}
},
createRecord: function createRecord(id, collection) {
async createRecord(id, collection) {
let record = new ClientsRec(collection, id);
const commandsChanges = this.engine._currentlySyncingCommands ?
@ -869,10 +873,8 @@ ClientStore.prototype = {
// Package the individual components into a record for the local client
if (id == this.engine.localID) {
let cb = Async.makeSpinningCallback();
this.engine.fxAccounts.getDeviceId().then(id => cb(null, id), cb);
try {
record.fxaDeviceId = cb.wait();
record.fxaDeviceId = await this.engine.fxAccounts.getDeviceId();
} catch (error) {
this._log.warn("failed to get fxa device id", error);
}
@ -917,11 +919,11 @@ ClientStore.prototype = {
return record;
},
itemExists(id) {
return id in this.getAllIDs();
async itemExists(id) {
return id in (await this.getAllIDs());
},
getAllIDs: function getAllIDs() {
async getAllIDs() {
let ids = {};
ids[this.engine.localID] = true;
for (let id in this._remoteClients)
@ -929,7 +931,7 @@ ClientStore.prototype = {
return ids;
},
wipe: function wipe() {
async wipe() {
this._remoteClients = {};
},
};

5
services/sync/modules/engines/extension-storage.js
View File

@ -12,7 +12,6 @@ Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/async.js");
XPCOMUtils.defineLazyModuleGetter(this, "extensionStorageSync",
"resource://gre/modules/ExtensionStorageSync.jsm");
@ -37,8 +36,8 @@ ExtensionStorageEngine.prototype = {
syncPriority: 10,
allowSkippedRecord: false,
_sync() {
return Async.promiseSpinningly(extensionStorageSync.syncAll());
async _sync() {
return extensionStorageSync.syncAll();
},
get enabled() {

95
services/sync/modules/engines/forms.js
View File

@ -11,7 +11,6 @@ var Cu = Components.utils;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/collection_validator.js");
@ -39,7 +38,7 @@ var FormWrapper = {
_getEntryCols: ["fieldname", "value"],
_guidCols: ["guid"],
_promiseSearch(terms, searchData) {
async _search(terms, searchData) {
return new Promise(resolve => {
let results = [];
let callbacks = {
@ -54,52 +53,48 @@ var FormWrapper = {
})
},
// Do a "sync" search by spinning the event loop until it completes.
_searchSpinningly(terms, searchData) {
return Async.promiseSpinningly(this._promiseSearch(terms, searchData));
},
_updateSpinningly(changes) {
async _update(changes) {
if (!FormHistory.enabled) {
return; // update isn't going to do anything.
}
let cb = Async.makeSpinningCallback();
let callbacks = {
handleCompletion(reason) {
cb();
}
};
FormHistory.update(changes, callbacks);
cb.wait();
await new Promise(resolve => {
let callbacks = {
handleCompletion(reason) {
resolve();
}
};
FormHistory.update(changes, callbacks);
});
},
getEntry(guid) {
let results = this._searchSpinningly(this._getEntryCols, {guid});
async getEntry(guid) {
let results = await this._search(this._getEntryCols, {guid});
if (!results.length) {
return null;
}
return {name: results[0].fieldname, value: results[0].value};
},
getGUID(name, value) {
async getGUID(name, value) {
// Query for the provided entry.
let query = { fieldname: name, value };
let results = this._searchSpinningly(this._guidCols, query);
let results = await this._search(this._guidCols, query);
return results.length ? results[0].guid : null;
},
hasGUID(guid) {
// We could probably use a count function here, but searchSpinningly exists...
return this._searchSpinningly(this._guidCols, {guid}).length != 0;
async hasGUID(guid) {
// We could probably use a count function here, but search exists...
let results = await this._search(this._guidCols, {guid});
return results.length != 0;
},
replaceGUID(oldGUID, newGUID) {
async replaceGUID(oldGUID, newGUID) {
let changes = {
op: "update",
guid: oldGUID,
newGuid: newGUID,
}
this._updateSpinningly(changes);
await this._update(changes);
}
};
@ -120,7 +115,7 @@ FormEngine.prototype = {
return "history";
},
_findDupe: function _findDupe(item) {
async _findDupe(item) {
return FormWrapper.getGUID(item.name, item.value);
}
};
@ -131,7 +126,7 @@ function FormStore(name, engine) {
FormStore.prototype = {
__proto__: Store.prototype,
_processChange(change) {
async _processChange(change) {
// If this._changes is defined, then we are applying a batch, so we
// can defer it.
if (this._changes) {
@ -139,23 +134,23 @@ FormStore.prototype = {
return;
}
// Otherwise we must handle the change synchronously, right now.
FormWrapper._updateSpinningly(change);
// Otherwise we must handle the change right now.
await FormWrapper._update(change);
},
applyIncomingBatch(records) {
async applyIncomingBatch(records) {
// We collect all the changes to be made then apply them all at once.
this._changes = [];
let failures = Store.prototype.applyIncomingBatch.call(this, records);
let failures = await Store.prototype.applyIncomingBatch.call(this, records);
if (this._changes.length) {
FormWrapper._updateSpinningly(this._changes);
await FormWrapper._update(this._changes);
}
delete this._changes;
return failures;
},
getAllIDs() {
let results = FormWrapper._searchSpinningly(["guid"], [])
async getAllIDs() {
let results = await FormWrapper._search(["guid"], [])
let guids = {};
for (let result of results) {
guids[result.guid] = true;
@ -163,17 +158,17 @@ FormStore.prototype = {
return guids;
},
changeItemID(oldID, newID) {
FormWrapper.replaceGUID(oldID, newID);
async changeItemID(oldID, newID) {
await FormWrapper.replaceGUID(oldID, newID);
},
itemExists(id) {
async itemExists(id) {
return FormWrapper.hasGUID(id);
},
createRecord(id, collection) {
async createRecord(id, collection) {
let record = new FormRec(collection, id);
let entry = FormWrapper.getEntry(id);
let entry = await FormWrapper.getEntry(id);
if (entry != null) {
record.name = entry.name;
record.value = entry.value;
@ -183,34 +178,34 @@ FormStore.prototype = {
return record;
},
create(record) {
async create(record) {
this._log.trace("Adding form record for " + record.name);
let change = {
op: "add",
fieldname: record.name,
value: record.value
};
this._processChange(change);
await this._processChange(change);
},
remove(record) {
async remove(record) {
this._log.trace("Removing form record: " + record.id);
let change = {
op: "remove",
guid: record.id
};
this._processChange(change);
await this._processChange(change);
},
update(record) {
async update(record) {
this._log.trace("Ignoring form record update request!");
},
wipe() {
async wipe() {
let change = {
op: "remove"
};
FormWrapper._updateSpinningly(change);
await FormWrapper._update(change);
}
};
@ -273,8 +268,8 @@ class FormValidator extends CollectionValidator {
return new FormsProblemData();
}
getClientItems() {
return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {});
async getClientItems() {
return FormWrapper._search(["guid", "fieldname", "value"], {});
}
normalizeClientItem(item) {
@ -288,7 +283,7 @@ class FormValidator extends CollectionValidator {
};
}
normalizeServerItem(item) {
async normalizeServerItem(item) {
let res = Object.assign({
guid: item.id,
fieldname: item.name,
@ -296,7 +291,7 @@ class FormValidator extends CollectionValidator {
}, item);
// Missing `name` or `value` causes the getGUID call to throw
if (item.name !== undefined && item.value !== undefined) {
let guid = FormWrapper.getGUID(item.name, item.value);
let guid = await FormWrapper.getGUID(item.name, item.value);
if (guid) {
res.guid = guid;
res.id = guid;

57
services/sync/modules/engines/history.js
View File

@ -46,7 +46,7 @@ HistoryEngine.prototype = {
syncPriority: 7,
_processIncoming(newitems) {
async _processIncoming(newitems) {
// We want to notify history observers that a batch operation is underway
// so they don't do lots of work for each incoming record.
let observers = PlacesUtils.history.getObservers();
@ -59,13 +59,13 @@ HistoryEngine.prototype = {
}
notifyHistoryObservers("onBeginUpdateBatch");
try {
return SyncEngine.prototype._processIncoming.call(this, newitems);
await SyncEngine.prototype._processIncoming.call(this, newitems);
} finally {
notifyHistoryObservers("onEndUpdateBatch");
}
},
pullNewChanges() {
async pullNewChanges() {
let modifiedGUIDs = Object.keys(this._tracker.changedIDs);
if (!modifiedGUIDs.length) {
return {};
@ -236,12 +236,12 @@ HistoryStore.prototype = {
return Async.querySpinningly(this._urlStm, this._urlCols)[0];
},
changeItemID: function HStore_changeItemID(oldID, newID) {
async changeItemID(oldID, newID) {
this.setGUID(this._findURLByGUID(oldID).url, newID);
},
getAllIDs: function HistStore_getAllIDs() {
async getAllIDs() {
// Only get places visited within the last 30 days (30*24*60*60*1000ms)
this._allUrlStm.params.cutoff_date = (Date.now() - 2592000000) * 1000;
this._allUrlStm.params.max_results = MAX_HISTORY_UPLOAD;
@ -254,7 +254,7 @@ HistoryStore.prototype = {
}, {});
},
applyIncomingBatch: function applyIncomingBatch(records) {
async applyIncomingBatch(records) {
let failed = [];
let blockers = [];
@ -290,8 +290,6 @@ HistoryStore.prototype = {
}
records.length = k; // truncate array
let handleAsyncOperationsComplete = Async.makeSyncCallback();
if (records.length) {
blockers.push(new Promise(resolve => {
let updatePlacesCallback = {
@ -305,20 +303,9 @@ HistoryStore.prototype = {
}));
}
// Since `failed` is updated asynchronously and this function is
// synchronous, we need to spin-wait until we are sure that all
// updates to `fail` have completed.
Promise.all(blockers).then(
handleAsyncOperationsComplete,
ex => {
// In case of error, terminate wait, but make sure that the
// error is reported nevertheless and still causes test
// failures.
handleAsyncOperationsComplete();
throw ex;
});
Async.waitForSyncCallback(handleAsyncOperationsComplete);
return failed;
// failed is updated asynchronously, hence the await on blockers.
await Promise.all(blockers);
return failed;
},
/**
@ -404,23 +391,21 @@ HistoryStore.prototype = {
return true;
},
remove: function HistStore_remove(record) {
async remove(record) {
this._log.trace("Removing page: " + record.id);
return PlacesUtils.history.remove(record.id).then(
(removed) => {
if (removed) {
this._log.trace("Removed page: " + record.id);
} else {
this._log.debug("Page already removed: " + record.id);
}
});
let removed = await PlacesUtils.history.remove(record.id);
if (removed) {
this._log.trace("Removed page: " + record.id);
} else {
this._log.debug("Page already removed: " + record.id);
}
},
itemExists: function HistStore_itemExists(id) {
async itemExists(id) {
return !!this._findURLByGUID(id);
},
createRecord: function createRecord(id, collection) {
async createRecord(id, collection) {
let foo = this._findURLByGUID(id);
let record = new HistoryRec(collection, id);
if (foo) {
@ -435,10 +420,8 @@ HistoryStore.prototype = {
return record;
},
wipe: function HistStore_wipe() {
let cb = Async.makeSyncCallback();
PlacesUtils.history.clear().then(result => { cb(null, result) }, err => { cb(err) });
return Async.waitForSyncCallback(cb);
async wipe() {
return PlacesUtils.history.clear();
}
};

49
services/sync/modules/engines/passwords.js
View File

@ -73,8 +73,8 @@ PasswordEngine.prototype = {
syncPriority: 2,
_syncFinish() {
SyncEngine.prototype._syncFinish.call(this);
async _syncFinish() {
await SyncEngine.prototype._syncFinish.call(this);
// Delete the Weave credentials from the server once.
if (!Svc.Prefs.get("deletePwdFxA", false)) {
@ -88,7 +88,7 @@ PasswordEngine.prototype = {
if (ids.length) {
let coll = new Collection(this.engineURL, null, this.service);
coll.ids = ids;
let ret = coll.delete();
let ret = await coll.delete();
this._log.debug("Delete result: " + ret);
if (!ret.success && ret.status != 400) {
// A non-400 failure means try again next time.
@ -110,7 +110,7 @@ PasswordEngine.prototype = {
}
},
_findDupe(item) {
async _findDupe(item) {
let login = this._store._nsLoginInfoFromRecord(item);
if (!login) {
return null;
@ -118,7 +118,7 @@ PasswordEngine.prototype = {
let logins = Services.logins.findLogins({}, login.hostname, login.formSubmitURL, login.httpRealm);
this._store._sleep(0); // Yield back to main thread after synchronous operation.
await Async.promiseYield(); // Yield back to main thread after synchronous operation.
// Look for existing logins that match the hostname, but ignore the password.
for (let local of logins) {
@ -130,9 +130,10 @@ PasswordEngine.prototype = {
return null;
},
pullAllChanges() {
async pullAllChanges() {
let changes = {};
for (let [id, info] of Object.entries(this._store.getAllIDs())) {
let ids = await this._store.getAllIDs();
for (let [id, info] of Object.entries(ids)) {
changes[id] = info.timePasswordChanged / 1000;
}
return changes;
@ -186,12 +187,12 @@ PasswordStore.prototype = {
return info;
},
_getLoginFromGUID(id) {
async _getLoginFromGUID(id) {
let prop = this._newPropertyBag();
prop.setPropertyAsAUTF8String("guid", id);
let logins = Services.logins.searchLogins({}, prop);
this._sleep(0); // Yield back to main thread after synchronous operation.
await Async.promiseYield(); // Yield back to main thread after synchronous operation.
if (logins.length > 0) {
this._log.trace(logins.length + " items matching " + id + " found.");
@ -202,7 +203,7 @@ PasswordStore.prototype = {
return null;
},
getAllIDs() {
async getAllIDs() {
let items = {};
let logins = Services.logins.getAllLogins({});
@ -219,15 +220,15 @@ PasswordStore.prototype = {
return items;
},
changeItemID(oldID, newID) {
async changeItemID(oldID, newID) {
this._log.trace("Changing item ID: " + oldID + " to " + newID);
let oldLogin = this._getLoginFromGUID(oldID);
let oldLogin = await this._getLoginFromGUID(oldID);
if (!oldLogin) {
this._log.trace("Can't change item ID: item doesn't exist");
return;
}
if (this._getLoginFromGUID(newID)) {
if ((await this._getLoginFromGUID(newID))) {
this._log.trace("Can't change item ID: new ID already in use");
return;
}
@ -238,13 +239,13 @@ PasswordStore.prototype = {
Services.logins.modifyLogin(oldLogin, prop);
},
itemExists(id) {
return !!this._getLoginFromGUID(id);
async itemExists(id) {
return !!(await this._getLoginFromGUID(id));
},
createRecord(id, collection) {
async createRecord(id, collection) {
let record = new LoginRec(collection, id);
let login = this._getLoginFromGUID(id);
let login = await this._getLoginFromGUID(id);
if (!login) {
record.deleted = true;
@ -267,7 +268,7 @@ PasswordStore.prototype = {
return record;
},
create(record) {
async create(record) {
let login = this._nsLoginInfoFromRecord(record);
if (!login) {
return;
@ -283,10 +284,10 @@ PasswordStore.prototype = {
}
},
remove(record) {
async remove(record) {
this._log.trace("Removing login " + record.id);
let loginItem = this._getLoginFromGUID(record.id);
let loginItem = await this._getLoginFromGUID(record.id);
if (!loginItem) {
this._log.trace("Asked to remove record that doesn't exist, ignoring");
return;
@ -295,8 +296,8 @@ PasswordStore.prototype = {
Services.logins.removeLogin(loginItem);
},
update(record) {
let loginItem = this._getLoginFromGUID(record.id);
async update(record) {
let loginItem = await this._getLoginFromGUID(record.id);
if (!loginItem) {
this._log.debug("Skipping update for unknown item: " + record.hostname);
return;
@ -315,7 +316,7 @@ PasswordStore.prototype = {
}
},
wipe() {
async wipe() {
Services.logins.removeAllLogins();
},
};
@ -424,7 +425,7 @@ class PasswordValidator extends CollectionValidator {
}
}
normalizeServerItem(item) {
async normalizeServerItem(item) {
return Object.assign({ guid: item.id }, item);
}
}

24
services/sync/modules/engines/prefs.js
View File

@ -48,7 +48,7 @@ PrefsEngine.prototype = {
syncPriority: 1,
allowSkippedRecord: false,
getChangedIDs() {
async getChangedIDs() {
// No need for a proper timestamp (no conflict resolution needed).
let changedIDs = {};
if (this._tracker.modified)
@ -56,12 +56,12 @@ PrefsEngine.prototype = {
return changedIDs;
},
_wipeClient() {
SyncEngine.prototype._wipeClient.call(this);
async _wipeClient() {
await SyncEngine.prototype._wipeClient.call(this);
this.justWiped = true;
},
_reconcile(item) {
async _reconcile(item) {
// Apply the incoming item if we don't care about the local data
if (this.justWiped) {
this.justWiped = false;
@ -165,22 +165,22 @@ PrefStore.prototype = {
}
},
getAllIDs() {
async getAllIDs() {
/* We store all prefs in just one WBO, with just one GUID */
let allprefs = {};
allprefs[PREFS_GUID] = true;
return allprefs;
},
changeItemID(oldID, newID) {
async changeItemID(oldID, newID) {
this._log.trace("PrefStore GUID is constant!");
},
itemExists(id) {
async itemExists(id) {
return (id === PREFS_GUID);
},
createRecord(id, collection) {
async createRecord(id, collection) {
let record = new PrefRec(collection, id);
if (id == PREFS_GUID) {
@ -192,15 +192,15 @@ PrefStore.prototype = {
return record;
},
create(record) {
async create(record) {
this._log.trace("Ignoring create request");
},
remove(record) {
async remove(record) {
this._log.trace("Ignoring remove request");
},
update(record) {
async update(record) {
// Silently ignore pref updates that are for other apps.
if (record.id != PREFS_GUID)
return;
@ -209,7 +209,7 @@ PrefStore.prototype = {
this._setAllPrefs(record.value);
},
wipe() {
async wipe() {
this._log.trace("Ignoring wipe request");
}
};

36
services/sync/modules/engines/tabs.js
View File

@ -35,9 +35,6 @@ Utils.deferGetSet(TabSetRecord, "cleartext", ["clientName", "tabs"]);
this.TabEngine = function TabEngine(service) {
SyncEngine.call(this, "Tabs", service);
// Reset the client on every startup so that we fetch recent tabs.
this._resetClient();
}
TabEngine.prototype = {
__proto__: SyncEngine.prototype,
@ -52,7 +49,14 @@ TabEngine.prototype = {
syncPriority: 3,
getChangedIDs() {
async initialize() {
await SyncEngine.prototype.initialize.call(this);
// Reset the client on every startup so that we fetch recent tabs.
await this._resetClient();
},
async getChangedIDs() {
// No need for a proper timestamp (no conflict resolution needed).
let changedIDs = {};
if (this._tracker.modified)
@ -69,9 +73,9 @@ TabEngine.prototype = {
return this._store._remoteClients[id];
},
_resetClient() {
SyncEngine.prototype._resetClient.call(this);
this._store.wipe();
async _resetClient() {
await SyncEngine.prototype._resetClient.call(this);
await this._store.wipe();
this._tracker.modified = true;
this.hasSyncedThisSession = false;
},
@ -92,10 +96,10 @@ TabEngine.prototype = {
return urls;
},
_reconcile(item) {
async _reconcile(item) {
// Skip our own record.
// TabStore.itemExists tests only against our local client ID.
if (this._store.itemExists(item.id)) {
if ((await this._store.itemExists(item.id))) {
this._log.trace("Ignoring incoming tab item because of its id: " + item.id);
return false;
}
@ -103,7 +107,7 @@ TabEngine.prototype = {
return SyncEngine.prototype._reconcile.call(this, item);
},
_syncFinish() {
async _syncFinish() {
this.hasSyncedThisSession = true;
return SyncEngine.prototype._syncFinish.call(this);
},
@ -116,7 +120,7 @@ function TabStore(name, engine) {
TabStore.prototype = {
__proto__: Store.prototype,
itemExists(id) {
async itemExists(id) {
return id == this.engine.service.clientsEngine.localID;
},
@ -201,7 +205,7 @@ TabStore.prototype = {
return allTabs;
},
createRecord(id, collection) {
async createRecord(id, collection) {
let record = new TabSetRecord(collection, id);
record.clientName = this.engine.service.clientsEngine.localName;
@ -235,7 +239,7 @@ TabStore.prototype = {
return record;
},
getAllIDs() {
async getAllIDs() {
// Don't report any tabs if all windows are in private browsing for
// first syncs.
let ids = {};
@ -261,18 +265,18 @@ TabStore.prototype = {
return ids;
},
wipe() {
async wipe() {
this._remoteClients = {};
},
create(record) {
async create(record) {
this._log.debug("Adding remote tabs from " + record.clientName);
this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
lastModified: record.modified
});
},
update(record) {
async update(record) {
this._log.trace("Ignoring tab updates as local ones win");
},
};

18
services/sync/modules/policies.js
View File

@ -738,23 +738,21 @@ ErrorHandler.prototype = {
Utils.nextTick(this.service.sync, this.service);
},
_dumpAddons: function _dumpAddons() {
async _dumpAddons() {
// Just dump the items that sync may be concerned with. Specifically,
// active extensions that are not hidden.
let addonPromise = Promise.resolve([]);
let addons = [];
try {
addonPromise = AddonManager.getAddonsByTypes(["extension"]);
addons = await AddonManager.getAddonsByTypes(["extension"]);
} catch (e) {
this._log.warn("Failed to dump addons", e)
}
return addonPromise.then(addons => {
let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
this._log.debug("Addons installed", relevantAddons.length);
for (let addon of relevantAddons) {
this._log.debug(" - ${name}, version ${version}, id ${id}", addon);
}
});
let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
this._log.debug("Addons installed", relevantAddons.length);
for (let addon of relevantAddons) {
this._log.debug(" - ${name}, version ${version}, id ${id}", addon);
}
},
/**

17
services/sync/modules/record.js
View File

@ -874,7 +874,7 @@ function PostQueue(poster, timestamp, config, log, postCallback) {
}
PostQueue.prototype = {
enqueue(record) {
async enqueue(record) {
// We want to ensure the record has a .toJSON() method defined - even
// though JSON.stringify() would implicitly call it, the stringify might
// still work even if it isn't defined, which isn't what we want.
@ -911,7 +911,7 @@ PostQueue.prototype = {
// Note that if a single record is too big for the batch or post, then
// the batch may be empty, and so we don't flush in that case.
if (this.numQueued) {
this.flush(batchSizeExceeded || singleRecordTooBig);
await this.flush(batchSizeExceeded || singleRecordTooBig);
}
}
// Either a ',' or a '[' depending on whether this is the first record.
@ -921,7 +921,7 @@ PostQueue.prototype = {
return { enqueued: true };
},
flush(finalBatchPost) {
async flush(finalBatchPost) {
if (!this.queued) {
// nothing queued - we can't be in a batch, and something has gone very
// bad if we think we are.
@ -957,14 +957,13 @@ PostQueue.prototype = {
}
this.queued = "";
this.numQueued = 0;
let response = Async.promiseSpinningly(
this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null)));
let response = await this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null));
if (!response.success) {
this.log.trace("Server error response during a batch", response);
// not clear what we should do here - we expect the consumer of this to
// abort by throwing in the postCallback below.
this.postCallback(response, !finalBatchPost);
await this.postCallback(response, !finalBatchPost);
return;
}
@ -972,7 +971,7 @@ PostQueue.prototype = {
this.log.trace("Committed batch", this.batchID);
this.batchID = undefined; // we are now in "first post for the batch" state.
this.lastModified = response.headers["x-last-modified"];
this.postCallback(response, false);
await this.postCallback(response, false);
return;
}
@ -982,7 +981,7 @@ PostQueue.prototype = {
}
this.batchID = null; // no batch semantics are in place.
this.lastModified = response.headers["x-last-modified"];
this.postCallback(response, false);
await this.postCallback(response, false);
return;
}
@ -1009,6 +1008,6 @@ PostQueue.prototype = {
throw new Error(`Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`);
}
this.postCallback(response, true);
await this.postCallback(response, true);
},
}

216
services/sync/modules/service.js
View File

@ -110,7 +110,7 @@ Sync11Service.prototype = {
// A specialized variant of Utils.catch.
// This provides a more informative error message when we're already syncing:
// see Bug 616568.
_catch: function _catch(func) {
_catch(func) {
function lockExceptions(ex) {
if (Utils.isLockException(ex)) {
// This only happens if we're syncing already.
@ -191,7 +191,7 @@ Sync11Service.prototype = {
/*
* Returns whether to try again.
*/
handleHMACEvent: function handleHMACEvent() {
async handleHMACEvent() {
let now = Date.now();
// Leave a sizable delay between HMAC recovery attempts. This gives us
@ -208,7 +208,7 @@ Sync11Service.prototype = {
// Fetch keys.
let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
try {
let cryptoResp = Async.promiseSpinningly(cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
// Save out the ciphertext for when we reupload. If there's a bug in
// CollectionKeyManager, this will prevent us from uploading junk.
@ -219,8 +219,8 @@ Sync11Service.prototype = {
return false;
}
let keysChanged = this.handleFetchedKeys(this.identity.syncKeyBundle,
cryptoKeys, true);
let keysChanged = await this.handleFetchedKeys(this.identity.syncKeyBundle,
cryptoKeys, true);
if (keysChanged) {
// Did they change? If so, carry on.
this._log.info("Suggesting retry.");
@ -231,7 +231,7 @@ Sync11Service.prototype = {
cryptoKeys.ciphertext = cipherText;
cryptoKeys.cleartext = null;
let uploadResp = this._uploadCryptoKeys(cryptoKeys, cryptoResp.obj.modified);
let uploadResp = await this._uploadCryptoKeys(cryptoKeys, cryptoResp.obj.modified);
if (uploadResp.success) {
this._log.info("Successfully re-uploaded keys. Continuing sync.");
} else {
@ -248,7 +248,7 @@ Sync11Service.prototype = {
}
},
handleFetchedKeys: function handleFetchedKeys(syncKey, cryptoKeys, skipReset) {
async handleFetchedKeys(syncKey, cryptoKeys, skipReset) {
// Don't want to wipe if we're just starting up!
let wasBlank = this.collectionKeys.isClear;
let keysChanged = this.collectionKeys.updateContents(syncKey, cryptoKeys);
@ -261,10 +261,10 @@ Sync11Service.prototype = {
if (keysChanged.length) {
// Collection keys only. Reset individual engines.
this.resetClient(keysChanged);
await this.resetClient(keysChanged);
} else {
// Default key changed: wipe it all.
this.resetClient();
await this.resetClient();
}
this._log.info("Downloaded new keys, client reset. Proceeding.");
@ -277,7 +277,7 @@ Sync11Service.prototype = {
/**
* Prepare to initialize the rest of Weave after waiting a little bit
*/
onStartup: function onStartup() {
async onStartup() {
this.status = Status;
this.identity = Status._authManager;
this.collectionKeys = new CollectionKeyManager();
@ -295,7 +295,7 @@ Sync11Service.prototype = {
this.enabled = true;
this._registerEngines();
await this._registerEngines();
let ua = Cc["@mozilla.org/network/protocol;1?name=http"].
getService(Ci.nsIHttpProtocolHandler).userAgent;
@ -352,7 +352,7 @@ Sync11Service.prototype = {
/**
* Register the built-in engines for certain applications
*/
_registerEngines: function _registerEngines() {
async _registerEngines() {
this.engineManager = new EngineManager(this);
let engines = [];
@ -372,7 +372,11 @@ Sync11Service.prototype = {
declined = pref.split(",");
}
this.clientsEngine = new ClientEngine(this);
let clientsEngine = new ClientEngine(this);
// Ideally clientsEngine should not exist
// (or be a promise that calls initialize() before returning the engine)
await clientsEngine.initialize();
this.clientsEngine = clientsEngine;
for (let name of engines) {
if (!(name in ENGINE_MODULES)) {
@ -391,7 +395,7 @@ Sync11Service.prototype = {
continue;
}
this.engineManager.register(ns[symbol]);
await this.engineManager.register(ns[symbol]);
} catch (ex) {
this._log.warn("Could not register engine " + name, ex);
}
@ -414,13 +418,18 @@ Sync11Service.prototype = {
// couldn't get to get the sync lock, due to us currently syncing the
// clients engine.
if (data.includes("clients") && !Svc.Prefs.get("testing.tps", false)) {
this.sync([]); // [] = clients collection only
// Sync in the background (it's fine not to wait on the returned promise
// because sync() has a lock).
// [] = clients collection only
this.sync([]).catch(e => {
this._log.error(e);
});
}
break;
case "fxaccounts:device_disconnected":
data = JSON.parse(data);
if (!data.isLocalDevice) {
this.clientsEngine.updateKnownStaleClients();
Async.promiseSpinningly(this.clientsEngine.updateKnownStaleClients());
}
break;
case "weave:service:setup-complete":
@ -472,13 +481,13 @@ Sync11Service.prototype = {
* Perform the info fetch as part of a login or key fetch, or
* inside engine sync.
*/
_fetchInfo(url) {
async _fetchInfo(url) {
let infoURL = url || this.infoURL;
this._log.trace("In _fetchInfo: " + infoURL);
let info;
try {
info = Async.promiseSpinningly(this.resource(infoURL).get());
info = await this.resource(infoURL).get();
} catch (ex) {
this.errorHandler.checkServerError(ex);
throw ex;
@ -493,7 +502,7 @@ Sync11Service.prototype = {
return info;
},
verifyAndFetchSymmetricKeys: function verifyAndFetchSymmetricKeys(infoResponse) {
async verifyAndFetchSymmetricKeys(infoResponse) {
this._log.debug("Fetching and verifying -- or generating -- symmetric keys.");
@ -506,7 +515,7 @@ Sync11Service.prototype = {
try {
if (!infoResponse)
infoResponse = this._fetchInfo(); // Will throw an exception on failure.
infoResponse = await this._fetchInfo(); // Will throw an exception on failure.
// This only applies when the server is already at version 4.
if (infoResponse.status != 200) {
@ -531,10 +540,10 @@ Sync11Service.prototype = {
if (infoCollections && (CRYPTO_COLLECTION in infoCollections)) {
try {
cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
let cryptoResp = Async.promiseSpinningly(cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
if (cryptoResp.success) {
this.handleFetchedKeys(syncKeyBundle, cryptoKeys);
await this.handleFetchedKeys(syncKeyBundle, cryptoKeys);
return true;
} else if (cryptoResp.status == 404) {
// On failure, ask to generate new keys and upload them.
@ -574,7 +583,7 @@ Sync11Service.prototype = {
// consistency. This is all achieved via _freshStart.
// If _freshStart fails to clear the server or upload keys, it will
// throw.
this._freshStart();
await this._freshStart();
return true;
}
@ -591,7 +600,7 @@ Sync11Service.prototype = {
}
},
verifyLogin: function verifyLogin(allow40XRecovery = true) {
async verifyLogin(allow40XRecovery = true) {
if (!this.identity.username) {
this._log.warn("No username in verifyLogin.");
this.status.login = LOGIN_FAILED_NO_USERNAME;
@ -604,12 +613,7 @@ Sync11Service.prototype = {
// So we ask the identity to verify the login state after unlocking the
// master password (ie, this call is expected to prompt for MP unlock
// if necessary) while we still have control.
let cb = Async.makeSpinningCallback();
this.identity.unlockAndVerifyAuthState().then(
result => cb(null, result),
cb
);
let unlockedState = cb.wait();
let unlockedState = await this.identity.unlockAndVerifyAuthState();
this._log.debug("Fetching unlocked auth state returned " + unlockedState);
if (unlockedState != STATUS_OK) {
this.status.login = unlockedState;
@ -626,7 +630,7 @@ Sync11Service.prototype = {
}
// Fetch collection info on every startup.
let test = Async.promiseSpinningly(this.resource(this.infoURL).get());
let test = await this.resource(this.infoURL).get();
switch (test.status) {
case 200:
@ -643,7 +647,7 @@ Sync11Service.prototype = {
// Go ahead and do remote setup, so that we can determine
// conclusively that our passphrase is correct.
if (this._remoteSetup(test)) {
if ((await this._remoteSetup(test))) {
// Username/password verified.
this.status.login = LOGIN_SUCCEEDED;
return true;
@ -660,7 +664,7 @@ Sync11Service.prototype = {
case 404:
// Check that we're verifying with the correct cluster
if (allow40XRecovery && this._clusterManager.setCluster()) {
return this.verifyLogin(false);
return await this.verifyLogin(false);
}
// We must have the right cluster, but the server doesn't expect us.
@ -686,13 +690,13 @@ Sync11Service.prototype = {
}
},
generateNewSymmetricKeys: function generateNewSymmetricKeys() {
async generateNewSymmetricKeys() {
this._log.info("Generating new keys WBO...");
let wbo = this.collectionKeys.generateNewKeysWBO();
this._log.info("Encrypting new key bundle.");
wbo.encrypt(this.identity.syncKeyBundle);
let uploadRes = this._uploadCryptoKeys(wbo, 0);
let uploadRes = await this._uploadCryptoKeys(wbo, 0);
if (uploadRes.status != 200) {
this._log.warn("Got status " + uploadRes.status + " uploading new keys. What to do? Throw!");
this.errorHandler.checkServerError(uploadRes);
@ -704,7 +708,7 @@ Sync11Service.prototype = {
// Now verify that info/collections shows them!
this._log.debug("Verifying server collection records.");
let info = this._fetchInfo();
let info = await this._fetchInfo();
this._log.debug("info/collections is: " + info);
if (info.status != 200) {
@ -730,19 +734,19 @@ Sync11Service.prototype = {
// Download and install them.
let cryptoKeys = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
let cryptoResp = Async.promiseSpinningly(cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
let cryptoResp = (await cryptoKeys.fetch(this.resource(this.cryptoKeysURL))).response;
if (cryptoResp.status != 200) {
this._log.warn("Failed to download keys.");
throw new Error("Symmetric key download failed.");
}
let keysChanged = this.handleFetchedKeys(this.identity.syncKeyBundle,
cryptoKeys, true);
let keysChanged = await this.handleFetchedKeys(this.identity.syncKeyBundle,
cryptoKeys, true);
if (keysChanged) {
this._log.info("Downloaded keys differed, as expected.");
}
},
startOver: function startOver() {
async startOver() {
this._log.trace("Invoking Service.startOver.");
Svc.Obs.notify("weave:engine:stop-tracking");
this.status.resetSync();
@ -752,11 +756,7 @@ Sync11Service.prototype = {
// Clear client-specific data from the server, including disabled engines.
for (let engine of [this.clientsEngine].concat(this.engineManager.getAll())) {
try {
// Note the additional Promise.resolve here is to handle the fact that
// some 3rd party engines probably don't return a promise. We can
// probably nuke this once webextensions become mandatory as then
// no 3rd party engines will be allowed to exist.
Async.promiseSpinningly(Promise.resolve().then(() => engine.removeClientData()));
await engine.removeClientData();
} catch (ex) {
this._log.warn(`Deleting client data for ${engine.name} failed`, ex);
}
@ -776,7 +776,7 @@ Sync11Service.prototype = {
Svc.Obs.notify("weave:service:start-over");
// Reset all engines and clear keys.
this.resetClient();
await this.resetClient();
this.collectionKeys.clear();
this.status.resetBackoff();
@ -804,8 +804,8 @@ Sync11Service.prototype = {
}
},
login: function login() {
function onNotify() {
async login() {
async function onNotify() {
this._loggedIn = false;
if (Services.io.offline) {
this.status.login = LOGIN_FAILED_NETWORK_ERROR;
@ -815,7 +815,7 @@ Sync11Service.prototype = {
this._log.info("Logging in the user.");
// Just let any errors bubble up - they've more context than we do!
try {
Async.promiseSpinningly(this.identity.ensureLoggedIn());
await this.identity.ensureLoggedIn();
} finally {
this._checkSetup(); // _checkSetup has a side effect of setting the right state.
}
@ -823,7 +823,7 @@ Sync11Service.prototype = {
this._updateCachedURLs();
this._log.info("User logged in successfully - verifying login.");
if (!this.verifyLogin()) {
if (!(await this.verifyLogin())) {
// verifyLogin sets the failure states here.
throw "Login failed: " + this.status.login;
}
@ -850,14 +850,14 @@ Sync11Service.prototype = {
// Note: returns false if we failed for a reason other than the server not yet
// supporting the api.
_fetchServerConfiguration() {
async _fetchServerConfiguration() {
// This is similar to _fetchInfo, but with different error handling.
let infoURL = this.userBaseURL + "info/configuration";
this._log.debug("Fetching server configuration", infoURL);
let configResponse;
try {
configResponse = Async.promiseSpinningly(this.resource(infoURL).get());
configResponse = await this.resource(infoURL).get();
} catch (ex) {
// This is probably a network or similar error.
this._log.warn("Failed to fetch info/configuration", ex);
@ -881,13 +881,13 @@ Sync11Service.prototype = {
// Stuff we need to do after login, before we can really do
// anything (e.g. key setup).
_remoteSetup: function _remoteSetup(infoResponse) {
if (!this._fetchServerConfiguration()) {
async _remoteSetup(infoResponse) {
if (!(await this._fetchServerConfiguration())) {
return false;
}
this._log.debug("Fetching global metadata record");
let meta = Async.promiseSpinningly(this.recordManager.get(this.metaURL));
let meta = await this.recordManager.get(this.metaURL);
// Checking modified time of the meta record.
if (infoResponse &&
@ -902,7 +902,7 @@ Sync11Service.prototype = {
this.recordManager.del(this.metaURL);
// ... fetch the current record from the server, and COPY THE FLAGS.
let newMeta = Async.promiseSpinningly(this.recordManager.get(this.metaURL));
let newMeta = await this.recordManager.get(this.metaURL);
// If we got a 401, we do not want to create a new meta/global - we
// should be able to get the existing meta after we get a new node.
@ -915,7 +915,7 @@ Sync11Service.prototype = {
if (this.recordManager.response.status == 404) {
this._log.debug("No meta/global record on the server. Creating one.");
try {
this._uploadNewMetaGlobal();
await this._uploadNewMetaGlobal();
} catch (uploadRes) {
this._log.warn("Unable to upload new meta/global. Failing remote setup.");
this.errorHandler.checkServerError(uploadRes);
@ -965,7 +965,7 @@ Sync11Service.prototype = {
this._log.warn("No sync id, server wipe needed");
this._log.info("Wiping server data");
this._freshStart();
await this._freshStart();
if (status == 404)
this._log.info("Metadata record not found, server was wiped to ensure " +
@ -981,18 +981,18 @@ Sync11Service.prototype = {
} else if (meta.payload.syncID != this.syncID) {
this._log.info("Sync IDs differ. Local is " + this.syncID + ", remote is " + meta.payload.syncID);
this.resetClient();
await this.resetClient();
this.collectionKeys.clear();
this.syncID = meta.payload.syncID;
this._log.debug("Clear cached values and take syncId: " + this.syncID);
if (!this.verifyAndFetchSymmetricKeys(infoResponse)) {
if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) {
this._log.warn("Failed to fetch symmetric keys. Failing remote setup.");
return false;
}
// bug 545725 - re-verify creds and fail sanely
if (!this.verifyLogin()) {
if (!(await this.verifyLogin())) {
this.status.sync = CREDENTIALS_CHANGED;
this._log.info("Credentials have changed, aborting sync and forcing re-login.");
return false;
@ -1000,7 +1000,7 @@ Sync11Service.prototype = {
return true;
}
if (!this.verifyAndFetchSymmetricKeys(infoResponse)) {
if (!(await this.verifyAndFetchSymmetricKeys(infoResponse))) {
this._log.warn("Failed to fetch symmetric keys. Failing remote setup.");
return false;
}
@ -1049,43 +1049,37 @@ Sync11Service.prototype = {
return reason;
},
sync: function sync(engineNamesToSync) {
async sync(engineNamesToSync) {
let dateStr = Utils.formatTimestamp(new Date());
this._log.debug("User-Agent: " + Utils.userAgent);
this._log.info(`Starting sync at ${dateStr} in browser session ${browserSessionID}`);
this._catch(function() {
return this._catch(async function() {
// Make sure we're logged in.
if (this._shouldLogin()) {
this._log.debug("In sync: should login.");
if (!this.login()) {
if (!(await this.login())) {
this._log.debug("Not syncing: login returned false.");
return;
}
} else {
this._log.trace("In sync: no need to login.");
}
this._lockedSync(engineNamesToSync);
await this._lockedSync(engineNamesToSync);
})();
},
/**
* Sync up engines with the server.
*/
_lockedSync: function _lockedSync(engineNamesToSync) {
async _lockedSync(engineNamesToSync) {
return this._lock("service.js: sync",
this._notify("sync", "", function onNotify() {
this._notify("sync", "", async function onNotify() {
let histogram = Services.telemetry.getHistogramById("WEAVE_START_COUNT");
histogram.add(1);
let synchronizer = new EngineSynchronizer(this);
let cb = Async.makeSpinningCallback();
synchronizer.onComplete = cb;
synchronizer.sync(engineNamesToSync);
// wait() throws if the first argument is truthy, which is exactly what
// we want.
cb.wait();
await synchronizer.sync(engineNamesToSync); // Might throw!
histogram = Services.telemetry.getHistogramById("WEAVE_COMPLETE_SUCCESS_COUNT");
histogram.add(1);
@ -1102,7 +1096,7 @@ Sync11Service.prototype = {
// Now let's update our declined engines (but only if we have a metaURL;
// if Sync failed due to no node we will not have one)
if (this.metaURL) {
let meta = Async.promiseSpinningly(this.recordManager.get(this.metaURL));
let meta = await this.recordManager.get(this.metaURL);
if (!meta) {
this._log.warn("No meta/global; can't update declined state.");
return;
@ -1115,7 +1109,7 @@ Sync11Service.prototype = {
return;
}
this.uploadMetaGlobal(meta);
await this.uploadMetaGlobal(meta);
}
}))();
},
@ -1124,7 +1118,7 @@ Sync11Service.prototype = {
* Upload a fresh meta/global record
* @throws the response object if the upload request was not a success
*/
_uploadNewMetaGlobal() {
async _uploadNewMetaGlobal() {
let meta = new WBORecord("meta", "global");
meta.payload.syncID = this.syncID;
meta.payload.storageVersion = STORAGE_VERSION;
@ -1132,7 +1126,7 @@ Sync11Service.prototype = {
meta.modified = 0;
meta.isNew = true;
this.uploadMetaGlobal(meta);
await this.uploadMetaGlobal(meta);
},
/**
@ -1140,11 +1134,11 @@ Sync11Service.prototype = {
* @param {WBORecord} meta meta/global record
* @throws the response object if the request was not a success
*/
uploadMetaGlobal(meta) {
async uploadMetaGlobal(meta) {
this._log.debug("Uploading meta/global", meta);
let res = this.resource(this.metaURL);
res.setHeader("X-If-Unmodified-Since", meta.modified);
let response = Async.promiseSpinningly(res.put(meta));
let response = await res.put(meta);
if (!response.success) {
throw response;
}
@ -1160,34 +1154,34 @@ Sync11Service.prototype = {
* @param {Number} lastModified known last modified timestamp (in decimal seconds),
* will be used to set the X-If-Unmodified-Since header
*/
_uploadCryptoKeys(cryptoKeys, lastModified) {
async _uploadCryptoKeys(cryptoKeys, lastModified) {
this._log.debug(`Uploading crypto/keys (lastModified: ${lastModified})`);
let res = this.resource(this.cryptoKeysURL);
res.setHeader("X-If-Unmodified-Since", lastModified);
return Async.promiseSpinningly(res.put(cryptoKeys));
return res.put(cryptoKeys);
},
_freshStart: function _freshStart() {
async _freshStart() {
this._log.info("Fresh start. Resetting client.");
this.resetClient();
await this.resetClient();
this.collectionKeys.clear();
// Wipe the server.
this.wipeServer();
await this.wipeServer();
// Upload a new meta/global record.
// _uploadNewMetaGlobal throws on failure -- including race conditions.
// If we got into a race condition, we'll abort the sync this way, too.
// That's fine. We'll just wait till the next sync. The client that we're
// racing is probably busy uploading stuff right now anyway.
this._uploadNewMetaGlobal();
await this._uploadNewMetaGlobal();
// Wipe everything we know about except meta because we just uploaded it
// TODO: there's a bug here. We should be calling resetClient, no?
// Generate, upload, and download new keys. Do this last so we don't wipe
// them...
this.generateNewSymmetricKeys();
await this.generateNewSymmetricKeys();
},
/**
@ -1199,7 +1193,7 @@ Sync11Service.prototype = {
*
* @return the server's timestamp of the (last) DELETE.
*/
wipeServer: function wipeServer(collections) {
async wipeServer(collections) {
let response;
let histogram = Services.telemetry.getHistogramById("WEAVE_WIPE_SERVER_SUCCEEDED");
if (!collections) {
@ -1207,7 +1201,7 @@ Sync11Service.prototype = {
let res = this.resource(this.storageURL.slice(0, -1));
res.setHeader("X-Confirm-Delete", "1");
try {
response = Async.promiseSpinningly(res.delete());
response = await res.delete();
} catch (ex) {
this._log.debug("Failed to wipe server", ex);
histogram.add(false);
@ -1227,7 +1221,7 @@ Sync11Service.prototype = {
for (let name of collections) {
let url = this.storageURL + name;
try {
response = Async.promiseSpinningly(this.resource(url).delete());
response = await this.resource(url).delete();
} catch (ex) {
this._log.debug("Failed to wipe '" + name + "' collection", ex);
histogram.add(false);
@ -1255,11 +1249,11 @@ Sync11Service.prototype = {
* @param engines [optional]
* Array of engine names to wipe. If not given, all engines are used.
*/
wipeClient: function wipeClient(engines) {
async wipeClient(engines) {
// If we don't have any engines, reset the service and wipe all engines
if (!engines) {
// Clear out any service data
this.resetService();
await this.resetService();
engines = [this.clientsEngine].concat(this.engineManager.getAll());
} else {
@ -1269,8 +1263,8 @@ Sync11Service.prototype = {
// Fully wipe each engine if it's able to decrypt data
for (let engine of engines) {
if (engine.canDecrypt()) {
engine.wipeClient();
if ((await engine.canDecrypt())) {
await engine.wipeClient();
}
}
},
@ -1282,27 +1276,27 @@ Sync11Service.prototype = {
* @param engines [optional]
* Array of engine names to wipe. If not given, all engines are used.
*/
wipeRemote: function wipeRemote(engines) {
async wipeRemote(engines) {
try {
// Make sure stuff gets uploaded.
this.resetClient(engines);
await this.resetClient(engines);
// Clear out any server data.
this.wipeServer(engines);
await this.wipeServer(engines);
// Only wipe the engines provided.
let extra = { reason: "wipe-remote" };
if (engines) {
engines.forEach(function(e) {
this.clientsEngine.sendCommand("wipeEngine", [e], null, extra);
}, this);
for (const e of engines) {
await this.clientsEngine.sendCommand("wipeEngine", [e], null, extra);
}
} else {
// Tell the remote machines to wipe themselves.
this.clientsEngine.sendCommand("wipeAll", [], null, extra);
await this.clientsEngine.sendCommand("wipeAll", [], null, extra);
}
// Make sure the changed clients get updated.
this.clientsEngine.sync();
await this.clientsEngine.sync();
} catch (ex) {
this.errorHandler.checkServerError(ex);
throw ex;
@ -1312,8 +1306,8 @@ Sync11Service.prototype = {
/**
* Reset local service information like logs, sync times, caches.
*/
resetService: function resetService() {
this._catch(function reset() {
async resetService() {
return this._catch(async function reset() {
this._log.info("Service reset.");
// Pretend we've never synced to the server and drop cached data
@ -1328,12 +1322,12 @@ Sync11Service.prototype = {
* @param engines [optional]
* Array of engine names to reset. If not given, all engines are used.
*/
resetClient: function resetClient(engines) {
this._catch(function doResetClient() {
async resetClient(engines) {
return this._catch(async function doResetClient() {
// If we don't have any engines, reset everything including the service
if (!engines) {
// Clear out any service data
this.resetService();
await this.resetService();
engines = [this.clientsEngine].concat(this.engineManager.getAll());
} else {
@ -1343,7 +1337,7 @@ Sync11Service.prototype = {
// Have each engine drop any temporary meta data
for (let engine of engines) {
engine.resetClient();
await engine.resetClient();
}
})();
},
@ -1401,4 +1395,6 @@ Sync11Service.prototype = {
};
this.Service = new Sync11Service();
this.Service.onStartup();
this.Service.promiseInitialized = new Promise(resolve => {
this.Service.onStartup().then(resolve);
});

70
services/sync/modules/stages/enginesync.js
View File

@ -28,16 +28,10 @@ this.EngineSynchronizer = function EngineSynchronizer(service) {
this._log.level = Log.Level[Svc.Prefs.get("log.logger.synchronizer")];
this.service = service;
this.onComplete = null;
}
EngineSynchronizer.prototype = {
sync: function sync(engineNamesToSync) {
if (!this.onComplete) {
throw new Error("onComplete handler not installed.");
}
async sync(engineNamesToSync) {
let startTime = Date.now();
this.service.status.resetSync();
@ -52,15 +46,13 @@ EngineSynchronizer.prototype = {
// this is a purposeful abort rather than a failure, so don't set
// any status bits
reason = "Can't sync: " + reason;
this.onComplete(new Error("Can't sync: " + reason));
return;
throw new Error(reason);
}
// If we don't have a node, get one. If that fails, retry in 10 minutes.
if (!this.service.clusterURL && !this.service._clusterManager.setCluster()) {
this.service.status.sync = NO_SYNC_NODE_FOUND;
this._log.info("No cluster URL found. Cannot sync.");
this.onComplete(null);
return;
}
@ -76,25 +68,23 @@ EngineSynchronizer.prototype = {
let engineManager = this.service.engineManager;
// Figure out what the last modified time is for each collection
let info = this.service._fetchInfo(infoURL);
let info = await this.service._fetchInfo(infoURL);
// Convert the response to an object and read out the modified times
for (let engine of [this.service.clientsEngine].concat(engineManager.getAll())) {
engine.lastModified = info.obj[engine.name] || 0;
}
if (!(this.service._remoteSetup(info))) {
this.onComplete(new Error("Aborting sync, remote setup failed"));
return;
if (!(await this.service._remoteSetup(info))) {
throw new Error("Aborting sync, remote setup failed");
}
// Make sure we have an up-to-date list of clients before sending commands
this._log.debug("Refreshing client list.");
if (!this._syncEngine(this.service.clientsEngine)) {
if (!(await this._syncEngine(this.service.clientsEngine))) {
// Clients is an engine like any other; it can fail with a 401,
// and we can elect to abort the sync.
this._log.warn("Client engine sync failed. Aborting.");
this.onComplete(null);
return;
}
@ -104,13 +94,13 @@ EngineSynchronizer.prototype = {
// Wipe data in the desired direction if necessary
switch (Svc.Prefs.get("firstSync")) {
case "resetClient":
this.service.resetClient(engineManager.enabledEngineNames);
await this.service.resetClient(engineManager.enabledEngineNames);
break;
case "wipeClient":
this.service.wipeClient(engineManager.enabledEngineNames);
await this.service.wipeClient(engineManager.enabledEngineNames);
break;
case "wipeRemote":
this.service.wipeRemote(engineManager.enabledEngineNames);
await this.service.wipeRemote(engineManager.enabledEngineNames);
break;
default:
allowEnginesHint = true;
@ -119,34 +109,31 @@ EngineSynchronizer.prototype = {
if (this.service.clientsEngine.localCommands) {
try {
if (!(this.service.clientsEngine.processIncomingCommands())) {
if (!(await this.service.clientsEngine.processIncomingCommands())) {
this.service.status.sync = ABORT_SYNC_COMMAND;
this.onComplete(new Error("Processed command aborted sync."));
return;
throw new Error("Processed command aborted sync.");
}
// Repeat remoteSetup in-case the commands forced us to reset
if (!(this.service._remoteSetup(info))) {
this.onComplete(new Error("Remote setup failed after processing commands."));
return;
if (!(await this.service._remoteSetup(info))) {
throw new Error("Remote setup failed after processing commands.");
}
} finally {
// Always immediately attempt to push back the local client (now
// without commands).
// Note that we don't abort here; if there's a 401 because we've
// been reassigned, we'll handle it around another engine.
this._syncEngine(this.service.clientsEngine);
await this._syncEngine(this.service.clientsEngine);
}
}
// Update engines because it might change what we sync.
try {
this._updateEnabledEngines();
await this._updateEnabledEngines();
} catch (ex) {
this._log.debug("Updating enabled engines failed", ex);
this.service.errorHandler.checkServerError(ex);
this.onComplete(ex);
return;
throw ex;
}
// If the engines to sync has been specified, we sync in the order specified.
@ -163,7 +150,7 @@ EngineSynchronizer.prototype = {
let enginesToValidate = [];
for (let engine of enginesToSync) {
// If there's any problems with syncing the engine, report the failure
if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) {
if (!(await this._syncEngine(engine)) || this.service.status.enforceBackoff) {
this._log.info("Aborting sync for failure in " + engine.name);
break;
}
@ -176,16 +163,15 @@ EngineSynchronizer.prototype = {
if (!this.service.clusterURL) {
this._log.debug("Aborting sync, no cluster URL: " +
"not uploading new meta/global.");
this.onComplete(null);
return;
}
// Upload meta/global if any engines changed anything.
let meta = Async.promiseSpinningly(this.service.recordManager.get(this.service.metaURL));
let meta = await this.service.recordManager.get(this.service.metaURL);
if (meta.isNew || meta.changed) {
this._log.info("meta/global changed locally: reuploading.");
try {
this.service.uploadMetaGlobal(meta);
await this.service.uploadMetaGlobal(meta);
delete meta.isNew;
delete meta.changed;
} catch (error) {
@ -193,7 +179,7 @@ EngineSynchronizer.prototype = {
}
}
Async.promiseSpinningly(Doctor.consult(enginesToValidate));
await Doctor.consult(enginesToValidate);
// If there were no sync engine failures
if (this.service.status.service != SYNC_FAILED_PARTIAL) {
@ -208,15 +194,13 @@ EngineSynchronizer.prototype = {
this._log.info("Sync completed at " + dateStr
+ " after " + syncTime + " secs.");
}
this.onComplete(null);
},
// Returns true if sync should proceed.
// false / no return value means sync should be aborted.
_syncEngine: function _syncEngine(engine) {
async _syncEngine(engine) {
try {
engine.sync();
await engine.sync();
} catch (e) {
if (e.status == 401) {
// Maybe a 401, cluster update perhaps needed?
@ -238,7 +222,7 @@ EngineSynchronizer.prototype = {
return true;
},
_updateEnabledFromMeta(meta, numClients, engineManager = this.service.engineManager) {
async _updateEnabledFromMeta(meta, numClients, engineManager = this.service.engineManager) {
this._log.info("Updating enabled engines: " +
numClients + " clients.");
@ -307,7 +291,7 @@ EngineSynchronizer.prototype = {
// disable it everywhere.
if (!engine.enabled) {
this._log.trace("Wiping data for " + engineName + " engine.");
engine.wipeServer();
await engine.wipeServer();
delete meta.payload.engines[engineName];
meta.changed = true; // the new enabled state must propagate
// We also here mark the engine as declined, because the pref
@ -342,12 +326,12 @@ EngineSynchronizer.prototype = {
this.service._ignorePrefObserver = false;
},
_updateEnabledEngines() {
let meta = Async.promiseSpinningly(this.service.recordManager.get(this.service.metaURL));
async _updateEnabledEngines() {
let meta = await this.service.recordManager.get(this.service.metaURL);
let numClients = this.service.scheduler.numClients;
let engineManager = this.service.engineManager;
this._updateEnabledFromMeta(meta, numClients, engineManager);
await this._updateEnabledFromMeta(meta, numClients, engineManager);
},
};
Object.freeze(EngineSynchronizer.prototype);

96
services/sync/modules/util.js
View File

@ -76,7 +76,7 @@ this.Utils = {
},
/**
* Wrap a function to catch all exceptions and log them
* Wrap a [promise-returning] function to catch all exceptions and log them.
*
* @usage MyObj._catch = Utils.catch;
* MyObj.foo = function() { this._catch(func)(); }
@ -84,11 +84,11 @@ this.Utils = {
* Optionally pass a function which will be called if an
* exception occurs.
*/
catch: function Utils_catch(func, exceptionCallback) {
catch(func, exceptionCallback) {
let thisArg = this;
return function WrappedCatch() {
return async function WrappedCatch() {
try {
return func.call(thisArg);
return await func.call(thisArg);
} catch (ex) {
thisArg._log.debug("Exception calling " + (func.name || "anonymous function"), ex);
if (exceptionCallback) {
@ -100,20 +100,21 @@ this.Utils = {
},
/**
* Wrap a function to call lock before calling the function then unlock.
* Wrap a [promise-returning] function to call lock before calling the function
* then unlock when it finishes executing or if it threw an error.
*
* @usage MyObj._lock = Utils.lock;
* MyObj.foo = function() { this._lock(func)(); }
* MyObj.foo = async function() { await this._lock(func)(); }
*/
lock: function lock(label, func) {
lock(label, func) {
let thisArg = this;
return function WrappedLock() {
return async function WrappedLock() {
if (!thisArg.lock()) {
throw "Could not acquire lock. Label: \"" + label + "\".";
}
try {
return func.call(thisArg);
return await func.call(thisArg);
} finally {
thisArg.unlock();
}
@ -125,8 +126,8 @@ this.Utils = {
},
/**
* Wrap functions to notify when it starts and finishes executing or if it
* threw an error.
* Wrap [promise-returning] functions to notify when it starts and
* finishes executing or if it threw an error.
*
* The message is a combination of a provided prefix, the local name, and
* the event. Possible events are: "start", "finish", "error". The subject
@ -140,12 +141,12 @@ this.Utils = {
* this._notify = Utils.notify("obj:");
* }
* MyObj.prototype = {
* foo: function() this._notify("func", "data-arg", function () {
* foo: function() this._notify("func", "data-arg", async function () {
* //...
* }(),
* };
*/
notify: function Utils_notify(prefix) {
notify(prefix) {
return function NotifyMaker(name, data, func) {
let thisArg = this;
let notify = function(state, subject) {
@ -154,10 +155,10 @@ this.Utils = {
Observers.notify(mesg, subject, data);
};
return function WrappedNotify() {
return async function WrappedNotify() {
notify("start", null);
try {
notify("start", null);
let ret = func.call(thisArg);
let ret = await func.call(thisArg);
notify("finish", ret);
return ret;
} catch (ex) {
@ -299,35 +300,28 @@ this.Utils = {
* <profile>/<filePath>.json. i.e. Do not specify the ".json"
* extension.
* @param that
* Object to use for logging and "this" for callback.
* @param callback
* Function to process json object as its first argument. If the file
* could not be loaded, the first argument will be undefined.
* Object to use for logging.
*
* @return Promise<>
* Promise resolved when the write has been performed.
*/
async jsonLoad(filePath, that, callback) {
async jsonLoad(filePath, that) {
let path = Utils.jsonFilePath(filePath);
if (that._log) {
if (that._log && that._log.trace) {
that._log.trace("Loading json from disk: " + filePath);
}
let json;
try {
json = await CommonUtils.readJSON(path);
return await CommonUtils.readJSON(path);
} catch (e) {
if (e instanceof OS.File.Error && e.becauseNoSuchFile) {
// Ignore non-existent files, but explicitly return null.
json = null;
} else if (that._log) {
if (!(e instanceof OS.File.Error && e.becauseNoSuchFile)) {
if (that._log) {
that._log.debug("Failed to load json", e);
}
}
return null;
}
if (callback) {
callback.call(that, json);
}
return json;
},
/**
@ -336,39 +330,29 @@ this.Utils = {
* @param filePath
* JSON file path save to <filePath>.json
* @param that
* Object to use for logging and "this" for callback
* Object to use for logging.
* @param obj
* Function to provide json-able object to save. If this isn't a
* function, it'll be used as the object to make a json string.
* @param callback
* function, it'll be used as the object to make a json string.*
* Function called when the write has been performed. Optional.
* The first argument will be a Components.results error
* constant on error or null if no error was encountered (and
* the file saved successfully).
*
* @return Promise<>
* Promise resolved when the write has been performed.
*/
async jsonSave(filePath, that, obj, callback) {
async jsonSave(filePath, that, obj) {
let path = OS.Path.join(OS.Constants.Path.profileDir, "weave",
...(filePath + ".json").split("/"));
let dir = OS.Path.dirname(path);
let error = null;
try {
await OS.File.makeDir(dir, { from: OS.Constants.Path.profileDir });
await OS.File.makeDir(dir, { from: OS.Constants.Path.profileDir });
if (that._log) {
that._log.trace("Saving json to disk: " + path);
}
let json = typeof obj == "function" ? obj.call(that) : obj;
await CommonUtils.writeJSON(json, path);
} catch (e) {
error = e
if (that._log) {
that._log.trace("Saving json to disk: " + path);
}
if (typeof callback == "function") {
callback.call(that, error);
}
let json = typeof obj == "function" ? obj.call(that) : obj;
return CommonUtils.writeJSON(json, path);
},
/**