mirror of
https://github.com/mozilla/gecko-dev.git
synced 2024-10-24 18:55:30 +00:00
Store each delta as a separate file on the server
This commit is contained in:
parent
52a8ad1e41
commit
4b9dbfb59b
@ -379,20 +379,18 @@ Engine.prototype = {
|
||||
this._snapshot.data = newSnapshot;
|
||||
this._snapshot.version = ++this._remote.status.data.maxVersion;
|
||||
|
||||
// XXX don't append delta if we do a full upload?
|
||||
if (this._remote.status.data.formatVersion != ENGINE_STORAGE_FORMAT_VERSION)
|
||||
yield this._remote.initialize(self.cb, this._snapshot);
|
||||
|
||||
this._remote.appendDelta(self.cb, serverDelta);
|
||||
yield;
|
||||
|
||||
let c = 0;
|
||||
for (GUID in this._snapshot.data)
|
||||
c++;
|
||||
|
||||
this._remote.status.data.maxVersion = this._snapshot.version;
|
||||
this._remote.status.data.snapEncryption = Crypto.defaultAlgorithm;
|
||||
this._remote.status.data.itemCount = c;
|
||||
this._remote.status.put(self.cb, this._remote.status.data);
|
||||
this._remote.appendDelta(self.cb, serverDelta,
|
||||
{maxVersion: this._snapshot.version,
|
||||
deltasEncryption: Crypto.defaultAlgorithm,
|
||||
itemCount: c});
|
||||
yield;
|
||||
|
||||
this._log.info("Successfully updated deltas and status on server");
|
||||
|
@ -230,6 +230,49 @@ Resource.prototype = {
|
||||
}
|
||||
};
|
||||
|
||||
function ResourceSet(basePath) {
|
||||
this._init(basePath);
|
||||
}
|
||||
ResourceSet.prototype = {
|
||||
__proto__: new Resource(),
|
||||
_init: function ResSet__init(basePath) {
|
||||
this.__proto__.__proto__._init.call(this);
|
||||
this._basePath = basePath;
|
||||
this._log = Log4Moz.Service.getLogger("Service.ResourceSet");
|
||||
},
|
||||
|
||||
_hack: function ResSet__hack(action, id, data) {
|
||||
let self = yield;
|
||||
let savedData = this._data;
|
||||
|
||||
if ("PUT" == action)
|
||||
this._data = data;
|
||||
|
||||
this._path = this._basePath + id;
|
||||
yield this._request.async(this, self.cb, action, data);
|
||||
|
||||
let newData = this._data;
|
||||
this._data = savedData;
|
||||
if (this._data == null)
|
||||
this._data = {};
|
||||
this._data[id] = newData;
|
||||
|
||||
self.done(this._data[id]);
|
||||
},
|
||||
|
||||
get: function ResSet_get(onComplete, id) {
|
||||
this._hack.async(this, onComplete, "GET", id);
|
||||
},
|
||||
|
||||
put: function ResSet_put(onComplete, id, data) {
|
||||
this._hack.async(this, onComplete, "PUT", id, data);
|
||||
},
|
||||
|
||||
delete: function ResSet_delete(onComplete, id) {
|
||||
this._hack.async(this, onComplete, "DELETE", id);
|
||||
}
|
||||
};
|
||||
|
||||
function ResourceFilter() {
|
||||
this._log = Log4Moz.Service.getLogger("Service.ResourceFilter");
|
||||
}
|
||||
@ -298,18 +341,6 @@ CryptoFilter.prototype = {
|
||||
}
|
||||
};
|
||||
|
||||
function Status(remoteStore) {
|
||||
this._init(remoteStore);
|
||||
}
|
||||
Status.prototype = {
|
||||
__proto__: new Resource(),
|
||||
_init: function Status__init(remoteStore) {
|
||||
this._remote = remoteStore;
|
||||
this.__proto__.__proto__._init.call(this, this._remote.serverPrefix + "status.json");
|
||||
this.pushFilter(new JsonFilter());
|
||||
}
|
||||
};
|
||||
|
||||
function Keychain(remoteStore) {
|
||||
this._init(remoteStore);
|
||||
}
|
||||
@ -352,7 +383,8 @@ RemoteStore.prototype = {
|
||||
get engineId() this._engine.engineId,
|
||||
|
||||
get status() {
|
||||
let status = new Status(this);
|
||||
let status = new Resource(this.serverPrefix + "status.json");
|
||||
status.pushFilter(new JsonFilter());
|
||||
this.__defineGetter__("status", function() status);
|
||||
return status;
|
||||
},
|
||||
@ -372,7 +404,7 @@ RemoteStore.prototype = {
|
||||
},
|
||||
|
||||
get _deltas() {
|
||||
let deltas = new Resource(this.serverPrefix + "deltas.json");
|
||||
let deltas = new ResourceSet(this.serverPrefix + "deltas/");
|
||||
deltas.pushFilter(new JsonFilter());
|
||||
deltas.pushFilter(new CryptoFilter(this, "deltasEncryption"));
|
||||
this.__defineGetter__("_deltas", function() deltas);
|
||||
@ -390,8 +422,7 @@ RemoteStore.prototype = {
|
||||
this._snapshot.data = null;
|
||||
this._deltas.data = null;
|
||||
|
||||
DAV.MKCOL(this.serverPrefix, self.cb);
|
||||
let ret = yield;
|
||||
let ret = yield DAV.MKCOL(this.serverPrefix + "deltas", self.cb);
|
||||
if (!ret)
|
||||
throw "Could not create remote folder";
|
||||
|
||||
@ -421,6 +452,7 @@ RemoteStore.prototype = {
|
||||
},
|
||||
|
||||
// Does a fresh upload of the given snapshot to a new store
|
||||
// FIXME: add 'metadata' arg here like appendDelta's
|
||||
_initialize: function RStore__initialize(snapshot) {
|
||||
let self = yield;
|
||||
let wrappedSymkey;
|
||||
@ -440,7 +472,7 @@ RemoteStore.prototype = {
|
||||
yield this.keys.put(self.cb, keys);
|
||||
|
||||
yield this._snapshot.put(self.cb, snapshot.data);
|
||||
yield this._deltas.put(self.cb, []);
|
||||
//yield this._deltas.put(self.cb, []);
|
||||
|
||||
let c = 0;
|
||||
for (GUID in snapshot.data)
|
||||
@ -467,7 +499,7 @@ RemoteStore.prototype = {
|
||||
yield this.status.delete(self.cb);
|
||||
yield this.keys.delete(self.cb);
|
||||
yield this._snapshot.delete(self.cb);
|
||||
yield this._deltas.delete(self.cb);
|
||||
//yield this._deltas.delete(self.cb);
|
||||
this._log.debug("Server files deleted");
|
||||
},
|
||||
wipe: function Engine_wipe(onComplete) {
|
||||
@ -478,20 +510,17 @@ RemoteStore.prototype = {
|
||||
// (snapshot + deltas)
|
||||
_getLatestFromScratch: function RStore__getLatestFromScratch() {
|
||||
let self = yield;
|
||||
let status = this.status.data;
|
||||
|
||||
this._log.info("Downloading all server data from scratch");
|
||||
|
||||
this._log.debug("Downloading server snapshot");
|
||||
let data = yield this._snapshot.get(self.cb);
|
||||
this._log.debug("Downloading server deltas");
|
||||
let deltas = yield this._deltas.get(self.cb);
|
||||
|
||||
let snap = new SnapshotStore();
|
||||
snap.version = this.status.data.maxVersion;
|
||||
snap.data = data;
|
||||
for (let i = 0; i < deltas.length; i++) {
|
||||
snap.applyCommands.async(snap, self.cb, deltas[i]);
|
||||
yield;
|
||||
snap.data = yield this._snapshot.get(self.cb);
|
||||
snap.version = status.maxVersion;
|
||||
|
||||
for (let id = status.snapVersion + 1; id <= status.maxVersion; id++) {
|
||||
let delta = yield this._deltas.get(self.cb, id);
|
||||
yield snap.applyCommands.async(snap, self.cb, delta);
|
||||
}
|
||||
|
||||
self.done(snap);
|
||||
@ -514,8 +543,12 @@ RemoteStore.prototype = {
|
||||
this._log.debug("Using last sync snapshot as starting point for server snapshot");
|
||||
snap.data = Utils.deepCopy(lastSyncSnap.data);
|
||||
this._log.info("Downloading server deltas");
|
||||
let allDeltas = yield this._deltas.get(self.cb);
|
||||
deltas = allDeltas.slice(lastSyncSnap.version - this.status.data.snapVersion);
|
||||
deltas = [];
|
||||
let min = lastSyncSnap.version + 1;
|
||||
let max = this.status.data.maxVersion;
|
||||
for (let id = min; id <= max; id++) {
|
||||
deltas.push(yield this._deltas.get(self.cb, id));
|
||||
}
|
||||
|
||||
} else if (lastSyncSnap.version == this.status.data.maxVersion) {
|
||||
this._log.debug("Using last sync snapshot as server snapshot (snap version == max version)");
|
||||
@ -557,17 +590,19 @@ RemoteStore.prototype = {
|
||||
},
|
||||
|
||||
// Adds a new set of changes (a delta) to this store
|
||||
_appendDelta: function RStore__appendDelta(delta) {
|
||||
_appendDelta: function RStore__appendDelta(delta, metadata) {
|
||||
let self = yield;
|
||||
if (this._deltas.data == null) {
|
||||
yield this._deltas.get(self.cb);
|
||||
if (this._deltas.data == null)
|
||||
this._deltas.data = [];
|
||||
|
||||
if (metadata) {
|
||||
for (let key in metadata)
|
||||
this.status.data[key] = metadata[key];
|
||||
}
|
||||
this._deltas.data.push(delta);
|
||||
yield this._deltas.put(self.cb, this._deltas.data);
|
||||
|
||||
let id = this.status.data.maxVersion; // FIXME: we increment maxVersion in Engine
|
||||
yield this._deltas.put(self.cb, id, delta);
|
||||
yield this.status.put(self.cb, this.status.data);
|
||||
},
|
||||
appendDelta: function RStore_appendDelta(onComplete, delta) {
|
||||
this._appendDelta.async(this, onComplete, delta);
|
||||
appendDelta: function RStore_appendDelta(onComplete, delta, metadata) {
|
||||
this._appendDelta.async(this, onComplete, delta, metadata);
|
||||
}
|
||||
};
|
||||
|
Loading…
Reference in New Issue
Block a user