Bug 569968 - Migration requires client to remove absolute URLs in crypto records

Make WBORecord.uri, CryptoWrapper.encryption, PubKey.privateKeyUri, PrivKey.publicKeyUri as well as the CryptoMeta keyring keys relative URLs in the payload.  As a result, all records now *must* know their URI.

Bump storage version from 2 to 3 to trigger reupload of all data and exclude older clients.

services/sync/modules/base_records/collection.js

@@ -138,6 +138,12 @@ Collection.prototype = {
     // Save this because onProgress is called with this as the ChannelListener
     let coll = this;
 
+    // Prepare a dummyUri so that records can generate the correct
+    // relative URLs.  The last bit will be replaced with record.id.
+    let dummyUri = this.uri.clone().QueryInterface(Ci.nsIURL);
+    dummyUri.filePath += "/replaceme";
+    dummyUri.query = "";
+
     // Switch to newline separated records for incremental parsing
     coll.setHeader("Accept", "application/newlines");
 
@@ -149,9 +155,8 @@ Collection.prototype = {
         this._data = this._data.slice(newline + 1);
 
         // Deserialize a record from json and give it to the callback
-        let record = new coll._recordObj();
+        let record = new coll._recordObj(dummyUri);
         record.deserialize(json);
-        record.baseUri = coll.uri;
         onRecord(record);
       }
     };

services/sync/modules/base_records/crypto.js

@@ -49,13 +49,19 @@ Cu.import("resource://services-sync/util.js");
 function CryptoWrapper(uri) {
   this.cleartext = {};
   WBORecord.call(this, uri);
-  this.encryption = "";
   this.ciphertext = null;
 }
 CryptoWrapper.prototype = {
   __proto__: WBORecord.prototype,
   _logName: "Record.CryptoWrapper",
 
+  get encryption() {
+    return this.uri.resolve(this.payload.encryption);
+  },
+  set encryption(value) {
+    this.payload.encryption = this.uri.getRelativeSpec(Utils.makeURI(value));
+  },
+
   encrypt: function CryptoWrapper_encrypt(passphrase) {
     let pubkey = PubKeys.getDefaultKey();
     let privkey = PrivKeys.get(pubkey.privateKeyUri);
@@ -109,8 +115,7 @@ CryptoWrapper.prototype = {
   },
 };
 
-Utils.deferGetSet(CryptoWrapper, "payload", ["ciphertext", "encryption", "IV",
-                                             "hmac"]);
+Utils.deferGetSet(CryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]);
 Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted");
 
 function CryptoMeta(uri) {
@@ -180,7 +185,7 @@ CryptoMeta.prototype = {
 
     // Wrap the symmetric key and generate a HMAC for it
     let wrapped = Svc.Crypto.wrapSymmetricKey(symkey, new_pubkey.keyData);
-    this.keyring[new_pubkey.uri.spec] = {
+    this.keyring[this.uri.getRelativeSpec(new_pubkey.uri)] = {
       wrapped: wrapped,
       hmac: Utils.sha256HMAC(wrapped, this.hmacKey)
     };

services/sync/modules/base_records/keys.js

@@ -52,7 +52,6 @@ function PubKey(uri) {
   WBORecord.call(this, uri);
   this.type = "pubkey";
   this.keyData = null;
-  this.privateKeyUri = null;
 }
 PubKey.prototype = {
   __proto__: WBORecord.prototype,
@@ -66,13 +65,16 @@ PubKey.prototype = {
     let key = this.payload.privateKeyUri;
     return Utils.makeURI(this.uri.resolve(key) || key);
   },
+  set privateKeyUri(value) {
+    this.payload.privateKeyUri = this.uri.getRelativeSpec(Utils.makeURI(value));
+  },
 
   get publicKeyUri() {
     throw "attempted to get public key url from a public key!";
   }
 };
 
-Utils.deferGetSet(PubKey, "payload", ["keyData", "privateKeyUri", "type"]);
+Utils.deferGetSet(PubKey, "payload", ["keyData", "type"]);
 
 function PrivKey(uri) {
   WBORecord.call(this, uri);
@@ -80,7 +82,6 @@ function PrivKey(uri) {
   this.salt = null;
   this.iv = null;
   this.keyData = null;
-  this.publicKeyUri = null;
 }
 PrivKey.prototype = {
   __proto__: WBORecord.prototype,
@@ -94,13 +95,16 @@ PrivKey.prototype = {
     let key = this.payload.publicKeyUri;
     return Utils.makeURI(this.uri.resolve(key) || key);
   },
+  set publicKeyUri(value) {
+    this.payload.publicKeyUri = this.uri.getRelativeSpec(Utils.makeURI(value));
+  },
 
   get privateKeyUri() {
     throw "attempted to get private key url from a private key!";
   }
 };
 
-Utils.deferGetSet(PrivKey, "payload", ["salt", "iv", "keyData", "publicKeyUri", "type"]);
+Utils.deferGetSet(PrivKey, "payload", ["salt", "iv", "keyData", "type"]);
 
 // XXX unused/unfinished
 function SymKey(keyData, wrapped) {
@@ -137,9 +141,14 @@ PubKeyManager.prototype = {
   },
 
   createKeypair: function KeyMgr_createKeypair(passphrase, pubkeyUri, privkeyUri) {
+    if (!pubkeyUri)
+      throw "Missing or null parameter 'pubkeyUri'.";
+    if (!privkeyUri)
+      throw "Missing or null parameter 'privkeyUri'.";
+
     this._log.debug("Generating RSA keypair");
-    let pubkey = new PubKey();
-    let privkey = new PrivKey();
+    let pubkey = new PubKey(pubkeyUri);
+    let privkey = new PrivKey(privkeyUri);
     privkey.salt = Svc.Crypto.generateRandomBytes(16);
     privkey.iv = Svc.Crypto.generateRandomIV();
 
@@ -148,14 +157,8 @@ PubKeyManager.prototype = {
                                privkey.iv, pub, priv);
     [pubkey.keyData, privkey.keyData] = [pub.value, priv.value];
 
-    if (pubkeyUri) {
-      pubkey.uri = pubkeyUri;
-      privkey.publicKeyUri = pubkeyUri;
-    }
-    if (privkeyUri) {
-      privkey.uri = privkeyUri;
-      pubkey.privateKeyUri = privkeyUri;
-    }
+    pubkey.privateKeyUri = privkeyUri;
+    privkey.publicKeyUri = pubkeyUri;
 
     this._log.debug("Generating RSA keypair... done");
     return {pubkey: pubkey, privkey: privkey};

services/sync/modules/base_records/wbo.js

@@ -46,10 +46,12 @@ Cu.import("resource://services-sync/resource.js");
 Cu.import("resource://services-sync/util.js");
 
 function WBORecord(uri) {
+  if (uri == null)
+    throw "WBOs must have a URI!";
+
   this.data = {};
   this.payload = {};
-  if (uri)
-    this.uri = uri;
+  this.uri = uri;
 }
 WBORecord.prototype = {
   _logName: "Record.WBO",
@@ -57,14 +59,14 @@ WBORecord.prototype = {
   // NOTE: baseUri must have a trailing slash, or baseUri.resolve() will omit
   //       the collection name
   get uri() {
-    return Utils.makeURI(this.baseUri.resolve(encodeURI(this.id)));
+    return Utils.makeURL(this.baseUri.resolve(encodeURI(this.id)));
   },
   set uri(value) {
     if (typeof(value) != "string")
       value = value.spec;
-    let foo = value.split('/');
-    this.id = foo.pop();
-    this.baseUri = Utils.makeURI(foo.join('/') + '/');
+    let parts = value.split('/');
+    this.id = parts.pop();
+    this.baseUri = Utils.makeURI(parts.join('/') + '/');
   },
 
   get sortindex() {
@@ -122,9 +124,8 @@ RecordManager.prototype = {
       if (!this.response.success)
         return null;
 
-      let record = new this._recordType();
+      let record = new this._recordType(url);
       record.deserialize(this.response);
-      record.uri = url;
 
       return this.set(url, record);
     }

services/sync/modules/constants.js

@@ -44,7 +44,7 @@ WEAVE_ID:                              "@weave_id@",
 // Version of the data format this client supports. The data format describes
 // how records are packaged; this is separate from the Server API version and
 // the per-engine cleartext formats.
-STORAGE_VERSION:                       2,
+STORAGE_VERSION:                       3,
 
 UPDATED_DEV_URL:                       "https://services.mozilla.com/sync/updated/?version=@weave_version@&channel=@xpi_type@",
 UPDATED_REL_URL:                       "http://www.mozilla.com/firefox/sync/updated.html",

services/sync/modules/engines.js

@@ -337,7 +337,7 @@ SyncEngine.prototype = {
 
   // Create a new record using the store and add in crypto fields
   _createRecord: function SyncEngine__createRecord(id) {
-    let record = this._store.createRecord(id);
+    let record = this._store.createRecord(id, this.engineURL + "/" + id);
     record.id = id;
     record.encryption = this.cryptoMetaURL;
     return record;

services/sync/modules/engines/bookmarks.js

@@ -756,11 +756,11 @@ BookmarksStore.prototype = {
   },
 
   // Create a record starting from the weave id (places guid)
-  createRecord: function createRecord(guid) {
+  createRecord: function createRecord(guid, uri) {
     let placeId = idForGUID(guid);
     let record;
     if (placeId <= 0) { // deleted item
-      record = new PlacesItem();
+      record = new PlacesItem(uri);
       record.deleted = true;
       return record;
     }
@@ -770,14 +770,14 @@ BookmarksStore.prototype = {
     case this._bms.TYPE_BOOKMARK:
       let bmkUri = this._bms.getBookmarkURI(placeId).spec;
       if (this._ms && this._ms.hasMicrosummary(placeId)) {
-        record = new BookmarkMicsum();
+        record = new BookmarkMicsum(uri);
         let micsum = this._ms.getMicrosummary(placeId);
         record.generatorUri = micsum.generator.uri.spec; // breaks local generators
         record.staticTitle = this._getStaticTitle(placeId);
       }
       else {
         if (bmkUri.search(/^place:/) == 0) {
-          record = new BookmarkQuery();
+          record = new BookmarkQuery(uri);
 
           // Get the actual tag name instead of the local itemId
           let folder = bmkUri.match(/[:&]folder=(\d+)/);
@@ -792,7 +792,7 @@ BookmarksStore.prototype = {
           catch(ex) {}
         }
         else
-          record = new Bookmark();
+          record = new Bookmark(uri);
         record.title = this._bms.getItemTitle(placeId);
       }
 
@@ -806,7 +806,7 @@ BookmarksStore.prototype = {
 
     case this._bms.TYPE_FOLDER:
       if (this._ls.isLivemark(placeId)) {
-        record = new Livemark();
+        record = new Livemark(uri);
 
         let siteURI = this._ls.getSiteURI(placeId);
         if (siteURI != null)
@@ -814,7 +814,7 @@ BookmarksStore.prototype = {
         record.feedUri = this._ls.getFeedURI(placeId).spec;
 
       } else {
-        record = new BookmarkFolder();
+        record = new BookmarkFolder(uri);
       }
 
       record.parentName = Svc.Bookmark.getItemTitle(parent);
@@ -823,19 +823,19 @@ BookmarksStore.prototype = {
       break;
 
     case this._bms.TYPE_SEPARATOR:
-      record = new BookmarkSeparator();
+      record = new BookmarkSeparator(uri);
       // Create a positioning identifier for the separator
       record.parentName = Svc.Bookmark.getItemTitle(parent);
       record.pos = Svc.Bookmark.getItemIndex(placeId);
       break;
 
     case this._bms.TYPE_DYNAMIC_CONTAINER:
-      record = new PlacesItem();
+      record = new PlacesItem(uri);
       this._log.warn("Don't know how to serialize dynamic containers yet");
       break;
 
     default:
-      record = new PlacesItem();
+      record = new PlacesItem(uri);
       this._log.warn("Unknown item type, cannot serialize: " +
                      this._bms.getItemType(placeId));
     }

services/sync/modules/engines/clients.js

@@ -180,8 +180,8 @@ ClientStore.prototype = {
       this._remoteClients[record.id] = record.cleartext;
   },
 
-  createRecord: function createRecord(guid) {
-    let record = new ClientsRec();
+  createRecord: function createRecord(guid, uri) {
+    let record = new ClientsRec(uri);
 
     // Package the individual components into a record for the local client
     if (guid == Clients.localID) {

services/sync/modules/engines/forms.js

@@ -163,8 +163,8 @@ FormStore.prototype = {
     return FormWrapper.hasGUID(id);
   },
 
-  createRecord: function createRecord(guid) {
-    let record = new FormRec();
+  createRecord: function createRecord(guid, uri) {
+    let record = new FormRec(uri);
     let entry = FormWrapper.getEntry(guid);
     if (entry != null) {
       record.name = entry.name;

services/sync/modules/engines/history.js

@@ -279,9 +279,9 @@ HistoryStore.prototype = {
     return url ? this._hsvc.isVisited(url) : false;
   },
 
-  createRecord: function createRecord(guid) {
+  createRecord: function createRecord(guid, uri) {
     let foo = this._findURLByGUID(guid);
-    let record = new HistoryRec();
+    let record = new HistoryRec(uri);
     if (foo) {
       record.histUri = foo.url;
       record.title = foo.title;

services/sync/modules/engines/passwords.js

@@ -169,8 +169,8 @@ PasswordStore.prototype = {
     return false;
   },
 
-  createRecord: function createRecord(guid) {
-    let record = new LoginRec();
+  createRecord: function createRecord(guid, uri) {
+    let record = new LoginRec(uri);
     let login = this._getLoginFromGUID(guid);
 
     if (login) {

services/sync/modules/engines/prefs.js

@@ -199,8 +199,8 @@ PrefStore.prototype = {
     return (id === WEAVE_PREFS_GUID);
   },
 
-  createRecord: function createRecord(guid) {
-    let record = new PrefRec();
+  createRecord: function createRecord(guid, uri) {
+    let record = new PrefRec(uri);
 
     if (guid == WEAVE_PREFS_GUID) {
       record.value = this._getAllPrefs();

services/sync/modules/engines/tabs.js

@@ -146,8 +146,8 @@ TabStore.prototype = {
     return allTabs;
   },
 
-  createRecord: function createRecord(guid) {
-    let record = new TabSetRecord();
+  createRecord: function createRecord(guid, uri) {
+    let record = new TabSetRecord(uri);
     record.clientName = Clients.localName;
 
     // Don't provide any tabs to compare against and ignore the update later.

services/sync/modules/stores.js

@@ -86,7 +86,7 @@ Store.prototype = {
     throw "override itemExists in a subclass";
   },
 
-  createRecord: function Store_createRecord(id) {
+  createRecord: function Store_createRecord(id, uri) {
     throw "override createRecord in a subclass";
   },