Bug 1317841 - Update version of kinto.js to v6.0.0, r=MattN

This major version of kinto.js released without a FirefoxStorage
adapter. Since Gecko is the only project with a need for this adapter
as well as the only one who can use it, that file moves to this repo
as a new kinto-storage-adapter.js. This file is mostly a copy of the
most recent FirefoxStorage.js, plus some other non-exported utility
functions copied from kinto.js to make it work.

This changes the export of the kinto-offline-client.js from a
`loadKinto` function which returns the Kinto class, to the actual
Kinto class itself, with the user expected to "hook up" the
adapter. It turns out that this doesn't change much in how we actually
use Kinto, since we were always previously explicitly selecting the
Kinto adapter.

This release of kinto.js also changes some of the build options
around, which changes the minified output of kinto-offline-client.js.

There are still some outstanding stylistic complaints about
FirefoxAdapter having to do with its _init static method which is
public and the fact that sometimes FirefoxAdapter manages its own
Sqlite connection and sometimes it doesn't. These will be addressed in
a future patch.

MozReview-Commit-ID: HF0oNCEDcFs

--HG--
rename : services/common/kinto-offline-client.js => services/common/kinto-storage-adapter.js
extra : rebase_source : 11d01e573b259798305494ac072575247ac01e2c

services/common/blocklist-clients.js

@@ -19,8 +19,9 @@ const { Task } = Cu.import("resource://gre/modules/Task.jsm");
 const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
 Cu.importGlobalProperties(["fetch"]);
 
-const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js");
 const { KintoHttpClient } = Cu.import("resource://services-common/kinto-http-client.js");
+const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js");
 const { CanonicalJSON } = Components.utils.import("resource://gre/modules/CanonicalJSON.jsm");
 
 const PREF_SETTINGS_SERVER                   = "services.settings.server";
@@ -91,10 +92,6 @@ function kintoClient() {
   let base = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
   let bucket = Services.prefs.getCharPref(PREF_BLOCKLIST_BUCKET);
 
-  let Kinto = loadKinto();
-
-  let FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
-
   let config = {
     remote: base,
     bucket: bucket,

services/common/kinto-storage-adapter.js

@@ -0,0 +1,463 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+const { utils: Cu } = Components;
+const { Sqlite } = Cu.import("resource://gre/modules/Sqlite.jsm");
+const { Task } = Cu.import("resource://gre/modules/Task.jsm");
+const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+
+const SQLITE_PATH = "kinto.sqlite";
+
+/**
+ * Filter and sort list against provided filters and order.
+ *
+ * @param  {Object} filters  The filters to apply.
+ * @param  {String} order    The order to apply.
+ * @param  {Array}  list     The list to reduce.
+ * @return {Array}
+ */
+function reduceRecords(filters, order, list) {
+  const filtered = filters ? filterObjects(filters, list) : list;
+  return order ? sortObjects(order, filtered) : filtered;
+}
+
+/**
+ * Checks if a value is undefined.
+ *
+ * This is a copy of `_isUndefined` from kinto.js/src/utils.js.
+ * @param  {Any}  value
+ * @return {Boolean}
+ */
+function _isUndefined(value) {
+  return typeof value === "undefined";
+}
+
+/**
+ * Sorts records in a list according to a given ordering.
+ *
+ * This is a copy of `sortObjects` from kinto.js/src/utils.js.
+ *
+ * @param  {String} order The ordering, eg. `-last_modified`.
+ * @param  {Array}  list  The collection to order.
+ * @return {Array}
+ */
+function sortObjects(order, list) {
+  const hasDash = order[0] === "-";
+  const field = hasDash ? order.slice(1) : order;
+  const direction = hasDash ? -1 : 1;
+  return list.slice().sort((a, b) => {
+    if (a[field] && _isUndefined(b[field])) {
+      return direction;
+    }
+    if (b[field] && _isUndefined(a[field])) {
+      return -direction;
+    }
+    if (_isUndefined(a[field]) && _isUndefined(b[field])) {
+      return 0;
+    }
+    return a[field] > b[field] ? direction : -direction;
+  });
+}
+
+/**
+ * Test if a single object matches all given filters.
+ *
+ * This is a copy of `filterObject` from kinto.js/src/utils.js.
+ *
+ * @param  {Object} filters  The filters object.
+ * @param  {Object} entry    The object to filter.
+ * @return {Function}
+ */
+function filterObject(filters, entry) {
+  return Object.keys(filters).every(filter => {
+    const value = filters[filter];
+    if (Array.isArray(value)) {
+      return value.some(candidate => candidate === entry[filter]);
+    }
+    return entry[filter] === value;
+  });
+}
+
+/**
+ * Filters records in a list matching all given filters.
+ *
+ * This is a copy of `filterObjects` from kinto.js/src/utils.js.
+ *
+ * @param  {Object} filters  The filters object.
+ * @param  {Array}  list     The collection to filter.
+ * @return {Array}
+ */
+function filterObjects(filters, list) {
+  return list.filter((entry) => {
+    return filterObject(filters, entry);
+  });
+}
+
+const statements = {
+  "createCollectionData": `
+    CREATE TABLE collection_data (
+      collection_name TEXT,
+      record_id TEXT,
+      record TEXT
+    );`,
+
+  "createCollectionMetadata": `
+    CREATE TABLE collection_metadata (
+      collection_name TEXT PRIMARY KEY,
+      last_modified INTEGER
+    ) WITHOUT ROWID;`,
+
+  "createCollectionDataRecordIdIndex": `
+    CREATE UNIQUE INDEX unique_collection_record
+      ON collection_data(collection_name, record_id);`,
+
+  "clearData": `
+    DELETE FROM collection_data
+      WHERE collection_name = :collection_name;`,
+
+  "createData": `
+    INSERT INTO collection_data (collection_name, record_id, record)
+      VALUES (:collection_name, :record_id, :record);`,
+
+  "updateData": `
+    INSERT OR REPLACE INTO collection_data (collection_name, record_id, record)
+      VALUES (:collection_name, :record_id, :record);`,
+
+  "deleteData": `
+    DELETE FROM collection_data
+      WHERE collection_name = :collection_name
+      AND record_id = :record_id;`,
+
+  "saveLastModified": `
+    REPLACE INTO collection_metadata (collection_name, last_modified)
+      VALUES (:collection_name, :last_modified);`,
+
+  "getLastModified": `
+    SELECT last_modified
+      FROM collection_metadata
+        WHERE collection_name = :collection_name;`,
+
+  "getRecord": `
+    SELECT record
+      FROM collection_data
+        WHERE collection_name = :collection_name
+        AND record_id = :record_id;`,
+
+  "listRecords": `
+    SELECT record
+      FROM collection_data
+        WHERE collection_name = :collection_name;`,
+
+  // N.B. we have to have a dynamic number of placeholders, which you
+  // can't do without building your own statement. See `execute` for details
+  "listRecordsById": `
+    SELECT record_id, record
+      FROM collection_data
+        WHERE collection_name = ?
+          AND record_id IN `,
+
+  "importData": `
+    REPLACE INTO collection_data (collection_name, record_id, record)
+      VALUES (:collection_name, :record_id, :record);`,
+
+  "scanAllRecords": `SELECT * FROM collection_data;`,
+
+  "clearCollectionMetadata": `DELETE FROM collection_metadata;`
+};
+
+const createStatements = ["createCollectionData", "createCollectionMetadata", "createCollectionDataRecordIdIndex"];
+
+const currentSchemaVersion = 1;
+
+/**
+ * Firefox adapter.
+ *
+ * Uses Sqlite as a backing store.
+ *
+ * Options:
+ *  - path: the filename/path for the Sqlite database. If absent, use SQLITE_PATH.
+ */
+class FirefoxAdapter extends Kinto.adapters.BaseAdapter {
+  constructor(collection, options = {}) {
+    super();
+    const { sqliteHandle = null } = options;
+    this.collection = collection;
+    this._connection = sqliteHandle;
+    this._options = options;
+  }
+
+  // We need to be capable of calling this from "outside" the adapter
+  // so that someone can initialize a connection and pass it to us in
+  // adapterOptions.
+  static _init(connection) {
+    return Task.spawn(function* () {
+      yield connection.executeTransaction(function* doSetup() {
+        const schema = yield connection.getSchemaVersion();
+
+        if (schema == 0) {
+
+          for (let statementName of createStatements) {
+            yield connection.execute(statements[statementName]);
+          }
+
+          yield connection.setSchemaVersion(currentSchemaVersion);
+        } else if (schema != 1) {
+          throw new Error("Unknown database schema: " + schema);
+        }
+      });
+      return connection;
+    });
+  }
+
+  _executeStatement(statement, params) {
+    if (!this._connection) {
+      throw new Error("The storage adapter is not open");
+    }
+    return this._connection.executeCached(statement, params);
+  }
+
+  open() {
+    const self = this;
+    return Task.spawn(function* () {
+      if (!self._connection) {
+        const path = self._options.path || SQLITE_PATH;
+        const opts = { path, sharedMemoryCache: false };
+        self._connection = yield Sqlite.openConnection(opts).then(FirefoxAdapter._init);
+      }
+    });
+  }
+
+  close() {
+    if (this._connection) {
+      const promise = this._connection.close();
+      this._connection = null;
+      return promise;
+    }
+    return Promise.resolve();
+  }
+
+  clear() {
+    const params = { collection_name: this.collection };
+    return this._executeStatement(statements.clearData, params);
+  }
+
+  execute(callback, options = { preload: [] }) {
+    if (!this._connection) {
+      throw new Error("The storage adapter is not open");
+    }
+
+    let result;
+    const conn = this._connection;
+    const collection = this.collection;
+
+    return conn.executeTransaction(function* doExecuteTransaction() {
+      // Preload specified records from DB, within transaction.
+      const parameters = [collection, ...options.preload];
+      const placeholders = options.preload.map(_ => "?");
+      const stmt = statements.listRecordsById + "(" + placeholders.join(",") + ");";
+      const rows = yield conn.execute(stmt, parameters);
+
+      const preloaded = rows.reduce((acc, row) => {
+        const record = JSON.parse(row.getResultByName("record"));
+        acc[row.getResultByName("record_id")] = record;
+        return acc;
+      }, {});
+
+      const proxy = transactionProxy(collection, preloaded);
+      result = callback(proxy);
+
+      for (let { statement, params } of proxy.operations) {
+        yield conn.executeCached(statement, params);
+      }
+    }, conn.TRANSACTION_EXCLUSIVE).then(_ => result);
+  }
+
+  get(id) {
+    const params = {
+      collection_name: this.collection,
+      record_id: id
+    };
+    return this._executeStatement(statements.getRecord, params).then(result => {
+      if (result.length == 0) {
+        return;
+      }
+      return JSON.parse(result[0].getResultByName("record"));
+    });
+  }
+
+  list(params = { filters: {}, order: "" }) {
+    const parameters = {
+      collection_name: this.collection
+    };
+    return this._executeStatement(statements.listRecords, parameters).then(result => {
+      const records = [];
+      for (let k = 0; k < result.length; k++) {
+        const row = result[k];
+        records.push(JSON.parse(row.getResultByName("record")));
+      }
+      return records;
+    }).then(results => {
+      // The resulting list of records is filtered and sorted.
+      // XXX: with some efforts, this could be implemented using SQL.
+      return reduceRecords(params.filters, params.order, results);
+    });
+  }
+
+  /**
+   * Load a list of records into the local database.
+   *
+   * Note: The adapter is not in charge of filtering the already imported
+   * records. This is done in `Collection#loadDump()`, as a common behaviour
+   * between every adapters.
+   *
+   * @param  {Array} records.
+   * @return {Array} imported records.
+   */
+  loadDump(records) {
+    const connection = this._connection;
+    const collection_name = this.collection;
+    return Task.spawn(function* () {
+      yield connection.executeTransaction(function* doImport() {
+        for (let record of records) {
+          const params = {
+            collection_name: collection_name,
+            record_id: record.id,
+            record: JSON.stringify(record)
+          };
+          yield connection.execute(statements.importData, params);
+        }
+        const lastModified = Math.max(...records.map(record => record.last_modified));
+        const params = {
+          collection_name: collection_name
+        };
+        const previousLastModified = yield connection.execute(statements.getLastModified, params).then(result => {
+          return result.length > 0 ? result[0].getResultByName("last_modified") : -1;
+        });
+        if (lastModified > previousLastModified) {
+          const params = {
+            collection_name: collection_name,
+            last_modified: lastModified
+          };
+          yield connection.execute(statements.saveLastModified, params);
+        }
+      });
+      return records;
+    });
+  }
+
+  saveLastModified(lastModified) {
+    const parsedLastModified = parseInt(lastModified, 10) || null;
+    const params = {
+      collection_name: this.collection,
+      last_modified: parsedLastModified
+    };
+    return this._executeStatement(statements.saveLastModified, params).then(() => parsedLastModified);
+  }
+
+  getLastModified() {
+    const params = {
+      collection_name: this.collection
+    };
+    return this._executeStatement(statements.getLastModified, params).then(result => {
+      if (result.length == 0) {
+        return 0;
+      }
+      return result[0].getResultByName("last_modified");
+    });
+  }
+
+  /**
+   * Reset the sync status of every record and collection we have
+   * access to.
+   */
+  resetSyncStatus() {
+    // We're going to use execute instead of executeCached, so build
+    // in our own sanity check
+    if (!this._connection) {
+      throw new Error("The storage adapter is not open");
+    }
+
+    return this._connection.executeTransaction(function* (conn) {
+      const promises = [];
+      yield conn.execute(statements.scanAllRecords, null, function (row) {
+        const record = JSON.parse(row.getResultByName("record"));
+        const record_id = row.getResultByName("record_id");
+        const collection_name = row.getResultByName("collection_name");
+        if (record._status === "deleted") {
+          // Garbage collect deleted records.
+          promises.push(conn.execute(statements.deleteData, { collection_name, record_id }));
+        }
+        else {
+          const newRecord = Object.assign({}, record, {
+            _status: "created",
+            last_modified: undefined
+          });
+          promises.push(conn.execute(statements.updateData, { record: JSON.stringify(newRecord), record_id, collection_name }));
+        }
+      });
+      yield Promise.all(promises);
+      yield conn.execute(statements.clearCollectionMetadata);
+    });
+  }
+}
+
+
+function transactionProxy(collection, preloaded) {
+  const _operations = [];
+
+  return {
+    get operations() {
+      return _operations;
+    },
+
+    create(record) {
+      _operations.push({
+        statement: statements.createData,
+        params: {
+          collection_name: collection,
+          record_id: record.id,
+          record: JSON.stringify(record)
+        }
+      });
+    },
+
+    update(record) {
+      _operations.push({
+        statement: statements.updateData,
+        params: {
+          collection_name: collection,
+          record_id: record.id,
+          record: JSON.stringify(record)
+        }
+      });
+    },
+
+    delete(id) {
+      _operations.push({
+        statement: statements.deleteData,
+        params: {
+          collection_name: collection,
+          record_id: id
+        }
+      });
+    },
+
+    get(id) {
+      // Gecko JS engine outputs undesired warnings if id is not in preloaded.
+      return id in preloaded ? preloaded[id] : undefined;
+    }
+  };
+}
+this.FirefoxAdapter = FirefoxAdapter;
+
+this.EXPORTED_SYMBOLS = ["FirefoxAdapter"];

services/common/moz.build

@@ -19,6 +19,7 @@ EXTRA_JS_MODULES['services-common'] += [
     'blocklist-updater.js',
     'kinto-http-client.js',
     'kinto-offline-client.js',
+    'kinto-storage-adapter.js',
     'logmanager.js',
     'observers.js',
     'rest.js',

services/common/tests/unit/test_blocklist_certificates.js

@@ -3,7 +3,8 @@ const { Constructor: CC } = Components;
 Cu.import("resource://testing-common/httpd.js");
 
 const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
-const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js");
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
@@ -11,8 +12,6 @@ const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
 let server;
 
 // set up what we need to make storage adapters
-const Kinto = loadKinto();
-const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
 const kintoFilename = "kinto.sqlite";
 
 let kintoClient;

services/common/tests/unit/test_blocklist_clients.js

@@ -8,7 +8,8 @@ Cu.import("resource://gre/modules/Timer.jsm");
 const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm");
 const { OS } = Cu.import("resource://gre/modules/osfile.jsm");
 
-const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js");
 const BlocklistClients = Cu.import("resource://services-common/blocklist-clients.js");
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
@@ -26,8 +27,6 @@ let kintoClient;
 
 function kintoCollection(collectionName) {
   if (!kintoClient) {
-    const Kinto = loadKinto();
-    const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
     const config = {
       // Set the remote to be some server that will cause test failure when
       // hit since we should never hit the server directly, only via maybeSync()

services/common/tests/unit/test_blocklist_signatures.js

@@ -3,7 +3,8 @@
 Cu.import("resource://services-common/blocklist-updater.js");
 Cu.import("resource://testing-common/httpd.js");
 
-const { loadKinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { Kinto } = Cu.import("resource://services-common/kinto-offline-client.js");
+const { FirefoxAdapter } = Cu.import("resource://services-common/kinto-storage-adapter.js");
 const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
 const { OneCRLBlocklistClient } = Cu.import("resource://services-common/blocklist-clients.js");
 
@@ -60,10 +61,6 @@ function* checkRecordCount(count) {
   const collectionName =
       Services.prefs.getCharPref(PREF_BLOCKLIST_ONECRL_COLLECTION);
 
-  const Kinto = loadKinto();
-
-  const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
-
   const config = {
     remote: base,
     bucket: bucket,

services/common/tests/unit/test_kinto.js

@@ -2,6 +2,7 @@
    http://creativecommons.org/publicdomain/zero/1.0/ */
 
 Cu.import("resource://services-common/kinto-offline-client.js");
+Cu.import("resource://services-common/kinto-storage-adapter.js");
 Cu.import("resource://testing-common/httpd.js");
 
 const BinaryInputStream = Components.Constructor("@mozilla.org/binaryinputstream;1",
@@ -10,8 +11,6 @@ const BinaryInputStream = Components.Constructor("@mozilla.org/binaryinputstream
 var server;
 
 // set up what we need to make storage adapters
-const Kinto = loadKinto();
-const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
 const kintoFilename = "kinto.sqlite";
 
 let kintoClient;

services/common/tests/unit/test_storage_adapter.js

@@ -2,10 +2,9 @@
    http://creativecommons.org/publicdomain/zero/1.0/ */
 
 Cu.import("resource://services-common/kinto-offline-client.js");
+Cu.import("resource://services-common/kinto-storage-adapter.js");
 
 // set up what we need to make storage adapters
-const Kinto = loadKinto();
-const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
 const kintoFilename = "kinto.sqlite";
 
 let gFirefoxAdapter = null;

toolkit/components/extensions/ExtensionStorageSync.jsm

@@ -54,8 +54,10 @@ XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
                                   "resource://gre/modules/FxAccounts.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "KintoHttpClient",
                                   "resource://services-common/kinto-http-client.js");
-XPCOMUtils.defineLazyModuleGetter(this, "loadKinto",
+XPCOMUtils.defineLazyModuleGetter(this, "Kinto",
                                   "resource://services-common/kinto-offline-client.js");
+XPCOMUtils.defineLazyModuleGetter(this, "FirefoxAdapter",
+                                  "resource://services-common/kinto-storage-adapter.js");
 XPCOMUtils.defineLazyModuleGetter(this, "Log",
                                   "resource://gre/modules/Log.jsm");
 XPCOMUtils.defineLazyModuleGetter(this, "Observers",
@@ -93,15 +95,14 @@ const log = Log.repository.getLogger("Sync.Engine.Extension-Storage");
  *   collections in this database will use the same Sqlite connection.
  */
 const storageSyncInit = Task.spawn(function* () {
-  const Kinto = loadKinto();
   const path = "storage-sync.sqlite";
   const opts = {path, sharedMemoryCache: false};
   const connection = yield Sqlite.openConnection(opts);
-  yield Kinto.adapters.FirefoxAdapter._init(connection);
+  yield FirefoxAdapter._init(connection);
   return {
     connection,
     kinto: new Kinto({
-      adapter: Kinto.adapters.FirefoxAdapter,
+      adapter: FirefoxAdapter,
       adapterOptions: {sqliteHandle: connection},
       timeout: KINTO_REQUEST_TIMEOUT,
     }),