Bug 1855455 - Implement a loadExtension() API in mozStorage. r=asuth

The API can only load from a predetermined list of extensions statically built
in the same library as SQLite (either nss3 or mozsqlite3 at this time).
New extensions must be audited and their code updated through the update.sh
script.
All the extensions should be compilable and usable across all the tier1
platforms and from Rusqlite.

Differential Revision: https://phabricator.services.mozilla.com/D191316

dom/cache/Connection.cpp

@@ -246,6 +246,11 @@ Connection::SetGrowthIncrement(int32_t aIncrement,
   return mBase->SetGrowthIncrement(aIncrement, aDatabase);
 }
 
+NS_IMETHODIMP
+Connection::LoadExtension(const nsACString& aExtensionName,
+                          mozIStorageCompletionCallback* aCallback) {
+  return mBase->LoadExtension(aExtensionName, aCallback);
+}
 NS_IMETHODIMP
 Connection::EnableModule(const nsACString& aModule) {
   return mBase->EnableModule(aModule);

storage/mozIStorageAsyncConnection.idl

@@ -282,6 +282,41 @@ interface mozIStorageAsyncConnection : nsISupports {
     in AUTF8String aSQLStatement,
     [optional] in mozIStorageStatementCallback aCallback);
 
+
+  /**
+   * Loads a Sqlite Run-Time Loadable Extension as defined at
+   * https://www.sqlite.org/loadext.html.
+   * Only a predetermined list of extensions can be loaded, that are statically
+   * linked in the shared library containing SQLite. The currently supported
+   * extensions are:
+   *   - fts5
+   *     A Full-Text search module, see https://www.sqlite.org/fts5.html
+   *
+   * New extensions can be added to the third_party/sqlite3/ext/ folder and then
+   * to this list, after a Storage peer has reviewed the request by verifying
+   * licensing, and code reliability.
+   * Extensions that must be loaded for all the connections should instead use
+   * sqlite3_auto_extension() (this must happen after sqlite3_config(), as it
+   * implicitly calls sqlite3_initialize()).
+   *
+   * @param aExtensionName
+   *        The extension to load, see the above list for supported values.
+   * @param aCallback
+   *        A callback that will be notified when the operation is complete,
+   *        with the following arguments:
+   *        - status: the status of the operation, use this to check if loading
+   *          the extension was successful as it may be partly asynchronous.
+   *        - value: unused.
+   * @throws NS_ERROR_INVALID_ARG
+   *         For unsupported extension names.
+   * @throws NS_ERROR_NOT_INITIALIZED
+   *         If the connection is not open.
+   * @throws NS_ERROR_UEXPECTED
+   *         If it was not possible to enable extensions loading.
+   */
+  void loadExtension(in AUTF8String aExtensionName,
+                     [optional] in mozIStorageCompletionCallback aCallback);
+
   //////////////////////////////////////////////////////////////////////////////
   //// Functions
 

storage/mozStorageConnection.cpp

@@ -5,6 +5,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 #include "BaseVFS.h"
+#include "ErrorList.h"
 #include "nsError.h"
 #include "nsThreadUtils.h"
 #include "nsIFile.h"
@@ -1869,6 +1870,17 @@ nsresult Connection::initializeClone(Connection* aClone, bool aReadOnly) {
     }
   }
 
+  // Load SQLite extensions that were on this connection.
+  // Copy into an array rather than holding the mutex while we load extensions.
+  nsTArray<nsCString> loadedExtensions;
+  {
+    MutexAutoLock lockedScope(sharedAsyncExecutionMutex);
+    AppendToArray(loadedExtensions, mLoadedExtensions);
+  }
+  for (const auto& extension : loadedExtensions) {
+    (void)aClone->LoadExtension(extension, nullptr);
+  }
+
   guard.release();
   return NS_OK;
 }
@@ -2541,6 +2553,103 @@ int32_t Connection::RemovablePagesInFreeList(const nsACString& aSchemaName) {
   return std::max(0, freeListPagesCount - (mGrowthChunkSize / pageSize));
 }
 
+NS_IMETHODIMP
+Connection::LoadExtension(const nsACString& aExtensionName,
+                          mozIStorageCompletionCallback* aCallback) {
+  AUTO_PROFILER_LABEL("Connection::LoadExtension", OTHER);
+
+  // This is a static list of extensions we can load.
+  // Please use lowercase ASCII names and keep this list alphabetically ordered.
+  static constexpr nsLiteralCString sSupportedExtensions[] = {
+      // clang-format off
+      "fts5"_ns,
+      // clang-format on
+  };
+  if (std::find(std::begin(sSupportedExtensions),
+                std::end(sSupportedExtensions),
+                aExtensionName) == std::end(sSupportedExtensions)) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  if (!connectionReady()) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  int srv = ::sqlite3_db_config(mDBConn, SQLITE_DBCONFIG_ENABLE_LOAD_EXTENSION,
+                                1, nullptr);
+  if (srv != SQLITE_OK) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  // Track the loaded extension for later connection cloning operations.
+  {
+    MutexAutoLock lockedScope(sharedAsyncExecutionMutex);
+    if (!mLoadedExtensions.EnsureInserted(aExtensionName)) {
+      // Already loaded, bail out but issue a warning.
+      NS_WARNING(nsPrintfCString(
+                     "Tried to register '%s' SQLite extension multiple times!",
+                     PromiseFlatCString(aExtensionName).get())
+                     .get());
+      return NS_OK;
+    }
+  }
+
+  nsAutoCString entryPoint("sqlite3_");
+  entryPoint.Append(aExtensionName);
+  entryPoint.AppendLiteral("_init");
+
+  RefPtr<Runnable> loadTask = NS_NewRunnableFunction(
+      "mozStorageConnection::LoadExtension",
+      [this, self = RefPtr(this), entryPoint,
+       callback = RefPtr(aCallback)]() mutable {
+        MOZ_ASSERT(
+            !NS_IsMainThread() ||
+                (operationSupported(Connection::SYNCHRONOUS) &&
+                 eventTargetOpenedOn == GetMainThreadSerialEventTarget()),
+            "Should happen on main-thread only for synchronous connections "
+            "opened on the main thread");
+#ifdef MOZ_FOLD_LIBS
+        int srv = ::sqlite3_load_extension(mDBConn,
+                                           MOZ_DLL_PREFIX "nss3" MOZ_DLL_SUFFIX,
+                                           entryPoint.get(), nullptr);
+#else
+        int srv = ::sqlite3_load_extension(
+            mDBConn, MOZ_DLL_PREFIX "mozsqlite3" MOZ_DLL_SUFFIX,
+            entryPoint.get(), nullptr);
+#endif
+        if (!callback) {
+          return;
+        };
+        RefPtr<Runnable> callbackTask = NS_NewRunnableFunction(
+            "mozStorageConnection::LoadExtension_callback",
+            [callback = std::move(callback), srv]() {
+              (void)callback->Complete(convertResultCode(srv), nullptr);
+            });
+        if (IsOnCurrentSerialEventTarget(eventTargetOpenedOn)) {
+          MOZ_ALWAYS_SUCCEEDS(callbackTask->Run());
+        } else {
+          // Redispatch the callback to the calling thread.
+          MOZ_ALWAYS_SUCCEEDS(eventTargetOpenedOn->Dispatch(
+              callbackTask.forget(), NS_DISPATCH_NORMAL));
+        }
+      });
+
+  if (NS_IsMainThread() && !operationSupported(Connection::SYNCHRONOUS)) {
+    // This is a main-thread call to an async-only connection, thus we should
+    // load the library in the helper thread.
+    nsIEventTarget* helperThread = getAsyncExecutionTarget();
+    if (!helperThread) {
+      return NS_ERROR_NOT_INITIALIZED;
+    }
+    MOZ_ALWAYS_SUCCEEDS(
+        helperThread->Dispatch(loadTask.forget(), NS_DISPATCH_NORMAL));
+  } else {
+    // In any other case we just load the extension on the current thread.
+    MOZ_ALWAYS_SUCCEEDS(loadTask->Run());
+  }
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 Connection::EnableModule(const nsACString& aModuleName) {
   if (!connectionReady()) {

storage/mozStorageConnection.h

@@ -15,6 +15,7 @@
 #include "nsIInterfaceRequestor.h"
 
 #include "nsTHashMap.h"
+#include "nsTHashSet.h"
 #include "mozIStorageProgressHandler.h"
 #include "SQLiteMutex.h"
 #include "mozIStorageConnection.h"
@@ -166,6 +167,7 @@ class Connection final : public mozIStorageConnection,
    *  - Connection.mAsyncExecutionThreadShuttingDown
    *  - Connection.mConnectionClosed
    *  - AsyncExecuteStatements.mCancelRequested
+   *  - Connection.mLoadedExtensions
    */
   Mutex sharedAsyncExecutionMutex MOZ_UNANNOTATED;
 
@@ -506,6 +508,14 @@ class Connection final : public mozIStorageConnection,
    * Stores the growth increment chunk size, set through SetGrowthIncrement().
    */
   Atomic<int32_t> mGrowthChunkSize;
+
+  /**
+   * Stores a list of the SQLite extensions loaded for this connections.
+   * This is used to properly clone the connection.
+   * @note Hold sharedAsyncExecutionMutex while using this.
+   */
+  nsTHashSet<nsCString> mLoadedExtensions
+      MOZ_GUARDED_BY(sharedAsyncExecutionMutex);
 };
 
 /**

storage/test/unit/head_storage.js

@@ -287,7 +287,7 @@ function asyncClone(db, readOnly) {
   return new Promise((resolve, reject) => {
     db.asyncClone(readOnly, function (status, db2) {
       if (Components.isSuccessCode(status)) {
-        resolve(db2);
+        resolve(db2.QueryInterface(Ci.mozIStorageAsyncConnection));
       } else {
         reject(status);
       }

storage/test/unit/test_storage_ext.js

@@ -0,0 +1,84 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file tests basics of loading SQLite extension.
+
+const VALID_EXTENSION_NAME = "fts5";
+
+add_setup(async function () {
+  cleanup();
+});
+
+add_task(async function test_valid_call() {
+  info("Testing valid call");
+  let conn = getOpenedUnsharedDatabase();
+
+  await new Promise((resolve, reject) => {
+    conn.loadExtension(VALID_EXTENSION_NAME, status => {
+      if (Components.isSuccessCode(status)) {
+        resolve();
+      } else {
+        reject(status);
+      }
+    });
+  });
+
+  cleanup();
+});
+
+add_task(async function test_invalid_calls() {
+  info("Testing invalid calls");
+  let conn = getOpenedUnsharedDatabase();
+
+  await Assert.rejects(
+    new Promise((resolve, reject) => {
+      conn.loadExtension("unknown", status => {
+        if (Components.isSuccessCode(status)) {
+          resolve();
+        } else {
+          reject(status);
+        }
+      });
+    }),
+    /NS_ERROR_ILLEGAL_VALUE/,
+    "Should fail loading unknown extension"
+  );
+
+  cleanup();
+
+  await Assert.rejects(
+    new Promise((resolve, reject) => {
+      conn.loadExtension(VALID_EXTENSION_NAME, status => {
+        if (Components.isSuccessCode(status)) {
+          resolve();
+        } else {
+          reject(status);
+        }
+      });
+    }),
+    /NS_ERROR_NOT_INITIALIZED/,
+    "Should fail loading extension on a closed connection"
+  );
+});
+
+add_task(async function test_more_invalid_calls() {
+  let conn = getOpenedUnsharedDatabase();
+  let promiseClosed = asyncClose(conn);
+
+  await Assert.rejects(
+    new Promise((resolve, reject) => {
+      conn.loadExtension(VALID_EXTENSION_NAME, status => {
+        if (Components.isSuccessCode(status)) {
+          resolve();
+        } else {
+          reject(status);
+        }
+      });
+    }),
+    /NS_ERROR_NOT_INITIALIZED/,
+    "Should fail loading extension on a closing connection"
+  );
+
+  await promiseClosed;
+});

storage/test/unit/test_storage_ext_fts5.js

@@ -0,0 +1,122 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// This file tests support for the fts5 extension.
+
+// Some example statements in this tests are taken from the SQLite FTS5
+// documentation page: https://sqlite.org/fts5.html
+
+add_setup(async function () {
+  cleanup();
+});
+
+add_task(async function test_synchronous() {
+  info("Testing synchronous connection");
+  let conn = getOpenedUnsharedDatabase();
+  Assert.throws(
+    () =>
+      conn.executeSimpleSQL(
+        "CREATE VIRTUAL TABLE email USING fts5(sender, title, body);"
+      ),
+    /NS_ERROR_FAILURE/,
+    "Should not be able to use FTS5 without loading the extension"
+  );
+
+  await loadFTS5Extension(conn);
+
+  conn.executeSimpleSQL(
+    "CREATE VIRTUAL TABLE email USING fts5(sender, title, body, tokenize='trigram');"
+  );
+
+  conn.executeSimpleSQL(
+    `INSERT INTO email(sender, title, body) VALUES
+    ('Mark', 'Fox', 'The quick brown fox jumps over the lazy dog.'),
+    ('Marco', 'Cat', 'The quick brown cat jumps over the lazy dog.'),
+    ('James', 'Hamster', 'The quick brown hamster jumps over the lazy dog.')`
+  );
+
+  var stmt = conn.createStatement(
+    `SELECT sender, title, highlight(email, 2, '<', '>')
+     FROM email
+     WHERE email MATCH 'ham'
+     ORDER BY bm25(email)`
+  );
+  Assert.ok(stmt.executeStep());
+  Assert.equal(stmt.getString(0), "James");
+  Assert.equal(stmt.getString(1), "Hamster");
+  Assert.equal(
+    stmt.getString(2),
+    "The quick brown <ham>ster jumps over the lazy dog."
+  );
+  stmt.reset();
+  stmt.finalize();
+
+  cleanup();
+});
+
+add_task(async function test_asynchronous() {
+  info("Testing asynchronous connection");
+  let conn = await openAsyncDatabase(getTestDB());
+
+  await Assert.rejects(
+    executeSimpleSQLAsync(
+      conn,
+      "CREATE VIRTUAL TABLE email USING fts5(sender, title, body);"
+    ),
+    err => err.message.startsWith("no such module"),
+    "Should not be able to use FTS5 without loading the extension"
+  );
+
+  await loadFTS5Extension(conn);
+
+  await executeSimpleSQLAsync(
+    conn,
+    "CREATE VIRTUAL TABLE email USING fts5(sender, title, body);"
+  );
+
+  await asyncClose(conn);
+  await IOUtils.remove(getTestDB().path, { ignoreAbsent: true });
+});
+
+add_task(async function test_clone() {
+  info("Testing cloning synchronous connection loads extensions in clone");
+  let conn1 = getOpenedUnsharedDatabase();
+  await loadFTS5Extension(conn1);
+
+  let conn2 = conn1.clone(false);
+  conn2.executeSimpleSQL(
+    "CREATE VIRTUAL TABLE email USING fts5(sender, title, body);"
+  );
+
+  conn2.close();
+  cleanup();
+});
+
+add_task(async function test_asyncClone() {
+  info("Testing asynchronously cloning connection loads extensions in clone");
+  let conn1 = getOpenedUnsharedDatabase();
+  await loadFTS5Extension(conn1);
+
+  let conn2 = await asyncClone(conn1, false);
+  await executeSimpleSQLAsync(
+    conn2,
+    "CREATE VIRTUAL TABLE email USING fts5(sender, title, body);"
+  );
+
+  await asyncClose(conn2);
+  await asyncClose(conn1);
+  await IOUtils.remove(getTestDB().path, { ignoreAbsent: true });
+});
+
+async function loadFTS5Extension(conn) {
+  await new Promise((resolve, reject) => {
+    conn.loadExtension("fts5", status => {
+      if (Components.isSuccessCode(status)) {
+        resolve();
+      } else {
+        reject(status);
+      }
+    });
+  });
+}

storage/test/unit/xpcshell.ini

@@ -41,8 +41,10 @@ skip-if = debug
 [test_storage_connection.js]
 # Bug 676981: test fails consistently on Android
 fail-if = os == "android"
+[test_storage_ext.js]
 [test_storage_ext_fts3.js]
 skip-if = appname != "thunderbird" && appname != "seamonkey"
+[test_storage_ext_fts5.js]
 [test_storage_function.js]
 [test_storage_progresshandler.js]
 [test_storage_service.js]