Bug 1621916 - Extract profile creation from test_unknownFiles.js into make_unknownFiles.js and make it more sophisticated; r=dom-workers-and-storage-reviewers,ttung

A generated profile now matches real world profiles:
- contains directory metadata
- contains real quota client specific files
- contains unique non-empty unknown files (instead of empty foo.bar files)
- contains unique non-empty unknown directories (instead of empty foo dirs)

There's now only one origin directory which contains unknown files and
directories. All methods that take a principal are verified to work with the
origin direcrtory.

Depends on D76095

Differential Revision: https://phabricator.services.mozilla.com/D76700

dom/quota/test/common/content.js

@@ -5,6 +5,8 @@
 
 const NS_ERROR_STORAGE_BUSY = SpecialPowers.Cr.NS_ERROR_STORAGE_BUSY;
 
+loadScript("dom/quota/test/common/global.js");
+
 function clearAllDatabases(callback) {
   let qms = SpecialPowers.Services.qms;
   let principal = SpecialPowers.wrap(document).nodePrincipal;

dom/quota/test/common/file.js

@@ -35,3 +35,11 @@ function compareBuffers(buffer1, buffer2) {
   }
   return true;
 }
+
+function getBlob(type, object) {
+  return new Blob([object], { type });
+}
+
+function getNullBlob(size) {
+  return getBlob("binary/null", getBuffer(size));
+}

dom/quota/test/common/global.js

@@ -0,0 +1,36 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+function openDBRequestUpgradeNeeded(request) {
+  return new Promise(function(resolve, reject) {
+    request.onerror = function(event) {
+      ok(false, "indexedDB error, '" + event.target.error.name + "'");
+      reject(event);
+    };
+    request.onupgradeneeded = function(event) {
+      resolve(event);
+    };
+    request.onsuccess = function(event) {
+      ok(false, "Got success, but did not expect it!");
+      reject(event);
+    };
+  });
+}
+
+function openDBRequestSucceeded(request) {
+  return new Promise(function(resolve, reject) {
+    request.onerror = function(event) {
+      ok(false, "indexedDB error, '" + event.target.error.name + "'");
+      reject(event);
+    };
+    request.onupgradeneeded = function(event) {
+      ok(false, "Got upgrade, but did not expect it!");
+      reject(event);
+    };
+    request.onsuccess = function(event) {
+      resolve(event);
+    };
+  });
+}

dom/quota/test/common/system.js

@@ -5,6 +5,8 @@
 
 const NS_ERROR_STORAGE_BUSY = Cr.NS_ERROR_STORAGE_BUSY;
 
+loadScript("dom/quota/test/common/global.js");
+
 function getProfileDir() {
   return Services.dirsvc.get("ProfD", Ci.nsIFile);
 }

dom/quota/test/moz.build

@@ -20,6 +20,7 @@ TEST_HARNESS_FILES.testing.mochitest.browser.dom.quota.test.common += [
     'common/browser.js',
     'common/content.js',
     'common/file.js',
+    'common/global.js',
     'common/nestedtest.js',
     'common/system.js',
 ]
@@ -27,6 +28,7 @@ TEST_HARNESS_FILES.testing.mochitest.browser.dom.quota.test.common += [
 TEST_HARNESS_FILES.testing.mochitest.tests.dom.quota.test.common += [
     'common/content.js',
     'common/file.js',
+    'common/global.js',
     'common/mochitest.js',
     'common/test_simpledb.js',
     'common/test_storage_manager_persist_allow.js',
@@ -36,6 +38,7 @@ TEST_HARNESS_FILES.testing.mochitest.tests.dom.quota.test.common += [
 
 TEST_HARNESS_FILES.xpcshell.dom.quota.test.common += [
     'common/file.js',
+    'common/global.js',
     'common/system.js',
     'common/test_simpledb.js',
     'common/xpcshell.js',

dom/quota/test/xpcshell/make_unknownFiles.js

@@ -0,0 +1,150 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+loadScript("dom/quota/test/common/file.js");
+
+async function testSteps() {
+  const principal = getPrincipal("http://example.com");
+
+  const repoRelativePath = "storage/default";
+  const originRelativePath = `${repoRelativePath}/http+++example.com`;
+
+  let unknownFileCounter = 1;
+  let unknownDirCounter = 1;
+
+  function createUnknownFileIn(dirRelativePath) {
+    const file = getRelativeFile(
+      dirRelativePath + "/foo" + unknownFileCounter + ".bar"
+    );
+
+    const ostream = Cc[
+      "@mozilla.org/network/file-output-stream;1"
+    ].createInstance(Ci.nsIFileOutputStream);
+
+    ostream.init(file, -1, parseInt("0644", 8), 0);
+
+    ostream.write("x".repeat(unknownFileCounter), unknownFileCounter);
+
+    ostream.close();
+
+    unknownFileCounter++;
+  }
+
+  function createUnknownDirectoryIn(dirRelativePath) {
+    createUnknownFileIn(dirRelativePath + "/foo" + unknownDirCounter++);
+  }
+
+  // storage.sqlite and storage/ls-archive.sqlite
+  {
+    const request = init();
+    await requestFinished(request);
+  }
+
+  // Unknown file in the repository
+  {
+    createUnknownFileIn(repoRelativePath);
+  }
+
+  // Unknown file in the origin directory
+  {
+    const request = initStorageAndOrigin(principal, "default");
+    await requestFinished(request);
+
+    ok(request.result === true, "The origin directory was created");
+
+    createUnknownFileIn(originRelativePath);
+  }
+
+  // Unknown directory in the origin directory
+  {
+    const request = initStorageAndOrigin(principal, "default");
+    await requestFinished(request);
+
+    ok(request.result === false, "The origin directory was not created");
+
+    createUnknownDirectoryIn(originRelativePath);
+  }
+
+  // Unknown file in idb client directory
+  {
+    const request = indexedDB.openForPrincipal(principal, "myIndexedDB");
+    await openDBRequestUpgradeNeeded(request);
+
+    const database = request.result;
+
+    const objectStore = database.createObjectStore("Blobs", {});
+
+    objectStore.add(getNullBlob(200), 42);
+
+    await openDBRequestSucceeded(request);
+
+    database.close();
+
+    // TODO: Fix IndexedDB to ignore unknown files as well
+    // createUnknownFileIn(`${originRelativePath}/idb`);
+  }
+
+  // Unknown file in cache client directory
+  {
+    async function sandboxScript() {
+      const cache = await caches.open("myCache");
+      const request = new Request("http://example.com/index.html");
+      const response = new Response("hello world");
+      await cache.put(request, response);
+    }
+
+    const sandbox = new Cu.Sandbox(principal, {
+      wantGlobalProperties: ["caches", "fetch"],
+    });
+
+    const promise = new Promise(function(resolve, reject) {
+      sandbox.resolve = resolve;
+      sandbox.reject = reject;
+    });
+
+    Cu.evalInSandbox(
+      sandboxScript.toSource() + " sandboxScript().then(resolve, reject);",
+      sandbox
+    );
+    await promise;
+
+    createUnknownFileIn(`${originRelativePath}/cache`);
+  }
+
+  // Unknown file in sdb client directory
+  {
+    const database = getSimpleDatabase(principal);
+
+    let request = database.open("mySimpleDB");
+    await requestFinished(request);
+
+    request = database.write(getBuffer(100));
+    await requestFinished(request);
+
+    request = database.close();
+    await requestFinished(request);
+
+    createUnknownFileIn(`${originRelativePath}/sdb`);
+  }
+
+  // Unknown file in ls client directory
+  {
+    Services.prefs.setBoolPref("dom.storage.testing", true);
+    Services.prefs.setBoolPref("dom.storage.client_validation", false);
+
+    const storage = Services.domStorageManager.createStorage(
+      null,
+      principal,
+      principal,
+      ""
+    );
+
+    storage.setItem("foo", "bar");
+
+    storage.close();
+
+    createUnknownFileIn(`${originRelativePath}/ls`);
+  }
+}

dom/quota/test/xpcshell/test_unknownFiles.js

@@ -13,81 +13,31 @@ async function testSteps() {
   const principal = getPrincipal("http://example.com");
 
   async function testFunctionality(testFunction) {
-    const originDirectory = getRelativeFile(
-      "storage/default/http+++example.com"
-    );
+    info("Clearing");
 
-    const unknownFiles = [
-      {
-        path: "storage/default/foo.bar",
-        dir: false,
-      },
+    let request = clear();
+    await requestFinished(request);
 
-      {
-        path: "storage/default/http+++foobar.com/foo.bar",
-        dir: false,
-      },
+    info("Installing package");
 
-      {
-        path: "storage/default/http+++foobar.com/foo",
-        dir: true,
-      },
+    // The profile contains unknown files and unknown directories placed across
+    // the repositories, origin directories and client directories. The file
+    // make_unknownFiles.js was run locally, specifically it was temporarily
+    // enabled in xpcshell.ini and then executed:
+    // mach test --interactive dom/quota/test/xpcshell/make_unknownFiles.js
+    installPackage("unknownFiles_profile");
 
-      // TODO: Fix IndexedDB to ignore unknown files as well
-      /*
-      {
-        path: "storage/default/http+++foobar.com/idb/foo.bar",
-        dir: false,
-      },
-      */
+    info("Initializing storage");
 
-      {
-        path: "storage/default/http+++foobar.com/cache/foo.bar",
-        dir: false,
-      },
+    request = init();
+    await requestFinished(request);
 
-      {
-        path: "storage/default/http+++foobar.com/sdb/foo.bar",
-        dir: false,
-      },
+    await testFunction();
 
-      {
-        path: "storage/default/http+++foobar.com/ls/foo.bar",
-        dir: false,
-      },
-    ];
+    info("Clearing");
 
-    for (const createOriginDirectory of [false, true]) {
-      info("Initializing storage");
-
-      let request = init();
-      await requestFinished(request);
-
-      if (createOriginDirectory) {
-        info("Creating origin directory");
-
-        originDirectory.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
-      }
-
-      info("Creating unknown files");
-
-      for (const unknownFile of unknownFiles) {
-        const file = getRelativeFile(unknownFile.path);
-
-        if (unknownFile.dir) {
-          file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
-        } else {
-          file.create(Ci.nsIFile.NORMAL_FILE_TYPE, parseInt("0644", 8));
-        }
-      }
-
-      await testFunction(createOriginDirectory);
-
-      info("Clearing");
-
-      request = clear();
-      await requestFinished(request);
-    }
+    request = clear();
+    await requestFinished(request);
   }
 
   info("Testing initTemporaryStorage functionality");
@@ -101,17 +51,13 @@ async function testSteps() {
 
   info("Testing initStorageAndOrigin functionality");
 
-  await testFunctionality(async function(createdOriginDirectory) {
+  await testFunctionality(async function() {
     info("Initializing origin");
 
     request = initStorageAndOrigin(principal, "default");
     await requestFinished(request);
 
-    if (createdOriginDirectory) {
-      ok(request.result === false, "The origin directory was not created");
-    } else {
-      ok(request.result === true, "The origin directory was created");
-    }
+    ok(request.result === false, "The origin directory was not created");
   });
 
   info("Testing getUsageForPrincipal functionality");
@@ -126,8 +72,10 @@ async function testSteps() {
       request.result instanceof Ci.nsIQuotaOriginUsageResult,
       "The result is nsIQuotaOriginUsageResult instance"
     );
-    ok(request.result.usage === 0, "The usage is 0");
-    ok(request.result.fileUsage === 0, "The fileUsage is 0");
+    // TODO: Fix SimpleDB to ignore unknown files during usage calculation
+    //is(request.result.usage, 115025, "Correct total usage");
+    is(request.result.usage, 115030, "Correct total usage");
+    is(request.result.fileUsage, 231, "Correct file usage");
   });
 
   info("Testing clearStoragesForPrincipal functionality");

dom/quota/test/xpcshell/xpcshell.ini

@@ -20,7 +20,10 @@ support-files =
   removeLocalStorage1_profile.zip
   removeLocalStorage2_profile.zip
   tempMetadataCleanup_profile.zip
+  unknownFiles_profile.zip
 
+[make_unknownFiles.js]
+skip-if = true # Only used for recreating unknownFiles_profile.zip
 [test_allowListFiles.js]
 [test_basics.js]
 [test_bad_origin_directory.js]