From 68b229f05112e08e079d191cbc1c422b68d1327a Mon Sep 17 00:00:00 2001
From: Luke Schwalfenberg <lschwalfenberg@gmail.com>
Date: Thu, 29 Nov 2018 14:19:29 +0000
Subject: [PATCH 01/78] Bug 1502856 - Display locked preferences in a different
 style. r=paolo

Differential Revision: https://phabricator.services.mozilla.com/D12871

--HG--
extra : rebase_source : 006612b428dd7f74eea43e585d53b30132904999
extra : amend_source : 2bf169f7307630547bcb855f12e3f4ea72662d41
---
 .../aboutconfig/content/aboutconfig.css       | 13 ++++
 .../aboutconfig/content/aboutconfig.js        | 19 +++--
 .../aboutconfig/test/browser/browser.ini      |  2 +
 .../test/browser/browser_locked.js            | 70 +++++++++++++++++++
 4 files changed, 100 insertions(+), 4 deletions(-)
 create mode 100644 browser/components/aboutconfig/test/browser/browser_locked.js

diff --git a/browser/components/aboutconfig/content/aboutconfig.css b/browser/components/aboutconfig/content/aboutconfig.css
index a39b683e0851..2fca8f31a893 100644
--- a/browser/components/aboutconfig/content/aboutconfig.css
+++ b/browser/components/aboutconfig/content/aboutconfig.css
@@ -14,6 +14,7 @@
   background-position: 9px center;
   background-size: 12px 12px;
   padding-left: 30px;
+  z-index: 1;
 }
 
 #prefs {
@@ -39,11 +40,23 @@
   font-weight: bold;
 }
 
+#prefs > tr.locked {
+  opacity: 0.4;
+  background-image: url("chrome://browser/skin/preferences/in-content/privacy-security.svg");
+  background-repeat: no-repeat;
+  background-position: 9px center;
+  background-size: 16px 16px;
+}
+
 #prefs > tr > td {
   padding: 4px;
   width: 50%;
 }
 
+#prefs > tr > td.cell-name {
+  padding-inline-start: 30px;
+}
+
 .cell-value {
   word-break: break-all;
 }
diff --git a/browser/components/aboutconfig/content/aboutconfig.js b/browser/components/aboutconfig/content/aboutconfig.js
index 3580bf5926be..2b26609b89e7 100644
--- a/browser/components/aboutconfig/content/aboutconfig.js
+++ b/browser/components/aboutconfig/content/aboutconfig.js
@@ -19,14 +19,16 @@ function onLoad() {
     let hasUserValue = Services.prefs.prefHasUserValue(name);
     let pref = {
       name,
-      value: Preferences.get(name),
       hasUserValue,
       hasDefaultValue: hasUserValue ? prefHasDefaultValue(name) : true,
+      isLocked: Services.prefs.prefIsLocked(name),
     };
-    // Try in case it's a localized string.
-    // Throws an exception if there is no equivalent value in the localized files for the pref.
+    // Try in case it's a localized string or locked user added pref
     // If an execption is thrown the pref value is set to the empty string.
     try {
+      // Throws an exception in case locked user added pref without default value
+      pref.value = Preferences.get(name);
+      // Throws an exception if there is no equivalent value in the localized files for the pref.
       if (!pref.hasUserValue && /^chrome:\/\/.+\/locale\/.+\.properties/.test(pref.value)) {
         pref.value = Services.prefs.getComplexValue(name, Ci.nsIPrefLocalizedString).data;
       }
@@ -121,6 +123,9 @@ function createPrefsFragment(prefArray) {
     if (pref.hasUserValue) {
       row.classList.add("has-user-value");
     }
+    if (pref.isLocked) {
+      row.classList.add("locked");
+    }
     row.setAttribute("aria-label", pref.name);
 
     row.appendChild(getPrefRow(pref));
@@ -135,6 +140,7 @@ function createNewPrefFragment(name) {
   row.classList.add("has-user-value");
   row.setAttribute("aria-label", name);
   let nameCell = document.createElement("td");
+  nameCell.className = "cell-name";
   nameCell.append(name);
   row.appendChild(nameCell);
 
@@ -164,6 +170,7 @@ function createNewPrefFragment(name) {
 function getPrefRow(pref) {
   let rowFragment = document.createDocumentFragment();
   let nameCell = document.createElement("td");
+  nameCell.className = "cell-name";
   // Add <wbr> behind dots to prevent line breaking in random mid-word places.
   let parts = pref.name.split(".");
   for (let i = 0; i < parts.length - 1; i++) {
@@ -187,11 +194,14 @@ function getPrefRow(pref) {
     document.l10n.setAttributes(button, "about-config-pref-edit");
     button.className = "button-edit";
   }
+  if (pref.isLocked) {
+    button.disabled = true;
+  }
   editCell.appendChild(button);
   rowFragment.appendChild(editCell);
 
   let buttonCell = document.createElement("td");
-  if (pref.hasUserValue) {
+  if (!pref.isLocked && pref.hasUserValue) {
     let resetButton = document.createElement("button");
     if (!pref.hasDefaultValue) {
       document.l10n.setAttributes(resetButton, "about-config-pref-delete");
@@ -201,6 +211,7 @@ function getPrefRow(pref) {
     }
     buttonCell.appendChild(resetButton);
   }
+
   rowFragment.appendChild(buttonCell);
   return rowFragment;
 }
diff --git a/browser/components/aboutconfig/test/browser/browser.ini b/browser/components/aboutconfig/test/browser/browser.ini
index bbd3d89aac8a..832af5a57ce4 100644
--- a/browser/components/aboutconfig/test/browser/browser.ini
+++ b/browser/components/aboutconfig/test/browser/browser.ini
@@ -5,3 +5,5 @@
 skip-if = debug # Bug 1507747
 [browser_search.js]
 skip-if = debug # Bug 1507747
+[browser_locked.js]
+
diff --git a/browser/components/aboutconfig/test/browser/browser_locked.js b/browser/components/aboutconfig/test/browser/browser_locked.js
new file mode 100644
index 000000000000..e5254e37e2aa
--- /dev/null
+++ b/browser/components/aboutconfig/test/browser/browser_locked.js
@@ -0,0 +1,70 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const PAGE_URL = "chrome://browser/content/aboutconfig/aboutconfig.html";
+
+add_task(async function setup() {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["test.aboutconfig.a", "some value"],
+    ],
+  });
+});
+
+add_task(async function test_locked() {
+  registerCleanupFunction(() => {
+    Services.prefs.unlockPref("browser.search.searchEnginesURL");
+    Services.prefs.unlockPref("test.aboutconfig.a");
+    Services.prefs.unlockPref("accessibility.AOM.enabled");
+  });
+
+  Services.prefs.lockPref("browser.search.searchEnginesURL");
+  Services.prefs.lockPref("test.aboutconfig.a");
+  Services.prefs.lockPref("accessibility.AOM.enabled");
+  await BrowserTestUtils.withNewTab({
+    gBrowser,
+    url: PAGE_URL,
+  }, async browser => {
+    await ContentTask.spawn(browser, null, () => {
+      let list = [...content.document.getElementById("prefs")
+        .getElementsByTagName("tr")];
+      function getRow(name) {
+        return list.find(row => row.querySelector("td").textContent == name);
+      }
+      function getValue(name) {
+          return getRow(name).querySelector("td.cell-value").textContent;
+      }
+      function getButton(name) {
+        return getRow(name).querySelector("button");
+      }
+
+      // Test locked default string pref.
+      let lockedPref = getRow("browser.search.searchEnginesURL");
+      Assert.ok(lockedPref.classList.contains("locked"));
+      Assert.equal(getValue("browser.search.searchEnginesURL"), "https://addons.mozilla.org/%LOCALE%/firefox/search-engines/");
+      Assert.equal(getButton("browser.search.searchEnginesURL").textContent, "Edit");
+      Assert.equal(getButton("browser.search.searchEnginesURL").disabled, true);
+
+      // Test locked default boolean pref.
+      lockedPref = getRow("accessibility.AOM.enabled");
+      Assert.ok(lockedPref.classList.contains("locked"));
+      Assert.equal(getValue("accessibility.AOM.enabled"), "false");
+      Assert.equal(getButton("accessibility.AOM.enabled").textContent, "Toggle");
+      Assert.equal(getButton("accessibility.AOM.enabled").disabled, true);
+
+      // Test locked user added pref.
+      lockedPref = getRow("test.aboutconfig.a");
+      Assert.ok(lockedPref.classList.contains("locked"));
+      Assert.equal(getValue("test.aboutconfig.a"), "");
+      Assert.equal(getButton("test.aboutconfig.a").textContent, "Edit");
+      Assert.equal(getButton("test.aboutconfig.a").disabled, true);
+
+      // Test pref not locked
+      let unlockedPref = getRow("accessibility.indicator.enabled");
+      Assert.equal(unlockedPref.classList.contains("locked"), false);
+      Assert.equal(getValue("accessibility.indicator.enabled"), "false");
+      Assert.equal(getButton("accessibility.indicator.enabled").textContent, "Toggle");
+      Assert.equal(getButton("accessibility.indicator.enabled").disabled, false);
+    });
+  });
+});

From fb1756d92cc0a1d959786c6252fdea3173ef9cae Mon Sep 17 00:00:00 2001
From: James Willcox <snorp@snorp.net>
Date: Thu, 29 Nov 2018 08:35:07 -0600
Subject: [PATCH 02/78] Bug 1507702 - Don't make about:crash* accessible to web
 content r=Ehsan,bzbarsky

Differential Revision: https://phabricator.services.mozilla.com/D12133
---
 docshell/base/nsAboutRedirector.cpp          | 43 ++++++++++++---
 docshell/test/mochitest/mochitest.ini        |  1 +
 docshell/test/mochitest/test_bug1507702.html | 57 ++++++++++++++++++++
 3 files changed, 93 insertions(+), 8 deletions(-)
 create mode 100644 docshell/test/mochitest/test_bug1507702.html

diff --git a/docshell/base/nsAboutRedirector.cpp b/docshell/base/nsAboutRedirector.cpp
index 5c95be2d6133..76a6bd2cac40 100644
--- a/docshell/base/nsAboutRedirector.cpp
+++ b/docshell/base/nsAboutRedirector.cpp
@@ -7,6 +7,7 @@
 #include "nsAboutRedirector.h"
 #include "nsNetUtil.h"
 #include "nsAboutProtocolUtils.h"
+#include "nsBaseChannel.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsIProtocolHandler.h"
 
@@ -27,6 +28,36 @@ struct RedirEntry
   uint32_t flags;
 };
 
+class CrashChannel final : public nsBaseChannel
+{
+public:
+  CrashChannel(nsIURI* aURI)
+  {
+    SetURI(aURI);
+  }
+
+  nsresult OpenContentStream(bool async, nsIInputStream **stream,
+                             nsIChannel** channel) override
+  {
+    nsAutoCString spec;
+    mURI->GetSpec(spec);
+
+    if (spec.EqualsASCII("about:crashparent") && XRE_IsParentProcess()) {
+      MOZ_CRASH("Crash via about:crashparent");
+    }
+
+    if (spec.EqualsASCII("about:crashcontent") && XRE_IsContentProcess()) {
+      MOZ_CRASH("Crash via about:crashcontent");
+    }
+
+    NS_WARNING("Unhandled about:crash* URI or wrong process");
+    return NS_ERROR_NOT_IMPLEMENTED;
+  }
+
+protected:
+  virtual ~CrashChannel() = default;
+};
+
 /*
   Entries which do not have URI_SAFE_FOR_UNTRUSTED_CONTENT will run with chrome
   privileges. This is potentially dangerous. Please use
@@ -145,12 +176,10 @@ static const RedirEntry kRedirMap[] = {
   },
   {
     "crashparent", "about:blank",
-    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT
   },
   {
     "crashcontent", "about:blank",
-    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT |
     nsIAboutModule::URI_CAN_LOAD_IN_CHILD |
     nsIAboutModule::URI_MUST_LOAD_IN_CHILD
@@ -174,12 +203,10 @@ nsAboutRedirector::NewChannel(nsIURI* aURI,
   nsCOMPtr<nsIIOService> ioService = do_GetIOService(&rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
-  if (XRE_IsParentProcess() && path.EqualsASCII("crashparent")) {
-    MOZ_CRASH("Crash via about:crashparent");
-  }
-
-  if (XRE_IsContentProcess() && path.EqualsASCII("crashcontent")) {
-    MOZ_CRASH("Crash via about:crashcontent");
+  if (path.EqualsASCII("crashparent") || path.EqualsASCII("crashcontent")) {
+    nsCOMPtr<nsIChannel> channel = new CrashChannel(aURI);
+    channel.forget(aResult);
+    return NS_OK;
   }
 
 #ifdef ABOUT_CONFIG_BLOCKED_GV
diff --git a/docshell/test/mochitest/mochitest.ini b/docshell/test/mochitest/mochitest.ini
index 4455fcb7d2f4..8f12715a905e 100644
--- a/docshell/test/mochitest/mochitest.ini
+++ b/docshell/test/mochitest/mochitest.ini
@@ -115,3 +115,4 @@ support-files = file_framedhistoryframes.html
 [test_pushState_after_document_open.html]
 [test_windowedhistoryframes.html]
 [test_triggeringprincipal_location_seturi.html]
+[test_bug1507702.html]
diff --git a/docshell/test/mochitest/test_bug1507702.html b/docshell/test/mochitest/test_bug1507702.html
new file mode 100644
index 000000000000..94e140df4b19
--- /dev/null
+++ b/docshell/test/mochitest/test_bug1507702.html
@@ -0,0 +1,57 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1507702
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1507702</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <link rel="icon" href="about:crashparent"/>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1507702">Mozilla Bug 1507702</a>
+<img src="about:crashparent">
+<img src="about:crashcontent">
+<iframe src="about:crashparent"></iframe>
+<iframe src="about:crashcontent"></iframe>
+<script>
+  let urls = ["about:crashparent", "about:crashcontent"];
+  async function testFetch() {
+    const url = urls.shift();
+    if (!url) {
+      return Promise.resolve();
+    }
+
+    let threw;
+    try {
+      await fetch(url);
+      threw = false;
+    } catch (e) {
+      threw = true;
+    };
+
+    ok(threw === true, "fetch should reject");
+    return testFetch();
+  }
+
+  document.body.onload = async () => {
+    for (const url of ["about:crashparent", "about:crashcontent"]) {
+      SimpleTest.doesThrow(() => {
+        top.location.href = url;
+      }, "navigation should throw");
+
+      SimpleTest.doesThrow(() => {
+        location.href = url;
+      }, "navigation should throw");
+    }
+
+    await testFetch();
+    SimpleTest.finish();
+  };
+
+  SimpleTest.waitForExplicitFinish();
+</script>
+</body>
+</html>

From c89eda391ed93c330252d7e5509a04e5de30a51a Mon Sep 17 00:00:00 2001
From: Daniel Varga <dvarga@mozilla.com>
Date: Thu, 29 Nov 2018 16:50:35 +0200
Subject: [PATCH 03/78] Backed out changeset 50171af401fc (bug 1507702) for
 build bustages at
 /builds/worker/workspace/build/src/docshell/base/nsAboutRedirector.cpp:34:3
 on a CLOSED TREE

--HG--
extra : amend_source : 3ee43847a33693e3a47ecc586561a21f41074f13
---
 docshell/base/nsAboutRedirector.cpp          | 43 +++------------
 docshell/test/mochitest/mochitest.ini        |  1 -
 docshell/test/mochitest/test_bug1507702.html | 57 --------------------
 3 files changed, 8 insertions(+), 93 deletions(-)
 delete mode 100644 docshell/test/mochitest/test_bug1507702.html

diff --git a/docshell/base/nsAboutRedirector.cpp b/docshell/base/nsAboutRedirector.cpp
index 76a6bd2cac40..5c95be2d6133 100644
--- a/docshell/base/nsAboutRedirector.cpp
+++ b/docshell/base/nsAboutRedirector.cpp
@@ -7,7 +7,6 @@
 #include "nsAboutRedirector.h"
 #include "nsNetUtil.h"
 #include "nsAboutProtocolUtils.h"
-#include "nsBaseChannel.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsIProtocolHandler.h"
 
@@ -28,36 +27,6 @@ struct RedirEntry
   uint32_t flags;
 };
 
-class CrashChannel final : public nsBaseChannel
-{
-public:
-  CrashChannel(nsIURI* aURI)
-  {
-    SetURI(aURI);
-  }
-
-  nsresult OpenContentStream(bool async, nsIInputStream **stream,
-                             nsIChannel** channel) override
-  {
-    nsAutoCString spec;
-    mURI->GetSpec(spec);
-
-    if (spec.EqualsASCII("about:crashparent") && XRE_IsParentProcess()) {
-      MOZ_CRASH("Crash via about:crashparent");
-    }
-
-    if (spec.EqualsASCII("about:crashcontent") && XRE_IsContentProcess()) {
-      MOZ_CRASH("Crash via about:crashcontent");
-    }
-
-    NS_WARNING("Unhandled about:crash* URI or wrong process");
-    return NS_ERROR_NOT_IMPLEMENTED;
-  }
-
-protected:
-  virtual ~CrashChannel() = default;
-};
-
 /*
   Entries which do not have URI_SAFE_FOR_UNTRUSTED_CONTENT will run with chrome
   privileges. This is potentially dangerous. Please use
@@ -176,10 +145,12 @@ static const RedirEntry kRedirMap[] = {
   },
   {
     "crashparent", "about:blank",
+    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT
   },
   {
     "crashcontent", "about:blank",
+    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT |
     nsIAboutModule::URI_CAN_LOAD_IN_CHILD |
     nsIAboutModule::URI_MUST_LOAD_IN_CHILD
@@ -203,10 +174,12 @@ nsAboutRedirector::NewChannel(nsIURI* aURI,
   nsCOMPtr<nsIIOService> ioService = do_GetIOService(&rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
-  if (path.EqualsASCII("crashparent") || path.EqualsASCII("crashcontent")) {
-    nsCOMPtr<nsIChannel> channel = new CrashChannel(aURI);
-    channel.forget(aResult);
-    return NS_OK;
+  if (XRE_IsParentProcess() && path.EqualsASCII("crashparent")) {
+    MOZ_CRASH("Crash via about:crashparent");
+  }
+
+  if (XRE_IsContentProcess() && path.EqualsASCII("crashcontent")) {
+    MOZ_CRASH("Crash via about:crashcontent");
   }
 
 #ifdef ABOUT_CONFIG_BLOCKED_GV
diff --git a/docshell/test/mochitest/mochitest.ini b/docshell/test/mochitest/mochitest.ini
index 8f12715a905e..4455fcb7d2f4 100644
--- a/docshell/test/mochitest/mochitest.ini
+++ b/docshell/test/mochitest/mochitest.ini
@@ -115,4 +115,3 @@ support-files = file_framedhistoryframes.html
 [test_pushState_after_document_open.html]
 [test_windowedhistoryframes.html]
 [test_triggeringprincipal_location_seturi.html]
-[test_bug1507702.html]
diff --git a/docshell/test/mochitest/test_bug1507702.html b/docshell/test/mochitest/test_bug1507702.html
deleted file mode 100644
index 94e140df4b19..000000000000
--- a/docshell/test/mochitest/test_bug1507702.html
+++ /dev/null
@@ -1,57 +0,0 @@
-<!DOCTYPE HTML>
-<html>
-<!--
-https://bugzilla.mozilla.org/show_bug.cgi?id=1507702
--->
-<head>
-  <meta charset="utf-8">
-  <title>Test for Bug 1507702</title>
-  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
-  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
-  <link rel="icon" href="about:crashparent"/>
-</head>
-<body>
-<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1507702">Mozilla Bug 1507702</a>
-<img src="about:crashparent">
-<img src="about:crashcontent">
-<iframe src="about:crashparent"></iframe>
-<iframe src="about:crashcontent"></iframe>
-<script>
-  let urls = ["about:crashparent", "about:crashcontent"];
-  async function testFetch() {
-    const url = urls.shift();
-    if (!url) {
-      return Promise.resolve();
-    }
-
-    let threw;
-    try {
-      await fetch(url);
-      threw = false;
-    } catch (e) {
-      threw = true;
-    };
-
-    ok(threw === true, "fetch should reject");
-    return testFetch();
-  }
-
-  document.body.onload = async () => {
-    for (const url of ["about:crashparent", "about:crashcontent"]) {
-      SimpleTest.doesThrow(() => {
-        top.location.href = url;
-      }, "navigation should throw");
-
-      SimpleTest.doesThrow(() => {
-        location.href = url;
-      }, "navigation should throw");
-    }
-
-    await testFetch();
-    SimpleTest.finish();
-  };
-
-  SimpleTest.waitForExplicitFinish();
-</script>
-</body>
-</html>

From ae4fa6387c4b8cb574a691f39546ef1c7575cf39 Mon Sep 17 00:00:00 2001
From: "Nicolas B. Pierron" <nicolas.b.pierron@nbp.name>
Date: Wed, 28 Nov 2018 15:27:35 +0100
Subject: [PATCH 04/78] Bug 1499323 - Prepare the check_macroassembler_style
 python script to accept clang-format rewritting. r=jandem

---
 config/check_macroassembler_style.py          | 64 +++++++++++++------
 js/src/jit/MacroAssembler-inl.h               |  2 -
 js/src/jit/MacroAssembler.cpp                 |  2 -
 js/src/jit/MacroAssembler.h                   |  4 --
 js/src/jit/arm/MacroAssembler-arm-inl.h       |  2 -
 js/src/jit/arm/MacroAssembler-arm.cpp         |  2 -
 js/src/jit/arm64/MacroAssembler-arm64-inl.h   |  2 -
 js/src/jit/arm64/MacroAssembler-arm64.cpp     |  2 -
 .../MacroAssembler-mips-shared-inl.h          |  2 -
 .../MacroAssembler-mips-shared.cpp            |  2 -
 js/src/jit/mips32/MacroAssembler-mips32-inl.h |  2 -
 js/src/jit/mips32/MacroAssembler-mips32.cpp   |  2 -
 js/src/jit/mips64/MacroAssembler-mips64-inl.h |  2 -
 js/src/jit/mips64/MacroAssembler-mips64.cpp   |  2 -
 js/src/jit/x64/MacroAssembler-x64-inl.h       |  2 -
 js/src/jit/x64/MacroAssembler-x64.cpp         |  2 -
 .../MacroAssembler-x86-shared-inl.h           |  2 -
 .../x86-shared/MacroAssembler-x86-shared.cpp  |  2 -
 js/src/jit/x86/MacroAssembler-x86-inl.h       |  2 -
 js/src/jit/x86/MacroAssembler-x86.cpp         |  2 -
 20 files changed, 43 insertions(+), 61 deletions(-)

diff --git a/config/check_macroassembler_style.py b/config/check_macroassembler_style.py
index d9b1383dcbd6..1b2dcb2fe599 100644
--- a/config/check_macroassembler_style.py
+++ b/config/check_macroassembler_style.py
@@ -48,6 +48,8 @@ def get_normalized_signatures(signature, fileAnnot=None):
     signature = signature.replace(';', ' ')
     # Normalize spaces.
     signature = re.sub(r'\s+', ' ', signature).strip()
+    # Remove new-line induced spaces after opening braces.
+    signature = re.sub(r'\(\s+', '(', signature).strip()
     # Match arguments, and keep only the type.
     signature = reMatchArg.sub('\g<type>', signature)
     # Remove class name
@@ -144,45 +146,62 @@ def get_macroassembler_definitions(filename):
         return []
 
     style_section = False
-    code_section = False
     lines = ''
     signatures = []
     with open(filename) as f:
         for line in f:
             if '//{{{ check_macroassembler_style' in line:
+                if style_section:
+                    raise 'check_macroassembler_style section already opened.'
                 style_section = True
+                braces_depth = 0
             elif '//}}} check_macroassembler_style' in line:
                 style_section = False
             if not style_section:
                 continue
 
+            # Remove comments from the processed line.
             line = re.sub(r'//.*', '', line)
-            if line.startswith('{') or line.strip() == "{}":
+
+            # Locate and count curly braces.
+            open_curly_brace = line.find('{')
+            was_braces_depth = braces_depth
+            braces_depth = braces_depth + line.count('{') - line.count('}')
+
+            # Raise an error if the check_macroassembler_style macro is used
+            # across namespaces / classes scopes.
+            if braces_depth < 0:
+                raise 'check_macroassembler_style annotations are not well scoped.'
+
+            # If the current line contains an opening curly brace, check if
+            # this line combines with the previous one can be identified as a
+            # MacroAssembler function signature.
+            if open_curly_brace != -1 and was_braces_depth == 0:
+                lines = lines + line[:open_curly_brace]
                 if 'MacroAssembler::' in lines:
                     signatures.extend(
                         get_normalized_signatures(lines, fileAnnot))
-                if line.strip() != "{}":  # Empty declaration, no need to declare
-                    # a new code section
-                    code_section = True
-                continue
-            if line.startswith('}'):
-                code_section = False
                 lines = ''
                 continue
-            if code_section:
-                continue
 
-            if len(line.strip()) == 0:
-                lines = ''
+            # We do not aggregate any lines if we are scanning lines which are
+            # in-between a set of curly braces.
+            if braces_depth > 0:
                 continue
+            if was_braces_depth != 0:
+                line = line[line.rfind('}') + 1:]
+
+            # This logic is used to remove template instantiation, static
+            # variable definitions and function declaration from the next
+            # function definition.
+            last_semi_colon = line.rfind(';')
+            if last_semi_colon != -1:
+                lines = ''
+                line = line[last_semi_colon + 1:]
+
+            # Aggregate lines of non-braced text, which corresponds to the space
+            # where we are expecting to find function definitions.
             lines = lines + line
-            # Continue until we have a complete declaration
-            if '{' not in lines:
-                continue
-            # Skip variable declarations
-            if ')' not in lines:
-                lines = ''
-                continue
 
     return signatures
 
@@ -201,14 +220,17 @@ def get_macroassembler_declaration(filename):
                 continue
 
             line = re.sub(r'//.*', '', line)
-            if len(line.strip()) == 0:
+            if len(line.strip()) == 0 or 'public:' in line or 'private:' in line:
                 lines = ''
                 continue
             lines = lines + line
+
             # Continue until we have a complete declaration
             if ';' not in lines:
                 continue
-            # Skip variable declarations
+
+            # Skip member declarations: which are lines ending with a
+            # semi-colon without any list of arguments.
             if ')' not in lines:
                 lines = ''
                 continue
diff --git a/js/src/jit/MacroAssembler-inl.h b/js/src/jit/MacroAssembler-inl.h
index dead510c82a5..be6a641ef9ef 100644
--- a/js/src/jit/MacroAssembler-inl.h
+++ b/js/src/jit/MacroAssembler-inl.h
@@ -33,7 +33,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // Stack manipulation functions.
@@ -800,7 +799,6 @@ template void MacroAssembler::storeFloat32(FloatRegister src, const Address& des
 template void MacroAssembler::storeFloat32(FloatRegister src, const BaseIndex& dest);
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 #ifndef JS_CODEGEN_ARM64
diff --git a/js/src/jit/MacroAssembler.cpp b/js/src/jit/MacroAssembler.cpp
index c8c0554369d2..4eabb20b9060 100644
--- a/js/src/jit/MacroAssembler.cpp
+++ b/js/src/jit/MacroAssembler.cpp
@@ -2954,7 +2954,6 @@ MacroAssembler::subFromStackPtr(Register reg)
 }
 #endif // JS_CODEGEN_ARM64
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // Stack manipulation functions.
@@ -3807,7 +3806,6 @@ MacroAssembler::boundsCheck32PowerOfTwo(Register index, uint32_t length, Label*
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 
 void
 MacroAssembler::memoryBarrierBefore(const Synchronization& sync) {
diff --git a/js/src/jit/MacroAssembler.h b/js/src/jit/MacroAssembler.h
index 98cbd4b05e23..657d9fa1b304 100644
--- a/js/src/jit/MacroAssembler.h
+++ b/js/src/jit/MacroAssembler.h
@@ -350,7 +350,6 @@ class MacroAssembler : public MacroAssemblerSpecific
     void Push(RegisterOrSP reg);
 #endif
 
-    // clang-format off
     //{{{ check_macroassembler_decl_style
   public:
     // ===============================================================
@@ -2136,7 +2135,6 @@ class MacroAssembler : public MacroAssemblerSpecific
     void speculationBarrier() PER_SHARED_ARCH;
 
     //}}} check_macroassembler_decl_style
-    // clang-format on
   public:
 
     // Emits a test of a value against all types in a TypeSet. A scratch
@@ -2902,7 +2900,6 @@ class IonHeapMacroAssembler : public MacroAssembler
     }
 };
 
-// clang-format off
 //{{{ check_macroassembler_style
 inline uint32_t
 MacroAssembler::framePushed() const
@@ -2931,7 +2928,6 @@ MacroAssembler::implicitPop(uint32_t bytes)
     adjustFrame(-int32_t(bytes));
 }
 //}}} check_macroassembler_style
-// clang-format on
 
 static inline Assembler::DoubleCondition
 JSOpToDoubleCondition(JSOp op)
diff --git a/js/src/jit/arm/MacroAssembler-arm-inl.h b/js/src/jit/arm/MacroAssembler-arm-inl.h
index de8808ca971b..370e68960522 100644
--- a/js/src/jit/arm/MacroAssembler-arm-inl.h
+++ b/js/src/jit/arm/MacroAssembler-arm-inl.h
@@ -12,7 +12,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -2335,7 +2334,6 @@ MacroAssembler::clampIntToUint8(Register reg)
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 void
diff --git a/js/src/jit/arm/MacroAssembler-arm.cpp b/js/src/jit/arm/MacroAssembler-arm.cpp
index c5330f2f45c1..51e3ec34d8e6 100644
--- a/js/src/jit/arm/MacroAssembler-arm.cpp
+++ b/js/src/jit/arm/MacroAssembler-arm.cpp
@@ -4362,7 +4362,6 @@ MacroAssembler::subFromStackPtr(Imm32 imm32)
     }
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // MacroAssembler high-level usage.
@@ -6176,7 +6175,6 @@ MacroAssembler::speculationBarrier()
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 
 void
 MacroAssemblerARM::wasmTruncateToInt32(FloatRegister input, Register output, MIRType fromType,
diff --git a/js/src/jit/arm64/MacroAssembler-arm64-inl.h b/js/src/jit/arm64/MacroAssembler-arm64-inl.h
index 07fbf4b5b196..21196471ee39 100644
--- a/js/src/jit/arm64/MacroAssembler-arm64-inl.h
+++ b/js/src/jit/arm64/MacroAssembler-arm64-inl.h
@@ -12,7 +12,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -1982,7 +1981,6 @@ MacroAssembler::clampIntToUint8(Register reg)
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 void
diff --git a/js/src/jit/arm64/MacroAssembler-arm64.cpp b/js/src/jit/arm64/MacroAssembler-arm64.cpp
index 49a015fdb2d8..823cff5b00ed 100644
--- a/js/src/jit/arm64/MacroAssembler-arm64.cpp
+++ b/js/src/jit/arm64/MacroAssembler-arm64.cpp
@@ -413,7 +413,6 @@ MacroAssembler::Push(RegisterOrSP reg)
     adjustFrame(sizeof(intptr_t));
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // MacroAssembler high-level usage.
@@ -2025,7 +2024,6 @@ MacroAssembler::speculationBarrier()
 }
 
 //}}} check_macroassembler_style
-// clang-format off
 
 } // namespace jit
 } // namespace js
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h b/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h
index a1513230494f..dcbb4868a8d6 100644
--- a/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h
+++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared-inl.h
@@ -12,7 +12,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -1111,7 +1110,6 @@ MacroAssembler::clampIntToUint8(Register reg)
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 } // namespace jit
diff --git a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
index 6ab1f9a3151c..28efa47c4379 100644
--- a/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
+++ b/js/src/jit/mips-shared/MacroAssembler-mips-shared.cpp
@@ -1439,7 +1439,6 @@ MacroAssemblerMIPSShared::asMasm() const
     return *static_cast<const MacroAssembler*>(this);
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // MacroAssembler high-level usage.
@@ -2929,4 +2928,3 @@ MacroAssembler::speculationBarrier()
     MOZ_CRASH();
 }
 //}}} check_macroassembler_style
-// clang-format on
diff --git a/js/src/jit/mips32/MacroAssembler-mips32-inl.h b/js/src/jit/mips32/MacroAssembler-mips32-inl.h
index 894055dd2697..3d1592ce0e56 100644
--- a/js/src/jit/mips32/MacroAssembler-mips32-inl.h
+++ b/js/src/jit/mips32/MacroAssembler-mips32-inl.h
@@ -14,7 +14,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -1012,7 +1011,6 @@ MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, Register
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 void
diff --git a/js/src/jit/mips32/MacroAssembler-mips32.cpp b/js/src/jit/mips32/MacroAssembler-mips32.cpp
index 7f4baec910c5..5907140c9cb5 100644
--- a/js/src/jit/mips32/MacroAssembler-mips32.cpp
+++ b/js/src/jit/mips32/MacroAssembler-mips32.cpp
@@ -2109,7 +2109,6 @@ MacroAssembler::subFromStackPtr(Imm32 imm32)
     }
 }
 
-// clang-format on
 //{{{ check_macroassembler_style
 // ===============================================================
 // Stack manipulation functions.
@@ -2968,4 +2967,3 @@ MacroAssembler::convertUInt64ToDouble(Register64 src, FloatRegister dest, Regist
 }
 
 //}}} check_macroassembler_style
-// clang-format on
diff --git a/js/src/jit/mips64/MacroAssembler-mips64-inl.h b/js/src/jit/mips64/MacroAssembler-mips64-inl.h
index 58859812a2b6..867aade6383e 100644
--- a/js/src/jit/mips64/MacroAssembler-mips64-inl.h
+++ b/js/src/jit/mips64/MacroAssembler-mips64-inl.h
@@ -14,7 +14,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -759,7 +758,6 @@ MacroAssembler::branchTruncateFloat32MaybeModUint32(FloatRegister src, Register
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 // The specializations for cmpPtrSet are outside the braces because check_macroassembler_style can't yet
diff --git a/js/src/jit/mips64/MacroAssembler-mips64.cpp b/js/src/jit/mips64/MacroAssembler-mips64.cpp
index 64381d4ec651..4bbc4d4e87e8 100644
--- a/js/src/jit/mips64/MacroAssembler-mips64.cpp
+++ b/js/src/jit/mips64/MacroAssembler-mips64.cpp
@@ -1948,7 +1948,6 @@ MacroAssembler::subFromStackPtr(Imm32 imm32)
     }
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // Stack manipulation functions.
@@ -2760,4 +2759,3 @@ MacroAssembler::convertUInt64ToFloat32(Register64 src_, FloatRegister dest, Regi
 }
 
 //}}} check_macroassembler_style
-// clang-format on
diff --git a/js/src/jit/x64/MacroAssembler-x64-inl.h b/js/src/jit/x64/MacroAssembler-x64-inl.h
index 4361cb73773a..e192c0c32ba9 100644
--- a/js/src/jit/x64/MacroAssembler-x64-inl.h
+++ b/js/src/jit/x64/MacroAssembler-x64-inl.h
@@ -14,7 +14,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 
@@ -965,7 +964,6 @@ MacroAssembler::truncateDoubleToUInt64(Address src, Address dest, Register temp,
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 void
diff --git a/js/src/jit/x64/MacroAssembler-x64.cpp b/js/src/jit/x64/MacroAssembler-x64.cpp
index 827741269d5d..4e41b499de9e 100644
--- a/js/src/jit/x64/MacroAssembler-x64.cpp
+++ b/js/src/jit/x64/MacroAssembler-x64.cpp
@@ -303,7 +303,6 @@ MacroAssembler::subFromStackPtr(Imm32 imm32)
     }
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // ABI function calls.
@@ -1039,4 +1038,3 @@ MacroAssembler::wasmAtomicEffectOp64(const wasm::MemoryAccessDesc& access, Atomi
 }
 
 //}}} check_macroassembler_style
-// clang-format on
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h b/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h
index a2612cf28535..4bbfbb388ffb 100644
--- a/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared-inl.h
@@ -12,7 +12,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // Move instructions
@@ -1332,7 +1331,6 @@ MacroAssembler::clampIntToUint8(Register reg)
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 } // namespace jit
diff --git a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
index a0ebe90d646b..d50e8e74bb19 100644
--- a/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
+++ b/js/src/jit/x86-shared/MacroAssembler-x86-shared.cpp
@@ -270,7 +270,6 @@ MacroAssemblerX86Shared::minMaxFloat32(FloatRegister first, FloatRegister second
     bind(&done);
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // MacroAssembler high-level usage.
@@ -1674,4 +1673,3 @@ MacroAssembler::speculationBarrier()
 }
 
 //}}} check_macroassembler_style
-// clang-format on
diff --git a/js/src/jit/x86/MacroAssembler-x86-inl.h b/js/src/jit/x86/MacroAssembler-x86-inl.h
index ecddad7558ab..6a80a4c97266 100644
--- a/js/src/jit/x86/MacroAssembler-x86-inl.h
+++ b/js/src/jit/x86/MacroAssembler-x86-inl.h
@@ -14,7 +14,6 @@
 namespace js {
 namespace jit {
 
-// clang-format off
 //{{{ check_macroassembler_style
 
 void
@@ -1183,7 +1182,6 @@ MacroAssembler::truncateDoubleToUInt64(Address src, Address dest, Register temp,
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 // ===============================================================
 
 // Note: this function clobbers the source register.
diff --git a/js/src/jit/x86/MacroAssembler-x86.cpp b/js/src/jit/x86/MacroAssembler-x86.cpp
index 60ffda510ef5..bccf7e42dacb 100644
--- a/js/src/jit/x86/MacroAssembler-x86.cpp
+++ b/js/src/jit/x86/MacroAssembler-x86.cpp
@@ -302,7 +302,6 @@ MacroAssembler::subFromStackPtr(Imm32 imm32)
     }
 }
 
-// clang-format off
 //{{{ check_macroassembler_style
 // ===============================================================
 // ABI function calls.
@@ -1282,5 +1281,4 @@ MacroAssembler::convertInt64ToFloat32(Register64 input, FloatRegister output)
 }
 
 //}}} check_macroassembler_style
-// clang-format on
 

From 9a8768d7980a5ef57b37dd27aad79fbecc3dbd95 Mon Sep 17 00:00:00 2001
From: Paolo Amadini <paolo.mozmail@amadzone.org>
Date: Thu, 29 Nov 2018 15:12:24 +0000
Subject: [PATCH 05/78] Bug 1502856 - Fix intermittent failure surfaced by
 test-verify. r=me

Just waiting for l10n.ready isn't enough, so the test now verifies the button class instead of its text content.

--HG--
extra : rebase_source : e0a9f1cffe6bae98334f71ada7d09f2d204ad268
---
 .../components/aboutconfig/test/browser/browser_locked.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/browser/components/aboutconfig/test/browser/browser_locked.js b/browser/components/aboutconfig/test/browser/browser_locked.js
index e5254e37e2aa..2e8f8cac7770 100644
--- a/browser/components/aboutconfig/test/browser/browser_locked.js
+++ b/browser/components/aboutconfig/test/browser/browser_locked.js
@@ -42,28 +42,28 @@ add_task(async function test_locked() {
       let lockedPref = getRow("browser.search.searchEnginesURL");
       Assert.ok(lockedPref.classList.contains("locked"));
       Assert.equal(getValue("browser.search.searchEnginesURL"), "https://addons.mozilla.org/%LOCALE%/firefox/search-engines/");
-      Assert.equal(getButton("browser.search.searchEnginesURL").textContent, "Edit");
+      Assert.ok(getButton("browser.search.searchEnginesURL").classList.contains("button-edit"));
       Assert.equal(getButton("browser.search.searchEnginesURL").disabled, true);
 
       // Test locked default boolean pref.
       lockedPref = getRow("accessibility.AOM.enabled");
       Assert.ok(lockedPref.classList.contains("locked"));
       Assert.equal(getValue("accessibility.AOM.enabled"), "false");
-      Assert.equal(getButton("accessibility.AOM.enabled").textContent, "Toggle");
+      Assert.ok(getButton("accessibility.AOM.enabled").classList.contains("button-toggle"));
       Assert.equal(getButton("accessibility.AOM.enabled").disabled, true);
 
       // Test locked user added pref.
       lockedPref = getRow("test.aboutconfig.a");
       Assert.ok(lockedPref.classList.contains("locked"));
       Assert.equal(getValue("test.aboutconfig.a"), "");
-      Assert.equal(getButton("test.aboutconfig.a").textContent, "Edit");
+      Assert.ok(getButton("test.aboutconfig.a").classList.contains("button-edit"));
       Assert.equal(getButton("test.aboutconfig.a").disabled, true);
 
       // Test pref not locked
       let unlockedPref = getRow("accessibility.indicator.enabled");
       Assert.equal(unlockedPref.classList.contains("locked"), false);
       Assert.equal(getValue("accessibility.indicator.enabled"), "false");
-      Assert.equal(getButton("accessibility.indicator.enabled").textContent, "Toggle");
+      Assert.ok(getButton("accessibility.indicator.enabled").classList.contains("button-toggle"));
       Assert.equal(getButton("accessibility.indicator.enabled").disabled, false);
     });
   });

From c4557c9062bd4a4b1ad0fcc26f1b13feeadc71d9 Mon Sep 17 00:00:00 2001
From: Nicolas Silva <nsilva@mozilla.com>
Date: Thu, 29 Nov 2018 15:11:58 +0100
Subject: [PATCH 06/78] Bug 1510447 - Adjust reftest expectation. r=kats

Differential Revision: https://phabricator.services.mozilla.com/D13412

--HG--
extra : source : e92d34bd8ebc56f3dfb1f4951e0d28e6a4c84ae2
---
 layout/reftests/bugs/reftest.list | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/layout/reftests/bugs/reftest.list b/layout/reftests/bugs/reftest.list
index 984e5f604f70..28dcbed45563 100644
--- a/layout/reftests/bugs/reftest.list
+++ b/layout/reftests/bugs/reftest.list
@@ -305,7 +305,7 @@ fuzzy-if(Android,0-3,0-50) fuzzy-if(skiaContent,0-1,0-133) == 273681-1.html 2736
 == 283686-2.html 283686-2-ref.html
 == 283686-3.html about:blank
 == 289384-1.xhtml 289384-ref.xhtml
-fails-if(webrender&&gtkWidget) random-if(d2d) fuzzy-if(Android,0-8,0-1439) HTTP == 289480.html#top 289480-ref.html # basically-verbatim acid2 test, HTTP for a 404 page -- bug 578114 for the d2d failures
+fails-if(webrender&&gtkWidget) random-if(d2d) fuzzy-if(webrender&&cocoaWidget,1-2,400-900) fuzzy-if(Android,0-8,0-1439) HTTP == 289480.html#top 289480-ref.html # basically-verbatim acid2 test, HTTP for a 404 page -- bug 578114 for the d2d failures
 == 290129-1.html 290129-1-ref.html
 == 291078-1.html 291078-1-ref.html
 == 291078-2.html 291078-2-ref.html

From 9f96be2f266c23b69793a7c5b9985c4b72e3f70e Mon Sep 17 00:00:00 2001
From: Nicolas Silva <nsilva@mozilla.com>
Date: Wed, 28 Nov 2018 16:55:10 +0100
Subject: [PATCH 07/78] bug 1510447 - Fix a blob image key leak. r=jrmuizel

Differential Revision: https://phabricator.services.mozilla.com/D13262

--HG--
extra : source : a58f38fb47f3f436584a816ccbbc192ae9684d02
extra : intermediate-source : 6cca1b66a9dd1951abd51f6c3bf588a2ae46934c
---
 gfx/layers/wr/WebRenderUserData.cpp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/gfx/layers/wr/WebRenderUserData.cpp b/gfx/layers/wr/WebRenderUserData.cpp
index aafcface38ab..3ca7057021b0 100644
--- a/gfx/layers/wr/WebRenderUserData.cpp
+++ b/gfx/layers/wr/WebRenderUserData.cpp
@@ -306,6 +306,7 @@ WebRenderFallbackData::WebRenderFallbackData(WebRenderLayerManager* aWRManager,
 
 WebRenderFallbackData::~WebRenderFallbackData()
 {
+  ClearImageKey();
 }
 
 nsDisplayItemGeometry*

From d950eb11c5b4499622612b716959522d4429d8a2 Mon Sep 17 00:00:00 2001
From: Geoff Brown <gbrown@mozilla.com>
Date: Thu, 29 Nov 2018 11:03:05 -0700
Subject: [PATCH 08/78] Bug 1511084 - Increase some linux build task
 max-run-times to avoid intermittent timeouts; r=dustin

---
 taskcluster/ci/build/linux.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/taskcluster/ci/build/linux.yml b/taskcluster/ci/build/linux.yml
index 6c21f174d2b9..fe6f4cfb92b0 100644
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -106,7 +106,7 @@ linux64-fuzzing/debug:
         symbol: Bf
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        max-run-time: 3600
+        max-run-time: 5400
         env:
             PERFHERDER_EXTRA_OPTIONS: fuzzing
             FORCE_GCC: '1'
@@ -143,7 +143,7 @@ linux64/debug:
         symbol: B
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]
@@ -353,7 +353,7 @@ linux/debug:
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: debian7-i386-build}
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]
@@ -575,7 +575,7 @@ linux64-asan/opt:
     worker:
         env:
             PERFHERDER_EXTRA_OPTIONS: "opt asan"
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]
@@ -609,7 +609,7 @@ linux64-asan-fuzzing/opt:
     worker:
         env:
             PERFHERDER_EXTRA_OPTIONS: asan-fuzzing
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]
@@ -644,7 +644,7 @@ linux64-asan-fuzzing-ccov/opt:
     worker:
         env:
             PERFHERDER_EXTRA_OPTIONS: asan-fuzzing-ccov
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]
@@ -676,7 +676,7 @@ linux64-fuzzing-ccov/opt:
     run-on-projects: ['mozilla-central', 'try']
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
-        max-run-time: 3600
+        max-run-time: 5400
         env:
             PERFHERDER_EXTRA_OPTIONS: fuzzing-ccov
             FORCE_GCC: '1'
@@ -750,7 +750,7 @@ linux64-asan/debug:
     worker:
         env:
             PERFHERDER_EXTRA_OPTIONS: "debug asan"
-        max-run-time: 3600
+        max-run-time: 5400
     run:
         using: mozharness
         actions: [get-secrets, build, check-test]

From 1ec332e1cbe53cb580085a2459d347494e888d0e Mon Sep 17 00:00:00 2001
From: Geoff Brown <gbrown@mozilla.com>
Date: Thu, 29 Nov 2018 11:03:05 -0700
Subject: [PATCH 09/78] Bug 1511101 - Enable a few android 7.0 x86 mochitest;
 r=me,a=test-only

---
 dom/base/test/mochitest.ini                          | 2 +-
 dom/browser-element/mochitest/mochitest.ini          | 2 +-
 editor/libeditor/tests/mochitest.ini                 | 9 +++++----
 testing/mochitest/tests/Harness_sanity/mochitest.ini | 2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/dom/base/test/mochitest.ini b/dom/base/test/mochitest.ini
index b59608e00ae1..18b5f73d6ac8 100644
--- a/dom/base/test/mochitest.ini
+++ b/dom/base/test/mochitest.ini
@@ -632,7 +632,7 @@ subsuite = clipboard
 skip-if = toolkit == 'android' #bug 904183
 [test_copypaste.xhtml]
 subsuite = clipboard
-skip-if = toolkit == 'android' #bug 904183
+skip-if = toolkit == 'android' && !e10s #bug 904183
 [test_createHTMLDocument.html]
 [test_data_uri.html]
 skip-if = verify
diff --git a/dom/browser-element/mochitest/mochitest.ini b/dom/browser-element/mochitest/mochitest.ini
index d4d65b5822a6..5ce50d7a7b5b 100644
--- a/dom/browser-element/mochitest/mochitest.ini
+++ b/dom/browser-element/mochitest/mochitest.ini
@@ -107,7 +107,7 @@ support-files =
 [test_browserElement_inproc_CookiesNotThirdParty.html]
 [test_browserElement_inproc_CopyPaste.html]
 subsuite = clipboard
-skip-if = (os == "android") # Disabled on Android, see bug 1230421
+skip-if = (os == "android" && !e10s) # Disabled on Android, see bug 1230421
 [test_browserElement_inproc_DataURI.html]
 [test_browserElement_inproc_ExposableURI.html]
 [test_browserElement_inproc_FirstPaint.html]
diff --git a/editor/libeditor/tests/mochitest.ini b/editor/libeditor/tests/mochitest.ini
index e3e67377e7df..c334d745d9aa 100644
--- a/editor/libeditor/tests/mochitest.ini
+++ b/editor/libeditor/tests/mochitest.ini
@@ -107,15 +107,15 @@ skip-if = toolkit == 'android'
 [test_bug596506.html]
 [test_bug597331.html]
 subsuite = clipboard
-skip-if = toolkit == 'android' # Bug 718316
+skip-if = toolkit == 'android' && !e10s # Bug 718316
 [test_bug597784.html]
 [test_bug599322.html]
 subsuite = clipboard
-skip-if = toolkit == 'android'
+skip-if = toolkit == 'android' && !e10s
 [test_bug599983.html]
 [test_bug600570.html]
 subsuite = clipboard
-skip-if = toolkit == 'android' # Bug 718316
+skip-if = toolkit == 'android' && !e10s # Bug 718316
 [test_bug602130.html]
 [test_bug603556.html]
 subsuite = clipboard
@@ -185,6 +185,7 @@ subsuite = clipboard
 [test_bug796839.html]
 [test_bug830600.html]
 subsuite = clipboard
+skip-if = toolkit == 'android' && e10s
 [test_bug832025.html]
 [test_bug850043.html]
 [test_bug857487.html]
@@ -214,7 +215,7 @@ subsuite = clipboard
 [test_bug1140105.html]
 [test_bug1140617.html]
 subsuite = clipboard
-skip-if = toolkit == 'android' # bug 1299578
+skip-if = toolkit == 'android' && !e10s # bug 1299578
 [test_bug1153237.html]
 [test_bug1154791.html]
 skip-if = os == 'android'
diff --git a/testing/mochitest/tests/Harness_sanity/mochitest.ini b/testing/mochitest/tests/Harness_sanity/mochitest.ini
index 61c9fcbc41aa..c34ed6679be0 100644
--- a/testing/mochitest/tests/Harness_sanity/mochitest.ini
+++ b/testing/mochitest/tests/Harness_sanity/mochitest.ini
@@ -34,7 +34,7 @@ support-files = SpecialPowersLoadChromeScript.js
 skip-if = toolkit == 'android' || (verify && (os == 'win')) # bug 688052
 [test_sanitySimpletest.html]
 subsuite = clipboard
-skip-if = toolkit == 'android'  # bug 688052
+skip-if = toolkit == 'android' && !e10s # bug 688052
 [test_sanity_manifest.html]
 skip-if = toolkit == 'android' # we use the old manifest style on android
 fail-if = true

From c010fd41c08dc415d1185fd6b3db82846d7e89ec Mon Sep 17 00:00:00 2001
From: James Willcox <snorp@snorp.net>
Date: Thu, 29 Nov 2018 08:35:07 -0600
Subject: [PATCH 10/78] Bug 1507702 - Don't make about:crash* accessible to web
 content r=Ehsan,bzbarsky

Differential Revision: https://phabricator.services.mozilla.com/D12133
---
 docshell/base/nsAboutRedirector.cpp          | 43 ++++++++++++---
 docshell/test/mochitest/mochitest.ini        |  1 +
 docshell/test/mochitest/test_bug1507702.html | 57 ++++++++++++++++++++
 3 files changed, 93 insertions(+), 8 deletions(-)
 create mode 100644 docshell/test/mochitest/test_bug1507702.html

diff --git a/docshell/base/nsAboutRedirector.cpp b/docshell/base/nsAboutRedirector.cpp
index 5c95be2d6133..4bdd53de3a62 100644
--- a/docshell/base/nsAboutRedirector.cpp
+++ b/docshell/base/nsAboutRedirector.cpp
@@ -7,6 +7,7 @@
 #include "nsAboutRedirector.h"
 #include "nsNetUtil.h"
 #include "nsAboutProtocolUtils.h"
+#include "nsBaseChannel.h"
 #include "mozilla/ArrayUtils.h"
 #include "nsIProtocolHandler.h"
 
@@ -27,6 +28,36 @@ struct RedirEntry
   uint32_t flags;
 };
 
+class CrashChannel final : public nsBaseChannel
+{
+public:
+  explicit CrashChannel(nsIURI* aURI)
+  {
+    SetURI(aURI);
+  }
+
+  nsresult OpenContentStream(bool async, nsIInputStream **stream,
+                             nsIChannel** channel) override
+  {
+    nsAutoCString spec;
+    mURI->GetSpec(spec);
+
+    if (spec.EqualsASCII("about:crashparent") && XRE_IsParentProcess()) {
+      MOZ_CRASH("Crash via about:crashparent");
+    }
+
+    if (spec.EqualsASCII("about:crashcontent") && XRE_IsContentProcess()) {
+      MOZ_CRASH("Crash via about:crashcontent");
+    }
+
+    NS_WARNING("Unhandled about:crash* URI or wrong process");
+    return NS_ERROR_NOT_IMPLEMENTED;
+  }
+
+protected:
+  virtual ~CrashChannel() = default;
+};
+
 /*
   Entries which do not have URI_SAFE_FOR_UNTRUSTED_CONTENT will run with chrome
   privileges. This is potentially dangerous. Please use
@@ -145,12 +176,10 @@ static const RedirEntry kRedirMap[] = {
   },
   {
     "crashparent", "about:blank",
-    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT
   },
   {
     "crashcontent", "about:blank",
-    nsIAboutModule::URI_SAFE_FOR_UNTRUSTED_CONTENT |
     nsIAboutModule::HIDE_FROM_ABOUTABOUT |
     nsIAboutModule::URI_CAN_LOAD_IN_CHILD |
     nsIAboutModule::URI_MUST_LOAD_IN_CHILD
@@ -174,12 +203,10 @@ nsAboutRedirector::NewChannel(nsIURI* aURI,
   nsCOMPtr<nsIIOService> ioService = do_GetIOService(&rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
-  if (XRE_IsParentProcess() && path.EqualsASCII("crashparent")) {
-    MOZ_CRASH("Crash via about:crashparent");
-  }
-
-  if (XRE_IsContentProcess() && path.EqualsASCII("crashcontent")) {
-    MOZ_CRASH("Crash via about:crashcontent");
+  if (path.EqualsASCII("crashparent") || path.EqualsASCII("crashcontent")) {
+    nsCOMPtr<nsIChannel> channel = new CrashChannel(aURI);
+    channel.forget(aResult);
+    return NS_OK;
   }
 
 #ifdef ABOUT_CONFIG_BLOCKED_GV
diff --git a/docshell/test/mochitest/mochitest.ini b/docshell/test/mochitest/mochitest.ini
index 4455fcb7d2f4..8f12715a905e 100644
--- a/docshell/test/mochitest/mochitest.ini
+++ b/docshell/test/mochitest/mochitest.ini
@@ -115,3 +115,4 @@ support-files = file_framedhistoryframes.html
 [test_pushState_after_document_open.html]
 [test_windowedhistoryframes.html]
 [test_triggeringprincipal_location_seturi.html]
+[test_bug1507702.html]
diff --git a/docshell/test/mochitest/test_bug1507702.html b/docshell/test/mochitest/test_bug1507702.html
new file mode 100644
index 000000000000..94e140df4b19
--- /dev/null
+++ b/docshell/test/mochitest/test_bug1507702.html
@@ -0,0 +1,57 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1507702
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1507702</title>
+  <script type="application/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <link rel="icon" href="about:crashparent"/>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1507702">Mozilla Bug 1507702</a>
+<img src="about:crashparent">
+<img src="about:crashcontent">
+<iframe src="about:crashparent"></iframe>
+<iframe src="about:crashcontent"></iframe>
+<script>
+  let urls = ["about:crashparent", "about:crashcontent"];
+  async function testFetch() {
+    const url = urls.shift();
+    if (!url) {
+      return Promise.resolve();
+    }
+
+    let threw;
+    try {
+      await fetch(url);
+      threw = false;
+    } catch (e) {
+      threw = true;
+    };
+
+    ok(threw === true, "fetch should reject");
+    return testFetch();
+  }
+
+  document.body.onload = async () => {
+    for (const url of ["about:crashparent", "about:crashcontent"]) {
+      SimpleTest.doesThrow(() => {
+        top.location.href = url;
+      }, "navigation should throw");
+
+      SimpleTest.doesThrow(() => {
+        location.href = url;
+      }, "navigation should throw");
+    }
+
+    await testFetch();
+    SimpleTest.finish();
+  };
+
+  SimpleTest.waitForExplicitFinish();
+</script>
+</body>
+</html>

From e7644eaace0af2636ba226c3d8e5dfbc8018f573 Mon Sep 17 00:00:00 2001
From: Dylan Roeh <droeh@mozilla.com>
Date: Thu, 29 Nov 2018 12:46:05 -0600
Subject: [PATCH 11/78] Bug 1510587 - Back out FennecKiller service for causing
 start-up crashes. r=snorp

---
 mobile/android/base/AndroidManifest.xml.in    |  2 --
 .../java/org/mozilla/gecko/BrowserApp.java    |  3 --
 .../java/org/mozilla/gecko/FennecKiller.java  | 36 -------------------
 3 files changed, 41 deletions(-)
 delete mode 100644 mobile/android/base/java/org/mozilla/gecko/FennecKiller.java

diff --git a/mobile/android/base/AndroidManifest.xml.in b/mobile/android/base/AndroidManifest.xml.in
index 341fbce6629c..78f4cdc37130 100644
--- a/mobile/android/base/AndroidManifest.xml.in
+++ b/mobile/android/base/AndroidManifest.xml.in
@@ -40,8 +40,6 @@
         <service android:name="org.mozilla.gecko.RemotePresentationService" android:exported="false"/>
 #endif
 
-        <service android:name="org.mozilla.gecko.FennecKiller" />
-
         <!-- This activity handles all incoming Intents and dispatches them to other activities. -->
         <activity android:name="org.mozilla.gecko.LauncherActivity"
             android:theme="@android:style/Theme.Translucent.NoTitleBar"
diff --git a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java
index 723670c58698..6c16c2d045c9 100644
--- a/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java
+++ b/mobile/android/base/java/org/mozilla/gecko/BrowserApp.java
@@ -1023,9 +1023,6 @@ public class BrowserApp extends GeckoApp
             return;
         }
 
-        final Intent killerIntent = new Intent(this, FennecKiller.class);
-        startService(killerIntent);
-
         if (!mHasResumed) {
             getAppEventDispatcher().unregisterUiThreadListener(this, "Prompt:ShowTop");
             mHasResumed = true;
diff --git a/mobile/android/base/java/org/mozilla/gecko/FennecKiller.java b/mobile/android/base/java/org/mozilla/gecko/FennecKiller.java
deleted file mode 100644
index 8f8d878a6a58..000000000000
--- a/mobile/android/base/java/org/mozilla/gecko/FennecKiller.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/* -*- Mode: Java; c-basic-offset: 4; tab-width: 20; indent-tabs-mode: nil; -*-
- * This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-package org.mozilla.gecko;
-
-import android.app.Service;
-import android.content.Intent;
-import android.os.IBinder;
-import android.os.Process;
-import android.util.Log;
-
-public class FennecKiller extends Service {
-    private static final String LOGTAG = "GeckoFennecKiller";
-
-    private static final String BROWSERAPP = "org.mozilla.gecko.BrowserApp";
-
-    @Override
-    public IBinder onBind(Intent intent) {
-        return null;
-    }
-
-    @Override
-    public void onTaskRemoved(Intent intent) {
-        if (intent != null && intent.getComponent() != null &&
-                BROWSERAPP.equals(intent.getComponent().getClassName())) {
-            Process.killProcess(Process.myPid());
-        }
-    }
-
-    @Override
-    public int onStartCommand(Intent intent, int flags, int startId) {
-        return Service.START_NOT_STICKY;
-    }
-}

From cd408214891fbe91f2bac4929516ca5305e582c3 Mon Sep 17 00:00:00 2001
From: Kartikaya Gupta <kgupta@mozilla.com>
Date: Thu, 29 Nov 2018 14:22:37 -0500
Subject: [PATCH 12/78] Bug 1511042 - Back out some commits from bug 1503447
 for introducing correctness and perf regressions. r=backout

This backs out hg commits 1e214baf8fc1, 7d4adeee5236, and f5ffebdcc014.
---
 gfx/layers/ipc/WebRenderMessages.ipdlh        |   3 +-
 gfx/layers/wr/AsyncImagePipelineManager.cpp   |  25 +++--
 gfx/layers/wr/AsyncImagePipelineManager.h     |  10 +-
 gfx/layers/wr/ClipManager.cpp                 |  22 +++-
 gfx/layers/wr/StackingContextHelper.cpp       |  26 +----
 gfx/layers/wr/StackingContextHelper.h         |  16 +--
 gfx/layers/wr/WebRenderBridgeParent.cpp       |   4 +-
 gfx/layers/wr/WebRenderCommandBuilder.cpp     |   6 +-
 gfx/layers/wr/WebRenderUserData.cpp           |  14 +--
 gfx/layers/wr/WebRenderUserData.h             |   1 +
 gfx/webrender_bindings/WebRenderAPI.cpp       |  31 +-----
 gfx/webrender_bindings/WebRenderAPI.h         |   6 +-
 gfx/webrender_bindings/src/bindings.rs        | 103 +++++++-----------
 .../webrender_ffi_generated.h                 |  54 ++++-----
 layout/base/nsLayoutUtils.h                   |   1 -
 layout/generic/nsHTMLCanvasFrame.cpp          |   3 +-
 layout/painting/nsDisplayList.cpp             |  32 ++----
 layout/reftests/bugs/reftest.list             |   2 +-
 layout/reftests/svg/conditions-07.svg         |   4 +-
 19 files changed, 143 insertions(+), 220 deletions(-)

diff --git a/gfx/layers/ipc/WebRenderMessages.ipdlh b/gfx/layers/ipc/WebRenderMessages.ipdlh
index 0a7a83058a70..43a7711f9712 100644
--- a/gfx/layers/ipc/WebRenderMessages.ipdlh
+++ b/gfx/layers/ipc/WebRenderMessages.ipdlh
@@ -26,7 +26,6 @@ using mozilla::wr::BlobImageKey from "mozilla/webrender/WebRenderTypes.h";
 using mozilla::wr::PipelineId from "mozilla/webrender/WebRenderTypes.h";
 using mozilla::gfx::MaybeIntSize from "mozilla/gfx/Point.h";
 using mozilla::LayoutDeviceRect from "Units.h";
-using mozilla::LayoutDeviceSize from "Units.h";
 using mozilla::ImageIntRect from "Units.h";
 using mozilla::gfx::Rect from "mozilla/gfx/Rect.h";
 using class mozilla::gfx::Matrix4x4 from "mozilla/gfx/Matrix.h";
@@ -81,7 +80,7 @@ struct OpReleaseTextureOfImage {
 
 struct OpUpdateAsyncImagePipeline {
   PipelineId pipelineId;
-  LayoutDeviceSize scSize;
+  LayoutDeviceRect scBounds;
   Matrix4x4 scTransform;
   MaybeIntSize scaleToSize;
   ImageRendering filter;
diff --git a/gfx/layers/wr/AsyncImagePipelineManager.cpp b/gfx/layers/wr/AsyncImagePipelineManager.cpp
index 1fb192361465..c0d4c04a6fd0 100644
--- a/gfx/layers/wr/AsyncImagePipelineManager.cpp
+++ b/gfx/layers/wr/AsyncImagePipelineManager.cpp
@@ -195,7 +195,7 @@ AsyncImagePipelineManager::RemoveAsyncImagePipeline(const wr::PipelineId& aPipel
 
 void
 AsyncImagePipelineManager::UpdateAsyncImagePipeline(const wr::PipelineId& aPipelineId,
-                                                    const LayoutDeviceSize& aSize,
+                                                    const LayoutDeviceRect& aScBounds,
                                                     const gfx::Matrix4x4& aScTransform,
                                                     const gfx::MaybeIntSize& aScaleToSize,
                                                     const wr::ImageRendering& aFilter,
@@ -209,7 +209,7 @@ AsyncImagePipelineManager::UpdateAsyncImagePipeline(const wr::PipelineId& aPipel
     return;
   }
   pipeline->mInitialised = true;
-  pipeline->Update(aSize,
+  pipeline->Update(aScBounds,
                    aScTransform,
                    aScaleToSize,
                    aFilter,
@@ -417,10 +417,13 @@ AsyncImagePipelineManager::ApplyAsyncImageForPipeline(const wr::Epoch& aEpoch,
   }
 
   aPipeline->mIsChanged = false;
-  wr::DisplayListBuilder builder(aPipelineId, wr::ToLayoutSize(aPipeline->mSize));
+
+  wr::LayoutSize contentSize { aPipeline->mScBounds.Width(), aPipeline->mScBounds.Height() };
+  wr::DisplayListBuilder builder(aPipelineId, contentSize);
 
   float opacity = 1.0f;
   Maybe<wr::WrClipId> referenceFrameId = builder.PushStackingContext(
+    wr::ToRoundedLayoutRect(aPipeline->mScBounds),
     nullptr,
     nullptr,
     &opacity,
@@ -466,7 +469,7 @@ AsyncImagePipelineManager::ApplyAsyncImageForPipeline(const wr::Epoch& aEpoch,
   aSceneBuilderTxn.SetDisplayList(
     gfx::Color(0.f, 0.f, 0.f, 0.f),
     aEpoch,
-    aPipeline->mSize,
+    LayerSize(aPipeline->mScBounds.Width(), aPipeline->mScBounds.Height()),
     aPipelineId, builderContentSize,
     dl.dl_desc, dl.dl);
 }
@@ -513,17 +516,17 @@ AsyncImagePipelineManager::SetEmptyDisplayList(const wr::PipelineId& aPipelineId
   auto& txn = pipeline->mImageHost->GetAsyncRef() ? aTxnForImageBridge : aTxn;
 
   wr::Epoch epoch = GetNextImageEpoch();
-  wr::DisplayListBuilder builder(aPipelineId, wr::ToLayoutSize(pipeline->mSize));
+  wr::LayoutSize contentSize { pipeline->mScBounds.Width(), pipeline->mScBounds.Height() };
+  wr::DisplayListBuilder builder(aPipelineId, contentSize);
 
   wr::BuiltDisplayList dl;
   wr::LayoutSize builderContentSize;
   builder.Finalize(builderContentSize, dl);
-  txn.SetDisplayList(
-    gfx::Color(0.f, 0.f, 0.f, 0.f),
-    epoch,
-    pipeline->mSize,
-    aPipelineId, builderContentSize,
-    dl.dl_desc, dl.dl);
+  txn.SetDisplayList(gfx::Color(0.f, 0.f, 0.f, 0.f),
+                     epoch,
+                     LayerSize(pipeline->mScBounds.Width(), pipeline->mScBounds.Height()),
+                     aPipelineId, builderContentSize,
+                     dl.dl_desc, dl.dl);
 }
 
 void
diff --git a/gfx/layers/wr/AsyncImagePipelineManager.h b/gfx/layers/wr/AsyncImagePipelineManager.h
index feaaaecd44df..bfacab01ad61 100644
--- a/gfx/layers/wr/AsyncImagePipelineManager.h
+++ b/gfx/layers/wr/AsyncImagePipelineManager.h
@@ -89,7 +89,7 @@ public:
                                 wr::TransactionBuilder& aTxn);
 
   void UpdateAsyncImagePipeline(const wr::PipelineId& aPipelineId,
-                                const LayoutDeviceSize& aSize,
+                                const LayoutDeviceRect& aScBounds,
                                 const gfx::Matrix4x4& aScTransform,
                                 const gfx::MaybeIntSize& aScaleToSize,
                                 const wr::ImageRendering& aFilter,
@@ -170,18 +170,18 @@ private:
 
   struct AsyncImagePipeline {
     AsyncImagePipeline();
-    void Update(const LayoutDeviceSize& aSize,
+    void Update(const LayoutDeviceRect& aScBounds,
                 const gfx::Matrix4x4& aScTransform,
                 const gfx::MaybeIntSize& aScaleToSize,
                 const wr::ImageRendering& aFilter,
                 const wr::MixBlendMode& aMixBlendMode)
     {
-      mIsChanged |= mSize != aSize ||
+      mIsChanged |= !mScBounds.IsEqualEdges(aScBounds) ||
                     mScTransform != aScTransform ||
                     mScaleToSize != aScaleToSize ||
                     mFilter != aFilter ||
                     mMixBlendMode != aMixBlendMode;
-      mSize = aSize;
+      mScBounds = aScBounds;
       mScTransform = aScTransform;
       mScaleToSize = aScaleToSize;
       mFilter = aFilter;
@@ -191,7 +191,7 @@ private:
     bool mInitialised;
     bool mIsChanged;
     bool mUseExternalImage;
-    LayoutDeviceSize mSize;
+    LayoutDeviceRect mScBounds;
     gfx::Matrix4x4 mScTransform;
     gfx::MaybeIntSize mScaleToSize;
     wr::ImageRendering mFilter;
diff --git a/gfx/layers/wr/ClipManager.cpp b/gfx/layers/wr/ClipManager.cpp
index bd87eb70faf3..e61a57cca5e5 100644
--- a/gfx/layers/wr/ClipManager.cpp
+++ b/gfx/layers/wr/ClipManager.cpp
@@ -61,10 +61,15 @@ ClipManager::EndBuild()
 void
 ClipManager::BeginList(const StackingContextHelper& aStackingContext)
 {
-  if (aStackingContext.ReferenceFrameId()) {
-    PushOverrideForASR(
-        mItemClipStack.empty() ? nullptr : mItemClipStack.top().mASR,
-        aStackingContext.ReferenceFrameId().ref());
+  if (aStackingContext.AffectsClipPositioning()) {
+    if (aStackingContext.ReferenceFrameId()) {
+      PushOverrideForASR(
+          mItemClipStack.empty() ? nullptr : mItemClipStack.top().mASR,
+          aStackingContext.ReferenceFrameId().ref());
+    } else {
+      // Start a new cache
+      mCacheStack.emplace();
+    }
   }
 
   ItemClips clips(nullptr, nullptr, false);
@@ -81,9 +86,14 @@ ClipManager::EndList(const StackingContextHelper& aStackingContext)
   mItemClipStack.top().Unapply(mBuilder);
   mItemClipStack.pop();
 
-  if (aStackingContext.ReferenceFrameId()) {
-    PopOverrideForASR(
+  if (aStackingContext.AffectsClipPositioning()) {
+    if (aStackingContext.ReferenceFrameId()) {
+      PopOverrideForASR(
         mItemClipStack.empty() ? nullptr : mItemClipStack.top().mASR);
+    } else {
+      MOZ_ASSERT(!mCacheStack.empty());
+      mCacheStack.pop();
+    }
   }
 }
 
diff --git a/gfx/layers/wr/StackingContextHelper.cpp b/gfx/layers/wr/StackingContextHelper.cpp
index 13d8b004fcbc..7ede1042b034 100644
--- a/gfx/layers/wr/StackingContextHelper.cpp
+++ b/gfx/layers/wr/StackingContextHelper.cpp
@@ -15,6 +15,7 @@ namespace layers {
 StackingContextHelper::StackingContextHelper()
   : mBuilder(nullptr)
   , mScale(1.0f, 1.0f)
+  , mAffectsClipPositioning(false)
   , mIsPreserve3D(false)
   , mRasterizeLocally(false)
 {
@@ -25,10 +26,10 @@ StackingContextHelper::StackingContextHelper(const StackingContextHelper& aParen
                                              const ActiveScrolledRoot* aAsr,
                                              wr::DisplayListBuilder& aBuilder,
                                              const nsTArray<wr::WrFilterOp>& aFilters,
+                                             const LayoutDeviceRect& aBounds,
                                              const gfx::Matrix4x4* aBoundTransform,
                                              const wr::WrAnimationProperty* aAnimation,
                                              const float* aOpacityPtr,
-                                             const LayoutDevicePoint& aOrigin,
                                              const gfx::Matrix4x4* aTransformPtr,
                                              const gfx::Matrix4x4* aPerspectivePtr,
                                              const gfx::CompositionOp& aMixBlendMode,
@@ -39,15 +40,10 @@ StackingContextHelper::StackingContextHelper(const StackingContextHelper& aParen
                                              bool aAnimated)
   : mBuilder(&aBuilder)
   , mScale(1.0f, 1.0f)
-  , mInheritedStickyOrigin(aOrigin)
   , mDeferredTransformItem(aDeferredTransformItem)
   , mIsPreserve3D(aIsPreserve3D)
   , mRasterizeLocally(aAnimated || aParentSC.mRasterizeLocally)
 {
-  if (aOrigin != LayoutDevicePoint()) {
-    mOriginFrameId = Some(mBuilder->PushOrigin(wr::ToLayoutPoint(aOrigin)));
-  }
-
   // Compute scale for fallback rendering. We don't try to guess a scale for 3d
   // transformed items
   gfx::Matrix transform2d;
@@ -71,6 +67,7 @@ StackingContextHelper::StackingContextHelper(const StackingContextHelper& aParen
     : wr::RasterSpace::Screen();
 
   mReferenceFrameId = mBuilder->PushStackingContext(
+          wr::ToLayoutRect(aBounds),
           aClipNodeId,
           aAnimation,
           aOpacityPtr,
@@ -82,6 +79,9 @@ StackingContextHelper::StackingContextHelper(const StackingContextHelper& aParen
           aBackfaceVisible,
           rasterSpace);
 
+  mAffectsClipPositioning = mReferenceFrameId.isSome() ||
+      (aBounds.TopLeft() != LayoutDevicePoint());
+
   // If the parent stacking context has a deferred transform item, inherit it
   // into this stacking context, as long as the ASR hasn't changed. Refer to
   // the comments on StackingContextHelper::mDeferredTransformItem for an
@@ -99,29 +99,15 @@ StackingContextHelper::StackingContextHelper(const StackingContextHelper& aParen
       mDeferredAncestorTransform = aParentSC.mDeferredAncestorTransform;
     }
   }
-
-  // Update the origin for use by sticky frames.
-  if (!mReferenceFrameId) {
-    mInheritedStickyOrigin += aParentSC.mInheritedStickyOrigin;
-  }
 }
 
 StackingContextHelper::~StackingContextHelper()
 {
   if (mBuilder) {
     mBuilder->PopStackingContext(mReferenceFrameId.isSome());
-    if (mOriginFrameId) {
-      mBuilder->PopOrigin();
-    }
   }
 }
 
-Maybe<wr::WrClipId>
-StackingContextHelper::ReferenceFrameId() const
-{
-  return mReferenceFrameId ? mReferenceFrameId : mOriginFrameId;
-}
-
 const Maybe<nsDisplayTransform*>&
 StackingContextHelper::GetDeferredTransformItem() const
 {
diff --git a/gfx/layers/wr/StackingContextHelper.h b/gfx/layers/wr/StackingContextHelper.h
index 1b5996226784..9deaf4468817 100644
--- a/gfx/layers/wr/StackingContextHelper.h
+++ b/gfx/layers/wr/StackingContextHelper.h
@@ -32,10 +32,10 @@ public:
                         const ActiveScrolledRoot* aAsr,
                         wr::DisplayListBuilder& aBuilder,
                         const nsTArray<wr::WrFilterOp>& aFilters = nsTArray<wr::WrFilterOp>(),
+                        const LayoutDeviceRect& aBounds = LayoutDeviceRect(),
                         const gfx::Matrix4x4* aBoundTransform = nullptr,
                         const wr::WrAnimationProperty* aAnimation = nullptr,
                         const float* aOpacityPtr = nullptr,
-                        const LayoutDevicePoint& aOrigin = LayoutDevicePoint(),
                         const gfx::Matrix4x4* aTransformPtr = nullptr,
                         const gfx::Matrix4x4* aPerspectivePtr = nullptr,
                         const gfx::CompositionOp& aMixBlendMode = gfx::CompositionOp::OP_OVER,
@@ -69,30 +69,22 @@ public:
   const Maybe<nsDisplayTransform*>& GetDeferredTransformItem() const;
   Maybe<gfx::Matrix4x4> GetDeferredTransformMatrix() const;
 
-  Maybe<wr::WrClipId> ReferenceFrameId() const;
-
-  const LayoutDevicePoint& GetInheritedStickyOrigin() const {
-    return mInheritedStickyOrigin;
-  }
+  bool AffectsClipPositioning() const { return mAffectsClipPositioning; }
+  Maybe<wr::WrClipId> ReferenceFrameId() const { return mReferenceFrameId; }
 
 private:
   wr::DisplayListBuilder* mBuilder;
   gfx::Size mScale;
   gfx::Matrix mInheritedTransform;
 
-  // A stacking context may insert a special WR reference frame if we have
-  // origin provided. It only affects sticky frames below it, which need to
-  // compensate for the origin when computing the viewport.
-  LayoutDevicePoint mInheritedStickyOrigin;
-
   // The "snapping surface" defines the space that we want to snap in.
   // You can think of it as the nearest physical surface.
   // Animated transforms create a new snapping surface, so that changes to their transform don't affect the snapping of their contents.
   // Non-animated transforms do *not* create a new snapping surface,
   // so that for example the existence of a non-animated identity transform does not affect snapping.
   gfx::Matrix mSnappingSurfaceTransform;
+  bool mAffectsClipPositioning;
   Maybe<wr::WrClipId> mReferenceFrameId;
-  Maybe<wr::WrClipId> mOriginFrameId;
 
   // The deferred transform item is used when building the WebRenderScrollData
   // structure. The backstory is that APZ needs to know about transforms that
diff --git a/gfx/layers/wr/WebRenderBridgeParent.cpp b/gfx/layers/wr/WebRenderBridgeParent.cpp
index c6994dec9c65..34bb2c7b79cd 100644
--- a/gfx/layers/wr/WebRenderBridgeParent.cpp
+++ b/gfx/layers/wr/WebRenderBridgeParent.cpp
@@ -994,7 +994,7 @@ WebRenderBridgeParent::RecvSetDisplayList(const gfx::IntSize& aSize,
       txn.SetWindowParameters(widgetSize, docRect);
     }
     gfx::Color clearColor(0.f, 0.f, 0.f, 0.f);
-    txn.SetDisplayList(clearColor, wrEpoch, LayoutDeviceSize(aSize.width, aSize.height),
+    txn.SetDisplayList(clearColor, wrEpoch, LayerSize(aSize.width, aSize.height),
                        mPipelineId, aContentSize,
                        dlDesc, dlData);
 
@@ -1233,7 +1233,7 @@ WebRenderBridgeParent::ProcessWebRenderParentCommands(const InfallibleTArray<Web
       case WebRenderParentCommand::TOpUpdateAsyncImagePipeline: {
         const OpUpdateAsyncImagePipeline& op = cmd.get_OpUpdateAsyncImagePipeline();
         mAsyncImageManager->UpdateAsyncImagePipeline(op.pipelineId(),
-                                                     op.scSize(),
+                                                     op.scBounds(),
                                                      op.scTransform(),
                                                      op.scaleToSize(),
                                                      op.filter(),
diff --git a/gfx/layers/wr/WebRenderCommandBuilder.cpp b/gfx/layers/wr/WebRenderCommandBuilder.cpp
index fbc00c0bfdca..296c993d0448 100644
--- a/gfx/layers/wr/WebRenderCommandBuilder.cpp
+++ b/gfx/layers/wr/WebRenderCommandBuilder.cpp
@@ -1149,7 +1149,7 @@ Grouper::ConstructItemInsideInactive(WebRenderCommandBuilder* aCommandBuilder,
 
   // we compute the geometry change here because we have the transform around still
   aGroup->ComputeGeometryChange(aItem, data, mTransform, mDisplayListBuilder);
-
+  
   // Temporarily restrict the image bounds to the bounds of the container so that
   // clipped children within the container know about the clip.
   IntRect oldImageBounds = aGroup->mImageBounds;
@@ -1362,7 +1362,7 @@ WebRenderCommandBuilder::BuildWebRenderCommands(wr::DisplayListBuilder& aBuilder
     }
 
     StackingContextHelper pageRootSc(sc, nullptr, aBuilder, aFilters,
-        nullptr, mZoomProp.ptrOr(nullptr));
+        LayoutDeviceRect(), nullptr, mZoomProp.ptrOr(nullptr));
     if (ShouldDumpDisplayList(aDisplayListBuilder)) {
       mBuilderDumpIndex = aBuilder.Dump(mDumpIndent + 1, Some(mBuilderDumpIndex), Nothing());
     }
@@ -1624,6 +1624,7 @@ WebRenderCommandBuilder::CreateImageKey(nsDisplayItem* aItem,
     MOZ_ASSERT(aAsyncImageBounds);
 
     LayoutDeviceRect rect = aAsyncImageBounds.value();
+    LayoutDeviceRect scBounds(LayoutDevicePoint(0, 0), rect.Size());
     gfx::MaybeIntSize scaleToSize;
     if (!aContainer->GetScaleHint().IsEmpty()) {
       scaleToSize = Some(aContainer->GetScaleHint());
@@ -1636,6 +1637,7 @@ WebRenderCommandBuilder::CreateImageKey(nsDisplayItem* aItem,
                                                  aContainer,
                                                  aSc,
                                                  rect,
+                                                 scBounds,
                                                  transform,
                                                  scaleToSize,
                                                  aRendering,
diff --git a/gfx/layers/wr/WebRenderUserData.cpp b/gfx/layers/wr/WebRenderUserData.cpp
index 3ca7057021b0..352d4065f4b3 100644
--- a/gfx/layers/wr/WebRenderUserData.cpp
+++ b/gfx/layers/wr/WebRenderUserData.cpp
@@ -229,6 +229,7 @@ WebRenderImageData::CreateAsyncImageWebRenderCommands(mozilla::wr::DisplayListBu
                                                       ImageContainer* aContainer,
                                                       const StackingContextHelper& aSc,
                                                       const LayoutDeviceRect& aBounds,
+                                                      const LayoutDeviceRect& aSCBounds,
                                                       const gfx::Matrix4x4& aSCTransform,
                                                       const gfx::MaybeIntSize& aScaleToSize,
                                                       const wr::ImageRendering& aFilter,
@@ -262,21 +263,10 @@ WebRenderImageData::CreateAsyncImageWebRenderCommands(mozilla::wr::DisplayListBu
   // where it will be done when we build the display list for the iframe.
   // That happens in AsyncImagePipelineManager.
   wr::LayoutRect r = wr::ToRoundedLayoutRect(aBounds);
-
-  Maybe<wr::WrClipId> originFrameId;
-  if (r.origin.x != 0.0 || r.origin.y != 0.0) {
-    originFrameId = Some(aBuilder.PushOrigin(r.origin));
-    r.origin = wr::LayoutPoint { 0.0, 0.0 };
-  }
-
   aBuilder.PushIFrame(r, aIsBackfaceVisible, mPipelineId.ref(), /*ignoreMissingPipelines*/ false);
 
-  if (originFrameId) {
-    aBuilder.PopOrigin();
-  }
-
   WrBridge()->AddWebRenderParentCommand(OpUpdateAsyncImagePipeline(mPipelineId.value(),
-                                                                   aBounds.Size(),
+                                                                   aSCBounds,
                                                                    aSCTransform,
                                                                    aScaleToSize,
                                                                    aFilter,
diff --git a/gfx/layers/wr/WebRenderUserData.h b/gfx/layers/wr/WebRenderUserData.h
index dc281aaf93f3..a91cd6ced4f9 100644
--- a/gfx/layers/wr/WebRenderUserData.h
+++ b/gfx/layers/wr/WebRenderUserData.h
@@ -139,6 +139,7 @@ public:
                                          ImageContainer* aContainer,
                                          const StackingContextHelper& aSc,
                                          const LayoutDeviceRect& aBounds,
+                                         const LayoutDeviceRect& aSCBounds,
                                          const gfx::Matrix4x4& aSCTransform,
                                          const gfx::MaybeIntSize& aScaleToSize,
                                          const wr::ImageRendering& aFilter,
diff --git a/gfx/webrender_bindings/WebRenderAPI.cpp b/gfx/webrender_bindings/WebRenderAPI.cpp
index 2e97083dde9a..38cf26e3d778 100644
--- a/gfx/webrender_bindings/WebRenderAPI.cpp
+++ b/gfx/webrender_bindings/WebRenderAPI.cpp
@@ -189,7 +189,7 @@ TransactionBuilder::RemovePipeline(PipelineId aPipelineId)
 void
 TransactionBuilder::SetDisplayList(gfx::Color aBgColor,
                                    Epoch aEpoch,
-                                   LayoutDeviceSize aViewportSize,
+                                   mozilla::LayerSize aViewportSize,
                                    wr::WrPipelineId pipeline_id,
                                    const wr::LayoutSize& content_size,
                                    wr::BuiltDisplayListDescriptor dl_descriptor,
@@ -870,29 +870,9 @@ DisplayListBuilder::Finalize(wr::LayoutSize& aOutContentSize,
                           &aOutDisplayList.dl.inner);
 }
 
-wr::WrClipId
-DisplayListBuilder::PushOrigin(const wr::LayoutPoint& aOrigin)
-{
-  WRDL_LOG("PushOrigin t=%s\n", mWrState,
-      Stringify(aOrigin).c_str());
-
-  //Note: there could be a simpler way to convert LayoutPoint -> LayoutTransform
-  wr::LayoutTransform transform = ToLayoutTransform(
-    gfx::Matrix4x4::Translation(aOrigin.x, aOrigin.y, 0.0));
-
-  auto id = wr_dp_push_reference_frame(mWrState, &transform);
-  return wr::WrClipId { id };
-}
-
-void
-DisplayListBuilder::PopOrigin()
-{
-  WRDL_LOG("PopOrigin\n", mWrState);
-  wr_dp_pop_reference_frame(mWrState);
-}
-
 Maybe<wr::WrClipId>
-DisplayListBuilder::PushStackingContext(const wr::WrClipId* aClipNodeId,
+DisplayListBuilder::PushStackingContext(const wr::LayoutRect& aBounds,
+                                        const wr::WrClipId* aClipNodeId,
                                         const WrAnimationProperty* aAnimation,
                                         const float* aOpacity,
                                         const gfx::Matrix4x4* aTransform,
@@ -915,13 +895,14 @@ DisplayListBuilder::PushStackingContext(const wr::WrClipId* aClipNodeId,
   if (aPerspective) {
     perspective = ToLayoutTransform(*aPerspective);
   }
+
   const wr::LayoutTransform* maybePerspective = aPerspective ? &perspective : nullptr;
-  WRDL_LOG("PushStackingContext t=%s\n", mWrState,
+  WRDL_LOG("PushStackingContext b=%s t=%s\n", mWrState, Stringify(aBounds).c_str(),
       aTransform ? Stringify(*aTransform).c_str() : "none");
 
   bool outIsReferenceFrame = false;
   uintptr_t outReferenceFrameId = 0;
-  wr_dp_push_stacking_context(mWrState, aClipNodeId, aAnimation,
+  wr_dp_push_stacking_context(mWrState, aBounds, aClipNodeId, aAnimation,
                               aOpacity, maybeTransform, aTransformStyle,
                               maybePerspective, aMixBlendMode,
                               aFilters.Elements(), aFilters.Length(),
diff --git a/gfx/webrender_bindings/WebRenderAPI.h b/gfx/webrender_bindings/WebRenderAPI.h
index 526d8c073dd0..575366c5be82 100644
--- a/gfx/webrender_bindings/WebRenderAPI.h
+++ b/gfx/webrender_bindings/WebRenderAPI.h
@@ -87,7 +87,7 @@ public:
 
   void SetDisplayList(gfx::Color aBgColor,
                       Epoch aEpoch,
-                      LayoutDeviceSize aViewportSize,
+                      mozilla::LayerSize aViewportSize,
                       wr::WrPipelineId pipeline_id,
                       const wr::LayoutSize& content_size,
                       wr::BuiltDisplayListDescriptor dl_descriptor,
@@ -335,10 +335,8 @@ public:
   void Finalize(wr::LayoutSize& aOutContentSize,
                 wr::BuiltDisplayList& aOutDisplayList);
 
-  wr::WrClipId PushOrigin(const wr::LayoutPoint& aOrigin);
-  void PopOrigin();
-
   Maybe<wr::WrClipId> PushStackingContext(
+          const wr::LayoutRect& aBounds, // TODO: We should work with strongly typed rects
           const wr::WrClipId* aClipNodeId,
           const wr::WrAnimationProperty* aAnimation,
           const float* aOpacity,
diff --git a/gfx/webrender_bindings/src/bindings.rs b/gfx/webrender_bindings/src/bindings.rs
index 3d2854166dce..3d5e6b4e5311 100644
--- a/gfx/webrender_bindings/src/bindings.rs
+++ b/gfx/webrender_bindings/src/bindings.rs
@@ -1853,32 +1853,9 @@ pub extern "C" fn wr_dp_clear_save(state: &mut WrState) {
     state.frame_builder.dl_builder.clear_save();
 }
 
-#[no_mangle]
-pub extern "C" fn wr_dp_push_reference_frame(state: &mut WrState,
-                                             transform: &LayoutTransform) -> usize {
-    debug_assert!(unsafe { !is_in_render_thread() });
-
-    let perspective = None;
-    let ref_frame_id = state.frame_builder.dl_builder.push_reference_frame(
-        &LayoutPrimitiveInfo::new(LayoutRect::zero()),
-        Some(PropertyBinding::Value(*transform)),
-        perspective,
-    );
-
-    state.frame_builder.dl_builder.push_clip_id(ref_frame_id);
-    pack_clip_id(ref_frame_id)
-}
-
-#[no_mangle]
-pub extern "C" fn wr_dp_pop_reference_frame(state: &mut WrState) {
-    debug_assert!(unsafe { !is_in_render_thread() });
-
-    state.frame_builder.dl_builder.pop_clip_id();
-    state.frame_builder.dl_builder.pop_reference_frame();
-}
-
 #[no_mangle]
 pub extern "C" fn wr_dp_push_stacking_context(state: &mut WrState,
+                                              bounds: LayoutRect,
                                               clip_node_id: *const WrClipId,
                                               animation: *const WrAnimationProperty,
                                               opacity: *const f32,
@@ -1915,54 +1892,56 @@ pub extern "C" fn wr_dp_push_stacking_context(state: &mut WrState,
         }
     }).collect();
 
-    let clip_node_id = match unsafe { clip_node_id.as_ref() } {
+    let clip_node_id_ref = unsafe { clip_node_id.as_ref() };
+    let clip_node_id = match clip_node_id_ref {
         Some(clip_node_id) => Some(unpack_clip_id(*clip_node_id, state.pipeline_id)),
         None => None,
     };
 
-    let transform = unsafe { transform.as_ref() }.cloned();
-    let mut transform_binding = transform.map(PropertyBinding::Value);
-    let opacity = unsafe { opacity.as_ref() }.cloned();
+    let transform_ref = unsafe { transform.as_ref() };
+    let mut transform_binding = match transform_ref {
+        Some(transform) => Some(PropertyBinding::Value(transform.clone())),
+        None => None,
+    };
 
+    let opacity_ref = unsafe { opacity.as_ref() };
     let mut has_opacity_animation = false;
-    if let Some(anim) = unsafe { animation.as_ref() } {
+    let anim = unsafe { animation.as_ref() };
+    if let Some(anim) = anim {
         debug_assert!(anim.id > 0);
         match anim.effect_type {
             WrAnimationType::Opacity => {
-                filters.push(FilterOp::Opacity(
-                    PropertyBinding::Binding(
-                        PropertyBindingKey::new(anim.id),
-                        // We have to set the static opacity value as
-                        // the value for the case where the animation is
-                        // in not in-effect (e.g. in the delay phase
-                        // with no corresponding fill mode).
-                        opacity.unwrap_or(1.0),
-                    ),
-                    1.0,
-                ));
+                filters.push(FilterOp::Opacity(PropertyBinding::Binding(PropertyBindingKey::new(anim.id),
+                                                                        // We have to set the static opacity value as
+                                                                        // the value for the case where the animation is
+                                                                        // in not in-effect (e.g. in the delay phase
+                                                                        // with no corresponding fill mode).
+                                                                        opacity_ref.cloned().unwrap_or(1.0)),
+                                                                        1.0));
                 has_opacity_animation = true;
             },
             WrAnimationType::Transform => {
                 transform_binding =
-                    Some(PropertyBinding::Binding(
-                        PropertyBindingKey::new(anim.id),
-                        // Same as above opacity case.
-                        transform.unwrap_or(LayoutTransform::identity()),
-                ));
+                    Some(PropertyBinding::Binding(PropertyBindingKey::new(anim.id),
+                                                  // Same as above opacity case.
+                                                  transform_ref.cloned().unwrap_or(LayoutTransform::identity())));
             },
         }
     }
 
-    if let Some(opacity) = opacity {
-        if !has_opacity_animation && opacity < 1.0 {
-            filters.push(FilterOp::Opacity(PropertyBinding::Value(opacity), opacity));
+    if let Some(opacity) = opacity_ref {
+        if !has_opacity_animation && *opacity < 1.0 {
+            filters.push(FilterOp::Opacity(PropertyBinding::Value(*opacity), *opacity));
         }
     }
 
-    let perspective = unsafe { perspective.as_ref() }.cloned();
-    // The only field of primitive info currently used by WR for stacking contexts
-    // is backface visibility. Layout rectangles don't matter.
-    let mut prim_info = LayoutPrimitiveInfo::new(LayoutRect::zero());
+    let perspective_ref = unsafe { perspective.as_ref() };
+    let perspective = match perspective_ref {
+        Some(perspective) => Some(perspective.clone()),
+        None => None,
+    };
+
+    let mut prim_info = LayoutPrimitiveInfo::new(bounds);
 
     *out_is_reference_frame = transform_binding.is_some() || perspective.is_some();
     if *out_is_reference_frame {
@@ -1971,27 +1950,28 @@ pub extern "C" fn wr_dp_push_stacking_context(state: &mut WrState,
             .push_reference_frame(&prim_info, transform_binding, perspective);
         *out_reference_frame_id = pack_clip_id(ref_frame_id);
 
+        prim_info.rect.origin = LayoutPoint::zero();
+        prim_info.clip_rect.origin = LayoutPoint::zero();
         state.frame_builder.dl_builder.push_clip_id(ref_frame_id);
     }
 
     prim_info.is_backface_visible = is_backface_visible;
     prim_info.tag = state.current_tag;
 
-    state.frame_builder.dl_builder.push_stacking_context(
-        &prim_info,
-        clip_node_id,
-        transform_style,
-        mix_blend_mode,
-        &filters,
-        glyph_raster_space,
-    );
+    state.frame_builder
+         .dl_builder
+         .push_stacking_context(&prim_info,
+                                clip_node_id,
+                                transform_style,
+                                mix_blend_mode,
+                                &filters,
+                                glyph_raster_space);
 }
 
 #[no_mangle]
 pub extern "C" fn wr_dp_pop_stacking_context(state: &mut WrState,
                                              is_reference_frame: bool) {
     debug_assert!(unsafe { !is_in_render_thread() });
-
     state.frame_builder.dl_builder.pop_stacking_context();
     if is_reference_frame {
         state.frame_builder.dl_builder.pop_clip_id();
@@ -2006,7 +1986,6 @@ pub extern "C" fn wr_dp_define_clipchain(state: &mut WrState,
                                          clips_count: usize)
                                          -> u64 {
     debug_assert!(unsafe { is_in_main_thread() });
-
     let parent = unsafe { parent_clipchain_id.as_ref() }.map(|id| ClipChainId(*id, state.pipeline_id));
     let pipeline_id = state.pipeline_id;
     let clips = make_slice(clips, clips_count)
diff --git a/gfx/webrender_bindings/webrender_ffi_generated.h b/gfx/webrender_bindings/webrender_ffi_generated.h
index 25b8d23a7a91..2da5db1a1e51 100644
--- a/gfx/webrender_bindings/webrender_ffi_generated.h
+++ b/gfx/webrender_bindings/webrender_ffi_generated.h
@@ -816,6 +816,28 @@ struct GradientStop {
   }
 };
 
+struct Shadow {
+  LayoutVector2D offset;
+  ColorF color;
+  float blur_radius;
+
+  bool operator==(const Shadow& aOther) const {
+    return offset == aOther.offset &&
+           color == aOther.color &&
+           blur_radius == aOther.blur_radius;
+  }
+};
+
+struct WrAnimationProperty {
+  WrAnimationType effect_type;
+  uint64_t id;
+
+  bool operator==(const WrAnimationProperty& aOther) const {
+    return effect_type == aOther.effect_type &&
+           id == aOther.id;
+  }
+};
+
 // A 3d transform stored as a 4 by 4 matrix in row-major order in memory.
 //
 // Transforms can be parametrized over the source and destination units, to describe a
@@ -868,28 +890,6 @@ struct TypedTransform3D {
 
 using LayoutTransform = TypedTransform3D<float, LayoutPixel, LayoutPixel>;
 
-struct Shadow {
-  LayoutVector2D offset;
-  ColorF color;
-  float blur_radius;
-
-  bool operator==(const Shadow& aOther) const {
-    return offset == aOther.offset &&
-           color == aOther.color &&
-           blur_radius == aOther.blur_radius;
-  }
-};
-
-struct WrAnimationProperty {
-  WrAnimationType effect_type;
-  uint64_t id;
-
-  bool operator==(const WrAnimationProperty& aOther) const {
-    return effect_type == aOther.effect_type &&
-           id == aOther.id;
-  }
-};
-
 struct WrFilterOp {
   WrFilterOpType filter_type;
   float argument;
@@ -1359,10 +1359,6 @@ WR_INLINE
 void wr_dp_pop_clip_and_scroll_info(WrState *aState)
 WR_FUNC;
 
-WR_INLINE
-void wr_dp_pop_reference_frame(WrState *aState)
-WR_FUNC;
-
 WR_INLINE
 void wr_dp_pop_scroll_layer(WrState *aState)
 WR_FUNC;
@@ -1532,11 +1528,6 @@ void wr_dp_push_rect(WrState *aState,
                      ColorF aColor)
 WR_FUNC;
 
-WR_INLINE
-uintptr_t wr_dp_push_reference_frame(WrState *aState,
-                                     const LayoutTransform *aTransform)
-WR_FUNC;
-
 WR_INLINE
 void wr_dp_push_scroll_layer(WrState *aState,
                              WrClipId aScrollId)
@@ -1552,6 +1543,7 @@ WR_FUNC;
 
 WR_INLINE
 void wr_dp_push_stacking_context(WrState *aState,
+                                 LayoutRect aBounds,
                                  const WrClipId *aClipNodeId,
                                  const WrAnimationProperty *aAnimation,
                                  const float *aOpacity,
diff --git a/layout/base/nsLayoutUtils.h b/layout/base/nsLayoutUtils.h
index 73af47db815d..557c1289a54a 100644
--- a/layout/base/nsLayoutUtils.h
+++ b/layout/base/nsLayoutUtils.h
@@ -187,7 +187,6 @@ public:
   typedef mozilla::ScreenMargin ScreenMargin;
   typedef mozilla::LayoutDeviceIntSize LayoutDeviceIntSize;
   typedef mozilla::LayoutDeviceRect LayoutDeviceRect;
-  typedef mozilla::LayoutDeviceSize LayoutDeviceSize;
   typedef mozilla::StyleGeometryBox StyleGeometryBox;
   typedef mozilla::SVGImageContext SVGImageContext;
   typedef mozilla::LogicalSize LogicalSize;
diff --git a/layout/generic/nsHTMLCanvasFrame.cpp b/layout/generic/nsHTMLCanvasFrame.cpp
index 6be166c646ff..3912b9c4c8bb 100644
--- a/layout/generic/nsHTMLCanvasFrame.cpp
+++ b/layout/generic/nsHTMLCanvasFrame.cpp
@@ -185,10 +185,11 @@ public:
         }
 
         MaybeIntSize scaleToSize;
+        LayoutDeviceRect scBounds(LayoutDevicePoint(0, 0), bounds.Size());
         wr::ImageRendering filter = wr::ToImageRendering(nsLayoutUtils::GetSamplingFilterForFrame(mFrame));
         wr::MixBlendMode mixBlendMode = wr::MixBlendMode::Normal;
         aManager->WrBridge()->AddWebRenderParentCommand(OpUpdateAsyncImagePipeline(data->GetPipelineId().value(),
-                                                                                   bounds.Size(),
+                                                                                   scBounds,
                                                                                    scTransform,
                                                                                    scaleToSize,
                                                                                    filter,
diff --git a/layout/painting/nsDisplayList.cpp b/layout/painting/nsDisplayList.cpp
index 39b50a6c192c..173e89125916 100644
--- a/layout/painting/nsDisplayList.cpp
+++ b/layout/painting/nsDisplayList.cpp
@@ -6830,6 +6830,7 @@ nsDisplayOpacity::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            filters,
+                           LayoutDeviceRect(),
                            nullptr,
                            animationsId ? &prop : nullptr,
                            opacityForSC);
@@ -6883,10 +6884,10 @@ nsDisplayBlendMode::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            filters,
+                           LayoutDeviceRect(),
                            nullptr,
                            nullptr,
                            nullptr,
-                           LayoutDevicePoint(),
                            nullptr,
                            nullptr,
                            nsCSSRendering::GetGFXBlendMode(mBlendMode));
@@ -7175,6 +7176,7 @@ nsDisplayOwnLayer::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            nsTArray<wr::WrFilterOp>(),
+                           LayoutDeviceRect(),
                            nullptr,
                            &prop);
 
@@ -7833,14 +7835,6 @@ nsDisplayStickyPosition::CreateWebRenderCommands(
     nsRect scrollPort =
       stickyScrollContainer->ScrollFrame()->GetScrollPortRect();
     scrollPort += offset;
-    // It would be cleaner to just modify `scrollPort` here instead of
-    // adjusting the computed margins, but the `scrollOrigin` is in a different
-    // space.
-    // Note: we aren't applying the reference frame origin if the scroll frame
-    // is already attached to it.
-    auto& scrollOrigin =
-      nsLayoutUtils::GetReferenceFrame(scrollFrame) == ReferenceFrame() ?
-      LayoutDevicePoint() : aSc.GetInheritedStickyOrigin();
 
     // The following computations make more sense upon understanding the
     // semantics of "inner" and "outer", which is explained in the comment on
@@ -7864,8 +7858,7 @@ nsDisplayStickyPosition::CreateWebRenderCommands(
       // -distance works.
       nscoord distance = DistanceToRange(inner.YMost(), outer.YMost());
       topMargin = Some(NSAppUnitsToFloatPixels(
-        itemBounds.y - scrollPort.y - distance, auPerDevPixel
-        ) - scrollOrigin.y);
+        itemBounds.y - scrollPort.y - distance, auPerDevPixel));
       // Question: What is the maximum positive ("downward") offset that WR
       // will have to apply to this item in order to prevent the item from
       // visually moving?
@@ -7891,8 +7884,7 @@ nsDisplayStickyPosition::CreateWebRenderCommands(
       // the distance from itemBounds.YMost() to scrollPort.YMost().
       nscoord distance = DistanceToRange(outer.Y(), inner.Y());
       bottomMargin = Some(NSAppUnitsToFloatPixels(
-        scrollPort.YMost() - itemBounds.YMost() + distance, auPerDevPixel
-        ) - scrollOrigin.y);
+        scrollPort.YMost() - itemBounds.YMost() + distance, auPerDevPixel));
       // And here WR will be moving the item upwards rather than downwards so
       // again things are inverted from the previous block.
       vBounds.min =
@@ -7909,8 +7901,7 @@ nsDisplayStickyPosition::CreateWebRenderCommands(
     if (outer.XMost() != inner.XMost()) {
       nscoord distance = DistanceToRange(inner.XMost(), outer.XMost());
       leftMargin = Some(NSAppUnitsToFloatPixels(
-        itemBounds.x - scrollPort.x - distance, auPerDevPixel
-        ) - scrollOrigin.x);
+        itemBounds.x - scrollPort.x - distance, auPerDevPixel));
       hBounds.max =
         NSAppUnitsToFloatPixels(outer.XMost() - inner.XMost(), auPerDevPixel);
       if (inner.XMost() < 0) {
@@ -7921,8 +7912,7 @@ nsDisplayStickyPosition::CreateWebRenderCommands(
     if (outer.X() != inner.X()) {
       nscoord distance = DistanceToRange(outer.X(), inner.X());
       rightMargin = Some(NSAppUnitsToFloatPixels(
-        scrollPort.XMost() - itemBounds.XMost() + distance, auPerDevPixel
-        ) - scrollOrigin.x);
+        scrollPort.XMost() - itemBounds.XMost() + distance, auPerDevPixel));
       hBounds.min =
         NSAppUnitsToFloatPixels(outer.X() - inner.X(), auPerDevPixel);
       if (appliedOffset.x == 0 && inner.X() > 0) {
@@ -8926,10 +8916,10 @@ nsDisplayTransform::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            filters,
+                           LayoutDeviceRect(position, LayoutDeviceSize()),
                            &newTransformMatrix,
                            animationsId ? &prop : nullptr,
                            nullptr,
-                           position,
                            transformForSC,
                            nullptr,
                            gfx::CompositionOp::OP_OVER,
@@ -9576,10 +9566,10 @@ nsDisplayPerspective::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            filters,
+                           LayoutDeviceRect(),
                            nullptr,
                            nullptr,
                            nullptr,
-                           LayoutDevicePoint(),
                            &transformForSC,
                            &perspectiveMatrix,
                            gfx::CompositionOp::OP_OVER,
@@ -10279,10 +10269,10 @@ nsDisplayMasksAndClipPaths::CreateWebRenderCommands(
                   GetActiveScrolledRoot(),
                   aBuilder,
                   /*aFilters: */ nsTArray<wr::WrFilterOp>(),
+                  /*aBounds: */ bounds,
                   /*aBoundTransform: */ nullptr,
                   /*aAnimation: */ nullptr,
                   /*aOpacity: */ opacity.ptrOr(nullptr),
-                  /*aOrigin: */ LayoutDevicePoint(),
                   /*aTransform: */ nullptr,
                   /*aPerspective: */ nullptr,
                   /*aMixBlendMode: */ gfx::CompositionOp::OP_OVER,
@@ -10602,11 +10592,11 @@ nsDisplayFilters::CreateWebRenderCommands(
                            GetActiveScrolledRoot(),
                            aBuilder,
                            wrFilters,
+                           LayoutDeviceRect(),
                            nullptr,
                            nullptr,
                            opacity != 1.0f && mHandleOpacity ? &opacity
                                                              : nullptr,
-                           LayoutDevicePoint(),
                            nullptr,
                            nullptr,
                            gfx::CompositionOp::OP_OVER,
diff --git a/layout/reftests/bugs/reftest.list b/layout/reftests/bugs/reftest.list
index 28dcbed45563..7e82e0229cb6 100644
--- a/layout/reftests/bugs/reftest.list
+++ b/layout/reftests/bugs/reftest.list
@@ -2077,7 +2077,7 @@ test-pref(font.size.systemFontScale,200) == 1412743.html 1412743-ref.html
 == 1424680.html 1424680-ref.html
 == 1424798-1.html 1424798-ref.html
 fuzzy-if(!webrender,0-74,0-2234) == 1425243-1.html 1425243-1-ref.html
-fuzzy-if(Android,0-66,0-574) fuzzy-if(d2d,0-89,0-777) fuzzy-if(!Android&&!d2d,0-1,0-31341) fuzzy-if(webrender&&winWidget,1-1,31308-31320) == 1425243-2.html 1425243-2-ref.html
+fuzzy-if(Android,0-66,0-574) fuzzy-if(d2d,0-89,0-777) fuzzy-if(!Android&&!d2d,0-1,0-31341) fuzzy-if(webrender&&winWidget,1-1,31320-31320) == 1425243-2.html 1425243-2-ref.html
 == 1430869.html 1430869-ref.html
 == 1432541.html 1432541-ref.html
 pref(layout.css.moz-document.url-prefix-hack.enabled,true) == 1446470.html 1035091-ref.html
diff --git a/layout/reftests/svg/conditions-07.svg b/layout/reftests/svg/conditions-07.svg
index 7ed4ef12b81e..15b5ff0dbdc9 100644
--- a/layout/reftests/svg/conditions-07.svg
+++ b/layout/reftests/svg/conditions-07.svg
@@ -17,7 +17,7 @@
             return;
           }
           f1.removeAttribute("systemLanguage");
-
+          
         } catch(e) {
           var f = document.getElementById("fail");
           f.setAttribute("fill", "red");
@@ -28,7 +28,7 @@
 
   <rect width="100%" height="100%" fill="lime"/>
   <!-- background images -->
-  <rect x="100" y="100" width="100" height="100" fill="yellow"/>
+  <rect x="100" y="100" width="100" height="100" fill="red"/>
 
   <!-- tests -->
   <foreignObject id="f1" x="100" y="100" width="100" height="100" systemLanguage="x">

From 04087c53c1958b1483a5f8b71c9fe7339af39d94 Mon Sep 17 00:00:00 2001
From: Marco Castelluccio <mcastelluccio@mozilla.com>
Date: Tue, 27 Nov 2018 23:53:33 +0100
Subject: [PATCH 13/78] Bug 1510452 - Use srcdir_relpath instead of
 file_relpath for tests and test support files. r=ahal

--HG--
extra : rebase_source : ec726b7b2a409e6e30dcdee2e5795eabacd56fb6
---
 tools/tryselect/selectors/coverage.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py
index ae17a38707d3..e1e8ac464323 100644
--- a/tools/tryselect/selectors/coverage.py
+++ b/tools/tryselect/selectors/coverage.py
@@ -74,7 +74,7 @@ def read_test_manifests():
     tests = set()
 
     for test in test_resolver.resolve_tests(build.topsrcdir):
-        tests.add(test['file_relpath'])
+        tests.add(test['srcdir_relpath'])
         if 'support-files' not in test:
             continue
 
@@ -91,11 +91,11 @@ def read_test_manifests():
             # If it doesn't have a glob, then it's a single file.
             if '*' not in support_file_pattern:
                 # Simple case: single support file, just add it here.
-                support_files_map[support_file_pattern].append(test['file_relpath'])
+                support_files_map[support_file_pattern].append(test['srcdir_relpath'])
                 continue
 
             for support_file, _ in file_finder.find(support_file_pattern):
-                support_files_map[support_file].append(test['file_relpath'])
+                support_files_map[support_file].append(test['srcdir_relpath'])
 
     return tests, support_files_map
 

From 33976b9bd258f1166b321acd10357a46bfdda4ac Mon Sep 17 00:00:00 2001
From: Marco Castelluccio <mcastelluccio@mozilla.com>
Date: Tue, 27 Nov 2018 17:29:34 +0100
Subject: [PATCH 14/78] Bug 1510391 - When a task is not found, don't add None
 to the list of tasks to run. r=ahal

--HG--
extra : rebase_source : ef53880797df740bbbd289099cb602539b208599
---
 tools/tryselect/selectors/coverage.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tools/tryselect/selectors/coverage.py b/tools/tryselect/selectors/coverage.py
index e1e8ac464323..5b2c1b3a8895 100644
--- a/tools/tryselect/selectors/coverage.py
+++ b/tools/tryselect/selectors/coverage.py
@@ -330,8 +330,8 @@ def filter_tasks_by_chunks(tasks, chunks):
 
         if selected_task is None:
             print('Warning: no task found for chunk', platform, chunk)
-
-        selected_tasks.add(selected_task)
+        else:
+            selected_tasks.add(selected_task)
 
     return list(selected_tasks)
 

From 33208bfd8ee8314fad91ecae361591ec6876bee3 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:14 +0100
Subject: [PATCH 15/78] Bug 1286798 - Part 1: Allow
 BackgroundChild::GetOrCreateForCurrentThread() to use a custom main event
 target; r=mccr8,asuth

---
 ipc/glue/BackgroundChild.h  |  4 +++-
 ipc/glue/BackgroundImpl.cpp | 42 ++++++++++++++++++++++++++-----------
 2 files changed, 33 insertions(+), 13 deletions(-)

diff --git a/ipc/glue/BackgroundChild.h b/ipc/glue/BackgroundChild.h
index 34590c23c75c..efecb67951b5 100644
--- a/ipc/glue/BackgroundChild.h
+++ b/ipc/glue/BackgroundChild.h
@@ -10,6 +10,8 @@
 #include "mozilla/Attributes.h"
 #include "mozilla/ipc/Transport.h"
 
+class nsIEventTarget;
+
 namespace mozilla {
 namespace dom {
 
@@ -55,7 +57,7 @@ public:
 
   // See above.
   static PBackgroundChild*
-  GetOrCreateForCurrentThread();
+  GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget = nullptr);
 
   // See above.
   static void
diff --git a/ipc/glue/BackgroundImpl.cpp b/ipc/glue/BackgroundImpl.cpp
index 528e1ce8f6dc..e83582cfeffe 100644
--- a/ipc/glue/BackgroundImpl.cpp
+++ b/ipc/glue/BackgroundImpl.cpp
@@ -177,7 +177,7 @@ private:
 
 public:
   static already_AddRefed<ChildImpl>
-  CreateActorForSameProcess();
+  CreateActorForSameProcess(nsIEventTarget* aMainEventTarget);
 
   static bool
   IsOnBackgroundThread()
@@ -399,7 +399,7 @@ private:
 
   // Forwarded from BackgroundChild.
   static PBackgroundChild*
-  GetOrCreateForCurrentThread();
+  GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget);
 
   // Forwarded from BackgroundChild.
   static void
@@ -569,7 +569,8 @@ public:
   }
 
   nsresult
-  BlockAndGetResults(RefPtr<ParentImpl>& aParentActor,
+  BlockAndGetResults(nsIEventTarget* aMainEventTarget,
+                     RefPtr<ParentImpl>& aParentActor,
                      nsCOMPtr<nsIThread>& aThread);
 
 private:
@@ -716,9 +717,9 @@ BackgroundChild::GetForCurrentThread()
 
 // static
 PBackgroundChild*
-BackgroundChild::GetOrCreateForCurrentThread()
+BackgroundChild::GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget)
 {
-  return ChildImpl::GetOrCreateForCurrentThread();
+  return ChildImpl::GetOrCreateForCurrentThread(aMainEventTarget);
 }
 
 // static
@@ -914,7 +915,7 @@ ParentImpl::Alloc(ContentParent* aContent,
 
 // static
 already_AddRefed<ChildImpl>
-ParentImpl::CreateActorForSameProcess()
+ParentImpl::CreateActorForSameProcess(nsIEventTarget* aMainEventTarget)
 {
   AssertIsInMainProcess();
 
@@ -936,7 +937,9 @@ ParentImpl::CreateActorForSameProcess()
   } else {
     RefPtr<CreateActorHelper> helper = new CreateActorHelper();
 
-    nsresult rv = helper->BlockAndGetResults(parentActor, backgroundThread);
+    nsresult rv = helper->BlockAndGetResults(aMainEventTarget,
+                                             parentActor,
+                                             backgroundThread);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return nullptr;
     }
@@ -1302,10 +1305,17 @@ ParentImpl::ConnectActorRunnable::Run()
 
 nsresult
 ParentImpl::
-CreateActorHelper::BlockAndGetResults(RefPtr<ParentImpl>& aParentActor,
+CreateActorHelper::BlockAndGetResults(nsIEventTarget* aMainEventTarget,
+                                      RefPtr<ParentImpl>& aParentActor,
                                       nsCOMPtr<nsIThread>& aThread)
 {
-  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+  AssertIsNotOnMainThread();
+
+  if (aMainEventTarget) {
+    MOZ_ALWAYS_SUCCEEDS(aMainEventTarget->Dispatch(this, NS_DISPATCH_NORMAL));
+  } else {
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+  }
 
   mozilla::MonitorAutoLock lock(mMonitor);
   while (mWaiting) {
@@ -1449,8 +1459,10 @@ ChildImpl::GetForCurrentThread()
 
 /* static */
 PBackgroundChild*
-ChildImpl::GetOrCreateForCurrentThread()
+ChildImpl::GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget)
 {
+  MOZ_ASSERT_IF(NS_IsMainThread(), !aMainEventTarget);
+
   MOZ_ASSERT(sThreadLocalIndex != kBadThreadLocalIndex,
              "BackgroundChild::Startup() was never called!");
 
@@ -1481,7 +1493,8 @@ ChildImpl::GetOrCreateForCurrentThread()
   }
 
   if (XRE_IsParentProcess()) {
-    RefPtr<ChildImpl> strongActor = ParentImpl::CreateActorForSameProcess();
+    RefPtr<ChildImpl> strongActor =
+      ParentImpl::CreateActorForSameProcess(aMainEventTarget);
     if (NS_WARN_IF(!strongActor)) {
       return nullptr;
     }
@@ -1534,7 +1547,12 @@ ChildImpl::GetOrCreateForCurrentThread()
         content,
         &ContentChild::SendInitBackground,
         std::move(parent));
-    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
+    if (aMainEventTarget) {
+      MOZ_ALWAYS_SUCCEEDS(aMainEventTarget->Dispatch(runnable,
+                                                     NS_DISPATCH_NORMAL));
+    } else {
+      MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
+    }
   }
 
   RefPtr<ChildImpl>& actor = threadLocalInfo->mActor;

From 89fb42ab069321087be8cc49f25b8ff5a3a956a6 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:17 +0100
Subject: [PATCH 16/78] Bug 1286798 - Part 2: Add IsOnDOMFileThread() and
 AssertIsOnDOMFileThread() generic helper methods; r=asuth

---
 dom/file/ipc/IPCBlobInputStreamThread.cpp | 17 +++++++++++++++++
 dom/file/ipc/IPCBlobInputStreamThread.h   |  6 ++++++
 2 files changed, 23 insertions(+)

diff --git a/dom/file/ipc/IPCBlobInputStreamThread.cpp b/dom/file/ipc/IPCBlobInputStreamThread.cpp
index 15671b7cb769..a4cfa52a442b 100644
--- a/dom/file/ipc/IPCBlobInputStreamThread.cpp
+++ b/dom/file/ipc/IPCBlobInputStreamThread.cpp
@@ -247,5 +247,22 @@ IPCBlobInputStreamThread::DelayedDispatch(already_AddRefed<nsIRunnable>, uint32_
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
+bool
+IsOnDOMFileThread()
+{
+  mozilla::StaticMutexAutoLock lock(gIPCBlobThreadMutex);
+
+  MOZ_ASSERT(!gShutdownHasStarted);
+  MOZ_ASSERT(gIPCBlobThread);
+
+  return gIPCBlobThread->IsOnCurrentThreadInfallible();
+}
+
+void
+AssertIsOnDOMFileThread()
+{
+  MOZ_ASSERT(IsOnDOMFileThread());
+}
+
 } // dom namespace
 } // mozilla namespace
diff --git a/dom/file/ipc/IPCBlobInputStreamThread.h b/dom/file/ipc/IPCBlobInputStreamThread.h
index 45900a6182a6..02bda56e1de7 100644
--- a/dom/file/ipc/IPCBlobInputStreamThread.h
+++ b/dom/file/ipc/IPCBlobInputStreamThread.h
@@ -54,6 +54,12 @@ private:
   nsTArray<RefPtr<IPCBlobInputStreamChild>> mPendingActors;
 };
 
+bool
+IsOnDOMFileThread();
+
+void
+AssertIsOnDOMFileThread();
+
 } // dom namespace
 } // mozilla namespace
 

From 60831f2e38f0a14a4face761e3f2949d8d07bfb7 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:20 +0100
Subject: [PATCH 17/78] Bug 1286798 - Part 3: New basic (memory only)
 implementation of LocalStorage; r=asuth,mccr8

The implementation is based on a cache (datastore) living in the parent process and sync IPC calls initiated from content processes.
IPC communication is done using per principal/origin database actors which connect to the datastore.
The synchronous blocking of the main thread is done by creating a nested event target and spinning the event loop.
---
 .../test/browser/browser.ini                  |   2 +-
 .../test/browser/browser-common.ini           |   2 +-
 .../privatebrowsing/test/browser/browser.ini  |   2 +-
 .../components/sessionstore/test/browser.ini  |   2 +-
 devtools/client/storage/test/browser.ini      |  18 +-
 devtools/server/tests/browser/browser.ini     |   6 +-
 dom/base/nsGlobalWindowInner.cpp              |  50 +-
 dom/base/test/mochitest.ini                   |   2 +-
 dom/ipc/ContentChild.cpp                      |   9 +-
 dom/ipc/ContentChild.h                        |   2 +-
 dom/localstorage/ActorsChild.cpp              | 118 +++
 dom/localstorage/ActorsChild.h                | 116 +++
 dom/localstorage/ActorsParent.cpp             | 975 ++++++++++++++++++
 dom/localstorage/ActorsParent.h               |  49 +
 dom/localstorage/LSDatabase.cpp               | 141 +++
 dom/localstorage/LSDatabase.h                 |  73 ++
 dom/localstorage/LSObject.cpp                 | 629 +++++++++++
 dom/localstorage/LSObject.h                   | 110 ++
 dom/localstorage/LocalStorageCommon.cpp       |  32 +
 dom/localstorage/LocalStorageCommon.h         | 193 ++++
 dom/localstorage/LocalStorageManager2.cpp     |  88 ++
 dom/localstorage/LocalStorageManager2.h       |  31 +
 dom/localstorage/PBackgroundLSDatabase.ipdl   |  49 +
 dom/localstorage/PBackgroundLSRequest.ipdl    |  49 +
 .../PBackgroundLSSharedTypes.ipdlh            |  32 +
 dom/localstorage/moz.build                    |  41 +
 dom/moz.build                                 |   1 +
 dom/storage/LocalStorageManager.cpp           |  14 +
 dom/storage/LocalStorageManager.h             |   2 +-
 dom/storage/StorageIPC.cpp                    |   3 +
 dom/storage/StorageObserver.cpp               |  63 +-
 dom/tests/browser/browser.ini                 |   4 +-
 dom/tests/mochitest/localstorage/chrome.ini   |   4 +-
 .../mochitest/localstorage/mochitest.ini      |  22 +-
 .../mochitest/storageevent/mochitest.ini      |   4 +-
 dom/tests/moz.build                           |   2 +-
 editor/libeditor/tests/mochitest.ini          |   6 +-
 ipc/glue/BackgroundChildImpl.cpp              |  72 +-
 ipc/glue/BackgroundChildImpl.h                |  13 +
 ipc/glue/BackgroundParentImpl.cpp             |  74 ++
 ipc/glue/BackgroundParentImpl.h               |  22 +
 ipc/glue/PBackground.ipdl                     |   9 +
 ipc/ipdl/sync-messages.ini                    |  14 +
 layout/build/nsLayoutModule.cpp               |  16 +-
 modules/libpref/init/all.js                   |   6 +
 .../opener-closed.html.ini                    |   2 +
 .../opener-noopener.html.ini                  |   2 +
 .../opener-noreferrer.html.ini                |   2 +
 .../choose-_blank-002.html.ini                |   2 +
 .../choose-_parent-004.html.ini               |   2 +
 .../choose-_self-002.html.ini                 |   2 +
 .../choose-_top-001.html.ini                  |   2 +
 .../choose-_top-002.html.ini                  |   2 +
 .../choose-_top-003.html.ini                  |   2 +
 .../windows/noreferrer-null-opener.html.ini   |   2 +
 .../windows/noreferrer-window-name.html.ini   |   3 +-
 .../meta/webstorage/document-domain.html.ini  |   2 +-
 .../meta/webstorage/event_basic.html.ini      |   2 +
 .../webstorage/event_body_attribute.html.ini  |   2 +
 .../webstorage/event_case_sensitive.html.ini  |   2 +
 .../meta/webstorage/event_local_key.html.ini  |   2 +
 .../webstorage/event_local_newvalue.html.ini  |   2 +
 .../webstorage/event_local_oldvalue.html.ini  |   2 +
 .../event_local_removeitem.html.ini           |   2 +
 .../event_local_storagearea.html.ini          |   2 +
 .../meta/webstorage/event_local_url.html.ini  |   2 +
 .../webstorage/event_no_duplicates.html.ini   |   2 +
 .../webstorage/event_setattribute.html.ini    |   2 +
 ...ge_local_setitem_quotaexceedederr.html.ini |   2 +
 .../antitracking/test/browser/browser.ini     |   8 +-
 .../test/mochitest/mochitest-common.ini       |   6 +-
 .../extensions/test/mochitest/mochitest.ini   |   2 +-
 toolkit/forgetaboutsite/moz.build             |   2 +-
 .../forgetaboutsite/test/unit/xpcshell.ini    |   2 +-
 74 files changed, 3134 insertions(+), 107 deletions(-)
 create mode 100644 dom/localstorage/ActorsChild.cpp
 create mode 100644 dom/localstorage/ActorsChild.h
 create mode 100644 dom/localstorage/ActorsParent.cpp
 create mode 100644 dom/localstorage/ActorsParent.h
 create mode 100644 dom/localstorage/LSDatabase.cpp
 create mode 100644 dom/localstorage/LSDatabase.h
 create mode 100644 dom/localstorage/LSObject.cpp
 create mode 100644 dom/localstorage/LSObject.h
 create mode 100644 dom/localstorage/LocalStorageCommon.cpp
 create mode 100644 dom/localstorage/LocalStorageCommon.h
 create mode 100644 dom/localstorage/LocalStorageManager2.cpp
 create mode 100644 dom/localstorage/LocalStorageManager2.h
 create mode 100644 dom/localstorage/PBackgroundLSDatabase.ipdl
 create mode 100644 dom/localstorage/PBackgroundLSRequest.ipdl
 create mode 100644 dom/localstorage/PBackgroundLSSharedTypes.ipdlh
 create mode 100644 dom/localstorage/moz.build
 create mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
 create mode 100644 testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_basic.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_body_attribute.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_key.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_local_url.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/event_setattribute.html.ini
 create mode 100644 testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini

diff --git a/browser/components/contextualidentity/test/browser/browser.ini b/browser/components/contextualidentity/test/browser/browser.ini
index 7cc9cd57c6ec..451bcd062722 100644
--- a/browser/components/contextualidentity/test/browser/browser.ini
+++ b/browser/components/contextualidentity/test/browser/browser.ini
@@ -10,7 +10,7 @@ support-files =
 [browser_aboutURLs.js]
 [browser_eme.js]
 [browser_favicon.js]
-[browser_forgetaboutsite.js]
+#[browser_forgetaboutsite.js]
 [browser_forgetAPI_cookie_getCookiesWithOriginAttributes.js]
 [browser_restore_getCookiesWithOriginAttributes.js]
 [browser_forgetAPI_EME_forgetThisSite.js]
diff --git a/browser/components/extensions/test/browser/browser-common.ini b/browser/components/extensions/test/browser/browser-common.ini
index 62d5d0703641..9fd71872c21b 100644
--- a/browser/components/extensions/test/browser/browser-common.ini
+++ b/browser/components/extensions/test/browser/browser-common.ini
@@ -65,7 +65,7 @@ skip-if = (os == 'mac' && debug) # Bug 1482004, also fails in test-verify
 [browser_ext_browsingData_formData.js]
 [browser_ext_browsingData_history.js]
 [browser_ext_browsingData_indexedDB.js]
-[browser_ext_browsingData_localStorage.js]
+#[browser_ext_browsingData_localStorage.js]
 [browser_ext_browsingData_pluginData.js]
 [browser_ext_browsingData_serviceWorkers.js]
 [browser_ext_chrome_settings_overrides_home.js]
diff --git a/browser/components/privatebrowsing/test/browser/browser.ini b/browser/components/privatebrowsing/test/browser/browser.ini
index 43947f3de878..6db62a2fa6fe 100644
--- a/browser/components/privatebrowsing/test/browser/browser.ini
+++ b/browser/components/privatebrowsing/test/browser/browser.ini
@@ -25,7 +25,7 @@ tags = trackingprotection
 [browser_privatebrowsing_aboutSessionRestore.js]
 [browser_privatebrowsing_cache.js]
 [browser_privatebrowsing_certexceptionsui.js]
-[browser_privatebrowsing_concurrent.js]
+#[browser_privatebrowsing_concurrent.js]
 [browser_privatebrowsing_context_and_chromeFlags.js]
 [browser_privatebrowsing_crh.js]
 [browser_privatebrowsing_downloadLastDir.js]
diff --git a/browser/components/sessionstore/test/browser.ini b/browser/components/sessionstore/test/browser.ini
index 3a05d3734099..b1ecf4b2bdb2 100644
--- a/browser/components/sessionstore/test/browser.ini
+++ b/browser/components/sessionstore/test/browser.ini
@@ -154,7 +154,7 @@ skip-if = true
 [browser_461634.js]
 [browser_463205.js]
 [browser_463206.js]
-[browser_464199.js]
+#[browser_464199.js]
 [browser_465215.js]
 [browser_465223.js]
 [browser_466937.js]
diff --git a/devtools/client/storage/test/browser.ini b/devtools/client/storage/test/browser.ini
index d9f9115ab09d..ab13a1931d9e 100644
--- a/devtools/client/storage/test/browser.ini
+++ b/devtools/client/storage/test/browser.ini
@@ -45,23 +45,23 @@ tags = usercontextid
 [browser_storage_cookies_samesite.js]
 skip-if = true # Bug 1448484 - sameSite1 is "Unset" - Got undefined, expected Unset
 [browser_storage_cookies_tab_navigation.js]
-[browser_storage_delete.js]
-[browser_storage_delete_all.js]
-[browser_storage_delete_tree.js]
-[browser_storage_delete_usercontextid.js]
-tags = usercontextid
+#[browser_storage_delete.js]
+#[browser_storage_delete_all.js]
+#[browser_storage_delete_tree.js]
+#[browser_storage_delete_usercontextid.js]
+#tags = usercontextid
 [browser_storage_dom_cache_disabled.js]
 [browser_storage_dynamic_updates_cookies.js]
-[browser_storage_dynamic_updates_localStorage.js]
-[browser_storage_dynamic_updates_sessionStorage.js]
+#[browser_storage_dynamic_updates_localStorage.js]
+#[browser_storage_dynamic_updates_sessionStorage.js]
 [browser_storage_empty_objectstores.js]
 [browser_storage_file_url.js]
 [browser_storage_indexeddb_delete.js]
 [browser_storage_indexeddb_delete_blocked.js]
 [browser_storage_indexeddb_duplicate_names.js]
 [browser_storage_indexeddb_overflow.js]
-[browser_storage_localstorage_add.js]
-[browser_storage_localstorage_edit.js]
+#[browser_storage_localstorage_add.js]
+#[browser_storage_localstorage_edit.js]
 [browser_storage_localstorage_error.js]
 [browser_storage_localstorage_rapid_add_remove.js]
 [browser_storage_overflow.js]
diff --git a/devtools/server/tests/browser/browser.ini b/devtools/server/tests/browser/browser.ini
index d9988c6bfeea..5deafdc8318e 100644
--- a/devtools/server/tests/browser/browser.ini
+++ b/devtools/server/tests/browser/browser.ini
@@ -94,10 +94,10 @@ skip-if = e10s # Bug 1183605 - devtools/server/tests/browser/ tests are still di
 [browser_spawn_actor_in_parent.js]
 [browser_storage_browser_toolbox_indexeddb.js]
 [browser_storage_cookies-duplicate-names.js]
-[browser_storage_dynamic_windows.js]
+#[browser_storage_dynamic_windows.js]
 [browser_storage_listings.js]
-[browser_storage_updates.js]
-skip-if = (verify && debug && (os == 'mac' || os == 'linux'))
+#[browser_storage_updates.js]
+#skip-if = (verify && debug && (os == 'mac' || os == 'linux'))
 [browser_stylesheets_getTextEmpty.js]
 [browser_stylesheets_nested-iframes.js]
 [browser_register_actor.js]
diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index 112502b61ffc..05e12ecb2f9a 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -23,6 +23,8 @@
 #include "mozilla/dom/DOMPrefs.h"
 #include "mozilla/dom/EventTarget.h"
 #include "mozilla/dom/LocalStorage.h"
+#include "mozilla/dom/LocalStorageCommon.h"
+#include "mozilla/dom/LSObject.h"
 #include "mozilla/dom/PartitionedLocalStorage.h"
 #include "mozilla/dom/Storage.h"
 #include "mozilla/dom/IdleRequest.h"
@@ -4908,32 +4910,38 @@ nsGlobalWindowInner::GetLocalStorage(ErrorResult& aError)
   if (access != nsContentUtils::StorageAccess::ePartitionedOrDeny &&
       (!mLocalStorage ||
         mLocalStorage->Type() == Storage::ePartitionedLocalStorage)) {
-    nsresult rv;
-    nsCOMPtr<nsIDOMStorageManager> storageManager =
-      do_GetService("@mozilla.org/dom/localStorage-manager;1", &rv);
-    if (NS_FAILED(rv)) {
-      aError.Throw(rv);
-      return nullptr;
-    }
+    RefPtr<Storage> storage;
 
-    nsString documentURI;
-    if (mDoc) {
-      aError = mDoc->GetDocumentURI(documentURI);
-      if (NS_WARN_IF(aError.Failed())) {
+    if (NextGenLocalStorageEnabled()) {
+      aError = LSObject::Create(this, getter_AddRefs(storage));
+    } else {
+      nsresult rv;
+      nsCOMPtr<nsIDOMStorageManager> storageManager =
+        do_GetService("@mozilla.org/dom/localStorage-manager;1", &rv);
+      if (NS_FAILED(rv)) {
+        aError.Throw(rv);
         return nullptr;
       }
+
+      nsString documentURI;
+      if (mDoc) {
+        aError = mDoc->GetDocumentURI(documentURI);
+        if (NS_WARN_IF(aError.Failed())) {
+          return nullptr;
+        }
+      }
+
+      nsIPrincipal *principal = GetPrincipal();
+      if (!principal) {
+        aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+        return nullptr;
+      }
+
+      aError = storageManager->CreateStorage(this, principal, documentURI,
+                                             IsPrivateBrowsing(),
+                                             getter_AddRefs(storage));
     }
 
-    nsIPrincipal *principal = GetPrincipal();
-    if (!principal) {
-      aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
-      return nullptr;
-    }
-
-    RefPtr<Storage> storage;
-    aError = storageManager->CreateStorage(this, principal, documentURI,
-                                           IsPrivateBrowsing(),
-                                           getter_AddRefs(storage));
     if (aError.Failed()) {
       return nullptr;
     }
diff --git a/dom/base/test/mochitest.ini b/dom/base/test/mochitest.ini
index 18b5f73d6ac8..9523fa410710 100644
--- a/dom/base/test/mochitest.ini
+++ b/dom/base/test/mochitest.ini
@@ -577,7 +577,7 @@ skip-if = toolkit == 'android' || (verify && !debug && (os == 'linux')) #bug 687
 [test_bug1025933.html]
 [test_bug1037687.html]
 support-files = test_bug1037687_subframe.html
-[test_bug1043106.html]
+#[test_bug1043106.html]
 [test_bug1057176.html]
 [test_bug1060938.html]
 [test_bug1064481.html]
diff --git a/dom/ipc/ContentChild.cpp b/dom/ipc/ContentChild.cpp
index fd51a6c42f7e..78321b413c2a 100644
--- a/dom/ipc/ContentChild.cpp
+++ b/dom/ipc/ContentChild.cpp
@@ -41,6 +41,7 @@
 #include "mozilla/dom/FileCreatorHelper.h"
 #include "mozilla/dom/GetFilesHelper.h"
 #include "mozilla/dom/IPCBlobUtils.h"
+#include "mozilla/dom/LSObject.h"
 #include "mozilla/dom/MemoryReportRequest.h"
 #include "mozilla/dom/PLoginReputationChild.h"
 #include "mozilla/dom/PushNotifier.h"
@@ -3811,15 +3812,21 @@ ContentChild::GetSpecificMessageEventTarget(const Message& aMsg)
   }
 }
 
-#ifdef NIGHTLY_BUILD
 void
 ContentChild::OnChannelReceivedMessage(const Message& aMsg)
 {
+  if (aMsg.is_sync()) {
+    LSObject::CancelSyncLoop();
+  }
+
+#ifdef NIGHTLY_BUILD
   if (nsContentUtils::IsMessageInputEvent(aMsg)) {
     mPendingInputEvents++;
   }
+#endif
 }
 
+#ifdef NIGHTLY_BUILD
 PContentChild::Result
 ContentChild::OnMessageReceived(const Message& aMsg)
 {
diff --git a/dom/ipc/ContentChild.h b/dom/ipc/ContentChild.h
index 9cf18b87a9aa..1b960f3841ae 100644
--- a/dom/ipc/ContentChild.h
+++ b/dom/ipc/ContentChild.h
@@ -768,10 +768,10 @@ private:
   virtual already_AddRefed<nsIEventTarget>
   GetSpecificMessageEventTarget(const Message& aMsg) override;
 
-#ifdef NIGHTLY_BUILD
   virtual void
   OnChannelReceivedMessage(const Message& aMsg) override;
 
+#ifdef NIGHTLY_BUILD
   virtual PContentChild::Result
   OnMessageReceived(const Message& aMsg) override;
 
diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
new file mode 100644
index 000000000000..9a9c9e45cc79
--- /dev/null
+++ b/dom/localstorage/ActorsChild.cpp
@@ -0,0 +1,118 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ActorsChild.h"
+
+#include "LSDatabase.h"
+
+namespace mozilla {
+namespace dom {
+
+/*******************************************************************************
+ * LSDatabaseChild
+ ******************************************************************************/
+
+LSDatabaseChild::LSDatabaseChild(LSDatabase* aDatabase)
+  : mDatabase(aDatabase)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aDatabase);
+
+  MOZ_COUNT_CTOR(LSDatabaseChild);
+}
+
+LSDatabaseChild::~LSDatabaseChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSDatabaseChild);
+}
+
+void
+LSDatabaseChild::SendDeleteMeInternal()
+{
+  AssertIsOnOwningThread();
+
+  if (mDatabase) {
+    mDatabase->ClearActor();
+    mDatabase = nullptr;
+
+    MOZ_ALWAYS_TRUE(PBackgroundLSDatabaseChild::SendDeleteMe());
+  }
+}
+
+void
+LSDatabaseChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  if (mDatabase) {
+    mDatabase->ClearActor();
+#ifdef DEBUG
+    mDatabase = nullptr;
+#endif
+  }
+}
+
+/*******************************************************************************
+ * LocalStorageRequestChild
+ ******************************************************************************/
+
+LSRequestChild::LSRequestChild(LSRequestChildCallback* aCallback)
+  : mCallback(aCallback)
+  , mFinishing(false)
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_CTOR(LSRequestChild);
+}
+
+LSRequestChild::~LSRequestChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSRequestChild);
+}
+
+bool
+LSRequestChild::Finishing() const
+{
+  AssertIsOnOwningThread();
+
+  return mFinishing;
+}
+
+void
+LSRequestChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+}
+
+mozilla::ipc::IPCResult
+LSRequestChild::Recv__delete__(const LSRequestResponse& aResponse)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mCallback);
+
+  mCallback->OnResponse(aResponse);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+LSRequestChild::RecvReady()
+{
+  AssertIsOnOwningThread();
+
+  mFinishing = true;
+
+  SendFinish();
+
+  return IPC_OK();
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
new file mode 100644
index 000000000000..271d1b921b3d
--- /dev/null
+++ b/dom/localstorage/ActorsChild.h
@@ -0,0 +1,116 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_ActorsChild_h
+#define mozilla_dom_localstorage_ActorsChild_h
+
+#include "mozilla/dom/PBackgroundLSDatabaseChild.h"
+#include "mozilla/dom/PBackgroundLSRequestChild.h"
+
+namespace mozilla {
+
+namespace ipc {
+
+class BackgroundChildImpl;
+
+} // namespace ipc
+
+namespace dom {
+
+class LSDatabase;
+class LSObject;
+class LSRequestChildCallback;
+
+class LSDatabaseChild final
+  : public PBackgroundLSDatabaseChild
+{
+  friend class mozilla::ipc::BackgroundChildImpl;
+  friend class LSDatabase;
+  friend class LSObject;
+
+  LSDatabase* mDatabase;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSDatabaseChild);
+  }
+
+private:
+  // Only created by LSObject.
+  explicit LSDatabaseChild(LSDatabase* aDatabase);
+
+  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
+  ~LSDatabaseChild();
+
+  void
+  SendDeleteMeInternal();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+};
+
+class LSRequestChild final
+  : public PBackgroundLSRequestChild
+{
+  friend class LSObject;
+  friend class LSObjectChild;
+
+  RefPtr<LSRequestChildCallback> mCallback;
+
+  bool mFinishing;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSReqeustChild);
+  }
+
+  bool
+  Finishing() const;
+
+private:
+  // Only created by LSObject.
+  explicit LSRequestChild(LSRequestChildCallback* aCallback);
+
+  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
+  ~LSRequestChild();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  Recv__delete__(const LSRequestResponse& aResponse) override;
+
+  mozilla::ipc::IPCResult
+  RecvReady() override;
+};
+
+class NS_NO_VTABLE LSRequestChildCallback
+{
+public:
+  NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
+
+  virtual void
+  OnResponse(const LSRequestResponse& aResponse) = 0;
+
+protected:
+  virtual ~LSRequestChildCallback()
+  { }
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_ActorsChild_h
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
new file mode 100644
index 000000000000..77f29ecb53e7
--- /dev/null
+++ b/dom/localstorage/ActorsParent.cpp
@@ -0,0 +1,975 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ActorsParent.h"
+
+#include "LocalStorageCommon.h"
+#include "mozilla/Unused.h"
+#include "mozilla/dom/PBackgroundLSDatabaseParent.h"
+#include "mozilla/dom/PBackgroundLSRequestParent.h"
+#include "mozilla/dom/PBackgroundLSSharedTypes.h"
+#include "mozilla/dom/quota/QuotaManager.h"
+#include "mozilla/ipc/BackgroundParent.h"
+#include "mozilla/ipc/PBackgroundParent.h"
+#include "mozilla/ipc/PBackgroundSharedTypes.h"
+#include "nsDataHashtable.h"
+#include "nsRefPtrHashtable.h"
+
+#define DISABLE_ASSERTS_FOR_FUZZING 0
+
+#if DISABLE_ASSERTS_FOR_FUZZING
+#define ASSERT_UNLESS_FUZZING(...) do { } while (0)
+#else
+#define ASSERT_UNLESS_FUZZING(...) MOZ_ASSERT(false, __VA_ARGS__)
+#endif
+
+namespace mozilla {
+namespace dom {
+
+using namespace mozilla::dom::quota;
+using namespace mozilla::ipc;
+
+namespace {
+
+/*******************************************************************************
+ * Non-actor class declarations
+ ******************************************************************************/
+
+class DatastoreOperationBase
+  : public Runnable
+{
+  nsCOMPtr<nsIEventTarget> mOwningEventTarget;
+  nsresult mResultCode;
+  Atomic<bool> mMayProceedOnNonOwningThread;
+  bool mMayProceed;
+
+public:
+  nsIEventTarget*
+  OwningEventTarget() const
+  {
+    MOZ_ASSERT(mOwningEventTarget);
+
+    return mOwningEventTarget;
+  }
+
+  bool
+  IsOnOwningThread() const
+  {
+    MOZ_ASSERT(mOwningEventTarget);
+
+    bool current;
+    return
+      NS_SUCCEEDED(mOwningEventTarget->IsOnCurrentThread(&current)) && current;
+  }
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    MOZ_ASSERT(IsOnBackgroundThread());
+    MOZ_ASSERT(IsOnOwningThread());
+  }
+
+  nsresult
+  ResultCode() const
+  {
+    return mResultCode;
+  }
+
+  void
+  MaybeSetFailureCode(nsresult aErrorCode)
+  {
+    MOZ_ASSERT(NS_FAILED(aErrorCode));
+
+    if (NS_SUCCEEDED(mResultCode)) {
+      mResultCode = aErrorCode;
+    }
+  }
+
+  void
+  NoteComplete()
+  {
+    AssertIsOnOwningThread();
+
+    mMayProceed = false;
+    mMayProceedOnNonOwningThread = false;
+  }
+
+  bool
+  MayProceed() const
+  {
+    AssertIsOnOwningThread();
+
+    return mMayProceed;
+  }
+
+  // May be called on any thread, but you should call MayProceed() if you know
+  // you're on the background thread because it is slightly faster.
+  bool
+  MayProceedOnNonOwningThread() const
+  {
+    return mMayProceedOnNonOwningThread;
+  }
+
+protected:
+  DatastoreOperationBase()
+    : Runnable("dom::DatastoreOperationBase")
+    , mOwningEventTarget(GetCurrentThreadEventTarget())
+    , mResultCode(NS_OK)
+    , mMayProceedOnNonOwningThread(true)
+    , mMayProceed(true)
+  { }
+
+  ~DatastoreOperationBase() override
+  {
+    MOZ_ASSERT(!mMayProceed);
+  }
+};
+
+class Datastore final
+{
+  nsDataHashtable<nsStringHashKey, nsString> mValues;
+  const nsCString mOrigin;
+
+public:
+  // Created by PrepareDatastoreOp.
+  explicit Datastore(const nsACString& aOrigin);
+
+  const nsCString&
+  Origin() const
+  {
+    return mOrigin;
+  }
+
+  uint32_t
+  GetLength() const;
+
+  void
+  GetKey(uint32_t aIndex, nsString& aKey) const;
+
+  void
+  GetItem(const nsString& aKey, nsString& aValue) const;
+
+  void
+  SetItem(const nsString& aKey, const nsString& aValue);
+
+  void
+  RemoveItem(const nsString& aKey);
+
+  void
+  Clear();
+
+  void
+  GetKeys(nsTArray<nsString>& aKeys) const;
+
+  NS_INLINE_DECL_REFCOUNTING(Datastore)
+
+private:
+  // Reference counted.
+  ~Datastore();
+};
+
+/*******************************************************************************
+ * Actor class declarations
+ ******************************************************************************/
+
+class Database final
+  : public PBackgroundLSDatabaseParent
+{
+  RefPtr<Datastore> mDatastore;
+
+#ifdef DEBUG
+  bool mActorDestroyed;
+  bool mActorWasAlive;
+#endif
+
+public:
+  // Created in AllocPBackgroundLSDatabaseParent.
+  Database();
+
+  void
+  SetActorAlive(already_AddRefed<Datastore>&& aDatastore);
+
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Database)
+
+private:
+  // Reference counted.
+  ~Database();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvDeleteMe() override;
+
+  mozilla::ipc::IPCResult
+  RecvGetLength(uint32_t* aLength) override;
+
+  mozilla::ipc::IPCResult
+  RecvGetKey(const uint32_t& aIndex, nsString* aKey) override;
+
+  mozilla::ipc::IPCResult
+  RecvGetItem(const nsString& aKey, nsString* aValue) override;
+
+  mozilla::ipc::IPCResult
+  RecvGetKeys(nsTArray<nsString>* aKeys) override;
+
+  mozilla::ipc::IPCResult
+  RecvSetItem(const nsString& aKey, const nsString& aValue) override;
+
+  mozilla::ipc::IPCResult
+  RecvRemoveItem(const nsString& aKey) override;
+
+  mozilla::ipc::IPCResult
+  RecvClear() override;
+};
+
+class LSRequestBase
+  : public DatastoreOperationBase
+  , public PBackgroundLSRequestParent
+{
+public:
+  virtual void
+  Dispatch() = 0;
+
+private:
+  // IPDL methods.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvCancel() override;
+};
+
+class PrepareDatastoreOp
+  : public LSRequestBase
+{
+  enum class State
+  {
+    // Just created on the PBackground thread. Next step is OpeningOnMainThread
+    // or OpeningOnOwningThread.
+    Initial,
+
+    // Waiting to open/opening on the main thread. Next step is
+    // SendingReadyMessage.
+    OpeningOnMainThread,
+
+    // Waiting to open/opening on the owning thread. Next step is
+    // SendingReadyMessage.
+    OpeningOnOwningThread,
+
+    // Waiting to send/sending the ready message on the PBackground thread. Next
+    // step is WaitingForFinish.
+    SendingReadyMessage,
+
+    // Waiting for the finish message on the PBackground thread. Next step is
+    // SendingResults.
+    WaitingForFinish,
+
+    // Waiting to send/sending results on the PBackground thread. Next step is
+    // Completed.
+    SendingResults,
+
+    // All done.
+    Completed
+  };
+
+  const LSRequestPrepareDatastoreParams mParams;
+  nsCString mSuffix;
+  nsCString mGroup;
+  nsCString mOrigin;
+  State mState;
+
+public:
+  explicit PrepareDatastoreOp(const LSRequestParams& aParams);
+
+  void
+  Dispatch() override;
+
+private:
+  ~PrepareDatastoreOp() override;
+
+  nsresult
+  OpenOnMainThread();
+
+  nsresult
+  OpenOnOwningThread();
+
+  void
+  SendReadyMessage();
+
+  void
+  SendResults();
+
+  NS_IMETHOD
+  Run() override;
+
+  // IPDL overrides.
+  mozilla::ipc::IPCResult
+  RecvFinish() override;
+};
+
+/*******************************************************************************
+ * Globals
+ ******************************************************************************/
+
+typedef nsDataHashtable<nsCStringHashKey, Datastore*> DatastoreHashtable;
+
+StaticAutoPtr<DatastoreHashtable> gDatastores;
+
+uint64_t gLastDatastoreId = 0;
+
+typedef nsRefPtrHashtable<nsUint64HashKey, Datastore>
+  TemporaryStrongDatastoreHashtable;
+
+StaticAutoPtr<TemporaryStrongDatastoreHashtable> gTemporaryStrongDatastores;
+
+typedef nsTArray<Database*> LiveDatabaseArray;
+
+StaticAutoPtr<LiveDatabaseArray> gLiveDatabases;
+
+} // namespace
+
+/*******************************************************************************
+ * Exported functions
+ ******************************************************************************/
+
+PBackgroundLSDatabaseParent*
+AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(!gTemporaryStrongDatastores)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  Datastore* datastore = gTemporaryStrongDatastores->GetWeak(aDatastoreId);
+  if (NS_WARN_IF(!datastore)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  // If we ever decide to return null from this point on, we need to make sure
+  // that the prepared datastore is removed from the gTemporaryStrongDatastores
+  // hashtable.
+  // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
+  // once we return a valid actor in this method.
+
+  RefPtr<Database> database = new Database();
+
+  // Transfer ownership to IPDL.
+  return database.forget().take();
+}
+
+bool
+RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                     const uint64_t& aDatastoreId)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(gTemporaryStrongDatastores);
+  MOZ_ASSERT(gTemporaryStrongDatastores->GetWeak(aDatastoreId));
+
+  // The actor is now completely built (it has a manager, channel and it's
+  // registered as a subprotocol).
+  // ActorDestroy will be called if we fail here.
+
+  RefPtr<Datastore> datastore;
+  gTemporaryStrongDatastores->Remove(aDatastoreId, datastore.StartAssignment());
+  MOZ_ASSERT(datastore);
+
+  auto* database = static_cast<Database*>(aActor);
+
+  database->SetActorAlive(datastore.forget());
+
+  return true;
+}
+
+bool
+DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  // Transfer ownership back from IPDL.
+  RefPtr<Database> actor = dont_AddRef(static_cast<Database*>(aActor));
+
+  return true;
+}
+
+PBackgroundLSRequestParent*
+AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
+                                const LSRequestParams& aParams)
+{
+  AssertIsOnBackgroundThread();
+
+  RefPtr<LSRequestBase> actor;
+
+  switch (aParams.type()) {
+    case LSRequestParams::TLSRequestPrepareDatastoreParams: {
+      bool isOtherProcess =
+        BackgroundParent::IsOtherProcessActor(aBackgroundActor);
+
+      const LSRequestPrepareDatastoreParams& params =
+        aParams.get_LSRequestPrepareDatastoreParams();
+
+      const PrincipalOrQuotaInfo& info = params.info();
+
+      PrincipalOrQuotaInfo::Type infoType = info.type();
+
+      bool paramsOk =
+        (isOtherProcess && infoType == PrincipalOrQuotaInfo::TPrincipalInfo) ||
+        (!isOtherProcess && infoType == PrincipalOrQuotaInfo::TQuotaInfo);
+
+      if (NS_WARN_IF(!paramsOk)) {
+        ASSERT_UNLESS_FUZZING();
+        return nullptr;
+      }
+
+      actor = new PrepareDatastoreOp(aParams);
+      break;
+    }
+
+    default:
+      MOZ_CRASH("Should never get here!");
+  }
+
+  // Transfer ownership to IPDL.
+  return actor.forget().take();
+}
+
+bool
+RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
+                                    const LSRequestParams& aParams)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(aParams.type() != LSRequestParams::T__None);
+
+  // The actor is now completely built.
+
+  auto* op = static_cast<LSRequestBase*>(aActor);
+
+  op->Dispatch();
+
+  return true;
+}
+
+bool
+DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+
+  // Transfer ownership back from IPDL.
+  RefPtr<LSRequestBase> actor =
+    dont_AddRef(static_cast<LSRequestBase*>(aActor));
+
+  return true;
+}
+
+/*******************************************************************************
+ * DatastoreOperationBase
+ ******************************************************************************/
+
+/*******************************************************************************
+ * Datastore
+ ******************************************************************************/
+
+Datastore::Datastore(const nsACString& aOrigin)
+  : mOrigin(aOrigin)
+{
+  AssertIsOnBackgroundThread();
+}
+
+Datastore::~Datastore()
+{
+  AssertIsOnBackgroundThread();
+
+  MOZ_ASSERT(gDatastores);
+  MOZ_ASSERT(gDatastores->Get(mOrigin));
+  gDatastores->Remove(mOrigin);
+
+  if (!gDatastores->Count()) {
+    gDatastores = nullptr;
+  }
+}
+
+uint32_t
+Datastore::GetLength() const
+{
+  AssertIsOnBackgroundThread();
+
+  return mValues.Count();
+}
+
+void
+Datastore::GetKey(uint32_t aIndex, nsString& aKey) const
+{
+  AssertIsOnBackgroundThread();
+
+  aKey.SetIsVoid(true);
+  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+    if (aIndex == 0) {
+      aKey = iter.Key();
+      return;
+    }
+    aIndex--;
+  }
+}
+
+void
+Datastore::GetItem(const nsString& aKey, nsString& aValue) const
+{
+  AssertIsOnBackgroundThread();
+
+  if (!mValues.Get(aKey, &aValue)) {
+    aValue.SetIsVoid(true);
+  }
+}
+
+void
+Datastore::SetItem(const nsString& aKey, const nsString& aValue)
+{
+  AssertIsOnBackgroundThread();
+
+  mValues.Put(aKey, aValue);
+}
+
+void
+Datastore::RemoveItem(const nsString& aKey)
+{
+  AssertIsOnBackgroundThread();
+
+  mValues.Remove(aKey);
+}
+
+void
+Datastore::Clear()
+{
+  AssertIsOnBackgroundThread();
+
+  mValues.Clear();
+}
+
+void
+Datastore::GetKeys(nsTArray<nsString>& aKeys) const
+{
+  AssertIsOnBackgroundThread();
+
+  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+    aKeys.AppendElement(iter.Key());
+  }
+}
+
+/*******************************************************************************
+ * Database
+ ******************************************************************************/
+
+Database::Database()
+#ifdef DEBUG
+  : mActorDestroyed(false)
+  , mActorWasAlive(false)
+#endif
+{
+  AssertIsOnBackgroundThread();
+}
+
+Database::~Database()
+{
+  MOZ_ASSERT_IF(mActorWasAlive, mActorDestroyed);
+}
+
+void
+Database::SetActorAlive(already_AddRefed<Datastore>&& aDatastore)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorWasAlive);
+  MOZ_ASSERT(!mActorDestroyed);
+
+#ifdef DEBUG
+  mActorWasAlive = true;
+#endif
+
+  mDatastore = std::move(aDatastore);
+
+  if (!gLiveDatabases) {
+    gLiveDatabases = new LiveDatabaseArray();
+  }
+
+  gLiveDatabases->AppendElement(this);
+}
+
+void
+Database::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+  MOZ_ASSERT(mDatastore);
+
+#ifdef DEBUG
+  mActorDestroyed = true;
+#endif
+
+  mDatastore = nullptr;
+
+  MOZ_ASSERT(gLiveDatabases);
+  gLiveDatabases->RemoveElement(this);
+
+  if (gLiveDatabases->IsEmpty()) {
+    gLiveDatabases = nullptr;
+  }
+}
+
+mozilla::ipc::IPCResult
+Database::RecvDeleteMe()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  IProtocol* mgr = Manager();
+  if (!PBackgroundLSDatabaseParent::Send__delete__(this)) {
+    return IPC_FAIL_NO_REASON(mgr);
+  }
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvGetLength(uint32_t* aLength)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aLength);
+  MOZ_ASSERT(mDatastore);
+
+  *aLength = mDatastore->GetLength();
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvGetKey(const uint32_t& aIndex, nsString* aKey)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aKey);
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->GetKey(aIndex, *aKey);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvGetItem(const nsString& aKey, nsString* aValue)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aValue);
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->GetItem(aKey, *aValue);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvSetItem(const nsString& aKey, const nsString& aValue)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->SetItem(aKey, aValue);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvRemoveItem(const nsString& aKey)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->RemoveItem(aKey);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvClear()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->Clear();
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvGetKeys(nsTArray<nsString>* aKeys)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aKeys);
+  MOZ_ASSERT(mDatastore);
+
+  mDatastore->GetKeys(*aKeys);
+
+  return IPC_OK();
+}
+
+/*******************************************************************************
+ * LSRequestBase
+ ******************************************************************************/
+
+void
+LSRequestBase::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  NoteComplete();
+}
+
+mozilla::ipc::IPCResult
+LSRequestBase::RecvCancel()
+{
+  AssertIsOnOwningThread();
+
+  IProtocol* mgr = Manager();
+  if (!PBackgroundLSRequestParent::Send__delete__(this, NS_ERROR_FAILURE)) {
+    return IPC_FAIL_NO_REASON(mgr);
+  }
+
+  return IPC_OK();
+}
+
+/*******************************************************************************
+ * PrepareDatastoreOp
+ ******************************************************************************/
+
+PrepareDatastoreOp::PrepareDatastoreOp(const LSRequestParams& aParams)
+  : mParams(aParams.get_LSRequestPrepareDatastoreParams())
+  , mState(State::Initial)
+{
+  MOZ_ASSERT(aParams.type() ==
+               LSRequestParams::TLSRequestPrepareDatastoreParams);
+}
+
+PrepareDatastoreOp::~PrepareDatastoreOp()
+{
+  MOZ_ASSERT_IF(MayProceedOnNonOwningThread(),
+                mState == State::Initial || mState == State::Completed);
+}
+
+void
+PrepareDatastoreOp::Dispatch()
+{
+  AssertIsOnOwningThread();
+
+  const PrincipalOrQuotaInfo& info = mParams.info();
+
+  if (info.type() == PrincipalOrQuotaInfo::TPrincipalInfo) {
+    mState = State::OpeningOnMainThread;
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+  } else {
+    MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TQuotaInfo);
+
+    mState = State::OpeningOnOwningThread;
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
+  }
+}
+
+nsresult
+PrepareDatastoreOp::OpenOnMainThread()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mState == State::OpeningOnMainThread);
+
+  if (!MayProceedOnNonOwningThread()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  const PrincipalOrQuotaInfo& info = mParams.info();
+
+  MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TPrincipalInfo);
+
+  const PrincipalInfo& principalInfo = info.get_PrincipalInfo();
+
+  if (principalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) {
+    QuotaManager::GetInfoForChrome(&mSuffix, &mGroup, &mOrigin);
+  } else {
+    MOZ_ASSERT(principalInfo.type() == PrincipalInfo::TContentPrincipalInfo);
+
+    nsresult rv;
+    nsCOMPtr<nsIPrincipal> principal =
+      PrincipalInfoToPrincipal(principalInfo, &rv);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = QuotaManager::GetInfoFromPrincipal(principal,
+                                            &mSuffix,
+                                            &mGroup,
+                                            &mOrigin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  mState = State::SendingReadyMessage;
+  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::OpenOnOwningThread()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::OpeningOnOwningThread);
+
+  if (!MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  const PrincipalOrQuotaInfo& info = mParams.info();
+
+  MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TQuotaInfo);
+
+  const QuotaInfo& quotaInfo = info.get_QuotaInfo();
+
+  mSuffix = quotaInfo.suffix();
+  mGroup = quotaInfo.group();
+  mOrigin = quotaInfo.origin();
+
+  mState = State::SendingReadyMessage;
+  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+
+  return NS_OK;
+}
+
+void
+PrepareDatastoreOp::SendReadyMessage()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingReadyMessage);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+
+    mState = State::Completed;
+  } else {
+    Unused << SendReady();
+
+    mState = State::WaitingForFinish;
+  }
+}
+
+void
+PrepareDatastoreOp::SendResults()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingResults);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+  } else {
+    LSRequestResponse response;
+
+    if (NS_SUCCEEDED(ResultCode())) {
+      RefPtr<Datastore> datastore;
+
+      if (gDatastores) {
+        datastore = gDatastores->Get(mOrigin);
+      }
+
+      if (!datastore) {
+        datastore = new Datastore(mOrigin);
+
+        if (!gDatastores) {
+          gDatastores = new DatastoreHashtable();
+        }
+
+        gDatastores->Put(mOrigin, datastore);
+      }
+
+      uint64_t datastoreId = ++gLastDatastoreId;
+
+      if (!gTemporaryStrongDatastores) {
+        gTemporaryStrongDatastores = new TemporaryStrongDatastoreHashtable();
+      }
+      gTemporaryStrongDatastores->Put(datastoreId, datastore);
+
+      LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
+      prepareDatastoreResponse.datastoreId() = datastoreId;
+
+      response = prepareDatastoreResponse;
+    } else {
+      response = ResultCode();
+    }
+
+    Unused <<
+      PBackgroundLSRequestParent::Send__delete__(this, response);
+  }
+
+  mState = State::Completed;
+}
+
+NS_IMETHODIMP
+PrepareDatastoreOp::Run()
+{
+  nsresult rv;
+
+  switch (mState) {
+    case State::OpeningOnMainThread:
+      rv = OpenOnMainThread();
+      break;
+
+    case State::OpeningOnOwningThread:
+      rv = OpenOnOwningThread();
+      break;
+
+    case State::SendingReadyMessage:
+      SendReadyMessage();
+      return NS_OK;
+
+    case State::SendingResults:
+      SendResults();
+      return NS_OK;
+
+    default:
+      MOZ_CRASH("Bad state!");
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv)) && mState != State::SendingReadyMessage) {
+    MaybeSetFailureCode(rv);
+
+    // Must set mState before dispatching otherwise we will race with the owning
+    // thread.
+    mState = State::SendingReadyMessage;
+
+    if (IsOnOwningThread()) {
+      SendReadyMessage();
+    } else {
+      MOZ_ALWAYS_SUCCEEDS(
+        OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+    }
+  }
+
+  return NS_OK;
+}
+
+mozilla::ipc::IPCResult
+PrepareDatastoreOp::RecvFinish()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::WaitingForFinish);
+
+  mState = State::SendingResults;
+  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
+
+  return IPC_OK();
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/ActorsParent.h b/dom/localstorage/ActorsParent.h
new file mode 100644
index 000000000000..1fdcf4eb0e8d
--- /dev/null
+++ b/dom/localstorage/ActorsParent.h
@@ -0,0 +1,49 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_ActorsParent_h
+#define mozilla_dom_localstorage_ActorsParent_h
+
+namespace mozilla {
+
+namespace ipc {
+
+class PBackgroundParent;
+
+} // namespace ipc
+
+namespace dom {
+
+class LSRequestParams;
+class PBackgroundLSDatabaseParent;
+class PBackgroundLSRequestParent;
+
+PBackgroundLSDatabaseParent*
+AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId);
+
+bool
+RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                     const uint64_t& aDatastoreId);
+
+bool
+DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor);
+
+PBackgroundLSRequestParent*
+AllocPBackgroundLSRequestParent(
+                              mozilla::ipc::PBackgroundParent* aBackgroundActor,
+                              const LSRequestParams& aParams);
+
+bool
+RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
+                                    const LSRequestParams& aParams);
+
+bool
+DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor);
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_ActorsParent_h
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
new file mode 100644
index 000000000000..0a9e1a5a5ddf
--- /dev/null
+++ b/dom/localstorage/LSDatabase.cpp
@@ -0,0 +1,141 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LSDatabase.h"
+
+namespace mozilla {
+namespace dom {
+
+LSDatabase::LSDatabase()
+  : mActor(nullptr)
+{
+  AssertIsOnOwningThread();
+}
+
+LSDatabase::~LSDatabase()
+{
+  AssertIsOnOwningThread();
+
+  if (mActor) {
+    mActor->SendDeleteMeInternal();
+    MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
+  }
+}
+
+void
+LSDatabase::SetActor(LSDatabaseChild* aActor)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(!mActor);
+
+  mActor = aActor;
+}
+
+nsresult
+LSDatabase::GetLength(uint32_t* aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  uint32_t result;
+  if (NS_WARN_IF(!mActor->SendGetLength(&result))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  *aResult = result;
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetKey(uint32_t aIndex,
+                   nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  nsString result;
+  if (NS_WARN_IF(!mActor->SendGetKey(aIndex, &result))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  aResult = result;
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetItem(const nsAString& aKey,
+                    nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  nsString result;
+  if (NS_WARN_IF(!mActor->SendGetItem(nsString(aKey), &result))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  aResult = result;
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  nsTArray<nsString> result;
+  if (NS_WARN_IF(!mActor->SendGetKeys(&result))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  aKeys.SwapElements(result);
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::SetItem(const nsAString& aKey,
+                    const nsAString& aValue)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey), nsString(aValue)))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::RemoveItem(const nsAString& aKey)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey)))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::Clear()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+
+  if (NS_WARN_IF(!mActor->SendClear())) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
new file mode 100644
index 000000000000..7b1402336eed
--- /dev/null
+++ b/dom/localstorage/LSDatabase.h
@@ -0,0 +1,73 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LSDatabase_h
+#define mozilla_dom_localstorage_LSDatabase_h
+
+namespace mozilla {
+namespace dom {
+
+class LSDatabaseChild;
+
+class LSDatabase final
+{
+  LSDatabaseChild* mActor;
+
+public:
+  LSDatabase();
+
+  NS_INLINE_DECL_REFCOUNTING(LSDatabase)
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSDatabase);
+  }
+
+  void
+  SetActor(LSDatabaseChild* aActor);
+
+  void
+  ClearActor()
+  {
+    AssertIsOnOwningThread();
+    MOZ_ASSERT(mActor);
+
+    mActor = nullptr;
+  }
+
+  nsresult
+  GetLength(uint32_t* aResult);
+
+  nsresult
+  GetKey(uint32_t aIndex,
+         nsAString& aResult);
+
+  nsresult
+  GetItem(const nsAString& aKey,
+          nsAString& aResult);
+
+  nsresult
+  GetKeys(nsTArray<nsString>& aKeys);
+
+  nsresult
+  SetItem(const nsAString& aKey,
+          const nsAString& aValue);
+
+  nsresult
+  RemoveItem(const nsAString& aKey);
+
+  nsresult
+  Clear();
+
+private:
+  ~LSDatabase();
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LSDatabase_h
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
new file mode 100644
index 000000000000..8f3f25bff4c9
--- /dev/null
+++ b/dom/localstorage/LSObject.cpp
@@ -0,0 +1,629 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LSObject.h"
+
+#include "ActorsChild.h"
+#include "IPCBlobInputStreamThread.h"
+#include "LocalStorageCommon.h"
+#include "mozilla/ThreadEventQueue.h"
+#include "mozilla/dom/quota/QuotaManager.h"
+#include "mozilla/ipc/BackgroundChild.h"
+#include "mozilla/ipc/BackgroundUtils.h"
+#include "mozilla/ipc/PBackgroundChild.h"
+#include "nsContentUtils.h"
+#include "nsIScriptObjectPrincipal.h"
+#include "nsThread.h"
+
+namespace mozilla {
+namespace dom {
+
+namespace {
+
+class RequestHelper;
+
+StaticMutex gRequestHelperMutex;
+RequestHelper* gRequestHelper = nullptr;
+
+class RequestHelper final
+  : public Runnable
+  , public LSRequestChildCallback
+{
+  enum class State
+  {
+    Initial,
+    ResponsePending,
+    Finishing,
+    Complete
+  };
+
+  RefPtr<LSObject> mObject;
+  nsCOMPtr<nsIEventTarget> mOwningEventTarget;
+  nsCOMPtr<nsIEventTarget> mNestedEventTarget;
+  LSRequestChild* mActor;
+  const LSRequestParams mParams;
+  LSRequestResponse mResponse;
+  nsresult mResultCode;
+  State mState;
+  bool mWaiting;
+
+public:
+  RequestHelper(LSObject* aObject,
+                const LSRequestParams& aParams)
+    : Runnable("dom::RequestHelper")
+    , mObject(aObject)
+    , mOwningEventTarget(GetCurrentThreadEventTarget())
+    , mActor(nullptr)
+    , mParams(aParams)
+    , mResultCode(NS_OK)
+    , mState(State::Initial)
+    , mWaiting(true)
+  {
+    StaticMutexAutoLock lock(gRequestHelperMutex);
+    gRequestHelper = this;
+  }
+
+  bool
+  IsOnOwningThread() const
+  {
+    MOZ_ASSERT(mOwningEventTarget);
+
+    bool current;
+    return
+      NS_SUCCEEDED(mOwningEventTarget->IsOnCurrentThread(&current)) && current;
+  }
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+    MOZ_ASSERT(IsOnOwningThread());
+  }
+
+  nsresult
+  StartAndReturnResponse(LSRequestResponse& aResponse);
+
+  nsresult
+  CancelOnAnyThread();
+
+private:
+  ~RequestHelper()
+  {
+    StaticMutexAutoLock lock(gRequestHelperMutex);
+    gRequestHelper = nullptr;
+  }
+
+  nsresult
+  Start();
+
+  void
+  Finish();
+
+  NS_DECL_ISUPPORTS_INHERITED
+
+  NS_DECL_NSIRUNNABLE
+
+  // LSRequestChildCallback
+  void
+  OnResponse(const LSRequestResponse& aResponse) override;
+};
+
+} // namespace
+
+LSObject::LSObject(nsPIDOMWindowInner* aWindow,
+                   nsIPrincipal* aPrincipal)
+  : Storage(aWindow, aPrincipal)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(NextGenLocalStorageEnabled());
+}
+
+LSObject::~LSObject()
+{
+  AssertIsOnOwningThread();
+}
+
+// static
+nsresult
+LSObject::Create(nsPIDOMWindowInner* aWindow,
+                 Storage** aStorage)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aWindow);
+  MOZ_ASSERT(aStorage);
+  MOZ_ASSERT(NextGenLocalStorageEnabled());
+  MOZ_ASSERT(nsContentUtils::StorageAllowedForWindow(aWindow) >
+               nsContentUtils::StorageAccess::eDeny);
+
+  nsCOMPtr<nsIScriptObjectPrincipal> sop = do_QueryInterface(aWindow);
+  MOZ_ASSERT(sop);
+
+  nsCOMPtr<nsIPrincipal> principal = sop->GetPrincipal();
+  if (NS_WARN_IF(!principal)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsresult rv;
+
+  nsAutoPtr<PrincipalOrQuotaInfo> info(new PrincipalOrQuotaInfo());
+  if (XRE_IsParentProcess()) {
+    nsCString suffix;
+    nsCString group;
+    nsCString origin;
+    rv = QuotaManager::GetInfoFromPrincipal(principal,
+                                            &suffix,
+                                            &group,
+                                            &origin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    QuotaInfo quotaInfo;
+    quotaInfo.suffix() = suffix;
+    quotaInfo.group() = group;
+    quotaInfo.origin() = origin;
+
+    *info = quotaInfo;
+  } else {
+    PrincipalInfo principalInfo;
+    rv = PrincipalToPrincipalInfo(principal, &principalInfo);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(principalInfo.type() == PrincipalInfo::TContentPrincipalInfo);
+
+    *info = principalInfo;
+  }
+
+  RefPtr<LSObject> object = new LSObject(aWindow, principal);
+  object->mInfo = std::move(info);
+
+  object.forget(aStorage);
+  return NS_OK;
+}
+
+// static
+void
+LSObject::CancelSyncLoop()
+{
+  RefPtr<RequestHelper> helper;
+
+  {
+    StaticMutexAutoLock lock(gRequestHelperMutex);
+    helper = gRequestHelper;
+  }
+
+  if (helper) {
+    Unused << NS_WARN_IF(NS_FAILED(helper->CancelOnAnyThread()));
+  }
+}
+
+LSRequestChild*
+LSObject::StartRequest(nsIEventTarget* aMainEventTarget,
+                       const LSRequestParams& aParams,
+                       LSRequestChildCallback* aCallback)
+{
+  AssertIsOnDOMFileThread();
+
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread(aMainEventTarget);
+  if (NS_WARN_IF(!backgroundActor)) {
+    return nullptr;
+  }
+
+  LSRequestChild* actor = new LSRequestChild(aCallback);
+
+  backgroundActor->SendPBackgroundLSRequestConstructor(actor, aParams);
+
+  return actor;
+}
+
+Storage::StorageType
+LSObject::Type() const
+{
+  AssertIsOnOwningThread();
+
+  return eLocalStorage;
+}
+
+bool
+LSObject::IsForkOf(const Storage* aStorage) const
+{
+  AssertIsOnOwningThread();
+
+  return false;
+}
+
+int64_t
+LSObject::GetOriginQuotaUsage() const
+{
+  AssertIsOnOwningThread();
+
+  return 0;
+}
+
+uint32_t
+LSObject::GetLength(nsIPrincipal& aSubjectPrincipal,
+                    ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return 0;
+  }
+
+  uint32_t result;
+  rv = mDatabase->GetLength(&result);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return 0;
+  }
+
+  return result;
+}
+
+void
+LSObject::Key(uint32_t aIndex,
+              nsAString& aResult,
+              nsIPrincipal& aSubjectPrincipal,
+              ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  nsString result;
+  rv = mDatabase->GetKey(aIndex, result);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  aResult = result;
+}
+
+void
+LSObject::GetItem(const nsAString& aKey,
+                  nsAString& aResult,
+                  nsIPrincipal& aSubjectPrincipal,
+                  ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  nsString result;
+  rv = mDatabase->GetItem(aKey, result);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  aResult = result;
+}
+
+void
+LSObject::GetSupportedNames(nsTArray<nsString>& aNames)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
+  rv = mDatabase->GetKeys(aNames);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+}
+
+void
+LSObject::SetItem(const nsAString& aKey,
+                  const nsAString& aValue,
+                  nsIPrincipal& aSubjectPrincipal,
+                  ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  rv = mDatabase->SetItem(aKey, aValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+}
+
+void
+LSObject::RemoveItem(const nsAString& aKey,
+                     nsIPrincipal& aSubjectPrincipal,
+                     ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  rv = mDatabase->RemoveItem(aKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+}
+
+void
+LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
+                ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  rv = mDatabase->Clear();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+}
+
+nsresult
+LSObject::EnsureDatabase()
+{
+  AssertIsOnOwningThread();
+
+  if (mDatabase) {
+    return NS_OK;
+  }
+
+  // We don't need this yet, but once the request successfully finishes, it's
+  // too late to initialize PBackground child on the owning thread, because
+  // it can fail and parent would keep an extra strong ref to the datastore.
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread();
+  if (NS_WARN_IF(!backgroundActor)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  LSRequestPrepareDatastoreParams params;
+  params.info() = *mInfo;
+
+  RefPtr<RequestHelper> helper = new RequestHelper(this, params);
+
+  LSRequestResponse response;
+
+  // This will start and finish the request on the DOM File thread.
+  // The owning thread is synchronously blocked while the request is
+  // asynchronously processed on the DOM File thread.
+  nsresult rv = helper->StartAndReturnResponse(response);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (response.type() == LSRequestResponse::Tnsresult) {
+    return response.get_nsresult();
+  }
+
+  MOZ_ASSERT(response.type() ==
+               LSRequestResponse::TLSRequestPrepareDatastoreResponse);
+
+  const LSRequestPrepareDatastoreResponse& prepareDatastoreResponse =
+    response.get_LSRequestPrepareDatastoreResponse();
+
+  uint64_t datastoreId = prepareDatastoreResponse.datastoreId();
+
+  // The datastore is now ready on the parent side (prepared by the asynchronous
+  // request on the DOM File thread).
+  // Let's create a direct connection to the datastore (through a database
+  // actor) from the owning thread.
+  // Note that we now can't error out, otherwise parent will keep an extra
+  // strong reference to the datastore.
+  RefPtr<LSDatabase> database = new LSDatabase();
+
+  LSDatabaseChild* actor = new LSDatabaseChild(database);
+
+  MOZ_ALWAYS_TRUE(
+    backgroundActor->SendPBackgroundLSDatabaseConstructor(actor, datastoreId));
+
+  database->SetActor(actor);
+
+  mDatabase = std::move(database);
+
+  return NS_OK;
+}
+
+nsresult
+RequestHelper::StartAndReturnResponse(LSRequestResponse& aResponse)
+{
+  AssertIsOnOwningThread();
+
+  // Normally, we would use the standard way of blocking the thread using
+  // a monitor.
+  // The problem is that BackgroundChild::GetOrCreateForCurrentThread()
+  // called on the DOM File thread may dispatch a runnable to the main
+  // thread to finish initialization of PBackground. A monitor would block
+  // the main thread and the runnable would never get executed causing the
+  // helper to be stuck in a wait loop.
+  // However, BackgroundChild::GetOrCreateForCurrentThread() supports passing
+  // a custom main event target, so we can create a nested event target and
+  // spin the event loop. Nothing can dispatch to the nested event target
+  // except BackgroundChild::GetOrCreateForCurrentThread(), so spinning of the
+  // event loop can't fire any other events.
+  // This way the thread is synchronously blocked in a safe manner and the
+  // runnable gets executed.
+  {
+    auto thread = static_cast<nsThread*>(NS_GetCurrentThread());
+
+    auto queue =
+      static_cast<ThreadEventQueue<EventQueue>*>(thread->EventQueue());
+
+    mNestedEventTarget = queue->PushEventQueue();
+    MOZ_ASSERT(mNestedEventTarget);
+
+    auto autoPopEventQueue = mozilla::MakeScopeExit([&] {
+      queue->PopEventQueue(mNestedEventTarget);
+    });
+
+    nsCOMPtr<nsIEventTarget> domFileThread =
+      IPCBlobInputStreamThread::GetOrCreate();
+    if (NS_WARN_IF(!domFileThread)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    nsresult rv = domFileThread->Dispatch(this, NS_DISPATCH_NORMAL);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ALWAYS_TRUE(SpinEventLoopUntil([&]() {
+      return !mWaiting;
+    }));
+  }
+
+  if (NS_WARN_IF(NS_FAILED(mResultCode))) {
+    return mResultCode;
+  }
+
+  aResponse = std::move(mResponse);
+  return NS_OK;
+}
+
+nsresult
+RequestHelper::CancelOnAnyThread()
+{
+  RefPtr<RequestHelper> self = this;
+
+  RefPtr<Runnable> runnable = NS_NewRunnableFunction(
+    "RequestHelper::CancelOnAnyThread",
+    [self] () {
+      LSRequestChild* actor = self->mActor;
+      if (actor && !actor->Finishing()) {
+        actor->SendCancel();
+      }
+    });
+
+  nsCOMPtr<nsIEventTarget> domFileThread =
+    IPCBlobInputStreamThread::GetOrCreate();
+  if (NS_WARN_IF(!domFileThread)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsresult rv = domFileThread->Dispatch(runnable, NS_DISPATCH_NORMAL);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+RequestHelper::Start()
+{
+  AssertIsOnDOMFileThread();
+  MOZ_ASSERT(mState == State::Initial);
+
+  mState = State::ResponsePending;
+
+  LSRequestChild* actor =
+    mObject->StartRequest(mNestedEventTarget, mParams, this);
+  if (NS_WARN_IF(!actor)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  mActor = actor;
+
+  return NS_OK;
+}
+
+void
+RequestHelper::Finish()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::Finishing);
+
+  mObject = nullptr;
+
+  mWaiting = false;
+
+  mState = State::Complete;
+}
+
+NS_IMPL_ISUPPORTS_INHERITED0(RequestHelper, Runnable)
+
+NS_IMETHODIMP
+RequestHelper::Run()
+{
+  nsresult rv;
+
+  switch (mState) {
+    case State::Initial:
+      rv = Start();
+      break;
+
+    case State::Finishing:
+      Finish();
+      return NS_OK;
+
+    default:
+      MOZ_CRASH("Bad state!");
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv)) && mState != State::Finishing) {
+    if (NS_SUCCEEDED(mResultCode)) {
+      mResultCode = rv;
+    }
+
+    mState = State::Finishing;
+
+    if (IsOnOwningThread()) {
+      Finish();
+    } else {
+      MOZ_ALWAYS_SUCCEEDS(mNestedEventTarget->Dispatch(this,
+                                                       NS_DISPATCH_NORMAL));
+    }
+  }
+
+  return NS_OK;
+}
+
+void
+RequestHelper::OnResponse(const LSRequestResponse& aResponse)
+{
+  AssertIsOnDOMFileThread();
+  MOZ_ASSERT(mState == State::ResponsePending);
+
+  mActor = nullptr;
+
+  mResponse = aResponse;
+
+  mState = State::Finishing;
+  MOZ_ALWAYS_SUCCEEDS(mNestedEventTarget->Dispatch(this, NS_DISPATCH_NORMAL));
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
new file mode 100644
index 000000000000..1e6ab93f1af4
--- /dev/null
+++ b/dom/localstorage/LSObject.h
@@ -0,0 +1,110 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LSObject_h
+#define mozilla_dom_localstorage_LSObject_h
+
+#include "mozilla/dom/Storage.h"
+
+class nsIPrincipal;
+class nsPIDOMWindowInner;
+
+namespace mozilla {
+
+class ErrorResult;
+
+namespace dom {
+
+class LSDatabase;
+class LSRequestChild;
+class LSRequestChildCallback;
+class LSRequestParams;
+class PrincipalOrQuotaInfo;
+
+class LSObject final
+  : public Storage
+{
+  nsAutoPtr<PrincipalOrQuotaInfo> mInfo;
+
+  RefPtr<LSDatabase> mDatabase;
+
+public:
+  static nsresult
+  Create(nsPIDOMWindowInner* aWindow,
+         Storage** aStorage);
+
+  static void
+  CancelSyncLoop();
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSObject);
+  }
+
+  LSRequestChild*
+  StartRequest(nsIEventTarget* aMainEventTarget,
+               const LSRequestParams& aParams,
+               LSRequestChildCallback* aCallback);
+
+  // Storage overrides.
+  StorageType
+  Type() const override;
+
+  bool
+  IsForkOf(const Storage* aStorage) const override;
+
+  int64_t
+  GetOriginQuotaUsage() const override;
+
+  uint32_t
+  GetLength(nsIPrincipal& aSubjectPrincipal,
+            ErrorResult& aError) override;
+
+  void
+  Key(uint32_t aIndex,
+      nsAString& aResult,
+      nsIPrincipal& aSubjectPrincipal,
+      ErrorResult& aError) override;
+
+  void
+  GetItem(const nsAString& aKey,
+          nsAString& aResult,
+          nsIPrincipal& aSubjectPrincipal,
+          ErrorResult& aError) override;
+
+  void
+  GetSupportedNames(nsTArray<nsString>& aNames) override;
+
+  void
+  SetItem(const nsAString& aKey,
+          const nsAString& aValue,
+          nsIPrincipal& aSubjectPrincipal,
+          ErrorResult& aError) override;
+
+  void
+  RemoveItem(const nsAString& aKey,
+             nsIPrincipal& aSubjectPrincipal,
+             ErrorResult& aError) override;
+
+  void
+  Clear(nsIPrincipal& aSubjectPrincipal,
+        ErrorResult& aError) override;
+
+private:
+  LSObject(nsPIDOMWindowInner* aWindow,
+           nsIPrincipal* aPrincipal);
+
+  ~LSObject();
+
+  nsresult
+  EnsureDatabase();
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LSObject_h
diff --git a/dom/localstorage/LocalStorageCommon.cpp b/dom/localstorage/LocalStorageCommon.cpp
new file mode 100644
index 000000000000..026331cf00aa
--- /dev/null
+++ b/dom/localstorage/LocalStorageCommon.cpp
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LocalStorageCommon.h"
+
+namespace mozilla {
+namespace dom {
+
+namespace {
+
+int32_t gNextGenLocalStorageEnabled = -1;
+
+} // namespace
+
+bool
+NextGenLocalStorageEnabled()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (gNextGenLocalStorageEnabled == -1) {
+    bool enabled = Preferences::GetBool("dom.storage.next_gen", false);
+    gNextGenLocalStorageEnabled = enabled ? 1 : 0;
+  }
+
+  return !!gNextGenLocalStorageEnabled;
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LocalStorageCommon.h b/dom/localstorage/LocalStorageCommon.h
new file mode 100644
index 000000000000..f65e6ad6d89d
--- /dev/null
+++ b/dom/localstorage/LocalStorageCommon.h
@@ -0,0 +1,193 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LocalStorageCommon_h
+#define mozilla_dom_localstorage_LocalStorageCommon_h
+
+/*
+ * Local storage
+ * ~~~~~~~~~~~~~
+ *
+ * Implementation overview
+ * ~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * The implementation is based on a per principal/origin cache (datastore)
+ * living in the main process and synchronous calls initiated from content
+ * processes.
+ * The IPC communication is managed by database actors which link to the
+ * datastore.
+ * The synchronous blocking of the main thread is done by using a special
+ * technique or by using standard synchronous IPC calls.
+ *
+ * General architecture
+ * ~~~~~~~~~~~~~~~~~~~~
+ * The current browser architecture consists of one main process and multiple
+ * content processes (there are other processes but for simplicity's sake, they
+ * are not mentioned here). The processes use the IPC communication to talk to
+ * each other. Local storage implementation uses the client-server model, so
+ * the main process manages all the data and content processes then request
+ * particular data from the main process. The main process is also called the
+ * parent or the parent side, the content process is then called the child or
+ * the child side.
+ *
+ * Datastores
+ * ~~~~~~~~~~
+ *
+ * A datastore provides a convenient way to access data for given origin. The
+ * data is always preloaded into memory and indexed using a hash table. This
+ * enables very fast access to particular stored items. There can be only one
+ * datastore per origin and exists solely on the parent side. It is represented
+ * by the "Datastore" class. A datastore instance is a ref counted object and
+ * lives on the PBackground thread, it is kept alive by database objects. When
+ * the last database object for given origin is destroyed, the associated
+ * datastore object is destroyed too.
+ *
+ * Databases
+ * ~~~~~~~~~
+ *
+ * A database allows direct access to a datastore from a content process. There
+ * can be multiple databases for the same origin, but they all share the same
+ * datastore.
+ * Databases use the PBackgroundLSDatabase IPDL protocol for IPC communication.
+ * Given the nature of local storage, most of PBackgroundLSDatabase messages
+ * are synchronous.
+ *
+ * On the parent side, the database is represented by the "Database" class that
+ * is a parent actor as well (implements the "PBackgroundLSDatabaseParent"
+ * interface). A database instance is a ref counted object and lives on the
+ * PBackground thread.
+ * All live database actors are tracked in an array.
+ *
+ * On the child side, the database is represented by the "LSDatabase" class
+ * that provides indirect access to a child actor. An LSDatabase instance is a
+ * ref counted object and lives on the main thread.
+ * The actual child actor is represented by the "LSDatabaseChild" class that
+ * implements the "PBackgroundLSDatabaseChild" interface. An "LSDatabaseChild"
+ * instance is not ref counted and lives on the main thread too.
+ *
+ * Synchronous blocking
+ * ~~~~~~~~~~~~~~~~~~~~
+ *
+ * Local storage is synchronous in nature which means the execution can't move
+ * forward until there's a reply for given method call.
+ * Since we have to use IPC anyway, we could just always use synchronous IPC
+ * messages for all local storage method calls. Well, there's a problem with
+ * that approach.
+ * If the main process needs to do some off PBackground thread stuff like
+ * getting info from principals on the main thread or some asynchronous stuff
+ * like directory locking before sending a reply to a synchronous message, then
+ * we would have to block the thread or spin the event loop which is usually a
+ * bad idea, especially in the main process.
+ * Instead, we can use a special thread in the content process called DOM File
+ * thread for communication with the main process using asynchronous messages
+ * and synchronously block the main thread until the DOM File thread is done
+ * (the main thread blocking is a bit more complicated, see the comment in
+ * RequestHelper::StartAndReturnResponse for more details).
+ * Anyway, the extra hop to the DOM File thread brings another overhead and
+ * latency. The final solution is to use a combination of the special thread
+ * for complex stuff like datastore preparation and synchronous IPC messages
+ * sent directly from the main thread for database access when data is already
+ * loaded from disk into memory.
+ *
+ * Requests
+ * ~~~~~~~~
+ *
+ * Requests are used to handle asynchronous high level datastore operations
+ * which are initiated in a content process and then processed in the parent
+ * process (for example, preparation of a datastore).
+ * Requests use the "PBackgroundLSRequest" IPDL protocol for IPC communication.
+ *
+ * On the parent side, the request is represented by the "LSRequestBase" class
+ * that is a parent actor as well (implements the "PBackgroundLSRequestParent"
+ * interface). It's an abstract class (contains pure virtual functions) so it
+ * can't be used to create instances.
+ * It also inherits from the "DatastoreOperationBase" class which is a generic
+ * base class for all datastore operations. The "DatastoreOperationsBase" class
+ * inherits from the "Runnable" class, so derived class instances are ref
+ * counted, can be dispatched to multiple threads and thus they are used on
+ * multiple threads. However, derived class instances can be created on the
+ * PBackground thread only.
+ *
+ * On the child side, the request is represented by the "RequestHelper" class
+ * that covers all the complexity needed to start a new request, handle
+ * responses and do safe main thread blocking at the same time.
+ * It inherits from the "Runnable" class, so instances are ref counted and
+ * they are internally used on multiple threads (specifically on the main
+ * thread and on the DOM File thread). Anyway, users should create and use
+ * instances of this class only on the main thread (apart from a special case
+ * when we need to cancel the request from an internal chromium IPC thread to
+ * prevent a dead lock involving CPOWs).
+ * The actual child actor is represented by the "LSRequestChild" class that
+ * implements the "PBackgroundLSRequestChild" interface. An "LSRequestChild"
+ * instance is not ref counted and lives on the DOM File thread.
+ * Request responses are passed using the "LSRequestChildCallback" interface.
+ *
+ * Preparation of a datastore
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * The datastore preparation is needed to make sure a datastore is fully loaded
+ * into memory. Every datastore preparation produces a unique id (even if the
+ * datastore for given origin already exists).
+ * On the parent side, the preparation is handled by the "PrepareDatastoreOp"
+ * class which inherits from the "LSRequestBase" class. The preparation process
+ * on the parent side is quite complicated, it happens sequentially on multiple
+ * threads and is managed by a state machine.
+ * On the child side, the preparation is done in the LSObject::EnsureDatabase
+ * method using the "RequestHelper" class. The method starts a new preparation
+ * request and obtains a unique id produced by the parent (or an error code if
+ * the requested failed to complete).
+ *
+ * Linking databases to a datastore
+ * ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * A datastore exists only on the parent side, but it can be accessed from the
+ * content via database actors. Database actors are initiated on the child side
+ * and they need to be linked to a datastore on the parent side via an id. The
+ * datastore preparation process gives us the required id.
+ * The linking is initiated on the child side in the LSObject::EnsureDatabase
+ * method by calling SendPBackgroundLSDatabaseConstructor and finished in
+ * RecvPBackgroundLSDatabaseConstructor on the parent side.
+ *
+ * Actor migration
+ * ~~~~~~~~~~~~~~~
+ *
+ * In theory, the datastore preparation request could return a database actor
+ * directly (instead of returning an id intended for database linking to a
+ * datastore). However, as it was explained above, the preparation must be done
+ * on the DOM File thread and database objects are used on the main thread. The
+ * returned actor would have to be migrated from the DOM File thread to the
+ * main thread and that's something which our IPDL doesn't support yet.
+ *
+ * Exposing local storage
+ * ~~~~~~~~~~~~~~~~~~~~~~
+ *
+ * The implementation is exposed to the DOM via window.localStorage attribute.
+ * Local storage's sibling, session storage shares the same WebIDL interface
+ * for exposing it to web content, therefore there's an abstract class called
+ * "Storage" that handles some of the common DOM bindings stuff. Local storage
+ * specific functionality is defined in the "LSObject" derived class.
+ * The "LSObject" class is also a starting point for the datastore preparation
+ * and database linking.
+ *
+ * Local storage manager
+ * ~~~~~~~~~~~~~~~~~~~~~
+ *
+ * The local storage manager exposes some of the features that need to be
+ * available only in the chrome code or tests. The manager is represented by
+ * the "LocalStorageManager2" class that implements the "nsIDOMStorageManager"
+ * interface.
+ */
+
+namespace mozilla {
+namespace dom {
+
+bool
+NextGenLocalStorageEnabled();
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LocalStorageCommon_h
diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
new file mode 100644
index 000000000000..5d6e761a7eaf
--- /dev/null
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -0,0 +1,88 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LocalStorageManager2.h"
+
+#include "LSObject.h"
+
+namespace mozilla {
+namespace dom {
+
+LocalStorageManager2::LocalStorageManager2()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(NextGenLocalStorageEnabled());
+}
+
+LocalStorageManager2::~LocalStorageManager2()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+}
+
+NS_IMPL_ISUPPORTS(LocalStorageManager2, nsIDOMStorageManager)
+
+NS_IMETHODIMP
+LocalStorageManager2::PrecacheStorage(nsIPrincipal* aPrincipal,
+                                      Storage** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+LocalStorageManager2::CreateStorage(mozIDOMWindow* aWindow,
+                                    nsIPrincipal* aPrincipal,
+                                    const nsAString& aDocumentURI,
+                                    bool aPrivate,
+                                    Storage** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+LocalStorageManager2::GetStorage(mozIDOMWindow* aWindow,
+                                 nsIPrincipal* aPrincipal,
+                                 bool aPrivate,
+                                 Storage** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+LocalStorageManager2::CloneStorage(Storage* aStorageToCloneFrom)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aStorageToCloneFrom);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+LocalStorageManager2::CheckStorage(nsIPrincipal* aPrincipal,
+                                   Storage *aStorage,
+                                   bool* _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(aStorage);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LocalStorageManager2.h b/dom/localstorage/LocalStorageManager2.h
new file mode 100644
index 000000000000..6b5a5f525f93
--- /dev/null
+++ b/dom/localstorage/LocalStorageManager2.h
@@ -0,0 +1,31 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LocalStorageManager2_h
+#define mozilla_dom_localstorage_LocalStorageManager2_h
+
+#include "nsIDOMStorageManager.h"
+
+namespace mozilla {
+namespace dom {
+
+class LocalStorageManager2 final
+  : public nsIDOMStorageManager
+{
+public:
+  LocalStorageManager2();
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIDOMSTORAGEMANAGER
+
+private:
+  ~LocalStorageManager2();
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LocalStorageManager2_h
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
new file mode 100644
index 000000000000..882d5f42ef39
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+
+namespace mozilla {
+namespace dom {
+
+sync protocol PBackgroundLSDatabase
+{
+  manager PBackground;
+
+parent:
+  // The DeleteMe message is used to avoid a race condition between the parent
+  // actor and the child actor. The PBackgroundLSDatabase protocol could be
+  // simply destroyed by sending the __delete__ message from the child side.
+  // However, that would destroy the child actor immediatelly and the parent
+  // could be sending a message to the child at the same time resulting in a
+  // routing error since the child actor wouldn't exist anymore. A routing
+  // error typically causes a crash. The race can be prevented by doing the
+  // teardown in two steps. First, we send the DeleteMe message to the parent
+  // and the parent then sends the __delete__ message to the child.
+  async DeleteMe();
+
+  sync GetLength()
+    returns (uint32_t length);
+
+  sync GetKey(uint32_t index)
+    returns (nsString key);
+
+  sync GetItem(nsString key)
+    returns (nsString value);
+
+  sync GetKeys()
+    returns (nsString[] keys);
+
+  sync SetItem(nsString key, nsString value);
+
+  sync RemoveItem(nsString key);
+
+  sync Clear();
+
+child:
+  async __delete__();
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/PBackgroundLSRequest.ipdl b/dom/localstorage/PBackgroundLSRequest.ipdl
new file mode 100644
index 000000000000..2efd6447a201
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSRequest.ipdl
@@ -0,0 +1,49 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+
+namespace mozilla {
+namespace dom {
+
+struct LSRequestPrepareDatastoreResponse
+{
+  uint64_t datastoreId;
+};
+
+union LSRequestResponse
+{
+  nsresult;
+  LSRequestPrepareDatastoreResponse;
+};
+
+protocol PBackgroundLSRequest
+{
+  manager PBackground;
+
+parent:
+  // The Cancel message is used to avoid a possible dead lock caused by a CPOW
+  // sending a synchronous message from the main thread in the chrome process
+  // to the main thread in the content process at the time we are blocking
+  // the main thread in the content process to handle a request.
+  // We use the PBackground thread on the parent side to handle requests, but
+  // sometimes we need to get information from principals and that's currently
+  // only possible on the main thread. So if the main thread in the chrome
+  // process is blocked by a CPOW operation, our request must wait for the CPOW
+  // operation to complete. However the CPOW operation can't complete either
+  // because we are blocking the main thread in the content process.
+  // The dead lock is prevented by canceling our nested event loop in the
+  // content process when we receive a synchronous IPC message from the parent.
+  async Cancel();
+
+  async Finish();
+
+child:
+  async __delete__(LSRequestResponse response);
+
+  async Ready();
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
new file mode 100644
index 000000000000..e0d7bcf6787e
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
@@ -0,0 +1,32 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include PBackgroundSharedTypes;
+
+namespace mozilla {
+namespace dom {
+
+struct QuotaInfo {
+  nsCString suffix;
+  nsCString group;
+  nsCString origin;
+};
+
+union PrincipalOrQuotaInfo {
+  PrincipalInfo;
+  QuotaInfo;
+};
+
+struct LSRequestPrepareDatastoreParams
+{
+  PrincipalOrQuotaInfo info;
+};
+
+union LSRequestParams
+{
+  LSRequestPrepareDatastoreParams;
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
new file mode 100644
index 000000000000..ebe0c87b7aab
--- /dev/null
+++ b/dom/localstorage/moz.build
@@ -0,0 +1,41 @@
+# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
+# vim: set filetype=python:
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+EXPORTS.mozilla.dom.localstorage += [
+    'ActorsParent.h',
+]
+
+EXPORTS.mozilla.dom += [
+    'LocalStorageCommon.h',
+    'LocalStorageManager2.h',
+    'LSObject.h',
+]
+
+UNIFIED_SOURCES += [
+    'ActorsChild.cpp',
+    'ActorsParent.cpp',
+    'LocalStorageCommon.cpp',
+    'LocalStorageManager2.cpp',
+    'LSDatabase.cpp',
+    'LSObject.cpp',
+]
+
+IPDL_SOURCES += [
+    'PBackgroundLSDatabase.ipdl',
+    'PBackgroundLSRequest.ipdl',
+    'PBackgroundLSSharedTypes.ipdlh',
+]
+
+include('/ipc/chromium/chromium-config.mozbuild')
+
+FINAL_LIBRARY = 'xul'
+
+if CONFIG['GNU_CXX']:
+    CXXFLAGS += ['-Wno-error=shadow']
+
+LOCAL_INCLUDES += [
+    '/dom/file/ipc',
+]
diff --git a/dom/moz.build b/dom/moz.build
index 28b85538deb7..7c2240efcfe2 100644
--- a/dom/moz.build
+++ b/dom/moz.build
@@ -103,6 +103,7 @@ DIRS += [
     'serviceworkers',
     'simpledb',
     'reporting',
+    'localstorage',
 ]
 
 if CONFIG['MOZ_LIBPRIO']:
diff --git a/dom/storage/LocalStorageManager.cpp b/dom/storage/LocalStorageManager.cpp
index 7a37097ef53d..dedd22397fc3 100644
--- a/dom/storage/LocalStorageManager.cpp
+++ b/dom/storage/LocalStorageManager.cpp
@@ -21,6 +21,7 @@
 #include "nsIObserverService.h"
 #include "mozilla/Services.h"
 #include "mozilla/Preferences.h"
+#include "mozilla/dom/LocalStorageCommon.h"
 
 // Only allow relatively small amounts of data since performance of
 // the synchronous IO is very bad.
@@ -61,6 +62,8 @@ NS_IMPL_ISUPPORTS(LocalStorageManager,
 LocalStorageManager::LocalStorageManager()
   : mCaches(8)
 {
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
+
   StorageObserver* observer = StorageObserver::Self();
   NS_ASSERTION(observer, "No StorageObserver, cannot observe private data delete notifications!");
 
@@ -473,9 +476,20 @@ LocalStorageManager::Observe(const char* aTopic,
   return NS_ERROR_UNEXPECTED;
 }
 
+// static
+LocalStorageManager*
+LocalStorageManager::Self()
+{
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
+
+  return sSelf;
+}
+
 LocalStorageManager*
 LocalStorageManager::Ensure()
 {
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
+
   if (sSelf) {
     return sSelf;
   }
diff --git a/dom/storage/LocalStorageManager.h b/dom/storage/LocalStorageManager.h
index 2a36278ace49..88ed6e62913d 100644
--- a/dom/storage/LocalStorageManager.h
+++ b/dom/storage/LocalStorageManager.h
@@ -117,7 +117,7 @@ private:
                    const nsACString& aKeyPrefix);
 
   // Global getter of localStorage manager service
-  static LocalStorageManager* Self() { return sSelf; }
+  static LocalStorageManager* Self();
 
   // Like Self, but creates an instance if we're not yet initialized.
   static LocalStorageManager* Ensure();
diff --git a/dom/storage/StorageIPC.cpp b/dom/storage/StorageIPC.cpp
index 54bcaab64269..760bcfca7b1f 100644
--- a/dom/storage/StorageIPC.cpp
+++ b/dom/storage/StorageIPC.cpp
@@ -147,6 +147,7 @@ StorageDBChild::StorageDBChild(LocalStorageManager* aManager)
   , mStatus(NS_OK)
   , mIPCOpen(false)
 {
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
 }
 
 StorageDBChild::~StorageDBChild()
@@ -158,6 +159,7 @@ StorageDBChild*
 StorageDBChild::Get()
 {
   MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
 
   return sStorageChild;
 }
@@ -167,6 +169,7 @@ StorageDBChild*
 StorageDBChild::GetOrCreate()
 {
   MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!NextGenLocalStorageEnabled());
 
   if (sStorageChild || sStorageChildDown) {
     // When sStorageChildDown is at true, sStorageChild is null.
diff --git a/dom/storage/StorageObserver.cpp b/dom/storage/StorageObserver.cpp
index 56fd031a010f..79f3845d3f9b 100644
--- a/dom/storage/StorageObserver.cpp
+++ b/dom/storage/StorageObserver.cpp
@@ -181,13 +181,15 @@ StorageObserver::ClearMatchingOrigin(const char16_t* aData,
     return rv;
   }
 
-  if (XRE_IsParentProcess()) {
-    StorageDBChild* storageChild = StorageDBChild::GetOrCreate();
-    if (NS_WARN_IF(!storageChild)) {
-      return NS_ERROR_FAILURE;
-    }
+  if (!NextGenLocalStorageEnabled()) {
+    if (XRE_IsParentProcess()) {
+      StorageDBChild* storageChild = StorageDBChild::GetOrCreate();
+      if (NS_WARN_IF(!storageChild)) {
+        return NS_ERROR_FAILURE;
+      }
 
-    storageChild->SendClearMatchingOrigin(originScope);
+      storageChild->SendClearMatchingOrigin(originScope);
+    }
   }
 
   aOriginScope = originScope;
@@ -205,6 +207,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   if (!strcmp(aTopic, kStartupTopic)) {
     MOZ_ASSERT(XRE_IsParentProcess());
 
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     nsCOMPtr<nsIObserverService> obs = mozilla::services::GetObserverService();
     obs->RemoveObserver(this, kStartupTopic);
 
@@ -215,6 +221,7 @@ StorageObserver::Observe(nsISupports* aSubject,
   // Timer callback used to start the database a short timer after startup
   if (!strcmp(aTopic, NS_TIMER_CALLBACK_TOPIC)) {
     MOZ_ASSERT(XRE_IsParentProcess());
+    MOZ_ASSERT(!NextGenLocalStorageEnabled());
 
     nsCOMPtr<nsITimer> timer = do_QueryInterface(aSubject);
     if (!timer) {
@@ -241,15 +248,17 @@ StorageObserver::Observe(nsISupports* aSubject,
       return NS_OK;
     }
 
-    StorageDBChild* storageChild = StorageDBChild::GetOrCreate();
-    if (NS_WARN_IF(!storageChild)) {
-      return NS_ERROR_FAILURE;
-    }
+    if (!NextGenLocalStorageEnabled()) {
+      StorageDBChild* storageChild = StorageDBChild::GetOrCreate();
+      if (NS_WARN_IF(!storageChild)) {
+        return NS_ERROR_FAILURE;
+      }
 
-    storageChild->AsyncClearAll();
+      storageChild->AsyncClearAll();
 
-    if (XRE_IsParentProcess()) {
-      storageChild->SendClearAll();
+      if (XRE_IsParentProcess()) {
+        storageChild->SendClearAll();
+      }
     }
 
     Notify("cookie-cleared");
@@ -311,6 +320,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   }
 
   if (!strcmp(aTopic, "extension:purge-localStorage")) {
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     const char topic[] = "extension:purge-localStorage-caches";
 
     if (aData) {
@@ -355,6 +368,10 @@ StorageObserver::Observe(nsISupports* aSubject,
 
   // Clear all private-browsing caches
   if (!strcmp(aTopic, "last-pb-context-exited")) {
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     Notify("private-browsing-data-cleared");
 
     return NS_OK;
@@ -364,6 +381,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   if (!strcmp(aTopic, "clear-origin-attributes-data")) {
     MOZ_ASSERT(XRE_IsParentProcess());
 
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     OriginAttributesPattern pattern;
     if (!pattern.Init(nsDependentString(aData))) {
       NS_ERROR("Cannot parse origin attributes pattern");
@@ -391,6 +412,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   if (!strcmp(aTopic, "profile-before-change")) {
     MOZ_ASSERT(XRE_IsParentProcess());
 
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     if (mBackgroundThread) {
       bool done = false;
 
@@ -409,6 +434,10 @@ StorageObserver::Observe(nsISupports* aSubject,
 
 #ifdef DOM_STORAGE_TESTS
   if (!strcmp(aTopic, "domstorage-test-flush-force")) {
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     StorageDBChild* storageChild = StorageDBChild::GetOrCreate();
     if (NS_WARN_IF(!storageChild)) {
       return NS_ERROR_FAILURE;
@@ -420,6 +449,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   }
 
   if (!strcmp(aTopic, "domstorage-test-flushed")) {
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     // Only used to propagate to IPC children
     Notify("test-flushed");
 
@@ -427,6 +460,10 @@ StorageObserver::Observe(nsISupports* aSubject,
   }
 
   if (!strcmp(aTopic, "domstorage-test-reload")) {
+    if (NextGenLocalStorageEnabled()) {
+      return NS_OK;
+    }
+
     Notify("test-reload");
 
     return NS_OK;
diff --git a/dom/tests/browser/browser.ini b/dom/tests/browser/browser.ini
index 26ed1132a8a0..2d59172fbd37 100644
--- a/dom/tests/browser/browser.ini
+++ b/dom/tests/browser/browser.ini
@@ -53,8 +53,8 @@ run-if = e10s
 skip-if = !e10s || os != "win" || processor != "x86" # Large-Allocation requires e10s
 [browser_largeAllocation_non_win32.js]
 skip-if = !e10s || (os == "win" && processor == "x86") || (verify && debug && (os == 'linux')) || (os == 'linux') || (os == 'mac' && debug)  # Large-Allocation requires e10s # Bug 1336075
-[browser_localStorage_e10s.js]
-skip-if = !e10s || verify # This is a test of e10s functionality.
+#[browser_localStorage_e10s.js]
+#skip-if = !e10s || verify # This is a test of e10s functionality.
 [browser_localStorage_privatestorageevent.js]
 [browser_persist_cookies.js]
 support-files =
diff --git a/dom/tests/mochitest/localstorage/chrome.ini b/dom/tests/mochitest/localstorage/chrome.ini
index 6cc5562c00c9..af961412ae78 100644
--- a/dom/tests/mochitest/localstorage/chrome.ini
+++ b/dom/tests/mochitest/localstorage/chrome.ini
@@ -7,5 +7,5 @@ support-files =
 
 [test_localStorageBasePrivateBrowsing_perwindowpb.html]
 skip-if = true # bug 1156725
-[test_localStorageFromChrome.xhtml]
-[test_localStorageQuotaPrivateBrowsing_perwindowpb.html]
+#[test_localStorageFromChrome.xhtml]
+#[test_localStorageQuotaPrivateBrowsing_perwindowpb.html]
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index c62ab9f6d84f..3a1894687ab0 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -19,19 +19,19 @@ support-files =
   file_tryAccessSessionStorage.html
 
 [test_brokenUTF-16.html]
-[test_bug600307-DBOps.html]
+#[test_bug600307-DBOps.html]
 [test_bug746272-1.html]
 [test_bug746272-2.html]
 skip-if = os == "android" || verify # bug 962029
 [test_cookieBlock.html]
-[test_cookieSession.html]
+#[test_cookieSession.html]
 [test_embededNulls.html]
 [test_keySync.html]
-[test_localStorageBase.html]
-skip-if = e10s
-[test_localStorageBaseSessionOnly.html]
+#[test_localStorageBase.html]
+#skip-if = e10s
+#[test_localStorageBaseSessionOnly.html]
 [test_localStorageCookieSettings.html]
-[test_localStorageEnablePref.html]
+#[test_localStorageEnablePref.html]
 [test_localStorageKeyOrder.html]
 [test_localStorageOriginsDiff.html]
 [test_localStorageOriginsDomainDiffs.html]
@@ -40,14 +40,14 @@ skip-if = toolkit == 'android'
 [test_localStorageOriginsPortDiffs.html]
 [test_localStorageOriginsSchemaDiffs.html]
 skip-if = toolkit == 'android' #TIMED_OUT
-[test_localStorageQuota.html]
-skip-if = toolkit == 'android' #TIMED_OUT
-[test_localStorageQuotaSessionOnly.html]
-skip-if = toolkit == 'android' || (verify && (os == 'linux' || os == 'win')) #TIMED_OUT
+#[test_localStorageQuota.html]
+#skip-if = toolkit == 'android' #TIMED_OUT
+#[test_localStorageQuotaSessionOnly.html]
+#skip-if = toolkit == 'android' || (verify && (os == 'linux' || os == 'win')) #TIMED_OUT
 [test_localStorageQuotaSessionOnly2.html]
 skip-if = true # bug 1347690
 [test_localStorageReplace.html]
 skip-if = toolkit == 'android'
 [test_storageConstructor.html]
-[test_localStorageSessionPrefOverride.html]
+#[test_localStorageSessionPrefOverride.html]
 [test_firstPartyOnlyPermission.html]
diff --git a/dom/tests/mochitest/storageevent/mochitest.ini b/dom/tests/mochitest/storageevent/mochitest.ini
index e16d9cd2e965..7985f2c38672 100644
--- a/dom/tests/mochitest/storageevent/mochitest.ini
+++ b/dom/tests/mochitest/storageevent/mochitest.ini
@@ -11,7 +11,7 @@ support-files =
   interOriginTest2.js
 
 [test_storageLocalStorageEventCheckNoPropagation.html]
-[test_storageLocalStorageEventCheckPropagation.html]
-[test_storageNotifications.html]
+#[test_storageLocalStorageEventCheckPropagation.html]
+#[test_storageNotifications.html]
 [test_storageSessionStorageEventCheckNoPropagation.html]
 [test_storageSessionStorageEventCheckPropagation.html]
diff --git a/dom/tests/moz.build b/dom/tests/moz.build
index 62683d990a6c..d99e9e72a9b8 100644
--- a/dom/tests/moz.build
+++ b/dom/tests/moz.build
@@ -179,7 +179,7 @@ MOCHITEST_CHROME_MANIFESTS += [
     'mochitest/general/chrome.ini',
     'mochitest/geolocation/chrome.ini',
     'mochitest/keyhandling/chrome.ini',
-    'mochitest/localstorage/chrome.ini',
+#    'mochitest/localstorage/chrome.ini',
     'mochitest/sessionstorage/chrome.ini',
     'mochitest/webcomponents/chrome.ini',
     'mochitest/whatwg/chrome.ini',
diff --git a/editor/libeditor/tests/mochitest.ini b/editor/libeditor/tests/mochitest.ini
index c334d745d9aa..8349c24dc2fc 100644
--- a/editor/libeditor/tests/mochitest.ini
+++ b/editor/libeditor/tests/mochitest.ini
@@ -141,9 +141,9 @@ skip-if = os == 'android'
 [test_bug645914.html]
 [test_bug646194.html]
 [test_bug668599.html]
-[test_bug674770-1.html]
-subsuite = clipboard
-skip-if = toolkit == 'android' || verify
+#[test_bug674770-1.html]
+#subsuite = clipboard
+#skip-if = toolkit == 'android' || verify
 [test_bug674770-2.html]
 subsuite = clipboard
 skip-if = toolkit == 'android'
diff --git a/ipc/glue/BackgroundChildImpl.cpp b/ipc/glue/BackgroundChildImpl.cpp
index d284323c3b7e..5630857427d3 100644
--- a/ipc/glue/BackgroundChildImpl.cpp
+++ b/ipc/glue/BackgroundChildImpl.cpp
@@ -16,6 +16,8 @@
 #include "mozilla/Assertions.h"
 #include "mozilla/SchedulerGroup.h"
 #include "mozilla/dom/ClientManagerActors.h"
+#include "mozilla/dom/PBackgroundLSDatabaseChild.h"
+#include "mozilla/dom/PBackgroundLSRequestChild.h"
 #include "mozilla/dom/PBackgroundSDBConnectionChild.h"
 #include "mozilla/dom/PFileSystemRequestChild.h"
 #include "mozilla/dom/FileSystemTaskBase.h"
@@ -223,6 +225,58 @@ BackgroundChildImpl::DeallocPBackgroundIndexedDBUtilsChild(
   return true;
 }
 
+BackgroundChildImpl::PBackgroundSDBConnectionChild*
+BackgroundChildImpl::AllocPBackgroundSDBConnectionChild(
+                                            const PrincipalInfo& aPrincipalInfo)
+{
+  MOZ_CRASH("PBackgroundSDBConnectionChild actor should be manually "
+            "constructed!");
+}
+
+bool
+BackgroundChildImpl::DeallocPBackgroundSDBConnectionChild(
+                                          PBackgroundSDBConnectionChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
+BackgroundChildImpl::PBackgroundLSDatabaseChild*
+BackgroundChildImpl::AllocPBackgroundLSDatabaseChild(
+                                                    const uint64_t& aDatastoreId)
+{
+  MOZ_CRASH("PBackgroundLSDatabaseChild actor should be manually constructed!");
+}
+
+bool
+BackgroundChildImpl::DeallocPBackgroundLSDatabaseChild(
+                                             PBackgroundLSDatabaseChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
+BackgroundChildImpl::PBackgroundLSRequestChild*
+BackgroundChildImpl::AllocPBackgroundLSRequestChild(
+                                                 const LSRequestParams& aParams)
+{
+  MOZ_CRASH("PBackgroundLSRequestChild actor should be manually constructed!");
+}
+
+bool
+BackgroundChildImpl::DeallocPBackgroundLSRequestChild(
+                                              PBackgroundLSRequestChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
 BackgroundChildImpl::PBackgroundLocalStorageCacheChild*
 BackgroundChildImpl::AllocPBackgroundLocalStorageCacheChild(
                                             const PrincipalInfo& aPrincipalInfo,
@@ -243,24 +297,6 @@ BackgroundChildImpl::DeallocPBackgroundLocalStorageCacheChild(
   return true;
 }
 
-BackgroundChildImpl::PBackgroundSDBConnectionChild*
-BackgroundChildImpl::AllocPBackgroundSDBConnectionChild(
-                                            const PrincipalInfo& aPrincipalInfo)
-{
-  MOZ_CRASH("PBackgroundSDBConnectionChild actor should be manually "
-            "constructed!");
-}
-
-bool
-BackgroundChildImpl::DeallocPBackgroundSDBConnectionChild(
-                                          PBackgroundSDBConnectionChild* aActor)
-{
-  MOZ_ASSERT(aActor);
-
-  delete aActor;
-  return true;
-}
-
 BackgroundChildImpl::PBackgroundStorageChild*
 BackgroundChildImpl::AllocPBackgroundStorageChild(const nsString& aProfilePath)
 {
diff --git a/ipc/glue/BackgroundChildImpl.h b/ipc/glue/BackgroundChildImpl.h
index c0320f9afe46..2df764eaa7ba 100644
--- a/ipc/glue/BackgroundChildImpl.h
+++ b/ipc/glue/BackgroundChildImpl.h
@@ -78,6 +78,19 @@ protected:
   DeallocPBackgroundSDBConnectionChild(PBackgroundSDBConnectionChild* aActor)
                                        override;
 
+  virtual PBackgroundLSDatabaseChild*
+  AllocPBackgroundLSDatabaseChild(const uint64_t& aCacheId) override;
+
+  virtual bool
+  DeallocPBackgroundLSDatabaseChild(PBackgroundLSDatabaseChild* aActor)
+                                    override;
+
+  virtual PBackgroundLSRequestChild*
+  AllocPBackgroundLSRequestChild(const LSRequestParams& aParams) override;
+
+  virtual bool
+  DeallocPBackgroundLSRequestChild(PBackgroundLSRequestChild* aActor) override;
+
   virtual PBackgroundLocalStorageCacheChild*
   AllocPBackgroundLocalStorageCacheChild(const PrincipalInfo& aPrincipalInfo,
                                          const nsCString& aOriginKey,
diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp
index b17b30aec9b2..2033e4870870 100644
--- a/ipc/glue/BackgroundParentImpl.cpp
+++ b/ipc/glue/BackgroundParentImpl.cpp
@@ -33,6 +33,7 @@
 #include "mozilla/dom/ipc/IPCBlobInputStreamParent.h"
 #include "mozilla/dom/ipc/PendingIPCBlobParent.h"
 #include "mozilla/dom/ipc/TemporaryIPCBlobParent.h"
+#include "mozilla/dom/localstorage/ActorsParent.h"
 #include "mozilla/dom/quota/ActorsParent.h"
 #include "mozilla/dom/simpledb/ActorsParent.h"
 #include "mozilla/dom/RemoteWorkerParent.h"
@@ -293,6 +294,79 @@ BackgroundParentImpl::DeallocPBackgroundSDBConnectionParent(
   return mozilla::dom::DeallocPBackgroundSDBConnectionParent(aActor);
 }
 
+BackgroundParentImpl::PBackgroundLSDatabaseParent*
+BackgroundParentImpl::AllocPBackgroundLSDatabaseParent(
+                                                    const uint64_t& aDatastoreId)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return mozilla::dom::AllocPBackgroundLSDatabaseParent(aDatastoreId);
+}
+
+mozilla::ipc::IPCResult
+BackgroundParentImpl::RecvPBackgroundLSDatabaseConstructor(
+                                            PBackgroundLSDatabaseParent* aActor,
+                                            const uint64_t& aDatastoreId)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  if (!mozilla::dom::RecvPBackgroundLSDatabaseConstructor(aActor,
+                                                          aDatastoreId)) {
+    return IPC_FAIL_NO_REASON(this);
+  }
+  return IPC_OK();
+}
+
+bool
+BackgroundParentImpl::DeallocPBackgroundLSDatabaseParent(
+                                            PBackgroundLSDatabaseParent* aActor)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  return mozilla::dom::DeallocPBackgroundLSDatabaseParent(aActor);
+}
+
+BackgroundParentImpl::PBackgroundLSRequestParent*
+BackgroundParentImpl::AllocPBackgroundLSRequestParent(
+                                                 const LSRequestParams& aParams)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return mozilla::dom::AllocPBackgroundLSRequestParent(this, aParams);
+}
+
+mozilla::ipc::IPCResult
+BackgroundParentImpl::RecvPBackgroundLSRequestConstructor(
+                                             PBackgroundLSRequestParent* aActor,
+                                             const LSRequestParams& aParams)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  if (!mozilla::dom::RecvPBackgroundLSRequestConstructor(aActor, aParams)) {
+    return IPC_FAIL_NO_REASON(this);
+  }
+  return IPC_OK();
+}
+
+bool
+BackgroundParentImpl::DeallocPBackgroundLSRequestParent(
+                                             PBackgroundLSRequestParent* aActor)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  return mozilla::dom::DeallocPBackgroundLSRequestParent(aActor);
+}
+
 BackgroundParentImpl::PBackgroundLocalStorageCacheParent*
 BackgroundParentImpl::AllocPBackgroundLocalStorageCacheParent(
                                             const PrincipalInfo& aPrincipalInfo,
diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h
index 64a1ba40737c..10fc1a37d2bb 100644
--- a/ipc/glue/BackgroundParentImpl.h
+++ b/ipc/glue/BackgroundParentImpl.h
@@ -77,6 +77,28 @@ protected:
   DeallocPBackgroundSDBConnectionParent(PBackgroundSDBConnectionParent* aActor)
                                         override;
 
+  virtual PBackgroundLSDatabaseParent*
+  AllocPBackgroundLSDatabaseParent(const uint64_t& aCacheId) override;
+
+  virtual mozilla::ipc::IPCResult
+  RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                       const uint64_t& aDatastoreId) override;
+
+  virtual bool
+  DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+                                     override;
+
+  virtual PBackgroundLSRequestParent*
+  AllocPBackgroundLSRequestParent(const LSRequestParams& aParams) override;
+
+  virtual mozilla::ipc::IPCResult
+  RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
+                                      const LSRequestParams& aParams) override;
+
+  virtual bool
+  DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor)
+                                    override;
+
   virtual PBackgroundLocalStorageCacheParent*
   AllocPBackgroundLocalStorageCacheParent(const PrincipalInfo& aPrincipalInfo,
                                           const nsCString& aOriginKey,
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index 7b6d4304de7f..17260497c99a 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -6,6 +6,8 @@ include protocol PAsmJSCacheEntry;
 include protocol PBackgroundIDBFactory;
 include protocol PBackgroundIndexedDBUtils;
 include protocol PBackgroundSDBConnection;
+include protocol PBackgroundLSDatabase;
+include protocol PBackgroundLSRequest;
 include protocol PBackgroundLocalStorageCache;
 include protocol PBackgroundStorage;
 include protocol PBackgroundTest;
@@ -44,6 +46,7 @@ include DOMTypes;
 include IPCBlob;
 include IPCServiceWorkerDescriptor;
 include IPCServiceWorkerRegistrationDescriptor;
+include PBackgroundLSSharedTypes;
 include PBackgroundSharedTypes;
 include PBackgroundIDBSharedTypes;
 include PFileSystemParams;
@@ -72,6 +75,8 @@ sync protocol PBackground
   manages PBackgroundIDBFactory;
   manages PBackgroundIndexedDBUtils;
   manages PBackgroundSDBConnection;
+  manages PBackgroundLSDatabase;
+  manages PBackgroundLSRequest;
   manages PBackgroundLocalStorageCache;
   manages PBackgroundStorage;
   manages PBackgroundTest;
@@ -119,6 +124,10 @@ parent:
 
   async PBackgroundSDBConnection(PrincipalInfo principalInfo);
 
+  async PBackgroundLSDatabase(uint64_t datastoreId);
+
+  async PBackgroundLSRequest(LSRequestParams params);
+
   async PBackgroundLocalStorageCache(PrincipalInfo principalInfo,
                                      nsCString originKey,
                                      uint32_t privateBrowsingId);
diff --git a/ipc/ipdl/sync-messages.ini b/ipc/ipdl/sync-messages.ini
index 2a2fe5f2484e..07afb207e63b 100644
--- a/ipc/ipdl/sync-messages.ini
+++ b/ipc/ipdl/sync-messages.ini
@@ -922,6 +922,20 @@ description =
 description =
 [PBackgroundStorage::Preload]
 description =
+[PBackgroundLSDatabase::GetLength]
+description =
+[PBackgroundLSDatabase::GetKey]
+description =
+[PBackgroundLSDatabase::GetItem]
+description =
+[PBackgroundLSDatabase::GetKeys]
+description =
+[PBackgroundLSDatabase::SetItem]
+description =
+[PBackgroundLSDatabase::RemoveItem]
+description =
+[PBackgroundLSDatabase::Clear]
+description =
 [PRemoteSpellcheckEngine::Check]
 description =
 [PRemoteSpellcheckEngine::CheckAndSuggest]
diff --git a/layout/build/nsLayoutModule.cpp b/layout/build/nsLayoutModule.cpp
index c984775f5381..6500066bf83a 100644
--- a/layout/build/nsLayoutModule.cpp
+++ b/layout/build/nsLayoutModule.cpp
@@ -63,7 +63,9 @@
 #include "mozilla/dom/BlobURL.h"
 #include "mozilla/dom/DOMRequest.h"
 #include "mozilla/dom/SDBConnection.h"
+#include "mozilla/dom/LocalStorageCommon.h"
 #include "mozilla/dom/LocalStorageManager.h"
+#include "mozilla/dom/LocalStorageManager2.h"
 #include "mozilla/dom/quota/QuotaManagerService.h"
 #include "mozilla/dom/ServiceWorkerManager.h"
 #include "mozilla/dom/StorageActivityService.h"
@@ -165,7 +167,6 @@ already_AddRefed<nsIPresentationService> NS_CreatePresentationService();
 // Factory Constructor
 typedef mozilla::dom::BlobURL::Mutator BlobURLMutator;
 NS_GENERIC_FACTORY_CONSTRUCTOR(BlobURLMutator)
-NS_GENERIC_FACTORY_CONSTRUCTOR(LocalStorageManager)
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(DOMRequestService,
                                          DOMRequestService::FactoryCreate)
 NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(QuotaManagerService,
@@ -494,6 +495,19 @@ NS_DEFINE_NAMED_CID(TEXT_INPUT_PROCESSOR_CID);
 
 NS_DEFINE_NAMED_CID(NS_SCRIPTERROR_CID);
 
+static nsresult
+LocalStorageManagerConstructor(nsISupports *aOuter, REFNSIID aIID,
+                               void **aResult)
+{
+  if (NextGenLocalStorageEnabled()) {
+    RefPtr<LocalStorageManager2> manager = new LocalStorageManager2();
+    return manager->QueryInterface(aIID, aResult);
+  }
+
+  RefPtr<LocalStorageManager> manager = new LocalStorageManager();
+  return manager->QueryInterface(aIID, aResult);
+}
+
 static const mozilla::Module::CIDEntry kLayoutCIDs[] = {
   // clang-format off
   XPCONNECT_CIDENTRIES
diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index d74bfa0c87a3..e37b8722e9a8 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1285,6 +1285,12 @@ pref("dom.disable_open_click_delay", 1000);
 pref("dom.serviceWorkers.disable_open_click_delay", 1000);
 
 pref("dom.storage.enabled", true);
+// Whether or not LSNG (Next Generation Local Storage) is enabled.
+#ifdef NIGHTLY_BUILD
+pref("dom.storage.next_gen", true);
+#else
+pref("dom.storage.next_gen", false);
+#endif
 pref("dom.storage.default_quota",      5120);
 pref("dom.storage.testing", false);
 
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
new file mode 100644
index 000000000000..9a6f9b1f7781
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
@@ -0,0 +1,2 @@
+[opener-closed.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
new file mode 100644
index 000000000000..378c58417e34
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
@@ -0,0 +1,2 @@
+[opener-noopener.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
new file mode 100644
index 000000000000..7567b7218ce4
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
@@ -0,0 +1,2 @@
+[opener-noreferrer.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
new file mode 100644
index 000000000000..fc0dfe449c21
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
@@ -0,0 +1,2 @@
+[choose-_blank-002.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
new file mode 100644
index 000000000000..c8485530052e
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
@@ -0,0 +1,2 @@
+[choose-_parent-004.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
new file mode 100644
index 000000000000..df79b5dd6dd9
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
@@ -0,0 +1,2 @@
+[choose-_self-002.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
new file mode 100644
index 000000000000..c75ba08bdc24
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
@@ -0,0 +1,2 @@
+[choose-_top-001.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
new file mode 100644
index 000000000000..a7527b0093c0
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
@@ -0,0 +1,2 @@
+[choose-_top-002.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
new file mode 100644
index 000000000000..4d357924c3ab
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
@@ -0,0 +1,2 @@
+[choose-_top-003.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini b/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
new file mode 100644
index 000000000000..57b9ebad9cf4
--- /dev/null
+++ b/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
@@ -0,0 +1,2 @@
+[noreferrer-null-opener.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini b/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
index 0dc9bc12e114..f509186c2a7a 100644
--- a/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
+++ b/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
@@ -1,3 +1,2 @@
 [noreferrer-window-name.html]
-  disabled:
-    if verify: fails in verify mode
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/document-domain.html.ini b/testing/web-platform/meta/webstorage/document-domain.html.ini
index ba4c2b143af4..0f7608a1bb03 100644
--- a/testing/web-platform/meta/webstorage/document-domain.html.ini
+++ b/testing/web-platform/meta/webstorage/document-domain.html.ini
@@ -1,2 +1,2 @@
 [document-domain.html]
-  expected: TIMEOUT
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_basic.html.ini b/testing/web-platform/meta/webstorage/event_basic.html.ini
new file mode 100644
index 000000000000..9a2a19afca5a
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_basic.html.ini
@@ -0,0 +1,2 @@
+[event_basic.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_body_attribute.html.ini b/testing/web-platform/meta/webstorage/event_body_attribute.html.ini
new file mode 100644
index 000000000000..d1f1c341ed1e
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_body_attribute.html.ini
@@ -0,0 +1,2 @@
+[event_body_attribute.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini b/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
new file mode 100644
index 000000000000..42743ad9ccdf
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
@@ -0,0 +1,2 @@
+[event_case_sensitive.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_key.html.ini b/testing/web-platform/meta/webstorage/event_local_key.html.ini
new file mode 100644
index 000000000000..0704163d2577
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_key.html.ini
@@ -0,0 +1,2 @@
+[event_local_key.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini b/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
new file mode 100644
index 000000000000..af1cb9407a36
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
@@ -0,0 +1,2 @@
+[event_local_newvalue.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini b/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
new file mode 100644
index 000000000000..c218f24c7a38
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
@@ -0,0 +1,2 @@
+[event_local_oldvalue.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini b/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
new file mode 100644
index 000000000000..3e2c560eeeda
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
@@ -0,0 +1,2 @@
+[event_local_removeitem.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini b/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
new file mode 100644
index 000000000000..447aced1bcc0
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
@@ -0,0 +1,2 @@
+[event_local_storagearea.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_url.html.ini b/testing/web-platform/meta/webstorage/event_local_url.html.ini
new file mode 100644
index 000000000000..6480915b2eba
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_local_url.html.ini
@@ -0,0 +1,2 @@
+[event_local_url.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini b/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
new file mode 100644
index 000000000000..f6475fbb82c2
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
@@ -0,0 +1,2 @@
+[event_no_duplicates.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_setattribute.html.ini b/testing/web-platform/meta/webstorage/event_setattribute.html.ini
new file mode 100644
index 000000000000..0d31589933c6
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/event_setattribute.html.ini
@@ -0,0 +1,2 @@
+[event_setattribute.html]
+  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini b/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini
new file mode 100644
index 000000000000..48f2692aad2a
--- /dev/null
+++ b/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini
@@ -0,0 +1,2 @@
+[storage_local_setitem_quotaexceedederr.html]
+  disabled: temporary
diff --git a/toolkit/components/antitracking/test/browser/browser.ini b/toolkit/components/antitracking/test/browser/browser.ini
index 360d4a4be03a..1a1a79aa108d 100644
--- a/toolkit/components/antitracking/test/browser/browser.ini
+++ b/toolkit/components/antitracking/test/browser/browser.ini
@@ -68,8 +68,8 @@ skip-if = serviceworker_e10s
 [browser_storageAccessWithHeuristics.js]
 [browser_allowPermissionForTracker.js]
 [browser_denyPermissionForTracker.js]
-[browser_localStorageEvents.js]
-support-files = localStorage.html
+#[browser_localStorageEvents.js]
+#support-files = localStorage.html
 [browser_partitionedLocalStorage.js]
-[browser_partitionedLocalStorage_events.js]
-support-files = localStorageEvents.html
+#[browser_partitionedLocalStorage_events.js]
+#support-files = localStorageEvents.html
diff --git a/toolkit/components/extensions/test/mochitest/mochitest-common.ini b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
index 1f6d278a2c18..03a1e08b3ca2 100644
--- a/toolkit/components/extensions/test/mochitest/mochitest-common.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
@@ -112,11 +112,11 @@ scheme=https
 [test_ext_subframes_privileges.html]
 skip-if = os == 'android' || verify # bug 1489771
 [test_ext_test.html]
-[test_ext_unlimitedStorage.html]
-[test_ext_unlimitedStorage_legacy_persistent_indexedDB.html]
+#[test_ext_unlimitedStorage.html]
+#[test_ext_unlimitedStorage_legacy_persistent_indexedDB.html]
 # IndexedDB persistent storage mode is not allowed on Fennec from a non-chrome privileged code
 # (it has only been enabled for apps and privileged code). See Bug 1119462 for additional info.
-skip-if = os == 'android'
+#skip-if = os == 'android'
 [test_ext_web_accessible_resources.html]
 skip-if = os == 'android' && debug # bug 1397615
 [test_ext_webnavigation.html]
diff --git a/toolkit/components/extensions/test/mochitest/mochitest.ini b/toolkit/components/extensions/test/mochitest/mochitest.ini
index 12b838652671..9fca36408650 100644
--- a/toolkit/components/extensions/test/mochitest/mochitest.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest.ini
@@ -1,7 +1,7 @@
 [DEFAULT]
 tags = webextensions in-process-webextensions
 
-[test_ext_storage_cleanup.html]
+#[test_ext_storage_cleanup.html]
 # Bug 1426514 storage_cleanup: clearing localStorage fails with oop
 
 [include:mochitest-common.ini]
diff --git a/toolkit/forgetaboutsite/moz.build b/toolkit/forgetaboutsite/moz.build
index d18d10231432..c29be29b13b0 100644
--- a/toolkit/forgetaboutsite/moz.build
+++ b/toolkit/forgetaboutsite/moz.build
@@ -5,7 +5,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 BROWSER_CHROME_MANIFESTS += ['test/browser/browser.ini']
-XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
+#XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
 
 EXTRA_JS_MODULES += [
     'ForgetAboutSite.jsm',
diff --git a/toolkit/forgetaboutsite/test/unit/xpcshell.ini b/toolkit/forgetaboutsite/test/unit/xpcshell.ini
index 5267de336e2b..6217234970de 100644
--- a/toolkit/forgetaboutsite/test/unit/xpcshell.ini
+++ b/toolkit/forgetaboutsite/test/unit/xpcshell.ini
@@ -4,4 +4,4 @@ skip-if = toolkit == 'android'
 support-files =
   !/dom/push/test/xpcshell/head.js
 
-[test_removeDataFromDomain.js]
+#[test_removeDataFromDomain.js]

From 9f71846e2a2cad157cce35224bfac1b7acda4dd9 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:24 +0100
Subject: [PATCH 18/78] Bug 1286798 - Part 4: Basic integration with
 QuotaManager; r=asuth

This adds a new quota client implementation, but only implements ShutdownWorkThreads.
At shutdown we wait for all running operations to finish including database actors which are closed by using an extra IPC message which avoids races between the parent and child.
Databases are dropped on the child side as soon as they are not used (e.g. after unlinking by the cycle collector).
---
 dom/localstorage/ActorsChild.cpp            |  12 +
 dom/localstorage/ActorsChild.h              |   3 +
 dom/localstorage/ActorsParent.cpp           | 456 ++++++++++++++++++--
 dom/localstorage/ActorsParent.h             |  13 +
 dom/localstorage/LSDatabase.cpp             |  25 ++
 dom/localstorage/LSDatabase.h               |  13 +
 dom/localstorage/LSObject.cpp               |  42 +-
 dom/localstorage/LSObject.h                 |  10 +
 dom/localstorage/LocalStorageCommon.cpp     |  10 +-
 dom/localstorage/LocalStorageCommon.h       |   3 +
 dom/localstorage/PBackgroundLSDatabase.ipdl |   4 +
 dom/quota/ActorsParent.cpp                  |  27 +-
 dom/quota/Client.h                          |  23 +-
 dom/quota/QuotaManager.h                    |   2 +-
 dom/storage/Storage.cpp                     |   2 +-
 dom/storage/Storage.h                       |   4 +
 16 files changed, 608 insertions(+), 41 deletions(-)

diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 9a9c9e45cc79..8f00d6858cd1 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -57,6 +57,18 @@ LSDatabaseChild::ActorDestroy(ActorDestroyReason aWhy)
   }
 }
 
+mozilla::ipc::IPCResult
+LSDatabaseChild::RecvRequestAllowToClose()
+{
+  AssertIsOnOwningThread();
+
+  if (mDatabase) {
+    mDatabase->AllowToClose();
+  }
+
+  return IPC_OK();
+}
+
 /*******************************************************************************
  * LocalStorageRequestChild
  ******************************************************************************/
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 271d1b921b3d..d7c4034960e6 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -55,6 +55,9 @@ private:
   // IPDL methods are only called by IPDL.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvRequestAllowToClose() override;
 };
 
 class LSRequestChild final
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 77f29ecb53e7..cdaf1cdfbfb5 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -15,8 +15,8 @@
 #include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/ipc/PBackgroundParent.h"
 #include "mozilla/ipc/PBackgroundSharedTypes.h"
+#include "nsClassHashtable.h"
 #include "nsDataHashtable.h"
-#include "nsRefPtrHashtable.h"
 
 #define DISABLE_ASSERTS_FOR_FUZZING 0
 
@@ -171,6 +171,28 @@ private:
   ~Datastore();
 };
 
+class PreparedDatastore
+{
+  RefPtr<Datastore> mDatastore;
+
+public:
+  explicit PreparedDatastore(Datastore* aDatastore)
+    : mDatastore(aDatastore)
+  {
+    AssertIsOnBackgroundThread();
+    MOZ_ASSERT(aDatastore);
+  }
+
+  already_AddRefed<Datastore>
+  ForgetDatastore()
+  {
+    AssertIsOnBackgroundThread();
+    MOZ_ASSERT(mDatastore);
+
+    return mDatastore.forget();
+  }
+};
+
 /*******************************************************************************
  * Actor class declarations
  ******************************************************************************/
@@ -180,8 +202,10 @@ class Database final
 {
   RefPtr<Datastore> mDatastore;
 
-#ifdef DEBUG
+  bool mAllowedToClose;
   bool mActorDestroyed;
+  bool mRequestedAllowToClose;
+#ifdef DEBUG
   bool mActorWasAlive;
 #endif
 
@@ -192,12 +216,18 @@ public:
   void
   SetActorAlive(already_AddRefed<Datastore>&& aDatastore);
 
+  void
+  RequestAllowToClose();
+
   NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Database)
 
 private:
   // Reference counted.
   ~Database();
 
+  void
+  AllowToClose();
+
   // IPDL methods are only called by IPDL.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
@@ -205,6 +235,9 @@ private:
   mozilla::ipc::IPCResult
   RecvDeleteMe() override;
 
+  mozilla::ipc::IPCResult
+  RecvAllowToClose() override;
+
   mozilla::ipc::IPCResult
   RecvGetLength(uint32_t* aLength) override;
 
@@ -304,6 +337,9 @@ private:
   void
   SendResults();
 
+  void
+  Cleanup();
+
   NS_IMETHOD
   Run() override;
 
@@ -312,20 +348,112 @@ private:
   RecvFinish() override;
 };
 
+/*******************************************************************************
+ * Other class declarations
+ ******************************************************************************/
+
+class QuotaClient final
+  : public mozilla::dom::quota::Client
+{
+  static QuotaClient* sInstance;
+
+  bool mShutdownRequested;
+
+public:
+  QuotaClient();
+
+  static bool
+  IsShuttingDownOnBackgroundThread()
+  {
+    AssertIsOnBackgroundThread();
+
+    if (sInstance) {
+      return sInstance->IsShuttingDown();
+    }
+
+    return QuotaManager::IsShuttingDown();
+  }
+
+  static bool
+  IsShuttingDownOnNonBackgroundThread()
+  {
+    MOZ_ASSERT(!IsOnBackgroundThread());
+
+    return QuotaManager::IsShuttingDown();
+  }
+
+  bool
+  IsShuttingDown() const
+  {
+    AssertIsOnBackgroundThread();
+
+    return mShutdownRequested;
+  }
+
+  NS_INLINE_DECL_THREADSAFE_REFCOUNTING(mozilla::dom::QuotaClient, override)
+
+  Type
+  GetType() override;
+
+  nsresult
+  InitOrigin(PersistenceType aPersistenceType,
+             const nsACString& aGroup,
+             const nsACString& aOrigin,
+             const AtomicBool& aCanceled,
+             UsageInfo* aUsageInfo) override;
+
+  nsresult
+  GetUsageForOrigin(PersistenceType aPersistenceType,
+                    const nsACString& aGroup,
+                    const nsACString& aOrigin,
+                    const AtomicBool& aCanceled,
+                    UsageInfo* aUsageInfo) override;
+
+  void
+  OnOriginClearCompleted(PersistenceType aPersistenceType,
+                         const nsACString& aOrigin)
+                         override;
+
+  void
+  ReleaseIOThreadObjects() override;
+
+  void
+  AbortOperations(const nsACString& aOrigin) override;
+
+  void
+  AbortOperationsForProcess(ContentParentId aContentParentId) override;
+
+  void
+  StartIdleMaintenance() override;
+
+  void
+  StopIdleMaintenance() override;
+
+  void
+  ShutdownWorkThreads() override;
+
+private:
+  ~QuotaClient() override;
+};
+
 /*******************************************************************************
  * Globals
  ******************************************************************************/
 
+typedef nsTArray<PrepareDatastoreOp*> PrepareDatastoreOpArray;
+
+StaticAutoPtr<PrepareDatastoreOpArray> gPrepareDatastoreOps;
+
 typedef nsDataHashtable<nsCStringHashKey, Datastore*> DatastoreHashtable;
 
 StaticAutoPtr<DatastoreHashtable> gDatastores;
 
 uint64_t gLastDatastoreId = 0;
 
-typedef nsRefPtrHashtable<nsUint64HashKey, Datastore>
-  TemporaryStrongDatastoreHashtable;
+typedef nsClassHashtable<nsUint64HashKey, PreparedDatastore>
+  PreparedDatastoreHashtable;
 
-StaticAutoPtr<TemporaryStrongDatastoreHashtable> gTemporaryStrongDatastores;
+StaticAutoPtr<PreparedDatastoreHashtable> gPreparedDatastores;
 
 typedef nsTArray<Database*> LiveDatabaseArray;
 
@@ -342,19 +470,23 @@ AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
 {
   AssertIsOnBackgroundThread();
 
-  if (NS_WARN_IF(!gTemporaryStrongDatastores)) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
+    return nullptr;
+  }
+
+  if (NS_WARN_IF(!gPreparedDatastores)) {
     ASSERT_UNLESS_FUZZING();
     return nullptr;
   }
 
-  Datastore* datastore = gTemporaryStrongDatastores->GetWeak(aDatastoreId);
-  if (NS_WARN_IF(!datastore)) {
+  PreparedDatastore* preparedDatastore = gPreparedDatastores->Get(aDatastoreId);
+  if (NS_WARN_IF(!preparedDatastore)) {
     ASSERT_UNLESS_FUZZING();
     return nullptr;
   }
 
   // If we ever decide to return null from this point on, we need to make sure
-  // that the prepared datastore is removed from the gTemporaryStrongDatastores
+  // that the prepared datastore is removed from the gPreparedDatastores
   // hashtable.
   // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
   // once we return a valid actor in this method.
@@ -371,20 +503,21 @@ RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
-  MOZ_ASSERT(gTemporaryStrongDatastores);
-  MOZ_ASSERT(gTemporaryStrongDatastores->GetWeak(aDatastoreId));
+  MOZ_ASSERT(gPreparedDatastores);
+  MOZ_ASSERT(gPreparedDatastores->Get(aDatastoreId));
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
 
   // The actor is now completely built (it has a manager, channel and it's
   // registered as a subprotocol).
   // ActorDestroy will be called if we fail here.
 
-  RefPtr<Datastore> datastore;
-  gTemporaryStrongDatastores->Remove(aDatastoreId, datastore.StartAssignment());
-  MOZ_ASSERT(datastore);
+  nsAutoPtr<PreparedDatastore> preparedDatastore;
+  gPreparedDatastores->Remove(aDatastoreId, &preparedDatastore);
+  MOZ_ASSERT(preparedDatastore);
 
   auto* database = static_cast<Database*>(aActor);
 
-  database->SetActorAlive(datastore.forget());
+  database->SetActorAlive(preparedDatastore->ForgetDatastore());
 
   return true;
 }
@@ -407,6 +540,10 @@ AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
 {
   AssertIsOnBackgroundThread();
 
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
+    return nullptr;
+  }
+
   RefPtr<LSRequestBase> actor;
 
   switch (aParams.type()) {
@@ -430,7 +567,16 @@ AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
         return nullptr;
       }
 
-      actor = new PrepareDatastoreOp(aParams);
+      RefPtr<PrepareDatastoreOp> prepareDatastoreOp =
+        new PrepareDatastoreOp(aParams);
+
+      if (!gPrepareDatastoreOps) {
+        gPrepareDatastoreOps = new PrepareDatastoreOpArray();
+      }
+      gPrepareDatastoreOps->AppendElement(prepareDatastoreOp);
+
+      actor = std::move(prepareDatastoreOp);
+
       break;
     }
 
@@ -449,6 +595,7 @@ RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
   MOZ_ASSERT(aParams.type() != LSRequestParams::T__None);
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
 
   // The actor is now completely built.
 
@@ -471,6 +618,20 @@ DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor)
   return true;
 }
 
+namespace localstorage {
+
+already_AddRefed<mozilla::dom::quota::Client>
+CreateQuotaClient()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(CachedNextGenLocalStorageEnabled());
+
+  RefPtr<QuotaClient> client = new QuotaClient();
+  return client.forget();
+}
+
+} // namespace localstorage
+
 /*******************************************************************************
  * DatastoreOperationBase
  ******************************************************************************/
@@ -570,8 +731,10 @@ Datastore::GetKeys(nsTArray<nsString>& aKeys) const
  ******************************************************************************/
 
 Database::Database()
+  : mAllowedToClose(false)
+  , mActorDestroyed(false)
+  , mRequestedAllowToClose(false)
 #ifdef DEBUG
-  : mActorDestroyed(false)
   , mActorWasAlive(false)
 #endif
 {
@@ -580,6 +743,7 @@ Database::Database()
 
 Database::~Database()
 {
+  MOZ_ASSERT_IF(mActorWasAlive, mAllowedToClose);
   MOZ_ASSERT_IF(mActorWasAlive, mActorDestroyed);
 }
 
@@ -604,15 +768,33 @@ Database::SetActorAlive(already_AddRefed<Datastore>&& aDatastore)
 }
 
 void
-Database::ActorDestroy(ActorDestroyReason aWhy)
+Database::RequestAllowToClose()
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(!mActorDestroyed);
+
+  if (mRequestedAllowToClose) {
+    return;
+  }
+
+  mRequestedAllowToClose = true;
+
+  // Send the RequestAllowToClose message to the child to avoid racing with the
+  // child actor. Except the case when the actor was already destroyed.
+  if (mActorDestroyed) {
+    MOZ_ASSERT(mAllowedToClose);
+  } else {
+    Unused << SendRequestAllowToClose();
+  }
+}
+
+void
+Database::AllowToClose()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mAllowedToClose);
   MOZ_ASSERT(mDatastore);
 
-#ifdef DEBUG
-  mActorDestroyed = true;
-#endif
+  mAllowedToClose = true;
 
   mDatastore = nullptr;
 
@@ -624,6 +806,19 @@ Database::ActorDestroy(ActorDestroyReason aWhy)
   }
 }
 
+void
+Database::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  mActorDestroyed = true;
+
+  if (!mAllowedToClose) {
+    AllowToClose();
+  }
+}
+
 mozilla::ipc::IPCResult
 Database::RecvDeleteMe()
 {
@@ -637,6 +832,21 @@ Database::RecvDeleteMe()
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+Database::RecvAllowToClose()
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  AllowToClose();
+
+  return IPC_OK();
+}
+
 mozilla::ipc::IPCResult
 Database::RecvGetLength(uint32_t* aLength)
 {
@@ -644,6 +854,11 @@ Database::RecvGetLength(uint32_t* aLength)
   MOZ_ASSERT(aLength);
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   *aLength = mDatastore->GetLength();
 
   return IPC_OK();
@@ -656,6 +871,11 @@ Database::RecvGetKey(const uint32_t& aIndex, nsString* aKey)
   MOZ_ASSERT(aKey);
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->GetKey(aIndex, *aKey);
 
   return IPC_OK();
@@ -668,6 +888,11 @@ Database::RecvGetItem(const nsString& aKey, nsString* aValue)
   MOZ_ASSERT(aValue);
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->GetItem(aKey, *aValue);
 
   return IPC_OK();
@@ -679,6 +904,11 @@ Database::RecvSetItem(const nsString& aKey, const nsString& aValue)
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->SetItem(aKey, aValue);
 
   return IPC_OK();
@@ -690,6 +920,11 @@ Database::RecvRemoveItem(const nsString& aKey)
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->RemoveItem(aKey);
 
   return IPC_OK();
@@ -701,6 +936,11 @@ Database::RecvClear()
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->Clear();
 
   return IPC_OK();
@@ -713,6 +953,11 @@ Database::RecvGetKeys(nsTArray<nsString>* aKeys)
   MOZ_ASSERT(aKeys);
   MOZ_ASSERT(mDatastore);
 
+  if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
   mDatastore->GetKeys(*aKeys);
 
   return IPC_OK();
@@ -785,7 +1030,8 @@ PrepareDatastoreOp::OpenOnMainThread()
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(mState == State::OpeningOnMainThread);
 
-  if (!MayProceedOnNonOwningThread()) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
+      !MayProceedOnNonOwningThread()) {
     return NS_ERROR_FAILURE;
   }
 
@@ -828,7 +1074,8 @@ PrepareDatastoreOp::OpenOnOwningThread()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::OpeningOnOwningThread);
 
-  if (!MayProceed()) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
     return NS_ERROR_FAILURE;
   }
 
@@ -857,6 +1104,8 @@ PrepareDatastoreOp::SendReadyMessage()
   if (!MayProceed()) {
     MaybeSetFailureCode(NS_ERROR_FAILURE);
 
+    Cleanup();
+
     mState = State::Completed;
   } else {
     Unused << SendReady();
@@ -895,10 +1144,13 @@ PrepareDatastoreOp::SendResults()
 
       uint64_t datastoreId = ++gLastDatastoreId;
 
-      if (!gTemporaryStrongDatastores) {
-        gTemporaryStrongDatastores = new TemporaryStrongDatastoreHashtable();
+      nsAutoPtr<PreparedDatastore> preparedDatastore(
+        new PreparedDatastore(datastore));
+
+      if (!gPreparedDatastores) {
+        gPreparedDatastores = new PreparedDatastoreHashtable();
       }
-      gTemporaryStrongDatastores->Put(datastoreId, datastore);
+      gPreparedDatastores->Put(datastoreId, preparedDatastore.forget());
 
       LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
       prepareDatastoreResponse.datastoreId() = datastoreId;
@@ -912,9 +1164,24 @@ PrepareDatastoreOp::SendResults()
       PBackgroundLSRequestParent::Send__delete__(this, response);
   }
 
+  Cleanup();
+
   mState = State::Completed;
 }
 
+void
+PrepareDatastoreOp::Cleanup()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_ASSERT(gPrepareDatastoreOps);
+  gPrepareDatastoreOps->RemoveElement(this);
+
+  if (gPrepareDatastoreOps->IsEmpty()) {
+    gPrepareDatastoreOps = nullptr;
+  }
+}
+
 NS_IMETHODIMP
 PrepareDatastoreOp::Run()
 {
@@ -971,5 +1238,140 @@ PrepareDatastoreOp::RecvFinish()
   return IPC_OK();
 }
 
+/*******************************************************************************
+ * QuotaClient
+ ******************************************************************************/
+
+QuotaClient* QuotaClient::sInstance = nullptr;
+
+QuotaClient::QuotaClient()
+  : mShutdownRequested(false)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!sInstance, "We expect this to be a singleton!");
+
+  sInstance = this;
+}
+
+QuotaClient::~QuotaClient()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(sInstance == this, "We expect this to be a singleton!");
+
+  sInstance = nullptr;
+}
+
+mozilla::dom::quota::Client::Type
+QuotaClient::GetType()
+{
+  return QuotaClient::LS;
+}
+
+nsresult
+QuotaClient::InitOrigin(PersistenceType aPersistenceType,
+                        const nsACString& aGroup,
+                        const nsACString& aOrigin,
+                        const AtomicBool& aCanceled,
+                        UsageInfo* aUsageInfo)
+{
+  AssertIsOnIOThread();
+
+  if (!aUsageInfo) {
+    return NS_OK;
+  }
+
+  return GetUsageForOrigin(aPersistenceType,
+                           aGroup,
+                           aOrigin,
+                           aCanceled,
+                           aUsageInfo);
+}
+
+nsresult
+QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
+                               const nsACString& aGroup,
+                               const nsACString& aOrigin,
+                               const AtomicBool& aCanceled,
+                               UsageInfo* aUsageInfo)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aUsageInfo);
+
+  return NS_OK;
+}
+
+void
+QuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType,
+                                    const nsACString& aOrigin)
+{
+  AssertIsOnIOThread();
+}
+
+void
+QuotaClient::ReleaseIOThreadObjects()
+{
+  AssertIsOnIOThread();
+}
+
+void
+QuotaClient::AbortOperations(const nsACString& aOrigin)
+{
+  AssertIsOnBackgroundThread();
+}
+
+void
+QuotaClient::AbortOperationsForProcess(ContentParentId aContentParentId)
+{
+  AssertIsOnBackgroundThread();
+}
+
+void
+QuotaClient::StartIdleMaintenance()
+{
+  AssertIsOnBackgroundThread();
+}
+
+void
+QuotaClient::StopIdleMaintenance()
+{
+  AssertIsOnBackgroundThread();
+}
+
+void
+QuotaClient::ShutdownWorkThreads()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mShutdownRequested);
+
+  mShutdownRequested = true;
+
+  // gPrepareDatastoreOps are short lived objects running a state machine.
+  // The shutdown flag is checked between states, so we don't have to notify
+  // all the objects here.
+  // Allocation of a new PrepareDatastoreOp object is prevented once the
+  // shutdown flag is set.
+  // When the last PrepareDatastoreOp finishes, the gPrepareDatastoreOps array
+  // is destroyed.
+
+  // If database actors haven't been created yet, don't do anything special.
+  // We are shutting down and we can release prepared datastores immediatelly
+  // since database actors will never be created for them.
+  if (gPreparedDatastores) {
+    gPreparedDatastores->Clear();
+    gPreparedDatastores = nullptr;
+  }
+
+  if (gLiveDatabases) {
+    for (Database* database : *gLiveDatabases) {
+      database->RequestAllowToClose();
+    }
+  }
+
+  MOZ_ALWAYS_TRUE(SpinEventLoopUntil([&]() {
+    // Don't have to check gPreparedDatastores since we nulled it out above.
+    return !gPrepareDatastoreOps && !gDatastores && !gLiveDatabases;
+  }));
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/ActorsParent.h b/dom/localstorage/ActorsParent.h
index 1fdcf4eb0e8d..4e505ea9fffd 100644
--- a/dom/localstorage/ActorsParent.h
+++ b/dom/localstorage/ActorsParent.h
@@ -21,6 +21,12 @@ class LSRequestParams;
 class PBackgroundLSDatabaseParent;
 class PBackgroundLSRequestParent;
 
+namespace quota {
+
+class Client;
+
+} // namespace quota
+
 PBackgroundLSDatabaseParent*
 AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId);
 
@@ -43,6 +49,13 @@ RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
 bool
 DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor);
 
+namespace localstorage {
+
+already_AddRefed<mozilla::dom::quota::Client>
+CreateQuotaClient();
+
+} // namespace localstorage
+
 } // namespace dom
 } // namespace mozilla
 
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index 0a9e1a5a5ddf..e5735fef9aa3 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -11,6 +11,7 @@ namespace dom {
 
 LSDatabase::LSDatabase()
   : mActor(nullptr)
+  , mAllowedToClose(false)
 {
   AssertIsOnOwningThread();
 }
@@ -19,6 +20,10 @@ LSDatabase::~LSDatabase()
 {
   AssertIsOnOwningThread();
 
+  if (!mAllowedToClose) {
+    AllowToClose();
+  }
+
   if (mActor) {
     mActor->SendDeleteMeInternal();
     MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
@@ -35,11 +40,25 @@ LSDatabase::SetActor(LSDatabaseChild* aActor)
   mActor = aActor;
 }
 
+void
+LSDatabase::AllowToClose()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!mAllowedToClose);
+
+  mAllowedToClose = true;
+
+  if (mActor) {
+    mActor->SendAllowToClose();
+  }
+}
+
 nsresult
 LSDatabase::GetLength(uint32_t* aResult)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   uint32_t result;
   if (NS_WARN_IF(!mActor->SendGetLength(&result))) {
@@ -56,6 +75,7 @@ LSDatabase::GetKey(uint32_t aIndex,
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   nsString result;
   if (NS_WARN_IF(!mActor->SendGetKey(aIndex, &result))) {
@@ -72,6 +92,7 @@ LSDatabase::GetItem(const nsAString& aKey,
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   nsString result;
   if (NS_WARN_IF(!mActor->SendGetItem(nsString(aKey), &result))) {
@@ -87,6 +108,7 @@ LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   nsTArray<nsString> result;
   if (NS_WARN_IF(!mActor->SendGetKeys(&result))) {
@@ -103,6 +125,7 @@ LSDatabase::SetItem(const nsAString& aKey,
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey), nsString(aValue)))) {
     return NS_ERROR_FAILURE;
@@ -116,6 +139,7 @@ LSDatabase::RemoveItem(const nsAString& aKey)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey)))) {
     return NS_ERROR_FAILURE;
@@ -129,6 +153,7 @@ LSDatabase::Clear()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
 
   if (NS_WARN_IF(!mActor->SendClear())) {
     return NS_ERROR_FAILURE;
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index 7b1402336eed..8446f4a4b9e0 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -16,6 +16,8 @@ class LSDatabase final
 {
   LSDatabaseChild* mActor;
 
+  bool mAllowedToClose;
+
 public:
   LSDatabase();
 
@@ -39,6 +41,17 @@ public:
     mActor = nullptr;
   }
 
+  void
+  AllowToClose();
+
+  bool
+  IsAllowedToClose() const
+  {
+    AssertIsOnOwningThread();
+
+    return mAllowedToClose;
+  }
+
   nsresult
   GetLength(uint32_t* aResult);
 
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 8f3f25bff4c9..d623ead8ae9b 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -392,15 +392,34 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
   }
 }
 
+NS_IMPL_ADDREF_INHERITED(LSObject, Storage)
+NS_IMPL_RELEASE_INHERITED(LSObject, Storage)
+
+NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(LSObject)
+NS_INTERFACE_MAP_END_INHERITING(Storage)
+
+NS_IMPL_CYCLE_COLLECTION_CLASS(LSObject)
+
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(LSObject, Storage)
+  tmp->AssertIsOnOwningThread();
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
+
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(LSObject, Storage)
+  tmp->AssertIsOnOwningThread();
+  tmp->DropDatabase();
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END
+
 nsresult
 LSObject::EnsureDatabase()
 {
   AssertIsOnOwningThread();
 
-  if (mDatabase) {
+  if (mDatabase && !mDatabase->IsAllowedToClose()) {
     return NS_OK;
   }
 
+  mDatabase = nullptr;
+
   // We don't need this yet, but once the request successfully finishes, it's
   // too late to initialize PBackground child on the owning thread, because
   // it can fail and parent would keep an extra strong ref to the datastore.
@@ -457,6 +476,27 @@ LSObject::EnsureDatabase()
   return NS_OK;
 }
 
+void
+LSObject::DropDatabase()
+{
+  AssertIsOnOwningThread();
+
+  if (mDatabase) {
+    if (!mDatabase->IsAllowedToClose()) {
+      mDatabase->AllowToClose();
+    }
+    mDatabase = nullptr;
+  }
+}
+
+void
+LSObject::LastRelease()
+{
+  AssertIsOnOwningThread();
+
+  DropDatabase();
+}
+
 nsresult
 RequestHelper::StartAndReturnResponse(LSRequestResponse& aResponse)
 {
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index 1e6ab93f1af4..a4d8f797b6c3 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -94,6 +94,9 @@ public:
   Clear(nsIPrincipal& aSubjectPrincipal,
         ErrorResult& aError) override;
 
+  NS_DECL_ISUPPORTS_INHERITED
+  NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(LSObject, Storage)
+
 private:
   LSObject(nsPIDOMWindowInner* aWindow,
            nsIPrincipal* aPrincipal);
@@ -102,6 +105,13 @@ private:
 
   nsresult
   EnsureDatabase();
+
+  void
+  DropDatabase();
+
+  // Storage overrides.
+  void
+  LastRelease() override;
 };
 
 } // namespace dom
diff --git a/dom/localstorage/LocalStorageCommon.cpp b/dom/localstorage/LocalStorageCommon.cpp
index 026331cf00aa..288c5a570358 100644
--- a/dom/localstorage/LocalStorageCommon.cpp
+++ b/dom/localstorage/LocalStorageCommon.cpp
@@ -11,7 +11,7 @@ namespace dom {
 
 namespace {
 
-int32_t gNextGenLocalStorageEnabled = -1;
+Atomic<int32_t> gNextGenLocalStorageEnabled(-1);
 
 } // namespace
 
@@ -28,5 +28,13 @@ NextGenLocalStorageEnabled()
   return !!gNextGenLocalStorageEnabled;
 }
 
+bool
+CachedNextGenLocalStorageEnabled()
+{
+  MOZ_ASSERT(gNextGenLocalStorageEnabled != -1);
+
+  return !!gNextGenLocalStorageEnabled;
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/LocalStorageCommon.h b/dom/localstorage/LocalStorageCommon.h
index f65e6ad6d89d..53553b9dcbac 100644
--- a/dom/localstorage/LocalStorageCommon.h
+++ b/dom/localstorage/LocalStorageCommon.h
@@ -187,6 +187,9 @@ namespace dom {
 bool
 NextGenLocalStorageEnabled();
 
+bool
+CachedNextGenLocalStorageEnabled();
+
 } // namespace dom
 } // namespace mozilla
 
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 882d5f42ef39..5a3724c81af5 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -23,6 +23,8 @@ parent:
   // and the parent then sends the __delete__ message to the child.
   async DeleteMe();
 
+  async AllowToClose();
+
   sync GetLength()
     returns (uint32_t length);
 
@@ -43,6 +45,8 @@ parent:
 
 child:
   async __delete__();
+
+  async RequestAllowToClose();
 };
 
 } // namespace dom
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 7a9df9c283f9..280aeb0b84a5 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -34,6 +34,7 @@
 #include "mozilla/dom/asmjscache/AsmJSCache.h"
 #include "mozilla/dom/cache/QuotaClient.h"
 #include "mozilla/dom/indexedDB/ActorsParent.h"
+#include "mozilla/dom/localstorage/ActorsParent.h"
 #include "mozilla/dom/quota/PQuotaParent.h"
 #include "mozilla/dom/quota/PQuotaRequestParent.h"
 #include "mozilla/dom/quota/PQuotaUsageRequestParent.h"
@@ -2561,7 +2562,7 @@ DirectoryLockImpl::DirectoryLockImpl(QuotaManager* aQuotaManager,
   MOZ_ASSERT_IF(!aInternal, !aGroup.IsEmpty());
   MOZ_ASSERT_IF(!aInternal, aOriginScope.IsOrigin());
   MOZ_ASSERT_IF(!aInternal, !aClientType.IsNull());
-  MOZ_ASSERT_IF(!aInternal, aClientType.Value() != Client::TYPE_MAX);
+  MOZ_ASSERT_IF(!aInternal, aClientType.Value() < Client::TypeMax());
   MOZ_ASSERT_IF(!aInternal, aOpenListener);
 }
 
@@ -2666,6 +2667,8 @@ CreateRunnable::Init()
     return rv;
   }
 
+  Unused << NextGenLocalStorageEnabled();
+
   return NS_OK;
 }
 
@@ -3299,7 +3302,7 @@ QuotaManager::CreateDirectoryLock(const Nullable<PersistenceType>& aPersistenceT
   MOZ_ASSERT_IF(!aInternal, !aGroup.IsEmpty());
   MOZ_ASSERT_IF(!aInternal, aOriginScope.IsOrigin());
   MOZ_ASSERT_IF(!aInternal, !aClientType.IsNull());
-  MOZ_ASSERT_IF(!aInternal, aClientType.Value() != Client::TYPE_MAX);
+  MOZ_ASSERT_IF(!aInternal, aClientType.Value() < Client::TypeMax());
   MOZ_ASSERT_IF(!aInternal, aOpenListener);
 
   RefPtr<DirectoryLockImpl> lock = new DirectoryLockImpl(this,
@@ -3647,7 +3650,8 @@ QuotaManager::Init(const nsAString& aBasePath)
                 Client::ASMJS == 1 &&
                 Client::DOMCACHE == 2 &&
                 Client::SDB == 3 &&
-                Client::TYPE_MAX == 4,
+                Client::LS == 4 &&
+                Client::TYPE_MAX == 5,
                 "Fix the registration!");
 
   MOZ_ASSERT(mClients.Capacity() == Client::TYPE_MAX,
@@ -3658,6 +3662,11 @@ QuotaManager::Init(const nsAString& aBasePath)
   mClients.AppendElement(asmjscache::CreateClient());
   mClients.AppendElement(cache::CreateQuotaClient());
   mClients.AppendElement(simpledb::CreateQuotaClient());
+  if (CachedNextGenLocalStorageEnabled()) {
+    mClients.AppendElement(localstorage::CreateQuotaClient());
+  } else {
+    mClients.SetLength(Client::TypeMax());
+  }
 
   return NS_OK;
 }
@@ -3685,7 +3694,7 @@ QuotaManager::Shutdown()
 
   // Each client will spin the event loop while we wait on all the threads
   // to close. Our timer may fire during that loop.
-  for (uint32_t index = 0; index < Client::TYPE_MAX; index++) {
+  for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
     mClients[index]->ShutdownWorkThreads();
   }
 
@@ -5222,7 +5231,7 @@ QuotaManager::OpenDirectoryInternal(const Nullable<PersistenceType>& aPersistenc
   // We also need to notify clients to abort operations for them.
   AutoTArray<nsAutoPtr<nsTHashtable<nsCStringHashKey>>,
                Client::TYPE_MAX> origins;
-  origins.SetLength(Client::TYPE_MAX);
+  origins.SetLength(Client::TypeMax());
 
   const nsTArray<DirectoryLockImpl*>& blockedOnLocks =
     lock->GetBlockedOnLocks();
@@ -5233,7 +5242,7 @@ QuotaManager::OpenDirectoryInternal(const Nullable<PersistenceType>& aPersistenc
     if (!blockedOnLock->IsInternal()) {
       MOZ_ASSERT(!blockedOnLock->GetClientType().IsNull());
       Client::Type clientType = blockedOnLock->GetClientType().Value();
-      MOZ_ASSERT(clientType < Client::TYPE_MAX);
+      MOZ_ASSERT(clientType < Client::TypeMax());
 
       const OriginScope& originScope = blockedOnLock->GetOriginScope();
       MOZ_ASSERT(originScope.IsOrigin());
@@ -5247,7 +5256,7 @@ QuotaManager::OpenDirectoryInternal(const Nullable<PersistenceType>& aPersistenc
     }
   }
 
-  for (uint32_t index : IntegerRange(uint32_t(Client::TYPE_MAX))) {
+  for (uint32_t index : IntegerRange(uint32_t(Client::TypeMax()))) {
     if (origins[index]) {
       for (auto iter = origins[index]->Iter(); !iter.Done(); iter.Next()) {
         MOZ_ASSERT(mClients[index]);
@@ -5486,7 +5495,7 @@ QuotaManager::OriginClearCompleted(PersistenceType aPersistenceType,
     mInitializedOrigins.RemoveElement(aOrigin);
   }
 
-  for (uint32_t index = 0; index < Client::TYPE_MAX; index++) {
+  for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
     mClients[index]->OnOriginClearCompleted(aPersistenceType, aOrigin);
   }
 }
@@ -5507,7 +5516,7 @@ Client*
 QuotaManager::GetClient(Client::Type aClientType)
 {
   MOZ_ASSERT(aClientType >= Client::IDB);
-  MOZ_ASSERT(aClientType < Client::TYPE_MAX);
+  MOZ_ASSERT(aClientType < Client::TypeMax());
 
   return mClients.ElementAt(aClientType);
 }
diff --git a/dom/quota/Client.h b/dom/quota/Client.h
index dce4762b1c30..7a92b208886d 100644
--- a/dom/quota/Client.h
+++ b/dom/quota/Client.h
@@ -9,6 +9,7 @@
 
 #include "mozilla/dom/quota/QuotaCommon.h"
 
+#include "mozilla/dom/LocalStorageCommon.h"
 #include "mozilla/dom/ipc/IdType.h"
 
 #include "PersistenceType.h"
@@ -39,14 +40,23 @@ public:
 
   enum Type {
     IDB = 0,
-    //LS,
     //APPCACHE,
     ASMJS,
     DOMCACHE,
     SDB,
+    LS,
     TYPE_MAX
   };
 
+  static Type
+  TypeMax()
+  {
+    if (CachedNextGenLocalStorageEnabled()) {
+      return TYPE_MAX;
+    }
+    return LS;
+  }
+
   virtual Type
   GetType() = 0;
 
@@ -70,6 +80,13 @@ public:
         aText.AssignLiteral(SDB_DIRECTORY_NAME);
         break;
 
+      case LS:
+        if (CachedNextGenLocalStorageEnabled()) {
+          aText.AssignLiteral(LS_DIRECTORY_NAME);
+          break;
+        }
+        MOZ_FALLTHROUGH;
+
       case TYPE_MAX:
       default:
         MOZ_ASSERT_UNREACHABLE("Bad id value!");
@@ -94,6 +111,10 @@ public:
     else if (aText.EqualsLiteral(SDB_DIRECTORY_NAME)) {
       aType = SDB;
     }
+    else if (CachedNextGenLocalStorageEnabled() &&
+             aText.EqualsLiteral(LS_DIRECTORY_NAME)) {
+      aType = LS;
+    }
     else {
       return NS_ERROR_FAILURE;
     }
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index 7085aa9a0a2b..8b9705cbb233 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -524,7 +524,7 @@ private:
   {
     AssertIsOnIOThread();
 
-    for (uint32_t index = 0; index < Client::TYPE_MAX; index++) {
+    for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
       mClients[index]->ReleaseIOThreadObjects();
     }
   }
diff --git a/dom/storage/Storage.cpp b/dom/storage/Storage.cpp
index 79aa0a77ef36..f54c06050b13 100644
--- a/dom/storage/Storage.cpp
+++ b/dom/storage/Storage.cpp
@@ -19,7 +19,7 @@ static const char kStorageEnabled[] = "dom.storage.enabled";
 NS_IMPL_CYCLE_COLLECTION_WRAPPERCACHE(Storage, mWindow, mPrincipal)
 
 NS_IMPL_CYCLE_COLLECTING_ADDREF(Storage)
-NS_IMPL_CYCLE_COLLECTING_RELEASE(Storage)
+NS_IMPL_CYCLE_COLLECTING_RELEASE_WITH_LAST_RELEASE(Storage, LastRelease())
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(Storage)
   NS_WRAPPERCACHE_INTERFACE_MAP_ENTRY
diff --git a/dom/storage/Storage.h b/dom/storage/Storage.h
index 550f83ac85e3..b4a9cf4cbf8a 100644
--- a/dom/storage/Storage.h
+++ b/dom/storage/Storage.h
@@ -138,6 +138,10 @@ protected:
   // hand together).
   bool CanUseStorage(nsIPrincipal& aSubjectPrincipal);
 
+  virtual void
+  LastRelease()
+  { }
+
 private:
   nsCOMPtr<nsPIDOMWindowInner> mWindow;
   nsCOMPtr<nsIPrincipal> mPrincipal;

From c5676a58c73b5ab274305ead293329f39081205c Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:27 +0100
Subject: [PATCH 19/78] Bug 1286798 - Part 5: More integration with
 QuotaManager; r=asuth

Preparation of datastores now creates real database files on disk. The LocalStorage directory is protected by a directory lock.
Infrastructure for database schema upgrades is in place too.
Database actors are now explicitely tracked by the datastore. When the last actor finishes the directory lock is released.
Added also implementation for QuotaClient::GetUsageForOrigin() and QuotaClient::AbortOperations().
---
 .../test/browser/browser.ini                  |    2 +-
 dom/localstorage/ActorsChild.cpp              |    5 +
 dom/localstorage/ActorsParent.cpp             | 1191 ++++++++++++++++-
 dom/localstorage/LSObject.cpp                 |    6 +
 dom/localstorage/ReportInternalError.cpp      |   36 +
 dom/localstorage/ReportInternalError.h        |   32 +
 dom/localstorage/moz.build                    |    1 +
 dom/quota/ActorsParent.cpp                    |    4 +
 .../test/unit/test_removeLocalStorage.js      |    4 +
 .../extensions/test/mochitest/mochitest.ini   |    2 +-
 .../test/xpcshell/xpcshell-common.ini         |    2 +-
 11 files changed, 1253 insertions(+), 32 deletions(-)
 create mode 100644 dom/localstorage/ReportInternalError.cpp
 create mode 100644 dom/localstorage/ReportInternalError.h

diff --git a/browser/components/contextualidentity/test/browser/browser.ini b/browser/components/contextualidentity/test/browser/browser.ini
index 451bcd062722..7cc9cd57c6ec 100644
--- a/browser/components/contextualidentity/test/browser/browser.ini
+++ b/browser/components/contextualidentity/test/browser/browser.ini
@@ -10,7 +10,7 @@ support-files =
 [browser_aboutURLs.js]
 [browser_eme.js]
 [browser_favicon.js]
-#[browser_forgetaboutsite.js]
+[browser_forgetaboutsite.js]
 [browser_forgetAPI_cookie_getCookiesWithOriginAttributes.js]
 [browser_restore_getCookiesWithOriginAttributes.js]
 [browser_forgetAPI_EME_forgetThisSite.js]
diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 8f00d6858cd1..788664ffbf97 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -64,6 +64,11 @@ LSDatabaseChild::RecvRequestAllowToClose()
 
   if (mDatabase) {
     mDatabase->AllowToClose();
+
+    // TODO: A new datastore will be prepared at first LocalStorage API
+    //       synchronous call. It would be better to start preparing a new
+    //       datastore right here, but asynchronously.
+    //       However, we probably shouldn't do that if we are shutting down.
   }
 
   return IPC_OK();
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index cdaf1cdfbfb5..b9f421e379a3 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -7,16 +7,23 @@
 #include "ActorsParent.h"
 
 #include "LocalStorageCommon.h"
+#include "mozIStorageConnection.h"
+#include "mozIStorageService.h"
+#include "mozStorageCID.h"
+#include "mozStorageHelper.h"
 #include "mozilla/Unused.h"
 #include "mozilla/dom/PBackgroundLSDatabaseParent.h"
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
 #include "mozilla/dom/quota/QuotaManager.h"
+#include "mozilla/dom/quota/UsageInfo.h"
 #include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/ipc/PBackgroundParent.h"
 #include "mozilla/ipc/PBackgroundSharedTypes.h"
 #include "nsClassHashtable.h"
 #include "nsDataHashtable.h"
+#include "nsISimpleEnumerator.h"
+#include "ReportInternalError.h"
 
 #define DISABLE_ASSERTS_FOR_FUZZING 0
 
@@ -26,6 +33,10 @@
 #define ASSERT_UNLESS_FUZZING(...) MOZ_ASSERT(false, __VA_ARGS__)
 #endif
 
+#if defined(MOZ_WIDGET_ANDROID)
+#define LS_MOBILE
+#endif
+
 namespace mozilla {
 namespace dom {
 
@@ -34,6 +45,352 @@ using namespace mozilla::ipc;
 
 namespace {
 
+class Database;
+
+/*******************************************************************************
+ * Constants
+ ******************************************************************************/
+
+// Major schema version. Bump for almost everything.
+const uint32_t kMajorSchemaVersion = 1;
+
+// Minor schema version. Should almost always be 0 (maybe bump on release
+// branches if we have to).
+const uint32_t kMinorSchemaVersion = 0;
+
+// The schema version we store in the SQLite database is a (signed) 32-bit
+// integer. The major version is left-shifted 4 bits so the max value is
+// 0xFFFFFFF. The minor version occupies the lower 4 bits and its max is 0xF.
+static_assert(kMajorSchemaVersion <= 0xFFFFFFF,
+              "Major version needs to fit in 28 bits.");
+static_assert(kMinorSchemaVersion <= 0xF,
+              "Minor version needs to fit in 4 bits.");
+
+const int32_t kSQLiteSchemaVersion =
+  int32_t((kMajorSchemaVersion << 4) + kMinorSchemaVersion);
+
+// Changing the value here will override the page size of new databases only.
+// A journal mode change and VACUUM are needed to change existing databases, so
+// the best way to do that is to use the schema version upgrade mechanism.
+const uint32_t kSQLitePageSizeOverride =
+#ifdef LS_MOBILE
+  512;
+#else
+  1024;
+#endif
+
+static_assert(kSQLitePageSizeOverride == /* mozStorage default */ 0 ||
+              (kSQLitePageSizeOverride % 2 == 0 &&
+               kSQLitePageSizeOverride >= 512  &&
+               kSQLitePageSizeOverride <= 65536),
+              "Must be 0 (disabled) or a power of 2 between 512 and 65536!");
+
+// Set to some multiple of the page size to grow the database in larger chunks.
+const uint32_t kSQLiteGrowthIncrement = kSQLitePageSizeOverride * 2;
+
+static_assert(kSQLiteGrowthIncrement >= 0 &&
+              kSQLiteGrowthIncrement % kSQLitePageSizeOverride == 0 &&
+              kSQLiteGrowthIncrement < uint32_t(INT32_MAX),
+              "Must be 0 (disabled) or a positive multiple of the page size!");
+
+#define DATA_FILE_NAME "data.sqlite"
+#define JOURNAL_FILE_NAME "data.sqlite-journal"
+
+/*******************************************************************************
+ * SQLite functions
+ ******************************************************************************/
+
+#if 0
+int32_t
+MakeSchemaVersion(uint32_t aMajorSchemaVersion,
+                  uint32_t aMinorSchemaVersion)
+{
+  return int32_t((aMajorSchemaVersion << 4) + aMinorSchemaVersion);
+}
+#endif
+
+nsresult
+CreateTables(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  // Table `database`
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "CREATE TABLE database"
+      "( origin TEXT NOT NULL"
+      ", last_vacuum_time INTEGER NOT NULL DEFAULT 0"
+      ", last_analyze_time INTEGER NOT NULL DEFAULT 0"
+      ", last_vacuum_size INTEGER NOT NULL DEFAULT 0"
+      ");"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // Table `data`
+  rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "CREATE TABLE data"
+      "( key TEXT PRIMARY KEY"
+      ", value TEXT NOT NULL"
+      ", compressed INTEGER NOT NULL DEFAULT 0"
+      ", lastAccessTime INTEGER NOT NULL DEFAULT 0"
+      ");"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->SetSchemaVersion(kSQLiteSchemaVersion);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+#if 0
+nsresult
+UpgradeSchemaFrom1_0To2_0(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->SetSchemaVersion(MakeSchemaVersion(2, 0));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+#endif
+
+nsresult
+SetDefaultPragmas(mozIStorageConnection* aConnection)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "PRAGMA synchronous = FULL;"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+#ifndef LS_MOBILE
+  if (kSQLiteGrowthIncrement) {
+    // This is just an optimization so ignore the failure if the disk is
+    // currently too full.
+    rv = aConnection->SetGrowthIncrement(kSQLiteGrowthIncrement,
+                                         EmptyCString());
+    if (rv != NS_ERROR_FILE_TOO_BIG && NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+#endif // LS_MOBILE
+
+  return NS_OK;
+}
+
+nsresult
+CreateStorageConnection(nsIFile* aDBFile,
+                        const nsACString& aOrigin,
+                        mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aDBFile);
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv;
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenDatabase(aDBFile, getter_AddRefs(connection));
+  if (rv == NS_ERROR_FILE_CORRUPTED) {
+    // Nuke the database file.
+    rv = aDBFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = ss->OpenDatabase(aDBFile, getter_AddRefs(connection));
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = SetDefaultPragmas(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // Check to make sure that the database schema is correct.
+  int32_t schemaVersion;
+  rv = connection->GetSchemaVersion(&schemaVersion);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (schemaVersion > kSQLiteSchemaVersion) {
+    LS_WARNING("Unable to open LocalStorage database, schema is too high!");
+    return NS_ERROR_FAILURE;
+  }
+
+  if (schemaVersion != kSQLiteSchemaVersion) {
+    const bool newDatabase = !schemaVersion;
+
+    if (newDatabase) {
+      // Set the page size first.
+      if (kSQLitePageSizeOverride) {
+        rv = connection->ExecuteSimpleSQL(
+          nsPrintfCString("PRAGMA page_size = %" PRIu32 ";", kSQLitePageSizeOverride)
+        );
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          return rv;
+        }
+      }
+
+      // We have to set the auto_vacuum mode before opening a transaction.
+      rv = connection->ExecuteSimpleSQL(
+#ifdef LS_MOBILE
+        // Turn on full auto_vacuum mode to reclaim disk space on mobile
+        // devices (at the cost of some COMMIT speed).
+        NS_LITERAL_CSTRING("PRAGMA auto_vacuum = FULL;")
+#else
+        // Turn on incremental auto_vacuum mode on desktop builds.
+        NS_LITERAL_CSTRING("PRAGMA auto_vacuum = INCREMENTAL;")
+#endif
+      );
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    mozStorageTransaction transaction(connection, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+    if (newDatabase) {
+      rv = CreateTables(connection);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      MOZ_ASSERT(NS_SUCCEEDED(connection->GetSchemaVersion(&schemaVersion)));
+      MOZ_ASSERT(schemaVersion == kSQLiteSchemaVersion);
+
+      nsCOMPtr<mozIStorageStatement> stmt;
+      nsresult rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+        "INSERT INTO database (origin) "
+        "VALUES (:origin)"
+      ), getter_AddRefs(stmt));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("origin"), aOrigin);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = stmt->Execute();
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    } else  {
+      // This logic needs to change next time we change the schema!
+      static_assert(kSQLiteSchemaVersion == int32_t((1 << 4) + 0),
+                    "Upgrade function needed due to schema version increase.");
+
+      while (schemaVersion != kSQLiteSchemaVersion) {
+#if 0
+        if (schemaVersion == MakeSchemaVersion(1, 0)) {
+          rv = UpgradeSchemaFrom1_0To2_0(connection);
+        } else {
+#endif
+          LS_WARNING("Unable to open LocalStorage database, no upgrade path is "
+                     "available!");
+          return NS_ERROR_FAILURE;
+#if 0
+        }
+#endif
+
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          return rv;
+        }
+
+        rv = connection->GetSchemaVersion(&schemaVersion);
+        if (NS_WARN_IF(NS_FAILED(rv))) {
+          return rv;
+        }
+      }
+
+      MOZ_ASSERT(schemaVersion == kSQLiteSchemaVersion);
+    }
+
+    rv = transaction.Commit();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (newDatabase) {
+      // Windows caches the file size, let's force it to stat the file again.
+      bool dummy;
+      rv = aDBFile->Exists(&dummy);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      int64_t fileSize;
+      rv = aDBFile->GetFileSize(&fileSize);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      MOZ_ASSERT(fileSize > 0);
+
+      PRTime vacuumTime = PR_Now();
+      MOZ_ASSERT(vacuumTime);
+
+      nsCOMPtr<mozIStorageStatement> vacuumTimeStmt;
+      rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+        "UPDATE database "
+          "SET last_vacuum_time = :time"
+            ", last_vacuum_size = :size;"
+      ), getter_AddRefs(vacuumTimeStmt));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = vacuumTimeStmt->BindInt64ByName(NS_LITERAL_CSTRING("time"),
+                                           vacuumTime);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = vacuumTimeStmt->BindInt64ByName(NS_LITERAL_CSTRING("size"),
+                                           fileSize);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = vacuumTimeStmt->Execute();
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Non-actor class declarations
  ******************************************************************************/
@@ -130,12 +487,16 @@ protected:
 
 class Datastore final
 {
+  RefPtr<DirectoryLock> mDirectoryLock;
+  nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const nsCString mOrigin;
+  bool mClosed;
 
 public:
   // Created by PrepareDatastoreOp.
-  explicit Datastore(const nsACString& aOrigin);
+  Datastore(const nsACString& aOrigin,
+            already_AddRefed<DirectoryLock>&& aDirectoryLock);
 
   const nsCString&
   Origin() const
@@ -143,6 +504,26 @@ public:
     return mOrigin;
   }
 
+  void
+  Close();
+
+  bool
+  IsClosed() const
+  {
+    AssertIsOnBackgroundThread();
+
+    return mClosed;
+  }
+
+  void
+  NoteLiveDatabase(Database* aDatabase);
+
+  void
+  NoteFinishedDatabase(Database* aDatabase);
+
+  bool
+  HasLiveDatabases() const;
+
   uint32_t
   GetLength() const;
 
@@ -174,10 +555,17 @@ private:
 class PreparedDatastore
 {
   RefPtr<Datastore> mDatastore;
+  // Strings share buffers if possible, so it's not a problem to duplicate the
+  // origin here.
+  const nsCString mOrigin;
+  bool mInvalidated;
 
 public:
-  explicit PreparedDatastore(Datastore* aDatastore)
+  PreparedDatastore(Datastore* aDatastore,
+                    const nsACString& aOrigin)
     : mDatastore(aDatastore)
+    , mOrigin(aOrigin)
+    , mInvalidated(false)
   {
     AssertIsOnBackgroundThread();
     MOZ_ASSERT(aDatastore);
@@ -191,6 +579,28 @@ public:
 
     return mDatastore.forget();
   }
+
+  const nsCString&
+  Origin() const
+  {
+    return mOrigin;
+  }
+
+  void
+  Invalidate()
+  {
+    AssertIsOnBackgroundThread();
+
+    mInvalidated = true;
+  }
+
+  bool
+  IsInvalidated() const
+  {
+    AssertIsOnBackgroundThread();
+
+    return mInvalidated;
+  }
 };
 
 /*******************************************************************************
@@ -201,7 +611,9 @@ class Database final
   : public PBackgroundLSDatabaseParent
 {
   RefPtr<Datastore> mDatastore;
-
+  // Strings share buffers if possible, so it's not a problem to duplicate the
+  // origin here.
+  nsCString mOrigin;
   bool mAllowedToClose;
   bool mActorDestroyed;
   bool mRequestedAllowToClose;
@@ -211,7 +623,13 @@ class Database final
 
 public:
   // Created in AllocPBackgroundLSDatabaseParent.
-  Database();
+  explicit Database(const nsACString& aOrigin);
+
+  const nsCString&
+  Origin() const
+  {
+    return mOrigin;
+  }
 
   void
   SetActorAlive(already_AddRefed<Datastore>&& aDatastore);
@@ -279,6 +697,7 @@ private:
 
 class PrepareDatastoreOp
   : public LSRequestBase
+  , public OpenDirectoryListener
 {
   enum class State
   {
@@ -286,14 +705,38 @@ class PrepareDatastoreOp
     // or OpeningOnOwningThread.
     Initial,
 
-    // Waiting to open/opening on the main thread. Next step is
-    // SendingReadyMessage.
+    // Waiting to open/opening on the main thread. Next step is FinishOpen.
     OpeningOnMainThread,
 
-    // Waiting to open/opening on the owning thread. Next step is
-    // SendingReadyMessage.
+    // Waiting to open/opening on the owning thread. Next step is FinishOpen.
     OpeningOnOwningThread,
 
+    // Checking if a prepare datastore operation is already running for given
+    // origin on the PBackground thread. Next step is PreparationPending.
+    FinishOpen,
+
+    // Opening directory or initializing quota manager on the PBackground
+    // thread. Next step is either DirectoryOpenPending if quota manager is
+    // already initialized or QuotaManagerPending if quota manager needs to be
+    // initialized.
+    // If a datastore already exists for given origin then the next state is
+    // SendingReadyMessage.
+    PreparationPending,
+
+    // Waiting for quota manager initialization to complete on the PBackground
+    // thread. Next step is either SendingReadyMessage if initialization failed
+    // or DirectoryOpenPending if initialization succeeded.
+    QuotaManagerPending,
+
+    // Waiting for directory open allowed on the PBackground thread. The next
+    // step is either SendingReadyMessage if directory lock failed to acquire,
+    // or DatabaseWorkOpen if directory lock is acquired.
+    DirectoryOpenPending,
+
+    // Waiting to do/doing work on the QuotaManager IO thread. Its next step is
+    // SendingReadyMessage.
+    DatabaseWorkOpen,
+
     // Waiting to send/sending the ready message on the PBackground thread. Next
     // step is WaitingForFinish.
     SendingReadyMessage,
@@ -310,15 +753,54 @@ class PrepareDatastoreOp
     Completed
   };
 
+  RefPtr<PrepareDatastoreOp> mDelayedOp;
+  RefPtr<DirectoryLock> mDirectoryLock;
+  RefPtr<Datastore> mDatastore;
   const LSRequestPrepareDatastoreParams mParams;
   nsCString mSuffix;
   nsCString mGroup;
+  nsCString mMainThreadOrigin;
   nsCString mOrigin;
   State mState;
+  bool mRequestedDirectoryLock;
+  bool mInvalidated;
 
 public:
   explicit PrepareDatastoreOp(const LSRequestParams& aParams);
 
+  bool
+  OriginIsKnown() const
+  {
+    AssertIsOnOwningThread();
+
+    return !mOrigin.IsEmpty();
+  }
+
+  const nsCString&
+  Origin() const
+  {
+    AssertIsOnOwningThread();
+    MOZ_ASSERT(OriginIsKnown());
+
+    return mOrigin;
+  }
+
+  bool
+  RequestedDirectoryLock() const
+  {
+    AssertIsOnOwningThread();
+
+    return mRequestedDirectoryLock;
+  }
+
+  void
+  Invalidate()
+  {
+    AssertIsOnOwningThread();
+
+    mInvalidated = true;
+  }
+
   void
   Dispatch() override;
 
@@ -331,6 +813,30 @@ private:
   nsresult
   OpenOnOwningThread();
 
+  nsresult
+  FinishOpen();
+
+  nsresult
+  PreparationOpen();
+
+  nsresult
+  BeginDatastorePreparation();
+
+  nsresult
+  QuotaManagerOpen();
+
+  nsresult
+  OpenDirectory();
+
+  nsresult
+  SendToIOThread();
+
+  nsresult
+  DatabaseWork();
+
+  nsresult
+  VerifyDatabaseInformation(mozIStorageConnection* aConnection);
+
   void
   SendReadyMessage();
 
@@ -340,12 +846,21 @@ private:
   void
   Cleanup();
 
+  NS_DECL_ISUPPORTS_INHERITED
+
   NS_IMETHOD
   Run() override;
 
   // IPDL overrides.
   mozilla::ipc::IPCResult
   RecvFinish() override;
+
+  // OpenDirectoryListener overrides.
+  void
+  DirectoryLockAcquired(DirectoryLock* aLock) override;
+
+  void
+  DirectoryLockFailed() override;
 };
 
 /*******************************************************************************
@@ -486,12 +1001,12 @@ AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
   }
 
   // If we ever decide to return null from this point on, we need to make sure
-  // that the prepared datastore is removed from the gPreparedDatastores
-  // hashtable.
+  // that the datastore is closed and the prepared datastore is removed from the
+  // gPreparedDatastores hashtable.
   // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
   // once we return a valid actor in this method.
 
-  RefPtr<Database> database = new Database();
+  RefPtr<Database> database = new Database(preparedDatastore->Origin());
 
   // Transfer ownership to IPDL.
   return database.forget().take();
@@ -519,6 +1034,13 @@ RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
 
   database->SetActorAlive(preparedDatastore->ForgetDatastore());
 
+  // It's possible that AbortOperations was called before the database actor
+  // was created and became live. Let the child know that the database in no
+  // longer valid.
+  if (preparedDatastore->IsInvalidated()) {
+    database->RequestAllowToClose();
+  }
+
   return true;
 }
 
@@ -640,8 +1162,11 @@ CreateQuotaClient()
  * Datastore
  ******************************************************************************/
 
-Datastore::Datastore(const nsACString& aOrigin)
-  : mOrigin(aOrigin)
+Datastore::Datastore(const nsACString& aOrigin,
+                     already_AddRefed<DirectoryLock>&& aDirectoryLock)
+  : mDirectoryLock(std::move(aDirectoryLock))
+  , mOrigin(aOrigin)
+  , mClosed(false)
 {
   AssertIsOnBackgroundThread();
 }
@@ -649,6 +1174,20 @@ Datastore::Datastore(const nsACString& aOrigin)
 Datastore::~Datastore()
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mClosed);
+}
+
+void
+Datastore::Close()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(!mDatabases.Count());
+  MOZ_ASSERT(mDirectoryLock);
+
+  mClosed = true;
+
+  mDirectoryLock = nullptr;
 
   MOZ_ASSERT(gDatastores);
   MOZ_ASSERT(gDatastores->Get(mOrigin));
@@ -659,6 +1198,42 @@ Datastore::~Datastore()
   }
 }
 
+void
+Datastore::NoteLiveDatabase(Database* aDatabase)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+  MOZ_ASSERT(!mDatabases.GetEntry(aDatabase));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mDatabases.PutEntry(aDatabase);
+}
+
+void
+Datastore::NoteFinishedDatabase(Database* aDatabase)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+  MOZ_ASSERT(mDatabases.GetEntry(aDatabase));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mDatabases.RemoveEntry(aDatabase);
+
+  if (!mDatabases.Count()) {
+    Close();
+  }
+}
+
+bool
+Datastore::HasLiveDatabases() const
+{
+  AssertIsOnBackgroundThread();
+
+  return mDatabases.Count();
+}
+
 uint32_t
 Datastore::GetLength() const
 {
@@ -730,8 +1305,9 @@ Datastore::GetKeys(nsTArray<nsString>& aKeys) const
  * Database
  ******************************************************************************/
 
-Database::Database()
-  : mAllowedToClose(false)
+Database::Database(const nsACString& aOrigin)
+  : mOrigin(aOrigin)
+  , mAllowedToClose(false)
   , mActorDestroyed(false)
   , mRequestedAllowToClose(false)
 #ifdef DEBUG
@@ -760,6 +1336,8 @@ Database::SetActorAlive(already_AddRefed<Datastore>&& aDatastore)
 
   mDatastore = std::move(aDatastore);
 
+  mDatastore->NoteLiveDatabase(this);
+
   if (!gLiveDatabases) {
     gLiveDatabases = new LiveDatabaseArray();
   }
@@ -796,6 +1374,8 @@ Database::AllowToClose()
 
   mAllowedToClose = true;
 
+  mDatastore->NoteFinishedDatabase(this);
+
   mDatastore = nullptr;
 
   MOZ_ASSERT(gLiveDatabases);
@@ -995,6 +1575,8 @@ LSRequestBase::RecvCancel()
 PrepareDatastoreOp::PrepareDatastoreOp(const LSRequestParams& aParams)
   : mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mState(State::Initial)
+  , mRequestedDirectoryLock(false)
+  , mInvalidated(false)
 {
   MOZ_ASSERT(aParams.type() ==
                LSRequestParams::TLSRequestPrepareDatastoreParams);
@@ -1002,6 +1584,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(const LSRequestParams& aParams)
 
 PrepareDatastoreOp::~PrepareDatastoreOp()
 {
+  MOZ_ASSERT(!mDirectoryLock);
   MOZ_ASSERT_IF(MayProceedOnNonOwningThread(),
                 mState == State::Initial || mState == State::Completed);
 }
@@ -1056,13 +1639,19 @@ PrepareDatastoreOp::OpenOnMainThread()
     rv = QuotaManager::GetInfoFromPrincipal(principal,
                                             &mSuffix,
                                             &mGroup,
-                                            &mOrigin);
+                                            &mMainThreadOrigin);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
   }
 
-  mState = State::SendingReadyMessage;
+  // This service has to be started on the main thread currently.
+  nsCOMPtr<mozIStorageService> ss;
+  if (NS_WARN_IF(!(ss = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID)))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  mState = State::FinishOpen;
   MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
 
   return NS_OK;
@@ -1089,12 +1678,287 @@ PrepareDatastoreOp::OpenOnOwningThread()
   mGroup = quotaInfo.group();
   mOrigin = quotaInfo.origin();
 
-  mState = State::SendingReadyMessage;
+  mState = State::FinishOpen;
   MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
 
   return NS_OK;
 }
 
+nsresult
+PrepareDatastoreOp::FinishOpen()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::FinishOpen);
+  MOZ_ASSERT(gPrepareDatastoreOps);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  // Normally it's safe to access member variables without a mutex because even
+  // though we hop between threads, the variables are never accessed by multiple
+  // threads at the same time.
+  // However, the methods OriginIsKnown and Origin can be called at any time.
+  // So we have to make sure the member variable is set on the same thread as
+  // those methods are called.
+  if (mParams.info().type() == PrincipalOrQuotaInfo::TPrincipalInfo) {
+    mOrigin = mMainThreadOrigin;
+  }
+
+  MOZ_ASSERT(!mOrigin.IsEmpty());
+
+  mState = State::PreparationPending;
+
+  // See if this PrepareDatastoreOp needs to wait.
+  bool foundThis = false;
+  for (uint32_t index = gPrepareDatastoreOps->Length(); index > 0; index--) {
+    PrepareDatastoreOp* existingOp = (*gPrepareDatastoreOps)[index - 1];
+
+    if (existingOp == this) {
+      foundThis = true;
+      continue;
+    }
+
+    if (foundThis && existingOp->Origin() == mOrigin) {
+      // Only one op can be delayed.
+      MOZ_ASSERT(!existingOp->mDelayedOp);
+      existingOp->mDelayedOp = this;
+
+      return NS_OK;
+    }
+  }
+
+  nsresult rv = BeginDatastorePreparation();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::BeginDatastorePreparation()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::PreparationPending);
+
+  if (gDatastores && (mDatastore = gDatastores->Get(mOrigin))) {
+    mState = State::SendingReadyMessage;
+    Unused << this->Run();
+
+    return NS_OK;
+  }
+
+  if (QuotaManager::Get()) {
+    nsresult rv = OpenDirectory();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    return NS_OK;
+  }
+
+  mState = State::QuotaManagerPending;
+  QuotaManager::GetOrCreate(this);
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::QuotaManagerOpen()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::QuotaManagerPending);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  if (NS_WARN_IF(!QuotaManager::Get())) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsresult rv = OpenDirectory();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::OpenDirectory()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::PreparationPending ||
+             mState == State::QuotaManagerPending);
+  MOZ_ASSERT(!mOrigin.IsEmpty());
+  MOZ_ASSERT(!mDirectoryLock);
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+  MOZ_ASSERT(QuotaManager::Get());
+
+  mState = State::DirectoryOpenPending;
+  QuotaManager::Get()->OpenDirectory(PERSISTENCE_TYPE_DEFAULT,
+                                     mGroup,
+                                     mOrigin,
+                                     mozilla::dom::quota::Client::LS,
+                                     /* aExclusive */ false,
+                                     this);
+
+  mRequestedDirectoryLock = true;
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::SendToIOThread()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  // Must set this before dispatching otherwise we will race with the IO thread.
+  mState = State::DatabaseWorkOpen;
+
+  nsresult rv = quotaManager->IOThread()->Dispatch(this, NS_DISPATCH_NORMAL);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::DatabaseWork()
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(mState == State::DatabaseWorkOpen);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
+      !MayProceedOnNonOwningThread()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsCOMPtr<nsIFile> dbDirectory;
+  nsresult rv =
+    quotaManager->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT,
+                                            mSuffix,
+                                            mGroup,
+                                            mOrigin,
+                                            getter_AddRefs(dbDirectory));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = dbDirectory->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+  rv = dbDirectory->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!exists) {
+    rv = dbDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+#ifdef DEBUG
+  else {
+    bool isDirectory;
+    MOZ_ASSERT(NS_SUCCEEDED(dbDirectory->IsDirectory(&isDirectory)));
+    MOZ_ASSERT(isDirectory);
+  }
+#endif
+
+  nsCOMPtr<nsIFile> dbFile;
+  rv = dbDirectory->Clone(getter_AddRefs(dbFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = dbFile->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = CreateStorageConnection(dbFile, mOrigin, getter_AddRefs(connection));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = VerifyDatabaseInformation(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // Must set mState before dispatching otherwise we will race with the owning
+  // thread.
+  mState = State::SendingReadyMessage;
+
+  rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::VerifyDatabaseInformation(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  nsresult rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT origin "
+    "FROM database"
+  ), getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!hasResult)) {
+    return NS_ERROR_FILE_CORRUPTED;
+  }
+
+  nsCString origin;
+  rv = stmt->GetUTF8String(0, origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!QuotaManager::AreOriginsEqualOnDisk(mOrigin, origin))) {
+    return NS_ERROR_FILE_CORRUPTED;
+  }
+
+  return NS_OK;
+}
+
 void
 PrepareDatastoreOp::SendReadyMessage()
 {
@@ -1126,31 +1990,32 @@ PrepareDatastoreOp::SendResults()
     LSRequestResponse response;
 
     if (NS_SUCCEEDED(ResultCode())) {
-      RefPtr<Datastore> datastore;
-
-      if (gDatastores) {
-        datastore = gDatastores->Get(mOrigin);
-      }
-
-      if (!datastore) {
-        datastore = new Datastore(mOrigin);
+      if (!mDatastore) {
+        mDatastore = new Datastore(mOrigin, mDirectoryLock.forget());
 
         if (!gDatastores) {
           gDatastores = new DatastoreHashtable();
         }
 
-        gDatastores->Put(mOrigin, datastore);
+        MOZ_ASSERT(!gDatastores->Get(mOrigin));
+        gDatastores->Put(mOrigin, mDatastore);
       }
 
       uint64_t datastoreId = ++gLastDatastoreId;
 
       nsAutoPtr<PreparedDatastore> preparedDatastore(
-        new PreparedDatastore(datastore));
+        new PreparedDatastore(mDatastore, mOrigin));
 
       if (!gPreparedDatastores) {
         gPreparedDatastores = new PreparedDatastoreHashtable();
       }
-      gPreparedDatastores->Put(datastoreId, preparedDatastore.forget());
+      gPreparedDatastores->Put(datastoreId, preparedDatastore);
+
+      if (mInvalidated) {
+        preparedDatastore->Invalidate();
+      }
+
+      preparedDatastore.forget();
 
       LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
       prepareDatastoreResponse.datastoreId() = datastoreId;
@@ -1174,6 +2039,28 @@ PrepareDatastoreOp::Cleanup()
 {
   AssertIsOnOwningThread();
 
+  if (mDatastore) {
+    MOZ_ASSERT(!mDirectoryLock);
+
+    if (NS_FAILED(ResultCode())) {
+      MOZ_ASSERT(!mDatastore->IsClosed());
+      MOZ_ASSERT(!mDatastore->HasLiveDatabases());
+      mDatastore->Close();
+    }
+
+    // Make sure to release the datastore on this thread.
+    mDatastore = nullptr;
+  } else if (mDirectoryLock) {
+    // If we have a directory lock then the operation must have failed.
+    MOZ_ASSERT(NS_FAILED(ResultCode()));
+
+    mDirectoryLock = nullptr;
+  }
+
+  if (mDelayedOp) {
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(mDelayedOp.forget()));
+  }
+
   MOZ_ASSERT(gPrepareDatastoreOps);
   gPrepareDatastoreOps->RemoveElement(this);
 
@@ -1182,6 +2069,8 @@ PrepareDatastoreOp::Cleanup()
   }
 }
 
+NS_IMPL_ISUPPORTS_INHERITED0(PrepareDatastoreOp, LSRequestBase)
+
 NS_IMETHODIMP
 PrepareDatastoreOp::Run()
 {
@@ -1196,6 +2085,22 @@ PrepareDatastoreOp::Run()
       rv = OpenOnOwningThread();
       break;
 
+    case State::FinishOpen:
+      rv = FinishOpen();
+      break;
+
+    case State::PreparationPending:
+      rv = BeginDatastorePreparation();
+      break;
+
+    case State::QuotaManagerPending:
+      rv = QuotaManagerOpen();
+      break;
+
+    case State::DatabaseWorkOpen:
+      rv = DatabaseWork();
+      break;
+
     case State::SendingReadyMessage:
       SendReadyMessage();
       return NS_OK;
@@ -1238,6 +2143,45 @@ PrepareDatastoreOp::RecvFinish()
   return IPC_OK();
 }
 
+void
+PrepareDatastoreOp::DirectoryLockAcquired(DirectoryLock* aLock)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+  MOZ_ASSERT(!mDirectoryLock);
+
+  mDirectoryLock = aLock;
+
+  nsresult rv = SendToIOThread();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    MaybeSetFailureCode(rv);
+
+    // The caller holds a strong reference to us, no need for a self reference
+    // before calling Run().
+
+    mState = State::SendingReadyMessage;
+    MOZ_ALWAYS_SUCCEEDS(Run());
+
+    return;
+  }
+}
+
+void
+PrepareDatastoreOp::DirectoryLockFailed()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+  MOZ_ASSERT(!mDirectoryLock);
+
+  MaybeSetFailureCode(NS_ERROR_FAILURE);
+
+  // The caller holds a strong reference to us, no need for a self reference
+  // before calling Run().
+
+  mState = State::SendingReadyMessage;
+  MOZ_ALWAYS_SUCCEEDS(Run());
+}
+
 /*******************************************************************************
  * QuotaClient
  ******************************************************************************/
@@ -1295,8 +2239,122 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
                                UsageInfo* aUsageInfo)
 {
   AssertIsOnIOThread();
+  MOZ_ASSERT(aPersistenceType == PERSISTENCE_TYPE_DEFAULT);
   MOZ_ASSERT(aUsageInfo);
 
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsCOMPtr<nsIFile> directory;
+  nsresult rv = quotaManager->GetDirectoryForOrigin(aPersistenceType,
+                                                    aOrigin,
+                                                    getter_AddRefs(directory));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(directory);
+
+  rv = directory->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+#ifdef DEBUG
+  rv = directory->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(exists);
+#endif
+
+  nsCOMPtr<nsIFile> file;
+  rv = directory->Clone(getter_AddRefs(file));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = file->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = file->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (exists) {
+    bool isDirectory;
+    rv = file->IsDirectory(&isDirectory);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (NS_WARN_IF(isDirectory)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    // TODO: Use a special file that contains logical size of the database.
+    //       For now, don't add to origin usage.
+  }
+
+  // Report unknown files, don't fail, just warn.
+
+  nsCOMPtr<nsIDirectoryEnumerator> directoryEntries;
+  rv = directory->GetDirectoryEntries(getter_AddRefs(directoryEntries));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!directoryEntries) {
+    return NS_OK;
+  }
+
+  while (true) {
+    if (aCanceled) {
+      break;
+    }
+
+    nsCOMPtr<nsIFile> file;
+    rv = directoryEntries->GetNextFile(getter_AddRefs(file));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (!file) {
+      break;
+    }
+
+    nsString leafName;
+    rv = file->GetLeafName(leafName);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (leafName.EqualsLiteral(DATA_FILE_NAME)) {
+      // Don't need to check if it is a directory or file. We did that above.
+      continue;
+    }
+
+    if (leafName.EqualsLiteral(JOURNAL_FILE_NAME)) {
+      bool isDirectory;
+      rv = file->IsDirectory(&isDirectory);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      if (!isDirectory) {
+        continue;
+      }
+    }
+
+    LS_WARNING("Something (%s) in the directory that doesn't belong!", \
+               NS_ConvertUTF16toUTF8(leafName).get());
+  }
+
   return NS_OK;
 }
 
@@ -1317,6 +2375,67 @@ void
 QuotaClient::AbortOperations(const nsACString& aOrigin)
 {
   AssertIsOnBackgroundThread();
+
+  // A PrepareDatastoreOp object could already acquire a directory lock for
+  // the given origin. Its last step is creation of a Datastore object (which
+  // will take ownership of the directory lock) and a PreparedDatastore object
+  // which keeps the Datastore alive until a database actor is created.
+  // We need to invalidate the PreparedDatastore object when it's created,
+  // otherwise the Datastore object can block the origin clear operation for
+  // long time. It's not a problem that we don't fail the PrepareDatastoreOp
+  // immediatelly (avoiding the creation of the Datastore and PreparedDatastore
+  // object). We will call RequestAllowToClose on the database actor once it's
+  // created and the child actor will respond by sending AllowToClose which
+  // will close the Datastore on the parent side (the closing releases the
+  // directory lock).
+
+  if (gPrepareDatastoreOps) {
+    for (PrepareDatastoreOp* prepareDatastoreOp : *gPrepareDatastoreOps) {
+      MOZ_ASSERT(prepareDatastoreOp);
+
+      // Explicitely check if a directory lock has been requested.
+      // Origin clearing can't be blocked by this PrepareDatastoreOp if it
+      // hasn't requested a directory lock yet, so we can just ignore it.
+      // This will also guarantee that PrepareDatastoreOp has a known origin.
+      // And it also ensures that the ordering is right. Without the check we
+      // could invalidate ops whose directory locks were requested after we
+      // requested a directory lock for origin clearing.
+      if (!prepareDatastoreOp->RequestedDirectoryLock()) {
+        continue;
+      }
+
+      MOZ_ASSERT(prepareDatastoreOp->OriginIsKnown());
+
+      if (aOrigin.IsVoid() || prepareDatastoreOp->Origin() == aOrigin) {
+        prepareDatastoreOp->Invalidate();
+      }
+    }
+  }
+
+  if (gPreparedDatastores) {
+    for (auto iter = gPreparedDatastores->ConstIter();
+         !iter.Done();
+         iter.Next()) {
+      PreparedDatastore* preparedDatastore = iter.Data();
+      MOZ_ASSERT(preparedDatastore);
+
+      if (aOrigin.IsVoid() || preparedDatastore->Origin() == aOrigin) {
+        preparedDatastore->Invalidate();
+      }
+    }
+  }
+
+  if (gLiveDatabases) {
+    for (Database* database : *gLiveDatabases) {
+      if (aOrigin.IsVoid() || database->Origin() == aOrigin) {
+        // TODO: This just allows the database to close, but we can actually
+        //       set a flag to abort any existing operations, so we can
+        //       eventually close faster.
+
+        database->RequestAllowToClose();
+      }
+    }
+  }
 }
 
 void
@@ -1353,6 +2472,20 @@ QuotaClient::ShutdownWorkThreads()
   // When the last PrepareDatastoreOp finishes, the gPrepareDatastoreOps array
   // is destroyed.
 
+  // There may be datastores that are only held alive by prepared datastores
+  // (ones which have no live database actors). We need to explicitly close
+  // them here.
+  if (gDatastores) {
+    for (auto iter = gDatastores->ConstIter(); !iter.Done(); iter.Next()) {
+      Datastore* datastore = iter.Data();
+      MOZ_ASSERT(datastore);
+
+      if (!datastore->IsClosed() && !datastore->HasLiveDatabases()) {
+        datastore->Close();
+      }
+    }
+  }
+
   // If database actors haven't been created yet, don't do anything special.
   // We are shutting down and we can release prepared datastores immediatelly
   // since database actors will never be created for them.
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index d623ead8ae9b..4c441b43d964 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -167,6 +167,12 @@ LSObject::Create(nsPIDOMWindowInner* aWindow,
     quotaInfo.origin() = origin;
 
     *info = quotaInfo;
+
+    // This service has to be started on the main thread currently.
+    nsCOMPtr<mozIStorageService> ss;
+    if (NS_WARN_IF(!(ss = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID)))) {
+      return NS_ERROR_FAILURE;
+    }
   } else {
     PrincipalInfo principalInfo;
     rv = PrincipalToPrincipalInfo(principal, &principalInfo);
diff --git a/dom/localstorage/ReportInternalError.cpp b/dom/localstorage/ReportInternalError.cpp
new file mode 100644
index 000000000000..a15209687f9d
--- /dev/null
+++ b/dom/localstorage/ReportInternalError.cpp
@@ -0,0 +1,36 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "ReportInternalError.h"
+
+#include "mozilla/IntegerPrintfMacros.h"
+
+#include "nsContentUtils.h"
+#include "nsPrintfCString.h"
+
+namespace mozilla {
+namespace dom {
+namespace localstorage {
+
+void
+ReportInternalError(const char* aFile, uint32_t aLine, const char* aStr)
+{
+  // Get leaf of file path
+  for (const char* p = aFile; *p; ++p) {
+    if (*p == '/' && *(p + 1)) {
+      aFile = p + 1;
+    }
+  }
+
+  nsContentUtils::LogSimpleConsoleError(
+    NS_ConvertUTF8toUTF16(nsPrintfCString(
+                          "LocalStorage %s: %s:%" PRIu32, aStr, aFile, aLine)),
+    "localstorage", false);
+}
+
+} // namespace localstorage
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/ReportInternalError.h b/dom/localstorage/ReportInternalError.h
new file mode 100644
index 000000000000..1988a880d5f8
--- /dev/null
+++ b/dom/localstorage/ReportInternalError.h
@@ -0,0 +1,32 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_ReportInternalError_h
+#define mozilla_dom_localstorage_ReportInternalError_h
+
+#include "nsDebug.h"
+
+#define LS_WARNING(...)                                                        \
+  do {                                                                         \
+    nsPrintfCString s(__VA_ARGS__);                                            \
+    mozilla::dom::localstorage::ReportInternalError(__FILE__,                  \
+                                                    __LINE__,                  \
+                                                    s.get());                  \
+    NS_WARNING(s.get());                                                       \
+  } while (0)
+
+namespace mozilla {
+namespace dom {
+namespace localstorage {
+
+void
+ReportInternalError(const char* aFile, uint32_t aLine, const char* aStr);
+
+} // namespace localstorage
+} // namespace dom
+} // namespace mozilla
+
+#endif  // mozilla_dom_localstorage_ReportInternalError_h
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index ebe0c87b7aab..64cd2c951b4e 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -21,6 +21,7 @@ UNIFIED_SOURCES += [
     'LocalStorageManager2.cpp',
     'LSDatabase.cpp',
     'LSObject.cpp',
+    'ReportInternalError.cpp',
 ]
 
 IPDL_SOURCES += [
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 280aeb0b84a5..68d4ee926e85 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -4799,6 +4799,10 @@ QuotaManager::MaybeRemoveLocalStorageData()
 {
   AssertIsOnIOThread();
 
+  if (CachedNextGenLocalStorageEnabled()) {
+    return NS_OK;
+  }
+
   // Cleanup the tmp file first, if there's any.
   nsCOMPtr<nsIFile> lsArchiveTmpFile;
   nsresult rv = NS_NewLocalFile(mStoragePath,
diff --git a/dom/quota/test/unit/test_removeLocalStorage.js b/dom/quota/test/unit/test_removeLocalStorage.js
index 9d4a3b80050a..162c10145412 100644
--- a/dom/quota/test/unit/test_removeLocalStorage.js
+++ b/dom/quota/test/unit/test_removeLocalStorage.js
@@ -11,6 +11,10 @@ function* testSteps()
   const lsArchiveTmpFile = "storage/ls-archive-tmp.sqlite";
   const lsDir = "storage/default/http+++localhost/ls";
 
+  info("Setting pref");
+
+  SpecialPowers.setBoolPref("dom.storage.next_gen", false);
+
   // Profile 1
   info("Clearing");
 
diff --git a/toolkit/components/extensions/test/mochitest/mochitest.ini b/toolkit/components/extensions/test/mochitest/mochitest.ini
index 9fca36408650..12b838652671 100644
--- a/toolkit/components/extensions/test/mochitest/mochitest.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest.ini
@@ -1,7 +1,7 @@
 [DEFAULT]
 tags = webextensions in-process-webextensions
 
-#[test_ext_storage_cleanup.html]
+[test_ext_storage_cleanup.html]
 # Bug 1426514 storage_cleanup: clearing localStorage fails with oop
 
 [include:mochitest-common.ini]
diff --git a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
index 8b013bb42947..c490ddf304f1 100644
--- a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
+++ b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
@@ -60,7 +60,7 @@ skip-if = os == "android" # checking for telemetry needs to be updated: 1384923
 [test_ext_extension_startup_telemetry.js]
 [test_ext_geturl.js]
 [test_ext_idle.js]
-[test_ext_localStorage.js]
+#[test_ext_localStorage.js]
 [test_ext_management.js]
 skip-if = (os == "win" && !debug) #Bug 1419183 disable on Windows
 [test_ext_management_uninstall_self.js]

From c934ad618d0c37389e5cc8ceaa334e4f9eeca22e Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:30 +0100
Subject: [PATCH 20/78] Bug 1286798 - Part 6: Fix a dead lock in the single
 process case; r=asuth,janv

Expose the nested main event target, so it can be used in the single process case by the parent side to process runnables which need to run on the main thread.
After this change we don't have use hacks like getting profile directory path on the child side and sending it to the parent. The parent side can now do it freely even in the single process case.

This patch was enhanced by asuth to not tunnel the nested main event target through IPC.
---
 dom/localstorage/ActorsParent.cpp             | 112 +++++-------------
 dom/localstorage/LSObject.cpp                 |  77 ++++++------
 dom/localstorage/LSObject.h                   |  19 ++-
 .../PBackgroundLSSharedTypes.ipdlh            |  13 +-
 dom/quota/ActorsParent.cpp                    |  23 +++-
 dom/quota/QuotaManager.h                      |   3 +-
 .../test/xpcshell/xpcshell-common.ini         |   2 +-
 7 files changed, 106 insertions(+), 143 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index b9f421e379a3..f4fbf22ce54d 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -7,6 +7,7 @@
 #include "ActorsParent.h"
 
 #include "LocalStorageCommon.h"
+#include "LSObject.h"
 #include "mozIStorageConnection.h"
 #include "mozIStorageService.h"
 #include "mozStorageCID.h"
@@ -706,10 +707,7 @@ class PrepareDatastoreOp
     Initial,
 
     // Waiting to open/opening on the main thread. Next step is FinishOpen.
-    OpeningOnMainThread,
-
-    // Waiting to open/opening on the owning thread. Next step is FinishOpen.
-    OpeningOnOwningThread,
+    Opening,
 
     // Checking if a prepare datastore operation is already running for given
     // origin on the PBackground thread. Next step is PreparationPending.
@@ -753,6 +751,7 @@ class PrepareDatastoreOp
     Completed
   };
 
+  nsCOMPtr<nsIEventTarget> mMainEventTarget;
   RefPtr<PrepareDatastoreOp> mDelayedOp;
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Datastore> mDatastore;
@@ -766,7 +765,8 @@ class PrepareDatastoreOp
   bool mInvalidated;
 
 public:
-  explicit PrepareDatastoreOp(const LSRequestParams& aParams);
+  PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
+                     const LSRequestParams& aParams);
 
   bool
   OriginIsKnown() const
@@ -808,10 +808,7 @@ private:
   ~PrepareDatastoreOp() override;
 
   nsresult
-  OpenOnMainThread();
-
-  nsresult
-  OpenOnOwningThread();
+  Open();
 
   nsresult
   FinishOpen();
@@ -1066,31 +1063,19 @@ AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
     return nullptr;
   }
 
+  // If we're in the same process as the actor, we need to get the target event
+  // queue from the current RequestHelper.
+  nsCOMPtr<nsIEventTarget> mainEventTarget;
+  if (!BackgroundParent::IsOtherProcessActor(aBackgroundActor)) {
+    mainEventTarget = LSObject::GetSyncLoopEventTarget();
+  }
+
   RefPtr<LSRequestBase> actor;
 
   switch (aParams.type()) {
     case LSRequestParams::TLSRequestPrepareDatastoreParams: {
-      bool isOtherProcess =
-        BackgroundParent::IsOtherProcessActor(aBackgroundActor);
-
-      const LSRequestPrepareDatastoreParams& params =
-        aParams.get_LSRequestPrepareDatastoreParams();
-
-      const PrincipalOrQuotaInfo& info = params.info();
-
-      PrincipalOrQuotaInfo::Type infoType = info.type();
-
-      bool paramsOk =
-        (isOtherProcess && infoType == PrincipalOrQuotaInfo::TPrincipalInfo) ||
-        (!isOtherProcess && infoType == PrincipalOrQuotaInfo::TQuotaInfo);
-
-      if (NS_WARN_IF(!paramsOk)) {
-        ASSERT_UNLESS_FUZZING();
-        return nullptr;
-      }
-
       RefPtr<PrepareDatastoreOp> prepareDatastoreOp =
-        new PrepareDatastoreOp(aParams);
+        new PrepareDatastoreOp(mainEventTarget, aParams);
 
       if (!gPrepareDatastoreOps) {
         gPrepareDatastoreOps = new PrepareDatastoreOpArray();
@@ -1572,8 +1557,10 @@ LSRequestBase::RecvCancel()
  * PrepareDatastoreOp
  ******************************************************************************/
 
-PrepareDatastoreOp::PrepareDatastoreOp(const LSRequestParams& aParams)
-  : mParams(aParams.get_LSRequestPrepareDatastoreParams())
+PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
+                                       const LSRequestParams& aParams)
+  : mMainEventTarget(aMainEventTarget)
+  , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mState(State::Initial)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
@@ -1594,35 +1581,27 @@ PrepareDatastoreOp::Dispatch()
 {
   AssertIsOnOwningThread();
 
-  const PrincipalOrQuotaInfo& info = mParams.info();
+  mState = State::Opening;
 
-  if (info.type() == PrincipalOrQuotaInfo::TPrincipalInfo) {
-    mState = State::OpeningOnMainThread;
-    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+  if (mMainEventTarget) {
+    MOZ_ALWAYS_SUCCEEDS(mMainEventTarget->Dispatch(this, NS_DISPATCH_NORMAL));
   } else {
-    MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TQuotaInfo);
-
-    mState = State::OpeningOnOwningThread;
-    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
   }
 }
 
 nsresult
-PrepareDatastoreOp::OpenOnMainThread()
+PrepareDatastoreOp::Open()
 {
   MOZ_ASSERT(NS_IsMainThread());
-  MOZ_ASSERT(mState == State::OpeningOnMainThread);
+  MOZ_ASSERT(mState == State::Opening);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
     return NS_ERROR_FAILURE;
   }
 
-  const PrincipalOrQuotaInfo& info = mParams.info();
-
-  MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TPrincipalInfo);
-
-  const PrincipalInfo& principalInfo = info.get_PrincipalInfo();
+  const PrincipalInfo& principalInfo = mParams.principalInfo();
 
   if (principalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) {
     QuotaManager::GetInfoForChrome(&mSuffix, &mGroup, &mOrigin);
@@ -1657,33 +1636,6 @@ PrepareDatastoreOp::OpenOnMainThread()
   return NS_OK;
 }
 
-nsresult
-PrepareDatastoreOp::OpenOnOwningThread()
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::OpeningOnOwningThread);
-
-  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
-      !MayProceed()) {
-    return NS_ERROR_FAILURE;
-  }
-
-  const PrincipalOrQuotaInfo& info = mParams.info();
-
-  MOZ_ASSERT(info.type() == PrincipalOrQuotaInfo::TQuotaInfo);
-
-  const QuotaInfo& quotaInfo = info.get_QuotaInfo();
-
-  mSuffix = quotaInfo.suffix();
-  mGroup = quotaInfo.group();
-  mOrigin = quotaInfo.origin();
-
-  mState = State::FinishOpen;
-  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
-
-  return NS_OK;
-}
-
 nsresult
 PrepareDatastoreOp::FinishOpen()
 {
@@ -1702,9 +1654,7 @@ PrepareDatastoreOp::FinishOpen()
   // However, the methods OriginIsKnown and Origin can be called at any time.
   // So we have to make sure the member variable is set on the same thread as
   // those methods are called.
-  if (mParams.info().type() == PrincipalOrQuotaInfo::TPrincipalInfo) {
-    mOrigin = mMainThreadOrigin;
-  }
+  mOrigin = mMainThreadOrigin;
 
   MOZ_ASSERT(!mOrigin.IsEmpty());
 
@@ -1760,7 +1710,7 @@ PrepareDatastoreOp::BeginDatastorePreparation()
   }
 
   mState = State::QuotaManagerPending;
-  QuotaManager::GetOrCreate(this);
+  QuotaManager::GetOrCreate(this, mMainEventTarget);
 
   return NS_OK;
 }
@@ -2077,12 +2027,8 @@ PrepareDatastoreOp::Run()
   nsresult rv;
 
   switch (mState) {
-    case State::OpeningOnMainThread:
-      rv = OpenOnMainThread();
-      break;
-
-    case State::OpeningOnOwningThread:
-      rv = OpenOnOwningThread();
+    case State::Opening:
+      rv = Open();
       break;
 
     case State::FinishOpen:
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 4c441b43d964..a45f1317bab5 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -83,6 +83,17 @@ public:
     MOZ_ASSERT(IsOnOwningThread());
   }
 
+  // Used for requests from the parent process to the parent process; in that
+  // case we want ActorsParent to know our event-target and this is better than
+  // trying to tunnel the pointer through IPC.
+  const nsCOMPtr<nsIEventTarget>&
+  GetSyncLoopEventTarget() const
+  {
+    MOZ_ASSERT(XRE_IsParentProcess());
+
+    return mNestedEventTarget;
+  }
+
   nsresult
   StartAndReturnResponse(LSRequestResponse& aResponse);
 
@@ -146,52 +157,40 @@ LSObject::Create(nsPIDOMWindowInner* aWindow,
     return NS_ERROR_FAILURE;
   }
 
-  nsresult rv;
-
-  nsAutoPtr<PrincipalOrQuotaInfo> info(new PrincipalOrQuotaInfo());
-  if (XRE_IsParentProcess()) {
-    nsCString suffix;
-    nsCString group;
-    nsCString origin;
-    rv = QuotaManager::GetInfoFromPrincipal(principal,
-                                            &suffix,
-                                            &group,
-                                            &origin);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    QuotaInfo quotaInfo;
-    quotaInfo.suffix() = suffix;
-    quotaInfo.group() = group;
-    quotaInfo.origin() = origin;
-
-    *info = quotaInfo;
-
-    // This service has to be started on the main thread currently.
-    nsCOMPtr<mozIStorageService> ss;
-    if (NS_WARN_IF(!(ss = do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID)))) {
-      return NS_ERROR_FAILURE;
-    }
-  } else {
-    PrincipalInfo principalInfo;
-    rv = PrincipalToPrincipalInfo(principal, &principalInfo);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    MOZ_ASSERT(principalInfo.type() == PrincipalInfo::TContentPrincipalInfo);
-
-    *info = principalInfo;
+  nsAutoPtr<PrincipalInfo> principalInfo(new PrincipalInfo());
+  nsresult rv = PrincipalToPrincipalInfo(principal, principalInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
   }
 
+  MOZ_ASSERT(principalInfo->type() == PrincipalInfo::TContentPrincipalInfo);
+
   RefPtr<LSObject> object = new LSObject(aWindow, principal);
-  object->mInfo = std::move(info);
+  object->mPrincipalInfo = std::move(principalInfo);
 
   object.forget(aStorage);
   return NS_OK;
 }
 
+// static
+already_AddRefed<nsIEventTarget>
+LSObject::GetSyncLoopEventTarget()
+{
+  RefPtr<RequestHelper> helper;
+
+  {
+    StaticMutexAutoLock lock(gRequestHelperMutex);
+    helper = gRequestHelper;
+  }
+
+  nsCOMPtr<nsIEventTarget> target;
+  if (helper) {
+    target = helper->GetSyncLoopEventTarget();
+  }
+
+  return target.forget();
+}
+
 // static
 void
 LSObject::CancelSyncLoop()
@@ -436,7 +435,7 @@ LSObject::EnsureDatabase()
   }
 
   LSRequestPrepareDatastoreParams params;
-  params.info() = *mInfo;
+  params.principalInfo() = *mPrincipalInfo;
 
   RefPtr<RequestHelper> helper = new RequestHelper(this, params);
 
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index a4d8f797b6c3..d46f538d2e3e 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -16,18 +16,25 @@ namespace mozilla {
 
 class ErrorResult;
 
+namespace ipc {
+
+class PrincipalInfo;
+
+} // namespace ipc
+
 namespace dom {
 
 class LSDatabase;
 class LSRequestChild;
 class LSRequestChildCallback;
 class LSRequestParams;
-class PrincipalOrQuotaInfo;
 
 class LSObject final
   : public Storage
 {
-  nsAutoPtr<PrincipalOrQuotaInfo> mInfo;
+  typedef mozilla::ipc::PrincipalInfo PrincipalInfo;
+
+  nsAutoPtr<PrincipalInfo> mPrincipalInfo;
 
   RefPtr<LSDatabase> mDatabase;
 
@@ -36,6 +43,14 @@ public:
   Create(nsPIDOMWindowInner* aWindow,
          Storage** aStorage);
 
+  /**
+   * Used for requests from the parent process to the parent process; in that
+   * case we want ActorsParent to know our event-target and this is better than
+   * trying to tunnel the pointer through IPC.
+   */
+  static already_AddRefed<nsIEventTarget>
+  GetSyncLoopEventTarget();
+
   static void
   CancelSyncLoop();
 
diff --git a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
index e0d7bcf6787e..8c2ea9e967f4 100644
--- a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
+++ b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
@@ -7,20 +7,9 @@ include PBackgroundSharedTypes;
 namespace mozilla {
 namespace dom {
 
-struct QuotaInfo {
-  nsCString suffix;
-  nsCString group;
-  nsCString origin;
-};
-
-union PrincipalOrQuotaInfo {
-  PrincipalInfo;
-  QuotaInfo;
-};
-
 struct LSRequestPrepareDatastoreParams
 {
-  PrincipalOrQuotaInfo info;
+  PrincipalInfo principalInfo;
 };
 
 union LSRequestParams
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 68d4ee926e85..e87ec42ef258 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -405,6 +405,7 @@ class QuotaManager::CreateRunnable final
   : public BackgroundThreadObject
   , public Runnable
 {
+  nsCOMPtr<nsIEventTarget> mMainEventTarget;
   nsTArray<nsCOMPtr<nsIRunnable>> mCallbacks;
   nsString mBaseDirPath;
   RefPtr<QuotaManager> mManager;
@@ -422,8 +423,9 @@ class QuotaManager::CreateRunnable final
   State mState;
 
 public:
-  CreateRunnable()
+  explicit CreateRunnable(nsIEventTarget* aMainEventTarget)
     : Runnable("dom::quota::QuotaManager::CreateRunnable")
+    , mMainEventTarget(aMainEventTarget)
     , mResultCode(NS_OK)
     , mState(State::Initial)
   {
@@ -2780,7 +2782,11 @@ CreateRunnable::GetNextState(nsCOMPtr<nsIEventTarget>& aThread) -> State
       aThread = mOwningThread;
       return State::CreatingManager;
     case State::CreatingManager:
-      aThread = GetMainThreadEventTarget();
+      if (mMainEventTarget) {
+        aThread = mMainEventTarget;
+      } else {
+        aThread = GetMainThreadEventTarget();
+      }
       return State::RegisteringObserver;
     case State::RegisteringObserver:
       aThread = mOwningThread;
@@ -3245,7 +3251,8 @@ QuotaManager::~QuotaManager()
 }
 
 void
-QuotaManager::GetOrCreate(nsIRunnable* aCallback)
+QuotaManager::GetOrCreate(nsIRunnable* aCallback,
+                          nsIEventTarget* aMainEventTarget)
 {
   AssertIsOnBackgroundThread();
 
@@ -3261,8 +3268,14 @@ QuotaManager::GetOrCreate(nsIRunnable* aCallback)
     MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(aCallback));
   } else {
     if (!gCreateRunnable) {
-      gCreateRunnable = new CreateRunnable();
-      MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(gCreateRunnable));
+      gCreateRunnable = new CreateRunnable(aMainEventTarget);
+      if (aMainEventTarget) {
+        MOZ_ALWAYS_SUCCEEDS(aMainEventTarget->Dispatch(gCreateRunnable,
+                                                       NS_DISPATCH_NORMAL));
+      } else {
+        MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(gCreateRunnable));
+      }
+
     }
 
     gCreateRunnable->AddCallback(aCallback);
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index 8b9705cbb233..f8a0a7ac10c2 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -117,7 +117,8 @@ public:
   static const char kReplaceChars[];
 
   static void
-  GetOrCreate(nsIRunnable* aCallback);
+  GetOrCreate(nsIRunnable* aCallback,
+              nsIEventTarget* aMainEventTarget = nullptr);
 
   // Returns a non-owning reference.
   static QuotaManager*
diff --git a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
index c490ddf304f1..8b013bb42947 100644
--- a/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
+++ b/toolkit/components/extensions/test/xpcshell/xpcshell-common.ini
@@ -60,7 +60,7 @@ skip-if = os == "android" # checking for telemetry needs to be updated: 1384923
 [test_ext_extension_startup_telemetry.js]
 [test_ext_geturl.js]
 [test_ext_idle.js]
-#[test_ext_localStorage.js]
+[test_ext_localStorage.js]
 [test_ext_management.js]
 skip-if = (os == "win" && !debug) #Bug 1419183 disable on Windows
 [test_ext_management_uninstall_self.js]

From a47c296511148d8de6bcc20ceeabb431b2aeb7a9 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:34 +0100
Subject: [PATCH 21/78] Bug 1286798 - Part 7: Teach QuotaManager's OriginParser
 to handle file://UNIVERSAL_FILE_URI_ORIGIN; r=asuth

See bug 1340710 which introduced this kind of URI.
---
 dom/quota/ActorsParent.cpp                 | 47 +++++++++++++++-------
 dom/quota/test/unit/test_specialOrigins.js | 44 ++++++++++++++++++++
 dom/quota/test/unit/xpcshell.ini           |  1 +
 3 files changed, 78 insertions(+), 14 deletions(-)
 create mode 100644 dom/quota/test/unit/test_specialOrigins.js

diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index e87ec42ef258..837619a25f96 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -1741,7 +1741,7 @@ private:
     eExpectingScheme,
     eExpectingEmptyToken1,
     eExpectingEmptyToken2,
-    eExpectingEmptyToken3,
+    eExpectingEmptyTokenOrUniversalFileOrigin,
     eExpectingHost,
     eExpectingPort,
     eExpectingEmptyTokenOrDriveLetterOrPathnameComponent,
@@ -1764,6 +1764,7 @@ private:
   SchemeType mSchemeType;
   State mState;
   bool mInIsolatedMozBrowser;
+  bool mUniversalFileOrigin;
   bool mMaybeDriveLetter;
   bool mError;
 
@@ -1778,6 +1779,7 @@ public:
     , mSchemeType(eNone)
     , mState(eExpectingAppIdOrScheme)
     , mInIsolatedMozBrowser(false)
+    , mUniversalFileOrigin(false)
     , mMaybeDriveLetter(false)
     , mError(false)
   { }
@@ -8491,11 +8493,17 @@ OriginParser::Parse(nsACString& aSpec, OriginAttributes* aAttrs) -> ResultType
   if (mSchemeType == eFile) {
     spec.AppendLiteral("://");
 
-    for (uint32_t count = mPathnameComponents.Length(), index = 0;
-         index < count;
-         index++) {
-      spec.Append('/');
-      spec.Append(mPathnameComponents[index]);
+    if (mUniversalFileOrigin) {
+      MOZ_ASSERT(mPathnameComponents.Length() == 1);
+
+      spec.Append(mPathnameComponents[0]);
+    } else {
+      for (uint32_t count = mPathnameComponents.Length(), index = 0;
+           index < count;
+           index++) {
+        spec.Append('/');
+        spec.Append(mPathnameComponents[index]);
+      }
     }
 
     aSpec = spec;
@@ -8663,7 +8671,7 @@ OriginParser::HandleToken(const nsDependentCSubstring& aToken)
       }
 
       if (mSchemeType == eFile) {
-        mState = eExpectingEmptyToken3;
+        mState = eExpectingEmptyTokenOrUniversalFileOrigin;
       } else {
         mState = eExpectingHost;
       }
@@ -8671,20 +8679,31 @@ OriginParser::HandleToken(const nsDependentCSubstring& aToken)
       return;
     }
 
-    case eExpectingEmptyToken3: {
+    case eExpectingEmptyTokenOrUniversalFileOrigin: {
       MOZ_ASSERT(mSchemeType == eFile);
 
-      if (!aToken.IsEmpty()) {
-        QM_WARNING("Expected the third empty token!");
+      if (aToken.IsEmpty()) {
+        mState = mTokenizer.hasMoreTokens()
+                   ? eExpectingEmptyTokenOrDriveLetterOrPathnameComponent
+                   : eComplete;
 
-        mError = true;
         return;
       }
 
-      mState = mTokenizer.hasMoreTokens()
-                 ? eExpectingEmptyTokenOrDriveLetterOrPathnameComponent
-                 : eComplete;
+      if (aToken.EqualsLiteral("UNIVERSAL_FILE_URI_ORIGIN")) {
+        mUniversalFileOrigin = true;
 
+        mPathnameComponents.AppendElement(aToken);
+
+        mState = eComplete;
+
+        return;
+      }
+
+      QM_WARNING("Expected the third empty token or "
+                 "UNIVERSAL_FILE_URI_ORIGIN!");
+
+      mError = true;
       return;
     }
 
diff --git a/dom/quota/test/unit/test_specialOrigins.js b/dom/quota/test/unit/test_specialOrigins.js
new file mode 100644
index 000000000000..d464e6edcbcc
--- /dev/null
+++ b/dom/quota/test/unit/test_specialOrigins.js
@@ -0,0 +1,44 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+async function testSteps()
+{
+  const origins = [
+    {
+      path: "storage/default/file+++UNIVERSAL_FILE_URI_ORIGIN",
+      url: "file:///Test/test.html",
+      persistence: "default"
+    },
+  ];
+
+  info("Setting pref");
+
+  SpecialPowers.setBoolPref("security.fileuri.strict_origin_policy", false);
+
+  info("Creating origin directories");
+
+  for (let origin of origins) {
+    let originDir = getRelativeFile(origin.path);
+    originDir.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+  }
+
+  info("Initializing origin directories");
+
+  for (let origin of origins) {
+    let result;
+
+    try {
+      let request = initOrigin(getPrincipal(origin.url),
+                               origin.persistence);
+      result = await requestFinished(request);
+
+      ok(true, "Should not have thrown");
+    } catch(ex) {
+      ok(false, "Should not have thrown");
+    }
+
+    ok(!result, "Origin directory wasn't created");
+  }
+}
diff --git a/dom/quota/test/unit/xpcshell.ini b/dom/quota/test/unit/xpcshell.ini
index dfeb62e1b2d6..6104caae4659 100644
--- a/dom/quota/test/unit/xpcshell.ini
+++ b/dom/quota/test/unit/xpcshell.ini
@@ -36,6 +36,7 @@ support-files =
 [test_removeAppsUpgrade.js]
 [test_removeLocalStorage.js]
 [test_simpledb.js]
+[test_specialOrigins.js]
 [test_storagePersistentUpgrade.js]
 [test_storagePressure.js]
 [test_tempMetadataCleanup.js]

From 845fc3a9917de61594866418d063129a0faaa2d0 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:38 +0100
Subject: [PATCH 22/78] Bug 1286798 - Part 8: Persist datastores to disk;
 r=asuth

Introduced a Connection and a ConnectioThread object.
All I/O requests are processed by a new single thread managed by ConnectionThread object.
Connection objects are prepared by the prepare datastore operation and then kept alive by datastores just like directory locks.
All datastore I/O operations are wrapped in a transaction which automaticaly commits after 5 seconds.
Datastore preparation now also loads all items from the database.
---
 dom/localstorage/ActorsParent.cpp | 1356 ++++++++++++++++++++++++++++-
 1 file changed, 1332 insertions(+), 24 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index f4fbf22ce54d..3d144d9f8639 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -23,6 +23,7 @@
 #include "mozilla/ipc/PBackgroundSharedTypes.h"
 #include "nsClassHashtable.h"
 #include "nsDataHashtable.h"
+#include "nsInterfaceHashtable.h"
 #include "nsISimpleEnumerator.h"
 #include "ReportInternalError.h"
 
@@ -46,6 +47,8 @@ using namespace mozilla::ipc;
 
 namespace {
 
+class Connection;
+class ConnectionThread;
 class Database;
 
 /*******************************************************************************
@@ -97,6 +100,14 @@ static_assert(kSQLiteGrowthIncrement >= 0 &&
 #define DATA_FILE_NAME "data.sqlite"
 #define JOURNAL_FILE_NAME "data.sqlite-journal"
 
+const uint32_t kAutoCommitTimeoutMs = 5000;
+
+bool
+IsOnConnectionThread();
+
+void
+AssertIsOnConnectionThread();
+
 /*******************************************************************************
  * SQLite functions
  ******************************************************************************/
@@ -392,6 +403,53 @@ CreateStorageConnection(nsIFile* aDBFile,
   return NS_OK;
 }
 
+nsresult
+GetStorageConnection(const nsAString& aDatabaseFilePath,
+                     mozIStorageConnection** aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(!aDatabaseFilePath.IsEmpty());
+  MOZ_ASSERT(StringEndsWith(aDatabaseFilePath, NS_LITERAL_STRING(".sqlite")));
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> databaseFile;
+  nsresult rv = NS_NewLocalFile(aDatabaseFilePath, false,
+                                getter_AddRefs(databaseFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+  rv = databaseFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!exists)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenDatabase(databaseFile, getter_AddRefs(connection));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = SetDefaultPragmas(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Non-actor class declarations
  ******************************************************************************/
@@ -436,6 +494,15 @@ public:
     return mResultCode;
   }
 
+  void
+  SetFailureCode(nsresult aErrorCode)
+  {
+    MOZ_ASSERT(NS_SUCCEEDED(mResultCode));
+    MOZ_ASSERT(NS_FAILED(aErrorCode));
+
+    mResultCode = aErrorCode;
+  }
+
   void
   MaybeSetFailureCode(nsresult aErrorCode)
   {
@@ -486,9 +553,303 @@ protected:
   }
 };
 
+class ConnectionDatastoreOperationBase
+  : public DatastoreOperationBase
+{
+protected:
+  RefPtr<Connection> mConnection;
+
+public:
+  // This callback will be called on the background thread before releasing the
+  // final reference to this request object. Subclasses may perform any
+  // additional cleanup here but must always call the base class implementation.
+  virtual void
+  Cleanup();
+
+protected:
+  ConnectionDatastoreOperationBase(Connection* aConnection);
+
+  ~ConnectionDatastoreOperationBase();
+
+  // Must be overridden in subclasses. Called on the target thread to allow the
+  // subclass to perform necessary datastore operations. A successful return
+  // value will trigger an OnSuccess callback on the background thread while
+  // while a failure value will trigger an OnFailure callback.
+  virtual nsresult
+  DoDatastoreWork() = 0;
+
+  // Methods that subclasses may implement.
+  virtual void
+  OnSuccess();
+
+  virtual void
+  OnFailure(nsresult aResultCode);
+
+private:
+  void
+  RunOnConnectionThread();
+
+  void
+  RunOnOwningThread();
+
+  // Not to be overridden by subclasses.
+  NS_DECL_NSIRUNNABLE
+};
+
+class Connection final
+{
+  friend class ConnectionThread;
+
+public:
+  class CachedStatement;
+
+private:
+  class BeginOp;
+  class CommitOp;
+  class CloseOp;
+
+  RefPtr<ConnectionThread> mConnectionThread;
+  nsCOMPtr<mozIStorageConnection> mStorageConnection;
+  nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
+    mCachedStatements;
+  const nsCString mOrigin;
+  const nsString mFilePath;
+  bool mInTransaction;
+
+public:
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Connection)
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(Connection);
+  }
+
+  // Methods which can only be called on the owning thread.
+
+  bool
+  InTransaction()
+  {
+    AssertIsOnOwningThread();
+    return mInTransaction;
+  }
+
+  // This method is used to asynchronously execute a connection datastore
+  // operation on the connection thread.
+  void
+  Dispatch(ConnectionDatastoreOperationBase* aOp);
+
+  // This method is used to asynchronously start a transaction on the connection
+  // thread.
+  void
+  Begin(bool aReadonly);
+
+  // This method is used to asynchronously end a transaction on the connection
+  // thread.
+  void
+  Commit();
+
+  // This method is used to asynchronously close the storage connection on the
+  // connection thread.
+  void
+  Close(nsIRunnable* aCallback);
+
+  // Methods which can only be called on the connection thread.
+
+  nsresult
+  EnsureStorageConnection();
+
+  mozIStorageConnection*
+  StorageConnection() const
+  {
+    AssertIsOnConnectionThread();
+    MOZ_ASSERT(mStorageConnection);
+
+    return mStorageConnection;
+  }
+
+  void
+  CloseStorageConnection();
+
+  nsresult
+  GetCachedStatement(const nsACString& aQuery,
+                     CachedStatement* aCachedStatement);
+
+private:
+  // Only created by ConnectionThread.
+  Connection(ConnectionThread* aConnectionThread,
+             const nsACString& aOrigin,
+             const nsAString& aFilePath);
+
+  ~Connection();
+};
+
+class Connection::CachedStatement final
+{
+  friend class Connection;
+
+  nsCOMPtr<mozIStorageStatement> mStatement;
+  Maybe<mozStorageStatementScoper> mScoper;
+
+public:
+  CachedStatement();
+  ~CachedStatement();
+
+  mozIStorageStatement*
+  operator->() const MOZ_NO_ADDREF_RELEASE_ON_RETURN;
+
+private:
+  // Only called by Connection.
+  void
+  Assign(Connection* aConnection,
+         already_AddRefed<mozIStorageStatement> aStatement);
+
+  // No funny business allowed.
+  CachedStatement(const CachedStatement&) = delete;
+  CachedStatement& operator=(const CachedStatement&) = delete;
+};
+
+class Connection::BeginOp final
+  : public ConnectionDatastoreOperationBase
+{
+  const bool mReadonly;
+
+public:
+  BeginOp(Connection* aConnection,
+          bool aReadonly)
+    : ConnectionDatastoreOperationBase(aConnection)
+    , mReadonly(aReadonly)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+};
+
+class Connection::CommitOp final
+  : public ConnectionDatastoreOperationBase
+{
+public:
+  explicit CommitOp(Connection* aConnection)
+    : ConnectionDatastoreOperationBase(aConnection)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+};
+
+class Connection::CloseOp final
+  : public ConnectionDatastoreOperationBase
+{
+  nsCOMPtr<nsIRunnable> mCallback;
+
+public:
+  CloseOp(Connection* aConnection,
+          nsIRunnable* aCallback)
+    : ConnectionDatastoreOperationBase(aConnection)
+    , mCallback(aCallback)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+
+  void
+  Cleanup() override;
+};
+
+class ConnectionThread final
+{
+  friend class Connection;
+
+  nsCOMPtr<nsIThread> mThread;
+  nsRefPtrHashtable<nsCStringHashKey, Connection> mConnections;
+
+public:
+  ConnectionThread();
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(ConnectionThread);
+  }
+
+  bool
+  IsOnConnectionThread();
+
+  void
+  AssertIsOnConnectionThread();
+
+  already_AddRefed<Connection>
+  CreateConnection(const nsACString& aOrigin,
+                   const nsAString& aFilePath);
+
+  void
+  Shutdown();
+
+  NS_INLINE_DECL_REFCOUNTING(ConnectionThread)
+
+private:
+  ~ConnectionThread();
+};
+
+class SetItemOp final
+  : public ConnectionDatastoreOperationBase
+{
+  nsString mKey;
+  nsString mValue;
+
+public:
+  SetItemOp(Connection* aConnection,
+            const nsAString& aKey,
+            const nsAString& aValue)
+    : ConnectionDatastoreOperationBase(aConnection)
+    , mKey(aKey)
+    , mValue(aValue)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+};
+
+class RemoveItemOp final
+  : public ConnectionDatastoreOperationBase
+{
+  nsString mKey;
+
+public:
+  RemoveItemOp(Connection* aConnection,
+               const nsAString& aKey)
+    : ConnectionDatastoreOperationBase(aConnection)
+    , mKey(aKey)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+};
+
+class ClearOp final
+  : public ConnectionDatastoreOperationBase
+{
+public:
+  explicit ClearOp(Connection* aConnection)
+    : ConnectionDatastoreOperationBase(aConnection)
+  { }
+
+private:
+  nsresult
+  DoDatastoreWork() override;
+};
+
 class Datastore final
 {
   RefPtr<DirectoryLock> mDirectoryLock;
+  RefPtr<Connection> mConnection;
+  nsCOMPtr<nsITimer> mAutoCommitTimer;
+  nsCOMPtr<nsIRunnable> mCompleteCallback;
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const nsCString mOrigin;
@@ -497,7 +858,9 @@ class Datastore final
 public:
   // Created by PrepareDatastoreOp.
   Datastore(const nsACString& aOrigin,
-            already_AddRefed<DirectoryLock>&& aDirectoryLock);
+            already_AddRefed<DirectoryLock>&& aDirectoryLock,
+            already_AddRefed<Connection>&& aConnection,
+            nsDataHashtable<nsStringHashKey, nsString>& aValues);
 
   const nsCString&
   Origin() const
@@ -516,6 +879,9 @@ public:
     return mClosed;
   }
 
+  void
+  WaitForConnectionToComplete(nsIRunnable* aCallback);
+
   void
   NoteLiveDatabase(Database* aDatabase);
 
@@ -551,6 +917,15 @@ public:
 private:
   // Reference counted.
   ~Datastore();
+
+  void
+  ConnectionClosedCallback();
+
+  void
+  EnsureTransaction();
+
+  static void
+  AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure);
 };
 
 class PreparedDatastore
@@ -687,7 +1062,7 @@ public:
   virtual void
   Dispatch() = 0;
 
-private:
+protected:
   // IPDL methods.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
@@ -700,18 +1075,25 @@ class PrepareDatastoreOp
   : public LSRequestBase
   , public OpenDirectoryListener
 {
+  class LoadDataOp;
+
   enum class State
   {
     // Just created on the PBackground thread. Next step is OpeningOnMainThread
     // or OpeningOnOwningThread.
     Initial,
 
-    // Waiting to open/opening on the main thread. Next step is FinishOpen.
+    // Waiting to open/opening on the main thread. Next step is
+    // CheckExistingOperations.
     Opening,
 
     // Checking if a prepare datastore operation is already running for given
-    // origin on the PBackground thread. Next step is PreparationPending.
-    FinishOpen,
+    // origin on the PBackground thread. Next step is CheckClosingDatastore.
+    CheckExistingOperations,
+
+    // Checking if a datastore is closing the connection for given origin on
+    // the PBackground thread. Next step is PreparationPending.
+    CheckClosingDatastore,
 
     // Opening directory or initializing quota manager on the PBackground
     // thread. Next step is either DirectoryOpenPending if quota manager is
@@ -732,9 +1114,18 @@ class PrepareDatastoreOp
     DirectoryOpenPending,
 
     // Waiting to do/doing work on the QuotaManager IO thread. Its next step is
-    // SendingReadyMessage.
+    // BeginLoadData.
     DatabaseWorkOpen,
 
+    // Starting a load data operation on the PBackground thread. Next step is
+    // DatabaseWorkLoadData.
+    BeginLoadData,
+
+    // Waiting to do/doing work on the connection thread. This involves waiting
+    // for the LoadDataOp to do its work. Eventually the state will transition
+    // to SendingReadyMessage.
+    DatabaseWorkLoadData,
+
     // Waiting to send/sending the ready message on the PBackground thread. Next
     // step is WaitingForFinish.
     SendingReadyMessage,
@@ -754,12 +1145,16 @@ class PrepareDatastoreOp
   nsCOMPtr<nsIEventTarget> mMainEventTarget;
   RefPtr<PrepareDatastoreOp> mDelayedOp;
   RefPtr<DirectoryLock> mDirectoryLock;
+  RefPtr<Connection> mConnection;
   RefPtr<Datastore> mDatastore;
+  LoadDataOp* mLoadDataOp;
+  nsDataHashtable<nsStringHashKey, nsString> mValues;
   const LSRequestPrepareDatastoreParams mParams;
   nsCString mSuffix;
   nsCString mGroup;
   nsCString mMainThreadOrigin;
   nsCString mOrigin;
+  nsString mDatabaseFilePath;
   State mState;
   bool mRequestedDirectoryLock;
   bool mInvalidated;
@@ -811,7 +1206,10 @@ private:
   Open();
 
   nsresult
-  FinishOpen();
+  CheckExistingOperations();
+
+  nsresult
+  CheckClosingDatastore();
 
   nsresult
   PreparationOpen();
@@ -834,6 +1232,9 @@ private:
   nsresult
   VerifyDatabaseInformation(mozIStorageConnection* aConnection);
 
+  nsresult
+  BeginLoadData();
+
   void
   SendReadyMessage();
 
@@ -843,12 +1244,21 @@ private:
   void
   Cleanup();
 
+  void
+  ConnectionClosedCallback();
+
+  void
+  CleanupMetadata();
+
   NS_DECL_ISUPPORTS_INHERITED
 
   NS_IMETHOD
   Run() override;
 
   // IPDL overrides.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
   mozilla::ipc::IPCResult
   RecvFinish() override;
 
@@ -860,6 +1270,33 @@ private:
   DirectoryLockFailed() override;
 };
 
+class PrepareDatastoreOp::LoadDataOp final
+  : public ConnectionDatastoreOperationBase
+{
+  RefPtr<PrepareDatastoreOp> mPrepareDatastoreOp;
+
+public:
+  explicit LoadDataOp(PrepareDatastoreOp* aPrepareDatastoreOp)
+    : ConnectionDatastoreOperationBase(aPrepareDatastoreOp->mConnection)
+    , mPrepareDatastoreOp(aPrepareDatastoreOp)
+  { }
+
+private:
+  ~LoadDataOp() = default;
+
+  nsresult
+  DoDatastoreWork() override;
+
+  void
+  OnSuccess() override;
+
+  void
+  OnFailure(nsresult aResultCode) override;
+
+  void
+  Cleanup() override;
+};
+
 /*******************************************************************************
  * Other class declarations
  ******************************************************************************/
@@ -971,6 +1408,22 @@ typedef nsTArray<Database*> LiveDatabaseArray;
 
 StaticAutoPtr<LiveDatabaseArray> gLiveDatabases;
 
+StaticRefPtr<ConnectionThread> gConnectionThread;
+
+bool
+IsOnConnectionThread()
+{
+  MOZ_ASSERT(gConnectionThread);
+  return gConnectionThread->IsOnConnectionThread();
+}
+
+void
+AssertIsOnConnectionThread()
+{
+  MOZ_ASSERT(gConnectionThread);
+  gConnectionThread->AssertIsOnConnectionThread();
+}
+
 } // namespace
 
 /*******************************************************************************
@@ -1143,17 +1596,518 @@ CreateQuotaClient()
  * DatastoreOperationBase
  ******************************************************************************/
 
+/*******************************************************************************
+ * ConnectionDatastoreOperationBase
+ ******************************************************************************/
+
+ConnectionDatastoreOperationBase::ConnectionDatastoreOperationBase(
+                                                        Connection* aConnection)
+  : mConnection(aConnection)
+{
+  MOZ_ASSERT(aConnection);
+}
+
+ConnectionDatastoreOperationBase::~ConnectionDatastoreOperationBase()
+{
+  MOZ_ASSERT(!mConnection,
+             "ConnectionDatabaseOperationBase::Cleanup() was not called by a "
+             "subclass!");
+}
+
+void
+ConnectionDatastoreOperationBase::Cleanup()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mConnection);
+
+  mConnection = nullptr;
+
+  NoteComplete();
+}
+
+void
+ConnectionDatastoreOperationBase::OnSuccess()
+{
+  AssertIsOnOwningThread();
+}
+
+void
+ConnectionDatastoreOperationBase::OnFailure(nsresult aResultCode)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(NS_FAILED(aResultCode));
+}
+
+void
+ConnectionDatastoreOperationBase::RunOnConnectionThread()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+  MOZ_ASSERT(NS_SUCCEEDED(ResultCode()));
+
+  if (!MayProceedOnNonOwningThread()) {
+    SetFailureCode(NS_ERROR_FAILURE);
+  } else {
+    nsresult rv = mConnection->EnsureStorageConnection();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      SetFailureCode(rv);
+    } else {
+      MOZ_ASSERT(mConnection->StorageConnection());
+
+      rv = DoDatastoreWork();
+      if (NS_FAILED(rv)) {
+        SetFailureCode(rv);
+      }
+    }
+  }
+
+  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+}
+
+void
+ConnectionDatastoreOperationBase::RunOnOwningThread()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mConnection);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+  } else {
+    if (NS_SUCCEEDED(ResultCode())) {
+      OnSuccess();
+    } else {
+      OnFailure(ResultCode());
+    }
+  }
+
+  Cleanup();
+}
+
+NS_IMETHODIMP
+ConnectionDatastoreOperationBase::Run()
+{
+  if (IsOnConnectionThread()) {
+    RunOnConnectionThread();
+  } else {
+    RunOnOwningThread();
+  }
+
+  return NS_OK;
+}
+
+/*******************************************************************************
+ * Connection implementation
+ ******************************************************************************/
+
+Connection::Connection(ConnectionThread* aConnectionThread,
+                       const nsACString& aOrigin,
+                       const nsAString& aFilePath)
+  : mConnectionThread(aConnectionThread)
+  , mOrigin(aOrigin)
+  , mFilePath(aFilePath)
+  , mInTransaction(false)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!aOrigin.IsEmpty());
+  MOZ_ASSERT(!aFilePath.IsEmpty());
+}
+
+Connection::~Connection()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!mStorageConnection);
+  MOZ_ASSERT(!mCachedStatements.Count());
+  MOZ_ASSERT(!mInTransaction);
+}
+
+void
+Connection::Dispatch(ConnectionDatastoreOperationBase* aOp)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mConnectionThread);
+
+  MOZ_ALWAYS_SUCCEEDS(mConnectionThread->mThread->Dispatch(aOp,
+                                                           NS_DISPATCH_NORMAL));
+}
+
+void
+Connection::Begin(bool aReadonly)
+{
+  AssertIsOnOwningThread();
+
+  RefPtr<BeginOp> op = new BeginOp(this, aReadonly);
+
+  Dispatch(op);
+
+  mInTransaction = true;
+}
+
+void
+Connection::Commit()
+{
+  AssertIsOnOwningThread();
+
+  RefPtr<CommitOp> op = new CommitOp(this);
+
+  Dispatch(op);
+
+  mInTransaction = false;
+}
+
+void
+Connection::Close(nsIRunnable* aCallback)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aCallback);
+
+  RefPtr<CloseOp> op = new CloseOp(this, aCallback);
+
+  Dispatch(op);
+}
+
+nsresult
+Connection::EnsureStorageConnection()
+{
+  AssertIsOnConnectionThread();
+
+  if (!mStorageConnection) {
+    nsCOMPtr<mozIStorageConnection> storageConnection;
+    nsresult rv =
+      GetStorageConnection(mFilePath,
+                           getter_AddRefs(storageConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    mStorageConnection = storageConnection;
+  }
+
+  return NS_OK;
+}
+
+void
+Connection::CloseStorageConnection()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mStorageConnection);
+
+  mCachedStatements.Clear();
+
+  MOZ_ALWAYS_SUCCEEDS(mStorageConnection->Close());
+  mStorageConnection = nullptr;
+}
+
+nsresult
+Connection::GetCachedStatement(const nsACString& aQuery,
+                                       CachedStatement* aCachedStatement)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(!aQuery.IsEmpty());
+  MOZ_ASSERT(aCachedStatement);
+  MOZ_ASSERT(mStorageConnection);
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+
+  if (!mCachedStatements.Get(aQuery, getter_AddRefs(stmt))) {
+    nsresult rv =
+      mStorageConnection->CreateStatement(aQuery, getter_AddRefs(stmt));
+    if (NS_FAILED(rv)) {
+#ifdef DEBUG
+      nsCString msg;
+      MOZ_ALWAYS_SUCCEEDS(mStorageConnection->GetLastErrorString(msg));
+
+      nsAutoCString error =
+        NS_LITERAL_CSTRING("The statement '") + aQuery +
+        NS_LITERAL_CSTRING("' failed to compile with the error message '") +
+        msg + NS_LITERAL_CSTRING("'.");
+
+      NS_WARNING(error.get());
+#endif
+      return rv;
+    }
+
+    mCachedStatements.Put(aQuery, stmt);
+  }
+
+  aCachedStatement->Assign(this, stmt.forget());
+  return NS_OK;
+}
+
+Connection::
+CachedStatement::CachedStatement()
+{
+  AssertIsOnConnectionThread();
+
+  MOZ_COUNT_CTOR(Connection::CachedStatement);
+}
+
+Connection::
+CachedStatement::~CachedStatement()
+{
+  AssertIsOnConnectionThread();
+
+  MOZ_COUNT_DTOR(Connection::CachedStatement);
+}
+
+mozIStorageStatement*
+Connection::
+CachedStatement::operator->() const
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mStatement);
+
+  return mStatement;
+}
+
+void
+Connection::
+CachedStatement::Assign(Connection* aConnection,
+                        already_AddRefed<mozIStorageStatement> aStatement)
+{
+  AssertIsOnConnectionThread();
+
+  mScoper.reset();
+
+  mStatement = aStatement;
+
+  if (mStatement) {
+    mScoper.emplace(mStatement);
+  }
+}
+
+nsresult
+Connection::
+BeginOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  CachedStatement stmt;
+  nsresult rv;
+  if (mReadonly) {
+    rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN;"), &stmt);
+  } else {
+    rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
+                                         &stmt);
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+Connection::
+CommitOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  CachedStatement stmt;
+  nsresult rv =
+    mConnection->GetCachedStatement(NS_LITERAL_CSTRING("COMMIT;"), &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+Connection::
+CloseOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  mConnection->CloseStorageConnection();
+
+  return NS_OK;
+}
+
+void
+Connection::
+CloseOp::Cleanup()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mConnection);
+
+  mConnection->mConnectionThread->mConnections.Remove(mConnection->mOrigin);
+
+  nsCOMPtr<nsIRunnable> callback;
+  mCallback.swap(callback);
+
+  callback->Run();
+
+  ConnectionDatastoreOperationBase::Cleanup();
+}
+
+/*******************************************************************************
+ * ConnectionThread implementation
+ ******************************************************************************/
+
+ConnectionThread::ConnectionThread()
+{
+  AssertIsOnOwningThread();
+  AssertIsOnBackgroundThread();
+
+  MOZ_ALWAYS_SUCCEEDS(NS_NewNamedThread("LS Thread", getter_AddRefs(mThread)));
+}
+
+ConnectionThread::~ConnectionThread()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!mConnections.Count());
+}
+
+bool
+ConnectionThread::IsOnConnectionThread()
+{
+  MOZ_ASSERT(mThread);
+
+  bool current;
+  return NS_SUCCEEDED(mThread->IsOnCurrentThread(&current)) && current;
+}
+
+void
+ConnectionThread::AssertIsOnConnectionThread()
+{
+  MOZ_ASSERT(IsOnConnectionThread());
+}
+
+already_AddRefed<Connection>
+ConnectionThread::CreateConnection(const nsACString& aOrigin,
+                                   const nsAString& aFilePath)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!aOrigin.IsEmpty());
+  MOZ_ASSERT(!mConnections.GetWeak(aOrigin));
+
+  RefPtr<Connection> connection = new Connection(this, aOrigin, aFilePath);
+  mConnections.Put(aOrigin, connection);
+
+  return connection.forget();
+}
+
+void
+ConnectionThread::Shutdown()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mThread);
+
+  mThread->Shutdown();
+}
+
+nsresult
+SetItemOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO data (key, value) "
+    "VALUES(:key, :value)"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+RemoveItemOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data "
+      "WHERE key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+ClearOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Datastore
  ******************************************************************************/
 
 Datastore::Datastore(const nsACString& aOrigin,
-                     already_AddRefed<DirectoryLock>&& aDirectoryLock)
+                     already_AddRefed<DirectoryLock>&& aDirectoryLock,
+                     already_AddRefed<Connection>&& aConnection,
+                     nsDataHashtable<nsStringHashKey, nsString>& aValues)
   : mDirectoryLock(std::move(aDirectoryLock))
+  , mConnection(std::move(aConnection))
   , mOrigin(aOrigin)
   , mClosed(false)
 {
   AssertIsOnBackgroundThread();
+
+  mValues.SwapElements(aValues);
 }
 
 Datastore::~Datastore()
@@ -1169,18 +2123,37 @@ Datastore::Close()
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(!mDatabases.Count());
   MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(mConnection);
 
   mClosed = true;
 
-  mDirectoryLock = nullptr;
+  if (mConnection->InTransaction()) {
+    MOZ_ASSERT(mAutoCommitTimer);
+    MOZ_ALWAYS_SUCCEEDS(mAutoCommitTimer->Cancel());
 
-  MOZ_ASSERT(gDatastores);
-  MOZ_ASSERT(gDatastores->Get(mOrigin));
-  gDatastores->Remove(mOrigin);
+    mConnection->Commit();
 
-  if (!gDatastores->Count()) {
-    gDatastores = nullptr;
+    mAutoCommitTimer = nullptr;
   }
+
+  // We can't release the directory lock and unregister itself from the
+  // hashtable until the connection is fully closed.
+  nsCOMPtr<nsIRunnable> callback =
+    NewRunnableMethod("dom::Datastore::ConnectionClosedCallback",
+                      this,
+                      &Datastore::ConnectionClosedCallback);
+  mConnection->Close(callback);
+}
+
+void
+Datastore::WaitForConnectionToComplete(nsIRunnable* aCallback)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aCallback);
+  MOZ_ASSERT(!mCompleteCallback);
+  MOZ_ASSERT(mClosed);
+
+  mCompleteCallback = aCallback;
 }
 
 void
@@ -1223,6 +2196,7 @@ uint32_t
 Datastore::GetLength() const
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   return mValues.Count();
 }
@@ -1231,6 +2205,7 @@ void
 Datastore::GetKey(uint32_t aIndex, nsString& aKey) const
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   aKey.SetIsVoid(true);
   for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
@@ -1246,6 +2221,7 @@ void
 Datastore::GetItem(const nsString& aKey, nsString& aValue) const
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   if (!mValues.Get(aKey, &aValue)) {
     aValue.SetIsVoid(true);
@@ -1256,36 +2232,121 @@ void
 Datastore::SetItem(const nsString& aKey, const nsString& aValue)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   mValues.Put(aKey, aValue);
+
+  EnsureTransaction();
+
+  RefPtr<SetItemOp> op = new SetItemOp(mConnection, aKey, aValue);
+  mConnection->Dispatch(op);
 }
 
 void
 Datastore::RemoveItem(const nsString& aKey)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   mValues.Remove(aKey);
+
+  EnsureTransaction();
+
+  RefPtr<RemoveItemOp> op = new RemoveItemOp(mConnection, aKey);
+  mConnection->Dispatch(op);
 }
 
 void
 Datastore::Clear()
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   mValues.Clear();
+
+  EnsureTransaction();
+
+  RefPtr<ClearOp> op = new ClearOp(mConnection);
+  mConnection->Dispatch(op);
 }
 
 void
 Datastore::GetKeys(nsTArray<nsString>& aKeys) const
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
 
   for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
     aKeys.AppendElement(iter.Key());
   }
 }
 
+void
+Datastore::ConnectionClosedCallback()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(mConnection);
+  MOZ_ASSERT(mClosed);
+
+  // Now it's safe to release the directory lock and unregister itself from
+  // the hashtable.
+
+  mDirectoryLock = nullptr;
+  mConnection = nullptr;
+
+  MOZ_ASSERT(gDatastores);
+  MOZ_ASSERT(gDatastores->Get(mOrigin));
+  gDatastores->Remove(mOrigin);
+
+  if (!gDatastores->Count()) {
+    gDatastores = nullptr;
+  }
+
+  if (mCompleteCallback) {
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(mCompleteCallback.forget()));
+  }
+}
+
+void
+Datastore::EnsureTransaction()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mConnection);
+
+  if (!mConnection->InTransaction()) {
+    mConnection->Begin(/* aReadonly */ false);
+
+    if (!mAutoCommitTimer) {
+      mAutoCommitTimer = NS_NewTimer();
+      MOZ_ASSERT(mAutoCommitTimer);
+    }
+
+    MOZ_ALWAYS_SUCCEEDS(
+      mAutoCommitTimer->InitWithNamedFuncCallback(
+                                          AutoCommitTimerCallback,
+                                          this,
+                                          kAutoCommitTimeoutMs,
+                                          nsITimer::TYPE_ONE_SHOT,
+                                          "Database::AutoCommitTimerCallback"));
+  }
+}
+
+// static
+void
+Datastore::AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure)
+{
+  MOZ_ASSERT(aClosure);
+
+  auto* self = static_cast<Datastore*>(aClosure);
+  MOZ_ASSERT(self);
+
+  MOZ_ASSERT(self->mConnection);
+  MOZ_ASSERT(self->mConnection->InTransaction());
+
+  self->mConnection->Commit();
+}
+
 /*******************************************************************************
  * Database
  ******************************************************************************/
@@ -1560,6 +2621,7 @@ LSRequestBase::RecvCancel()
 PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
                                        const LSRequestParams& aParams)
   : mMainEventTarget(aMainEventTarget)
+  , mLoadDataOp(nullptr)
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mState(State::Initial)
   , mRequestedDirectoryLock(false)
@@ -1574,6 +2636,7 @@ PrepareDatastoreOp::~PrepareDatastoreOp()
   MOZ_ASSERT(!mDirectoryLock);
   MOZ_ASSERT_IF(MayProceedOnNonOwningThread(),
                 mState == State::Initial || mState == State::Completed);
+  MOZ_ASSERT(!mLoadDataOp);
 }
 
 void
@@ -1630,17 +2693,17 @@ PrepareDatastoreOp::Open()
     return NS_ERROR_FAILURE;
   }
 
-  mState = State::FinishOpen;
+  mState = State::CheckExistingOperations;
   MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
 
   return NS_OK;
 }
 
 nsresult
-PrepareDatastoreOp::FinishOpen()
+PrepareDatastoreOp::CheckExistingOperations()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::FinishOpen);
+  MOZ_ASSERT(mState == State::CheckExistingOperations);
   MOZ_ASSERT(gPrepareDatastoreOps);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
@@ -1658,7 +2721,7 @@ PrepareDatastoreOp::FinishOpen()
 
   MOZ_ASSERT(!mOrigin.IsEmpty());
 
-  mState = State::PreparationPending;
+  mState = State::CheckClosingDatastore;
 
   // See if this PrepareDatastoreOp needs to wait.
   bool foundThis = false;
@@ -1679,6 +2742,36 @@ PrepareDatastoreOp::FinishOpen()
     }
   }
 
+  nsresult rv = CheckClosingDatastore();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::CheckClosingDatastore()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::CheckClosingDatastore);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  mState = State::PreparationPending;
+
+  RefPtr<Datastore> datastore;
+  if (gDatastores &&
+      (datastore = gDatastores->Get(mOrigin)) &&
+      datastore->IsClosed()) {
+    datastore->WaitForConnectionToComplete(this);
+
+    return NS_OK;
+  }
+
   nsresult rv = BeginDatastorePreparation();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -1694,6 +2787,8 @@ PrepareDatastoreOp::BeginDatastorePreparation()
   MOZ_ASSERT(mState == State::PreparationPending);
 
   if (gDatastores && (mDatastore = gDatastores->Get(mOrigin))) {
+    MOZ_ASSERT(!mDatastore->IsClosed());
+
     mState = State::SendingReadyMessage;
     Unused << this->Run();
 
@@ -1848,6 +2943,11 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
+  rv = dbFile->GetPath(mDatabaseFilePath);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   nsCOMPtr<mozIStorageConnection> connection;
   rv = CreateStorageConnection(dbFile, mOrigin, getter_AddRefs(connection));
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -1859,9 +2959,13 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
+  // Must close connection before dispatching otherwise we might race with the
+  // connection thread which needs to open the same database.
+  MOZ_ALWAYS_SUCCEEDS(connection->Close());
+
   // Must set mState before dispatching otherwise we will race with the owning
   // thread.
-  mState = State::SendingReadyMessage;
+  mState = State::BeginLoadData;
 
   rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -1909,6 +3013,46 @@ PrepareDatastoreOp::VerifyDatabaseInformation(mozIStorageConnection* aConnection
   return NS_OK;
 }
 
+nsresult
+PrepareDatastoreOp::BeginLoadData()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::BeginLoadData);
+  MOZ_ASSERT(!mConnection);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  if (!gConnectionThread) {
+    gConnectionThread = new ConnectionThread();
+  }
+
+  mConnection = gConnectionThread->CreateConnection(mOrigin,
+                                                    mDatabaseFilePath);
+  MOZ_ASSERT(mConnection);
+
+  MOZ_ASSERT(!mConnection->InTransaction());
+
+  // Must set this before dispatching otherwise we will race with the
+  // connection thread.
+  mState = State::DatabaseWorkLoadData;
+
+  // Can't assign to mLoadDataOp directly since that's a weak reference and
+  // LoadDataOp is reference counted.
+  RefPtr<LoadDataOp> loadDataOp = new LoadDataOp(this);
+
+  // This add refs loadDataOp.
+  mConnection->Dispatch(loadDataOp);
+
+  // This is cleared in LoadDataOp::Cleanup() before the load data op is
+  // destroyed.
+  mLoadDataOp = loadDataOp;
+
+  return NS_OK;
+}
+
 void
 PrepareDatastoreOp::SendReadyMessage()
 {
@@ -1941,7 +3085,10 @@ PrepareDatastoreOp::SendResults()
 
     if (NS_SUCCEEDED(ResultCode())) {
       if (!mDatastore) {
-        mDatastore = new Datastore(mOrigin, mDirectoryLock.forget());
+        mDatastore = new Datastore(mOrigin,
+                                   mDirectoryLock.forget(),
+                                   mConnection.forget(),
+                                   mValues);
 
         if (!gDatastores) {
           gDatastores = new DatastoreHashtable();
@@ -1991,6 +3138,7 @@ PrepareDatastoreOp::Cleanup()
 
   if (mDatastore) {
     MOZ_ASSERT(!mDirectoryLock);
+    MOZ_ASSERT(!mConnection);
 
     if (NS_FAILED(ResultCode())) {
       MOZ_ASSERT(!mDatastore->IsClosed());
@@ -2000,12 +3148,54 @@ PrepareDatastoreOp::Cleanup()
 
     // Make sure to release the datastore on this thread.
     mDatastore = nullptr;
-  } else if (mDirectoryLock) {
-    // If we have a directory lock then the operation must have failed.
+
+    CleanupMetadata();
+  } else if (mConnection) {
+    // If we have a connection then the operation must have failed and there
+    // must be a directory lock too.
     MOZ_ASSERT(NS_FAILED(ResultCode()));
+    MOZ_ASSERT(mDirectoryLock);
+
+    // We must close the connection on the connection thread before releasing
+    // it on this thread. The directory lock can't be released either.
+    nsCOMPtr<nsIRunnable> callback =
+      NewRunnableMethod("dom::OpenDatabaseOp::ConnectionClosedCallback",
+                        this,
+                        &PrepareDatastoreOp::ConnectionClosedCallback);
+
+    mConnection->Close(callback);
+  } else {
+    // If we don't have a connection, but we do have a directory lock then the
+    // operation must have failed too.
+    MOZ_ASSERT_IF(mDirectoryLock, NS_FAILED(ResultCode()));
+
+    // There's no connection, so it's safe to release the directory lock and
+    // unregister itself from the array.
 
     mDirectoryLock = nullptr;
+
+    CleanupMetadata();
   }
+}
+
+void
+PrepareDatastoreOp::ConnectionClosedCallback()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(NS_FAILED(ResultCode()));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(mConnection);
+
+  mConnection = nullptr;
+  mDirectoryLock = nullptr;
+
+  CleanupMetadata();
+}
+
+void
+PrepareDatastoreOp::CleanupMetadata()
+{
+  AssertIsOnOwningThread();
 
   if (mDelayedOp) {
     MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(mDelayedOp.forget()));
@@ -2031,8 +3221,12 @@ PrepareDatastoreOp::Run()
       rv = Open();
       break;
 
-    case State::FinishOpen:
-      rv = FinishOpen();
+    case State::CheckExistingOperations:
+      rv = CheckExistingOperations();
+      break;
+
+    case State::CheckClosingDatastore:
+      rv = CheckClosingDatastore();
       break;
 
     case State::PreparationPending:
@@ -2047,6 +3241,10 @@ PrepareDatastoreOp::Run()
       rv = DatabaseWork();
       break;
 
+    case State::BeginLoadData:
+      rv = BeginLoadData();
+      break;
+
     case State::SendingReadyMessage:
       SendReadyMessage();
       return NS_OK;
@@ -2077,6 +3275,18 @@ PrepareDatastoreOp::Run()
   return NS_OK;
 }
 
+void
+PrepareDatastoreOp::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  LSRequestBase::ActorDestroy(aWhy);
+
+  if (mLoadDataOp) {
+    mLoadDataOp->NoteComplete();
+  }
+}
+
 mozilla::ipc::IPCResult
 PrepareDatastoreOp::RecvFinish()
 {
@@ -2128,6 +3338,95 @@ PrepareDatastoreOp::DirectoryLockFailed()
   MOZ_ALWAYS_SUCCEEDS(Run());
 }
 
+nsresult
+PrepareDatastoreOp::
+LoadDataOp::DoDatastoreWork()
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(mConnection);
+  MOZ_ASSERT(mPrepareDatastoreOp);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
+      !MayProceedOnNonOwningThread()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  Connection::CachedStatement stmt;
+  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "SELECT key, value "
+      "FROM data;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&hasResult)) && hasResult) {
+    nsString key;
+    rv = stmt->GetString(0, key);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsString value;
+    rv = stmt->GetString(1, value);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    mPrepareDatastoreOp->mValues.Put(key, value);
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+void
+PrepareDatastoreOp::
+LoadDataOp::OnSuccess()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mPrepareDatastoreOp);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+  MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
+
+  mPrepareDatastoreOp->mState = State::SendingReadyMessage;
+
+  MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
+}
+
+void
+PrepareDatastoreOp::
+LoadDataOp::OnFailure(nsresult aResultCode)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mPrepareDatastoreOp);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+  MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
+
+  mPrepareDatastoreOp->SetFailureCode(aResultCode);
+  mPrepareDatastoreOp->mState = State::SendingReadyMessage;
+
+  MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
+}
+
+void
+PrepareDatastoreOp::
+LoadDataOp::Cleanup()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mPrepareDatastoreOp);
+  MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
+
+  mPrepareDatastoreOp->mLoadDataOp = nullptr;
+  mPrepareDatastoreOp = nullptr;
+
+  ConnectionDatastoreOperationBase::Cleanup();
+}
+
 /*******************************************************************************
  * QuotaClient
  ******************************************************************************/
@@ -2446,10 +3745,19 @@ QuotaClient::ShutdownWorkThreads()
     }
   }
 
+  // This should release any local storage related quota objects or directory
+  // locks.
   MOZ_ALWAYS_TRUE(SpinEventLoopUntil([&]() {
     // Don't have to check gPreparedDatastores since we nulled it out above.
     return !gPrepareDatastoreOps && !gDatastores && !gLiveDatabases;
   }));
+
+  // And finally, shutdown the connection thread.
+  if (gConnectionThread) {
+    gConnectionThread->Shutdown();
+
+    gConnectionThread = nullptr;
+  }
 }
 
 } // namespace dom

From 08a07f2f3646577ecaa054bc69aa528f4f8856ca Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:41 +0100
Subject: [PATCH 23/78] Bug 1286798 - Part 9: Support for private browsing;
 r=asuth

Since we keep/cache data in memory anyway, private browsing support is mostly about avoiding any persistence related calls and clearing private browsing datastores when we get a notification. The separation between normal and private browsing datastores is done by the privateBrowsingId attribute which is part of the origin string.
---
 .../privatebrowsing/test/browser/browser.ini  |   2 +-
 dom/localstorage/ActorsParent.cpp             | 230 ++++++++++++++++--
 2 files changed, 213 insertions(+), 19 deletions(-)

diff --git a/browser/components/privatebrowsing/test/browser/browser.ini b/browser/components/privatebrowsing/test/browser/browser.ini
index 6db62a2fa6fe..43947f3de878 100644
--- a/browser/components/privatebrowsing/test/browser/browser.ini
+++ b/browser/components/privatebrowsing/test/browser/browser.ini
@@ -25,7 +25,7 @@ tags = trackingprotection
 [browser_privatebrowsing_aboutSessionRestore.js]
 [browser_privatebrowsing_cache.js]
 [browser_privatebrowsing_certexceptionsui.js]
-#[browser_privatebrowsing_concurrent.js]
+[browser_privatebrowsing_concurrent.js]
 [browser_privatebrowsing_context_and_chromeFlags.js]
 [browser_privatebrowsing_crh.js]
 [browser_privatebrowsing_downloadLastDir.js]
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 3d144d9f8639..43b1ade07ed2 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -853,11 +853,13 @@ class Datastore final
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const nsCString mOrigin;
+  const uint32_t mPrivateBrowsingId;
   bool mClosed;
 
 public:
   // Created by PrepareDatastoreOp.
   Datastore(const nsACString& aOrigin,
+            uint32_t aPrivateBrowsingId,
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
             nsDataHashtable<nsStringHashKey, nsString>& aValues);
@@ -868,6 +870,18 @@ public:
     return mOrigin;
   }
 
+  uint32_t
+  PrivateBrowsingId() const
+  {
+    return mPrivateBrowsingId;
+  }
+
+  bool
+  IsPersistent() const
+  {
+    return mPrivateBrowsingId == 0;
+  }
+
   void
   Close();
 
@@ -921,6 +935,9 @@ private:
   void
   ConnectionClosedCallback();
 
+  void
+  CleanupMetadata();
+
   void
   EnsureTransaction();
 
@@ -1155,6 +1172,7 @@ class PrepareDatastoreOp
   nsCString mMainThreadOrigin;
   nsCString mOrigin;
   nsString mDatabaseFilePath;
+  uint32_t mPrivateBrowsingId;
   State mState;
   bool mRequestedDirectoryLock;
   bool mInvalidated;
@@ -1304,7 +1322,11 @@ private:
 class QuotaClient final
   : public mozilla::dom::quota::Client
 {
+  class ClearPrivateBrowsingRunnable;
+  class PrivateBrowsingObserver;
+
   static QuotaClient* sInstance;
+  static bool sPrivateBrowsingObserverRegistered;
 
   bool mShutdownRequested;
 
@@ -1331,6 +1353,9 @@ public:
     return QuotaManager::IsShuttingDown();
   }
 
+  static nsresult
+  RegisterObservers(nsIEventTarget* aBackgroundEventTarget);
+
   bool
   IsShuttingDown() const
   {
@@ -1385,6 +1410,45 @@ private:
   ~QuotaClient() override;
 };
 
+class QuotaClient::ClearPrivateBrowsingRunnable final
+  : public Runnable
+{
+public:
+  ClearPrivateBrowsingRunnable()
+    : Runnable("mozilla::dom::ClearPrivateBrowsingRunnable")
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+  }
+
+private:
+  ~ClearPrivateBrowsingRunnable() = default;
+
+  NS_DECL_NSIRUNNABLE
+};
+
+class QuotaClient::PrivateBrowsingObserver final
+  : public nsIObserver
+{
+  nsCOMPtr<nsIEventTarget> mBackgroundEventTarget;
+
+public:
+  explicit PrivateBrowsingObserver(nsIEventTarget* aBackgroundEventTarget)
+    : mBackgroundEventTarget(aBackgroundEventTarget)
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+  }
+
+  NS_DECL_ISUPPORTS
+
+private:
+  ~PrivateBrowsingObserver()
+  {
+    MOZ_ASSERT(NS_IsMainThread());
+  }
+
+  NS_DECL_NSIOBSERVER
+};
+
 /*******************************************************************************
  * Globals
  ******************************************************************************/
@@ -2097,12 +2161,14 @@ ClearOp::DoDatastoreWork()
  ******************************************************************************/
 
 Datastore::Datastore(const nsACString& aOrigin,
+                     uint32_t aPrivateBrowsingId,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
                      nsDataHashtable<nsStringHashKey, nsString>& aValues)
   : mDirectoryLock(std::move(aDirectoryLock))
   , mConnection(std::move(aConnection))
   , mOrigin(aOrigin)
+  , mPrivateBrowsingId(aPrivateBrowsingId)
   , mClosed(false)
 {
   AssertIsOnBackgroundThread();
@@ -2123,26 +2189,38 @@ Datastore::Close()
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(!mDatabases.Count());
   MOZ_ASSERT(mDirectoryLock);
-  MOZ_ASSERT(mConnection);
 
   mClosed = true;
 
-  if (mConnection->InTransaction()) {
-    MOZ_ASSERT(mAutoCommitTimer);
-    MOZ_ALWAYS_SUCCEEDS(mAutoCommitTimer->Cancel());
+  if (IsPersistent()) {
+    MOZ_ASSERT(mConnection);
 
-    mConnection->Commit();
+    if (mConnection->InTransaction()) {
+      MOZ_ASSERT(mAutoCommitTimer);
+      MOZ_ALWAYS_SUCCEEDS(mAutoCommitTimer->Cancel());
 
-    mAutoCommitTimer = nullptr;
+      mConnection->Commit();
+
+      mAutoCommitTimer = nullptr;
+    }
+
+    // We can't release the directory lock and unregister itself from the
+    // hashtable until the connection is fully closed.
+    nsCOMPtr<nsIRunnable> callback =
+      NewRunnableMethod("dom::Datastore::ConnectionClosedCallback",
+                        this,
+                        &Datastore::ConnectionClosedCallback);
+    mConnection->Close(callback);
+  } else {
+    MOZ_ASSERT(!mConnection);
+
+    // There's no connection, so it's safe to release the directory lock and
+    // unregister itself from the hashtable.
+
+    mDirectoryLock = nullptr;
+
+    CleanupMetadata();
   }
-
-  // We can't release the directory lock and unregister itself from the
-  // hashtable until the connection is fully closed.
-  nsCOMPtr<nsIRunnable> callback =
-    NewRunnableMethod("dom::Datastore::ConnectionClosedCallback",
-                      this,
-                      &Datastore::ConnectionClosedCallback);
-  mConnection->Close(callback);
 }
 
 void
@@ -2236,6 +2314,10 @@ Datastore::SetItem(const nsString& aKey, const nsString& aValue)
 
   mValues.Put(aKey, aValue);
 
+  if (!IsPersistent()) {
+    return;
+  }
+
   EnsureTransaction();
 
   RefPtr<SetItemOp> op = new SetItemOp(mConnection, aKey, aValue);
@@ -2250,6 +2332,10 @@ Datastore::RemoveItem(const nsString& aKey)
 
   mValues.Remove(aKey);
 
+  if (!IsPersistent()) {
+    return;
+  }
+
   EnsureTransaction();
 
   RefPtr<RemoveItemOp> op = new RemoveItemOp(mConnection, aKey);
@@ -2264,6 +2350,10 @@ Datastore::Clear()
 
   mValues.Clear();
 
+  if (!IsPersistent()) {
+    return;
+  }
+
   EnsureTransaction();
 
   RefPtr<ClearOp> op = new ClearOp(mConnection);
@@ -2295,6 +2385,18 @@ Datastore::ConnectionClosedCallback()
   mDirectoryLock = nullptr;
   mConnection = nullptr;
 
+  CleanupMetadata();
+
+  if (mCompleteCallback) {
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(mCompleteCallback.forget()));
+  }
+}
+
+void
+Datastore::CleanupMetadata()
+{
+  AssertIsOnBackgroundThread();
+
   MOZ_ASSERT(gDatastores);
   MOZ_ASSERT(gDatastores->Get(mOrigin));
   gDatastores->Remove(mOrigin);
@@ -2302,10 +2404,6 @@ Datastore::ConnectionClosedCallback()
   if (!gDatastores->Count()) {
     gDatastores = nullptr;
   }
-
-  if (mCompleteCallback) {
-    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(mCompleteCallback.forget()));
-  }
 }
 
 void
@@ -2623,6 +2721,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   : mMainEventTarget(aMainEventTarget)
   , mLoadDataOp(nullptr)
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
+  , mPrivateBrowsingId(0)
   , mState(State::Initial)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
@@ -2685,6 +2784,11 @@ PrepareDatastoreOp::Open()
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
+
+    rv = principal->GetPrivateBrowsingId(&mPrivateBrowsingId);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
   }
 
   // This service has to be started on the main thread currently.
@@ -2693,6 +2797,11 @@ PrepareDatastoreOp::Open()
     return NS_ERROR_FAILURE;
   }
 
+  nsresult rv = QuotaClient::RegisterObservers(OwningEventTarget());
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   mState = State::CheckExistingOperations;
   MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
 
@@ -2868,6 +2977,22 @@ PrepareDatastoreOp::SendToIOThread()
     return NS_ERROR_FAILURE;
   }
 
+  // Skip all disk related stuff and transition to SendingReadyMessage if we
+  // are preparing a datastore for private browsing.
+  // Note that we do use a directory lock for private browsing even though we
+  // don't do any stuff on disk. The thing is that without a directory lock,
+  // quota manager wouldn't call AbortOperations for our private browsing
+  // origin when a clear origin operation is requested. AbortOperations
+  // requests all databases to close and the datastore is destroyed in the end.
+  // Any following LocalStorage API call will trigger preparation of a new
+  // (empty) datastore.
+  if (mPrivateBrowsingId) {
+    mState = State::SendingReadyMessage;
+    Unused << this->Run();
+
+    return NS_OK;
+  }
+
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
@@ -3086,6 +3211,7 @@ PrepareDatastoreOp::SendResults()
     if (NS_SUCCEEDED(ResultCode())) {
       if (!mDatastore) {
         mDatastore = new Datastore(mOrigin,
+                                   mPrivateBrowsingId,
                                    mDirectoryLock.forget(),
                                    mConnection.forget(),
                                    mValues);
@@ -3432,6 +3558,7 @@ LoadDataOp::Cleanup()
  ******************************************************************************/
 
 QuotaClient* QuotaClient::sInstance = nullptr;
+bool QuotaClient::sPrivateBrowsingObserverRegistered = false;
 
 QuotaClient::QuotaClient()
   : mShutdownRequested(false)
@@ -3456,6 +3583,33 @@ QuotaClient::GetType()
   return QuotaClient::LS;
 }
 
+// static
+nsresult
+QuotaClient::RegisterObservers(nsIEventTarget* aBackgroundEventTarget)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aBackgroundEventTarget);
+
+  if (!sPrivateBrowsingObserverRegistered) {
+    nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
+    if (NS_WARN_IF(!obs)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    nsCOMPtr<nsIObserver> observer =
+      new PrivateBrowsingObserver(aBackgroundEventTarget);
+
+    nsresult rv = obs->AddObserver(observer, "last-pb-context-exited", false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    sPrivateBrowsingObserverRegistered = true;
+  }
+
+  return NS_OK;
+}
+
 nsresult
 QuotaClient::InitOrigin(PersistenceType aPersistenceType,
                         const nsACString& aGroup,
@@ -3760,5 +3914,45 @@ QuotaClient::ShutdownWorkThreads()
   }
 }
 
+NS_IMETHODIMP
+QuotaClient::
+ClearPrivateBrowsingRunnable::Run()
+{
+  AssertIsOnBackgroundThread();
+
+  if (gDatastores) {
+    for (auto iter = gDatastores->ConstIter(); !iter.Done(); iter.Next()) {
+      Datastore* datastore = iter.Data();
+      MOZ_ASSERT(datastore);
+
+      if (datastore->PrivateBrowsingId()) {
+        datastore->Clear();
+      }
+    }
+  }
+
+  return NS_OK;
+}
+
+NS_IMPL_ISUPPORTS(QuotaClient::PrivateBrowsingObserver, nsIObserver)
+
+NS_IMETHODIMP
+QuotaClient::
+PrivateBrowsingObserver::Observe(nsISupports* aSubject,
+                                 const char* aTopic,
+                                 const char16_t* aData)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!strcmp(aTopic, "last-pb-context-exited"));
+
+  RefPtr<ClearPrivateBrowsingRunnable> runnable =
+    new ClearPrivateBrowsingRunnable();
+
+  MOZ_ALWAYS_SUCCEEDS(
+    mBackgroundEventTarget->Dispatch(runnable, NS_DISPATCH_NORMAL));
+
+  return NS_OK;
+}
+
 } // namespace dom
 } // namespace mozilla

From be167c5e0b2baea7ae10c84df70c3cab660f9cc1 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:45 +0100
Subject: [PATCH 24/78] Bug 1286798 - Part 10: Support for storage events;
 r=asuth,janv

Storage events are fired either directly after getting response from synchronous SetItem call or through observers. When a new onstorage event listener is added, we sycnhronously register an observer in the parent process. There's always only one observer actor per content process.
PBackgroundLSDatabase is now managed by a new PBackgroundLSObject protocol. PBackgroundLSObject is needed to eliminate the need to pass the principal info and document URI everytime a write operation occurs.
Preparation of an observer shares some states with preparation of a datastore, so common stuff now lives in LSRequestBase and preparation of a datastore now implements a nested state machine.

This patch was enhanced by asuth to drop observers only when the last storage listener is removed.
EventListenerRemoved is invoked on any removal, not just the final removal, so we need to make sure it's the final removal before dropping observer.
---
 devtools/client/storage/test/browser.ini      |   18 +-
 devtools/server/tests/browser/browser.ini     |    6 +-
 dom/base/nsGlobalWindowInner.cpp              |   50 +-
 dom/base/nsGlobalWindowOuter.cpp              |    1 +
 dom/base/test/mochitest.ini                   |    2 +-
 dom/localstorage/ActorsChild.cpp              |  146 ++
 dom/localstorage/ActorsChild.h                |   83 ++
 dom/localstorage/ActorsParent.cpp             | 1282 +++++++++++++----
 dom/localstorage/ActorsParent.h               |   30 +-
 dom/localstorage/LSDatabase.cpp               |   34 +-
 dom/localstorage/LSDatabase.h                 |   10 +-
 dom/localstorage/LSObject.cpp                 |  224 ++-
 dom/localstorage/LSObject.h                   |   37 +
 dom/localstorage/LSObserver.cpp               |   76 +
 dom/localstorage/LSObserver.h                 |   56 +
 dom/localstorage/LocalStorageCommon.cpp       |    2 +
 dom/localstorage/LocalStorageCommon.h         |    2 +
 dom/localstorage/PBackgroundLSDatabase.ipdl   |   13 +-
 dom/localstorage/PBackgroundLSObject.ipdl     |   27 +
 dom/localstorage/PBackgroundLSObserver.ipdl   |   31 +
 dom/localstorage/PBackgroundLSRequest.ipdl    |    6 +
 .../PBackgroundLSSharedTypes.ipdlh            |    6 +
 dom/localstorage/moz.build                    |    4 +
 .../mochitest/localstorage/mochitest.ini      |    4 +-
 .../mochitest/storageevent/mochitest.ini      |    4 +-
 editor/libeditor/tests/mochitest.ini          |    6 +-
 ipc/glue/BackgroundChildImpl.cpp              |   34 +-
 ipc/glue/BackgroundChildImpl.h                |   15 +-
 ipc/glue/BackgroundParentImpl.cpp             |   69 +-
 ipc/glue/BackgroundParentImpl.h               |   26 +-
 ipc/glue/PBackground.ipdl                     |   12 +-
 .../opener-closed.html.ini                    |    2 -
 .../opener-noopener.html.ini                  |    2 -
 .../opener-noreferrer.html.ini                |    2 -
 .../choose-_blank-002.html.ini                |    2 -
 .../choose-_parent-004.html.ini               |    2 -
 .../choose-_self-002.html.ini                 |    2 -
 .../choose-_top-001.html.ini                  |    2 -
 .../choose-_top-002.html.ini                  |    2 -
 .../choose-_top-003.html.ini                  |    2 -
 .../windows/noreferrer-null-opener.html.ini   |    2 -
 .../windows/noreferrer-window-name.html.ini   |    3 +-
 .../meta/webstorage/document-domain.html.ini  |    2 +-
 .../meta/webstorage/event_basic.html.ini      |    2 -
 .../webstorage/event_body_attribute.html.ini  |    2 -
 .../webstorage/event_case_sensitive.html.ini  |    2 -
 .../meta/webstorage/event_local_key.html.ini  |    2 -
 .../webstorage/event_local_newvalue.html.ini  |    2 -
 .../webstorage/event_local_oldvalue.html.ini  |    2 -
 .../event_local_removeitem.html.ini           |    2 -
 .../event_local_storagearea.html.ini          |    2 -
 .../meta/webstorage/event_local_url.html.ini  |    2 -
 .../webstorage/event_no_duplicates.html.ini   |    2 -
 .../webstorage/event_setattribute.html.ini    |    2 -
 .../antitracking/test/browser/browser.ini     |    8 +-
 .../test/browser/localStorage.html            |    9 +-
 56 files changed, 1932 insertions(+), 448 deletions(-)
 create mode 100644 dom/localstorage/LSObserver.cpp
 create mode 100644 dom/localstorage/LSObserver.h
 create mode 100644 dom/localstorage/PBackgroundLSObject.ipdl
 create mode 100644 dom/localstorage/PBackgroundLSObserver.ipdl
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
 delete mode 100644 testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_basic.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_body_attribute.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_key.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_local_url.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
 delete mode 100644 testing/web-platform/meta/webstorage/event_setattribute.html.ini

diff --git a/devtools/client/storage/test/browser.ini b/devtools/client/storage/test/browser.ini
index ab13a1931d9e..d9f9115ab09d 100644
--- a/devtools/client/storage/test/browser.ini
+++ b/devtools/client/storage/test/browser.ini
@@ -45,23 +45,23 @@ tags = usercontextid
 [browser_storage_cookies_samesite.js]
 skip-if = true # Bug 1448484 - sameSite1 is "Unset" - Got undefined, expected Unset
 [browser_storage_cookies_tab_navigation.js]
-#[browser_storage_delete.js]
-#[browser_storage_delete_all.js]
-#[browser_storage_delete_tree.js]
-#[browser_storage_delete_usercontextid.js]
-#tags = usercontextid
+[browser_storage_delete.js]
+[browser_storage_delete_all.js]
+[browser_storage_delete_tree.js]
+[browser_storage_delete_usercontextid.js]
+tags = usercontextid
 [browser_storage_dom_cache_disabled.js]
 [browser_storage_dynamic_updates_cookies.js]
-#[browser_storage_dynamic_updates_localStorage.js]
-#[browser_storage_dynamic_updates_sessionStorage.js]
+[browser_storage_dynamic_updates_localStorage.js]
+[browser_storage_dynamic_updates_sessionStorage.js]
 [browser_storage_empty_objectstores.js]
 [browser_storage_file_url.js]
 [browser_storage_indexeddb_delete.js]
 [browser_storage_indexeddb_delete_blocked.js]
 [browser_storage_indexeddb_duplicate_names.js]
 [browser_storage_indexeddb_overflow.js]
-#[browser_storage_localstorage_add.js]
-#[browser_storage_localstorage_edit.js]
+[browser_storage_localstorage_add.js]
+[browser_storage_localstorage_edit.js]
 [browser_storage_localstorage_error.js]
 [browser_storage_localstorage_rapid_add_remove.js]
 [browser_storage_overflow.js]
diff --git a/devtools/server/tests/browser/browser.ini b/devtools/server/tests/browser/browser.ini
index 5deafdc8318e..d9988c6bfeea 100644
--- a/devtools/server/tests/browser/browser.ini
+++ b/devtools/server/tests/browser/browser.ini
@@ -94,10 +94,10 @@ skip-if = e10s # Bug 1183605 - devtools/server/tests/browser/ tests are still di
 [browser_spawn_actor_in_parent.js]
 [browser_storage_browser_toolbox_indexeddb.js]
 [browser_storage_cookies-duplicate-names.js]
-#[browser_storage_dynamic_windows.js]
+[browser_storage_dynamic_windows.js]
 [browser_storage_listings.js]
-#[browser_storage_updates.js]
-#skip-if = (verify && debug && (os == 'mac' || os == 'linux'))
+[browser_storage_updates.js]
+skip-if = (verify && debug && (os == 'mac' || os == 'linux'))
 [browser_stylesheets_getTextEmpty.js]
 [browser_stylesheets_nested-iframes.js]
 [browser_register_actor.js]
diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index 05e12ecb2f9a..b60acaaa432b 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -5817,7 +5817,8 @@ nsGlobalWindowInner::ObserveStorageNotification(StorageEvent* aEvent,
     MOZ_DIAGNOSTIC_ASSERT(StorageUtils::PrincipalsEqual(aEvent->GetPrincipal(),
                                                         principal));
 
-    fireMozStorageChanged = mLocalStorage == aEvent->GetStorageArea();
+    fireMozStorageChanged =
+       mLocalStorage && mLocalStorage == aEvent->GetStorageArea();
 
     if (fireMozStorageChanged) {
       eventType.AssignLiteral("MozLocalStorageChanged");
@@ -5864,16 +5865,18 @@ nsGlobalWindowInner::CloneStorageEvent(const nsAString& aType,
   // If null, this is a localStorage event received by IPC.
   if (!storageArea) {
     storage = GetLocalStorage(aRv);
-    if (aRv.Failed() || !storage) {
-      return nullptr;
-    }
+    if (!NextGenLocalStorageEnabled()) {
+      if (aRv.Failed() || !storage) {
+        return nullptr;
+      }
 
-    if (storage->Type() == Storage::eLocalStorage) {
-      RefPtr<LocalStorage> localStorage =
-        static_cast<LocalStorage*>(storage.get());
+      if (storage->Type() == Storage::eLocalStorage) {
+        RefPtr<LocalStorage> localStorage =
+          static_cast<LocalStorage*>(storage.get());
 
-      // We must apply the current change to the 'local' localStorage.
-      localStorage->ApplyEvent(aEvent);
+        // We must apply the current change to the 'local' localStorage.
+        localStorage->ApplyEvent(aEvent);
+      }
     }
   } else if (storageArea->Type() == Storage::eSessionStorage) {
     storage = GetSessionStorage(aRv);
@@ -6798,6 +6801,13 @@ nsGlobalWindowInner::EventListenerAdded(nsAtom* aType)
     ErrorResult rv;
     GetLocalStorage(rv);
     rv.SuppressException();
+
+    if (NextGenLocalStorageEnabled() &&
+        mLocalStorage && mLocalStorage->Type() == Storage::eLocalStorage) {
+      auto object = static_cast<LSObject*>(mLocalStorage.get());
+
+      Unused << NS_WARN_IF(NS_FAILED(object->EnsureObserver()));
+    }
   }
 }
 
@@ -6811,6 +6821,20 @@ nsGlobalWindowInner::EventListenerRemoved(nsAtom* aType)
     MOZ_ASSERT(mBeforeUnloadListenerCount >= 0);
     mTabChild->BeforeUnloadRemoved();
   }
+
+  if (aType == nsGkAtoms::onstorage) {
+    if (NextGenLocalStorageEnabled() &&
+        mLocalStorage &&
+        mLocalStorage->Type() == Storage::eLocalStorage &&
+        // The remove event is fired even if this isn't the last listener, so
+        // only remove if there are no other listeners left.
+        mListenerManager &&
+        !mListenerManager->HasListenersFor(nsGkAtoms::onstorage)) {
+      auto object = static_cast<LSObject*>(mLocalStorage.get());
+
+      object->DropObserver();
+    }
+  }
 }
 
 void
@@ -7972,6 +7996,14 @@ nsGlobalWindowInner::StorageAccessGranted()
 
     MOZ_ASSERT(mLocalStorage &&
                mLocalStorage->Type() == Storage::eLocalStorage);
+
+    if (NextGenLocalStorageEnabled() &&
+        mListenerManager &&
+        mListenerManager->HasListenersFor(nsGkAtoms::onstorage)) {
+      auto object = static_cast<LSObject*>(mLocalStorage.get());
+
+      object->EnsureObserver();
+    }
   }
 }
 
diff --git a/dom/base/nsGlobalWindowOuter.cpp b/dom/base/nsGlobalWindowOuter.cpp
index 3a3a984f1994..51a1b602afc9 100644
--- a/dom/base/nsGlobalWindowOuter.cpp
+++ b/dom/base/nsGlobalWindowOuter.cpp
@@ -27,6 +27,7 @@
 #include "mozilla/dom/ContentFrameMessageManager.h"
 #include "mozilla/dom/EventTarget.h"
 #include "mozilla/dom/LocalStorage.h"
+#include "mozilla/dom/LSObject.h"
 #include "mozilla/dom/Storage.h"
 #include "mozilla/dom/IdleRequest.h"
 #include "mozilla/dom/Performance.h"
diff --git a/dom/base/test/mochitest.ini b/dom/base/test/mochitest.ini
index 9523fa410710..18b5f73d6ac8 100644
--- a/dom/base/test/mochitest.ini
+++ b/dom/base/test/mochitest.ini
@@ -577,7 +577,7 @@ skip-if = toolkit == 'android' || (verify && !debug && (os == 'linux')) #bug 687
 [test_bug1025933.html]
 [test_bug1037687.html]
 support-files = test_bug1037687_subframe.html
-#[test_bug1043106.html]
+[test_bug1043106.html]
 [test_bug1057176.html]
 [test_bug1060938.html]
 [test_bug1064481.html]
diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 788664ffbf97..3ae09fba998d 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -6,11 +6,77 @@
 
 #include "ActorsChild.h"
 
+#include "LocalStorageCommon.h"
 #include "LSDatabase.h"
+#include "LSObject.h"
+#include "LSObserver.h"
 
 namespace mozilla {
 namespace dom {
 
+/*******************************************************************************
+ * LSObjectChild
+ ******************************************************************************/
+
+LSObjectChild::LSObjectChild(LSObject* aObject)
+  : mObject(aObject)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  aObject->AssertIsOnOwningThread();
+
+  MOZ_COUNT_CTOR(LSObjectChild);
+}
+
+LSObjectChild::~LSObjectChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSObjectChild);
+}
+
+void
+LSObjectChild::SendDeleteMeInternal()
+{
+  AssertIsOnOwningThread();
+
+  if (mObject) {
+    mObject->ClearActor();
+    mObject = nullptr;
+
+    MOZ_ALWAYS_TRUE(PBackgroundLSObjectChild::SendDeleteMe());
+  }
+}
+
+void
+LSObjectChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  if (mObject) {
+    mObject->ClearActor();
+#ifdef DEBUG
+    mObject = nullptr;
+#endif
+  }
+}
+
+LSObjectChild::PBackgroundLSDatabaseChild*
+LSObjectChild::AllocPBackgroundLSDatabaseChild(const uint64_t& aDatastoreId)
+{
+  MOZ_CRASH("PBackgroundLSDatabaseChild actor should be manually constructed!");
+}
+
+bool
+LSObjectChild::DeallocPBackgroundLSDatabaseChild(
+                                             PBackgroundLSDatabaseChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
 /*******************************************************************************
  * LSDatabaseChild
  ******************************************************************************/
@@ -74,6 +140,86 @@ LSDatabaseChild::RecvRequestAllowToClose()
   return IPC_OK();
 }
 
+/*******************************************************************************
+ * LSObserverChild
+ ******************************************************************************/
+
+LSObserverChild::LSObserverChild(LSObserver* aObserver)
+  : mObserver(aObserver)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObserver);
+
+  MOZ_COUNT_CTOR(LSObserverChild);
+}
+
+LSObserverChild::~LSObserverChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSObserverChild);
+}
+
+void
+LSObserverChild::SendDeleteMeInternal()
+{
+  AssertIsOnOwningThread();
+
+  if (mObserver) {
+    mObserver->ClearActor();
+    mObserver = nullptr;
+
+    MOZ_ALWAYS_TRUE(PBackgroundLSObserverChild::SendDeleteMe());
+  }
+}
+
+void
+LSObserverChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  if (mObserver) {
+    mObserver->ClearActor();
+#ifdef DEBUG
+    mObserver = nullptr;
+#endif
+  }
+}
+
+mozilla::ipc::IPCResult
+LSObserverChild::RecvObserve(const PrincipalInfo& aPrincipalInfo,
+                             const uint32_t& aPrivateBrowsingId,
+                             const nsString& aDocumentURI,
+                             const nsString& aKey,
+                             const nsString& aOldValue,
+                             const nsString& aNewValue)
+{
+  AssertIsOnOwningThread();
+
+  if (!mObserver) {
+    return IPC_OK();
+  }
+
+  nsresult rv;
+  nsCOMPtr<nsIPrincipal> principal =
+    PrincipalInfoToPrincipal(aPrincipalInfo, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  Storage::NotifyChange(/* aStorage */ nullptr,
+                        principal,
+                        aKey,
+                        aOldValue,
+                        aNewValue,
+                        /* aStorageType */ kLocalStorageType,
+                        aDocumentURI,
+                        /* aIsPrivate */ !!aPrivateBrowsingId,
+                        /* aImmediateDispatch */ true);
+
+  return IPC_OK();
+}
+
 /*******************************************************************************
  * LocalStorageRequestChild
  ******************************************************************************/
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index d7c4034960e6..43cbe8a57366 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -8,6 +8,8 @@
 #define mozilla_dom_localstorage_ActorsChild_h
 
 #include "mozilla/dom/PBackgroundLSDatabaseChild.h"
+#include "mozilla/dom/PBackgroundLSObjectChild.h"
+#include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
 
 namespace mozilla {
@@ -22,8 +24,48 @@ namespace dom {
 
 class LSDatabase;
 class LSObject;
+class LSObserver;
 class LSRequestChildCallback;
 
+class LSObjectChild final
+  : public PBackgroundLSObjectChild
+{
+  friend class mozilla::ipc::BackgroundChildImpl;
+  friend class LSObject;
+
+  LSObject* mObject;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSObjectChild);
+  }
+
+private:
+  // Only created by LSObject.
+  explicit LSObjectChild(LSObject* aObject);
+
+  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
+  ~LSObjectChild();
+
+  void
+  SendDeleteMeInternal();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  PBackgroundLSDatabaseChild*
+  AllocPBackgroundLSDatabaseChild(const uint64_t& aDatastoreId) override;
+
+  bool
+  DeallocPBackgroundLSDatabaseChild(PBackgroundLSDatabaseChild* aActor)
+                                    override;
+};
+
 class LSDatabaseChild final
   : public PBackgroundLSDatabaseChild
 {
@@ -60,6 +102,47 @@ private:
   RecvRequestAllowToClose() override;
 };
 
+class LSObserverChild final
+  : public PBackgroundLSObserverChild
+{
+  friend class mozilla::ipc::BackgroundChildImpl;
+  friend class LSObserver;
+  friend class LSObject;
+
+  LSObserver* mObserver;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSObserverChild);
+  }
+
+private:
+  // Only created by LSObject.
+  explicit LSObserverChild(LSObserver* aObserver);
+
+  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
+  ~LSObserverChild();
+
+  void
+  SendDeleteMeInternal();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvObserve(const PrincipalInfo& aPrinciplaInfo,
+              const uint32_t& aPrivateBrowsingId,
+              const nsString& aDocumentURI,
+              const nsString& aKey,
+              const nsString& aOldValue,
+              const nsString& aNewValue) override;
+};
+
 class LSRequestChild final
   : public PBackgroundLSRequestChild
 {
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 43b1ade07ed2..d8e24afe8ad4 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -14,6 +14,8 @@
 #include "mozStorageHelper.h"
 #include "mozilla/Unused.h"
 #include "mozilla/dom/PBackgroundLSDatabaseParent.h"
+#include "mozilla/dom/PBackgroundLSObjectParent.h"
+#include "mozilla/dom/PBackgroundLSObserverParent.h"
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
 #include "mozilla/dom/quota/QuotaManager.h"
@@ -25,6 +27,7 @@
 #include "nsDataHashtable.h"
 #include "nsInterfaceHashtable.h"
 #include "nsISimpleEnumerator.h"
+#include "nsRefPtrHashtable.h"
 #include "ReportInternalError.h"
 
 #define DISABLE_ASSERTS_FOR_FUZZING 0
@@ -915,13 +918,21 @@ public:
   GetItem(const nsString& aKey, nsString& aValue) const;
 
   void
-  SetItem(const nsString& aKey, const nsString& aValue);
+  SetItem(Database* aDatabase,
+          const nsString& aKey,
+          const nsString& aValue,
+          bool* aChanged,
+          nsString& aOldValue);
 
   void
-  RemoveItem(const nsString& aKey);
+  RemoveItem(Database* aDatabase,
+             const nsString& aKey,
+             bool* aChanged,
+             nsString& aOldValue);
 
   void
-  Clear();
+  Clear(Database* aDatabase,
+        bool* aChanged);
 
   void
   GetKeys(nsTArray<nsString>& aKeys) const;
@@ -938,6 +949,12 @@ private:
   void
   CleanupMetadata();
 
+  void
+  NotifyObservers(Database* aDatabase,
+                  const nsString& aKey,
+                  const nsString& aOldValue,
+                  const nsString& aNewValue);
+
   void
   EnsureTransaction();
 
@@ -1000,9 +1017,67 @@ public:
  * Actor class declarations
  ******************************************************************************/
 
+class Object final
+  : public PBackgroundLSObjectParent
+{
+  const PrincipalInfo mPrincipalInfo;
+  const nsString mDocumentURI;
+  uint32_t mPrivateBrowsingId;
+  bool mActorDestroyed;
+
+public:
+  // Created in AllocPBackgroundLSObjectParent.
+  Object(const PrincipalInfo& aPrincipalInfo,
+         const nsAString& aDocumentURI,
+         uint32_t aPrivateBrowsingId);
+
+  const PrincipalInfo&
+  GetPrincipalInfo() const
+  {
+    return mPrincipalInfo;
+  }
+
+  uint32_t
+  PrivateBrowsingId() const
+  {
+    return mPrivateBrowsingId;
+  }
+
+  const nsString&
+  DocumentURI() const
+  {
+    return mDocumentURI;
+  }
+
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Object)
+
+private:
+  // Reference counted.
+  ~Object();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvDeleteMe() override;
+
+  PBackgroundLSDatabaseParent*
+  AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId) override;
+
+  mozilla::ipc::IPCResult
+  RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                       const uint64_t& aDatastoreId) override;
+
+  bool
+  DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+                                     override;
+};
+
 class Database final
   : public PBackgroundLSDatabaseParent
 {
+  RefPtr<Object> mObject;
   RefPtr<Datastore> mDatastore;
   // Strings share buffers if possible, so it's not a problem to duplicate the
   // origin here.
@@ -1016,7 +1091,16 @@ class Database final
 
 public:
   // Created in AllocPBackgroundLSDatabaseParent.
-  explicit Database(const nsACString& aOrigin);
+  Database(Object* aObject,
+           const nsACString& aOrigin);
+
+  Object*
+  GetObject() const
+  {
+    AssertIsOnBackgroundThread();
+
+    return mObject;
+  }
 
   const nsCString&
   Origin() const
@@ -1062,30 +1146,137 @@ private:
   RecvGetKeys(nsTArray<nsString>* aKeys) override;
 
   mozilla::ipc::IPCResult
-  RecvSetItem(const nsString& aKey, const nsString& aValue) override;
+  RecvSetItem(const nsString& aKey,
+              const nsString& aValue,
+              bool* aChanged,
+              nsString* aOldValue) override;
 
   mozilla::ipc::IPCResult
-  RecvRemoveItem(const nsString& aKey) override;
+  RecvRemoveItem(const nsString& aKey,
+                 bool* aChanged,
+                 nsString* aOldValue) override;
 
   mozilla::ipc::IPCResult
-  RecvClear() override;
+  RecvClear(bool* aChanged) override;
+};
+
+class Observer final
+  : public PBackgroundLSObserverParent
+{
+  nsCString mOrigin;
+  bool mActorDestroyed;
+
+public:
+  // Created in AllocPBackgroundLSObserverParent.
+  explicit Observer(const nsACString& aOrigin);
+
+  const nsCString&
+  Origin() const
+  {
+    return mOrigin;
+  }
+
+  void
+  Observe(Database* aDatabase,
+          const nsString& aKey,
+          const nsString& aOldValue,
+          const nsString& aNewValue);
+
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Observer)
+
+private:
+  // Reference counted.
+  ~Observer();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvDeleteMe() override;
 };
 
 class LSRequestBase
   : public DatastoreOperationBase
   , public PBackgroundLSRequestParent
 {
+protected:
+  enum class State
+  {
+    // Just created on the PBackground thread. Next step is Opening.
+    Initial,
+
+    // Waiting to open/opening on the main thread. Next step is either
+    // Nesting if a subclass needs to process more nested states or
+    // SendingReadyMessage if a subclass doesn't need any nested processing.
+    Opening,
+
+    // Doing nested processing.
+    Nesting,
+
+    // Waiting to send/sending the ready message on the PBackground thread. Next
+    // step is WaitingForFinish.
+    SendingReadyMessage,
+
+    // Waiting for the finish message on the PBackground thread. Next step is
+    // SendingResults.
+    WaitingForFinish,
+
+    // Waiting to send/sending results on the PBackground thread. Next step is
+    // Completed.
+    SendingResults,
+
+    // All done.
+    Completed
+  };
+
+  nsCOMPtr<nsIEventTarget> mMainEventTarget;
+  State mState;
+
 public:
-  virtual void
-  Dispatch() = 0;
+  explicit LSRequestBase(nsIEventTarget* aMainEventTarget);
+
+  void
+  Dispatch();
 
 protected:
+  ~LSRequestBase() override;
+
+  virtual nsresult
+  Open() = 0;
+
+  virtual nsresult
+  NestedRun();
+
+  virtual void
+  GetResponse(LSRequestResponse& aResponse) = 0;
+
+  virtual void
+  Cleanup()
+  { }
+
+private:
+  void
+  SendReadyMessage();
+
+  void
+  SendResults();
+
+protected:
+  // Common nsIRunnable implementation that subclasses may not override.
+  NS_IMETHOD
+  Run() final;
+
   // IPDL methods.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
 
+private:
   mozilla::ipc::IPCResult
   RecvCancel() override;
+
+  mozilla::ipc::IPCResult
+  RecvFinish() override;
 };
 
 class PrepareDatastoreOp
@@ -1094,15 +1285,11 @@ class PrepareDatastoreOp
 {
   class LoadDataOp;
 
-  enum class State
+  enum class NestedState
   {
-    // Just created on the PBackground thread. Next step is OpeningOnMainThread
-    // or OpeningOnOwningThread.
-    Initial,
-
-    // Waiting to open/opening on the main thread. Next step is
+    // The nesting has not yet taken place. Next step is
     // CheckExistingOperations.
-    Opening,
+    BeforeNesting,
 
     // Checking if a prepare datastore operation is already running for given
     // origin on the PBackground thread. Next step is CheckClosingDatastore.
@@ -1143,20 +1330,8 @@ class PrepareDatastoreOp
     // to SendingReadyMessage.
     DatabaseWorkLoadData,
 
-    // Waiting to send/sending the ready message on the PBackground thread. Next
-    // step is WaitingForFinish.
-    SendingReadyMessage,
-
-    // Waiting for the finish message on the PBackground thread. Next step is
-    // SendingResults.
-    WaitingForFinish,
-
-    // Waiting to send/sending results on the PBackground thread. Next step is
-    // Completed.
-    SendingResults,
-
-    // All done.
-    Completed
+    // The nesting has completed.
+    AfterNesting
   };
 
   nsCOMPtr<nsIEventTarget> mMainEventTarget;
@@ -1173,7 +1348,7 @@ class PrepareDatastoreOp
   nsCString mOrigin;
   nsString mDatabaseFilePath;
   uint32_t mPrivateBrowsingId;
-  State mState;
+  NestedState mNestedState;
   bool mRequestedDirectoryLock;
   bool mInvalidated;
 
@@ -1214,14 +1389,11 @@ public:
     mInvalidated = true;
   }
 
-  void
-  Dispatch() override;
-
 private:
   ~PrepareDatastoreOp() override;
 
   nsresult
-  Open();
+  Open() override;
 
   nsresult
   CheckExistingOperations();
@@ -1229,9 +1401,6 @@ private:
   nsresult
   CheckClosingDatastore();
 
-  nsresult
-  PreparationOpen();
-
   nsresult
   BeginDatastorePreparation();
 
@@ -1253,14 +1422,14 @@ private:
   nsresult
   BeginLoadData();
 
-  void
-  SendReadyMessage();
+  nsresult
+  NestedRun() override;
 
   void
-  SendResults();
+  GetResponse(LSRequestResponse& aResponse) override;
 
   void
-  Cleanup();
+  Cleanup() override;
 
   void
   ConnectionClosedCallback();
@@ -1270,16 +1439,10 @@ private:
 
   NS_DECL_ISUPPORTS_INHERITED
 
-  NS_IMETHOD
-  Run() override;
-
   // IPDL overrides.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
 
-  mozilla::ipc::IPCResult
-  RecvFinish() override;
-
   // OpenDirectoryListener overrides.
   void
   DirectoryLockAcquired(DirectoryLock* aLock) override;
@@ -1315,6 +1478,24 @@ private:
   Cleanup() override;
 };
 
+class PrepareObserverOp
+  : public LSRequestBase
+{
+  const LSRequestPrepareObserverParams mParams;
+  nsCString mOrigin;
+
+public:
+  PrepareObserverOp(nsIEventTarget* aMainEventTarget,
+                    const LSRequestParams& aParams);
+
+private:
+  nsresult
+  Open() override;
+
+  void
+  GetResponse(LSRequestResponse& aResponse) override;
+};
+
 /*******************************************************************************
  * Other class declarations
  ******************************************************************************/
@@ -1474,6 +1655,17 @@ StaticAutoPtr<LiveDatabaseArray> gLiveDatabases;
 
 StaticRefPtr<ConnectionThread> gConnectionThread;
 
+uint64_t gLastObserverId = 0;
+
+typedef nsRefPtrHashtable<nsUint64HashKey, Observer> PreparedObserverHashtable;
+
+StaticAutoPtr<PreparedObserverHashtable> gPreparedObsevers;
+
+typedef nsClassHashtable<nsCStringHashKey, nsTArray<Observer*>>
+  ObserverHashtable;
+
+StaticAutoPtr<ObserverHashtable> gObservers;
+
 bool
 IsOnConnectionThread()
 {
@@ -1494,8 +1686,10 @@ AssertIsOnConnectionThread()
  * Exported functions
  ******************************************************************************/
 
-PBackgroundLSDatabaseParent*
-AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
+PBackgroundLSObjectParent*
+AllocPBackgroundLSObjectParent(const PrincipalInfo& aPrincipalInfo,
+                               const nsString& aDocumentURI,
+                               const uint32_t& aPrivateBrowsingId)
 {
   AssertIsOnBackgroundThread();
 
@@ -1503,69 +1697,103 @@ AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
     return nullptr;
   }
 
-  if (NS_WARN_IF(!gPreparedDatastores)) {
-    ASSERT_UNLESS_FUZZING();
-    return nullptr;
-  }
-
-  PreparedDatastore* preparedDatastore = gPreparedDatastores->Get(aDatastoreId);
-  if (NS_WARN_IF(!preparedDatastore)) {
-    ASSERT_UNLESS_FUZZING();
-    return nullptr;
-  }
-
-  // If we ever decide to return null from this point on, we need to make sure
-  // that the datastore is closed and the prepared datastore is removed from the
-  // gPreparedDatastores hashtable.
-  // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
-  // once we return a valid actor in this method.
-
-  RefPtr<Database> database = new Database(preparedDatastore->Origin());
+  RefPtr<Object> object =
+    new Object(aPrincipalInfo, aDocumentURI, aPrivateBrowsingId);
 
   // Transfer ownership to IPDL.
-  return database.forget().take();
+  return object.forget().take();
 }
 
 bool
-RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
-                                     const uint64_t& aDatastoreId)
+RecvPBackgroundLSObjectConstructor(PBackgroundLSObjectParent* aActor,
+                                   const PrincipalInfo& aPrincipalInfo,
+                                   const nsString& aDocumentURI,
+                                   const uint32_t& aPrivateBrowsingId)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
-  MOZ_ASSERT(gPreparedDatastores);
-  MOZ_ASSERT(gPreparedDatastores->Get(aDatastoreId));
   MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
 
-  // The actor is now completely built (it has a manager, channel and it's
-  // registered as a subprotocol).
-  // ActorDestroy will be called if we fail here.
-
-  nsAutoPtr<PreparedDatastore> preparedDatastore;
-  gPreparedDatastores->Remove(aDatastoreId, &preparedDatastore);
-  MOZ_ASSERT(preparedDatastore);
-
-  auto* database = static_cast<Database*>(aActor);
-
-  database->SetActorAlive(preparedDatastore->ForgetDatastore());
-
-  // It's possible that AbortOperations was called before the database actor
-  // was created and became live. Let the child know that the database in no
-  // longer valid.
-  if (preparedDatastore->IsInvalidated()) {
-    database->RequestAllowToClose();
-  }
-
   return true;
 }
 
 bool
-DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
   // Transfer ownership back from IPDL.
-  RefPtr<Database> actor = dont_AddRef(static_cast<Database*>(aActor));
+  RefPtr<Object> actor = dont_AddRef(static_cast<Object*>(aActor));
+
+  return true;
+}
+
+PBackgroundLSObserverParent*
+AllocPBackgroundLSObserverParent(const uint64_t& aObserverId)
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
+    return nullptr;
+  }
+
+  if (NS_WARN_IF(!gPreparedObsevers)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  RefPtr<Observer> observer = gPreparedObsevers->Get(aObserverId);
+  if (NS_WARN_IF(!observer)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  //observer->SetObject(this);
+
+  // Transfer ownership to IPDL.
+  return observer.forget().take();
+}
+
+bool
+RecvPBackgroundLSObserverConstructor(PBackgroundLSObserverParent* aActor,
+                                     const uint64_t& aObserverId)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(gPreparedObsevers);
+  MOZ_ASSERT(gPreparedObsevers->GetWeak(aObserverId));
+
+  RefPtr<Observer> observer;
+  gPreparedObsevers->Remove(aObserverId, observer.StartAssignment());
+  MOZ_ASSERT(observer);
+
+  if (!gPreparedObsevers->Count()) {
+    gPreparedObsevers = nullptr;
+  }
+
+  if (!gObservers) {
+    gObservers = new ObserverHashtable();
+  }
+
+  nsTArray<Observer*>* array;
+  if (!gObservers->Get(observer->Origin(), &array)) {
+    array = new nsTArray<Observer*>();
+    gObservers->Put(observer->Origin(), array);
+  }
+  array->AppendElement(observer);
+
+  return true;
+}
+
+bool
+DeallocPBackgroundLSObserverParent(PBackgroundLSObserverParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  // Transfer ownership back from IPDL.
+  RefPtr<Observer> actor = dont_AddRef(static_cast<Observer*>(aActor));
 
   return true;
 }
@@ -1604,6 +1832,15 @@ AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
       break;
     }
 
+    case LSRequestParams::TLSRequestPrepareObserverParams: {
+      RefPtr<PrepareObserverOp> prepareObserverOp =
+        new PrepareObserverOp(mainEventTarget, aParams);
+
+      actor = std::move(prepareObserverOp);
+
+      break;
+    }
+
     default:
       MOZ_CRASH("Should never get here!");
   }
@@ -2307,57 +2544,107 @@ Datastore::GetItem(const nsString& aKey, nsString& aValue) const
 }
 
 void
-Datastore::SetItem(const nsString& aKey, const nsString& aValue)
+Datastore::SetItem(Database* aDatabase,
+                   const nsString& aKey,
+                   const nsString& aValue,
+                   bool* aChanged,
+                   nsString& aOldValue)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(!mClosed);
 
-  mValues.Put(aKey, aValue);
-
-  if (!IsPersistent()) {
-    return;
+  nsString oldValue;
+  if (!mValues.Get(aKey, &oldValue)) {
+    oldValue.SetIsVoid(true);
   }
 
-  EnsureTransaction();
+  bool changed;
+  if (oldValue == aValue && oldValue.IsVoid() == aValue.IsVoid()) {
+    changed = false;
+  } else {
+    changed = true;
 
-  RefPtr<SetItemOp> op = new SetItemOp(mConnection, aKey, aValue);
-  mConnection->Dispatch(op);
+    mValues.Put(aKey, aValue);
+
+    NotifyObservers(aDatabase, aKey, oldValue, aValue);
+
+    if (IsPersistent()) {
+      EnsureTransaction();
+
+      RefPtr<SetItemOp> op = new SetItemOp(mConnection, aKey, aValue);
+      mConnection->Dispatch(op);
+    }
+  }
+
+  *aChanged = changed;
+  aOldValue = oldValue;
 }
 
 void
-Datastore::RemoveItem(const nsString& aKey)
+Datastore::RemoveItem(Database* aDatabase,
+                      const nsString& aKey,
+                      bool* aChanged,
+                      nsString& aOldValue)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(!mClosed);
 
-  mValues.Remove(aKey);
+  bool changed;
+  nsString oldValue;
+  if (!mValues.Get(aKey, &oldValue)) {
+    oldValue.SetIsVoid(true);
+    changed = false;
+  } else {
+    changed = true;
 
-  if (!IsPersistent()) {
-    return;
+    mValues.Remove(aKey);
+
+    NotifyObservers(aDatabase, aKey, oldValue, VoidString());
+
+    if (IsPersistent()) {
+      EnsureTransaction();
+
+      RefPtr<RemoveItemOp> op = new RemoveItemOp(mConnection, aKey);
+      mConnection->Dispatch(op);
+    }
   }
 
-  EnsureTransaction();
-
-  RefPtr<RemoveItemOp> op = new RemoveItemOp(mConnection, aKey);
-  mConnection->Dispatch(op);
+  *aChanged = changed;
+  aOldValue = oldValue;
 }
 
 void
-Datastore::Clear()
+Datastore::Clear(Database* aDatabase,
+                 bool* aChanged)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(!mClosed);
 
-  mValues.Clear();
+  bool changed;
+  if (!mValues.Count()) {
+    changed = false;
+  } else {
+    changed = true;
 
-  if (!IsPersistent()) {
-    return;
+    mValues.Clear();
+
+    if (aDatabase) {
+      NotifyObservers(aDatabase, VoidString(), VoidString(), VoidString());
+    }
+
+    if (IsPersistent()) {
+      EnsureTransaction();
+
+      RefPtr<ClearOp> op = new ClearOp(mConnection);
+      mConnection->Dispatch(op);
+    }
   }
 
-  EnsureTransaction();
-
-  RefPtr<ClearOp> op = new ClearOp(mConnection);
-  mConnection->Dispatch(op);
+  *aChanged = changed;
 }
 
 void
@@ -2406,6 +2693,35 @@ Datastore::CleanupMetadata()
   }
 }
 
+void
+Datastore::NotifyObservers(Database* aDatabase,
+                           const nsString& aKey,
+                           const nsString& aOldValue,
+                           const nsString& aNewValue)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+
+  if (!gObservers) {
+    return;
+  }
+
+  nsTArray<Observer*>* array;
+  if (!gObservers->Get(mOrigin, &array)) {
+    return;
+  }
+
+  MOZ_ASSERT(array);
+
+  PBackgroundParent* databaseBackgroundActor = aDatabase->Manager()->Manager();
+
+  for (Observer* observer : *array) {
+    if (observer->Manager() != databaseBackgroundActor) {
+      observer->Observe(aDatabase, aKey, aOldValue, aNewValue);
+    }
+  }
+}
+
 void
 Datastore::EnsureTransaction()
 {
@@ -2445,12 +2761,134 @@ Datastore::AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure)
   self->mConnection->Commit();
 }
 
+/*******************************************************************************
+ * Object
+ ******************************************************************************/
+
+Object::Object(const PrincipalInfo& aPrincipalInfo,
+               const nsAString& aDocumentURI,
+               uint32_t aPrivateBrowsingId)
+  : mPrincipalInfo(aPrincipalInfo)
+  , mDocumentURI(aDocumentURI)
+  , mPrivateBrowsingId(aPrivateBrowsingId)
+  , mActorDestroyed(false)
+{
+  AssertIsOnBackgroundThread();
+}
+
+Object::~Object()
+{
+  MOZ_ASSERT(mActorDestroyed);
+}
+
+void
+Object::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  mActorDestroyed = true;
+}
+
+mozilla::ipc::IPCResult
+Object::RecvDeleteMe()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  IProtocol* mgr = Manager();
+  if (!PBackgroundLSObjectParent::Send__delete__(this)) {
+    return IPC_FAIL_NO_REASON(mgr);
+  }
+  return IPC_OK();
+}
+
+PBackgroundLSDatabaseParent*
+Object::AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
+    return nullptr;
+  }
+
+  if (NS_WARN_IF(!gPreparedDatastores)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  PreparedDatastore* preparedDatastore = gPreparedDatastores->Get(aDatastoreId);
+  if (NS_WARN_IF(!preparedDatastore)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  // If we ever decide to return null from this point on, we need to make sure
+  // that the datastore is closed and the prepared datastore is removed from the
+  // gPreparedDatastores hashtable.
+  // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
+  // once we return a valid actor in this method.
+
+  RefPtr<Database> database = new Database(this,
+                                           preparedDatastore->Origin());
+
+  // Transfer ownership to IPDL.
+  return database.forget().take();
+}
+
+mozilla::ipc::IPCResult
+Object::RecvPBackgroundLSDatabaseConstructor(
+                                            PBackgroundLSDatabaseParent* aActor,
+                                            const uint64_t& aDatastoreId)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(gPreparedDatastores);
+  MOZ_ASSERT(gPreparedDatastores->Get(aDatastoreId));
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+
+  // The actor is now completely built (it has a manager, channel and it's
+  // registered as a subprotocol).
+  // ActorDestroy will be called if we fail here.
+
+  nsAutoPtr<PreparedDatastore> preparedDatastore;
+  gPreparedDatastores->Remove(aDatastoreId, &preparedDatastore);
+  MOZ_ASSERT(preparedDatastore);
+
+  auto* database = static_cast<Database*>(aActor);
+
+  database->SetActorAlive(preparedDatastore->ForgetDatastore());
+
+  // It's possible that AbortOperations was called before the database actor
+  // was created and became live. Let the child know that the database in no
+  // longer valid.
+  if (preparedDatastore->IsInvalidated()) {
+    database->RequestAllowToClose();
+  }
+
+  return IPC_OK();
+}
+
+bool
+Object::DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  // Transfer ownership back from IPDL.
+  RefPtr<Database> actor = dont_AddRef(static_cast<Database*>(aActor));
+
+  return true;
+}
+
 /*******************************************************************************
  * Database
  ******************************************************************************/
 
-Database::Database(const nsACString& aOrigin)
-  : mOrigin(aOrigin)
+Database::Database(Object* aObject,
+                   const nsACString& aOrigin)
+  : mObject(aObject)
+  , mOrigin(aOrigin)
   , mAllowedToClose(false)
   , mActorDestroyed(false)
   , mRequestedAllowToClose(false)
@@ -2623,9 +3061,14 @@ Database::RecvGetItem(const nsString& aKey, nsString* aValue)
 }
 
 mozilla::ipc::IPCResult
-Database::RecvSetItem(const nsString& aKey, const nsString& aValue)
+Database::RecvSetItem(const nsString& aKey,
+                      const nsString& aValue,
+                      bool* aChanged,
+                      nsString* aOldValue)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aChanged);
+  MOZ_ASSERT(aOldValue);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -2633,15 +3076,19 @@ Database::RecvSetItem(const nsString& aKey, const nsString& aValue)
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->SetItem(aKey, aValue);
+  mDatastore->SetItem(this, aKey, aValue, aChanged, *aOldValue);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvRemoveItem(const nsString& aKey)
+Database::RecvRemoveItem(const nsString& aKey,
+                         bool* aChanged,
+                         nsString* aOldValue)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aChanged);
+  MOZ_ASSERT(aOldValue);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -2649,15 +3096,16 @@ Database::RecvRemoveItem(const nsString& aKey)
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->RemoveItem(aKey);
+  mDatastore->RemoveItem(this, aKey, aChanged, *aOldValue);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvClear()
+Database::RecvClear(bool* aChanged)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -2665,7 +3113,7 @@ Database::RecvClear()
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->Clear();
+  mDatastore->Clear(this, aChanged);
 
   return IPC_OK();
 }
@@ -2687,10 +3135,205 @@ Database::RecvGetKeys(nsTArray<nsString>* aKeys)
   return IPC_OK();
 }
 
+/*******************************************************************************
+ * Observer
+ ******************************************************************************/
+
+Observer::Observer(const nsACString& aOrigin)
+  : mOrigin(aOrigin)
+  , mActorDestroyed(false)
+{
+  AssertIsOnBackgroundThread();
+}
+
+Observer::~Observer()
+{
+  MOZ_ASSERT(mActorDestroyed);
+}
+
+void
+Observer::Observe(Database* aDatabase,
+                  const nsString& aKey,
+                  const nsString& aOldValue,
+                  const nsString& aNewValue)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+
+  Object* object = aDatabase->GetObject();
+  MOZ_ASSERT(object);
+
+  Unused << SendObserve(object->GetPrincipalInfo(),
+                        object->PrivateBrowsingId(),
+                        object->DocumentURI(),
+                        aKey,
+                        aOldValue,
+                        aNewValue);
+}
+
+void
+Observer::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  mActorDestroyed = true;
+
+  MOZ_ASSERT(gObservers);
+
+  nsTArray<Observer*>* array;
+  gObservers->Get(mOrigin, &array);
+  MOZ_ASSERT(array);
+
+  array->RemoveElement(this);
+
+  if (array->IsEmpty()) {
+    gObservers->Remove(mOrigin);
+  }
+
+  if (!gObservers->Count()) {
+    gObservers = nullptr;
+  }
+}
+
+mozilla::ipc::IPCResult
+Observer::RecvDeleteMe()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  IProtocol* mgr = Manager();
+  if (!PBackgroundLSObserverParent::Send__delete__(this)) {
+    return IPC_FAIL_NO_REASON(mgr);
+  }
+  return IPC_OK();
+}
+
 /*******************************************************************************
  * LSRequestBase
  ******************************************************************************/
 
+LSRequestBase::LSRequestBase(nsIEventTarget* aMainEventTarget)
+  : mMainEventTarget(aMainEventTarget)
+  , mState(State::Initial)
+{
+}
+
+LSRequestBase::~LSRequestBase()
+{
+  MOZ_ASSERT_IF(MayProceedOnNonOwningThread(),
+                mState == State::Initial || mState == State::Completed);
+}
+
+void
+LSRequestBase::Dispatch()
+{
+  AssertIsOnOwningThread();
+
+  mState = State::Opening;
+
+  if (mMainEventTarget) {
+    MOZ_ALWAYS_SUCCEEDS(mMainEventTarget->Dispatch(this, NS_DISPATCH_NORMAL));
+  } else {
+    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+  }
+}
+
+nsresult
+LSRequestBase::NestedRun()
+{
+  return NS_OK;
+}
+
+void
+LSRequestBase::SendReadyMessage()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingReadyMessage);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+
+    Cleanup();
+
+    mState = State::Completed;
+  } else {
+    Unused << SendReady();
+
+    mState = State::WaitingForFinish;
+  }
+}
+
+void
+LSRequestBase::SendResults()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingResults);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+  } else {
+    LSRequestResponse response;
+
+    if (NS_SUCCEEDED(ResultCode())) {
+      GetResponse(response);
+    } else {
+      response = ResultCode();
+    }
+
+    Unused <<
+      PBackgroundLSRequestParent::Send__delete__(this, response);
+  }
+
+  Cleanup();
+
+  mState = State::Completed;
+}
+
+NS_IMETHODIMP
+LSRequestBase::Run()
+{
+  nsresult rv;
+
+  switch (mState) {
+    case State::Opening:
+      rv = Open();
+      break;
+
+    case State::Nesting:
+      rv = NestedRun();
+      break;
+
+    case State::SendingReadyMessage:
+      SendReadyMessage();
+      return NS_OK;
+
+    case State::SendingResults:
+      SendResults();
+      return NS_OK;
+
+    default:
+      MOZ_CRASH("Bad state!");
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv)) && mState != State::SendingReadyMessage) {
+    MaybeSetFailureCode(rv);
+
+    // Must set mState before dispatching otherwise we will race with the owning
+    // thread.
+    mState = State::SendingReadyMessage;
+
+    if (IsOnOwningThread()) {
+      SendReadyMessage();
+    } else {
+      MOZ_ALWAYS_SUCCEEDS(
+        OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+    }
+  }
+
+  return NS_OK;
+}
+
 void
 LSRequestBase::ActorDestroy(ActorDestroyReason aWhy)
 {
@@ -2712,17 +3355,30 @@ LSRequestBase::RecvCancel()
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+LSRequestBase::RecvFinish()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::WaitingForFinish);
+
+  mState = State::SendingResults;
+  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
+
+  return IPC_OK();
+}
+
 /*******************************************************************************
  * PrepareDatastoreOp
  ******************************************************************************/
 
 PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
                                        const LSRequestParams& aParams)
-  : mMainEventTarget(aMainEventTarget)
+  : LSRequestBase(aMainEventTarget)
+  , mMainEventTarget(aMainEventTarget)
   , mLoadDataOp(nullptr)
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mPrivateBrowsingId(0)
-  , mState(State::Initial)
+  , mNestedState(NestedState::BeforeNesting)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
 {
@@ -2738,25 +3394,12 @@ PrepareDatastoreOp::~PrepareDatastoreOp()
   MOZ_ASSERT(!mLoadDataOp);
 }
 
-void
-PrepareDatastoreOp::Dispatch()
-{
-  AssertIsOnOwningThread();
-
-  mState = State::Opening;
-
-  if (mMainEventTarget) {
-    MOZ_ALWAYS_SUCCEEDS(mMainEventTarget->Dispatch(this, NS_DISPATCH_NORMAL));
-  } else {
-    MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
-  }
-}
-
 nsresult
 PrepareDatastoreOp::Open()
 {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(mState == State::Opening);
+  MOZ_ASSERT(mNestedState == NestedState::BeforeNesting);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
@@ -2802,7 +3445,9 @@ PrepareDatastoreOp::Open()
     return rv;
   }
 
-  mState = State::CheckExistingOperations;
+  mState = State::Nesting;
+  mNestedState = NestedState::CheckExistingOperations;
+
   MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
 
   return NS_OK;
@@ -2812,7 +3457,8 @@ nsresult
 PrepareDatastoreOp::CheckExistingOperations()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::CheckExistingOperations);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::CheckExistingOperations);
   MOZ_ASSERT(gPrepareDatastoreOps);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
@@ -2830,7 +3476,7 @@ PrepareDatastoreOp::CheckExistingOperations()
 
   MOZ_ASSERT(!mOrigin.IsEmpty());
 
-  mState = State::CheckClosingDatastore;
+  mNestedState = NestedState::CheckClosingDatastore;
 
   // See if this PrepareDatastoreOp needs to wait.
   bool foundThis = false;
@@ -2863,14 +3509,15 @@ nsresult
 PrepareDatastoreOp::CheckClosingDatastore()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::CheckClosingDatastore);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::CheckClosingDatastore);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
       !MayProceed()) {
     return NS_ERROR_FAILURE;
   }
 
-  mState = State::PreparationPending;
+  mNestedState = NestedState::PreparationPending;
 
   RefPtr<Datastore> datastore;
   if (gDatastores &&
@@ -2893,12 +3540,15 @@ nsresult
 PrepareDatastoreOp::BeginDatastorePreparation()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::PreparationPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::PreparationPending);
 
   if (gDatastores && (mDatastore = gDatastores->Get(mOrigin))) {
     MOZ_ASSERT(!mDatastore->IsClosed());
 
     mState = State::SendingReadyMessage;
+    mNestedState = NestedState::AfterNesting;
+
     Unused << this->Run();
 
     return NS_OK;
@@ -2913,7 +3563,7 @@ PrepareDatastoreOp::BeginDatastorePreparation()
     return NS_OK;
   }
 
-  mState = State::QuotaManagerPending;
+  mNestedState = NestedState::QuotaManagerPending;
   QuotaManager::GetOrCreate(this, mMainEventTarget);
 
   return NS_OK;
@@ -2923,7 +3573,8 @@ nsresult
 PrepareDatastoreOp::QuotaManagerOpen()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::QuotaManagerPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::QuotaManagerPending);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
       !MayProceed()) {
@@ -2946,14 +3597,15 @@ nsresult
 PrepareDatastoreOp::OpenDirectory()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::PreparationPending ||
-             mState == State::QuotaManagerPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::PreparationPending ||
+             mNestedState == NestedState::QuotaManagerPending);
   MOZ_ASSERT(!mOrigin.IsEmpty());
   MOZ_ASSERT(!mDirectoryLock);
   MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
   MOZ_ASSERT(QuotaManager::Get());
 
-  mState = State::DirectoryOpenPending;
+  mNestedState = NestedState::DirectoryOpenPending;
   QuotaManager::Get()->OpenDirectory(PERSISTENCE_TYPE_DEFAULT,
                                      mGroup,
                                      mOrigin,
@@ -2970,7 +3622,8 @@ nsresult
 PrepareDatastoreOp::SendToIOThread()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::DirectoryOpenPending);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
       !MayProceed()) {
@@ -2988,6 +3641,8 @@ PrepareDatastoreOp::SendToIOThread()
   // (empty) datastore.
   if (mPrivateBrowsingId) {
     mState = State::SendingReadyMessage;
+    mNestedState = NestedState::AfterNesting;
+
     Unused << this->Run();
 
     return NS_OK;
@@ -2997,7 +3652,7 @@ PrepareDatastoreOp::SendToIOThread()
   MOZ_ASSERT(quotaManager);
 
   // Must set this before dispatching otherwise we will race with the IO thread.
-  mState = State::DatabaseWorkOpen;
+  mNestedState = NestedState::DatabaseWorkOpen;
 
   nsresult rv = quotaManager->IOThread()->Dispatch(this, NS_DISPATCH_NORMAL);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -3011,7 +3666,8 @@ nsresult
 PrepareDatastoreOp::DatabaseWork()
 {
   AssertIsOnIOThread();
-  MOZ_ASSERT(mState == State::DatabaseWorkOpen);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::DatabaseWorkOpen);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
@@ -3090,7 +3746,7 @@ PrepareDatastoreOp::DatabaseWork()
 
   // Must set mState before dispatching otherwise we will race with the owning
   // thread.
-  mState = State::BeginLoadData;
+  mNestedState = NestedState::BeginLoadData;
 
   rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -3142,7 +3798,8 @@ nsresult
 PrepareDatastoreOp::BeginLoadData()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::BeginLoadData);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::BeginLoadData);
   MOZ_ASSERT(!mConnection);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
@@ -3162,7 +3819,7 @@ PrepareDatastoreOp::BeginLoadData()
 
   // Must set this before dispatching otherwise we will race with the
   // connection thread.
-  mState = State::DatabaseWorkLoadData;
+  mNestedState = NestedState::DatabaseWorkLoadData;
 
   // Can't assign to mLoadDataOp directly since that's a weak reference and
   // LoadDataOp is reference counted.
@@ -3178,83 +3835,91 @@ PrepareDatastoreOp::BeginLoadData()
   return NS_OK;
 }
 
-void
-PrepareDatastoreOp::SendReadyMessage()
+nsresult
+PrepareDatastoreOp::NestedRun()
 {
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::SendingReadyMessage);
+  nsresult rv;
 
-  if (!MayProceed()) {
-    MaybeSetFailureCode(NS_ERROR_FAILURE);
+  switch (mNestedState) {
+    case NestedState::CheckExistingOperations:
+      rv = CheckExistingOperations();
+      break;
 
-    Cleanup();
+    case NestedState::CheckClosingDatastore:
+      rv = CheckClosingDatastore();
+      break;
 
-    mState = State::Completed;
-  } else {
-    Unused << SendReady();
+    case NestedState::PreparationPending:
+      rv = BeginDatastorePreparation();
+      break;
 
-    mState = State::WaitingForFinish;
+    case NestedState::QuotaManagerPending:
+      rv = QuotaManagerOpen();
+      break;
+
+    case NestedState::DatabaseWorkOpen:
+      rv = DatabaseWork();
+      break;
+
+    case NestedState::BeginLoadData:
+      rv = BeginLoadData();
+      break;
+
+    default:
+      MOZ_CRASH("Bad state!");
   }
+
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    mNestedState = NestedState::AfterNesting;
+
+    return rv;
+  }
+
+  return NS_OK;
 }
 
 void
-PrepareDatastoreOp::SendResults()
+PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::SendingResults);
+  MOZ_ASSERT(NS_SUCCEEDED(ResultCode()));
 
-  if (!MayProceed()) {
-    MaybeSetFailureCode(NS_ERROR_FAILURE);
-  } else {
-    LSRequestResponse response;
+  if (!mDatastore) {
+    mDatastore = new Datastore(mOrigin,
+                               mPrivateBrowsingId,
+                               mDirectoryLock.forget(),
+                               mConnection.forget(),
+                               mValues);
 
-    if (NS_SUCCEEDED(ResultCode())) {
-      if (!mDatastore) {
-        mDatastore = new Datastore(mOrigin,
-                                   mPrivateBrowsingId,
-                                   mDirectoryLock.forget(),
-                                   mConnection.forget(),
-                                   mValues);
-
-        if (!gDatastores) {
-          gDatastores = new DatastoreHashtable();
-        }
-
-        MOZ_ASSERT(!gDatastores->Get(mOrigin));
-        gDatastores->Put(mOrigin, mDatastore);
-      }
-
-      uint64_t datastoreId = ++gLastDatastoreId;
-
-      nsAutoPtr<PreparedDatastore> preparedDatastore(
-        new PreparedDatastore(mDatastore, mOrigin));
-
-      if (!gPreparedDatastores) {
-        gPreparedDatastores = new PreparedDatastoreHashtable();
-      }
-      gPreparedDatastores->Put(datastoreId, preparedDatastore);
-
-      if (mInvalidated) {
-        preparedDatastore->Invalidate();
-      }
-
-      preparedDatastore.forget();
-
-      LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
-      prepareDatastoreResponse.datastoreId() = datastoreId;
-
-      response = prepareDatastoreResponse;
-    } else {
-      response = ResultCode();
+    if (!gDatastores) {
+      gDatastores = new DatastoreHashtable();
     }
 
-    Unused <<
-      PBackgroundLSRequestParent::Send__delete__(this, response);
+    MOZ_ASSERT(!gDatastores->Get(mOrigin));
+    gDatastores->Put(mOrigin, mDatastore);
   }
 
-  Cleanup();
+  uint64_t datastoreId = ++gLastDatastoreId;
 
-  mState = State::Completed;
+  nsAutoPtr<PreparedDatastore> preparedDatastore(
+    new PreparedDatastore(mDatastore, mOrigin));
+
+  if (!gPreparedDatastores) {
+    gPreparedDatastores = new PreparedDatastoreHashtable();
+  }
+  gPreparedDatastores->Put(datastoreId, preparedDatastore);
+
+  if (mInvalidated) {
+    preparedDatastore->Invalidate();
+  }
+
+  preparedDatastore.forget();
+
+  LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
+  prepareDatastoreResponse.datastoreId() = datastoreId;
+
+  aResponse = prepareDatastoreResponse;
 }
 
 void
@@ -3337,70 +4002,6 @@ PrepareDatastoreOp::CleanupMetadata()
 
 NS_IMPL_ISUPPORTS_INHERITED0(PrepareDatastoreOp, LSRequestBase)
 
-NS_IMETHODIMP
-PrepareDatastoreOp::Run()
-{
-  nsresult rv;
-
-  switch (mState) {
-    case State::Opening:
-      rv = Open();
-      break;
-
-    case State::CheckExistingOperations:
-      rv = CheckExistingOperations();
-      break;
-
-    case State::CheckClosingDatastore:
-      rv = CheckClosingDatastore();
-      break;
-
-    case State::PreparationPending:
-      rv = BeginDatastorePreparation();
-      break;
-
-    case State::QuotaManagerPending:
-      rv = QuotaManagerOpen();
-      break;
-
-    case State::DatabaseWorkOpen:
-      rv = DatabaseWork();
-      break;
-
-    case State::BeginLoadData:
-      rv = BeginLoadData();
-      break;
-
-    case State::SendingReadyMessage:
-      SendReadyMessage();
-      return NS_OK;
-
-    case State::SendingResults:
-      SendResults();
-      return NS_OK;
-
-    default:
-      MOZ_CRASH("Bad state!");
-  }
-
-  if (NS_WARN_IF(NS_FAILED(rv)) && mState != State::SendingReadyMessage) {
-    MaybeSetFailureCode(rv);
-
-    // Must set mState before dispatching otherwise we will race with the owning
-    // thread.
-    mState = State::SendingReadyMessage;
-
-    if (IsOnOwningThread()) {
-      SendReadyMessage();
-    } else {
-      MOZ_ALWAYS_SUCCEEDS(
-        OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
-    }
-  }
-
-  return NS_OK;
-}
-
 void
 PrepareDatastoreOp::ActorDestroy(ActorDestroyReason aWhy)
 {
@@ -3413,23 +4014,12 @@ PrepareDatastoreOp::ActorDestroy(ActorDestroyReason aWhy)
   }
 }
 
-mozilla::ipc::IPCResult
-PrepareDatastoreOp::RecvFinish()
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::WaitingForFinish);
-
-  mState = State::SendingResults;
-  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
-
-  return IPC_OK();
-}
-
 void
 PrepareDatastoreOp::DirectoryLockAcquired(DirectoryLock* aLock)
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::DirectoryOpenPending);
   MOZ_ASSERT(!mDirectoryLock);
 
   mDirectoryLock = aLock;
@@ -3442,6 +4032,8 @@ PrepareDatastoreOp::DirectoryLockAcquired(DirectoryLock* aLock)
     // before calling Run().
 
     mState = State::SendingReadyMessage;
+    mNestedState = NestedState::AfterNesting;
+
     MOZ_ALWAYS_SUCCEEDS(Run());
 
     return;
@@ -3452,7 +4044,8 @@ void
 PrepareDatastoreOp::DirectoryLockFailed()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mState == State::DirectoryOpenPending);
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::DirectoryOpenPending);
   MOZ_ASSERT(!mDirectoryLock);
 
   MaybeSetFailureCode(NS_ERROR_FAILURE);
@@ -3461,6 +4054,8 @@ PrepareDatastoreOp::DirectoryLockFailed()
   // before calling Run().
 
   mState = State::SendingReadyMessage;
+  mNestedState = NestedState::AfterNesting;
+
   MOZ_ALWAYS_SUCCEEDS(Run());
 }
 
@@ -3471,7 +4066,9 @@ LoadDataOp::DoDatastoreWork()
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mConnection);
   MOZ_ASSERT(mPrepareDatastoreOp);
-  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::Nesting);
+  MOZ_ASSERT(mPrepareDatastoreOp->mNestedState ==
+               NestedState::DatabaseWorkLoadData);
 
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
@@ -3516,10 +4113,13 @@ LoadDataOp::OnSuccess()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mPrepareDatastoreOp);
-  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::Nesting);
+  MOZ_ASSERT(mPrepareDatastoreOp->mNestedState ==
+               NestedState::DatabaseWorkLoadData);
   MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
 
   mPrepareDatastoreOp->mState = State::SendingReadyMessage;
+  mPrepareDatastoreOp->mNestedState = NestedState::AfterNesting;
 
   MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
 }
@@ -3530,11 +4130,14 @@ LoadDataOp::OnFailure(nsresult aResultCode)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mPrepareDatastoreOp);
-  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::DatabaseWorkLoadData);
+  MOZ_ASSERT(mPrepareDatastoreOp->mState == State::Nesting);
+  MOZ_ASSERT(mPrepareDatastoreOp->mNestedState ==
+               NestedState::DatabaseWorkLoadData);
   MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
 
   mPrepareDatastoreOp->SetFailureCode(aResultCode);
   mPrepareDatastoreOp->mState = State::SendingReadyMessage;
+  mPrepareDatastoreOp->mNestedState = NestedState::AfterNesting;
 
   MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
 }
@@ -3553,6 +4156,81 @@ LoadDataOp::Cleanup()
   ConnectionDatastoreOperationBase::Cleanup();
 }
 
+/*******************************************************************************
+ * PrepareObserverOp
+ ******************************************************************************/
+
+PrepareObserverOp::PrepareObserverOp(nsIEventTarget* aMainEventTarget,
+                                     const LSRequestParams& aParams)
+  : LSRequestBase(aMainEventTarget)
+  , mParams(aParams.get_LSRequestPrepareObserverParams())
+{
+  MOZ_ASSERT(aParams.type() ==
+               LSRequestParams::TLSRequestPrepareObserverParams);
+}
+
+nsresult
+PrepareObserverOp::Open()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mState == State::Opening);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
+      !MayProceedOnNonOwningThread()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  const PrincipalInfo& principalInfo = mParams.principalInfo();
+
+  if (principalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) {
+    QuotaManager::GetInfoForChrome(nullptr, nullptr, &mOrigin);
+  } else {
+    MOZ_ASSERT(principalInfo.type() == PrincipalInfo::TContentPrincipalInfo);
+
+    nsresult rv;
+    nsCOMPtr<nsIPrincipal> principal =
+      PrincipalInfoToPrincipal(principalInfo, &rv);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = QuotaManager::GetInfoFromPrincipal(principal,
+                                            nullptr,
+                                            nullptr,
+                                            &mOrigin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  mState = State::SendingReadyMessage;
+  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+
+  return NS_OK;
+}
+
+void
+PrepareObserverOp::GetResponse(LSRequestResponse& aResponse)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingResults);
+  MOZ_ASSERT(NS_SUCCEEDED(ResultCode()));
+
+  uint64_t observerId = ++gLastObserverId;
+
+  RefPtr<Observer> observer = new Observer(mOrigin);
+
+  if (!gPreparedObsevers) {
+    gPreparedObsevers = new PreparedObserverHashtable();
+  }
+  gPreparedObsevers->Put(observerId, observer);
+
+  LSRequestPrepareObserverResponse prepareObserverResponse;
+  prepareObserverResponse.observerId() = observerId;
+
+  aResponse = prepareObserverResponse;
+}
+
 /*******************************************************************************
  * QuotaClient
  ******************************************************************************/
@@ -3899,6 +4577,11 @@ QuotaClient::ShutdownWorkThreads()
     }
   }
 
+  if (gPreparedObsevers) {
+    gPreparedObsevers->Clear();
+    gPreparedObsevers = nullptr;
+  }
+
   // This should release any local storage related quota objects or directory
   // locks.
   MOZ_ALWAYS_TRUE(SpinEventLoopUntil([&]() {
@@ -3926,7 +4609,8 @@ ClearPrivateBrowsingRunnable::Run()
       MOZ_ASSERT(datastore);
 
       if (datastore->PrivateBrowsingId()) {
-        datastore->Clear();
+        bool dummy;
+        datastore->Clear(nullptr, &dummy);
       }
     }
   }
diff --git a/dom/localstorage/ActorsParent.h b/dom/localstorage/ActorsParent.h
index 4e505ea9fffd..ee7396ff1331 100644
--- a/dom/localstorage/ActorsParent.h
+++ b/dom/localstorage/ActorsParent.h
@@ -12,13 +12,15 @@ namespace mozilla {
 namespace ipc {
 
 class PBackgroundParent;
+class PrincipalInfo;
 
 } // namespace ipc
 
 namespace dom {
 
 class LSRequestParams;
-class PBackgroundLSDatabaseParent;
+class PBackgroundLSObjectParent;
+class PBackgroundLSObserverParent;
 class PBackgroundLSRequestParent;
 
 namespace quota {
@@ -27,15 +29,31 @@ class Client;
 
 } // namespace quota
 
-PBackgroundLSDatabaseParent*
-AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId);
+PBackgroundLSObjectParent*
+AllocPBackgroundLSObjectParent(
+                              const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
+                              const nsString& aDocumentURI,
+                              const uint32_t& aPrivateBrowsingId);
 
 bool
-RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
-                                     const uint64_t& aDatastoreId);
+RecvPBackgroundLSObjectConstructor(
+                              PBackgroundLSObjectParent* aActor,
+                              const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
+                              const nsString& aDocumentURI,
+                              const uint32_t& aPrivateBrowsingId);
 
 bool
-DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor);
+DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor);
+
+PBackgroundLSObserverParent*
+AllocPBackgroundLSObserverParent(const uint64_t& aObserverId);
+
+bool
+RecvPBackgroundLSObserverConstructor(PBackgroundLSObserverParent* aActor,
+                                     const uint64_t& aObservereId);
+
+bool
+DeallocPBackgroundLSObserverParent(PBackgroundLSObserverParent* aActor);
 
 PBackgroundLSRequestParent*
 AllocPBackgroundLSRequestParent(
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index e5735fef9aa3..83c14c0eddf4 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -121,44 +121,66 @@ LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
 
 nsresult
 LSDatabase::SetItem(const nsAString& aKey,
-                    const nsAString& aValue)
+                    const nsAString& aValue,
+                    bool* aChanged,
+                    nsAString& aOldValue)
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey), nsString(aValue)))) {
+  bool changed;
+  nsString oldValue;
+  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey),
+                                      nsString(aValue),
+                                      &changed,
+                                      &oldValue))) {
     return NS_ERROR_FAILURE;
   }
 
+  *aChanged = changed;
+  aOldValue = oldValue;
   return NS_OK;
 }
 
 nsresult
-LSDatabase::RemoveItem(const nsAString& aKey)
+LSDatabase::RemoveItem(const nsAString& aKey,
+                       bool* aChanged,
+                       nsAString& aOldValue)
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey)))) {
+  bool changed;
+  nsString oldValue;
+  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey),
+                                         &changed,
+                                         &oldValue))) {
     return NS_ERROR_FAILURE;
   }
 
+  *aChanged = changed;
+  aOldValue = oldValue;
   return NS_OK;
 }
 
 nsresult
-LSDatabase::Clear()
+LSDatabase::Clear(bool* aChanged)
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  if (NS_WARN_IF(!mActor->SendClear())) {
+  bool changed;
+  if (NS_WARN_IF(!mActor->SendClear(&changed))) {
     return NS_ERROR_FAILURE;
   }
 
+  *aChanged = changed;
   return NS_OK;
 }
 
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index 8446f4a4b9e0..01035fc1b326 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -68,13 +68,17 @@ public:
 
   nsresult
   SetItem(const nsAString& aKey,
-          const nsAString& aValue);
+          const nsAString& aValue,
+          bool* aChanged,
+          nsAString& aOldValue);
 
   nsresult
-  RemoveItem(const nsAString& aKey);
+  RemoveItem(const nsAString& aKey,
+             bool* aChanged,
+             nsAString& aOldValue);
 
   nsresult
-  Clear();
+  Clear(bool* aChanged);
 
 private:
   ~LSDatabase();
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index a45f1317bab5..2fd3fbd64ae3 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -127,6 +127,9 @@ private:
 LSObject::LSObject(nsPIDOMWindowInner* aWindow,
                    nsIPrincipal* aPrincipal)
   : Storage(aWindow, aPrincipal)
+  , mActor(nullptr)
+  , mPrivateBrowsingId(0)
+  , mActorFailed(false)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(NextGenLocalStorageEnabled());
@@ -135,6 +138,14 @@ LSObject::LSObject(nsPIDOMWindowInner* aWindow,
 LSObject::~LSObject()
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT_IF(mActorFailed, !mActor);
+
+  DropObserver();
+
+  if (mActor) {
+    mActor->SendDeleteMeInternal();
+    MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
+  }
 }
 
 // static
@@ -165,8 +176,31 @@ LSObject::Create(nsPIDOMWindowInner* aWindow,
 
   MOZ_ASSERT(principalInfo->type() == PrincipalInfo::TContentPrincipalInfo);
 
+  nsCString origin;
+  rv = QuotaManager::GetInfoFromPrincipal(principal, nullptr, nullptr, &origin);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  uint32_t privateBrowsingId;
+  rv = principal->GetPrivateBrowsingId(&privateBrowsingId);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsString documentURI;
+  if (nsCOMPtr<nsIDocument> doc = aWindow->GetExtantDoc()) {
+    rv = doc->GetDocumentURI(documentURI);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
   RefPtr<LSObject> object = new LSObject(aWindow, principal);
   object->mPrincipalInfo = std::move(principalInfo);
+  object->mPrivateBrowsingId = privateBrowsingId;
+  object->mOrigin = origin;
+  object->mDocumentURI = documentURI;
 
   object.forget(aStorage);
   return NS_OK;
@@ -239,8 +273,13 @@ bool
 LSObject::IsForkOf(const Storage* aStorage) const
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(aStorage);
 
-  return false;
+  if (aStorage->Type() != eLocalStorage) {
+    return false;
+  }
+
+  return static_cast<const LSObject*>(aStorage)->mOrigin == mOrigin;
 }
 
 int64_t
@@ -351,11 +390,17 @@ LSObject::SetItem(const nsAString& aKey,
     return;
   }
 
-  rv = mDatabase->SetItem(aKey, aValue);
+  bool changed;
+  nsString oldValue;
+  rv = mDatabase->SetItem(aKey, aValue, &changed, oldValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
+
+  if (changed) {
+    OnChange(aKey, oldValue, aValue);
+  }
 }
 
 void
@@ -371,11 +416,17 @@ LSObject::RemoveItem(const nsAString& aKey,
     return;
   }
 
-  rv = mDatabase->RemoveItem(aKey);
+  bool changed;
+  nsString oldValue;
+  rv = mDatabase->RemoveItem(aKey, &changed, oldValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
+
+  if (changed) {
+    OnChange(aKey, oldValue, VoidString());
+  }
 }
 
 void
@@ -390,11 +441,16 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
     return;
   }
 
-  rv = mDatabase->Clear();
+  bool changed;
+  rv = mDatabase->Clear(&changed);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
+
+  if (changed) {
+    OnChange(VoidString(), VoidString(), VoidString());
+  }
 }
 
 NS_IMPL_ADDREF_INHERITED(LSObject, Storage)
@@ -414,6 +470,37 @@ NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(LSObject, Storage)
   tmp->DropDatabase();
 NS_IMPL_CYCLE_COLLECTION_UNLINK_END
 
+nsresult
+LSObject::DoRequestSynchronously(const LSRequestParams& aParams,
+                                 LSRequestResponse& aResponse)
+{
+  // We don't need this yet, but once the request successfully finishes, it's
+  // too late to initialize PBackground child on the owning thread, because
+  // it can fail and parent would keep an extra strong ref to the datastore or
+  // observer.
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread();
+  if (NS_WARN_IF(!backgroundActor)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  RefPtr<RequestHelper> helper = new RequestHelper(this, aParams);
+
+  // This will start and finish the request on the DOM File thread.
+  // The owning thread is synchronously blocked while the request is
+  // asynchronously processed on the DOM File thread.
+  nsresult rv = helper->StartAndReturnResponse(aResponse);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (aResponse.type() == LSRequestResponse::Tnsresult) {
+    return aResponse.get_nsresult();
+  }
+
+  return NS_OK;
+}
+
 nsresult
 LSObject::EnsureDatabase()
 {
@@ -425,34 +512,43 @@ LSObject::EnsureDatabase()
 
   mDatabase = nullptr;
 
-  // We don't need this yet, but once the request successfully finishes, it's
-  // too late to initialize PBackground child on the owning thread, because
-  // it can fail and parent would keep an extra strong ref to the datastore.
-  PBackgroundChild* backgroundActor =
-    BackgroundChild::GetOrCreateForCurrentThread();
-  if (NS_WARN_IF(!backgroundActor)) {
+  if (mActorFailed) {
     return NS_ERROR_FAILURE;
   }
 
+  if (!mActor) {
+    PBackgroundChild* backgroundActor =
+      BackgroundChild::GetOrCreateForCurrentThread();
+    if (NS_WARN_IF(!backgroundActor)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    LSObjectChild* actor = new LSObjectChild(this);
+
+    mActor =
+      static_cast<LSObjectChild*>(
+        backgroundActor->SendPBackgroundLSObjectConstructor(
+                                                           actor,
+                                                           *mPrincipalInfo,
+                                                           mDocumentURI,
+                                                           mPrivateBrowsingId));
+
+    if (NS_WARN_IF(!mActor)) {
+      mActorFailed = true;
+      return NS_ERROR_FAILURE;
+    }
+  }
+
   LSRequestPrepareDatastoreParams params;
   params.principalInfo() = *mPrincipalInfo;
 
-  RefPtr<RequestHelper> helper = new RequestHelper(this, params);
-
   LSRequestResponse response;
 
-  // This will start and finish the request on the DOM File thread.
-  // The owning thread is synchronously blocked while the request is
-  // asynchronously processed on the DOM File thread.
-  nsresult rv = helper->StartAndReturnResponse(response);
+  nsresult rv = DoRequestSynchronously(params, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  if (response.type() == LSRequestResponse::Tnsresult) {
-    return response.get_nsresult();
-  }
-
   MOZ_ASSERT(response.type() ==
                LSRequestResponse::TLSRequestPrepareDatastoreResponse);
 
@@ -467,12 +563,13 @@ LSObject::EnsureDatabase()
   // actor) from the owning thread.
   // Note that we now can't error out, otherwise parent will keep an extra
   // strong reference to the datastore.
+
   RefPtr<LSDatabase> database = new LSDatabase();
 
   LSDatabaseChild* actor = new LSDatabaseChild(database);
 
   MOZ_ALWAYS_TRUE(
-    backgroundActor->SendPBackgroundLSDatabaseConstructor(actor, datastoreId));
+    mActor->SendPBackgroundLSDatabaseConstructor(actor, datastoreId));
 
   database->SetActor(actor);
 
@@ -494,6 +591,91 @@ LSObject::DropDatabase()
   }
 }
 
+nsresult
+LSObject::EnsureObserver()
+{
+  AssertIsOnOwningThread();
+
+  if (mObserver) {
+    return NS_OK;
+  }
+
+  mObserver = LSObserver::Get(mOrigin);
+
+  if (mObserver) {
+    return NS_OK;
+  }
+
+  LSRequestPrepareObserverParams params;
+  params.principalInfo() = *mPrincipalInfo;
+
+  LSRequestResponse response;
+
+  nsresult rv = DoRequestSynchronously(params, response);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(response.type() ==
+               LSRequestResponse::TLSRequestPrepareObserverResponse);
+
+  const LSRequestPrepareObserverResponse& prepareObserverResponse =
+    response.get_LSRequestPrepareObserverResponse();
+
+  uint64_t observerId = prepareObserverResponse.observerId();
+
+  // The obsserver is now ready on the parent side (prepared by the asynchronous
+  // request on the DOM File thread).
+  // Let's create a direct connection to the observer (through an observer
+  // actor) from the owning thread.
+  // Note that we now can't error out, otherwise parent will keep an extra
+  // strong reference to the observer.
+
+  PBackgroundChild* backgroundActor = BackgroundChild::GetForCurrentThread();
+  MOZ_ASSERT(backgroundActor);
+
+  RefPtr<LSObserver> observer = new LSObserver(mOrigin);
+
+  LSObserverChild* actor = new LSObserverChild(observer);
+
+  MOZ_ALWAYS_TRUE(
+    backgroundActor->SendPBackgroundLSObserverConstructor(actor, observerId));
+
+  observer->SetActor(actor);
+
+  mObserver = std::move(observer);
+
+  return NS_OK;
+}
+
+void
+LSObject::DropObserver()
+{
+  AssertIsOnOwningThread();
+
+  if (mObserver) {
+    mObserver = nullptr;
+  }
+}
+
+void
+LSObject::OnChange(const nsAString& aKey,
+                   const nsAString& aOldValue,
+                   const nsAString& aNewValue)
+{
+  AssertIsOnOwningThread();
+
+  NotifyChange(/* aStorage */ this,
+               Principal(),
+               aKey,
+               aOldValue,
+               aNewValue,
+               /* aStorageType */ kLocalStorageType,
+               mDocumentURI,
+               /* aIsPrivate */ !!mPrivateBrowsingId,
+               /* aImmediateDispatch */ false);
+}
+
 void
 LSObject::LastRelease()
 {
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index d46f538d2e3e..c6746b9ad501 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -9,6 +9,7 @@
 
 #include "mozilla/dom/Storage.h"
 
+class nsGlobalWindowInner;
 class nsIPrincipal;
 class nsPIDOMWindowInner;
 
@@ -25,18 +26,31 @@ class PrincipalInfo;
 namespace dom {
 
 class LSDatabase;
+class LSObjectChild;
+class LSObserver;
 class LSRequestChild;
 class LSRequestChildCallback;
 class LSRequestParams;
+class LSRequestResponse;
 
 class LSObject final
   : public Storage
 {
   typedef mozilla::ipc::PrincipalInfo PrincipalInfo;
 
+  friend nsGlobalWindowInner;
+
   nsAutoPtr<PrincipalInfo> mPrincipalInfo;
 
   RefPtr<LSDatabase> mDatabase;
+  RefPtr<LSObserver> mObserver;
+
+  LSObjectChild* mActor;
+
+  uint32_t mPrivateBrowsingId;
+  nsCString mOrigin;
+  nsString mDocumentURI;
+  bool mActorFailed;
 
 public:
   static nsresult
@@ -60,6 +74,14 @@ public:
     NS_ASSERT_OWNINGTHREAD(LSObject);
   }
 
+  void
+  ClearActor()
+  {
+    AssertIsOnOwningThread();
+
+    mActor = nullptr;
+  }
+
   LSRequestChild*
   StartRequest(nsIEventTarget* aMainEventTarget,
                const LSRequestParams& aParams,
@@ -118,12 +140,27 @@ private:
 
   ~LSObject();
 
+  nsresult
+  DoRequestSynchronously(const LSRequestParams& aParams,
+                         LSRequestResponse& aResponse);
+
   nsresult
   EnsureDatabase();
 
   void
   DropDatabase();
 
+  nsresult
+  EnsureObserver();
+
+  void
+  DropObserver();
+
+  void
+  OnChange(const nsAString& aKey,
+           const nsAString& aOldValue,
+           const nsAString& aNewValue);
+
   // Storage overrides.
   void
   LastRelease() override;
diff --git a/dom/localstorage/LSObserver.cpp b/dom/localstorage/LSObserver.cpp
new file mode 100644
index 000000000000..8953dc23103c
--- /dev/null
+++ b/dom/localstorage/LSObserver.cpp
@@ -0,0 +1,76 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LSObserver.h"
+
+#include "mozilla/ipc/BackgroundChild.h"
+#include "mozilla/ipc/BackgroundUtils.h"
+#include "mozilla/ipc/PBackgroundChild.h"
+#include "nsContentUtils.h"
+#include "nsIScriptObjectPrincipal.h"
+
+namespace mozilla {
+namespace dom {
+
+namespace {
+
+typedef nsDataHashtable<nsCStringHashKey, LSObserver*> LSObserverHashtable;
+
+StaticAutoPtr<LSObserverHashtable> gLSObservers;
+
+} // namespace
+
+LSObserver::LSObserver(const nsACString& aOrigin)
+  : mActor(nullptr)
+  , mOrigin(aOrigin)
+{
+  AssertIsOnOwningThread();
+
+  if (!gLSObservers) {
+    gLSObservers = new LSObserverHashtable();
+  }
+
+  MOZ_ASSERT(!gLSObservers->Get(mOrigin));
+  gLSObservers->Put(mOrigin, this);
+}
+
+LSObserver::~LSObserver()
+{
+  AssertIsOnOwningThread();
+
+  if (mActor) {
+    mActor->SendDeleteMeInternal();
+    MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
+  }
+
+  MOZ_ASSERT(gLSObservers);
+  MOZ_ASSERT(gLSObservers->Get(mOrigin));
+  gLSObservers->Remove(mOrigin);
+
+  if (!gLSObservers->Count()) {
+    gLSObservers = nullptr;
+  }
+}
+
+// static
+LSObserver*
+LSObserver::Get(const nsACString& aOrigin)
+{
+  return gLSObservers ? gLSObservers->Get(aOrigin) : nullptr;
+}
+
+void
+LSObserver::SetActor(LSObserverChild* aActor)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(!mActor);
+
+  mActor = aActor;
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LSObserver.h b/dom/localstorage/LSObserver.h
new file mode 100644
index 000000000000..0266ceb31345
--- /dev/null
+++ b/dom/localstorage/LSObserver.h
@@ -0,0 +1,56 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LSObserver_h
+#define mozilla_dom_localstorage_LSObserver_h
+
+namespace mozilla {
+namespace dom {
+
+class LSObserverChild;
+
+class LSObserver final
+{
+  friend class LSObject;
+
+  LSObserverChild* mActor;
+
+  const nsCString mOrigin;
+
+public:
+  explicit LSObserver(const nsACString& aOrigin);
+
+  static LSObserver*
+  Get(const nsACString& aOrigin);
+
+  NS_INLINE_DECL_REFCOUNTING(LSObserver)
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSDatabase);
+  }
+
+  void
+  SetActor(LSObserverChild* aActor);
+
+  void
+  ClearActor()
+  {
+    AssertIsOnOwningThread();
+    MOZ_ASSERT(mActor);
+
+    mActor = nullptr;
+  }
+
+private:
+  ~LSObserver();
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LSObserver_h
diff --git a/dom/localstorage/LocalStorageCommon.cpp b/dom/localstorage/LocalStorageCommon.cpp
index 288c5a570358..e90dce61f7c4 100644
--- a/dom/localstorage/LocalStorageCommon.cpp
+++ b/dom/localstorage/LocalStorageCommon.cpp
@@ -15,6 +15,8 @@ Atomic<int32_t> gNextGenLocalStorageEnabled(-1);
 
 } // namespace
 
+const char16_t* kLocalStorageType = u"localStorage";
+
 bool
 NextGenLocalStorageEnabled()
 {
diff --git a/dom/localstorage/LocalStorageCommon.h b/dom/localstorage/LocalStorageCommon.h
index 53553b9dcbac..7c31b7a56cf6 100644
--- a/dom/localstorage/LocalStorageCommon.h
+++ b/dom/localstorage/LocalStorageCommon.h
@@ -184,6 +184,8 @@
 namespace mozilla {
 namespace dom {
 
+extern const char16_t* kLocalStorageType;
+
 bool
 NextGenLocalStorageEnabled();
 
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 5a3724c81af5..90d8d124bbd6 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -2,14 +2,14 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-include protocol PBackground;
+include protocol PBackgroundLSObject;
 
 namespace mozilla {
 namespace dom {
 
 sync protocol PBackgroundLSDatabase
 {
-  manager PBackground;
+  manager PBackgroundLSObject;
 
 parent:
   // The DeleteMe message is used to avoid a race condition between the parent
@@ -37,11 +37,14 @@ parent:
   sync GetKeys()
     returns (nsString[] keys);
 
-  sync SetItem(nsString key, nsString value);
+  sync SetItem(nsString key, nsString value)
+    returns (bool changed, nsString oldValue);
 
-  sync RemoveItem(nsString key);
+  sync RemoveItem(nsString key)
+    returns (bool changed, nsString oldValue);
 
-  sync Clear();
+  sync Clear()
+    returns (bool changed);
 
 child:
   async __delete__();
diff --git a/dom/localstorage/PBackgroundLSObject.ipdl b/dom/localstorage/PBackgroundLSObject.ipdl
new file mode 100644
index 000000000000..291e8acb29b2
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSObject.ipdl
@@ -0,0 +1,27 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+include protocol PBackgroundLSDatabase;
+
+namespace mozilla {
+namespace dom {
+
+sync protocol PBackgroundLSObject
+{
+  manager PBackground;
+
+  manages PBackgroundLSDatabase;
+
+parent:
+  async DeleteMe();
+
+  async PBackgroundLSDatabase(uint64_t datastoreId);
+
+child:
+  async __delete__();
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/PBackgroundLSObserver.ipdl b/dom/localstorage/PBackgroundLSObserver.ipdl
new file mode 100644
index 000000000000..155f760588f2
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSObserver.ipdl
@@ -0,0 +1,31 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+
+include PBackgroundSharedTypes;
+
+namespace mozilla {
+namespace dom {
+
+async protocol PBackgroundLSObserver
+{
+  manager PBackground;
+
+parent:
+  async DeleteMe();
+
+child:
+  async __delete__();
+
+  async Observe(PrincipalInfo principalInfo,
+                uint32_t privateBrowsingId,
+                nsString documentURI,
+                nsString key,
+                nsString oldValue,
+                nsString newValue);
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/PBackgroundLSRequest.ipdl b/dom/localstorage/PBackgroundLSRequest.ipdl
index 2efd6447a201..e87bcea22729 100644
--- a/dom/localstorage/PBackgroundLSRequest.ipdl
+++ b/dom/localstorage/PBackgroundLSRequest.ipdl
@@ -12,10 +12,16 @@ struct LSRequestPrepareDatastoreResponse
   uint64_t datastoreId;
 };
 
+struct LSRequestPrepareObserverResponse
+{
+  uint64_t observerId;
+};
+
 union LSRequestResponse
 {
   nsresult;
   LSRequestPrepareDatastoreResponse;
+  LSRequestPrepareObserverResponse;
 };
 
 protocol PBackgroundLSRequest
diff --git a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
index 8c2ea9e967f4..b7cf7d5acc6b 100644
--- a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
+++ b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
@@ -12,9 +12,15 @@ struct LSRequestPrepareDatastoreParams
   PrincipalInfo principalInfo;
 };
 
+struct LSRequestPrepareObserverParams
+{
+  PrincipalInfo principalInfo;
+};
+
 union LSRequestParams
 {
   LSRequestPrepareDatastoreParams;
+  LSRequestPrepareObserverParams;
 };
 
 } // namespace dom
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index 64cd2c951b4e..b15ebc898ccb 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -12,6 +12,7 @@ EXPORTS.mozilla.dom += [
     'LocalStorageCommon.h',
     'LocalStorageManager2.h',
     'LSObject.h',
+    'LSObserver.h',
 ]
 
 UNIFIED_SOURCES += [
@@ -21,11 +22,14 @@ UNIFIED_SOURCES += [
     'LocalStorageManager2.cpp',
     'LSDatabase.cpp',
     'LSObject.cpp',
+    'LSObserver.cpp',
     'ReportInternalError.cpp',
 ]
 
 IPDL_SOURCES += [
     'PBackgroundLSDatabase.ipdl',
+    'PBackgroundLSObject.ipdl',
+    'PBackgroundLSObserver.ipdl',
     'PBackgroundLSRequest.ipdl',
     'PBackgroundLSSharedTypes.ipdlh',
 ]
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index 3a1894687ab0..543da6ffcc77 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -27,8 +27,8 @@ skip-if = os == "android" || verify # bug 962029
 #[test_cookieSession.html]
 [test_embededNulls.html]
 [test_keySync.html]
-#[test_localStorageBase.html]
-#skip-if = e10s
+[test_localStorageBase.html]
+skip-if = e10s
 #[test_localStorageBaseSessionOnly.html]
 [test_localStorageCookieSettings.html]
 #[test_localStorageEnablePref.html]
diff --git a/dom/tests/mochitest/storageevent/mochitest.ini b/dom/tests/mochitest/storageevent/mochitest.ini
index 7985f2c38672..e16d9cd2e965 100644
--- a/dom/tests/mochitest/storageevent/mochitest.ini
+++ b/dom/tests/mochitest/storageevent/mochitest.ini
@@ -11,7 +11,7 @@ support-files =
   interOriginTest2.js
 
 [test_storageLocalStorageEventCheckNoPropagation.html]
-#[test_storageLocalStorageEventCheckPropagation.html]
-#[test_storageNotifications.html]
+[test_storageLocalStorageEventCheckPropagation.html]
+[test_storageNotifications.html]
 [test_storageSessionStorageEventCheckNoPropagation.html]
 [test_storageSessionStorageEventCheckPropagation.html]
diff --git a/editor/libeditor/tests/mochitest.ini b/editor/libeditor/tests/mochitest.ini
index 8349c24dc2fc..c334d745d9aa 100644
--- a/editor/libeditor/tests/mochitest.ini
+++ b/editor/libeditor/tests/mochitest.ini
@@ -141,9 +141,9 @@ skip-if = os == 'android'
 [test_bug645914.html]
 [test_bug646194.html]
 [test_bug668599.html]
-#[test_bug674770-1.html]
-#subsuite = clipboard
-#skip-if = toolkit == 'android' || verify
+[test_bug674770-1.html]
+subsuite = clipboard
+skip-if = toolkit == 'android' || verify
 [test_bug674770-2.html]
 subsuite = clipboard
 skip-if = toolkit == 'android'
diff --git a/ipc/glue/BackgroundChildImpl.cpp b/ipc/glue/BackgroundChildImpl.cpp
index 5630857427d3..464066b947f1 100644
--- a/ipc/glue/BackgroundChildImpl.cpp
+++ b/ipc/glue/BackgroundChildImpl.cpp
@@ -16,7 +16,8 @@
 #include "mozilla/Assertions.h"
 #include "mozilla/SchedulerGroup.h"
 #include "mozilla/dom/ClientManagerActors.h"
-#include "mozilla/dom/PBackgroundLSDatabaseChild.h"
+#include "mozilla/dom/PBackgroundLSObjectChild.h"
+#include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
 #include "mozilla/dom/PBackgroundSDBConnectionChild.h"
 #include "mozilla/dom/PFileSystemRequestChild.h"
@@ -243,16 +244,35 @@ BackgroundChildImpl::DeallocPBackgroundSDBConnectionChild(
   return true;
 }
 
-BackgroundChildImpl::PBackgroundLSDatabaseChild*
-BackgroundChildImpl::AllocPBackgroundLSDatabaseChild(
-                                                    const uint64_t& aDatastoreId)
+BackgroundChildImpl::PBackgroundLSObjectChild*
+BackgroundChildImpl::AllocPBackgroundLSObjectChild(
+                                            const PrincipalInfo& aPrincipalInfo,
+                                            const nsString& aDocumentURI,
+                                            const uint32_t& aPrivateBrowsingId)
 {
-  MOZ_CRASH("PBackgroundLSDatabaseChild actor should be manually constructed!");
+  MOZ_CRASH("PBackgroundLSObjectChild actor should be manually constructed!");
 }
 
 bool
-BackgroundChildImpl::DeallocPBackgroundLSDatabaseChild(
-                                             PBackgroundLSDatabaseChild* aActor)
+BackgroundChildImpl::DeallocPBackgroundLSObjectChild(
+                                               PBackgroundLSObjectChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
+BackgroundChildImpl::PBackgroundLSObserverChild*
+BackgroundChildImpl::AllocPBackgroundLSObserverChild(
+                                                    const uint64_t& aObserverId)
+{
+  MOZ_CRASH("PBackgroundLSObserverChild actor should be manually constructed!");
+}
+
+bool
+BackgroundChildImpl::DeallocPBackgroundLSObserverChild(
+                                             PBackgroundLSObserverChild* aActor)
 {
   MOZ_ASSERT(aActor);
 
diff --git a/ipc/glue/BackgroundChildImpl.h b/ipc/glue/BackgroundChildImpl.h
index 2df764eaa7ba..fc72d188d4d2 100644
--- a/ipc/glue/BackgroundChildImpl.h
+++ b/ipc/glue/BackgroundChildImpl.h
@@ -78,11 +78,20 @@ protected:
   DeallocPBackgroundSDBConnectionChild(PBackgroundSDBConnectionChild* aActor)
                                        override;
 
-  virtual PBackgroundLSDatabaseChild*
-  AllocPBackgroundLSDatabaseChild(const uint64_t& aCacheId) override;
+  virtual PBackgroundLSObjectChild*
+  AllocPBackgroundLSObjectChild(const PrincipalInfo& aPrincipalInfo,
+                                const nsString& aDocumentURI,
+                                const uint32_t& aPrivateBrowsingId) override;
 
   virtual bool
-  DeallocPBackgroundLSDatabaseChild(PBackgroundLSDatabaseChild* aActor)
+  DeallocPBackgroundLSObjectChild(PBackgroundLSObjectChild* aActor)
+                                  override;
+
+  virtual PBackgroundLSObserverChild*
+  AllocPBackgroundLSObserverChild(const uint64_t& aObserverId) override;
+
+  virtual bool
+  DeallocPBackgroundLSObserverChild(PBackgroundLSObserverChild* aActor)
                                     override;
 
   virtual PBackgroundLSRequestChild*
diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp
index 2033e4870870..4b5928e0f145 100644
--- a/ipc/glue/BackgroundParentImpl.cpp
+++ b/ipc/glue/BackgroundParentImpl.cpp
@@ -294,41 +294,86 @@ BackgroundParentImpl::DeallocPBackgroundSDBConnectionParent(
   return mozilla::dom::DeallocPBackgroundSDBConnectionParent(aActor);
 }
 
-BackgroundParentImpl::PBackgroundLSDatabaseParent*
-BackgroundParentImpl::AllocPBackgroundLSDatabaseParent(
-                                                    const uint64_t& aDatastoreId)
+BackgroundParentImpl::PBackgroundLSObjectParent*
+BackgroundParentImpl::AllocPBackgroundLSObjectParent(
+                                            const PrincipalInfo& aPrincipalInfo,
+                                            const nsString& aDocumentURI,
+                                            const uint32_t& aPrivateBrowsingId)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
 
-  return mozilla::dom::AllocPBackgroundLSDatabaseParent(aDatastoreId);
+  return mozilla::dom::AllocPBackgroundLSObjectParent(aPrincipalInfo,
+                                                      aDocumentURI,
+                                                      aPrivateBrowsingId);
 }
 
 mozilla::ipc::IPCResult
-BackgroundParentImpl::RecvPBackgroundLSDatabaseConstructor(
-                                            PBackgroundLSDatabaseParent* aActor,
-                                            const uint64_t& aDatastoreId)
+BackgroundParentImpl::RecvPBackgroundLSObjectConstructor(
+                                            PBackgroundLSObjectParent* aActor,
+                                            const PrincipalInfo& aPrincipalInfo,
+                                            const nsString& aDocumentURI,
+                                            const uint32_t& aPrivateBrowsingId)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
-  if (!mozilla::dom::RecvPBackgroundLSDatabaseConstructor(aActor,
-                                                          aDatastoreId)) {
+  if (!mozilla::dom::RecvPBackgroundLSObjectConstructor(aActor,
+                                                        aPrincipalInfo,
+                                                        aDocumentURI,
+                                                        aPrivateBrowsingId)) {
     return IPC_FAIL_NO_REASON(this);
   }
   return IPC_OK();
 }
 
 bool
-BackgroundParentImpl::DeallocPBackgroundLSDatabaseParent(
-                                            PBackgroundLSDatabaseParent* aActor)
+BackgroundParentImpl::DeallocPBackgroundLSObjectParent(
+                                              PBackgroundLSObjectParent* aActor)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
-  return mozilla::dom::DeallocPBackgroundLSDatabaseParent(aActor);
+  return mozilla::dom::DeallocPBackgroundLSObjectParent(aActor);
+}
+
+BackgroundParentImpl::PBackgroundLSObserverParent*
+BackgroundParentImpl::AllocPBackgroundLSObserverParent(
+                                                    const uint64_t& aObserverId)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return mozilla::dom::AllocPBackgroundLSObserverParent(aObserverId);
+}
+
+mozilla::ipc::IPCResult
+BackgroundParentImpl::RecvPBackgroundLSObserverConstructor(
+                                            PBackgroundLSObserverParent* aActor,
+                                            const uint64_t& aObserverId)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  if (!mozilla::dom::RecvPBackgroundLSObserverConstructor(aActor,
+                                                          aObserverId)) {
+    return IPC_FAIL_NO_REASON(this);
+  }
+  return IPC_OK();
+}
+
+bool
+BackgroundParentImpl::DeallocPBackgroundLSObserverParent(
+                                            PBackgroundLSObserverParent* aActor)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  return mozilla::dom::DeallocPBackgroundLSObserverParent(aActor);
 }
 
 BackgroundParentImpl::PBackgroundLSRequestParent*
diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h
index 10fc1a37d2bb..58b6ef628542 100644
--- a/ipc/glue/BackgroundParentImpl.h
+++ b/ipc/glue/BackgroundParentImpl.h
@@ -77,15 +77,31 @@ protected:
   DeallocPBackgroundSDBConnectionParent(PBackgroundSDBConnectionParent* aActor)
                                         override;
 
-  virtual PBackgroundLSDatabaseParent*
-  AllocPBackgroundLSDatabaseParent(const uint64_t& aCacheId) override;
+  virtual PBackgroundLSObjectParent*
+  AllocPBackgroundLSObjectParent(const PrincipalInfo& aPrincipalInfo,
+                                 const nsString& aDocumentURI,
+                                 const uint32_t& aPrivateBrowsingId) override;
 
   virtual mozilla::ipc::IPCResult
-  RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
-                                       const uint64_t& aDatastoreId) override;
+  RecvPBackgroundLSObjectConstructor(PBackgroundLSObjectParent* aActor,
+                                     const PrincipalInfo& aPrincipalInfo,
+                                     const nsString& aDocumentURI,
+                                     const uint32_t& aPrivateBrowsingId)
+                                     override;
 
   virtual bool
-  DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+  DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor)
+                                   override;
+
+  virtual PBackgroundLSObserverParent*
+  AllocPBackgroundLSObserverParent(const uint64_t& aObserverId) override;
+
+  virtual mozilla::ipc::IPCResult
+  RecvPBackgroundLSObserverConstructor(PBackgroundLSObserverParent* aActor,
+                                       const uint64_t& aObserverId) override;
+
+  virtual bool
+  DeallocPBackgroundLSObserverParent(PBackgroundLSObserverParent* aActor)
                                      override;
 
   virtual PBackgroundLSRequestParent*
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index 17260497c99a..b9738e9a325d 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -6,7 +6,8 @@ include protocol PAsmJSCacheEntry;
 include protocol PBackgroundIDBFactory;
 include protocol PBackgroundIndexedDBUtils;
 include protocol PBackgroundSDBConnection;
-include protocol PBackgroundLSDatabase;
+include protocol PBackgroundLSObject;
+include protocol PBackgroundLSObserver;
 include protocol PBackgroundLSRequest;
 include protocol PBackgroundLocalStorageCache;
 include protocol PBackgroundStorage;
@@ -75,7 +76,8 @@ sync protocol PBackground
   manages PBackgroundIDBFactory;
   manages PBackgroundIndexedDBUtils;
   manages PBackgroundSDBConnection;
-  manages PBackgroundLSDatabase;
+  manages PBackgroundLSObject;
+  manages PBackgroundLSObserver;
   manages PBackgroundLSRequest;
   manages PBackgroundLocalStorageCache;
   manages PBackgroundStorage;
@@ -124,7 +126,11 @@ parent:
 
   async PBackgroundSDBConnection(PrincipalInfo principalInfo);
 
-  async PBackgroundLSDatabase(uint64_t datastoreId);
+  async PBackgroundLSObject(PrincipalInfo principalInfo,
+                            nsString documentURI,
+                            uint32_t privateBrowsingId);
+
+  async PBackgroundLSObserver(uint64_t observerId);
 
   async PBackgroundLSRequest(LSRequestParams params);
 
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
deleted file mode 100644
index 9a6f9b1f7781..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-closed.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[opener-closed.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
deleted file mode 100644
index 378c58417e34..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noopener.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[opener-noopener.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini b/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
deleted file mode 100644
index 7567b7218ce4..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/auxiliary-browsing-contexts/opener-noreferrer.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[opener-noreferrer.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
deleted file mode 100644
index fc0dfe449c21..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_blank-002.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_blank-002.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
deleted file mode 100644
index c8485530052e..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_parent-004.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_parent-004.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
deleted file mode 100644
index df79b5dd6dd9..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_self-002.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_self-002.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
deleted file mode 100644
index c75ba08bdc24..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-001.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_top-001.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
deleted file mode 100644
index a7527b0093c0..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-002.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_top-002.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini b/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
deleted file mode 100644
index 4d357924c3ab..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/browsing-context-names/choose-_top-003.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[choose-_top-003.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini b/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
deleted file mode 100644
index 57b9ebad9cf4..000000000000
--- a/testing/web-platform/meta/html/browsers/windows/noreferrer-null-opener.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[noreferrer-null-opener.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini b/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
index f509186c2a7a..0dc9bc12e114 100644
--- a/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
+++ b/testing/web-platform/meta/html/browsers/windows/noreferrer-window-name.html.ini
@@ -1,2 +1,3 @@
 [noreferrer-window-name.html]
-  disabled: temporary
+  disabled:
+    if verify: fails in verify mode
diff --git a/testing/web-platform/meta/webstorage/document-domain.html.ini b/testing/web-platform/meta/webstorage/document-domain.html.ini
index 0f7608a1bb03..ba4c2b143af4 100644
--- a/testing/web-platform/meta/webstorage/document-domain.html.ini
+++ b/testing/web-platform/meta/webstorage/document-domain.html.ini
@@ -1,2 +1,2 @@
 [document-domain.html]
-  disabled: temporary
+  expected: TIMEOUT
diff --git a/testing/web-platform/meta/webstorage/event_basic.html.ini b/testing/web-platform/meta/webstorage/event_basic.html.ini
deleted file mode 100644
index 9a2a19afca5a..000000000000
--- a/testing/web-platform/meta/webstorage/event_basic.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_basic.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_body_attribute.html.ini b/testing/web-platform/meta/webstorage/event_body_attribute.html.ini
deleted file mode 100644
index d1f1c341ed1e..000000000000
--- a/testing/web-platform/meta/webstorage/event_body_attribute.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_body_attribute.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini b/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
deleted file mode 100644
index 42743ad9ccdf..000000000000
--- a/testing/web-platform/meta/webstorage/event_case_sensitive.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_case_sensitive.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_key.html.ini b/testing/web-platform/meta/webstorage/event_local_key.html.ini
deleted file mode 100644
index 0704163d2577..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_key.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_key.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini b/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
deleted file mode 100644
index af1cb9407a36..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_newvalue.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_newvalue.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini b/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
deleted file mode 100644
index c218f24c7a38..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_oldvalue.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_oldvalue.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini b/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
deleted file mode 100644
index 3e2c560eeeda..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_removeitem.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_removeitem.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini b/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
deleted file mode 100644
index 447aced1bcc0..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_storagearea.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_storagearea.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_local_url.html.ini b/testing/web-platform/meta/webstorage/event_local_url.html.ini
deleted file mode 100644
index 6480915b2eba..000000000000
--- a/testing/web-platform/meta/webstorage/event_local_url.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_local_url.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini b/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
deleted file mode 100644
index f6475fbb82c2..000000000000
--- a/testing/web-platform/meta/webstorage/event_no_duplicates.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_no_duplicates.html]
-  disabled: temporary
diff --git a/testing/web-platform/meta/webstorage/event_setattribute.html.ini b/testing/web-platform/meta/webstorage/event_setattribute.html.ini
deleted file mode 100644
index 0d31589933c6..000000000000
--- a/testing/web-platform/meta/webstorage/event_setattribute.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[event_setattribute.html]
-  disabled: temporary
diff --git a/toolkit/components/antitracking/test/browser/browser.ini b/toolkit/components/antitracking/test/browser/browser.ini
index 1a1a79aa108d..360d4a4be03a 100644
--- a/toolkit/components/antitracking/test/browser/browser.ini
+++ b/toolkit/components/antitracking/test/browser/browser.ini
@@ -68,8 +68,8 @@ skip-if = serviceworker_e10s
 [browser_storageAccessWithHeuristics.js]
 [browser_allowPermissionForTracker.js]
 [browser_denyPermissionForTracker.js]
-#[browser_localStorageEvents.js]
-#support-files = localStorage.html
+[browser_localStorageEvents.js]
+support-files = localStorage.html
 [browser_partitionedLocalStorage.js]
-#[browser_partitionedLocalStorage_events.js]
-#support-files = localStorageEvents.html
+[browser_partitionedLocalStorage_events.js]
+support-files = localStorageEvents.html
diff --git a/toolkit/components/antitracking/test/browser/localStorage.html b/toolkit/components/antitracking/test/browser/localStorage.html
index 934d7a5efa67..e08c25f2c42c 100644
--- a/toolkit/components/antitracking/test/browser/localStorage.html
+++ b/toolkit/components/antitracking/test/browser/localStorage.html
@@ -4,7 +4,14 @@
 if (window.opener) {
   SpecialPowers.wrap(document).userInteractionForTesting();
   localStorage.foo = "opener" + Math.random();
-  window.close();
+  // Don't call window.close immediatelly. It can happen that adding the
+  // "storage" event listener below takes more time than usual (it may need to
+  // synchronously subscribe in the parent process to receive storage
+  // notifications). Spending more time in the initial script can prevent
+  // the "load" event from being fired for the window opened by "open and test".
+  setTimeout(() => {
+    window.close();
+  }, 0);
 }
 
 if (parent) {

From 9bbba97e9191dff42206ccb6516e30e20a200c0f Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:48 +0100
Subject: [PATCH 25/78] Bug 1286798 - Part 11: Enable tests for session only
 mode (but only for the old local storage implementation); r=asuth

An attribute for checking if the next gen local storage implementation is enabled is exposed via a new interface nsILocalStorageManager which should be used for any other local storage specific stuff.
---
 dom/localstorage/LocalStorageManager2.cpp         | 14 +++++++++++++-
 dom/localstorage/LocalStorageManager2.h           |  3 +++
 dom/localstorage/moz.build                        |  6 ++++++
 dom/localstorage/nsILocalStorageManager.idl       | 15 +++++++++++++++
 dom/storage/LocalStorage.cpp                      | 11 ++++++++++-
 dom/storage/LocalStorageManager.cpp               | 13 ++++++++++++-
 dom/storage/LocalStorageManager.h                 |  3 +++
 dom/tests/mochitest/localstorage/mochitest.ini    | 10 +++++-----
 .../localstorage/test_cookieSession.html          |  7 +++++++
 .../test_localStorageBaseSessionOnly.html         |  6 ++++++
 .../test_localStorageQuotaSessionOnly.html        |  6 ++++++
 .../test_localStorageSessionPrefOverride.html     |  5 +++++
 toolkit/modules/Services.jsm                      |  1 +
 13 files changed, 92 insertions(+), 8 deletions(-)
 create mode 100644 dom/localstorage/nsILocalStorageManager.idl

diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
index 5d6e761a7eaf..124c612931d6 100644
--- a/dom/localstorage/LocalStorageManager2.cpp
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -22,7 +22,9 @@ LocalStorageManager2::~LocalStorageManager2()
   MOZ_ASSERT(NS_IsMainThread());
 }
 
-NS_IMPL_ISUPPORTS(LocalStorageManager2, nsIDOMStorageManager)
+NS_IMPL_ISUPPORTS(LocalStorageManager2,
+                  nsIDOMStorageManager,
+                  nsILocalStorageManager)
 
 NS_IMETHODIMP
 LocalStorageManager2::PrecacheStorage(nsIPrincipal* aPrincipal,
@@ -84,5 +86,15 @@ LocalStorageManager2::CheckStorage(nsIPrincipal* aPrincipal,
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
+NS_IMETHODIMP
+LocalStorageManager2::GetNextGenLocalStorageEnabled(bool* aResult)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aResult);
+
+  *aResult = NextGenLocalStorageEnabled();
+  return NS_OK;
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/LocalStorageManager2.h b/dom/localstorage/LocalStorageManager2.h
index 6b5a5f525f93..c64c884fd6e2 100644
--- a/dom/localstorage/LocalStorageManager2.h
+++ b/dom/localstorage/LocalStorageManager2.h
@@ -8,18 +8,21 @@
 #define mozilla_dom_localstorage_LocalStorageManager2_h
 
 #include "nsIDOMStorageManager.h"
+#include "nsILocalStorageManager.h"
 
 namespace mozilla {
 namespace dom {
 
 class LocalStorageManager2 final
   : public nsIDOMStorageManager
+  , public nsILocalStorageManager
 {
 public:
   LocalStorageManager2();
 
   NS_DECL_ISUPPORTS
   NS_DECL_NSIDOMSTORAGEMANAGER
+  NS_DECL_NSILOCALSTORAGEMANAGER
 
 private:
   ~LocalStorageManager2();
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index b15ebc898ccb..99f438373c82 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -4,6 +4,12 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+XPIDL_SOURCES += [
+    'nsILocalStorageManager.idl',
+]
+
+XPIDL_MODULE = 'dom_localstorage'
+
 EXPORTS.mozilla.dom.localstorage += [
     'ActorsParent.h',
 ]
diff --git a/dom/localstorage/nsILocalStorageManager.idl b/dom/localstorage/nsILocalStorageManager.idl
new file mode 100644
index 000000000000..8e9871994437
--- /dev/null
+++ b/dom/localstorage/nsILocalStorageManager.idl
@@ -0,0 +1,15 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsISupports.idl"
+
+interface nsIPrincipal;
+
+[scriptable, builtinclass, uuid(d4f534da-2744-4db3-8774-8b187c64ade9)]
+interface nsILocalStorageManager : nsISupports
+{
+  readonly attribute boolean nextGenLocalStorageEnabled;
+};
diff --git a/dom/storage/LocalStorage.cpp b/dom/storage/LocalStorage.cpp
index 5255ce013b6d..f396de886661 100644
--- a/dom/storage/LocalStorage.cpp
+++ b/dom/storage/LocalStorage.cpp
@@ -33,7 +33,16 @@ using namespace ipc;
 
 namespace dom {
 
-NS_IMPL_CYCLE_COLLECTION_INHERITED(LocalStorage, Storage, mManager);
+NS_IMPL_CYCLE_COLLECTION_CLASS(LocalStorage)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_BEGIN_INHERITED(LocalStorage, Storage)
+NS_IMPL_CYCLE_COLLECTION_UNLINK(mManager)
+NS_IMPL_CYCLE_COLLECTION_UNLINK_END
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_BEGIN_INHERITED(LocalStorage, Storage)
+  CycleCollectionNoteChild(cb,
+                           NS_ISUPPORTS_CAST(nsIDOMStorageManager*,
+                                             tmp->mManager.get()),
+                           "mManager");
+NS_IMPL_CYCLE_COLLECTION_TRAVERSE_END
 
 NS_INTERFACE_MAP_BEGIN_CYCLE_COLLECTION(LocalStorage)
   NS_INTERFACE_MAP_ENTRY(nsISupportsWeakReference)
diff --git a/dom/storage/LocalStorageManager.cpp b/dom/storage/LocalStorageManager.cpp
index dedd22397fc3..92d1b383d6fb 100644
--- a/dom/storage/LocalStorageManager.cpp
+++ b/dom/storage/LocalStorageManager.cpp
@@ -57,7 +57,8 @@ LocalStorageManager::GetQuota()
 }
 
 NS_IMPL_ISUPPORTS(LocalStorageManager,
-                  nsIDOMStorageManager)
+                  nsIDOMStorageManager,
+                  nsILocalStorageManager)
 
 LocalStorageManager::LocalStorageManager()
   : mCaches(8)
@@ -370,6 +371,16 @@ LocalStorageManager::CheckStorage(nsIPrincipal* aPrincipal,
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocalStorageManager::GetNextGenLocalStorageEnabled(bool* aResult)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aResult);
+
+  *aResult = NextGenLocalStorageEnabled();
+  return NS_OK;
+}
+
 void
 LocalStorageManager::ClearCaches(uint32_t aUnloadFlags,
                                  const OriginAttributesPattern& aPattern,
diff --git a/dom/storage/LocalStorageManager.h b/dom/storage/LocalStorageManager.h
index 88ed6e62913d..68f1516aa5a1 100644
--- a/dom/storage/LocalStorageManager.h
+++ b/dom/storage/LocalStorageManager.h
@@ -8,6 +8,7 @@
 #define mozilla_dom_StorageManager_h
 
 #include "nsIDOMStorageManager.h"
+#include "nsILocalStorageManager.h"
 #include "StorageObserver.h"
 
 #include "LocalStorage.h"
@@ -26,10 +27,12 @@ class OriginAttributesPattern;
 namespace dom {
 
 class LocalStorageManager final : public nsIDOMStorageManager
+                                , public nsILocalStorageManager
                                 , public StorageObserverSink
 {
   NS_DECL_ISUPPORTS
   NS_DECL_NSIDOMSTORAGEMANAGER
+  NS_DECL_NSILOCALSTORAGEMANAGER
 
 public:
   LocalStorageManager();
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index 543da6ffcc77..ad2c7692db06 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -24,12 +24,12 @@ support-files =
 [test_bug746272-2.html]
 skip-if = os == "android" || verify # bug 962029
 [test_cookieBlock.html]
-#[test_cookieSession.html]
+[test_cookieSession.html]
 [test_embededNulls.html]
 [test_keySync.html]
 [test_localStorageBase.html]
 skip-if = e10s
-#[test_localStorageBaseSessionOnly.html]
+[test_localStorageBaseSessionOnly.html]
 [test_localStorageCookieSettings.html]
 #[test_localStorageEnablePref.html]
 [test_localStorageKeyOrder.html]
@@ -42,12 +42,12 @@ skip-if = toolkit == 'android'
 skip-if = toolkit == 'android' #TIMED_OUT
 #[test_localStorageQuota.html]
 #skip-if = toolkit == 'android' #TIMED_OUT
-#[test_localStorageQuotaSessionOnly.html]
-#skip-if = toolkit == 'android' || (verify && (os == 'linux' || os == 'win')) #TIMED_OUT
+[test_localStorageQuotaSessionOnly.html]
+skip-if = toolkit == 'android' || (verify && (os == 'linux' || os == 'mac' || os == 'win')) #TIMED_OUT
 [test_localStorageQuotaSessionOnly2.html]
 skip-if = true # bug 1347690
 [test_localStorageReplace.html]
 skip-if = toolkit == 'android'
 [test_storageConstructor.html]
-#[test_localStorageSessionPrefOverride.html]
+[test_localStorageSessionPrefOverride.html]
 [test_firstPartyOnlyPermission.html]
diff --git a/dom/tests/mochitest/localstorage/test_cookieSession.html b/dom/tests/mochitest/localstorage/test_cookieSession.html
index 1fdd6f4d5656..34989553509a 100644
--- a/dom/tests/mochitest/localstorage/test_cookieSession.html
+++ b/dom/tests/mochitest/localstorage/test_cookieSession.html
@@ -118,6 +118,13 @@ function startTest() {
       break;
     default:
       SimpleTest.waitForExplicitFinish();
+
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        ok(true, "Test ignored when the next gen local storage is enabled.");
+        SimpleTest.finish();
+        return;
+      }
+
       var iframe = document.createElement('iframe');
       iframe.src = 'test_cookieSession.html?1';
       document.body.appendChild(iframe);
diff --git a/dom/tests/mochitest/localstorage/test_localStorageBaseSessionOnly.html b/dom/tests/mochitest/localstorage/test_localStorageBaseSessionOnly.html
index c55a8e48d152..9b7a086e845c 100644
--- a/dom/tests/mochitest/localstorage/test_localStorageBaseSessionOnly.html
+++ b/dom/tests/mochitest/localstorage/test_localStorageBaseSessionOnly.html
@@ -9,6 +9,12 @@
 
 function startTest()
 {
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    ok(true, "Test ignored when the next gen local storage is enabled.");
+    SimpleTest.finish();
+    return;
+  }
+
   SpecialPowers.pushPermissions([{'type': 'cookie', 'allow': SpecialPowers.Ci.nsICookiePermission.ACCESS_SESSION, 'context': document}], test1);
 }
 
diff --git a/dom/tests/mochitest/localstorage/test_localStorageQuotaSessionOnly.html b/dom/tests/mochitest/localstorage/test_localStorageQuotaSessionOnly.html
index 585da7ceb9d9..643efcbeb73b 100644
--- a/dom/tests/mochitest/localstorage/test_localStorageQuotaSessionOnly.html
+++ b/dom/tests/mochitest/localstorage/test_localStorageQuotaSessionOnly.html
@@ -96,6 +96,12 @@ function doStep()
 SimpleTest.waitForExplicitFinish();
 
 function startTest() {
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    ok(true, "Test ignored when the next gen local storage is enabled.");
+    SimpleTest.finish();
+    return;
+  }
+
   SpecialPowers.pushPermissions([{'type': 'cookie', 'allow': SpecialPowers.Ci.nsICookiePermission.ACCESS_SESSION, 'context': document}], function() {
     // Initialy setup the quota to testing value of 1024B and
     // set a 500 bytes key with name length 1 (allocate 501 bytes)
diff --git a/dom/tests/mochitest/localstorage/test_localStorageSessionPrefOverride.html b/dom/tests/mochitest/localstorage/test_localStorageSessionPrefOverride.html
index 97d7bb3b208f..57f9cacbdcac 100644
--- a/dom/tests/mochitest/localstorage/test_localStorageSessionPrefOverride.html
+++ b/dom/tests/mochitest/localstorage/test_localStorageSessionPrefOverride.html
@@ -8,6 +8,11 @@
       const ACCEPT_SESSION = 2;
 
       add_task(async function() {
+        if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+          ok(true, "Test ignored when the next gen local storage is enabled.");
+          return;
+        }
+
         await SpecialPowers.pushPrefEnv({"set": [["network.cookie.lifetimePolicy",
                                                   ACCEPT_SESSION]]});
 
diff --git a/toolkit/modules/Services.jsm b/toolkit/modules/Services.jsm
index 3e56626ec7ff..bf464534e9ed 100644
--- a/toolkit/modules/Services.jsm
+++ b/toolkit/modules/Services.jsm
@@ -80,6 +80,7 @@ var initTable = {
   scriptSecurityManager: ["@mozilla.org/scriptsecuritymanager;1", "nsIScriptSecurityManager"],
   storage: ["@mozilla.org/storage/service;1", "mozIStorageService"],
   domStorageManager: ["@mozilla.org/dom/localStorage-manager;1", "nsIDOMStorageManager"],
+  lsm: ["@mozilla.org/dom/localStorage-manager;1", "nsILocalStorageManager"],
   strings: ["@mozilla.org/intl/stringbundle;1", "nsIStringBundleService"],
   telemetry: ["@mozilla.org/base/telemetry;1", "nsITelemetry"],
   textToSubURI: ["@mozilla.org/intl/texttosuburi;1", "nsITextToSubURI"],

From ae769dcc6ba54e7622690614a164e5db2dce0e33 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:52 +0100
Subject: [PATCH 26/78] Bug 1286798 - Part 12: Honor the storage preference and
 cookie settings in all LocalStorage API methods; r=asuth

---
 dom/base/nsGlobalWindowInner.cpp              |  2 +-
 dom/localstorage/LSObject.cpp                 | 36 +++++++++++++++++++
 .../mochitest/localstorage/mochitest.ini      |  2 +-
 .../localstorage/test_cookieBlock.html        |  4 +++
 4 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index b60acaaa432b..5acd94dbd489 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -5830,7 +5830,7 @@ nsGlobalWindowInner::ObserveStorageNotification(StorageEvent* aEvent,
   IgnoredErrorResult error;
   RefPtr<StorageEvent> clonedEvent =
     CloneStorageEvent(eventType, aEvent, error);
-  if (error.Failed()) {
+  if (error.Failed() || !clonedEvent) {
     return;
   }
 
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 2fd3fbd64ae3..42735f9c2c08 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -296,6 +296,11 @@ LSObject::GetLength(nsIPrincipal& aSubjectPrincipal,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return 0;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
@@ -320,6 +325,11 @@ LSObject::Key(uint32_t aIndex,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
@@ -344,6 +354,11 @@ LSObject::GetItem(const nsAString& aKey,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
@@ -365,6 +380,12 @@ LSObject::GetSupportedNames(nsTArray<nsString>& aNames)
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(*nsContentUtils::SubjectPrincipal())) {
+    // Return just an empty array.
+    aNames.Clear();
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
@@ -384,6 +405,11 @@ LSObject::SetItem(const nsAString& aKey,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
@@ -410,6 +436,11 @@ LSObject::RemoveItem(const nsAString& aKey,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
@@ -435,6 +466,11 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
 {
   AssertIsOnOwningThread();
 
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
   nsresult rv = EnsureDatabase();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index ad2c7692db06..17e57beb61ed 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -31,7 +31,7 @@ skip-if = os == "android" || verify # bug 962029
 skip-if = e10s
 [test_localStorageBaseSessionOnly.html]
 [test_localStorageCookieSettings.html]
-#[test_localStorageEnablePref.html]
+[test_localStorageEnablePref.html]
 [test_localStorageKeyOrder.html]
 [test_localStorageOriginsDiff.html]
 [test_localStorageOriginsDomainDiffs.html]
diff --git a/dom/tests/mochitest/localstorage/test_cookieBlock.html b/dom/tests/mochitest/localstorage/test_cookieBlock.html
index c95cbca49784..5e3cae6c8f90 100644
--- a/dom/tests/mochitest/localstorage/test_cookieBlock.html
+++ b/dom/tests/mochitest/localstorage/test_cookieBlock.html
@@ -30,6 +30,10 @@ function startTest()
 
 SimpleTest.waitForExplicitFinish();
 
+// Initialize storage before setting the cookie, otherwise we won't be testing
+// the checks in setItem/getItem methods.
+var storage = localStorage;
+
 SpecialPowers.pushPermissions([{'type': 'cookie', 'allow': false, 'context': document}], startTest);
 
 </script>

From 65acbea52db86c04358d1828c9b52fd71e205d56 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:55 +0100
Subject: [PATCH 27/78] Bug 1286798 - Part 13: Preparation for quota checks;
 r=asuth

---
 dom/localstorage/ActorsParent.cpp           | 114 ++++++++++----------
 dom/localstorage/LSDatabase.cpp             |  36 +++----
 dom/localstorage/LSDatabase.h               |   9 +-
 dom/localstorage/LSObject.cpp               |  67 +++++++++---
 dom/localstorage/LSObject.h                 |   6 ++
 dom/localstorage/PBackgroundLSDatabase.ipdl |  18 +++-
 6 files changed, 148 insertions(+), 102 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index d8e24afe8ad4..595a9a621a4a 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -105,6 +105,8 @@ static_assert(kSQLiteGrowthIncrement >= 0 &&
 
 const uint32_t kAutoCommitTimeoutMs = 5000;
 
+const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
+
 bool
 IsOnConnectionThread();
 
@@ -921,18 +923,16 @@ public:
   SetItem(Database* aDatabase,
           const nsString& aKey,
           const nsString& aValue,
-          bool* aChanged,
-          nsString& aOldValue);
+          LSWriteOpResponse& aResponse);
 
   void
   RemoveItem(Database* aDatabase,
              const nsString& aKey,
-             bool* aChanged,
-             nsString& aOldValue);
+             LSWriteOpResponse& aResponse);
 
   void
   Clear(Database* aDatabase,
-        bool* aChanged);
+        LSWriteOpResponse& aResponse);
 
   void
   GetKeys(nsTArray<nsString>& aKeys) const;
@@ -1148,16 +1148,14 @@ private:
   mozilla::ipc::IPCResult
   RecvSetItem(const nsString& aKey,
               const nsString& aValue,
-              bool* aChanged,
-              nsString* aOldValue) override;
+              LSWriteOpResponse* aResponse) override;
 
   mozilla::ipc::IPCResult
   RecvRemoveItem(const nsString& aKey,
-                 bool* aChanged,
-                 nsString* aOldValue) override;
+                 LSWriteOpResponse* aResponse) override;
 
   mozilla::ipc::IPCResult
-  RecvClear(bool* aChanged) override;
+  RecvClear(LSWriteOpResponse* aResponse) override;
 };
 
 class Observer final
@@ -1504,10 +1502,10 @@ class QuotaClient final
   : public mozilla::dom::quota::Client
 {
   class ClearPrivateBrowsingRunnable;
-  class PrivateBrowsingObserver;
+  class Observer;
 
   static QuotaClient* sInstance;
-  static bool sPrivateBrowsingObserverRegistered;
+  static bool sObserversRegistered;
 
   bool mShutdownRequested;
 
@@ -1607,13 +1605,13 @@ private:
   NS_DECL_NSIRUNNABLE
 };
 
-class QuotaClient::PrivateBrowsingObserver final
+class QuotaClient::Observer final
   : public nsIObserver
 {
   nsCOMPtr<nsIEventTarget> mBackgroundEventTarget;
 
 public:
-  explicit PrivateBrowsingObserver(nsIEventTarget* aBackgroundEventTarget)
+  explicit Observer(nsIEventTarget* aBackgroundEventTarget)
     : mBackgroundEventTarget(aBackgroundEventTarget)
   {
     MOZ_ASSERT(NS_IsMainThread());
@@ -1622,7 +1620,7 @@ public:
   NS_DECL_ISUPPORTS
 
 private:
-  ~PrivateBrowsingObserver()
+  ~Observer()
   {
     MOZ_ASSERT(NS_IsMainThread());
   }
@@ -2547,12 +2545,10 @@ void
 Datastore::SetItem(Database* aDatabase,
                    const nsString& aKey,
                    const nsString& aValue,
-                   bool* aChanged,
-                   nsString& aOldValue)
+                   LSWriteOpResponse& aResponse)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
-  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(!mClosed);
 
   nsString oldValue;
@@ -2578,15 +2574,16 @@ Datastore::SetItem(Database* aDatabase,
     }
   }
 
-  *aChanged = changed;
-  aOldValue = oldValue;
+  LSNotifyInfo info;
+  info.changed() = changed;
+  info.oldValue() = oldValue;
+  aResponse = info;
 }
 
 void
 Datastore::RemoveItem(Database* aDatabase,
                       const nsString& aKey,
-                      bool* aChanged,
-                      nsString& aOldValue)
+                      LSWriteOpResponse& aResponse)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
@@ -2612,16 +2609,17 @@ Datastore::RemoveItem(Database* aDatabase,
     }
   }
 
-  *aChanged = changed;
-  aOldValue = oldValue;
+  LSNotifyInfo info;
+  info.changed() = changed;
+  info.oldValue() = oldValue;
+  aResponse = info;
 }
 
 void
 Datastore::Clear(Database* aDatabase,
-                 bool* aChanged)
+                 LSWriteOpResponse& aResponse)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(!mClosed);
 
   bool changed;
@@ -2644,7 +2642,9 @@ Datastore::Clear(Database* aDatabase,
     }
   }
 
-  *aChanged = changed;
+  LSNotifyInfo info;
+  info.changed() = changed;
+  aResponse = info;
 }
 
 void
@@ -3063,12 +3063,10 @@ Database::RecvGetItem(const nsString& aKey, nsString* aValue)
 mozilla::ipc::IPCResult
 Database::RecvSetItem(const nsString& aKey,
                       const nsString& aValue,
-                      bool* aChanged,
-                      nsString* aOldValue)
+                      LSWriteOpResponse* aResponse)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aChanged);
-  MOZ_ASSERT(aOldValue);
+  MOZ_ASSERT(aResponse);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -3076,19 +3074,17 @@ Database::RecvSetItem(const nsString& aKey,
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->SetItem(this, aKey, aValue, aChanged, *aOldValue);
+  mDatastore->SetItem(this, aKey, aValue, *aResponse);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
 Database::RecvRemoveItem(const nsString& aKey,
-                         bool* aChanged,
-                         nsString* aOldValue)
+                         LSWriteOpResponse* aResponse)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aChanged);
-  MOZ_ASSERT(aOldValue);
+  MOZ_ASSERT(aResponse);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -3096,16 +3092,16 @@ Database::RecvRemoveItem(const nsString& aKey,
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->RemoveItem(this, aKey, aChanged, *aOldValue);
+  mDatastore->RemoveItem(this, aKey, *aResponse);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvClear(bool* aChanged)
+Database::RecvClear(LSWriteOpResponse* aResponse)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aChanged);
+  MOZ_ASSERT(aResponse);
   MOZ_ASSERT(mDatastore);
 
   if (NS_WARN_IF(mAllowedToClose)) {
@@ -3113,7 +3109,7 @@ Database::RecvClear(bool* aChanged)
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->Clear(this, aChanged);
+  mDatastore->Clear(this, *aResponse);
 
   return IPC_OK();
 }
@@ -4236,7 +4232,7 @@ PrepareObserverOp::GetResponse(LSRequestResponse& aResponse)
  ******************************************************************************/
 
 QuotaClient* QuotaClient::sInstance = nullptr;
-bool QuotaClient::sPrivateBrowsingObserverRegistered = false;
+bool QuotaClient::sObserversRegistered = false;
 
 QuotaClient::QuotaClient()
   : mShutdownRequested(false)
@@ -4268,21 +4264,21 @@ QuotaClient::RegisterObservers(nsIEventTarget* aBackgroundEventTarget)
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(aBackgroundEventTarget);
 
-  if (!sPrivateBrowsingObserverRegistered) {
+  if (!sObserversRegistered) {
     nsCOMPtr<nsIObserverService> obs = services::GetObserverService();
     if (NS_WARN_IF(!obs)) {
       return NS_ERROR_FAILURE;
     }
 
-    nsCOMPtr<nsIObserver> observer =
-      new PrivateBrowsingObserver(aBackgroundEventTarget);
+    nsCOMPtr<nsIObserver> observer = new Observer(aBackgroundEventTarget);
 
-    nsresult rv = obs->AddObserver(observer, "last-pb-context-exited", false);
+    nsresult rv =
+      obs->AddObserver(observer, kPrivateBrowsingObserverTopic, false);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    sPrivateBrowsingObserverRegistered = true;
+    sObserversRegistered = true;
   }
 
   return NS_OK;
@@ -4609,8 +4605,8 @@ ClearPrivateBrowsingRunnable::Run()
       MOZ_ASSERT(datastore);
 
       if (datastore->PrivateBrowsingId()) {
-        bool dummy;
-        datastore->Clear(nullptr, &dummy);
+        LSWriteOpResponse dummy;
+        datastore->Clear(nullptr, dummy);
       }
     }
   }
@@ -4618,23 +4614,27 @@ ClearPrivateBrowsingRunnable::Run()
   return NS_OK;
 }
 
-NS_IMPL_ISUPPORTS(QuotaClient::PrivateBrowsingObserver, nsIObserver)
+NS_IMPL_ISUPPORTS(QuotaClient::Observer, nsIObserver)
 
 NS_IMETHODIMP
 QuotaClient::
-PrivateBrowsingObserver::Observe(nsISupports* aSubject,
-                                 const char* aTopic,
-                                 const char16_t* aData)
+Observer::Observe(nsISupports* aSubject,
+                  const char* aTopic,
+                  const char16_t* aData)
 {
   MOZ_ASSERT(NS_IsMainThread());
-  MOZ_ASSERT(!strcmp(aTopic, "last-pb-context-exited"));
 
-  RefPtr<ClearPrivateBrowsingRunnable> runnable =
-    new ClearPrivateBrowsingRunnable();
+  if (!strcmp(aTopic, kPrivateBrowsingObserverTopic)) {
+    RefPtr<ClearPrivateBrowsingRunnable> runnable =
+      new ClearPrivateBrowsingRunnable();
 
-  MOZ_ALWAYS_SUCCEEDS(
-    mBackgroundEventTarget->Dispatch(runnable, NS_DISPATCH_NORMAL));
+    MOZ_ALWAYS_SUCCEEDS(
+      mBackgroundEventTarget->Dispatch(runnable, NS_DISPATCH_NORMAL));
 
+    return NS_OK;
+  }
+
+  NS_WARNING("Unknown observer topic!");
   return NS_OK;
 }
 
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index 83c14c0eddf4..96cbff118684 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -122,65 +122,53 @@ LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
 nsresult
 LSDatabase::SetItem(const nsAString& aKey,
                     const nsAString& aValue,
-                    bool* aChanged,
-                    nsAString& aOldValue)
+                    LSWriteOpResponse& aResponse)
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  bool changed;
-  nsString oldValue;
+  LSWriteOpResponse response;
   if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey),
                                       nsString(aValue),
-                                      &changed,
-                                      &oldValue))) {
+                                      &response))) {
     return NS_ERROR_FAILURE;
   }
 
-  *aChanged = changed;
-  aOldValue = oldValue;
+  aResponse = response;
   return NS_OK;
 }
 
 nsresult
 LSDatabase::RemoveItem(const nsAString& aKey,
-                       bool* aChanged,
-                       nsAString& aOldValue)
+                       LSWriteOpResponse& aResponse)
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  bool changed;
-  nsString oldValue;
-  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey),
-                                         &changed,
-                                         &oldValue))) {
+  LSWriteOpResponse response;
+  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey), &response))) {
     return NS_ERROR_FAILURE;
   }
 
-  *aChanged = changed;
-  aOldValue = oldValue;
+  aResponse = response;
   return NS_OK;
 }
 
 nsresult
-LSDatabase::Clear(bool* aChanged)
+LSDatabase::Clear(LSWriteOpResponse& aResponse)
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(aChanged);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  bool changed;
-  if (NS_WARN_IF(!mActor->SendClear(&changed))) {
+  LSWriteOpResponse response;
+  if (NS_WARN_IF(!mActor->SendClear(&response))) {
     return NS_ERROR_FAILURE;
   }
 
-  *aChanged = changed;
+  aResponse = response;
   return NS_OK;
 }
 
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index 01035fc1b326..c0c391c3bc1e 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -11,6 +11,7 @@ namespace mozilla {
 namespace dom {
 
 class LSDatabaseChild;
+class LSWriteOpResponse;
 
 class LSDatabase final
 {
@@ -69,16 +70,14 @@ public:
   nsresult
   SetItem(const nsAString& aKey,
           const nsAString& aValue,
-          bool* aChanged,
-          nsAString& aOldValue);
+          LSWriteOpResponse& aResponse);
 
   nsresult
   RemoveItem(const nsAString& aKey,
-             bool* aChanged,
-             nsAString& aOldValue);
+             LSWriteOpResponse& aResponse);
 
   nsresult
-  Clear(bool* aChanged);
+  Clear(LSWriteOpResponse& aResponse);
 
 private:
   ~LSDatabase();
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 42735f9c2c08..4a7e4ab01a8b 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -416,16 +416,22 @@ LSObject::SetItem(const nsAString& aKey,
     return;
   }
 
-  bool changed;
-  nsString oldValue;
-  rv = mDatabase->SetItem(aKey, aValue, &changed, oldValue);
+  LSWriteOpResponse response;
+  rv = mDatabase->SetItem(aKey, aValue, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
 
-  if (changed) {
-    OnChange(aKey, oldValue, aValue);
+  LSNotifyInfo info;
+  rv = GetInfoFromResponse(response, info);
+  if (NS_FAILED(rv)) {
+    aError.Throw(rv);
+    return;
+  }
+
+  if (info.changed()) {
+    OnChange(aKey, info.oldValue(), aValue);
   }
 }
 
@@ -447,16 +453,22 @@ LSObject::RemoveItem(const nsAString& aKey,
     return;
   }
 
-  bool changed;
-  nsString oldValue;
-  rv = mDatabase->RemoveItem(aKey, &changed, oldValue);
+  LSWriteOpResponse response;
+  rv = mDatabase->RemoveItem(aKey, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
 
-  if (changed) {
-    OnChange(aKey, oldValue, VoidString());
+  LSNotifyInfo info;
+  rv = GetInfoFromResponse(response, info);
+  if (NS_FAILED(rv)) {
+    aError.Throw(rv);
+    return;
+  }
+
+  if (info.changed()) {
+    OnChange(aKey, info.oldValue(), VoidString());
   }
 }
 
@@ -477,14 +489,21 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
     return;
   }
 
-  bool changed;
-  rv = mDatabase->Clear(&changed);
+  LSWriteOpResponse response;
+  rv = mDatabase->Clear(response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
 
-  if (changed) {
+  LSNotifyInfo info;
+  rv = GetInfoFromResponse(response, info);
+  if (NS_FAILED(rv)) {
+    aError.Throw(rv);
+    return;
+  }
+
+  if (info.changed()) {
     OnChange(VoidString(), VoidString(), VoidString());
   }
 }
@@ -694,6 +713,28 @@ LSObject::DropObserver()
   }
 }
 
+nsresult
+LSObject::GetInfoFromResponse(const LSWriteOpResponse& aResponse,
+                              LSNotifyInfo& aInfo)
+{
+  AssertIsOnOwningThread();
+
+  if (aResponse.type() == LSWriteOpResponse::Tnsresult) {
+    nsresult errorCode = aResponse.get_nsresult();
+
+    if (errorCode == NS_ERROR_FILE_NO_DEVICE_SPACE) {
+      errorCode = NS_ERROR_DOM_QUOTA_EXCEEDED_ERR;
+    }
+
+    return errorCode;
+  }
+
+  MOZ_ASSERT(aResponse.type() == LSWriteOpResponse::TLSNotifyInfo);
+
+  aInfo = aResponse.get_LSNotifyInfo();
+  return NS_OK;
+}
+
 void
 LSObject::OnChange(const nsAString& aKey,
                    const nsAString& aOldValue,
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index c6746b9ad501..ffb3ba8df970 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -26,12 +26,14 @@ class PrincipalInfo;
 namespace dom {
 
 class LSDatabase;
+class LSNotifyInfo;
 class LSObjectChild;
 class LSObserver;
 class LSRequestChild;
 class LSRequestChildCallback;
 class LSRequestParams;
 class LSRequestResponse;
+class LSWriteOpResponse;
 
 class LSObject final
   : public Storage
@@ -156,6 +158,10 @@ private:
   void
   DropObserver();
 
+  nsresult
+  GetInfoFromResponse(const LSWriteOpResponse& aResponse,
+                      LSNotifyInfo& aInfo);
+
   void
   OnChange(const nsAString& aKey,
            const nsAString& aOldValue,
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 90d8d124bbd6..78d77c7e172d 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -7,6 +7,18 @@ include protocol PBackgroundLSObject;
 namespace mozilla {
 namespace dom {
 
+struct LSNotifyInfo
+{
+  bool changed;
+  nsString oldValue;
+};
+
+union LSWriteOpResponse
+{
+  nsresult;
+  LSNotifyInfo;
+};
+
 sync protocol PBackgroundLSDatabase
 {
   manager PBackgroundLSObject;
@@ -38,13 +50,13 @@ parent:
     returns (nsString[] keys);
 
   sync SetItem(nsString key, nsString value)
-    returns (bool changed, nsString oldValue);
+    returns (LSWriteOpResponse response);
 
   sync RemoveItem(nsString key)
-    returns (bool changed, nsString oldValue);
+    returns (LSWriteOpResponse response);
 
   sync Clear()
-    returns (bool changed);
+    returns (LSWriteOpResponse response);
 
 child:
   async __delete__();

From 709a73dabf2b7fd8b0df092260b92115a45bd583 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:47:58 +0100
Subject: [PATCH 28/78] Bug 1286798 - Part 14: Enhance
 clearStoragesForPrincipal() to support clearing of storages for specific
 quota client; r=asuth

See also bug 1402254, original patch by baku.
---
 .../test/sanitize/browser_cookiePermission.js |   4 +-
 ...rgetAPI_quota_clearStoragesForPrincipal.js |   2 +-
 .../extensions/parent/ext-browsingData.js     |  22 ++--
 .../browser_ext_browsingData_indexedDB.js     |   3 +
 .../in-content/tests/siteData/browser.ini     |   2 +-
 .../tests/siteData/browser_siteData.js        |   2 +-
 browser/modules/SiteDataManager.jsm           |   2 +-
 dom/quota/ActorsParent.cpp                    | 101 +++++++++++++++---
 dom/quota/Client.h                            |  18 ++++
 dom/quota/PQuota.ipdl                         |   5 +
 dom/quota/QuotaManager.h                      |   3 +-
 dom/quota/QuotaManagerService.cpp             |  14 +++
 dom/quota/SerializationHelpers.h              |   9 ++
 dom/quota/nsIQuotaManagerService.idl          |  16 ++-
 mobile/android/modules/Sanitizer.jsm          |   2 +-
 .../wptrunner/executors/executormarionette.py |   2 +-
 .../components/cleardata/ClearDataService.js  |   8 +-
 17 files changed, 180 insertions(+), 35 deletions(-)

diff --git a/browser/base/content/test/sanitize/browser_cookiePermission.js b/browser/base/content/test/sanitize/browser_cookiePermission.js
index 4ca839ce72d7..f800395193a5 100644
--- a/browser/base/content/test/sanitize/browser_cookiePermission.js
+++ b/browser/base/content/test/sanitize/browser_cookiePermission.js
@@ -277,7 +277,7 @@ add_task(async function deleteStorageInAboutURL() {
 
   let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin("about:newtab");
   await new Promise(aResolve => {
-    let req = Services.qms.clearStoragesForPrincipal(principal, null, false);
+    let req = Services.qms.clearStoragesForPrincipal(principal);
     req.callback = () => { aResolve(); };
   });
 });
@@ -313,7 +313,7 @@ add_task(async function deleteStorageOnlyCustomPermissionInAboutURL() {
 
   let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin("about:newtab");
   await new Promise(aResolve => {
-    let req = Services.qms.clearStoragesForPrincipal(principal, null, false);
+    let req = Services.qms.clearStoragesForPrincipal(principal);
     req.callback = () => { aResolve(); };
   });
 
diff --git a/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js b/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
index 80790ee4a1ea..e557c2dd569c 100644
--- a/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
+++ b/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
@@ -127,7 +127,7 @@ add_task(async function test_quota_clearStoragesForPrincipal() {
   let httpURI = caUtils.makeURI("http://" + TEST_HOST);
   let httpPrincipal = Services.scriptSecurityManager
                               .createCodebasePrincipal(httpURI, {});
-  Services.qms.clearStoragesForPrincipal(httpPrincipal, null, true);
+  Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
 
   for (let userContextId of Object.keys(USER_CONTEXTS)) {
     // Open our tab in the given user context.
diff --git a/browser/components/extensions/parent/ext-browsingData.js b/browser/components/extensions/parent/ext-browsingData.js
index 8b6f811d2b11..0156a3876e70 100644
--- a/browser/components/extensions/parent/ext-browsingData.js
+++ b/browser/components/extensions/parent/ext-browsingData.js
@@ -77,22 +77,26 @@ const clearHistory = options => {
 const clearIndexedDB = async function(options) {
   let promises = [];
 
-  await new Promise(resolve => {
+  await new Promise((resolve, reject) => {
     quotaManagerService.getUsage(request => {
       if (request.resultCode != Cr.NS_OK) {
-        // We are probably shutting down. We don't want to propagate the error,
-        // rejecting the promise.
-        resolve();
+        reject({message: "Clear indexedDB failed"});
         return;
       }
 
       for (let item of request.result) {
         let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(item.origin);
-        let uri = principal.URI;
-        if (uri.scheme == "http" || uri.scheme == "https" || uri.scheme == "file") {
-          promises.push(new Promise(r => {
-            let req = quotaManagerService.clearStoragesForPrincipal(principal, null, false);
-            req.callback = () => { r(); };
+        let scheme = principal.URI.scheme;
+        if (scheme == "http" || scheme == "https" || scheme == "file") {
+          promises.push(new Promise((resolve, reject) => {
+            let clearRequest = quotaManagerService.clearStoragesForPrincipal(principal, null, "idb");
+            clearRequest.callback = () => {
+              if (clearRequest.resultCode == Cr.NS_OK) {
+                resolve();
+              } else {
+                reject({message: "Clear indexedDB failed"});
+              }
+            };
           }));
         }
       }
diff --git a/browser/components/extensions/test/browser/browser_ext_browsingData_indexedDB.js b/browser/components/extensions/test/browser/browser_ext_browsingData_indexedDB.js
index b4825c7d6910..a65dc4ab71af 100644
--- a/browser/components/extensions/test/browser/browser_ext_browsingData_indexedDB.js
+++ b/browser/components/extensions/test/browser/browser_ext_browsingData_indexedDB.js
@@ -53,6 +53,9 @@ add_task(async function testIndexedDB() {
       let origins = [];
       Services.qms.getUsage(request => {
         for (let i = 0; i < request.result.length; ++i) {
+          if (request.result[i].usage === 0) {
+            continue;
+          }
           if (request.result[i].origin.startsWith("http://mochi.test") ||
               request.result[i].origin.startsWith("http://example.com")) {
             origins.push(request.result[i].origin);
diff --git a/browser/components/preferences/in-content/tests/siteData/browser.ini b/browser/components/preferences/in-content/tests/siteData/browser.ini
index 6c8b7fcba763..c5fe2caee189 100644
--- a/browser/components/preferences/in-content/tests/siteData/browser.ini
+++ b/browser/components/preferences/in-content/tests/siteData/browser.ini
@@ -9,7 +9,7 @@ support-files =
 
 [browser_clearSiteData.js]
 [browser_siteData.js]
-skip-if = (os == 'linux' && debug) # Bug 1439332
+skip-if = (os == 'linux' && debug) || verify # Bug 1439332 and bug 1436395
 [browser_siteData2.js]
 [browser_siteData3.js]
 [browser_siteData_multi_select.js]
diff --git a/browser/components/preferences/in-content/tests/siteData/browser_siteData.js b/browser/components/preferences/in-content/tests/siteData/browser_siteData.js
index 8a063c0ab3af..76f02cb90ff6 100644
--- a/browser/components/preferences/in-content/tests/siteData/browser_siteData.js
+++ b/browser/components/preferences/in-content/tests/siteData/browser_siteData.js
@@ -44,7 +44,7 @@ add_task(async function() {
   await new Promise(resolve => {
     let principal = Services.scriptSecurityManager
                             .createCodebasePrincipalFromOrigin(TEST_QUOTA_USAGE_ORIGIN);
-    let request = Services.qms.clearStoragesForPrincipal(principal, null, true);
+    let request = Services.qms.clearStoragesForPrincipal(principal, null, null, true);
     request.callback = resolve;
   });
 
diff --git a/browser/modules/SiteDataManager.jsm b/browser/modules/SiteDataManager.jsm
index e86aa4b9029e..6f0baac26304 100644
--- a/browser/modules/SiteDataManager.jsm
+++ b/browser/modules/SiteDataManager.jsm
@@ -304,7 +304,7 @@ var SiteDataManager = {
         // We are clearing *All* across OAs so need to ensure a principal without suffix here,
         // or the call of `clearStoragesForPrincipal` would fail.
         principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(originNoSuffix);
-        let request = this._qms.clearStoragesForPrincipal(principal, null, true);
+        let request = this._qms.clearStoragesForPrincipal(principal, null, null, true);
         request.callback = resolve;
       }));
     }
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 837619a25f96..1e7b047082a8 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -953,6 +953,7 @@ class NormalOriginOperationBase
 protected:
   Nullable<PersistenceType> mPersistenceType;
   OriginScope mOriginScope;
+  Nullable<Client::Type> mClientType;
   mozilla::Atomic<bool> mCanceled;
   const bool mExclusive;
 
@@ -5506,16 +5507,22 @@ QuotaManager::EnsureOriginDirectory(nsIFile* aDirectory,
 
 void
 QuotaManager::OriginClearCompleted(PersistenceType aPersistenceType,
-                                   const nsACString& aOrigin)
+                                   const nsACString& aOrigin,
+                                   const Nullable<Client::Type>& aClientType)
 {
   AssertIsOnIOThread();
 
-  if (aPersistenceType == PERSISTENCE_TYPE_PERSISTENT) {
-    mInitializedOrigins.RemoveElement(aOrigin);
-  }
+  if (aClientType.IsNull()) {
+    if (aPersistenceType == PERSISTENCE_TYPE_PERSISTENT) {
+      mInitializedOrigins.RemoveElement(aOrigin);
+    }
 
-  for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
-    mClients[index]->OnOriginClearCompleted(aPersistenceType, aOrigin);
+    for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
+      mClients[index]->OnOriginClearCompleted(aPersistenceType, aOrigin);
+    }
+  } else {
+    mClients[aClientType.Value()]->OnOriginClearCompleted(aPersistenceType,
+                                                          aOrigin);
   }
 }
 
@@ -5987,7 +5994,8 @@ QuotaManager::CheckTemporaryStorageLimits()
 
   for (const OriginParams& doomedOrigin : doomedOrigins) {
     OriginClearCompleted(doomedOrigin.mPersistenceType,
-                         doomedOrigin.mOrigin);
+                         doomedOrigin.mOrigin,
+                         Nullable<Client::Type>());
   }
 
   if (mTemporaryStorageUsage > mTemporaryStorageLimit) {
@@ -6527,7 +6535,8 @@ FinalizeOriginEvictionOp::DoDirectoryWork(QuotaManager* aQuotaManager)
 
   for (RefPtr<DirectoryLockImpl>& lock : mLocks) {
     aQuotaManager->OriginClearCompleted(lock->GetPersistenceType().Value(),
-                                        lock->GetOriginScope().GetOrigin());
+                                        lock->GetOriginScope().GetOrigin(),
+                                        Nullable<Client::Type>());
   }
 
   return NS_OK;
@@ -6561,7 +6570,7 @@ NormalOriginOperationBase::Open()
 
   QuotaManager::Get()->OpenDirectoryInternal(mPersistenceType,
                                              mOriginScope,
-                                             Nullable<Client::Type>(),
+                                             mClientType,
                                              mExclusive,
                                              this);
 }
@@ -7713,6 +7722,62 @@ ClearRequestBase::DeleteFiles(QuotaManager* aQuotaManager,
       return;
     }
 
+    UsageInfo usageInfo;
+
+    if (!mClientType.IsNull()) {
+      Client::Type clientType = mClientType.Value();
+
+      nsAutoString clientDirectoryName;
+      rv = Client::TypeToText(clientType, clientDirectoryName);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return;
+      }
+
+      rv = file->Append(clientDirectoryName);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return;
+      }
+
+      bool exists;
+      rv = file->Exists(&exists);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return;
+      }
+
+      if (!exists) {
+        continue;
+      }
+
+      bool initialized;
+      if (aPersistenceType == PERSISTENCE_TYPE_PERSISTENT) {
+        initialized = aQuotaManager->IsOriginInitialized(origin);
+      } else {
+        initialized = aQuotaManager->IsTemporaryStorageInitialized();
+      }
+
+      Client* client = aQuotaManager->GetClient(clientType);
+      MOZ_ASSERT(client);
+
+      Atomic<bool> dummy(false);
+      if (initialized) {
+        rv = client->GetUsageForOrigin(aPersistenceType,
+                                       group,
+                                       origin,
+                                       dummy,
+                                       &usageInfo);
+      } else {
+        rv = client->InitOrigin(aPersistenceType,
+                                group,
+                                origin,
+                                dummy,
+                                &usageInfo);
+
+      }
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return;
+      }
+    }
+
     for (uint32_t index = 0; index < 10; index++) {
       // We can't guarantee that this will always succeed on Windows...
       if (NS_SUCCEEDED((rv = file->Remove(true)))) {
@@ -7729,12 +7794,18 @@ ClearRequestBase::DeleteFiles(QuotaManager* aQuotaManager,
     }
 
     if (aPersistenceType != PERSISTENCE_TYPE_PERSISTENT) {
-      aQuotaManager->RemoveQuotaForOrigin(aPersistenceType, group, origin);
+      if (mClientType.IsNull()) {
+        aQuotaManager->RemoveQuotaForOrigin(aPersistenceType, group, origin);
+      } else {
+        aQuotaManager->DecreaseUsageForOrigin(aPersistenceType,
+                                              group,
+                                              origin,
+                                              usageInfo.TotalUsage());
+      }
     }
 
-    aQuotaManager->OriginClearCompleted(aPersistenceType, origin);
+    aQuotaManager->OriginClearCompleted(aPersistenceType, origin, mClientType);
   }
-
 }
 
 nsresult
@@ -7778,6 +7849,12 @@ ClearOriginOp::Init(Quota* aQuota)
     mPersistenceType.SetValue(mParams.persistenceType());
   }
 
+  if (mParams.clientTypeIsExplicit()) {
+    MOZ_ASSERT(mParams.clientType() != Client::TYPE_MAX);
+
+    mClientType.SetValue(mParams.clientType());
+  }
+
   mNeedsMainThreadInit = true;
 
   return true;
diff --git a/dom/quota/Client.h b/dom/quota/Client.h
index 7a92b208886d..c701e0d2bf94 100644
--- a/dom/quota/Client.h
+++ b/dom/quota/Client.h
@@ -122,6 +122,24 @@ public:
     return NS_OK;
   }
 
+  static nsresult
+  NullableTypeFromText(const nsAString& aText, Nullable<Type>* aType)
+  {
+    if (aText.IsVoid()) {
+      *aType = Nullable<Type>();
+      return NS_OK;
+    }
+
+    Type type;
+    nsresult rv = TypeFromText(aText, type);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    *aType = Nullable<Type>(type);
+    return NS_OK;
+  }
+
   // Methods which are called on the IO thread.
   virtual nsresult
   UpgradeStorageFrom1_0To2_0(nsIFile* aDirectory)
diff --git a/dom/quota/PQuota.ipdl b/dom/quota/PQuota.ipdl
index 81e2b4660eb8..d0f9dfbc4a2c 100644
--- a/dom/quota/PQuota.ipdl
+++ b/dom/quota/PQuota.ipdl
@@ -13,6 +13,9 @@ include "mozilla/dom/quota/SerializationHelpers.h";
 using mozilla::dom::quota::PersistenceType
   from "mozilla/dom/quota/PersistenceType.h";
 
+using mozilla::dom::quota::Client::Type
+  from "mozilla/dom/quota/Client.h";
+
 namespace mozilla {
 namespace dom {
 namespace quota {
@@ -53,6 +56,8 @@ struct ClearOriginParams
   PrincipalInfo principalInfo;
   PersistenceType persistenceType;
   bool persistenceTypeIsExplicit;
+  Type clientType;
+  bool clientTypeIsExplicit;
   bool clearAll;
 };
 
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index f8a0a7ac10c2..bc168b708c42 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -308,7 +308,8 @@ public:
 
   void
   OriginClearCompleted(PersistenceType aPersistenceType,
-                       const nsACString& aOrigin);
+                       const nsACString& aOrigin,
+                       const Nullable<Client::Type>& aClientType);
 
   void
   ResetOrClearCompleted();
diff --git a/dom/quota/QuotaManagerService.cpp b/dom/quota/QuotaManagerService.cpp
index 8b39cea75db8..db52c4a8f103 100644
--- a/dom/quota/QuotaManagerService.cpp
+++ b/dom/quota/QuotaManagerService.cpp
@@ -610,6 +610,7 @@ QuotaManagerService::Clear(nsIQuotaRequest** _retval)
 NS_IMETHODIMP
 QuotaManagerService::ClearStoragesForPrincipal(nsIPrincipal* aPrincipal,
                                                const nsACString& aPersistenceType,
+                                               const nsAString& aClientType,
                                                bool aClearAll,
                                                nsIQuotaRequest** _retval)
 {
@@ -648,6 +649,19 @@ QuotaManagerService::ClearStoragesForPrincipal(nsIPrincipal* aPrincipal,
     params.persistenceTypeIsExplicit() = true;
   }
 
+  Nullable<Client::Type> clientType;
+  rv = Client::NullableTypeFromText(aClientType, &clientType);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  if (clientType.IsNull()) {
+    params.clientTypeIsExplicit() = false;
+  } else {
+    params.clientType() = clientType.Value();
+    params.clientTypeIsExplicit() = true;
+  }
+
   params.clearAll() = aClearAll;
 
   nsAutoPtr<PendingRequestInfo> info(new RequestInfo(request, params));
diff --git a/dom/quota/SerializationHelpers.h b/dom/quota/SerializationHelpers.h
index 2f32b195e578..29b75e176f00 100644
--- a/dom/quota/SerializationHelpers.h
+++ b/dom/quota/SerializationHelpers.h
@@ -9,6 +9,7 @@
 
 #include "ipc/IPCMessageUtils.h"
 
+#include "mozilla/dom/quota/Client.h"
 #include "mozilla/dom/quota/PersistenceType.h"
 #include "mozilla/OriginAttributes.h"
 
@@ -22,6 +23,14 @@ struct ParamTraits<mozilla::dom::quota::PersistenceType> :
                                mozilla::dom::quota::PERSISTENCE_TYPE_INVALID>
 { };
 
+template <>
+struct ParamTraits<mozilla::dom::quota::Client::Type> :
+  public ContiguousEnumSerializer<
+                          mozilla::dom::quota::Client::Type,
+                          mozilla::dom::quota::Client::IDB,
+                          mozilla::dom::quota::Client::TYPE_MAX>
+{ };
+
 template <>
 struct ParamTraits<mozilla::OriginAttributesPattern>
 {
diff --git a/dom/quota/nsIQuotaManagerService.idl b/dom/quota/nsIQuotaManagerService.idl
index 196ea1316b5e..3de0f0a76dc9 100644
--- a/dom/quota/nsIQuotaManagerService.idl
+++ b/dom/quota/nsIQuotaManagerService.idl
@@ -104,7 +104,20 @@ interface nsIQuotaManagerService : nsISupports
    *        A principal for the origin whose storages are to be cleared.
    * @param aPersistenceType
    *        An optional string that tells what persistence type of storages
-   *        will be cleared.
+   *        will be cleared.  If omitted (or void), all persistence types will
+   *        be cleared for the principal.  If a single persistence type
+   *        ("persistent", "temporary", or "default") is provided, then only
+   *        that persistence directory will be considered.  Note that
+   *        "persistent" is different than being "persisted" via persist() and
+   *        is only for chrome principals.  See bug 1354500 for more info.
+   *        In general, null is the right thing to pass here.
+   * @param aClientType
+   *        An optional string that tells what client type of storages
+   *        will be cleared.  If omitted (or void), all client types will be
+   *        cleared for the principal.  If a single client type is provided
+   *        from Client.h, then only that client's storage will be cleared.
+   *        If you want to clear multiple client types (but not all), then you
+   *        must call this method multiple times.
    * @param aClearAll
    *        An optional boolean to indicate clearing all storages under the
    *        given origin.
@@ -112,6 +125,7 @@ interface nsIQuotaManagerService : nsISupports
   [must_use] nsIQuotaRequest
   clearStoragesForPrincipal(in nsIPrincipal aPrincipal,
                             [optional] in ACString aPersistenceType,
+                            [optional] in AString aClientType,
                             [optional] in boolean aClearAll);
 
   /**
diff --git a/mobile/android/modules/Sanitizer.jsm b/mobile/android/modules/Sanitizer.jsm
index 5b3ea9051ae2..a265cc3f4184 100644
--- a/mobile/android/modules/Sanitizer.jsm
+++ b/mobile/android/modules/Sanitizer.jsm
@@ -205,7 +205,7 @@ Sanitizer.prototype = {
               let uri = principal.URI;
               if (uri.scheme == "http" || uri.scheme == "https" || uri.scheme == "file") {
                 promises.push(new Promise(r => {
-                  let req = quotaManagerService.clearStoragesForPrincipal(principal, null, false);
+                  let req = quotaManagerService.clearStoragesForPrincipal(principal);
                   req.callback = () => { r(); };
                 }));
               }
diff --git a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
index 3d7282c33475..f5b67c0b9c73 100644
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/executors/executormarionette.py
@@ -292,7 +292,7 @@ class MarionetteStorageProtocolPart(StorageProtocolPart):
             let principal = ssm.createCodebasePrincipal(uri, {});
             let qms = Components.classes["@mozilla.org/dom/quota-manager-service;1"]
                                 .getService(Components.interfaces.nsIQuotaManagerService);
-            qms.clearStoragesForPrincipal(principal, "default", true);
+            qms.clearStoragesForPrincipal(principal, "default", null, true);
             """ % url
         with self.marionette.using_context(self.marionette.CONTEXT_CHROME):
             self.marionette.execute_script(script)
diff --git a/toolkit/components/cleardata/ClearDataService.js b/toolkit/components/cleardata/ClearDataService.js
index 2464610d0059..5325bb485c14 100644
--- a/toolkit/components/cleardata/ClearDataService.js
+++ b/toolkit/components/cleardata/ClearDataService.js
@@ -301,7 +301,7 @@ const QuotaCleaner = {
       .then(exceptionThrown => {
         // QuotaManager
         return new Promise((aResolve, aReject) => {
-          let req = Services.qms.clearStoragesForPrincipal(aPrincipal, null, false);
+          let req = Services.qms.clearStoragesForPrincipal(aPrincipal);
           req.callback = () => {
             if (exceptionThrown) {
               aReject();
@@ -334,11 +334,11 @@ const QuotaCleaner = {
                                      .createCodebasePrincipal(httpsURI, aOriginAttributes);
         return Promise.all([
           new Promise(aResolve => {
-            let req = Services.qms.clearStoragesForPrincipal(httpPrincipal, null, true);
+            let req = Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
             req.callback = () => { aResolve(); };
           }),
           new Promise(aResolve => {
-            let req = Services.qms.clearStoragesForPrincipal(httpsPrincipal, null, true);
+            let req = Services.qms.clearStoragesForPrincipal(httpsPrincipal, null, null, true);
             req.callback = () => { aResolve(); };
           }),
         ]).then(() => {
@@ -395,7 +395,7 @@ const QuotaCleaner = {
                   principal.URI.scheme == "https" ||
                   principal.URI.scheme == "file") {
                 promises.push(new Promise(aResolve => {
-                  let req = Services.qms.clearStoragesForPrincipal(principal, null, false);
+                  let req = Services.qms.clearStoragesForPrincipal(principal);
                   req.callback = () => { aResolve(); };
                 }));
               }

From 7313de76cde245e24c24b50ca528e056b6938325 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:01 +0100
Subject: [PATCH 29/78] Bug 1286798 - Part 15: Fix clearLocalStorage() in
 browser extensions; r=asuth

---
 .../extensions/parent/ext-browsingData.js     | 40 +++++++++++++++++++
 .../test/browser/browser-common.ini           |  2 +-
 2 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/browser/components/extensions/parent/ext-browsingData.js b/browser/components/extensions/parent/ext-browsingData.js
index 0156a3876e70..28ecc5ffcb4d 100644
--- a/browser/components/extensions/parent/ext-browsingData.js
+++ b/browser/components/extensions/parent/ext-browsingData.js
@@ -113,6 +113,46 @@ const clearLocalStorage = async function(options) {
     return Promise.reject(
       {message: "Firefox does not support clearing localStorage with 'since'."});
   }
+
+  if (Services.lsm.nextGenLocalStorageEnabled) {
+    // Ideally we could reuse the logic in Sanitizer.jsm or nsIClearDataService,
+    // but this API exposes an ability to wipe data at a much finger granularity
+    // than those APIs.  So custom logic is used here to wipe only the QM
+    // localStorage client (when in use).
+
+    let promises = [];
+
+    await new Promise((resolve, reject) => {
+      quotaManagerService.getUsage(request => {
+        if (request.resultCode != Cr.NS_OK) {
+          reject({message: "Clear localStorage failed"});
+          return;
+        }
+
+        for (let item of request.result) {
+          let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(item.origin);
+          let host = principal.URI.hostPort;
+          if (!options.hostnames || options.hostnames.includes(host)) {
+            promises.push(new Promise((resolve, reject) => {
+              let clearRequest = quotaManagerService.clearStoragesForPrincipal(principal, "default", "ls");
+              clearRequest.callback = () => {
+                if (clearRequest.resultCode == Cr.NS_OK) {
+                  resolve();
+                } else {
+                  reject({message: "Clear localStorage failed"});
+                }
+              };
+            }));
+          }
+        }
+
+        resolve();
+      });
+    });
+
+    return Promise.all(promises);
+  }
+
   if (options.hostnames) {
     for (let hostname of options.hostnames) {
       Services.obs.notifyObservers(null, "extension:purge-localStorage", hostname);
diff --git a/browser/components/extensions/test/browser/browser-common.ini b/browser/components/extensions/test/browser/browser-common.ini
index 9fd71872c21b..62d5d0703641 100644
--- a/browser/components/extensions/test/browser/browser-common.ini
+++ b/browser/components/extensions/test/browser/browser-common.ini
@@ -65,7 +65,7 @@ skip-if = (os == 'mac' && debug) # Bug 1482004, also fails in test-verify
 [browser_ext_browsingData_formData.js]
 [browser_ext_browsingData_history.js]
 [browser_ext_browsingData_indexedDB.js]
-#[browser_ext_browsingData_localStorage.js]
+[browser_ext_browsingData_localStorage.js]
 [browser_ext_browsingData_pluginData.js]
 [browser_ext_browsingData_serviceWorkers.js]
 [browser_ext_chrome_settings_overrides_home.js]

From 944443d4814de020dfa1aeafde1beecc5badd864 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:05 +0100
Subject: [PATCH 30/78] Bug 1286798 - Part 16: Adjust ClearDataService for new
 local storage implementation; r=asuth

This patch also adds support for creating LSObjects from chrome for any given origin which can be used for example by xpcshell tests.
---
 .../components/sessionstore/test/browser.ini  |   2 +-
 dom/base/nsGlobalWindowInner.cpp              |   2 +-
 dom/localstorage/LSObject.cpp                 |  53 +++++++-
 dom/localstorage/LSObject.h                   |  11 +-
 dom/localstorage/LocalStorageManager2.cpp     |  15 ++-
 dom/tests/mochitest/localstorage/chrome.ini   |   2 +-
 dom/tests/moz.build                           |   2 +-
 .../components/cleardata/ClearDataService.js  | 115 +++++++++++++-----
 toolkit/components/extensions/Extension.jsm   |  13 +-
 .../test/mochitest/mochitest-common.ini       |   6 +-
 toolkit/forgetaboutsite/moz.build             |   2 +-
 .../test/unit/test_removeDataFromDomain.js    |  31 -----
 .../forgetaboutsite/test/unit/xpcshell.ini    |   2 +-
 13 files changed, 179 insertions(+), 77 deletions(-)

diff --git a/browser/components/sessionstore/test/browser.ini b/browser/components/sessionstore/test/browser.ini
index b1ecf4b2bdb2..3a05d3734099 100644
--- a/browser/components/sessionstore/test/browser.ini
+++ b/browser/components/sessionstore/test/browser.ini
@@ -154,7 +154,7 @@ skip-if = true
 [browser_461634.js]
 [browser_463205.js]
 [browser_463206.js]
-#[browser_464199.js]
+[browser_464199.js]
 [browser_465215.js]
 [browser_465223.js]
 [browser_466937.js]
diff --git a/dom/base/nsGlobalWindowInner.cpp b/dom/base/nsGlobalWindowInner.cpp
index 5acd94dbd489..f37f8e4415ca 100644
--- a/dom/base/nsGlobalWindowInner.cpp
+++ b/dom/base/nsGlobalWindowInner.cpp
@@ -4913,7 +4913,7 @@ nsGlobalWindowInner::GetLocalStorage(ErrorResult& aError)
     RefPtr<Storage> storage;
 
     if (NextGenLocalStorageEnabled()) {
-      aError = LSObject::Create(this, getter_AddRefs(storage));
+      aError = LSObject::CreateForWindow(this, getter_AddRefs(storage));
     } else {
       nsresult rv;
       nsCOMPtr<nsIDOMStorageManager> storageManager =
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 4a7e4ab01a8b..86a01bee561a 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -150,8 +150,8 @@ LSObject::~LSObject()
 
 // static
 nsresult
-LSObject::Create(nsPIDOMWindowInner* aWindow,
-                 Storage** aStorage)
+LSObject::CreateForWindow(nsPIDOMWindowInner* aWindow,
+                          Storage** aStorage)
 {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(aWindow);
@@ -168,6 +168,10 @@ LSObject::Create(nsPIDOMWindowInner* aWindow,
     return NS_ERROR_FAILURE;
   }
 
+  if (nsContentUtils::IsSystemPrincipal(principal)) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   nsAutoPtr<PrincipalInfo> principalInfo(new PrincipalInfo());
   nsresult rv = PrincipalToPrincipalInfo(principal, principalInfo);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -206,6 +210,51 @@ LSObject::Create(nsPIDOMWindowInner* aWindow,
   return NS_OK;
 }
 
+// static
+nsresult
+LSObject::CreateForPrincipal(nsPIDOMWindowInner* aWindow,
+                             nsIPrincipal* aPrincipal,
+                             const nsAString& aDocumentURI,
+                             bool aPrivate,
+                             LSObject** aObject)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(aObject);
+
+  nsAutoPtr<PrincipalInfo> principalInfo(new PrincipalInfo());
+  nsresult rv = PrincipalToPrincipalInfo(aPrincipal, principalInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(principalInfo->type() == PrincipalInfo::TContentPrincipalInfo ||
+             principalInfo->type() == PrincipalInfo::TSystemPrincipalInfo);
+
+  nsCString origin;
+
+  if (principalInfo->type() == PrincipalInfo::TSystemPrincipalInfo) {
+    QuotaManager::GetInfoForChrome(nullptr, nullptr, &origin);
+  } else {
+    rv = QuotaManager::GetInfoFromPrincipal(aPrincipal,
+                                            nullptr,
+                                            nullptr,
+                                            &origin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  RefPtr<LSObject> object = new LSObject(aWindow, aPrincipal);
+  object->mPrincipalInfo = std::move(principalInfo);
+  object->mPrivateBrowsingId = aPrivate ? 1 : 0;
+  object->mOrigin = origin;
+  object->mDocumentURI = aDocumentURI;
+
+  object.forget(aObject);
+  return NS_OK;
+}
+
 // static
 already_AddRefed<nsIEventTarget>
 LSObject::GetSyncLoopEventTarget()
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index ffb3ba8df970..789918f87cf2 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -56,8 +56,15 @@ class LSObject final
 
 public:
   static nsresult
-  Create(nsPIDOMWindowInner* aWindow,
-         Storage** aStorage);
+  CreateForWindow(nsPIDOMWindowInner* aWindow,
+                  Storage** aStorage);
+
+  static nsresult
+  CreateForPrincipal(nsPIDOMWindowInner* aWindow,
+                     nsIPrincipal* aPrincipal,
+                     const nsAString& aDocumentURI,
+                     bool aPrivate,
+                     LSObject** aObject);
 
   /**
    * Used for requests from the parent process to the parent process; in that
diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
index 124c612931d6..b1fe2444c204 100644
--- a/dom/localstorage/LocalStorageManager2.cpp
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -48,7 +48,20 @@ LocalStorageManager2::CreateStorage(mozIDOMWindow* aWindow,
   MOZ_ASSERT(aPrincipal);
   MOZ_ASSERT(_retval);
 
-  return NS_ERROR_NOT_IMPLEMENTED;
+  nsCOMPtr<nsPIDOMWindowInner> inner = nsPIDOMWindowInner::From(aWindow);
+
+  RefPtr<LSObject> object;
+  nsresult rv = LSObject::CreateForPrincipal(inner,
+                                             aPrincipal,
+                                             aDocumentURI,
+                                             aPrivate,
+                                             getter_AddRefs(object));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  object.forget(_retval);
+  return NS_OK;
 }
 
 NS_IMETHODIMP
diff --git a/dom/tests/mochitest/localstorage/chrome.ini b/dom/tests/mochitest/localstorage/chrome.ini
index af961412ae78..10f164bfd6e6 100644
--- a/dom/tests/mochitest/localstorage/chrome.ini
+++ b/dom/tests/mochitest/localstorage/chrome.ini
@@ -7,5 +7,5 @@ support-files =
 
 [test_localStorageBasePrivateBrowsing_perwindowpb.html]
 skip-if = true # bug 1156725
-#[test_localStorageFromChrome.xhtml]
+[test_localStorageFromChrome.xhtml]
 #[test_localStorageQuotaPrivateBrowsing_perwindowpb.html]
diff --git a/dom/tests/moz.build b/dom/tests/moz.build
index d99e9e72a9b8..62683d990a6c 100644
--- a/dom/tests/moz.build
+++ b/dom/tests/moz.build
@@ -179,7 +179,7 @@ MOCHITEST_CHROME_MANIFESTS += [
     'mochitest/general/chrome.ini',
     'mochitest/geolocation/chrome.ini',
     'mochitest/keyhandling/chrome.ini',
-#    'mochitest/localstorage/chrome.ini',
+    'mochitest/localstorage/chrome.ini',
     'mochitest/sessionstorage/chrome.ini',
     'mochitest/webcomponents/chrome.ini',
     'mochitest/whatwg/chrome.ini',
diff --git a/toolkit/components/cleardata/ClearDataService.js b/toolkit/components/cleardata/ClearDataService.js
index 5325bb485c14..45a0f74542ba 100644
--- a/toolkit/components/cleardata/ClearDataService.js
+++ b/toolkit/components/cleardata/ClearDataService.js
@@ -291,20 +291,25 @@ const AppCacheCleaner = {
 
 const QuotaCleaner = {
   deleteByPrincipal(aPrincipal) {
-    // localStorage
-    Services.obs.notifyObservers(null, "browser:purge-domain-data",
-                                 aPrincipal.URI.host);
+    if (!Services.lsm.nextGenLocalStorageEnabled) {
+      // localStorage: The legacy LocalStorage implementation that will
+      // eventually be removed depends on this observer notification to clear by
+      // principal.  Only generate it if we're using the legacy implementation.
+      Services.obs.notifyObservers(null, "browser:purge-domain-data",
+                                   aPrincipal.URI.host);
+    }
 
     // ServiceWorkers: they must be removed before cleaning QuotaManager.
     return ServiceWorkerCleanUp.removeFromPrincipal(aPrincipal)
       .then(_ => /* exceptionThrown = */ false, _ => /* exceptionThrown = */ true)
       .then(exceptionThrown => {
-        // QuotaManager
+        // QuotaManager: In the event of a failure, we call reject to propagate
+        // the error upwards.
         return new Promise((aResolve, aReject) => {
           let req = Services.qms.clearStoragesForPrincipal(aPrincipal);
           req.callback = () => {
-            if (exceptionThrown) {
-              aReject();
+            if (exceptionThrown || req.resultCode != Cr.NS_OK) {
+              aReject({message: "Delete by principal failed"});
             } else {
               aResolve();
             }
@@ -314,8 +319,12 @@ const QuotaCleaner = {
   },
 
   deleteByHost(aHost, aOriginAttributes) {
-    // localStorage
-    Services.obs.notifyObservers(null, "browser:purge-domain-data", aHost);
+    if (!Services.lsm.nextGenLocalStorageEnabled) {
+      // localStorage: The legacy LocalStorage implementation that will
+      // eventually be removed depends on this observer notification to clear by
+      // principal.  Only generate it if we're using the legacy implementation.
+      Services.obs.notifyObservers(null, "browser:purge-domain-data", aHost);
+    }
 
     let exceptionThrown = false;
 
@@ -324,7 +333,9 @@ const QuotaCleaner = {
       ServiceWorkerCleanUp.removeFromHost("http://" + aHost).catch(_ => { exceptionThrown = true; }),
       ServiceWorkerCleanUp.removeFromHost("https://" + aHost).catch(_ => { exceptionThrown = true; }),
     ]).then(() => {
-        // QuotaManager
+        // QuotaManager: In the event of a failure, we call reject to propagate
+        // the error upwards.
+
         // delete data from both HTTP and HTTPS sites
         let httpURI = Services.io.newURI("http://" + aHost);
         let httpsURI = Services.io.newURI("https://" + aHost);
@@ -332,16 +343,62 @@ const QuotaCleaner = {
                                      .createCodebasePrincipal(httpURI, aOriginAttributes);
         let httpsPrincipal = Services.scriptSecurityManager
                                      .createCodebasePrincipal(httpsURI, aOriginAttributes);
-        return Promise.all([
-          new Promise(aResolve => {
-            let req = Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
-            req.callback = () => { aResolve(); };
-          }),
-          new Promise(aResolve => {
-            let req = Services.qms.clearStoragesForPrincipal(httpsPrincipal, null, null, true);
-            req.callback = () => { aResolve(); };
-          }),
-        ]).then(() => {
+        let promises = [];
+        promises.push(new Promise((aResolve, aReject) => {
+          let req = Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
+          req.callback = () => {
+            if (req.resultCode == Cr.NS_OK) {
+              aResolve();
+            } else {
+              aReject({message: "Delete by host failed"});
+            }
+          };
+        }));
+        promises.push(new Promise((aResolve, aReject) => {
+          let req = Services.qms.clearStoragesForPrincipal(httpsPrincipal, null, null, true);
+          req.callback = () => {
+            if (req.resultCode == Cr.NS_OK) {
+              aResolve();
+            } else {
+              aReject({message: "Delete by host failed"});
+            }
+          };
+        }));
+        if (Services.lsm.nextGenLocalStorageEnabled) {
+          // deleteByHost has the semantics that "foo.example.com" should be
+          // wiped if we are provided an aHost of "example.com".  QuotaManager
+          // doesn't have a way to directly do this, so we use getUsage() to
+          // get a list of all of the origins known to QuotaManager and then
+          // check whether the domain is a sub-domain of aHost.
+          promises.push(new Promise((aResolve, aReject) => {
+            Services.qms.getUsage(aRequest => {
+              if (aRequest.resultCode != Cr.NS_OK) {
+                aReject({message: "Delete by host failed"});
+                return;
+              }
+
+              let promises = [];
+              for (let item of aRequest.result) {
+                let principal = Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(item.origin);
+                if (eTLDService.hasRootDomain(principal.URI.host, aHost)) {
+                  promises.push(new Promise((aResolve, aReject) => {
+                    let clearRequest = Services.qms.clearStoragesForPrincipal(principal, null, "ls");
+                    clearRequest.callback = () => {
+                      if (clearRequest.resultCode == Cr.NS_OK) {
+                        aResolve();
+                      } else {
+                        aReject({message: "Delete by host failed"});
+                      }
+                    };
+                  }));
+                }
+              }
+
+              Promise.all(promises).then(aResolve);
+            });
+          }));
+        }
+        return Promise.all(promises).then(() => {
           return exceptionThrown ? Promise.reject() : Promise.resolve();
         });
       });
@@ -375,16 +432,12 @@ const QuotaCleaner = {
     return ServiceWorkerCleanUp.removeAll()
       .then(_ => /* exceptionThrown = */ false, _ => /* exceptionThrown = */ true)
       .then(exceptionThrown => {
-        // QuotaManager
+        // QuotaManager: In the event of a failure, we call reject to propagate
+        // the error upwards.
         return new Promise((aResolve, aReject) => {
           Services.qms.getUsage(aRequest => {
             if (aRequest.resultCode != Cr.NS_OK) {
-              // We are probably shutting down.
-              if (exceptionThrown) {
-                aReject();
-              } else {
-                aResolve();
-              }
+              aReject({message: "Delete all failed"});
               return;
             }
 
@@ -394,9 +447,15 @@ const QuotaCleaner = {
               if (principal.URI.scheme == "http" ||
                   principal.URI.scheme == "https" ||
                   principal.URI.scheme == "file") {
-                promises.push(new Promise(aResolve => {
+                promises.push(new Promise((aResolve, aReject) => {
                   let req = Services.qms.clearStoragesForPrincipal(principal);
-                  req.callback = () => { aResolve(); };
+                  req.callback = () => {
+                    if (req.resultCode == Cr.NS_OK) {
+                      aResolve();
+                    } else {
+                      aReject({message: "Delete all failed"});
+                    }
+                  };
                 }));
               }
             }
diff --git a/toolkit/components/extensions/Extension.jsm b/toolkit/components/extensions/Extension.jsm
index 39e819b1aef9..126e8fb0c0e4 100644
--- a/toolkit/components/extensions/Extension.jsm
+++ b/toolkit/components/extensions/Extension.jsm
@@ -244,6 +244,7 @@ var UninstallObserver = {
         ExtensionStorage.clear(addon.id, {shouldNotifyListeners: false}));
 
       // Clear any IndexedDB storage created by the extension
+      // If LSNG is enabled, this also clears localStorage.
       let baseURI = Services.io.newURI(`moz-extension://${uuid}/`);
       let principal = Services.scriptSecurityManager.createCodebasePrincipal(
         baseURI, {});
@@ -257,10 +258,14 @@ var UninstallObserver = {
 
       ExtensionStorageIDB.clearMigratedExtensionPref(addon.id);
 
-      // Clear localStorage created by the extension
-      let storage = Services.domStorageManager.getStorage(null, principal);
-      if (storage) {
-        storage.clear();
+      // If LSNG is not enabled, we need to clear localStorage explicitly using
+      // the old API.
+      if (!Services.lsm.nextGenLocalStorageEnabled) {
+        // Clear localStorage created by the extension
+        let storage = Services.domStorageManager.getStorage(null, principal);
+        if (storage) {
+          storage.clear();
+        }
       }
 
       // Remove any permissions related to the unlimitedStorage permission
diff --git a/toolkit/components/extensions/test/mochitest/mochitest-common.ini b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
index 03a1e08b3ca2..1f6d278a2c18 100644
--- a/toolkit/components/extensions/test/mochitest/mochitest-common.ini
+++ b/toolkit/components/extensions/test/mochitest/mochitest-common.ini
@@ -112,11 +112,11 @@ scheme=https
 [test_ext_subframes_privileges.html]
 skip-if = os == 'android' || verify # bug 1489771
 [test_ext_test.html]
-#[test_ext_unlimitedStorage.html]
-#[test_ext_unlimitedStorage_legacy_persistent_indexedDB.html]
+[test_ext_unlimitedStorage.html]
+[test_ext_unlimitedStorage_legacy_persistent_indexedDB.html]
 # IndexedDB persistent storage mode is not allowed on Fennec from a non-chrome privileged code
 # (it has only been enabled for apps and privileged code). See Bug 1119462 for additional info.
-#skip-if = os == 'android'
+skip-if = os == 'android'
 [test_ext_web_accessible_resources.html]
 skip-if = os == 'android' && debug # bug 1397615
 [test_ext_webnavigation.html]
diff --git a/toolkit/forgetaboutsite/moz.build b/toolkit/forgetaboutsite/moz.build
index c29be29b13b0..d18d10231432 100644
--- a/toolkit/forgetaboutsite/moz.build
+++ b/toolkit/forgetaboutsite/moz.build
@@ -5,7 +5,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 BROWSER_CHROME_MANIFESTS += ['test/browser/browser.ini']
-#XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
+XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']
 
 EXTRA_JS_MODULES += [
     'ForgetAboutSite.jsm',
diff --git a/toolkit/forgetaboutsite/test/unit/test_removeDataFromDomain.js b/toolkit/forgetaboutsite/test/unit/test_removeDataFromDomain.js
index 3927e4fe1d44..20f6ba7d8a9b 100644
--- a/toolkit/forgetaboutsite/test/unit/test_removeDataFromDomain.js
+++ b/toolkit/forgetaboutsite/test/unit/test_removeDataFromDomain.js
@@ -308,34 +308,13 @@ async function test_permission_manager_not_cleared_with_uri_contains_domain() {
   check_permission_exists(TEST_URI, false);
 }
 
-function waitForPurgeNotification() {
-  return new Promise(resolve => {
-
-    let observer = {
-      observe(aSubject, aTopic, aData) {
-        Services.obs.removeObserver(observer, "browser:purge-domain-data");
-        // test_storage_cleared needs this extra executeSoon because
-        // the DOMStorage clean-up is also listening to this same observer
-        // which is run synchronously.
-        Services.tm.dispatchToMainThread(function() {
-          resolve();
-        });
-      },
-    };
-    Services.obs.addObserver(observer, "browser:purge-domain-data");
-
-  });
-}
-
 // Content Preferences
 async function test_content_preferences_cleared_with_direct_match() {
   const TEST_URI = Services.io.newURI("http://mozilla.org");
   Assert.equal(false, await preference_exists(TEST_URI));
   await add_preference(TEST_URI);
   Assert.ok(await preference_exists(TEST_URI));
-  let promisePurgeNotification = waitForPurgeNotification();
   await ForgetAboutSite.removeDataFromDomain("mozilla.org");
-  await promisePurgeNotification;
   Assert.equal(false, await preference_exists(TEST_URI));
 }
 
@@ -344,9 +323,7 @@ async function test_content_preferences_cleared_with_subdomain() {
   Assert.equal(false, await preference_exists(TEST_URI));
   await add_preference(TEST_URI);
   Assert.ok(await preference_exists(TEST_URI));
-  let promisePurgeNotification = waitForPurgeNotification();
   await ForgetAboutSite.removeDataFromDomain("mozilla.org");
-  await promisePurgeNotification;
   Assert.equal(false, await preference_exists(TEST_URI));
 }
 
@@ -355,15 +332,11 @@ async function test_content_preferences_not_cleared_with_uri_contains_domain() {
   Assert.equal(false, await preference_exists(TEST_URI));
   await add_preference(TEST_URI);
   Assert.ok(await preference_exists(TEST_URI));
-  let promisePurgeNotification = waitForPurgeNotification();
   await ForgetAboutSite.removeDataFromDomain("mozilla.org");
-  await promisePurgeNotification;
   Assert.ok(await preference_exists(TEST_URI));
 
   // Reset state
-  promisePurgeNotification = waitForPurgeNotification();
   await ForgetAboutSite.removeDataFromDomain("ilovemozilla.org");
-  await promisePurgeNotification;
   Assert.equal(false, await preference_exists(TEST_URI));
 }
 
@@ -428,9 +401,7 @@ async function test_push_cleared() {
     });
     Assert.ok(await push_registration_exists(TEST_URL, ps));
 
-    let promisePurgeNotification = waitForPurgeNotification();
     await ForgetAboutSite.removeDataFromDomain("mozilla.org");
-    await promisePurgeNotification;
 
     Assert.equal(false, await push_registration_exists(TEST_URL, ps));
   } finally {
@@ -482,9 +453,7 @@ async function test_storage_cleared() {
     Assert.equal(storage.getItem("test"), "value" + i);
   }
 
-  let promisePurgeNotification = waitForPurgeNotification();
   await ForgetAboutSite.removeDataFromDomain("mozilla.org");
-  await promisePurgeNotification;
 
   Assert.equal(s[0].getItem("test"), null);
   Assert.equal(s[0].length, 0);
diff --git a/toolkit/forgetaboutsite/test/unit/xpcshell.ini b/toolkit/forgetaboutsite/test/unit/xpcshell.ini
index 6217234970de..5267de336e2b 100644
--- a/toolkit/forgetaboutsite/test/unit/xpcshell.ini
+++ b/toolkit/forgetaboutsite/test/unit/xpcshell.ini
@@ -4,4 +4,4 @@ skip-if = toolkit == 'android'
 support-files =
   !/dom/push/test/xpcshell/head.js
 
-#[test_removeDataFromDomain.js]
+[test_removeDataFromDomain.js]

From 36b0d5128d641c13b0edd201065333b3e798b45d Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:08 +0100
Subject: [PATCH 31/78] Bug 1286798 - Part 17: Fix a test failing in --verify
 mode; r=asuth

---
 ...owser_forgetAPI_quota_clearStoragesForPrincipal.js | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js b/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
index e557c2dd569c..46ae1cd4c091 100644
--- a/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
+++ b/browser/components/contextualidentity/test/browser/browser_forgetAPI_quota_clearStoragesForPrincipal.js
@@ -92,6 +92,10 @@ async function checkIndexedDB(browser) {
     } catch (e) {
       is(e.name, "NotFoundError", "The indexedDB does not exist as expected");
     }
+
+    db.close();
+
+    content.indexedDB.deleteDatabase("idb");
   });
 }
 
@@ -127,7 +131,12 @@ add_task(async function test_quota_clearStoragesForPrincipal() {
   let httpURI = caUtils.makeURI("http://" + TEST_HOST);
   let httpPrincipal = Services.scriptSecurityManager
                               .createCodebasePrincipal(httpURI, {});
-  Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
+  let clearRequest = Services.qms.clearStoragesForPrincipal(httpPrincipal, null, null, true);
+  await new Promise(resolve => {
+    clearRequest.callback = () => {
+      resolve();
+    };
+  });
 
   for (let userContextId of Object.keys(USER_CONTEXTS)) {
     // Open our tab in the given user context.

From b968ceb8137387dae0a19db0f93d8f8080d97fbf Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:11 +0100
Subject: [PATCH 32/78] Bug 1286798 - Part 18: Verify that data is persisted on
 disk; r=asuth,mrbkap

New methods open() and close() are added to the Storage WebIDL interface. They are only available when a pref is set and are only intended for testing. There's also a new method resetStoragesForPrincipal() which is used as a callback for close() since datastores don't release directory locks immediately. resetStoragesForPrincipal() requests an exclusive lock for given origin, so it must wait for any exising shared locks to be released.
---
 dom/localstorage/LSObject.cpp                 |  32 +++++
 dom/localstorage/LSObject.h                   |   8 ++
 dom/quota/ActorsChild.cpp                     |   1 +
 dom/quota/ActorsParent.cpp                    |  45 ++++--
 dom/quota/PQuota.ipdl                         |  15 +-
 dom/quota/PQuotaRequest.ipdl                  |   5 +
 dom/quota/QuotaManagerService.cpp             | 135 ++++++++++++++----
 dom/quota/nsIQuotaManagerService.idl          |  34 +++++
 dom/storage/Storage.h                         |   8 ++
 .../localstorage/localStorageCommon.js        | 100 +++++++++++--
 .../mochitest/localstorage/mochitest.ini      |   2 +-
 .../localstorage/test_bug600307-DBOps.html    |  18 ++-
 dom/webidl/Storage.webidl                     |   9 ++
 13 files changed, 347 insertions(+), 65 deletions(-)

diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 86a01bee561a..5cc79d1f0eaf 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -557,6 +557,38 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
   }
 }
 
+void
+LSObject::Open(nsIPrincipal& aSubjectPrincipal,
+               ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+}
+
+void
+LSObject::Close(nsIPrincipal& aSubjectPrincipal,
+                ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
+  DropDatabase();
+}
+
 NS_IMPL_ADDREF_INHERITED(LSObject, Storage)
 NS_IMPL_RELEASE_INHERITED(LSObject, Storage)
 
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index 789918f87cf2..3ed1d1867f51 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -140,6 +140,14 @@ public:
   Clear(nsIPrincipal& aSubjectPrincipal,
         ErrorResult& aError) override;
 
+  void
+  Open(nsIPrincipal& aSubjectPrincipal,
+       ErrorResult& aError) override;
+
+  void
+  Close(nsIPrincipal& aSubjectPrincipal,
+        ErrorResult& aError) override;
+
   NS_DECL_ISUPPORTS_INHERITED
   NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(LSObject, Storage)
 
diff --git a/dom/quota/ActorsChild.cpp b/dom/quota/ActorsChild.cpp
index 3f7911635685..ca09da17d7b2 100644
--- a/dom/quota/ActorsChild.cpp
+++ b/dom/quota/ActorsChild.cpp
@@ -318,6 +318,7 @@ QuotaRequestChild::Recv__delete__(const RequestResponse& aResponse)
     case RequestResponse::TInitResponse:
     case RequestResponse::TInitTemporaryStorageResponse:
     case RequestResponse::TClearOriginResponse:
+    case RequestResponse::TResetOriginResponse:
     case RequestResponse::TClearDataResponse:
     case RequestResponse::TClearAllResponse:
     case RequestResponse::TResetAllResponse:
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 1e7b047082a8..374c11de6149 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -1314,8 +1314,13 @@ class ClearRequestBase
   : public QuotaRequestBase
 {
 protected:
-  explicit ClearRequestBase(bool aExclusive)
+  const bool mClear;
+
+protected:
+  ClearRequestBase(bool aExclusive,
+                   bool aClear)
     : QuotaRequestBase(aExclusive)
+    , mClear(aClear)
   {
     AssertIsOnOwningThread();
   }
@@ -1331,7 +1336,7 @@ protected:
 class ClearOriginOp final
   : public ClearRequestBase
 {
-  const ClearOriginParams mParams;
+  const ClearResetOriginParams mParams;
 
 public:
   explicit ClearOriginOp(const RequestParams& aParams);
@@ -6815,6 +6820,7 @@ Quota::AllocPQuotaRequestParent(const RequestParams& aParams)
       break;
 
     case RequestParams::TClearOriginParams:
+    case RequestParams::TResetOriginParams:
       actor = new ClearOriginOp(aParams);
       break;
 
@@ -7651,7 +7657,6 @@ ClearRequestBase::DeleteFiles(QuotaManager* aQuotaManager,
   AssertIsOnIOThread();
   MOZ_ASSERT(aQuotaManager);
 
-
   nsCOMPtr<nsIFile> directory;
   nsresult rv = NS_NewLocalFile(aQuotaManager->GetStoragePath(aPersistenceType),
                                 false, getter_AddRefs(directory));
@@ -7815,22 +7820,29 @@ ClearRequestBase::DoDirectoryWork(QuotaManager* aQuotaManager)
 
   AUTO_PROFILER_LABEL("ClearRequestBase::DoDirectoryWork", OTHER);
 
-  if (mPersistenceType.IsNull()) {
-    for (const PersistenceType type : kAllPersistenceTypes) {
-      DeleteFiles(aQuotaManager, type);
+  if (mClear) {
+    if (mPersistenceType.IsNull()) {
+      for (const PersistenceType type : kAllPersistenceTypes) {
+        DeleteFiles(aQuotaManager, type);
+      }
+    } else {
+      DeleteFiles(aQuotaManager, mPersistenceType.Value());
     }
-  } else {
-    DeleteFiles(aQuotaManager, mPersistenceType.Value());
   }
 
   return NS_OK;
 }
 
 ClearOriginOp::ClearOriginOp(const RequestParams& aParams)
-  : ClearRequestBase(/* aExclusive */ true)
-  , mParams(aParams)
+  : ClearRequestBase(/* aExclusive */ true,
+                     aParams.type() == RequestParams::TClearOriginParams)
+  , mParams(aParams.type() == RequestParams::TClearOriginParams ?
+              aParams.get_ClearOriginParams().commonParams() :
+              aParams.get_ResetOriginParams().commonParams())
+
 {
-  MOZ_ASSERT(aParams.type() == RequestParams::TClearOriginParams);
+  MOZ_ASSERT(aParams.type() == RequestParams::TClearOriginParams ||
+             aParams.type() == RequestParams::TResetOriginParams);
 }
 
 bool
@@ -7884,7 +7896,7 @@ ClearOriginOp::DoInitOnMainThread()
     return rv;
   }
 
-  if (mParams.clearAll()) {
+  if (mParams.matchAll()) {
     mOriginScope.SetFromPrefix(origin);
   } else {
     mOriginScope.SetFromOrigin(origin);
@@ -7898,11 +7910,16 @@ ClearOriginOp::GetResponse(RequestResponse& aResponse)
 {
   AssertIsOnOwningThread();
 
-  aResponse = ClearOriginResponse();
+  if (mClear) {
+    aResponse = ClearOriginResponse();
+  } else {
+    aResponse = ResetOriginResponse();
+  }
 }
 
 ClearDataOp::ClearDataOp(const RequestParams& aParams)
-  : ClearRequestBase(/* aExclusive */ true)
+  : ClearRequestBase(/* aExclusive */ true,
+                     /* aClear */ true)
   , mParams(aParams)
 {
   MOZ_ASSERT(aParams.type() == RequestParams::TClearDataParams);
diff --git a/dom/quota/PQuota.ipdl b/dom/quota/PQuota.ipdl
index d0f9dfbc4a2c..61d5e3da6826 100644
--- a/dom/quota/PQuota.ipdl
+++ b/dom/quota/PQuota.ipdl
@@ -51,14 +51,24 @@ union UsageRequestParams
   OriginUsageParams;
 };
 
-struct ClearOriginParams
+struct ClearResetOriginParams
 {
   PrincipalInfo principalInfo;
   PersistenceType persistenceType;
   bool persistenceTypeIsExplicit;
   Type clientType;
   bool clientTypeIsExplicit;
-  bool clearAll;
+  bool matchAll;
+};
+
+struct ClearOriginParams
+{
+  ClearResetOriginParams commonParams;
+};
+
+struct ResetOriginParams
+{
+  ClearResetOriginParams commonParams;
 };
 
 struct ClearDataParams
@@ -90,6 +100,7 @@ union RequestParams
   InitTemporaryStorageParams;
   InitOriginParams;
   ClearOriginParams;
+  ResetOriginParams;
   ClearDataParams;
   ClearAllParams;
   ResetAllParams;
diff --git a/dom/quota/PQuotaRequest.ipdl b/dom/quota/PQuotaRequest.ipdl
index d9a0074bd8e7..9eae731fd9fc 100644
--- a/dom/quota/PQuotaRequest.ipdl
+++ b/dom/quota/PQuotaRequest.ipdl
@@ -25,6 +25,10 @@ struct ClearOriginResponse
 {
 };
 
+struct ResetOriginResponse
+{
+};
+
 struct ClearDataResponse
 {
 };
@@ -53,6 +57,7 @@ union RequestResponse
   InitTemporaryStorageResponse;
   InitOriginResponse;
   ClearOriginResponse;
+  ResetOriginResponse;
   ClearDataResponse;
   ClearAllResponse;
   ResetAllResponse;
diff --git a/dom/quota/QuotaManagerService.cpp b/dom/quota/QuotaManagerService.cpp
index db52c4a8f103..66f52c5ca245 100644
--- a/dom/quota/QuotaManagerService.cpp
+++ b/dom/quota/QuotaManagerService.cpp
@@ -78,6 +78,53 @@ CheckedPrincipalToPrincipalInfo(nsIPrincipal* aPrincipal,
   return NS_OK;
 }
 
+nsresult
+GetClearResetOriginParams(nsIPrincipal* aPrincipal,
+                          const nsACString& aPersistenceType,
+                          const nsAString& aClientType,
+                          bool aMatchAll,
+                          ClearResetOriginParams& aParams)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+
+  nsresult rv = CheckedPrincipalToPrincipalInfo(aPrincipal,
+                                                aParams.principalInfo());
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  Nullable<PersistenceType> persistenceType;
+  rv = NullablePersistenceTypeFromText(aPersistenceType, &persistenceType);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  if (persistenceType.IsNull()) {
+    aParams.persistenceTypeIsExplicit() = false;
+  } else {
+    aParams.persistenceType() = persistenceType.Value();
+    aParams.persistenceTypeIsExplicit() = true;
+  }
+
+  Nullable<Client::Type> clientType;
+  rv = Client::NullableTypeFromText(aClientType, &clientType);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  if (clientType.IsNull()) {
+    aParams.clientTypeIsExplicit() = false;
+  } else {
+    aParams.clientType() = clientType.Value();
+    aParams.clientTypeIsExplicit() = true;
+  }
+
+  aParams.matchAll() = aMatchAll;
+
+  return NS_OK;
+}
+
 class AbortOperationsRunnable final
   : public Runnable
 {
@@ -628,41 +675,19 @@ QuotaManagerService::ClearStoragesForPrincipal(nsIPrincipal* aPrincipal,
 
   RefPtr<Request> request = new Request(aPrincipal);
 
-  ClearOriginParams params;
+  ClearResetOriginParams commonParams;
 
-  nsresult rv = CheckedPrincipalToPrincipalInfo(aPrincipal,
-                                                params.principalInfo());
+  nsresult rv = GetClearResetOriginParams(aPrincipal,
+                                          aPersistenceType,
+                                          aClientType,
+                                          aClearAll,
+                                          commonParams);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  Nullable<PersistenceType> persistenceType;
-  rv = NullablePersistenceTypeFromText(aPersistenceType, &persistenceType);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return NS_ERROR_INVALID_ARG;
-  }
-
-  if (persistenceType.IsNull()) {
-    params.persistenceTypeIsExplicit() = false;
-  } else {
-    params.persistenceType() = persistenceType.Value();
-    params.persistenceTypeIsExplicit() = true;
-  }
-
-  Nullable<Client::Type> clientType;
-  rv = Client::NullableTypeFromText(aClientType, &clientType);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return NS_ERROR_INVALID_ARG;
-  }
-
-  if (clientType.IsNull()) {
-    params.clientTypeIsExplicit() = false;
-  } else {
-    params.clientType() = clientType.Value();
-    params.clientTypeIsExplicit() = true;
-  }
-
-  params.clearAll() = aClearAll;
+  RequestParams params;
+  params = ClearOriginParams(commonParams);
 
   nsAutoPtr<PendingRequestInfo> info(new RequestInfo(request, params));
 
@@ -699,6 +724,56 @@ QuotaManagerService::Reset(nsIQuotaRequest** _retval)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+QuotaManagerService::ResetStoragesForPrincipal(nsIPrincipal* aPrincipal,
+                                               const nsACString& aPersistenceType,
+                                               const nsAString& aClientType,
+                                               bool aResetAll,
+                                               nsIQuotaRequest** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+
+  if (NS_WARN_IF(!gTestingMode)) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  nsCString suffix;
+  aPrincipal->OriginAttributesRef().CreateSuffix(suffix);
+
+  if (NS_WARN_IF(aResetAll && !suffix.IsEmpty())) {
+    // The originAttributes should be default originAttributes when the
+    // aClearAll flag is set.
+    return NS_ERROR_INVALID_ARG;
+  }
+
+  RefPtr<Request> request = new Request(aPrincipal);
+
+  ClearResetOriginParams commonParams;
+
+  nsresult rv = GetClearResetOriginParams(aPrincipal,
+                                          aPersistenceType,
+                                          aClientType,
+                                          aResetAll,
+                                          commonParams);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  RequestParams params;
+  params = ResetOriginParams(commonParams);
+
+  nsAutoPtr<PendingRequestInfo> info(new RequestInfo(request, params));
+
+  rv = InitiateRequest(info);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  request.forget(_retval);
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 QuotaManagerService::Persisted(nsIPrincipal* aPrincipal,
                                nsIQuotaRequest** _retval)
diff --git a/dom/quota/nsIQuotaManagerService.idl b/dom/quota/nsIQuotaManagerService.idl
index 3de0f0a76dc9..b2098384e66f 100644
--- a/dom/quota/nsIQuotaManagerService.idl
+++ b/dom/quota/nsIQuotaManagerService.idl
@@ -140,6 +140,40 @@ interface nsIQuotaManagerService : nsISupports
   [must_use] nsIQuotaRequest
   reset();
 
+  /**
+   * Resets all storages stored for the given principal.
+   *
+   * If the dom.quotaManager.testing preference is not true the call will be
+   * a no-op.
+   *
+   * @param aPrincipal
+   *        A principal for the origin whose storages are to be reset.
+   * @param aPersistenceType
+   *        An optional string that tells what persistence type of storages
+   *        will be reset.  If omitted (or void), all persistence types will
+   *        be cleared for the principal.  If a single persistence type
+   *        ("persistent", "temporary", or "default") is provided, then only
+   *        that persistence directory will be considered.  Note that
+   *        "persistent" is different than being "persisted" via persist() and
+   *        is only for chrome principals.  See bug 1354500 for more info.
+   *        In general, null is the right thing to pass here.
+   * @param aClientType
+   *        An optional string that tells what client type of storages
+   *        will be reset.  If omitted (or void), all client types will be
+   *        cleared for the principal.  If a single client type is provided
+   *        from Client.h, then only that client's storage will be cleared.
+   *        If you want to clear multiple client types (but not all), then you
+   *        must call this method multiple times.
+   * @param aResetAll
+   *        An optional boolean to indicate resetting all storages under the
+   *        given origin.
+   */
+  [must_use] nsIQuotaRequest
+  resetStoragesForPrincipal(in nsIPrincipal aPrincipal,
+                            [optional] in ACString aPersistenceType,
+                            [optional] in AString aClientType,
+                            [optional] in boolean aResetAll);
+
   /**
    * Check if given origin is persisted.
    *
diff --git a/dom/storage/Storage.h b/dom/storage/Storage.h
index b4a9cf4cbf8a..00e470d5d30a 100644
--- a/dom/storage/Storage.h
+++ b/dom/storage/Storage.h
@@ -110,6 +110,14 @@ public:
 
   bool IsSessionOnly() const { return mIsSessionOnly; }
 
+  virtual void
+  Open(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
+  { }
+
+  virtual void
+  Close(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
+  { }
+
   // aStorage can be null if this method is called by LocalStorageCacheChild.
   //
   // aImmediateDispatch is for use by child IPC code (LocalStorageCacheChild)
diff --git a/dom/tests/mochitest/localstorage/localStorageCommon.js b/dom/tests/mochitest/localstorage/localStorageCommon.js
index ecbc80f309a5..4bb00c197844 100644
--- a/dom/tests/mochitest/localstorage/localStorageCommon.js
+++ b/dom/tests/mochitest/localstorage/localStorageCommon.js
@@ -1,5 +1,12 @@
 function localStorageFlush(cb)
 {
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    SimpleTest.executeSoon(function () {
+      cb();
+    });
+    return;
+  }
+
   var ob = {
     observe : function(sub, top, dat)
     {
@@ -11,27 +18,94 @@ function localStorageFlush(cb)
   notify("domstorage-test-flush-force");
 }
 
-function localStorageReload()
+function localStorageReload(callback)
 {
-  notify("domstorage-test-reload");
-}
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    localStorage.close();
+    let qms = SpecialPowers.Services.qms;
+    let principal = SpecialPowers.wrap(document).nodePrincipal;
+    let request = qms.resetStoragesForPrincipal(principal, "default", "ls");
+    request.callback = SpecialPowers.wrapCallback(function() {
+      localStorage.open();
+      callback();
+    });
+    return;
+  }
 
-function localStorageFlushAndReload(cb)
-{
-  localStorageFlush(function() {
-    localStorageReload();
-    cb();
+  notify("domstorage-test-reload");
+  SimpleTest.executeSoon(function () {
+    callback();
   });
 }
 
-function localStorageClearAll()
+function localStorageFlushAndReload(callback)
 {
-  os().notifyObservers(null, "cookie-changed", "cleared");
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    localStorage.close();
+    let qms = SpecialPowers.Services.qms;
+    let principal = SpecialPowers.wrap(document).nodePrincipal;
+    let request = qms.resetStoragesForPrincipal(principal, "default", "ls");
+    request.callback = SpecialPowers.wrapCallback(function() {
+      localStorage.open();
+      callback();
+    });
+    return;
+  }
+
+  localStorageFlush(function() {
+    localStorageReload(callback);
+  });
 }
 
-function localStorageClearDomain(domain)
+function localStorageClearAll(callback)
 {
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    let qms = SpecialPowers.Services.qms;
+    let ssm = SpecialPowers.Services.scriptSecurityManager;
+
+    qms.getUsage(SpecialPowers.wrapCallback(function(request) {
+      if (request.resultCode != SpecialPowers.Cr.NS_OK) {
+        callback();
+        return;
+      }
+
+      let clearRequestCount = 0;
+      for (let item of request.result) {
+        let principal = ssm.createCodebasePrincipalFromOrigin(item.origin);
+        let clearRequest =
+          qms.clearStoragesForPrincipal(principal, "default", "ls");
+        clearRequestCount++;
+        clearRequest.callback = SpecialPowers.wrapCallback(function() {
+          if (--clearRequestCount == 0) {
+            callback();
+          }
+        });
+      }
+    }));
+    return;
+  }
+
+  os().notifyObservers(null, "cookie-changed", "cleared");
+  SimpleTest.executeSoon(function () {
+    callback();
+  });
+}
+
+function localStorageClearDomain(domain, callback)
+{
+  if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+    let qms = SpecialPowers.Services.qms;
+    let principal = SpecialPowers.wrap(document).nodePrincipal;
+    let request = qms.clearStoragesForPrincipal(principal, "default", "ls");
+    let cb = SpecialPowers.wrapCallback(callback);
+    request.callback = cb;
+    return;
+  }
+
   os().notifyObservers(null, "browser:purge-domain-data", domain);
+  SimpleTest.executeSoon(function () {
+    callback();
+  });
 }
 
 function os()
@@ -49,5 +123,7 @@ function notify(top)
  */
 function localStorageEnableTestingMode(cb)
 {
-  SpecialPowers.pushPrefEnv({ "set": [["dom.storage.testing", true]] }, cb);
+  SpecialPowers.pushPrefEnv({ set: [["dom.storage.testing", true],
+                                    ["dom.quotaManager.testing", true]] },
+                            cb);
 }
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index 17e57beb61ed..8de589e7dfb7 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -19,7 +19,7 @@ support-files =
   file_tryAccessSessionStorage.html
 
 [test_brokenUTF-16.html]
-#[test_bug600307-DBOps.html]
+[test_bug600307-DBOps.html]
 [test_bug746272-1.html]
 [test_bug746272-2.html]
 skip-if = os == "android" || verify # bug 962029
diff --git a/dom/tests/mochitest/localstorage/test_bug600307-DBOps.html b/dom/tests/mochitest/localstorage/test_bug600307-DBOps.html
index dd1a7c2b5d07..8c3ae3f79301 100644
--- a/dom/tests/mochitest/localstorage/test_bug600307-DBOps.html
+++ b/dom/tests/mochitest/localstorage/test_bug600307-DBOps.html
@@ -84,10 +84,11 @@ function startTest()
   localStorage.setItem("item", "value");
   localStorage.setItem("item2", "value2");
   localStorage.clear();
-  localStorageReload();
+  localStorageReload(function() {
   is(localStorage.length, 0, "localStorage clean in case 4");
 
-  if (SpecialPowers.Cc["@mozilla.org/xre/app-info;1"].getService(
+  if (!SpecialPowers.Services.lsm.nextGenLocalStorageEnabled &&
+      SpecialPowers.Cc["@mozilla.org/xre/app-info;1"].getService(
       SpecialPowers.Ci.nsIXULRuntime).processType != SpecialPowers.Ci.nsIXULRuntime.PROCESS_TYPE_DEFAULT) {
     // Following tests cannot be run in a child/plugin process type
     SimpleTest.finish();
@@ -97,7 +98,7 @@ function startTest()
   // Cookies clean 1
   localStorageFlush(function() {
   localStorage.setItem("item", "value");
-  localStorageClearAll();
+  localStorageClearAll(function() {
   is(localStorage.length, 0, "localStorage clean after cookies deletion");
   localStorage.setItem("item2", "value2");
   is(localStorage.getItem("item"), null, "Unexpected key 1, cookies delete");
@@ -110,7 +111,7 @@ function startTest()
   localStorage.clear();
   localStorageFlush(function() {
   localStorage.setItem("item", "value");
-  localStorageClearAll();
+  localStorageClearAll(function() {
   is(localStorage.length, 0, "localStorage clean after cookies deletion 2");
   localStorageFlushAndReload(function() {
   is(localStorage.length, 0, "localStorage clean after cookies deletion 2");
@@ -119,7 +120,7 @@ function startTest()
   // Domain clean 1
   localStorageFlush(function() {
   localStorage.setItem("item", "value");
-  localStorageClearDomain("test");
+  localStorageClearDomain("test", function() {
   is(localStorage.length, 0, "localStorage clean after domain deletion");
   localStorage.setItem("item2", "value2");
   is(localStorage.getItem("item"), null, "Unexpected key 1, domain delete");
@@ -132,7 +133,7 @@ function startTest()
   localStorage.clear();
   localStorageFlush(function() {
   localStorage.setItem("item", "value");
-  localStorageClearDomain("test");
+  localStorageClearDomain("test", function() {
   is(localStorage.length, 0, "localStorage clean after domain deletion 2");
   localStorageFlushAndReload(function() {
   is(localStorage.length, 0, "localStorage clean after domain deletion 2");
@@ -157,6 +158,11 @@ function startTest()
   });
   });
   });
+  });
+  });
+  });
+  });
+  });
 }
 
 SimpleTest.waitForExplicitFinish();
diff --git a/dom/webidl/Storage.webidl b/dom/webidl/Storage.webidl
index b64974a287b2..dd5a31ba287f 100644
--- a/dom/webidl/Storage.webidl
+++ b/dom/webidl/Storage.webidl
@@ -33,3 +33,12 @@ interface Storage {
   [ChromeOnly]
   readonly attribute boolean isSessionOnly;
 };
+
+// Testing only.
+partial interface Storage {
+  [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
+  void open();
+
+  [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
+  void close();
+};

From 81c542fbe62c21ee5873f6858ebfef54f8935742 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:15 +0100
Subject: [PATCH 33/78] Bug 1286798 - Part 19: Implement a helper method for
 datastore preloading verification; r=asuth

A new type of request is introduced, PBackgroundLSSimpleRequest. This protocol is much simpler than PBackgroundLSRequest which needs to be cancelable.
---
 dom/localstorage/ActorsChild.cpp              |  37 +++
 dom/localstorage/ActorsChild.h                |  47 +++
 dom/localstorage/ActorsParent.cpp             | 297 ++++++++++++++++++
 dom/localstorage/ActorsParent.h               |  14 +
 dom/localstorage/LocalStorageManager2.cpp     | 168 ++++++++++
 dom/localstorage/LocalStorageManager2.h       |   7 +
 .../PBackgroundLSSharedTypes.ipdlh            |  10 +
 .../PBackgroundLSSimpleRequest.ipdl           |  30 ++
 dom/localstorage/moz.build                    |   1 +
 dom/localstorage/nsILocalStorageManager.idl   |   3 +
 dom/storage/LocalStorageManager.cpp           |  12 +
 dom/tests/browser/browser.ini                 |   4 +-
 .../browser/browser_localStorage_e10s.js      |  23 ++
 ipc/glue/BackgroundChildImpl.cpp              |  19 ++
 ipc/glue/BackgroundChildImpl.h                |   9 +
 ipc/glue/BackgroundParentImpl.cpp             |  37 +++
 ipc/glue/BackgroundParentImpl.h               |  15 +
 ipc/glue/PBackground.ipdl                     |   4 +
 18 files changed, 735 insertions(+), 2 deletions(-)
 create mode 100644 dom/localstorage/PBackgroundLSSimpleRequest.ipdl

diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 3ae09fba998d..c0e7a68c331a 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -277,5 +277,42 @@ LSRequestChild::RecvReady()
   return IPC_OK();
 }
 
+/*******************************************************************************
+ * LSSimpleRequestChild
+ ******************************************************************************/
+
+LSSimpleRequestChild::LSSimpleRequestChild(
+                                        LSSimpleRequestChildCallback* aCallback)
+  : mCallback(aCallback)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aCallback);
+
+  MOZ_COUNT_CTOR(LSSimpleRequestChild);
+}
+
+LSSimpleRequestChild::~LSSimpleRequestChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSSimpleRequestChild);
+}
+
+void
+LSSimpleRequestChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+}
+
+mozilla::ipc::IPCResult
+LSSimpleRequestChild::Recv__delete__(const LSSimpleRequestResponse& aResponse)
+{
+  AssertIsOnOwningThread();
+
+  mCallback->OnResponse(aResponse);
+
+  return IPC_OK();
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 43cbe8a57366..10b5881b93f0 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -11,6 +11,7 @@
 #include "mozilla/dom/PBackgroundLSObjectChild.h"
 #include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
+#include "mozilla/dom/PBackgroundLSSimpleRequestChild.h"
 
 namespace mozilla {
 
@@ -22,10 +23,12 @@ class BackgroundChildImpl;
 
 namespace dom {
 
+class LocalStorageManager2;
 class LSDatabase;
 class LSObject;
 class LSObserver;
 class LSRequestChildCallback;
+class LSSimpleRequestChildCallback;
 
 class LSObjectChild final
   : public PBackgroundLSObjectChild
@@ -196,6 +199,50 @@ protected:
   { }
 };
 
+class LSSimpleRequestChild final
+  : public PBackgroundLSSimpleRequestChild
+{
+  friend class LocalStorageManager2;
+
+  RefPtr<LSSimpleRequestChildCallback> mCallback;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSSimpleReqeustChild);
+  }
+
+private:
+  // Only created by LocalStorageManager2.
+  explicit LSSimpleRequestChild(LSSimpleRequestChildCallback* aCallback);
+
+  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
+  ~LSSimpleRequestChild();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  Recv__delete__(const LSSimpleRequestResponse& aResponse) override;
+};
+
+class NS_NO_VTABLE LSSimpleRequestChildCallback
+{
+public:
+  NS_INLINE_DECL_PURE_VIRTUAL_REFCOUNTING
+
+  virtual void
+  OnResponse(const LSSimpleRequestResponse& aResponse) = 0;
+
+protected:
+  virtual ~LSSimpleRequestChildCallback()
+  { }
+};
+
 } // namespace dom
 } // namespace mozilla
 
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 595a9a621a4a..4ce59a31aede 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -18,6 +18,7 @@
 #include "mozilla/dom/PBackgroundLSObserverParent.h"
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
+#include "mozilla/dom/PBackgroundLSSimpleRequestParent.h"
 #include "mozilla/dom/quota/QuotaManager.h"
 #include "mozilla/dom/quota/UsageInfo.h"
 #include "mozilla/ipc/BackgroundParent.h"
@@ -1494,6 +1495,74 @@ private:
   GetResponse(LSRequestResponse& aResponse) override;
 };
 
+class LSSimpleRequestBase
+  : public DatastoreOperationBase
+  , public PBackgroundLSSimpleRequestParent
+{
+protected:
+  enum class State
+  {
+    // Just created on the PBackground thread. Next step is Opening.
+    Initial,
+
+    // Waiting to open/opening on the main thread. Next step is SendingResults.
+    Opening,
+
+    // Waiting to send/sending results on the PBackground thread. Next step is
+    // Completed.
+    SendingResults,
+
+    // All done.
+    Completed
+  };
+
+  State mState;
+
+public:
+  LSSimpleRequestBase();
+
+  void
+  Dispatch();
+
+protected:
+  ~LSSimpleRequestBase() override;
+
+  virtual nsresult
+  Open() = 0;
+
+  virtual void
+  GetResponse(LSSimpleRequestResponse& aResponse) = 0;
+
+private:
+  void
+  SendResults();
+
+  // Common nsIRunnable implementation that subclasses may not override.
+  NS_IMETHOD
+  Run() final;
+
+  // IPDL methods.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+};
+
+class PreloadedOp
+  : public LSSimpleRequestBase
+{
+  const LSSimpleRequestPreloadedParams mParams;
+  nsCString mOrigin;
+
+public:
+  explicit PreloadedOp(const LSSimpleRequestParams& aParams);
+
+private:
+  nsresult
+  Open() override;
+
+  void
+  GetResponse(LSSimpleRequestResponse& aResponse) override;
+};
+
 /*******************************************************************************
  * Other class declarations
  ******************************************************************************/
@@ -1877,6 +1946,67 @@ DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor)
   return true;
 }
 
+PBackgroundLSSimpleRequestParent*
+AllocPBackgroundLSSimpleRequestParent(const LSSimpleRequestParams& aParams)
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
+    return nullptr;
+  }
+
+  RefPtr<LSSimpleRequestBase> actor;
+
+  switch (aParams.type()) {
+    case LSSimpleRequestParams::TLSSimpleRequestPreloadedParams: {
+      RefPtr<PreloadedOp> preloadedOp =
+        new PreloadedOp(aParams);
+
+      actor = std::move(preloadedOp);
+
+      break;
+    }
+
+    default:
+      MOZ_CRASH("Should never get here!");
+  }
+
+  // Transfer ownership to IPDL.
+  return actor.forget().take();
+}
+
+bool
+RecvPBackgroundLSSimpleRequestConstructor(
+                                       PBackgroundLSSimpleRequestParent* aActor,
+                                       const LSSimpleRequestParams& aParams)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(aParams.type() != LSSimpleRequestParams::T__None);
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+
+  // The actor is now completely built.
+
+  auto* op = static_cast<LSSimpleRequestBase*>(aActor);
+
+  op->Dispatch();
+
+  return true;
+}
+
+bool
+DeallocPBackgroundLSSimpleRequestParent(
+                                       PBackgroundLSSimpleRequestParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+
+  // Transfer ownership back from IPDL.
+  RefPtr<LSSimpleRequestBase> actor =
+    dont_AddRef(static_cast<LSSimpleRequestBase*>(aActor));
+
+  return true;
+}
+
 namespace localstorage {
 
 already_AddRefed<mozilla::dom::quota::Client>
@@ -4227,6 +4357,173 @@ PrepareObserverOp::GetResponse(LSRequestResponse& aResponse)
   aResponse = prepareObserverResponse;
 }
 
+/*******************************************************************************
++ * LSSimpleRequestBase
++ ******************************************************************************/
+
+LSSimpleRequestBase::LSSimpleRequestBase()
+  : mState(State::Initial)
+{
+}
+
+LSSimpleRequestBase::~LSSimpleRequestBase()
+{
+  MOZ_ASSERT_IF(MayProceedOnNonOwningThread(),
+                mState == State::Initial || mState == State::Completed);
+}
+
+void
+LSSimpleRequestBase::Dispatch()
+{
+  AssertIsOnOwningThread();
+
+  mState = State::Opening;
+
+  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+}
+
+void
+LSSimpleRequestBase::SendResults()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingResults);
+
+  if (!MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
+  } else {
+    LSSimpleRequestResponse response;
+
+    if (NS_SUCCEEDED(ResultCode())) {
+      GetResponse(response);
+    } else {
+      response = ResultCode();
+    }
+
+    Unused <<
+      PBackgroundLSSimpleRequestParent::Send__delete__(this, response);
+  }
+
+  mState = State::Completed;
+}
+
+NS_IMETHODIMP
+LSSimpleRequestBase::Run()
+{
+  nsresult rv;
+
+  switch (mState) {
+    case State::Opening:
+      rv = Open();
+      break;
+
+    case State::SendingResults:
+      SendResults();
+      return NS_OK;
+
+    default:
+      MOZ_CRASH("Bad state!");
+  }
+
+  if (NS_WARN_IF(NS_FAILED(rv)) && mState != State::SendingResults) {
+    MaybeSetFailureCode(rv);
+
+    // Must set mState before dispatching otherwise we will race with the owning
+    // thread.
+    mState = State::SendingResults;
+
+    if (IsOnOwningThread()) {
+      SendResults();
+    } else {
+      MOZ_ALWAYS_SUCCEEDS(
+        OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+    }
+  }
+
+  return NS_OK;
+}
+
+void
+LSSimpleRequestBase::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  NoteComplete();
+}
+
+/*******************************************************************************
+ * PreloadedOp
+ ******************************************************************************/
+
+PreloadedOp::PreloadedOp(const LSSimpleRequestParams& aParams)
+  : mParams(aParams.get_LSSimpleRequestPreloadedParams())
+{
+  MOZ_ASSERT(aParams.type() ==
+               LSSimpleRequestParams::TLSSimpleRequestPreloadedParams);
+}
+
+nsresult
+PreloadedOp::Open()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mState == State::Opening);
+
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
+      !MayProceedOnNonOwningThread()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  const PrincipalInfo& principalInfo = mParams.principalInfo();
+
+  if (principalInfo.type() == PrincipalInfo::TSystemPrincipalInfo) {
+    QuotaManager::GetInfoForChrome(nullptr, nullptr, &mOrigin);
+  } else {
+    MOZ_ASSERT(principalInfo.type() == PrincipalInfo::TContentPrincipalInfo);
+
+    nsresult rv;
+    nsCOMPtr<nsIPrincipal> principal =
+      PrincipalInfoToPrincipal(principalInfo, &rv);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = QuotaManager::GetInfoFromPrincipal(principal,
+                                            nullptr,
+                                            nullptr,
+                                            &mOrigin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  mState = State::SendingResults;
+  MOZ_ALWAYS_SUCCEEDS(OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL));
+
+  return NS_OK;
+}
+
+void
+PreloadedOp::GetResponse(LSSimpleRequestResponse& aResponse)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::SendingResults);
+  MOZ_ASSERT(NS_SUCCEEDED(ResultCode()));
+
+  bool preloaded;
+  RefPtr<Datastore> datastore;
+  if (gDatastores &&
+      (datastore = gDatastores->Get(mOrigin)) &&
+      !datastore->IsClosed()) {
+    preloaded = true;
+  } else {
+    preloaded = false;
+  }
+
+  LSSimpleRequestPreloadedResponse preloadedResponse;
+  preloadedResponse.preloaded() = preloaded;
+
+  aResponse = preloadedResponse;
+}
+
 /*******************************************************************************
  * QuotaClient
  ******************************************************************************/
diff --git a/dom/localstorage/ActorsParent.h b/dom/localstorage/ActorsParent.h
index ee7396ff1331..006ba0efaa01 100644
--- a/dom/localstorage/ActorsParent.h
+++ b/dom/localstorage/ActorsParent.h
@@ -19,9 +19,11 @@ class PrincipalInfo;
 namespace dom {
 
 class LSRequestParams;
+class LSSimpleRequestParams;
 class PBackgroundLSObjectParent;
 class PBackgroundLSObserverParent;
 class PBackgroundLSRequestParent;
+class PBackgroundLSSimpleRequestParent;
 
 namespace quota {
 
@@ -67,6 +69,18 @@ RecvPBackgroundLSRequestConstructor(PBackgroundLSRequestParent* aActor,
 bool
 DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor);
 
+PBackgroundLSSimpleRequestParent*
+AllocPBackgroundLSSimpleRequestParent(const LSSimpleRequestParams& aParams);
+
+bool
+RecvPBackgroundLSSimpleRequestConstructor(
+                                       PBackgroundLSSimpleRequestParent* aActor,
+                                       const LSSimpleRequestParams& aParams);
+
+bool
+DeallocPBackgroundLSSimpleRequestParent(
+                                      PBackgroundLSSimpleRequestParent* aActor);
+
 namespace localstorage {
 
 already_AddRefed<mozilla::dom::quota::Client>
diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
index b1fe2444c204..a9e2f18cd651 100644
--- a/dom/localstorage/LocalStorageManager2.cpp
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -7,10 +7,83 @@
 #include "LocalStorageManager2.h"
 
 #include "LSObject.h"
+#include "mozilla/dom/Promise.h"
 
 namespace mozilla {
 namespace dom {
 
+namespace {
+
+class SimpleRequestResolver final
+  : public LSSimpleRequestChildCallback
+{
+  RefPtr<Promise> mPromise;
+
+public:
+  explicit SimpleRequestResolver(Promise* aPromise)
+    : mPromise(aPromise)
+  { }
+
+  NS_INLINE_DECL_REFCOUNTING(SimpleRequestResolver, override);
+
+private:
+  ~SimpleRequestResolver() = default;
+
+  void
+  HandleResponse(nsresult aResponse);
+
+  void
+  HandleResponse(bool aResponse);
+
+  // LSRequestChildCallback
+  void
+  OnResponse(const LSSimpleRequestResponse& aResponse) override;
+};
+
+nsresult
+CreatePromise(JSContext* aContext, Promise** aPromise)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aContext);
+
+  nsIGlobalObject* global =
+    xpc::NativeGlobal(JS::CurrentGlobalOrNull(aContext));
+  if (NS_WARN_IF(!global)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  ErrorResult result;
+  RefPtr<Promise> promise = Promise::Create(global, result);
+  if (result.Failed()) {
+    return result.StealNSResult();
+  }
+
+  promise.forget(aPromise);
+  return NS_OK;
+}
+
+nsresult
+CheckedPrincipalToPrincipalInfo(nsIPrincipal* aPrincipal,
+                                PrincipalInfo& aPrincipalInfo)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+
+  nsresult rv = PrincipalToPrincipalInfo(aPrincipal, &aPrincipalInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (aPrincipalInfo.type() != PrincipalInfo::TContentPrincipalInfo &&
+      aPrincipalInfo.type() != PrincipalInfo::TSystemPrincipalInfo) {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  return NS_OK;
+}
+
+} // namespace
+
 LocalStorageManager2::LocalStorageManager2()
 {
   MOZ_ASSERT(NS_IsMainThread());
@@ -109,5 +182,100 @@ LocalStorageManager2::GetNextGenLocalStorageEnabled(bool* aResult)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocalStorageManager2::IsPreloaded(nsIPrincipal* aPrincipal,
+                                  JSContext* aContext,
+                                  nsISupports** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  RefPtr<Promise> promise;
+  nsresult rv = CreatePromise(aContext, getter_AddRefs(promise));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  LSSimpleRequestPreloadedParams params;
+
+  rv = CheckedPrincipalToPrincipalInfo(aPrincipal,
+                                       params.principalInfo());
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StartSimpleRequest(promise, params);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  promise.forget(_retval);
+  return NS_OK;
+}
+
+nsresult
+LocalStorageManager2::StartSimpleRequest(Promise* aPromise,
+                                         const LSSimpleRequestParams& aParams)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPromise);
+
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread();
+  if (NS_WARN_IF(!backgroundActor)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  RefPtr<SimpleRequestResolver> resolver = new SimpleRequestResolver(aPromise);
+
+  auto actor = new LSSimpleRequestChild(resolver);
+
+  if (!backgroundActor->SendPBackgroundLSSimpleRequestConstructor(actor,
+                                                                  aParams)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
+void
+SimpleRequestResolver::HandleResponse(nsresult aResponse)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mPromise);
+
+  mPromise->MaybeReject(aResponse);
+}
+
+void
+SimpleRequestResolver::HandleResponse(bool aResponse)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(mPromise);
+
+  mPromise->MaybeResolve(aResponse);
+}
+
+void
+SimpleRequestResolver::OnResponse(const LSSimpleRequestResponse& aResponse)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  switch (aResponse.type()) {
+    case LSSimpleRequestResponse::Tnsresult:
+      HandleResponse(aResponse.get_nsresult());
+      break;
+
+    case LSSimpleRequestResponse::TLSSimpleRequestPreloadedResponse:
+      HandleResponse(
+        aResponse.get_LSSimpleRequestPreloadedResponse().preloaded());
+      break;
+
+   default:
+      MOZ_CRASH("Unknown response type!");
+  }
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/LocalStorageManager2.h b/dom/localstorage/LocalStorageManager2.h
index c64c884fd6e2..3349b93fa8b4 100644
--- a/dom/localstorage/LocalStorageManager2.h
+++ b/dom/localstorage/LocalStorageManager2.h
@@ -13,6 +13,9 @@
 namespace mozilla {
 namespace dom {
 
+class LSSimpleRequestParams;
+class Promise;
+
 class LocalStorageManager2 final
   : public nsIDOMStorageManager
   , public nsILocalStorageManager
@@ -26,6 +29,10 @@ public:
 
 private:
   ~LocalStorageManager2();
+
+  nsresult
+  StartSimpleRequest(Promise* aPromise,
+                     const LSSimpleRequestParams& aParams);
 };
 
 } // namespace dom
diff --git a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
index b7cf7d5acc6b..75bdc3f56d5a 100644
--- a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
+++ b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
@@ -23,5 +23,15 @@ union LSRequestParams
   LSRequestPrepareObserverParams;
 };
 
+struct LSSimpleRequestPreloadedParams
+{
+  PrincipalInfo principalInfo;
+};
+
+union LSSimpleRequestParams
+{
+  LSSimpleRequestPreloadedParams;
+};
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/PBackgroundLSSimpleRequest.ipdl b/dom/localstorage/PBackgroundLSSimpleRequest.ipdl
new file mode 100644
index 000000000000..fc0c5132e921
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSSimpleRequest.ipdl
@@ -0,0 +1,30 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+
+namespace mozilla {
+namespace dom {
+
+struct LSSimpleRequestPreloadedResponse
+{
+  bool preloaded;
+};
+
+union LSSimpleRequestResponse
+{
+  nsresult;
+  LSSimpleRequestPreloadedResponse;
+};
+
+protocol PBackgroundLSSimpleRequest
+{
+  manager PBackground;
+
+child:
+  async __delete__(LSSimpleRequestResponse response);
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index 99f438373c82..66e58fd81dd3 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -38,6 +38,7 @@ IPDL_SOURCES += [
     'PBackgroundLSObserver.ipdl',
     'PBackgroundLSRequest.ipdl',
     'PBackgroundLSSharedTypes.ipdlh',
+    'PBackgroundLSSimpleRequest.ipdl',
 ]
 
 include('/ipc/chromium/chromium-config.mozbuild')
diff --git a/dom/localstorage/nsILocalStorageManager.idl b/dom/localstorage/nsILocalStorageManager.idl
index 8e9871994437..cab473000b59 100644
--- a/dom/localstorage/nsILocalStorageManager.idl
+++ b/dom/localstorage/nsILocalStorageManager.idl
@@ -12,4 +12,7 @@ interface nsIPrincipal;
 interface nsILocalStorageManager : nsISupports
 {
   readonly attribute boolean nextGenLocalStorageEnabled;
+
+  [implicit_jscontext] nsISupports
+  isPreloaded(in nsIPrincipal aPrincipal);
 };
diff --git a/dom/storage/LocalStorageManager.cpp b/dom/storage/LocalStorageManager.cpp
index 92d1b383d6fb..b9917eb624e9 100644
--- a/dom/storage/LocalStorageManager.cpp
+++ b/dom/storage/LocalStorageManager.cpp
@@ -381,6 +381,18 @@ LocalStorageManager::GetNextGenLocalStorageEnabled(bool* aResult)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocalStorageManager::IsPreloaded(nsIPrincipal* aPrincipal,
+                                 JSContext* aContext,
+                                 nsISupports** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
 void
 LocalStorageManager::ClearCaches(uint32_t aUnloadFlags,
                                  const OriginAttributesPattern& aPattern,
diff --git a/dom/tests/browser/browser.ini b/dom/tests/browser/browser.ini
index 2d59172fbd37..26ed1132a8a0 100644
--- a/dom/tests/browser/browser.ini
+++ b/dom/tests/browser/browser.ini
@@ -53,8 +53,8 @@ run-if = e10s
 skip-if = !e10s || os != "win" || processor != "x86" # Large-Allocation requires e10s
 [browser_largeAllocation_non_win32.js]
 skip-if = !e10s || (os == "win" && processor == "x86") || (verify && debug && (os == 'linux')) || (os == 'linux') || (os == 'mac' && debug)  # Large-Allocation requires e10s # Bug 1336075
-#[browser_localStorage_e10s.js]
-#skip-if = !e10s || verify # This is a test of e10s functionality.
+[browser_localStorage_e10s.js]
+skip-if = !e10s || verify # This is a test of e10s functionality.
 [browser_localStorage_privatestorageevent.js]
 [browser_persist_cookies.js]
 support-files =
diff --git a/dom/tests/browser/browser_localStorage_e10s.js b/dom/tests/browser/browser_localStorage_e10s.js
index 67d19784928d..49e564f90267 100644
--- a/dom/tests/browser/browser_localStorage_e10s.js
+++ b/dom/tests/browser/browser_localStorage_e10s.js
@@ -79,6 +79,10 @@ async function cleanupTabs(knownTabs) {
  * - Us generating a "domstorage-test-flush-force" observer notification.
  */
 function waitForLocalStorageFlush() {
+  if (Services.lsm.nextGenLocalStorageEnabled) {
+    return new Promise(resolve => executeSoon(resolve));
+  }
+
   return new Promise(function(resolve) {
     let observer = {
       observe: function() {
@@ -101,6 +105,10 @@ function waitForLocalStorageFlush() {
  * notifications, but correctness is guaranteed after the second notification.
  */
 function triggerAndWaitForLocalStorageFlush() {
+  if (Services.lsm.nextGenLocalStorageEnabled) {
+    return new Promise(resolve => executeSoon(resolve));
+  }
+
   SpecialPowers.notifyObservers(null, "domstorage-test-flush-force");
   // This first wait is ambiguous...
   return waitForLocalStorageFlush().then(function() {
@@ -127,6 +135,18 @@ function clearOriginStorageEnsuringNoPreload() {
   let principal =
     Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(
       HELPER_PAGE_ORIGIN);
+
+  if (Services.lsm.nextGenLocalStorageEnabled) {
+    let request =
+      Services.qms.clearStoragesForPrincipal(principal, "default", "ls");
+    let promise = new Promise(resolve => {
+      request.callback = () => {
+        resolve();
+      };
+    });
+    return promise;
+  }
+
   // We want to use createStorage to force the cache to be created so we can
   // issue the clear.  It's possible for getStorage to return false but for the
   // origin preload hash to still have our origin in it.
@@ -146,6 +166,9 @@ async function verifyTabPreload(knownTab, expectStorageExists) {
       let principal =
         Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(
           origin);
+      if (Services.lsm.nextGenLocalStorageEnabled) {
+        return Services.lsm.isPreloaded(principal);
+      }
       return !!Services.domStorageManager.getStorage(null, principal);
     });
   is(storageExists, expectStorageExists, "Storage existence === preload");
diff --git a/ipc/glue/BackgroundChildImpl.cpp b/ipc/glue/BackgroundChildImpl.cpp
index 464066b947f1..75c802601f1e 100644
--- a/ipc/glue/BackgroundChildImpl.cpp
+++ b/ipc/glue/BackgroundChildImpl.cpp
@@ -19,6 +19,7 @@
 #include "mozilla/dom/PBackgroundLSObjectChild.h"
 #include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
+#include "mozilla/dom/PBackgroundLSSimpleRequestChild.h"
 #include "mozilla/dom/PBackgroundSDBConnectionChild.h"
 #include "mozilla/dom/PFileSystemRequestChild.h"
 #include "mozilla/dom/FileSystemTaskBase.h"
@@ -317,6 +318,24 @@ BackgroundChildImpl::DeallocPBackgroundLocalStorageCacheChild(
   return true;
 }
 
+BackgroundChildImpl::PBackgroundLSSimpleRequestChild*
+BackgroundChildImpl::AllocPBackgroundLSSimpleRequestChild(
+                                           const LSSimpleRequestParams& aParams)
+{
+  MOZ_CRASH("PBackgroundLSSimpleRequestChild actor should be manually "
+            "constructed!");
+}
+
+bool
+BackgroundChildImpl::DeallocPBackgroundLSSimpleRequestChild(
+                                        PBackgroundLSSimpleRequestChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
 BackgroundChildImpl::PBackgroundStorageChild*
 BackgroundChildImpl::AllocPBackgroundStorageChild(const nsString& aProfilePath)
 {
diff --git a/ipc/glue/BackgroundChildImpl.h b/ipc/glue/BackgroundChildImpl.h
index fc72d188d4d2..ccb42555623e 100644
--- a/ipc/glue/BackgroundChildImpl.h
+++ b/ipc/glue/BackgroundChildImpl.h
@@ -100,6 +100,15 @@ protected:
   virtual bool
   DeallocPBackgroundLSRequestChild(PBackgroundLSRequestChild* aActor) override;
 
+  virtual PBackgroundLSSimpleRequestChild*
+  AllocPBackgroundLSSimpleRequestChild(const LSSimpleRequestParams& aParams)
+                                       override;
+
+  virtual bool
+  DeallocPBackgroundLSSimpleRequestChild(
+                                        PBackgroundLSSimpleRequestChild* aActor)
+                                        override;
+
   virtual PBackgroundLocalStorageCacheChild*
   AllocPBackgroundLocalStorageCacheChild(const PrincipalInfo& aPrincipalInfo,
                                          const nsCString& aOriginKey,
diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp
index 4b5928e0f145..ad88df7e83d3 100644
--- a/ipc/glue/BackgroundParentImpl.cpp
+++ b/ipc/glue/BackgroundParentImpl.cpp
@@ -412,6 +412,43 @@ BackgroundParentImpl::DeallocPBackgroundLSRequestParent(
   return mozilla::dom::DeallocPBackgroundLSRequestParent(aActor);
 }
 
+BackgroundParentImpl::PBackgroundLSSimpleRequestParent*
+BackgroundParentImpl::AllocPBackgroundLSSimpleRequestParent(
+                                           const LSSimpleRequestParams& aParams)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+
+  return mozilla::dom::AllocPBackgroundLSSimpleRequestParent(aParams);
+}
+
+mozilla::ipc::IPCResult
+BackgroundParentImpl::RecvPBackgroundLSSimpleRequestConstructor(
+                                       PBackgroundLSSimpleRequestParent* aActor,
+                                       const LSSimpleRequestParams& aParams)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  if (!mozilla::dom::RecvPBackgroundLSSimpleRequestConstructor(aActor,
+                                                               aParams)) {
+    return IPC_FAIL_NO_REASON(this);
+  }
+  return IPC_OK();
+}
+
+bool
+BackgroundParentImpl::DeallocPBackgroundLSSimpleRequestParent(
+                                       PBackgroundLSSimpleRequestParent* aActor)
+{
+  AssertIsInMainProcess();
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  return mozilla::dom::DeallocPBackgroundLSSimpleRequestParent(aActor);
+}
+
 BackgroundParentImpl::PBackgroundLocalStorageCacheParent*
 BackgroundParentImpl::AllocPBackgroundLocalStorageCacheParent(
                                             const PrincipalInfo& aPrincipalInfo,
diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h
index 58b6ef628542..c8c8b4b67251 100644
--- a/ipc/glue/BackgroundParentImpl.h
+++ b/ipc/glue/BackgroundParentImpl.h
@@ -115,6 +115,21 @@ protected:
   DeallocPBackgroundLSRequestParent(PBackgroundLSRequestParent* aActor)
                                     override;
 
+  virtual PBackgroundLSSimpleRequestParent*
+  AllocPBackgroundLSSimpleRequestParent(const LSSimpleRequestParams& aParams)
+                                        override;
+
+  virtual mozilla::ipc::IPCResult
+  RecvPBackgroundLSSimpleRequestConstructor(
+                                       PBackgroundLSSimpleRequestParent* aActor,
+                                       const LSSimpleRequestParams& aParams)
+                                       override;
+
+  virtual bool
+  DeallocPBackgroundLSSimpleRequestParent(
+                                       PBackgroundLSSimpleRequestParent* aActor)
+                                       override;
+
   virtual PBackgroundLocalStorageCacheParent*
   AllocPBackgroundLocalStorageCacheParent(const PrincipalInfo& aPrincipalInfo,
                                           const nsCString& aOriginKey,
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index b9738e9a325d..a1f03a747747 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -9,6 +9,7 @@ include protocol PBackgroundSDBConnection;
 include protocol PBackgroundLSObject;
 include protocol PBackgroundLSObserver;
 include protocol PBackgroundLSRequest;
+include protocol PBackgroundLSSimpleRequest;
 include protocol PBackgroundLocalStorageCache;
 include protocol PBackgroundStorage;
 include protocol PBackgroundTest;
@@ -79,6 +80,7 @@ sync protocol PBackground
   manages PBackgroundLSObject;
   manages PBackgroundLSObserver;
   manages PBackgroundLSRequest;
+  manages PBackgroundLSSimpleRequest;
   manages PBackgroundLocalStorageCache;
   manages PBackgroundStorage;
   manages PBackgroundTest;
@@ -134,6 +136,8 @@ parent:
 
   async PBackgroundLSRequest(LSRequestParams params);
 
+  async PBackgroundLSSimpleRequest(LSSimpleRequestParams params);
+
   async PBackgroundLocalStorageCache(PrincipalInfo principalInfo,
                                      nsCString originKey,
                                      uint32_t privateBrowsingId);

From d015b67aae211482913fd32bae94cec59469b7a1 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:19 +0100
Subject: [PATCH 34/78] Bug 1286798 - Part 20: Add checks for the 5 MB origin
 limit; r=asuth

The 5 MB limit is no longer applied to the whole group (eTLD+1). That will be controlled by quota manager.
---
 dom/localstorage/ActorsParent.cpp             | 61 +++++++++++++++++++
 dom/tests/mochitest/localstorage/chrome.ini   |  2 +-
 .../mochitest/localstorage/frameQuota.html    |  8 ---
 .../localstorage/frameQuotaSessionOnly.html   |  8 ---
 .../mochitest/localstorage/mochitest.ini      |  4 +-
 .../localstorage/test_localStorageQuota.html  | 50 ++++++++++-----
 ...orageQuotaPrivateBrowsing_perwindowpb.html | 56 ++++++++++-------
 ...ge_local_setitem_quotaexceedederr.html.ini |  2 -
 8 files changed, 133 insertions(+), 58 deletions(-)
 delete mode 100644 testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 4ce59a31aede..358d4d7e017b 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -12,6 +12,7 @@
 #include "mozIStorageService.h"
 #include "mozStorageCID.h"
 #include "mozStorageHelper.h"
+#include "mozilla/Preferences.h"
 #include "mozilla/Unused.h"
 #include "mozilla/dom/PBackgroundLSDatabaseParent.h"
 #include "mozilla/dom/PBackgroundLSObjectParent.h"
@@ -108,6 +109,9 @@ const uint32_t kAutoCommitTimeoutMs = 5000;
 
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 
+const uint32_t kDefaultOriginLimitKB = 5 * 1024;
+const char kDefaultQuotaPref[] = "dom.storage.default_quota";
+
 bool
 IsOnConnectionThread();
 
@@ -860,12 +864,14 @@ class Datastore final
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const nsCString mOrigin;
   const uint32_t mPrivateBrowsingId;
+  int64_t mUsage;
   bool mClosed;
 
 public:
   // Created by PrepareDatastoreOp.
   Datastore(const nsACString& aOrigin,
             uint32_t aPrivateBrowsingId,
+            int64_t aUsage,
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
             nsDataHashtable<nsStringHashKey, nsString>& aValues);
@@ -950,6 +956,9 @@ private:
   void
   CleanupMetadata();
 
+  bool
+  UpdateUsage(int64_t aDelta);
+
   void
   NotifyObservers(Database* aDatabase,
                   const nsString& aKey,
@@ -1347,6 +1356,7 @@ class PrepareDatastoreOp
   nsCString mOrigin;
   nsString mDatabaseFilePath;
   uint32_t mPrivateBrowsingId;
+  int64_t mUsage;
   NestedState mNestedState;
   bool mRequestedDirectoryLock;
   bool mInvalidated;
@@ -1733,6 +1743,8 @@ typedef nsClassHashtable<nsCStringHashKey, nsTArray<Observer*>>
 
 StaticAutoPtr<ObserverHashtable> gObservers;
 
+Atomic<uint32_t, Relaxed> gOriginLimitKB(kDefaultOriginLimitKB);
+
 bool
 IsOnConnectionThread()
 {
@@ -2527,6 +2539,7 @@ ClearOp::DoDatastoreWork()
 
 Datastore::Datastore(const nsACString& aOrigin,
                      uint32_t aPrivateBrowsingId,
+                     int64_t aUsage,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
                      nsDataHashtable<nsStringHashKey, nsString>& aValues)
@@ -2534,6 +2547,7 @@ Datastore::Datastore(const nsACString& aOrigin,
   , mConnection(std::move(aConnection))
   , mOrigin(aOrigin)
   , mPrivateBrowsingId(aPrivateBrowsingId)
+  , mUsage(aUsage)
   , mClosed(false)
 {
   AssertIsOnBackgroundThread();
@@ -2692,6 +2706,20 @@ Datastore::SetItem(Database* aDatabase,
   } else {
     changed = true;
 
+    int64_t delta = 0;
+
+    if (oldValue.IsVoid()) {
+      delta += static_cast<int64_t>(aKey.Length());
+    }
+
+    delta += static_cast<int64_t>(aValue.Length()) -
+             static_cast<int64_t>(oldValue.Length());
+
+    if (!UpdateUsage(delta)) {
+      aResponse = NS_ERROR_FILE_NO_DEVICE_SPACE;
+      return;
+    }
+
     mValues.Put(aKey, aValue);
 
     NotifyObservers(aDatabase, aKey, oldValue, aValue);
@@ -2727,6 +2755,12 @@ Datastore::RemoveItem(Database* aDatabase,
   } else {
     changed = true;
 
+    int64_t delta = -(static_cast<int64_t>(aKey.Length()) +
+                      static_cast<int64_t>(oldValue.Length()));
+
+    DebugOnly<bool> ok = UpdateUsage(delta);
+    MOZ_ASSERT(ok);
+
     mValues.Remove(aKey);
 
     NotifyObservers(aDatabase, aKey, oldValue, VoidString());
@@ -2758,6 +2792,9 @@ Datastore::Clear(Database* aDatabase,
   } else {
     changed = true;
 
+    DebugOnly<bool> ok = UpdateUsage(-mUsage);
+    MOZ_ASSERT(ok);
+
     mValues.Clear();
 
     if (aDatabase) {
@@ -2823,6 +2860,21 @@ Datastore::CleanupMetadata()
   }
 }
 
+bool
+Datastore::UpdateUsage(int64_t aDelta)
+{
+  AssertIsOnBackgroundThread();
+
+  int64_t newUsage = mUsage + aDelta;
+  if (newUsage > gOriginLimitKB * 1024) {
+    return false;
+  }
+
+  mUsage = newUsage;
+
+  return true;
+}
+
 void
 Datastore::NotifyObservers(Database* aDatabase,
                            const nsString& aKey,
@@ -3504,6 +3556,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   , mLoadDataOp(nullptr)
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mPrivateBrowsingId(0)
+  , mUsage(0)
   , mNestedState(NestedState::BeforeNesting)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
@@ -4014,6 +4067,7 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
   if (!mDatastore) {
     mDatastore = new Datastore(mOrigin,
                                mPrivateBrowsingId,
+                               mUsage,
                                mDirectoryLock.forget(),
                                mConnection.forget(),
                                mValues);
@@ -4225,6 +4279,7 @@ LoadDataOp::DoDatastoreWork()
     }
 
     mPrepareDatastoreOp->mValues.Put(key, value);
+    mPrepareDatastoreOp->mUsage += key.Length() + value.Length();
   }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -4575,6 +4630,12 @@ QuotaClient::RegisterObservers(nsIEventTarget* aBackgroundEventTarget)
       return rv;
     }
 
+    if (NS_FAILED(Preferences::AddAtomicUintVarCache(&gOriginLimitKB,
+                                                     kDefaultQuotaPref,
+                                                     kDefaultOriginLimitKB))) {
+      NS_WARNING("Unable to respond to default quota pref changes!");
+    }
+
     sObserversRegistered = true;
   }
 
diff --git a/dom/tests/mochitest/localstorage/chrome.ini b/dom/tests/mochitest/localstorage/chrome.ini
index 10f164bfd6e6..6cc5562c00c9 100644
--- a/dom/tests/mochitest/localstorage/chrome.ini
+++ b/dom/tests/mochitest/localstorage/chrome.ini
@@ -8,4 +8,4 @@ support-files =
 [test_localStorageBasePrivateBrowsing_perwindowpb.html]
 skip-if = true # bug 1156725
 [test_localStorageFromChrome.xhtml]
-#[test_localStorageQuotaPrivateBrowsing_perwindowpb.html]
+[test_localStorageQuotaPrivateBrowsing_perwindowpb.html]
diff --git a/dom/tests/mochitest/localstorage/frameQuota.html b/dom/tests/mochitest/localstorage/frameQuota.html
index 2b6c8f9ddecd..5ff9855c99ad 100644
--- a/dom/tests/mochitest/localstorage/frameQuota.html
+++ b/dom/tests/mochitest/localstorage/frameQuota.html
@@ -44,14 +44,6 @@ function doStep()
           localStorage.removeItem(keyName);
           is(localStorage.getItem(keyName), null, "Key "+keyName+" removed");
           break;
-
-        case "checkclean":
-          is(localStorage.getItem(keyName), null, "Key "+keyName+" not present");
-          break;
-
-        case "checknotclean":
-          is(localStorage.getItem(keyName), "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "Key "+keyName+" is present");
-          break;
       }
 
       break;
diff --git a/dom/tests/mochitest/localstorage/frameQuotaSessionOnly.html b/dom/tests/mochitest/localstorage/frameQuotaSessionOnly.html
index 68efde9b8b09..e687991faed4 100644
--- a/dom/tests/mochitest/localstorage/frameQuotaSessionOnly.html
+++ b/dom/tests/mochitest/localstorage/frameQuotaSessionOnly.html
@@ -44,14 +44,6 @@ function doStep()
           localStorage.removeItem(keyName);
           is(localStorage.getItem(keyName), null, "Key "+keyName+" removed");
           break;
-
-        case "checkclean":
-          is(localStorage.getItem(keyName), null, "Key "+keyName+" not present");
-          break;
-
-        case "checknotclean":
-          is(localStorage.getItem(keyName), "12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "Key "+keyName+" is present");
-          break;
       }
 
       break;
diff --git a/dom/tests/mochitest/localstorage/mochitest.ini b/dom/tests/mochitest/localstorage/mochitest.ini
index 8de589e7dfb7..48173aac3b72 100644
--- a/dom/tests/mochitest/localstorage/mochitest.ini
+++ b/dom/tests/mochitest/localstorage/mochitest.ini
@@ -40,8 +40,8 @@ skip-if = toolkit == 'android'
 [test_localStorageOriginsPortDiffs.html]
 [test_localStorageOriginsSchemaDiffs.html]
 skip-if = toolkit == 'android' #TIMED_OUT
-#[test_localStorageQuota.html]
-#skip-if = toolkit == 'android' #TIMED_OUT
+[test_localStorageQuota.html]
+skip-if = toolkit == 'android' #TIMED_OUT
 [test_localStorageQuotaSessionOnly.html]
 skip-if = toolkit == 'android' || (verify && (os == 'linux' || os == 'mac' || os == 'win')) #TIMED_OUT
 [test_localStorageQuotaSessionOnly2.html]
diff --git a/dom/tests/mochitest/localstorage/test_localStorageQuota.html b/dom/tests/mochitest/localstorage/test_localStorageQuota.html
index 6aeefe6bcb9d..3ed723c4d27f 100644
--- a/dom/tests/mochitest/localstorage/test_localStorageQuota.html
+++ b/dom/tests/mochitest/localstorage/test_localStorageQuota.html
@@ -18,12 +18,15 @@ function doNextTest()
   switch (currentTest)
   {
     case 1:
-      slaveOrigin = "http://example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "http://example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&A&success";
       break;
 
-    // In subdomain now set another key with length 500 bytes, i.e.
-    // allocate 501 bytes
+    // Now set another key with length 500 bytes, i.e. allocate 501 bytes
     case 2:
       slaveOrigin = "http://test1.example.com";
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&B&success";
@@ -44,16 +47,19 @@ function doNextTest()
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add2&B&failure";
       break;
 
-    // In a different subdomain try to set a new 500 bytes key
-    // and check we fail because we are over the quota
+    // Try to set a new 500 bytes key and check we fail because we are over the
+    // quota.
     case 5:
-      slaveOrigin = "https://test2.example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "https://test2.example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&C&failure";
       break;
 
-    // Remove from the second subdomain the second key, it must not fail
-    // This should release the allocated space of the quota assigned to
-    // example.com.
+    // Remove the second key, it must not fail. This should release the
+    // allocated space of the quota assigned to test1.example.com.
     case 6:
       slaveOrigin = "http://test1.example.com";
       slave.location = slaveOrigin + slavePath + "frameQuota.html?remove&B&success";
@@ -61,14 +67,22 @@ function doNextTest()
 
     // Now try again to set 500 bytes key, it must succeed.
     case 7:
-      slaveOrigin = "https://test2.example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "https://test2.example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&C&success";
       break;
 
     case 8:
-      // Do a clean up...
-      slaveOrigin = "http://example.com";
-      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        SimpleTest.executeSoon(doNextTest);
+      } else {
+        // Do a clean up...
+        slaveOrigin = "http://example.com";
+        slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      }
       break;
 
     case 9:
@@ -78,9 +92,13 @@ function doNextTest()
       break;
 
     case 10:
-      // Do a clean up...
-      slaveOrigin = "https://test2.example.com";
-      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        SimpleTest.executeSoon(doNextTest);
+      } else {
+        // Do a clean up...
+        slaveOrigin = "https://test2.example.com";
+        slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      }
       break;
 
     default: // end
diff --git a/dom/tests/mochitest/localstorage/test_localStorageQuotaPrivateBrowsing_perwindowpb.html b/dom/tests/mochitest/localstorage/test_localStorageQuotaPrivateBrowsing_perwindowpb.html
index 84147fa3cd89..03d5f9cd16ae 100644
--- a/dom/tests/mochitest/localstorage/test_localStorageQuotaPrivateBrowsing_perwindowpb.html
+++ b/dom/tests/mochitest/localstorage/test_localStorageQuotaPrivateBrowsing_perwindowpb.html
@@ -42,12 +42,15 @@ function doNextTest(aWindow) {
     // Initialy setup the quota to testing value of 1024B and
     // set a 500 bytes key with name length 1 (allocate 501 bytes)
     case 1:
-      slaveOrigin = "http://example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "http://example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&A&success";
       break;
 
-    // In subdomain now set another key with length 500 bytes, i.e.
-    // allocate 501 bytes
+    // Now set another key with length 500 bytes, i.e. allocate 501 bytes
     case 2:
       slaveOrigin = "http://test1.example.com";
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&B&success";
@@ -68,16 +71,19 @@ function doNextTest(aWindow) {
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add2&B&failure";
       break;
 
-    // In a different subdomain try to set a new 500 bytes key
-    // and check we fail because we are over the quota
+    // Try to set a new 500 bytes key and check we fail because we are over the
+    // quota.
     case 5:
-      slaveOrigin = "https://test2.example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "https://test2.example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&C&failure";
       break;
 
-    // Remove from the second subdomain the second key, it must not fail
-    // This should release the allocated space of the quota assigned to
-    // example.com.
+    // Remove the second key, it must not fail. This should release the
+    // allocated space of the quota assigned to test1.example.com.
     case 6:
       slaveOrigin = "http://test1.example.com";
       slave.location = slaveOrigin + slavePath + "frameQuota.html?remove&B&success";
@@ -85,30 +91,38 @@ function doNextTest(aWindow) {
 
     // Now try again to set 500 bytes key, it must succeed.
     case 7:
-      slaveOrigin = "https://test2.example.com";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        slaveOrigin = "http://test1.example.com";
+      } else {
+        slaveOrigin = "https://test2.example.com";
+      }
       slave.location = slaveOrigin + slavePath + "frameQuota.html?add&C&success";
       break;
 
     case 8:
-      // Do a clean up...
-      // TODO Bug 455070, use just ?clear what invokes call
-      // of clear() in the target frame. W/o clear method we must
-      // call clear implemented as removeItem for each item in
-      // the localStorage.
-      slaveOrigin = "http://example.com";
-      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear&A&";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        SimpleTest.executeSoon(() => doNextTest(aWindow));
+      } else {
+        // Do a clean up...
+        slaveOrigin = "http://example.com";
+        slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      }
       break;
 
     case 9:
       // Do a clean up...
       slaveOrigin = "http://test1.example.com";
-      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear&B&";
+      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
       break;
 
     case 10:
-      // Do a clean up...
-      slaveOrigin = "https://test2.example.com";
-      slave.location = slaveOrigin + slavePath + "frameQuota.html?clear&C&";
+      if (SpecialPowers.Services.lsm.nextGenLocalStorageEnabled) {
+        SimpleTest.executeSoon(() => doNextTest(aWindow));
+      } else {
+        // Do a clean up...
+        slaveOrigin = "https://test2.example.com";
+        slave.location = slaveOrigin + slavePath + "frameQuota.html?clear";
+      }
       break;
 
     default:
diff --git a/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini b/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini
deleted file mode 100644
index 48f2692aad2a..000000000000
--- a/testing/web-platform/meta/webstorage/storage_local_setitem_quotaexceedederr.html.ini
+++ /dev/null
@@ -1,2 +0,0 @@
-[storage_local_setitem_quotaexceedederr.html]
-  disabled: temporary

From de23109c5863b9bc9e622ca68be0695ae18d726a Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:22 +0100
Subject: [PATCH 35/78] Bug 1286798 - Part 21: Base domain needs to be handled
 too if strict file origin policy is not in effect; r=bholley,asuth,dholbert

---
 caps/ContentPrincipal.cpp        | 11 ++++++++++-
 layout/reftests/svg/reftest.list |  3 ++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/caps/ContentPrincipal.cpp b/caps/ContentPrincipal.cpp
index b6ced998078c..04481ebfa6f4 100644
--- a/caps/ContentPrincipal.cpp
+++ b/caps/ContentPrincipal.cpp
@@ -396,8 +396,17 @@ GetSpecialBaseDomain(const nsCOMPtr<nsIURI>& aCodebase,
 {
   *aHandled = false;
 
-  // For a file URI, we return the file path.
+  // Special handling for a file URI.
   if (NS_URIIsLocalFile(aCodebase)) {
+    // If strict file origin policy is not in effect, all local files are
+    // considered to be same-origin, so return a known dummy domain here.
+    if (!nsScriptSecurityManager::GetStrictFileOriginPolicy()) {
+      *aHandled = true;
+      aBaseDomain.AssignLiteral("UNIVERSAL_FILE_URI_ORIGIN");
+      return NS_OK;
+    }
+
+    // Otherwise, we return the file path.
     nsCOMPtr<nsIURL> url = do_QueryInterface(aCodebase);
 
     if (url) {
diff --git a/layout/reftests/svg/reftest.list b/layout/reftests/svg/reftest.list
index 9849518f7571..9128aef8a60b 100644
--- a/layout/reftests/svg/reftest.list
+++ b/layout/reftests/svg/reftest.list
@@ -199,10 +199,11 @@ fuzzy-if(skiaContent,0-1,0-2) == fallback-color-04.svg pass.svg
 == filter-basic-03.svg pass.svg
 == filter-bounds-01.svg pass.svg
 == filter-bounds-02.svg pass.svg
+# Disabled for now, see bug 1286798 comment 180, 187, 190 and 196.
 # This pref is normally on by default, but we turn it off in reftest runs to
 # disable an unnecessary security-check. This reftest is actually testing that
 # the security check works, though, so it needs the pref to be turned on:
-fails-if(Android) pref(security.fileuri.strict_origin_policy,true) == filter-extref-differentOrigin-01.svg pass.svg # Bug 695385
+skip pref(security.fileuri.strict_origin_policy,true) == filter-extref-differentOrigin-01.svg pass.svg # Bug 695385
 == filter-foreignObject-01.svg pass.svg
 == filter-in-mask-01.svg pass.svg
 == filter-invalidation-01.svg pass.svg

From 6e40a9dccbbc2a28f4c64a24732ba54e74a50801 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:25 +0100
Subject: [PATCH 36/78] Bug 1286798 - Part 22: Add support for preloading of
 datastores; r=asuth

Datastores are preloaded only for content principals. The preloading is triggered as soon as possible to lower the chance of blocking the main thread in content process. If there is no physical database on disk for given origin, datastore is not created. Preloaded datastores are kept alive for 20 seconds.
---
 dom/asmjscache/AsmJSCache.cpp                 |   4 +-
 dom/cache/Context.cpp                         |   1 +
 dom/indexedDB/ActorsParent.cpp                |  13 +-
 dom/ipc/ContentParent.cpp                     |  20 +
 dom/localstorage/ActorsChild.h                |   2 +-
 dom/localstorage/ActorsParent.cpp             | 357 +++++++++++++++---
 dom/localstorage/LSObject.cpp                 |   5 +-
 dom/localstorage/LocalStorageManager2.cpp     | 141 +++++++
 dom/localstorage/LocalStorageManager2.h       |   5 +
 dom/localstorage/PBackgroundLSRequest.ipdl    |  11 +-
 .../PBackgroundLSSharedTypes.ipdlh            |   1 +
 dom/localstorage/nsILocalStorageManager.idl   |   3 +
 dom/quota/ActorsParent.cpp                    |  34 +-
 dom/quota/QuotaManager.h                      |   3 +
 dom/simpledb/ActorsParent.cpp                 |   1 +
 dom/storage/LocalStorageManager.cpp           |  12 +
 16 files changed, 540 insertions(+), 73 deletions(-)

diff --git a/dom/asmjscache/AsmJSCache.cpp b/dom/asmjscache/AsmJSCache.cpp
index b92714a05bf8..368626706290 100644
--- a/dom/asmjscache/AsmJSCache.cpp
+++ b/dom/asmjscache/AsmJSCache.cpp
@@ -792,7 +792,9 @@ ParentRunnable::ReadMetadata()
 
   nsresult rv =
     qm->EnsureOriginIsInitialized(quota::PERSISTENCE_TYPE_TEMPORARY, mSuffix,
-                                  mGroup, mOrigin, getter_AddRefs(mDirectory));
+                                  mGroup, mOrigin,
+                                  /* aCreateIfNotExists */ true,
+                                  getter_AddRefs(mDirectory));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     mResult = JS::AsmJSCache_StorageInitFailure;
     return rv;
diff --git a/dom/cache/Context.cpp b/dom/cache/Context.cpp
index ea49046f0c67..9bfdce27a276 100644
--- a/dom/cache/Context.cpp
+++ b/dom/cache/Context.cpp
@@ -435,6 +435,7 @@ Context::QuotaInitRunnable::Run()
                                                   mQuotaInfo.mSuffix,
                                                   mQuotaInfo.mGroup,
                                                   mQuotaInfo.mOrigin,
+                                                  /* aCreateIfNotExists */ true,
                                                   getter_AddRefs(mQuotaInfo.mDir));
       if (NS_FAILED(rv)) {
         resolver->Resolve(rv);
diff --git a/dom/indexedDB/ActorsParent.cpp b/dom/indexedDB/ActorsParent.cpp
index 4e611ecb2139..fccd8b4b69c6 100644
--- a/dom/indexedDB/ActorsParent.cpp
+++ b/dom/indexedDB/ActorsParent.cpp
@@ -18419,11 +18419,13 @@ Maintenance::DirectoryWork()
         // Idle maintenance may occur before origin is initailized.
         // Ensure origin is initialized first. It will initialize all origins
         // for temporary storage including IDB origins.
-        rv = quotaManager->EnsureOriginIsInitialized(persistenceType,
-                                                     suffix,
-                                                     group,
-                                                     origin,
-                                                     getter_AddRefs(directory));
+        rv = quotaManager->EnsureOriginIsInitialized(
+                                                  persistenceType,
+                                                  suffix,
+                                                  group,
+                                                  origin,
+                                                  /* aCreateIfNotExists */ true,
+                                                  getter_AddRefs(directory));
 
         if (NS_WARN_IF(NS_FAILED(rv))) {
           return rv;
@@ -21319,6 +21321,7 @@ OpenDatabaseOp::DoDatabaseWork()
                                             mSuffix,
                                             mGroup,
                                             mOrigin,
+                                            /* aCreateIfNotExists */ true,
                                             getter_AddRefs(dbDirectory));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
diff --git a/dom/ipc/ContentParent.cpp b/dom/ipc/ContentParent.cpp
index 52ca081ca838..8294fdd69d4f 100644
--- a/dom/ipc/ContentParent.cpp
+++ b/dom/ipc/ContentParent.cpp
@@ -49,6 +49,7 @@
 #include "mozilla/dom/ExternalHelperAppParent.h"
 #include "mozilla/dom/GetFilesHelper.h"
 #include "mozilla/dom/GeolocationBinding.h"
+#include "mozilla/dom/LocalStorageCommon.h"
 #include "mozilla/dom/MemoryReportRequest.h"
 #include "mozilla/dom/Notification.h"
 #include "mozilla/dom/PContentBridgeParent.h"
@@ -141,6 +142,7 @@
 #include "nsIGfxInfo.h"
 #include "nsIIdleService.h"
 #include "nsIInterfaceRequestorUtils.h"
+#include "nsILocalStorageManager.h"
 #include "nsIMemoryInfoDumper.h"
 #include "nsIMemoryReporter.h"
 #include "nsIMozBrowserFrame.h"
@@ -5707,6 +5709,24 @@ ContentParent::AboutToLoadHttpFtpWyciwygDocumentForChild(nsIChannel* aChannel)
     UpdateCookieStatus(aChannel);
   }
 
+  if (!NextGenLocalStorageEnabled()) {
+    return NS_OK;
+  }
+
+  if (principal->GetIsCodebasePrincipal()) {
+    nsCOMPtr<nsILocalStorageManager> lsm =
+      do_GetService("@mozilla.org/dom/localStorage-manager;1");
+    if (NS_WARN_IF(!lsm)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    nsCOMPtr<nsISupports> dummy;
+    rv = lsm->Preload(principal, nullptr, getter_AddRefs(dummy));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
   return NS_OK;
 }
 
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 10b5881b93f0..100234ccb384 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -150,7 +150,7 @@ class LSRequestChild final
   : public PBackgroundLSRequestChild
 {
   friend class LSObject;
-  friend class LSObjectChild;
+  friend class LocalStorageManager2;
 
   RefPtr<LSRequestChildCallback> mCallback;
 
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 358d4d7e017b..013d3f97164b 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -55,6 +55,8 @@ namespace {
 class Connection;
 class ConnectionThread;
 class Database;
+class PrepareDatastoreOp;
+class PreparedDatastore;
 
 /*******************************************************************************
  * Constants
@@ -112,6 +114,8 @@ const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
 const char kDefaultQuotaPref[] = "dom.storage.default_quota";
 
+const uint32_t kPreparedDatastoreTimeoutMs = 20000;
+
 bool
 IsOnConnectionThread();
 
@@ -860,6 +864,8 @@ class Datastore final
   RefPtr<Connection> mConnection;
   nsCOMPtr<nsITimer> mAutoCommitTimer;
   nsCOMPtr<nsIRunnable> mCompleteCallback;
+  nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
+  nsTHashtable<nsPtrHashKey<PreparedDatastore>> mPreparedDatastores;
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const nsCString mOrigin;
@@ -908,14 +914,33 @@ public:
   void
   WaitForConnectionToComplete(nsIRunnable* aCallback);
 
+  void
+  NoteLivePrepareDatastoreOp(PrepareDatastoreOp* aPrepareDatastoreOp);
+
+  void
+  NoteFinishedPrepareDatastoreOp(PrepareDatastoreOp* aPrepareDatastoreOp);
+
+  void
+  NoteLivePreparedDatastore(PreparedDatastore* aPreparedDatastore);
+
+  void
+  NoteFinishedPreparedDatastore(PreparedDatastore* aPreparedDatastore);
+
+#ifdef DEBUG
+  bool
+  HasLivePreparedDatastores() const;
+#endif
+
   void
   NoteLiveDatabase(Database* aDatabase);
 
   void
   NoteFinishedDatabase(Database* aDatabase);
 
+#ifdef DEBUG
   bool
   HasLiveDatabases() const;
+#endif
 
   uint32_t
   GetLength() const;
@@ -950,6 +975,9 @@ private:
   // Reference counted.
   ~Datastore();
 
+  void
+  MaybeClose();
+
   void
   ConnectionClosedCallback();
 
@@ -975,29 +1003,57 @@ private:
 class PreparedDatastore
 {
   RefPtr<Datastore> mDatastore;
+  nsCOMPtr<nsITimer> mTimer;
   // Strings share buffers if possible, so it's not a problem to duplicate the
   // origin here.
   const nsCString mOrigin;
+  uint64_t mDatastoreId;
+  bool mForPreload;
   bool mInvalidated;
 
 public:
   PreparedDatastore(Datastore* aDatastore,
-                    const nsACString& aOrigin)
+                    const nsACString& aOrigin,
+                    uint64_t aDatastoreId,
+                    bool aForPreload)
     : mDatastore(aDatastore)
+    , mTimer(NS_NewTimer())
     , mOrigin(aOrigin)
+    , mDatastoreId(aDatastoreId)
+    , mForPreload(aForPreload)
     , mInvalidated(false)
   {
     AssertIsOnBackgroundThread();
     MOZ_ASSERT(aDatastore);
+    MOZ_ASSERT(mTimer);
+
+    aDatastore->NoteLivePreparedDatastore(this);
+
+    MOZ_ALWAYS_SUCCEEDS(
+      mTimer->InitWithNamedFuncCallback(TimerCallback,
+                                        this,
+                                        kPreparedDatastoreTimeoutMs,
+                                        nsITimer::TYPE_ONE_SHOT,
+                                        "PreparedDatastore::TimerCallback"));
   }
 
-  already_AddRefed<Datastore>
-  ForgetDatastore()
+  ~PreparedDatastore()
+  {
+    MOZ_ASSERT(mDatastore);
+    MOZ_ASSERT(mTimer);
+
+    mTimer->Cancel();
+
+    mDatastore->NoteFinishedPreparedDatastore(this);
+  }
+
+  Datastore*
+  GetDatastore() const
   {
     AssertIsOnBackgroundThread();
     MOZ_ASSERT(mDatastore);
 
-    return mDatastore.forget();
+    return mDatastore;
   }
 
   const nsCString&
@@ -1012,6 +1068,17 @@ public:
     AssertIsOnBackgroundThread();
 
     mInvalidated = true;
+
+    if (mForPreload) {
+      mTimer->Cancel();
+
+      MOZ_ALWAYS_SUCCEEDS(
+        mTimer->InitWithNamedFuncCallback(TimerCallback,
+                                          this,
+                                          0,
+                                          nsITimer::TYPE_ONE_SHOT,
+                                          "PreparedDatastore::TimerCallback"));
+    }
   }
 
   bool
@@ -1021,6 +1088,13 @@ public:
 
     return mInvalidated;
   }
+
+private:
+  void
+  Destroy();
+
+  static void
+  TimerCallback(nsITimer* aTimer, void* aClosure);
 };
 
 /*******************************************************************************
@@ -1119,7 +1193,7 @@ public:
   }
 
   void
-  SetActorAlive(already_AddRefed<Datastore>&& aDatastore);
+  SetActorAlive(Datastore* aDatastore);
 
   void
   RequestAllowToClose();
@@ -1358,6 +1432,7 @@ class PrepareDatastoreOp
   uint32_t mPrivateBrowsingId;
   int64_t mUsage;
   NestedState mNestedState;
+  bool mDatabaseNotAvailable;
   bool mRequestedDirectoryLock;
   bool mInvalidated;
 
@@ -1425,6 +1500,12 @@ private:
   nsresult
   DatabaseWork();
 
+  nsresult
+  DatabaseNotAvailable();
+
+  nsresult
+  EnsureDirectoryEntry(nsIFile* aEntry, bool aDirectory);
+
   nsresult
   VerifyDatabaseInformation(mozIStorageConnection* aConnection);
 
@@ -2613,6 +2694,69 @@ Datastore::WaitForConnectionToComplete(nsIRunnable* aCallback)
   mCompleteCallback = aCallback;
 }
 
+void
+Datastore::NoteLivePrepareDatastoreOp(PrepareDatastoreOp* aPrepareDatastoreOp)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aPrepareDatastoreOp);
+  MOZ_ASSERT(!mPrepareDatastoreOps.GetEntry(aPrepareDatastoreOp));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mPrepareDatastoreOps.PutEntry(aPrepareDatastoreOp);
+}
+
+void
+Datastore::NoteFinishedPrepareDatastoreOp(
+                                        PrepareDatastoreOp* aPrepareDatastoreOp)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aPrepareDatastoreOp);
+  MOZ_ASSERT(mPrepareDatastoreOps.GetEntry(aPrepareDatastoreOp));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mPrepareDatastoreOps.RemoveEntry(aPrepareDatastoreOp);
+
+  MaybeClose();
+}
+
+void
+Datastore::NoteLivePreparedDatastore(PreparedDatastore* aPreparedDatastore)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aPreparedDatastore);
+  MOZ_ASSERT(!mPreparedDatastores.GetEntry(aPreparedDatastore));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mPreparedDatastores.PutEntry(aPreparedDatastore);
+}
+
+void
+Datastore::NoteFinishedPreparedDatastore(PreparedDatastore* aPreparedDatastore)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aPreparedDatastore);
+  MOZ_ASSERT(mPreparedDatastores.GetEntry(aPreparedDatastore));
+  MOZ_ASSERT(mDirectoryLock);
+  MOZ_ASSERT(!mClosed);
+
+  mPreparedDatastores.RemoveEntry(aPreparedDatastore);
+
+  MaybeClose();
+}
+
+#ifdef DEBUG
+bool
+Datastore::HasLivePreparedDatastores() const
+{
+  AssertIsOnBackgroundThread();
+
+  return mPreparedDatastores.Count();
+}
+#endif
+
 void
 Datastore::NoteLiveDatabase(Database* aDatabase)
 {
@@ -2636,11 +2780,10 @@ Datastore::NoteFinishedDatabase(Database* aDatabase)
 
   mDatabases.RemoveEntry(aDatabase);
 
-  if (!mDatabases.Count()) {
-    Close();
-  }
+  MaybeClose();
 }
 
+#ifdef DEBUG
 bool
 Datastore::HasLiveDatabases() const
 {
@@ -2648,6 +2791,7 @@ Datastore::HasLiveDatabases() const
 
   return mDatabases.Count();
 }
+#endif
 
 uint32_t
 Datastore::GetLength() const
@@ -2825,6 +2969,18 @@ Datastore::GetKeys(nsTArray<nsString>& aKeys) const
   }
 }
 
+void
+Datastore::MaybeClose()
+{
+  AssertIsOnBackgroundThread();
+
+  if (!mPrepareDatastoreOps.Count() &&
+      !mPreparedDatastores.Count() &&
+      !mDatabases.Count()) {
+    Close();
+  }
+}
+
 void
 Datastore::ConnectionClosedCallback()
 {
@@ -2943,6 +3099,34 @@ Datastore::AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure)
   self->mConnection->Commit();
 }
 
+/*******************************************************************************
+ * PreparedDatastore
+ ******************************************************************************/
+
+void
+PreparedDatastore::Destroy()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(gPreparedDatastores);
+  MOZ_ASSERT(gPreparedDatastores->Get(mDatastoreId));
+
+  nsAutoPtr<PreparedDatastore> preparedDatastore;
+  gPreparedDatastores->Remove(mDatastoreId, &preparedDatastore);
+  MOZ_ASSERT(preparedDatastore);
+}
+
+// static
+void
+PreparedDatastore::TimerCallback(nsITimer* aTimer, void* aClosure)
+{
+  AssertIsOnBackgroundThread();
+
+  auto* self = static_cast<PreparedDatastore*>(aClosure);
+  MOZ_ASSERT(self);
+
+  self->Destroy();
+}
+
 /*******************************************************************************
  * Object
  ******************************************************************************/
@@ -3039,7 +3223,7 @@ Object::RecvPBackgroundLSDatabaseConstructor(
 
   auto* database = static_cast<Database*>(aActor);
 
-  database->SetActorAlive(preparedDatastore->ForgetDatastore());
+  database->SetActorAlive(preparedDatastore->GetDatastore());
 
   // It's possible that AbortOperations was called before the database actor
   // was created and became live. Let the child know that the database in no
@@ -3088,7 +3272,7 @@ Database::~Database()
 }
 
 void
-Database::SetActorAlive(already_AddRefed<Datastore>&& aDatastore)
+Database::SetActorAlive(Datastore* aDatastore)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!mActorWasAlive);
@@ -3098,7 +3282,7 @@ Database::SetActorAlive(already_AddRefed<Datastore>&& aDatastore)
   mActorWasAlive = true;
 #endif
 
-  mDatastore = std::move(aDatastore);
+  mDatastore = aDatastore;
 
   mDatastore->NoteLiveDatabase(this);
 
@@ -3558,6 +3742,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   , mPrivateBrowsingId(0)
   , mUsage(0)
   , mNestedState(NestedState::BeforeNesting)
+  , mDatabaseNotAvailable(false)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
 {
@@ -3725,6 +3910,8 @@ PrepareDatastoreOp::BeginDatastorePreparation()
   if (gDatastores && (mDatastore = gDatastores->Get(mOrigin))) {
     MOZ_ASSERT(!mDatastore->IsClosed());
 
+    mDatastore->NoteLivePrepareDatastoreOp(this);
+
     mState = State::SendingReadyMessage;
     mNestedState = NestedState::AfterNesting;
 
@@ -3856,60 +4043,56 @@ PrepareDatastoreOp::DatabaseWork()
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
-  nsCOMPtr<nsIFile> dbDirectory;
+  nsCOMPtr<nsIFile> directoryEntry;
   nsresult rv =
     quotaManager->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT,
                                             mSuffix,
                                             mGroup,
                                             mOrigin,
-                                            getter_AddRefs(dbDirectory));
+                                            mParams.createIfNotExists(),
+                                            getter_AddRefs(directoryEntry));
+  if (rv == NS_ERROR_NOT_AVAILABLE) {
+    return DatabaseNotAvailable();
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = dbDirectory->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
+  rv = directoryEntry->Append(NS_LITERAL_STRING(LS_DIRECTORY_NAME));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  bool exists;
-  rv = dbDirectory->Exists(&exists);
+  rv = EnsureDirectoryEntry(directoryEntry, /* aIsDirectory */ true);
+  if (rv == NS_ERROR_NOT_AVAILABLE) {
+    return DatabaseNotAvailable();
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  if (!exists) {
-    rv = dbDirectory->Create(nsIFile::DIRECTORY_TYPE, 0755);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-  }
-#ifdef DEBUG
-  else {
-    bool isDirectory;
-    MOZ_ASSERT(NS_SUCCEEDED(dbDirectory->IsDirectory(&isDirectory)));
-    MOZ_ASSERT(isDirectory);
-  }
-#endif
-
-  nsCOMPtr<nsIFile> dbFile;
-  rv = dbDirectory->Clone(getter_AddRefs(dbFile));
+  rv = directoryEntry->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = dbFile->Append(NS_LITERAL_STRING(DATA_FILE_NAME));
+  rv = EnsureDirectoryEntry(directoryEntry, /* aIsDirectory */ false);
+  if (rv == NS_ERROR_NOT_AVAILABLE) {
+    return DatabaseNotAvailable();
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = dbFile->GetPath(mDatabaseFilePath);
+  rv = directoryEntry->GetPath(mDatabaseFilePath);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
   nsCOMPtr<mozIStorageConnection> connection;
-  rv = CreateStorageConnection(dbFile, mOrigin, getter_AddRefs(connection));
+  rv = CreateStorageConnection(directoryEntry,
+                               mOrigin,
+                               getter_AddRefs(connection));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3923,7 +4106,7 @@ PrepareDatastoreOp::DatabaseWork()
   // connection thread which needs to open the same database.
   MOZ_ALWAYS_SUCCEEDS(connection->Close());
 
-  // Must set mState before dispatching otherwise we will race with the owning
+  // Must set this before dispatching otherwise we will race with the owning
   // thread.
   mNestedState = NestedState::BeginLoadData;
 
@@ -3935,6 +4118,63 @@ PrepareDatastoreOp::DatabaseWork()
   return NS_OK;
 }
 
+nsresult
+PrepareDatastoreOp::DatabaseNotAvailable()
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::DatabaseWorkOpen);
+
+  mDatabaseNotAvailable = true;
+
+  // Must set this before dispatching otherwise we will race with the owning
+  // thread.
+  mState = State::SendingReadyMessage;
+  mNestedState = NestedState::AfterNesting;
+
+  nsresult rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry, bool aIsDirectory)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aEntry);
+
+  bool exists;
+  nsresult rv = aEntry->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!exists) {
+    if (!mParams.createIfNotExists()) {
+      return NS_ERROR_NOT_AVAILABLE;
+    }
+
+    if (aIsDirectory) {
+      rv = aEntry->Create(nsIFile::DIRECTORY_TYPE, 0755);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+  }
+#ifdef DEBUG
+  else {
+    bool isDirectory;
+    MOZ_ASSERT(NS_SUCCEEDED(aEntry->IsDirectory(&isDirectory)));
+    MOZ_ASSERT(isDirectory == aIsDirectory);
+  }
+#endif
+
+  return NS_OK;
+}
+
 nsresult
 PrepareDatastoreOp::VerifyDatabaseInformation(mozIStorageConnection* aConnection)
 {
@@ -4064,6 +4304,17 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
   MOZ_ASSERT(mState == State::SendingResults);
   MOZ_ASSERT(NS_SUCCEEDED(ResultCode()));
 
+  if (mDatabaseNotAvailable) {
+    MOZ_ASSERT(!mParams.createIfNotExists());
+
+    LSRequestPrepareDatastoreResponse prepareDatastoreResponse;
+    prepareDatastoreResponse.datastoreId() = null_t();
+
+    aResponse = prepareDatastoreResponse;
+
+    return;
+  }
+
   if (!mDatastore) {
     mDatastore = new Datastore(mOrigin,
                                mPrivateBrowsingId,
@@ -4072,6 +4323,8 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
                                mConnection.forget(),
                                mValues);
 
+    mDatastore->NoteLivePrepareDatastoreOp(this);
+
     if (!gDatastores) {
       gDatastores = new DatastoreHashtable();
     }
@@ -4083,7 +4336,10 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
   uint64_t datastoreId = ++gLastDatastoreId;
 
   nsAutoPtr<PreparedDatastore> preparedDatastore(
-    new PreparedDatastore(mDatastore, mOrigin));
+    new PreparedDatastore(mDatastore,
+                          mOrigin,
+                          datastoreId,
+                          /* aForPreload */ !mParams.createIfNotExists()));
 
   if (!gPreparedDatastores) {
     gPreparedDatastores = new PreparedDatastoreHashtable();
@@ -4114,10 +4370,14 @@ PrepareDatastoreOp::Cleanup()
     if (NS_FAILED(ResultCode())) {
       MOZ_ASSERT(!mDatastore->IsClosed());
       MOZ_ASSERT(!mDatastore->HasLiveDatabases());
+      MOZ_ASSERT(!mDatastore->HasLivePreparedDatastores());
       mDatastore->Close();
     }
 
     // Make sure to release the datastore on this thread.
+
+    mDatastore->NoteFinishedPrepareDatastoreOp(this);
+
     mDatastore = nullptr;
 
     CleanupMetadata();
@@ -4137,8 +4397,10 @@ PrepareDatastoreOp::Cleanup()
     mConnection->Close(callback);
   } else {
     // If we don't have a connection, but we do have a directory lock then the
-    // operation must have failed too.
-    MOZ_ASSERT_IF(mDirectoryLock, NS_FAILED(ResultCode()));
+    // operation must have failed or we were preloading a datastore and there
+    // was no physical database on disk.
+    MOZ_ASSERT_IF(mDirectoryLock,
+                  NS_FAILED(ResultCode()) || mDatabaseNotAvailable);
 
     // There's no connection, so it's safe to release the directory lock and
     // unregister itself from the array.
@@ -4903,23 +5165,6 @@ QuotaClient::ShutdownWorkThreads()
   // When the last PrepareDatastoreOp finishes, the gPrepareDatastoreOps array
   // is destroyed.
 
-  // There may be datastores that are only held alive by prepared datastores
-  // (ones which have no live database actors). We need to explicitly close
-  // them here.
-  if (gDatastores) {
-    for (auto iter = gDatastores->ConstIter(); !iter.Done(); iter.Next()) {
-      Datastore* datastore = iter.Data();
-      MOZ_ASSERT(datastore);
-
-      if (!datastore->IsClosed() && !datastore->HasLiveDatabases()) {
-        datastore->Close();
-      }
-    }
-  }
-
-  // If database actors haven't been created yet, don't do anything special.
-  // We are shutting down and we can release prepared datastores immediatelly
-  // since database actors will never be created for them.
   if (gPreparedDatastores) {
     gPreparedDatastores->Clear();
     gPreparedDatastores = nullptr;
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 5cc79d1f0eaf..6af0533497fb 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -677,6 +677,7 @@ LSObject::EnsureDatabase()
 
   LSRequestPrepareDatastoreParams params;
   params.principalInfo() = *mPrincipalInfo;
+  params.createIfNotExists() = true;
 
   LSRequestResponse response;
 
@@ -691,7 +692,9 @@ LSObject::EnsureDatabase()
   const LSRequestPrepareDatastoreResponse& prepareDatastoreResponse =
     response.get_LSRequestPrepareDatastoreResponse();
 
-  uint64_t datastoreId = prepareDatastoreResponse.datastoreId();
+  const NullableDatastoreId& datastoreId = prepareDatastoreResponse.datastoreId();
+
+  MOZ_ASSERT(datastoreId.type() == NullableDatastoreId::Tuint64_t);
 
   // The datastore is now ready on the parent side (prepared by the asynchronous
   // request on the DOM File thread).
diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
index a9e2f18cd651..278b74b289fd 100644
--- a/dom/localstorage/LocalStorageManager2.cpp
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -14,6 +14,32 @@ namespace dom {
 
 namespace {
 
+class RequestResolver final
+  : public LSRequestChildCallback
+{
+  RefPtr<Promise> mPromise;
+
+public:
+  explicit RequestResolver(Promise* aPromise)
+    : mPromise(aPromise)
+  { }
+
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::RequestResolver, override);
+
+private:
+  ~RequestResolver() = default;
+
+  void
+  HandleResponse(nsresult aResponse);
+
+  void
+  HandleResponse(const NullableDatastoreId& aDatastoreId);
+
+  // LSRequestChildCallback
+  void
+  OnResponse(const LSRequestResponse& aResponse) override;
+};
+
 class SimpleRequestResolver final
   : public LSSimpleRequestChildCallback
 {
@@ -182,6 +208,44 @@ LocalStorageManager2::GetNextGenLocalStorageEnabled(bool* aResult)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocalStorageManager2::Preload(nsIPrincipal* aPrincipal,
+                              JSContext* aContext,
+                              nsISupports** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  nsresult rv;
+
+  RefPtr<Promise> promise;
+
+  if (aContext) {
+    rv = CreatePromise(aContext, getter_AddRefs(promise));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  LSRequestPrepareDatastoreParams params;
+  params.createIfNotExists() = false;
+
+  rv = CheckedPrincipalToPrincipalInfo(aPrincipal,
+                                       params.principalInfo());
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StartRequest(promise, params);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  promise.forget(_retval);
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 LocalStorageManager2::IsPreloaded(nsIPrincipal* aPrincipal,
                                   JSContext* aContext,
@@ -214,6 +278,29 @@ LocalStorageManager2::IsPreloaded(nsIPrincipal* aPrincipal,
   return NS_OK;
 }
 
+nsresult
+LocalStorageManager2::StartRequest(Promise* aPromise,
+                                   const LSRequestParams& aParams)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread();
+  if (NS_WARN_IF(!backgroundActor)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  RefPtr<RequestResolver> resolver = new RequestResolver(aPromise);
+
+  auto actor = new LSRequestChild(resolver);
+
+  if (!backgroundActor->SendPBackgroundLSRequestConstructor(actor, aParams)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
 nsresult
 LocalStorageManager2::StartSimpleRequest(Promise* aPromise,
                                          const LSSimpleRequestParams& aParams)
@@ -239,6 +326,60 @@ LocalStorageManager2::StartSimpleRequest(Promise* aPromise,
   return NS_OK;
 }
 
+void
+RequestResolver::HandleResponse(nsresult aResponse)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mPromise) {
+    return;
+  }
+
+  mPromise->MaybeReject(aResponse);
+}
+
+void
+RequestResolver::HandleResponse(const NullableDatastoreId& aDatastoreId)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  if (!mPromise) {
+    return;
+  }
+
+  switch (aDatastoreId.type()) {
+    case NullableDatastoreId::Tnull_t:
+      mPromise->MaybeResolve(JS::NullHandleValue);
+      break;
+
+    case NullableDatastoreId::Tuint64_t:
+      mPromise->MaybeResolve(aDatastoreId.get_uint64_t());
+      break;
+
+   default:
+      MOZ_CRASH("Unknown datastore id type!");
+  }
+}
+
+void
+RequestResolver::OnResponse(const LSRequestResponse& aResponse)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  switch (aResponse.type()) {
+    case LSRequestResponse::Tnsresult:
+      HandleResponse(aResponse.get_nsresult());
+      break;
+
+    case LSRequestResponse::TLSRequestPrepareDatastoreResponse:
+      HandleResponse(
+        aResponse.get_LSRequestPrepareDatastoreResponse().datastoreId());
+      break;
+   default:
+      MOZ_CRASH("Unknown response type!");
+  }
+}
+
 void
 SimpleRequestResolver::HandleResponse(nsresult aResponse)
 {
diff --git a/dom/localstorage/LocalStorageManager2.h b/dom/localstorage/LocalStorageManager2.h
index 3349b93fa8b4..6601466bd228 100644
--- a/dom/localstorage/LocalStorageManager2.h
+++ b/dom/localstorage/LocalStorageManager2.h
@@ -13,6 +13,7 @@
 namespace mozilla {
 namespace dom {
 
+class LSRequestParams;
 class LSSimpleRequestParams;
 class Promise;
 
@@ -30,6 +31,10 @@ public:
 private:
   ~LocalStorageManager2();
 
+  nsresult
+  StartRequest(Promise* aPromise,
+               const LSRequestParams& aParams);
+
   nsresult
   StartSimpleRequest(Promise* aPromise,
                      const LSSimpleRequestParams& aParams);
diff --git a/dom/localstorage/PBackgroundLSRequest.ipdl b/dom/localstorage/PBackgroundLSRequest.ipdl
index e87bcea22729..7432bfff6ecd 100644
--- a/dom/localstorage/PBackgroundLSRequest.ipdl
+++ b/dom/localstorage/PBackgroundLSRequest.ipdl
@@ -4,12 +4,21 @@
 
 include protocol PBackground;
 
+using struct mozilla::null_t
+  from "ipc/IPCMessageUtils.h";
+
 namespace mozilla {
 namespace dom {
 
+union NullableDatastoreId
+{
+  null_t;
+  uint64_t;
+};
+
 struct LSRequestPrepareDatastoreResponse
 {
-  uint64_t datastoreId;
+  NullableDatastoreId datastoreId;
 };
 
 struct LSRequestPrepareObserverResponse
diff --git a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
index 75bdc3f56d5a..3ed292a15d51 100644
--- a/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
+++ b/dom/localstorage/PBackgroundLSSharedTypes.ipdlh
@@ -10,6 +10,7 @@ namespace dom {
 struct LSRequestPrepareDatastoreParams
 {
   PrincipalInfo principalInfo;
+  bool createIfNotExists;
 };
 
 struct LSRequestPrepareObserverParams
diff --git a/dom/localstorage/nsILocalStorageManager.idl b/dom/localstorage/nsILocalStorageManager.idl
index cab473000b59..c3791d81f9b7 100644
--- a/dom/localstorage/nsILocalStorageManager.idl
+++ b/dom/localstorage/nsILocalStorageManager.idl
@@ -13,6 +13,9 @@ interface nsILocalStorageManager : nsISupports
 {
   readonly attribute boolean nextGenLocalStorageEnabled;
 
+  [implicit_jscontext] nsISupports
+  preload(in nsIPrincipal aPrincipal);
+
   [implicit_jscontext] nsISupports
   isPreloaded(in nsIPrincipal aPrincipal);
 };
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 374c11de6149..2dc036baf324 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -5297,6 +5297,7 @@ QuotaManager::EnsureOriginIsInitialized(PersistenceType aPersistenceType,
                                         const nsACString& aSuffix,
                                         const nsACString& aGroup,
                                         const nsACString& aOrigin,
+                                        bool aCreateIfNotExists,
                                         nsIFile** aDirectory)
 {
   AssertIsOnIOThread();
@@ -5308,8 +5309,12 @@ QuotaManager::EnsureOriginIsInitialized(PersistenceType aPersistenceType,
                                                   aSuffix,
                                                   aGroup,
                                                   aOrigin,
+                                                  aCreateIfNotExists,
                                                   getter_AddRefs(directory),
                                                   &created);
+  if (rv == NS_ERROR_NOT_AVAILABLE) {
+    return rv;
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -5324,6 +5329,7 @@ QuotaManager::EnsureOriginIsInitializedInternal(
                                                const nsACString& aSuffix,
                                                const nsACString& aGroup,
                                                const nsACString& aOrigin,
+                                               bool aCreateIfNotExists,
                                                nsIFile** aDirectory,
                                                bool* aCreated)
 {
@@ -5354,7 +5360,10 @@ QuotaManager::EnsureOriginIsInitializedInternal(
   }
 
   bool created;
-  rv = EnsureOriginDirectory(directory, &created);
+  rv = EnsureOriginDirectory(directory, aCreateIfNotExists, &created);
+  if (rv == NS_ERROR_NOT_AVAILABLE) {
+    return rv;
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -5475,6 +5484,7 @@ QuotaManager::EnsureTemporaryStorageIsInitialized()
 
 nsresult
 QuotaManager::EnsureOriginDirectory(nsIFile* aDirectory,
+                                    bool aCreateIfNotExists,
                                     bool* aCreated)
 {
   AssertIsOnIOThread();
@@ -5488,6 +5498,10 @@ QuotaManager::EnsureOriginDirectory(nsIFile* aDirectory,
   }
 
   if (!exists) {
+    if (!aCreateIfNotExists) {
+      return NS_ERROR_NOT_AVAILABLE;
+    }
+
     nsString leafName;
     rv = aDirectory->GetLeafName(leafName);
     if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -7551,12 +7565,14 @@ InitOriginOp::DoDirectoryWork(QuotaManager* aQuotaManager)
   nsCOMPtr<nsIFile> directory;
   bool created;
   nsresult rv =
-    aQuotaManager->EnsureOriginIsInitializedInternal(mPersistenceType.Value(),
-                                                     mSuffix,
-                                                     mGroup,
-                                                     mOriginScope.GetOrigin(),
-                                                     getter_AddRefs(directory),
-                                                     &created);
+    aQuotaManager->EnsureOriginIsInitializedInternal(
+                                                  mPersistenceType.Value(),
+                                                  mSuffix,
+                                                  mGroup,
+                                                  mOriginScope.GetOrigin(),
+                                                  /* aCreateIfNotExists */ true,
+                                                  getter_AddRefs(directory),
+                                                  &created);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -8112,7 +8128,9 @@ PersistOp::DoDirectoryWork(QuotaManager* aQuotaManager)
   }
 
   bool created;
-  rv = aQuotaManager->EnsureOriginDirectory(directory, &created);
+  rv = aQuotaManager->EnsureOriginDirectory(directory,
+                                            /* aCreateIfNotExists */ true,
+                                            &created);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index bc168b708c42..3905155abdd7 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -289,6 +289,7 @@ public:
                             const nsACString& aSuffix,
                             const nsACString& aGroup,
                             const nsACString& aOrigin,
+                            bool aCreateIfNotExists,
                             nsIFile** aDirectory);
 
   nsresult
@@ -296,6 +297,7 @@ public:
                                     const nsACString& aSuffix,
                                     const nsACString& aGroup,
                                     const nsACString& aOrigin,
+                                    bool aCreateIfNotExists,
                                     nsIFile** aDirectory,
                                     bool* aCreated);
 
@@ -304,6 +306,7 @@ public:
 
   nsresult
   EnsureOriginDirectory(nsIFile* aDirectory,
+                        bool aCreateIfNotExists,
                         bool* aCreated);
 
   void
diff --git a/dom/simpledb/ActorsParent.cpp b/dom/simpledb/ActorsParent.cpp
index c87c05d3ee52..22c9250c170c 100644
--- a/dom/simpledb/ActorsParent.cpp
+++ b/dom/simpledb/ActorsParent.cpp
@@ -1352,6 +1352,7 @@ OpenOp::DatabaseWork()
                                             mSuffix,
                                             mGroup,
                                             mOrigin,
+                                            /* aCreateIfNotExists */ true,
                                             getter_AddRefs(dbDirectory));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
diff --git a/dom/storage/LocalStorageManager.cpp b/dom/storage/LocalStorageManager.cpp
index b9917eb624e9..5a57a650fb44 100644
--- a/dom/storage/LocalStorageManager.cpp
+++ b/dom/storage/LocalStorageManager.cpp
@@ -381,6 +381,18 @@ LocalStorageManager::GetNextGenLocalStorageEnabled(bool* aResult)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocalStorageManager::Preload(nsIPrincipal* aPrincipal,
+                             JSContext* aContext,
+                             nsISupports** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+  MOZ_ASSERT(_retval);
+
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
 NS_IMETHODIMP
 LocalStorageManager::IsPreloaded(nsIPrincipal* aPrincipal,
                                  JSContext* aContext,

From a5c446d3d9d50e4a0adb96652f099baf752427c9 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:28 +0100
Subject: [PATCH 37/78] Bug 1286798 - Part 23: Fix
 GetOrCreateForCurrentThread() when the runnable for calling
 SendInitBackground() on the main thread was already dispatched, but the main
 thread then entered a nested event loop; r=asuth

See the big comment in the code for more details.
---
 ipc/glue/BackgroundImpl.cpp | 195 ++++++++++++++++++++++++++++++++----
 1 file changed, 174 insertions(+), 21 deletions(-)

diff --git a/ipc/glue/BackgroundImpl.cpp b/ipc/glue/BackgroundImpl.cpp
index e83582cfeffe..ea7ce8479aa0 100644
--- a/ipc/glue/BackgroundImpl.cpp
+++ b/ipc/glue/BackgroundImpl.cpp
@@ -24,6 +24,8 @@
 #include "mozilla/dom/ContentChild.h"
 #include "mozilla/dom/ContentParent.h"
 #include "mozilla/dom/File.h"
+#include "mozilla/dom/WorkerPrivate.h"
+#include "mozilla/dom/WorkerRef.h"
 #include "mozilla/ipc/ProtocolTypes.h"
 #include "nsAutoPtr.h"
 #include "nsCOMPtr.h"
@@ -293,7 +295,7 @@ class ChildImpl final : public BackgroundChildImpl
   typedef mozilla::ipc::Transport Transport;
 
   class ShutdownObserver;
-  class ActorCreatedRunnable;
+  class SendInitBackgroundRunnable;
 
   // A thread-local index that is not valid.
   static const unsigned int kBadThreadLocalIndex =
@@ -313,6 +315,7 @@ class ChildImpl final : public BackgroundChildImpl
     }
 
     RefPtr<ChildImpl> mActor;
+    RefPtr<SendInitBackgroundRunnable> mSendInitBackgroundRunnable;
     nsAutoPtr<BackgroundChildImpl::ThreadLocal> mConsumerThreadLocal;
 #ifdef DEBUG
     bool mClosed;
@@ -410,20 +413,7 @@ private:
   GetThreadLocalForCurrentThread();
 
   static void
-  ThreadLocalDestructor(void* aThreadLocal)
-  {
-    auto threadLocalInfo = static_cast<ThreadLocalInfo*>(aThreadLocal);
-
-    if (threadLocalInfo) {
-      MOZ_ASSERT(threadLocalInfo->mClosed);
-
-      if (threadLocalInfo->mActor) {
-        threadLocalInfo->mActor->Close();
-        threadLocalInfo->mActor->AssertActorDestroyed();
-      }
-      delete threadLocalInfo;
-    }
-  }
+  ThreadLocalDestructor(void* aThreadLocal);
 
   // This class is reference counted.
   ~ChildImpl()
@@ -623,6 +613,43 @@ private:
   }
 };
 
+class ChildImpl::SendInitBackgroundRunnable final
+  : public CancelableRunnable
+{
+  nsCOMPtr<nsISerialEventTarget> mOwningEventTarget;
+  RefPtr<StrongWorkerRef> mWorkerRef;
+  Endpoint<PBackgroundParent> mParent;
+  mozilla::Mutex mMutex;
+  bool mSentInitBackground;
+
+public:
+  static already_AddRefed<SendInitBackgroundRunnable>
+  Create(Endpoint<PBackgroundParent>&& aParent);
+
+  void
+  ClearEventTarget()
+  {
+    mWorkerRef = nullptr;
+
+    mozilla::MutexAutoLock lock(mMutex);
+    mOwningEventTarget = nullptr;
+  }
+
+private:
+  explicit SendInitBackgroundRunnable(Endpoint<PBackgroundParent>&& aParent)
+    : CancelableRunnable("Background::ChildImpl::SendInitBackgroundRunnable")
+    , mOwningEventTarget(GetCurrentThreadSerialEventTarget())
+    , mParent(std::move(aParent))
+    , mMutex("SendInitBackgroundRunnable::mMutex")
+    , mSentInitBackground(false)
+  { }
+
+  ~SendInitBackgroundRunnable()
+  { }
+
+  NS_DECL_NSIRUNNABLE
+};
+
 } // namespace
 
 namespace mozilla {
@@ -1489,6 +1516,29 @@ ChildImpl::GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget)
   }
 
   if (threadLocalInfo->mActor) {
+    RefPtr<SendInitBackgroundRunnable>& runnable =
+      threadLocalInfo->mSendInitBackgroundRunnable;
+
+    if (aMainEventTarget && runnable) {
+      // The SendInitBackgroundRunnable was already dispatched to the main
+      // thread to finish initialization of a new background child actor.
+      // However, the caller passed a custom main event target which indicates
+      // that synchronous blocking of the main thread is happening (done by
+      // creating a nested event target and spinning the event loop).
+      // It can happen that the SendInitBackgroundRunnable didn't have a chance
+      // to run before the synchronous blocking has occured. Unblocking of the
+      // main thread can depend on an IPC message received on this thread, so
+      // we have to dispatch the SendInitBackgroundRunnable to the custom main
+      // event target too, otherwise IPC will be only queueing messages on this
+      // thread. The runnable will run twice in the end, but that's a harmless
+      // race between the main and nested event queue of the main thread.
+      // There's a guard in the runnable implementation for calling
+      // SendInitBackground only once.
+
+      MOZ_ALWAYS_SUCCEEDS(aMainEventTarget->Dispatch(runnable,
+                                                     NS_DISPATCH_NORMAL));
+    }
+
     return threadLocalInfo->mActor;
   }
 
@@ -1525,6 +1575,14 @@ ChildImpl::GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget)
     return nullptr;
   }
 
+  RefPtr<SendInitBackgroundRunnable> runnable;
+  if (!NS_IsMainThread()) {
+    runnable = SendInitBackgroundRunnable::Create(std::move(parent));
+    if (!runnable) {
+      return nullptr;
+    }
+  }
+
   RefPtr<ChildImpl> strongActor = new ChildImpl();
 
   if (!child.Bind(strongActor)) {
@@ -1541,18 +1599,14 @@ ChildImpl::GetOrCreateForCurrentThread(nsIEventTarget* aMainEventTarget)
       return nullptr;
     }
   } else {
-    nsCOMPtr<nsIRunnable> runnable =
-      NewRunnableMethod<Endpoint<PBackgroundParent>&&>(
-        "dom::ContentChild::SendInitBackground",
-        content,
-        &ContentChild::SendInitBackground,
-        std::move(parent));
     if (aMainEventTarget) {
       MOZ_ALWAYS_SUCCEEDS(aMainEventTarget->Dispatch(runnable,
                                                      NS_DISPATCH_NORMAL));
     } else {
       MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(runnable));
     }
+
+    threadLocalInfo->mSendInitBackgroundRunnable = runnable;
   }
 
   RefPtr<ChildImpl>& actor = threadLocalInfo->mActor;
@@ -1611,6 +1665,28 @@ ChildImpl::GetThreadLocalForCurrentThread()
   return threadLocalInfo->mConsumerThreadLocal;
 }
 
+// static
+void
+ChildImpl::ThreadLocalDestructor(void* aThreadLocal)
+{
+  auto threadLocalInfo = static_cast<ThreadLocalInfo*>(aThreadLocal);
+
+  if (threadLocalInfo) {
+    MOZ_ASSERT(threadLocalInfo->mClosed);
+
+    if (threadLocalInfo->mActor) {
+      threadLocalInfo->mActor->Close();
+      threadLocalInfo->mActor->AssertActorDestroyed();
+    }
+
+    if (threadLocalInfo->mSendInitBackgroundRunnable) {
+      threadLocalInfo->mSendInitBackgroundRunnable->ClearEventTarget();
+    }
+
+    delete threadLocalInfo;
+  }
+}
+
 void
 ChildImpl::ActorDestroy(ActorDestroyReason aWhy)
 {
@@ -1638,3 +1714,80 @@ ChildImpl::ShutdownObserver::Observe(nsISupports* aSubject,
 
   return NS_OK;
 }
+
+// static
+already_AddRefed<ChildImpl::SendInitBackgroundRunnable>
+ChildImpl::
+SendInitBackgroundRunnable::Create(Endpoint<PBackgroundParent>&& aParent)
+{
+  MOZ_ASSERT(!NS_IsMainThread());
+
+  RefPtr<SendInitBackgroundRunnable> runnable =
+    new SendInitBackgroundRunnable(std::move(aParent));
+
+  WorkerPrivate* workerPrivate = mozilla::dom::GetCurrentThreadWorkerPrivate();
+  if (!workerPrivate) {
+    return runnable.forget();
+  }
+
+  workerPrivate->AssertIsOnWorkerThread();
+
+  runnable->mWorkerRef =
+    StrongWorkerRef::Create(workerPrivate,
+                            "ChildImpl::SendInitBackgroundRunnable");
+  if (NS_WARN_IF(!runnable->mWorkerRef)) {
+    return nullptr;
+  }
+
+  return runnable.forget();
+}
+
+NS_IMETHODIMP
+ChildImpl::
+SendInitBackgroundRunnable::Run()
+{
+  if (NS_IsMainThread()) {
+    if (mSentInitBackground) {
+      return NS_OK;
+    }
+
+    mSentInitBackground = true;
+
+    RefPtr<ContentChild> content = ContentChild::GetSingleton();
+    MOZ_ASSERT(content);
+
+    if (!content->SendInitBackground(std::move(mParent))) {
+      MOZ_CRASH("Failed to create top level actor!");
+    }
+
+    nsCOMPtr<nsISerialEventTarget> owningEventTarget;
+    {
+      mozilla::MutexAutoLock lock(mMutex);
+      owningEventTarget = mOwningEventTarget;
+    }
+
+    if (!owningEventTarget) {
+      return NS_OK;
+    }
+
+    nsresult rv = owningEventTarget->Dispatch(this, NS_DISPATCH_NORMAL);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    return NS_OK;
+  }
+
+  ClearEventTarget();
+
+  auto threadLocalInfo =
+    static_cast<ThreadLocalInfo*>(PR_GetThreadPrivate(sThreadLocalIndex));
+
+  if (!threadLocalInfo) {
+    return NS_OK;
+  }
+
+  threadLocalInfo->mSendInitBackgroundRunnable = nullptr;
+
+  return NS_OK;
+}

From 480f7ccea16e18b4cee3baa528aeed95dcba4cbd Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:31 +0100
Subject: [PATCH 38/78] Bug 1286798 - Part 24: A new exclusive directory lock
 shouldn't invalidate existing internal directory locks; r=asuth

This fixes an intermittent failure when multiple clearStoragesForPrincipal() are called at the same time.
---
 dom/quota/ActorsParent.cpp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 2dc036baf324..315aeffae083 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -5262,9 +5262,9 @@ QuotaManager::OpenDirectoryInternal(const Nullable<PersistenceType>& aPersistenc
     lock->GetBlockedOnLocks();
 
   for (DirectoryLockImpl* blockedOnLock : blockedOnLocks) {
-    blockedOnLock->Invalidate();
-
     if (!blockedOnLock->IsInternal()) {
+      blockedOnLock->Invalidate();
+
       MOZ_ASSERT(!blockedOnLock->GetClientType().IsNull());
       Client::Type clientType = blockedOnLock->GetClientType().Value();
       MOZ_ASSERT(clientType < Client::TypeMax());

From 7981be440b61345cf63d63284a7aed03f3ee8382 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:34 +0100
Subject: [PATCH 39/78] Bug 1286798 - Part 25: Add checks for the group and
 global limit; r=asuth

---
 .eslintignore                                 |   1 +
 dom/cache/FileUtils.cpp                       |   2 +-
 dom/localstorage/ActorsParent.cpp             | 203 +++++++++++++++---
 dom/localstorage/moz.build                    |   4 +
 dom/localstorage/test/unit/head.js            | 144 +++++++++++++
 dom/localstorage/test/unit/test_eviction.js   |  91 ++++++++
 dom/localstorage/test/unit/test_groupLimit.js |  77 +++++++
 dom/localstorage/test/unit/xpcshell.ini       |   9 +
 dom/quota/ActorsParent.cpp                    |  42 ++--
 dom/quota/QuotaManager.h                      |   2 +
 10 files changed, 536 insertions(+), 39 deletions(-)
 create mode 100644 dom/localstorage/test/unit/head.js
 create mode 100644 dom/localstorage/test/unit/test_eviction.js
 create mode 100644 dom/localstorage/test/unit/test_groupLimit.js
 create mode 100644 dom/localstorage/test/unit/xpcshell.ini

diff --git a/.eslintignore b/.eslintignore
index 1ba5b4a56970..680563f86fcb 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -190,6 +190,7 @@ dom/grid/**
 dom/html/**
 dom/ipc/**
 dom/jsurl/**
+dom/localstorage/**
 dom/manifest/**
 dom/media/test/**
 dom/media/tests/**
diff --git a/dom/cache/FileUtils.cpp b/dom/cache/FileUtils.cpp
index f51621d7a448..8b1f0eb675c1 100644
--- a/dom/cache/FileUtils.cpp
+++ b/dom/cache/FileUtils.cpp
@@ -294,7 +294,7 @@ BodyMaybeUpdatePaddingSize(const QuotaInfo& aQuotaInfo, nsIFile* aBaseDir,
   int64_t fileSize = 0;
   RefPtr<QuotaObject> quotaObject =
     quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT, aQuotaInfo.mGroup,
-                                 aQuotaInfo.mOrigin, bodyFile, &fileSize);
+                                 aQuotaInfo.mOrigin, bodyFile, -1, &fileSize);
   MOZ_DIAGNOSTIC_ASSERT(quotaObject);
   MOZ_DIAGNOSTIC_ASSERT(fileSize >= 0);
   // XXXtt: bug: https://bugzilla.mozilla.org/show_bug.cgi?id=1422815
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 013d3f97164b..a58bdb4a9a2a 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -21,6 +21,7 @@
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestParent.h"
 #include "mozilla/dom/quota/QuotaManager.h"
+#include "mozilla/dom/quota/QuotaObject.h"
 #include "mozilla/dom/quota/UsageInfo.h"
 #include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/ipc/PBackgroundParent.h"
@@ -862,6 +863,7 @@ class Datastore final
 {
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
+  RefPtr<QuotaObject> mQuotaObject;
   nsCOMPtr<nsITimer> mAutoCommitTimer;
   nsCOMPtr<nsIRunnable> mCompleteCallback;
   nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
@@ -880,6 +882,7 @@ public:
             int64_t aUsage,
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
+            already_AddRefed<QuotaObject>&& aQuotaObject,
             nsDataHashtable<nsStringHashKey, nsString>& aValues);
 
   const nsCString&
@@ -1436,6 +1439,10 @@ class PrepareDatastoreOp
   bool mRequestedDirectoryLock;
   bool mInvalidated;
 
+#ifdef DEBUG
+  int64_t mDEBUGUsage;
+#endif
+
 public:
   PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
                      const LSRequestParams& aParams);
@@ -1504,7 +1511,9 @@ private:
   DatabaseNotAvailable();
 
   nsresult
-  EnsureDirectoryEntry(nsIFile* aEntry, bool aDirectory);
+  EnsureDirectoryEntry(nsIFile* aEntry,
+                       bool aDirectory,
+                       bool* aAlreadyExisted = nullptr);
 
   nsresult
   VerifyDatabaseInformation(mozIStorageConnection* aConnection);
@@ -1826,6 +1835,11 @@ StaticAutoPtr<ObserverHashtable> gObservers;
 
 Atomic<uint32_t, Relaxed> gOriginLimitKB(kDefaultOriginLimitKB);
 
+typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
+
+// Can only be touched on the Quota Manager I/O thread.
+StaticAutoPtr<UsageHashtable> gUsages;
+
 bool
 IsOnConnectionThread()
 {
@@ -1840,6 +1854,19 @@ AssertIsOnConnectionThread()
   gConnectionThread->AssertIsOnConnectionThread();
 }
 
+void
+InitUsageForOrigin(const nsACString& aOrigin, int64_t aUsage)
+{
+  AssertIsOnIOThread();
+
+  if (!gUsages) {
+    gUsages = new UsageHashtable();
+  }
+
+  MOZ_ASSERT(!gUsages->Contains(aOrigin));
+  gUsages->Put(aOrigin, aUsage);
+}
+
 } // namespace
 
 /*******************************************************************************
@@ -2623,9 +2650,11 @@ Datastore::Datastore(const nsACString& aOrigin,
                      int64_t aUsage,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
+                     already_AddRefed<QuotaObject>&& aQuotaObject,
                      nsDataHashtable<nsStringHashKey, nsString>& aValues)
   : mDirectoryLock(std::move(aDirectoryLock))
   , mConnection(std::move(aConnection))
+  , mQuotaObject(std::move(aQuotaObject))
   , mOrigin(aOrigin)
   , mPrivateBrowsingId(aPrivateBrowsingId)
   , mUsage(aUsage)
@@ -2654,6 +2683,7 @@ Datastore::Close()
 
   if (IsPersistent()) {
     MOZ_ASSERT(mConnection);
+    MOZ_ASSERT(mQuotaObject);
 
     if (mConnection->InTransaction()) {
       MOZ_ASSERT(mAutoCommitTimer);
@@ -2673,6 +2703,7 @@ Datastore::Close()
     mConnection->Close(callback);
   } else {
     MOZ_ASSERT(!mConnection);
+    MOZ_ASSERT(!mQuotaObject);
 
     // There's no connection, so it's safe to release the directory lock and
     // unregister itself from the hashtable.
@@ -2987,8 +3018,12 @@ Datastore::ConnectionClosedCallback()
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(mDirectoryLock);
   MOZ_ASSERT(mConnection);
+  MOZ_ASSERT(mQuotaObject);
   MOZ_ASSERT(mClosed);
 
+  // Release the quota object first.
+  mQuotaObject = nullptr;
+
   // Now it's safe to release the directory lock and unregister itself from
   // the hashtable.
 
@@ -3021,13 +3056,42 @@ Datastore::UpdateUsage(int64_t aDelta)
 {
   AssertIsOnBackgroundThread();
 
+  // Check internal LocalStorage origin limit.
   int64_t newUsage = mUsage + aDelta;
   if (newUsage > gOriginLimitKB * 1024) {
     return false;
   }
 
+  // Check QuotaManager limits (group and global limit).
+  if (IsPersistent()) {
+    MOZ_ASSERT(mQuotaObject);
+
+    if (!mQuotaObject->MaybeUpdateSize(newUsage, /* aTruncate */ true)) {
+      return false;
+    }
+
+  }
+
+  // Quota checks passed, set new usage.
+
   mUsage = newUsage;
 
+  if (IsPersistent()) {
+    RefPtr<Runnable> runnable = NS_NewRunnableFunction(
+      "Datastore::UpdateUsage",
+      [origin = mOrigin, newUsage] () {
+        MOZ_ASSERT(gUsages);
+        MOZ_ASSERT(gUsages->Contains(origin));
+        gUsages->Put(origin, newUsage);
+      });
+
+    QuotaManager* quotaManager = QuotaManager::Get();
+    MOZ_ASSERT(quotaManager);
+
+    MOZ_ALWAYS_SUCCEEDS(
+      quotaManager->IOThread()->Dispatch(runnable, NS_DISPATCH_NORMAL));
+  }
+
   return true;
 }
 
@@ -3745,6 +3809,9 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   , mDatabaseNotAvailable(false)
   , mRequestedDirectoryLock(false)
   , mInvalidated(false)
+#ifdef DEBUG
+  , mDEBUGUsage(0)
+#endif
 {
   MOZ_ASSERT(aParams.type() ==
                LSRequestParams::TLSRequestPrepareDatastoreParams);
@@ -4076,7 +4143,10 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
-  rv = EnsureDirectoryEntry(directoryEntry, /* aIsDirectory */ false);
+  bool alreadyExisted;
+  rv = EnsureDirectoryEntry(directoryEntry,
+                            /* aIsDirectory */ false,
+                            &alreadyExisted);
   if (rv == NS_ERROR_NOT_AVAILABLE) {
     return DatabaseNotAvailable();
   }
@@ -4084,6 +4154,13 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
+  if (alreadyExisted) {
+    MOZ_ASSERT(gUsages);
+    MOZ_ASSERT(gUsages->Get(mOrigin, &mUsage));
+  } else {
+    InitUsageForOrigin(mOrigin, 0);
+  }
+
   rv = directoryEntry->GetPath(mDatabaseFilePath);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -4141,7 +4218,9 @@ PrepareDatastoreOp::DatabaseNotAvailable()
 }
 
 nsresult
-PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry, bool aIsDirectory)
+PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry,
+                                         bool aIsDirectory,
+                                         bool* aAlreadyExisted)
 {
   AssertIsOnIOThread();
   MOZ_ASSERT(aEntry);
@@ -4172,6 +4251,9 @@ PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry, bool aIsDirectory)
   }
 #endif
 
+  if (aAlreadyExisted) {
+    *aAlreadyExisted = exists;
+  }
   return NS_OK;
 }
 
@@ -4316,11 +4398,30 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
   }
 
   if (!mDatastore) {
+    MOZ_ASSERT(mUsage == mDEBUGUsage);
+
+    RefPtr<QuotaObject> quotaObject;
+
+    if (mPrivateBrowsingId == 0) {
+      MOZ_ASSERT(!mDatabaseFilePath.IsEmpty());
+
+      QuotaManager* quotaManager = QuotaManager::Get();
+      MOZ_ASSERT(quotaManager);
+
+      quotaObject = quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT,
+                                                 mGroup,
+                                                 mOrigin,
+                                                 mDatabaseFilePath,
+                                                 mUsage);
+      MOZ_ASSERT(quotaObject);
+    }
+
     mDatastore = new Datastore(mOrigin,
                                mPrivateBrowsingId,
                                mUsage,
                                mDirectoryLock.forget(),
                                mConnection.forget(),
+                               quotaObject.forget(),
                                mValues);
 
     mDatastore->NoteLivePrepareDatastoreOp(this);
@@ -4541,7 +4642,9 @@ LoadDataOp::DoDatastoreWork()
     }
 
     mPrepareDatastoreOp->mValues.Put(key, value);
-    mPrepareDatastoreOp->mUsage += key.Length() + value.Length();
+#ifdef DEBUG
+    mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
+#endif
   }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -4912,28 +5015,7 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
                         UsageInfo* aUsageInfo)
 {
   AssertIsOnIOThread();
-
-  if (!aUsageInfo) {
-    return NS_OK;
-  }
-
-  return GetUsageForOrigin(aPersistenceType,
-                           aGroup,
-                           aOrigin,
-                           aCanceled,
-                           aUsageInfo);
-}
-
-nsresult
-QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
-                               const nsACString& aGroup,
-                               const nsACString& aOrigin,
-                               const AtomicBool& aCanceled,
-                               UsageInfo* aUsageInfo)
-{
-  AssertIsOnIOThread();
   MOZ_ASSERT(aPersistenceType == PERSISTENCE_TYPE_DEFAULT);
-  MOZ_ASSERT(aUsageInfo);
 
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
@@ -4991,7 +5073,42 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
     }
 
     // TODO: Use a special file that contains logical size of the database.
-    //       For now, don't add to origin usage.
+    //       For now, get the usage from the database.
+
+    nsCOMPtr<mozIStorageConnection> connection;
+    rv = CreateStorageConnection(file, aOrigin, getter_AddRefs(connection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsCOMPtr<mozIStorageStatement> stmt;
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "SELECT sum(length(key) + length(value)) "
+      "FROM data"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    bool hasResult;
+    rv = stmt->ExecuteStep(&hasResult);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (NS_WARN_IF(!hasResult)) {
+      return NS_ERROR_FAILURE;
+    }
+
+    int64_t usage;
+    rv = stmt->GetInt64(0, &usage);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    InitUsageForOrigin(aOrigin, usage);
+
+    aUsageInfo->AppendToDatabaseUsage(uint64_t(usage));
   }
 
   // Report unknown files, don't fail, just warn.
@@ -5051,17 +5168,51 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
   return NS_OK;
 }
 
+nsresult
+QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
+                               const nsACString& aGroup,
+                               const nsACString& aOrigin,
+                               const AtomicBool& aCanceled,
+                               UsageInfo* aUsageInfo)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aPersistenceType == PERSISTENCE_TYPE_DEFAULT);
+  MOZ_ASSERT(aUsageInfo);
+
+  // We can't open the database at this point, since it can be already used
+  // by the connection thread. Use the cached value instead.
+
+  if (gUsages) {
+    int64_t usage;
+    if (gUsages->Get(aOrigin, &usage)) {
+      aUsageInfo->AppendToDatabaseUsage(usage);
+    }
+  }
+
+  return NS_OK;
+}
+
 void
 QuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType,
                                     const nsACString& aOrigin)
 {
   AssertIsOnIOThread();
+
+  if (aPersistenceType != PERSISTENCE_TYPE_DEFAULT) {
+    return;
+  }
+
+  if (gUsages) {
+    gUsages->Remove(aOrigin);
+  }
 }
 
 void
 QuotaClient::ReleaseIOThreadObjects()
 {
   AssertIsOnIOThread();
+
+  gUsages = nullptr;
 }
 
 void
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index 66e58fd81dd3..528c057258e6 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -4,6 +4,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+XPCSHELL_TESTS_MANIFESTS += [
+    'test/unit/xpcshell.ini'
+]
+
 XPIDL_SOURCES += [
     'nsILocalStorageManager.idl',
 ]
diff --git a/dom/localstorage/test/unit/head.js b/dom/localstorage/test/unit/head.js
new file mode 100644
index 000000000000..15320313f2eb
--- /dev/null
+++ b/dom/localstorage/test/unit/head.js
@@ -0,0 +1,144 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+const NS_ERROR_DOM_QUOTA_EXCEEDED_ERR = 22;
+
+ChromeUtils.import("resource://gre/modules/Services.jsm");
+
+function is(a, b, msg)
+{
+  Assert.equal(a, b, msg);
+}
+
+function ok(cond, msg)
+{
+  Assert.ok(!!cond, msg);
+}
+
+function run_test()
+{
+  runTest();
+};
+
+if (!this.runTest) {
+  this.runTest = function()
+  {
+    do_get_profile();
+
+    enableTesting();
+
+    do_test_pending();
+    testGenerator.next();
+  }
+}
+
+function finishTest()
+{
+  resetTesting();
+
+  executeSoon(function() {
+    do_test_finished();
+  })
+}
+
+function continueToNextStepSync()
+{
+  testGenerator.next();
+}
+
+function enableTesting()
+{
+  Services.prefs.setBoolPref("dom.storage.testing", true);
+  Services.prefs.setBoolPref("dom.quotaManager.testing", true);
+}
+
+function resetTesting()
+{
+  Services.prefs.clearUserPref("dom.quotaManager.testing");
+  Services.prefs.clearUserPref("dom.storage.testing");
+}
+
+function setGlobalLimit(globalLimit)
+{
+  Services.prefs.setIntPref("dom.quotaManager.temporaryStorage.fixedLimit",
+                            globalLimit);
+}
+
+function resetGlobalLimit()
+{
+  Services.prefs.clearUserPref("dom.quotaManager.temporaryStorage.fixedLimit");
+}
+
+function setOriginLimit(originLimit)
+{
+  Services.prefs.setIntPref("dom.storage.default_quota", originLimit);
+}
+
+function resetOriginLimit()
+{
+  Services.prefs.clearUserPref("dom.storage.default_quota");
+}
+
+function getOriginUsage(principal, callback)
+{
+  let request = Services.qms.getUsageForPrincipal(principal, callback);
+  request.callback = callback;
+
+  return request;
+}
+
+function clear(callback)
+{
+  let request = Services.qms.clear();
+  request.callback = callback;
+
+  return request;
+}
+
+function resetOrigin(principal, callback)
+{
+  let request =
+    Services.qms.resetStoragesForPrincipal(principal, "default", "ls");
+  request.callback = callback;
+
+  return request;
+}
+
+function repeatChar(count, ch) {
+  if (count == 0) {
+    return "";
+  }
+
+  let result = ch;
+  let count2 = count / 2;
+
+  // Double the input until it is long enough.
+  while (result.length <= count2) {
+    result += result;
+  }
+
+  // Use substring to hit the precise length target without using extra memory.
+  return result + result.substring(0, count - result.length);
+}
+
+function getPrincipal(url)
+{
+  let uri = Services.io.newURI(url);
+  return Services.scriptSecurityManager.createCodebasePrincipal(uri, {});
+}
+
+function getCurrentPrincipal()
+{
+  return Cc["@mozilla.org/systemprincipal;1"].createInstance(Ci.nsIPrincipal);
+}
+
+function getLocalStorage(principal)
+{
+  if (!principal) {
+    principal = getCurrentPrincipal();
+  }
+
+  return Services.domStorageManager.createStorage(null, principal, "");
+}
diff --git a/dom/localstorage/test/unit/test_eviction.js b/dom/localstorage/test/unit/test_eviction.js
new file mode 100644
index 000000000000..0af547fd47e0
--- /dev/null
+++ b/dom/localstorage/test/unit/test_eviction.js
@@ -0,0 +1,91 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const globalLimitKB = 5 * 1024;
+
+  const data = {};
+  data.sizeKB = 1 * 1024;
+  data.key = "A";
+  data.value = repeatChar(data.sizeKB * 1024 - data.key.length, ".");
+  data.urlCount = globalLimitKB / data.sizeKB;
+
+  function getSpec(index) {
+    return "http://example" + index + ".com";
+  }
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+
+  info("Setting limits");
+
+  setGlobalLimit(globalLimitKB);
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  info("Getting storages");
+
+  let storages = [];
+  for (let i = 0; i < data.urlCount; i++) {
+    let storage = getLocalStorage(getPrincipal(getSpec(i)));
+    storages.push(storage);
+  }
+
+  info("Filling up entire default storage");
+
+  for (let i = 0; i < data.urlCount; i++) {
+    storages[i].setItem(data.key, data.value);
+  }
+
+  info("Verifying no more data can be written");
+
+  for (let i = 0; i < data.urlCount; i++) {
+    try {
+      storages[i].setItem("B", "");
+      ok(false, "Should have thrown");
+    } catch(ex) {
+      ok(true, "Did throw");
+      ok(ex instanceof DOMException, "Threw DOMException");
+      is(ex.name, "QuotaExceededError", "Threw right DOMException");
+      is(ex.code, NS_ERROR_DOM_QUOTA_EXCEEDED_ERR, "Threw with right code");
+    }
+  }
+
+  info("Closing first origin");
+
+  storages[0].close();
+
+  let principal = getPrincipal("http://example0.com");
+
+  resetOrigin(principal, continueToNextStepSync);
+  yield undefined;
+
+  info("Getting usage for first origin");
+
+  let request = getOriginUsage(principal, continueToNextStepSync);
+  yield undefined;
+
+  is(request.result.usage, data.sizeKB * 1024, "Correct usage");
+
+  info("Verifying more data data can be written");
+
+  for (let i = 1; i < data.urlCount; i++) {
+    storages[i].setItem("B", "");
+  }
+
+  info("Getting usage for first origin");
+
+  request = getOriginUsage(principal, continueToNextStepSync);
+  yield undefined;
+
+  is(request.result.usage, 0, "Zero usage");
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_groupLimit.js b/dom/localstorage/test/unit/test_groupLimit.js
new file mode 100644
index 000000000000..f408a549e0e6
--- /dev/null
+++ b/dom/localstorage/test/unit/test_groupLimit.js
@@ -0,0 +1,77 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const groupLimitKB = 10 * 1024;
+
+  const globalLimitKB = groupLimitKB * 5;
+
+  const originLimit = 10 * 1024;
+
+  const urls = [
+    "http://example.com",
+    "http://test1.example.com",
+    "https://test2.example.com",
+    "http://test3.example.com:8080"
+  ];
+
+  const data = {};
+  data.sizeKB = 5 * 1024;
+  data.key = "A";
+  data.value = repeatChar(data.sizeKB * 1024 - data.key.length, ".");
+  data.urlCount = groupLimitKB / data.sizeKB;
+
+  info("Setting limits");
+
+  setGlobalLimit(globalLimitKB);
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  setOriginLimit(originLimit);
+
+  info("Getting storages");
+
+  let storages = [];
+  for (let i = 0; i < urls.length; i++) {
+    let storage = getLocalStorage(getPrincipal(urls[i]));
+    storages.push(storage);
+  }
+
+  info("Filling up the whole group");
+
+  for (let i = 0; i < data.urlCount; i++) {
+    storages[i].setItem(data.key, data.value);
+  }
+
+  info("Verifying no more data can be written");
+
+  for (let i = 0; i < urls.length; i++) {
+    try {
+      storages[i].setItem("B", "");
+      ok(false, "Should have thrown");
+    } catch(ex) {
+      ok(true, "Did throw");
+      ok(ex instanceof DOMException, "Threw DOMException");
+      is(ex.name, "QuotaExceededError", "Threw right DOMException");
+      is(ex.code, NS_ERROR_DOM_QUOTA_EXCEEDED_ERR, "Threw with right code");
+    }
+  }
+
+  info("Clearing first origin");
+
+  storages[0].clear();
+
+  info("Verifying more data can be written");
+
+  for (let i = 0; i < urls.length; i++) {
+    storages[i].setItem("B", "");
+  }
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/xpcshell.ini b/dom/localstorage/test/unit/xpcshell.ini
new file mode 100644
index 000000000000..4dd70fa4f0c6
--- /dev/null
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -0,0 +1,9 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+[DEFAULT]
+head = head.js
+
+[test_eviction.js]
+[test_groupLimit.js]
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 315aeffae083..58763e733330 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -3116,9 +3116,16 @@ QuotaObject::LockedMaybeUpdateSize(int64_t aSize, bool aTruncate)
     // This will block the thread without holding the lock while waitting.
 
     AutoTArray<RefPtr<DirectoryLockImpl>, 10> locks;
+    uint64_t sizeToBeFreed;
 
-    uint64_t sizeToBeFreed =
-      quotaManager->LockedCollectOriginsForEviction(delta, locks);
+    if (IsOnBackgroundThread()) {
+      MutexAutoUnlock autoUnlock(quotaManager->mQuotaMutex);
+
+      sizeToBeFreed = quotaManager->CollectOriginsForEviction(delta, locks);
+    } else {
+      sizeToBeFreed = quotaManager->LockedCollectOriginsForEviction(delta,
+                                                                    locks);
+    }
 
     if (!sizeToBeFreed) {
       uint64_t usage = quotaManager->mTemporaryStorageUsage;
@@ -3876,6 +3883,7 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
                              const nsACString& aGroup,
                              const nsACString& aOrigin,
                              nsIFile* aFile,
+                             int64_t aFileSize,
                              int64_t* aFileSizeOut /* = nullptr */)
 {
   NS_ASSERTION(!NS_IsMainThread(), "Wrong thread!");
@@ -3894,16 +3902,20 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
 
   int64_t fileSize;
 
-  bool exists;
-  rv = aFile->Exists(&exists);
-  NS_ENSURE_SUCCESS(rv, nullptr);
-
-  if (exists) {
-    rv = aFile->GetFileSize(&fileSize);
+  if (aFileSize == -1) {
+    bool exists;
+    rv = aFile->Exists(&exists);
     NS_ENSURE_SUCCESS(rv, nullptr);
-  }
-  else {
-    fileSize = 0;
+
+    if (exists) {
+      rv = aFile->GetFileSize(&fileSize);
+      NS_ENSURE_SUCCESS(rv, nullptr);
+    }
+    else {
+      fileSize = 0;
+    }
+  } else {
+    fileSize = aFileSize;
   }
 
   // Re-escape our parameters above to make sure we get the right quota group.
@@ -3969,6 +3981,7 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
                              const nsACString& aGroup,
                              const nsACString& aOrigin,
                              const nsAString& aPath,
+                             int64_t aFileSize,
                              int64_t* aFileSizeOut /* = nullptr */)
 {
   NS_ASSERTION(!NS_IsMainThread(), "Wrong thread!");
@@ -3981,7 +3994,12 @@ QuotaManager::GetQuotaObject(PersistenceType aPersistenceType,
   nsresult rv = NS_NewLocalFile(aPath, false, getter_AddRefs(file));
   NS_ENSURE_SUCCESS(rv, nullptr);
 
-  return GetQuotaObject(aPersistenceType, aGroup, aOrigin, file, aFileSizeOut);
+  return GetQuotaObject(aPersistenceType,
+                        aGroup,
+                        aOrigin,
+                        file,
+                        aFileSize,
+                        aFileSizeOut);
 }
 
 Nullable<bool>
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index 3905155abdd7..498379bb3769 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -179,6 +179,7 @@ public:
                  const nsACString& aGroup,
                  const nsACString& aOrigin,
                  nsIFile* aFile,
+                 int64_t aFileSize = -1,
                  int64_t* aFileSizeOut = nullptr);
 
   already_AddRefed<QuotaObject>
@@ -186,6 +187,7 @@ public:
                  const nsACString& aGroup,
                  const nsACString& aOrigin,
                  const nsAString& aPath,
+                 int64_t aFileSize = -1,
                  int64_t* aFileSizeOut = nullptr);
 
   Nullable<bool>

From 0649224345c47d4db7e2bc031b71dc757f95b32b Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:38 +0100
Subject: [PATCH 40/78] Bug 1286798 - Part 26: Implement a lazy data migration
 from old webappsstore.sqlite; r=asuth,janv

This patch was enhanced by asuth to bind attached database path.
Places has shown that argument binding is necessary.  (Profiles may include usernames in their path which can have cool emoji and such.)
---
 dom/localstorage/ActorsParent.cpp             | 547 ++++++++++++++++--
 dom/localstorage/LSObject.cpp                 |   8 +-
 .../test/unit/archive_profile.zip             | Bin 0 -> 1162 bytes
 dom/localstorage/test/unit/head.js            |  82 ++-
 .../test/unit/migration_profile.zip           | Bin 0 -> 1727 bytes
 dom/localstorage/test/unit/test_archive.js    |  80 +++
 dom/localstorage/test/unit/test_migration.js  |  57 ++
 dom/localstorage/test/unit/xpcshell.ini       |   5 +
 dom/quota/ActorsParent.cpp                    | 268 ++++++++-
 dom/quota/QuotaManager.h                      |   3 +
 .../test/unit/createLocalStorage_profile.zip  | Bin 0 -> 1006 bytes
 dom/quota/test/unit/head.js                   |   4 +-
 .../test/unit/test_createLocalStorage.js      | 161 ++++++
 dom/quota/test/unit/xpcshell.ini              |   2 +
 dom/storage/moz.build                         |   1 +
 15 files changed, 1169 insertions(+), 49 deletions(-)
 create mode 100644 dom/localstorage/test/unit/archive_profile.zip
 create mode 100644 dom/localstorage/test/unit/migration_profile.zip
 create mode 100644 dom/localstorage/test/unit/test_archive.js
 create mode 100644 dom/localstorage/test/unit/test_migration.js
 create mode 100644 dom/quota/test/unit/createLocalStorage_profile.zip
 create mode 100644 dom/quota/test/unit/test_createLocalStorage.js

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index a58bdb4a9a2a..5da3b1266961 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -20,6 +20,8 @@
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestParent.h"
+#include "mozilla/dom/StorageDBUpdater.h"
+#include "mozilla/dom/StorageUtils.h"
 #include "mozilla/dom/quota/QuotaManager.h"
 #include "mozilla/dom/quota/QuotaObject.h"
 #include "mozilla/dom/quota/UsageInfo.h"
@@ -49,10 +51,12 @@ namespace mozilla {
 namespace dom {
 
 using namespace mozilla::dom::quota;
+using namespace mozilla::dom::StorageUtils;
 using namespace mozilla::ipc;
 
 namespace {
 
+class ArchivedOriginInfo;
 class Connection;
 class ConnectionThread;
 class Database;
@@ -117,6 +121,8 @@ const char kDefaultQuotaPref[] = "dom.storage.default_quota";
 
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
 
+#define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
+
 bool
 IsOnConnectionThread();
 
@@ -465,6 +471,158 @@ GetStorageConnection(const nsAString& aDatabaseFilePath,
   return NS_OK;
 }
 
+nsresult
+GetArchiveFile(const nsAString& aStoragePath,
+               nsIFile** aArchiveFile)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aStoragePath.IsEmpty());
+  MOZ_ASSERT(aArchiveFile);
+
+  nsCOMPtr<nsIFile> archiveFile;
+  nsresult rv = NS_NewLocalFile(aStoragePath,
+                                false,
+                                getter_AddRefs(archiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = archiveFile->Append(NS_LITERAL_STRING(LS_ARCHIVE_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  archiveFile.forget(aArchiveFile);
+  return NS_OK;
+}
+
+nsresult
+CreateArchiveStorageConnection(const nsAString& aStoragePath,
+                               mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aStoragePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> archiveFile;
+  nsresult rv = GetArchiveFile(aStoragePath, getter_AddRefs(archiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // QuotaManager ensures this file always exists.
+  DebugOnly<bool> exists;
+  MOZ_ASSERT(NS_SUCCEEDED(archiveFile->Exists(&exists)));
+  MOZ_ASSERT(exists);
+
+  bool isDirectory;
+  rv = archiveFile->IsDirectory(&isDirectory);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (isDirectory) {
+    LS_WARNING("ls-archive is not a file!");
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenUnsharedDatabase(archiveFile, getter_AddRefs(connection));
+  if (rv == NS_ERROR_FILE_CORRUPTED) {
+    // Don't throw an error, leave a corrupted ls-archive database as it is.
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StorageDBUpdater::Update(connection);
+  if (NS_FAILED(rv)) {
+    // Don't throw an error, leave a non-updateable ls-archive database as
+    // it is.
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
+nsresult
+AttachArchiveDatabase(const nsAString& aStoragePath,
+                      mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aStoragePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+  nsCOMPtr<nsIFile> archiveFile;
+
+  nsresult rv = GetArchiveFile(aStoragePath, getter_AddRefs(archiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+#ifdef DEBUG
+  bool exists;
+  rv = archiveFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(exists);
+#endif
+
+  nsString path;
+  rv = archiveFile->GetPath(path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = aConnection->CreateStatement(
+    NS_LITERAL_CSTRING("ATTACH DATABASE :path AS archive;"),
+    getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("path"), path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+DetachArchiveDatabase(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "DETACH DATABASE archive"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Non-actor class declarations
  ******************************************************************************/
@@ -1424,6 +1582,7 @@ class PrepareDatastoreOp
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
   RefPtr<Datastore> mDatastore;
+  nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   LoadDataOp* mLoadDataOp;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   const LSRequestPrepareDatastoreParams mParams;
@@ -1512,12 +1671,16 @@ private:
 
   nsresult
   EnsureDirectoryEntry(nsIFile* aEntry,
+                       bool aCreateIfNotExists,
                        bool aDirectory,
                        bool* aAlreadyExisted = nullptr);
 
   nsresult
   VerifyDatabaseInformation(mozIStorageConnection* aConnection);
 
+  already_AddRefed<QuotaObject>
+  GetQuotaObject();
+
   nsresult
   BeginLoadData();
 
@@ -1667,6 +1830,44 @@ private:
  * Other class declarations
  ******************************************************************************/
 
+class ArchivedOriginInfo
+{
+  nsCString mOriginSuffix;
+  nsCString mOriginNoSuffix;
+
+public:
+  static ArchivedOriginInfo*
+  Create(nsIPrincipal* aPrincipal);
+
+  const nsCString&
+  OriginSuffix() const
+  {
+    return mOriginSuffix;
+  }
+
+  const nsCString&
+  OriginNoSuffix() const
+  {
+    return mOriginNoSuffix;
+  }
+
+  const nsCString
+  Origin() const
+  {
+    return mOriginSuffix + NS_LITERAL_CSTRING(":") + mOriginNoSuffix;
+  }
+
+  nsresult
+  BindToStatement(mozIStorageStatement* aStatement) const;
+
+private:
+  ArchivedOriginInfo(const nsACString& aOriginSuffix,
+                     const nsACString& aOriginNoSuffix)
+    : mOriginSuffix(aOriginSuffix)
+    , mOriginNoSuffix(aOriginNoSuffix)
+  { }
+};
+
 class QuotaClient final
   : public mozilla::dom::quota::Client
 {
@@ -1840,6 +2041,10 @@ typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
 // Can only be touched on the Quota Manager I/O thread.
 StaticAutoPtr<UsageHashtable> gUsages;
 
+typedef nsTHashtable<nsCStringHashKey> ArchivedOriginHashtable;
+
+StaticAutoPtr<ArchivedOriginHashtable> gArchivedOrigins;
+
 bool
 IsOnConnectionThread()
 {
@@ -1867,6 +2072,117 @@ InitUsageForOrigin(const nsACString& aOrigin, int64_t aUsage)
   gUsages->Put(aOrigin, aUsage);
 }
 
+nsresult
+LoadArchivedOrigins()
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!gArchivedOrigins);
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  // Ensure that the webappsstore.sqlite is moved to new place.
+  nsresult rv = quotaManager->EnsureStorageIsInitialized();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = CreateArchiveStorageConnection(quotaManager->GetStoragePath(),
+                                      getter_AddRefs(connection));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!connection) {
+    gArchivedOrigins = new ArchivedOriginHashtable();
+    return NS_OK;
+  }
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+    "SELECT DISTINCT originAttributes || ':' || originKey "
+      "FROM webappsstore2;"
+  ), getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsAutoPtr<ArchivedOriginHashtable> archivedOrigins(
+    new ArchivedOriginHashtable());
+
+  bool hasResult;
+  while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&hasResult)) && hasResult) {
+    nsCString origin;
+    rv = stmt->GetUTF8String(0, origin);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    archivedOrigins->PutEntry(origin);
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  gArchivedOrigins = archivedOrigins.forget();
+  return NS_OK;
+}
+
+nsresult
+GetUsage(mozIStorageConnection* aConnection,
+         ArchivedOriginInfo* aArchivedOriginInfo,
+         int64_t* aUsage)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+  MOZ_ASSERT(aUsage);
+
+  nsresult rv;
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  if (aArchivedOriginInfo) {
+    rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+      "SELECT sum(length(key) + length(value)) "
+      "FROM webappsstore2 "
+      "WHERE originKey = :originKey "
+      "AND originAttributes = :originAttributes;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = aArchivedOriginInfo->BindToStatement(stmt);
+  } else {
+    rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+      "SELECT sum(length(key) + length(value)) "
+      "FROM data"
+    ), getter_AddRefs(stmt));
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!hasResult)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  int64_t usage;
+  rv = stmt->GetInt64(0, &usage);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  *aUsage = usage;
+  return NS_OK;
+}
+
 } // namespace
 
 /*******************************************************************************
@@ -3863,6 +4179,11 @@ PrepareDatastoreOp::Open()
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
+
+    mArchivedOriginInfo = ArchivedOriginInfo::Create(principal);
+    if (NS_WARN_IF(!mArchivedOriginInfo)) {
+      return NS_ERROR_FAILURE;
+    }
   }
 
   // This service has to be started on the main thread currently.
@@ -4099,6 +4420,7 @@ nsresult
 PrepareDatastoreOp::DatabaseWork()
 {
   AssertIsOnIOThread();
+  MOZ_ASSERT(mArchivedOriginInfo);
   MOZ_ASSERT(mState == State::Nesting);
   MOZ_ASSERT(mNestedState == NestedState::DatabaseWorkOpen);
 
@@ -4110,14 +4432,28 @@ PrepareDatastoreOp::DatabaseWork()
   QuotaManager* quotaManager = QuotaManager::Get();
   MOZ_ASSERT(quotaManager);
 
+  nsresult rv;
+
+  if (!gArchivedOrigins) {
+    rv = LoadArchivedOrigins();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+    MOZ_ASSERT(gArchivedOrigins);
+  }
+
+  bool hasDataForMigration =
+    gArchivedOrigins->GetEntry(mArchivedOriginInfo->Origin());
+
+  bool createIfNotExists = mParams.createIfNotExists() || hasDataForMigration;
+
   nsCOMPtr<nsIFile> directoryEntry;
-  nsresult rv =
-    quotaManager->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT,
-                                            mSuffix,
-                                            mGroup,
-                                            mOrigin,
-                                            mParams.createIfNotExists(),
-                                            getter_AddRefs(directoryEntry));
+  rv = quotaManager->EnsureOriginIsInitialized(PERSISTENCE_TYPE_DEFAULT,
+                                               mSuffix,
+                                               mGroup,
+                                               mOrigin,
+                                               createIfNotExists,
+                                               getter_AddRefs(directoryEntry));
   if (rv == NS_ERROR_NOT_AVAILABLE) {
     return DatabaseNotAvailable();
   }
@@ -4130,7 +4466,9 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
-  rv = EnsureDirectoryEntry(directoryEntry, /* aIsDirectory */ true);
+  rv = EnsureDirectoryEntry(directoryEntry,
+                            createIfNotExists,
+                            /* aIsDirectory */ true);
   if (rv == NS_ERROR_NOT_AVAILABLE) {
     return DatabaseNotAvailable();
   }
@@ -4145,6 +4483,7 @@ PrepareDatastoreOp::DatabaseWork()
 
   bool alreadyExisted;
   rv = EnsureDirectoryEntry(directoryEntry,
+                            createIfNotExists,
                             /* aIsDirectory */ false,
                             &alreadyExisted);
   if (rv == NS_ERROR_NOT_AVAILABLE) {
@@ -4158,7 +4497,8 @@ PrepareDatastoreOp::DatabaseWork()
     MOZ_ASSERT(gUsages);
     MOZ_ASSERT(gUsages->Get(mOrigin, &mUsage));
   } else {
-    InitUsageForOrigin(mOrigin, 0);
+    MOZ_ASSERT(mUsage == 0);
+    InitUsageForOrigin(mOrigin, mUsage);
   }
 
   rv = directoryEntry->GetPath(mDatabaseFilePath);
@@ -4179,6 +4519,93 @@ PrepareDatastoreOp::DatabaseWork()
     return rv;
   }
 
+  if (hasDataForMigration) {
+    MOZ_ASSERT(mUsage == 0);
+
+    rv = AttachArchiveDatabase(quotaManager->GetStoragePath(), connection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    int64_t newUsage;
+    rv = GetUsage(connection, mArchivedOriginInfo, &newUsage);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    RefPtr<QuotaObject> quotaObject = GetQuotaObject();
+    MOZ_ASSERT(quotaObject);
+
+    if (!quotaObject->MaybeUpdateSize(newUsage, /* aTruncate */ true)) {
+      return NS_ERROR_FILE_NO_DEVICE_SPACE;
+    }
+
+    mozStorageTransaction transaction(connection, false,
+                                  mozIStorageConnection::TRANSACTION_IMMEDIATE);
+
+    nsCOMPtr<mozIStorageStatement> stmt;
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "INSERT INTO data (key, value) "
+        "SELECT key, value "
+        "FROM webappsstore2 "
+        "WHERE originKey = :originKey "
+        "AND originAttributes = :originAttributes;"
+
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = mArchivedOriginInfo->BindToStatement(stmt);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = stmt->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "DELETE FROM webappsstore2 "
+        "WHERE originKey = :originKey "
+        "AND originAttributes = :originAttributes;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = mArchivedOriginInfo->BindToStatement(stmt);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = stmt->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = transaction.Commit();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = DetachArchiveDatabase(connection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(gArchivedOrigins);
+    MOZ_ASSERT(gArchivedOrigins->GetEntry(mArchivedOriginInfo->Origin()));
+    gArchivedOrigins->RemoveEntry(mArchivedOriginInfo->Origin());
+
+    mUsage = newUsage;
+
+    MOZ_ASSERT(gUsages);
+    MOZ_ASSERT(gUsages->Contains(mOrigin));
+    gUsages->Put(mOrigin, newUsage);
+  }
+
   // Must close connection before dispatching otherwise we might race with the
   // connection thread which needs to open the same database.
   MOZ_ALWAYS_SUCCEEDS(connection->Close());
@@ -4219,6 +4646,7 @@ PrepareDatastoreOp::DatabaseNotAvailable()
 
 nsresult
 PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry,
+                                         bool aCreateIfNotExists,
                                          bool aIsDirectory,
                                          bool* aAlreadyExisted)
 {
@@ -4232,7 +4660,7 @@ PrepareDatastoreOp::EnsureDirectoryEntry(nsIFile* aEntry,
   }
 
   if (!exists) {
-    if (!mParams.createIfNotExists()) {
+    if (!aCreateIfNotExists) {
       return NS_ERROR_NOT_AVAILABLE;
     }
 
@@ -4295,6 +4723,28 @@ PrepareDatastoreOp::VerifyDatabaseInformation(mozIStorageConnection* aConnection
   return NS_OK;
 }
 
+already_AddRefed<QuotaObject>
+PrepareDatastoreOp::GetQuotaObject()
+{
+  MOZ_ASSERT(IsOnOwningThread() || IsOnIOThread());
+  MOZ_ASSERT(!mGroup.IsEmpty());
+  MOZ_ASSERT(!mOrigin.IsEmpty());
+  MOZ_ASSERT(!mDatabaseFilePath.IsEmpty());
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  RefPtr<QuotaObject> quotaObject =
+    quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT,
+                                 mGroup,
+                                 mOrigin,
+                                 mDatabaseFilePath,
+                                 mUsage);
+  MOZ_ASSERT(quotaObject);
+
+  return quotaObject.forget();
+}
+
 nsresult
 PrepareDatastoreOp::BeginLoadData()
 {
@@ -4403,16 +4853,7 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
     RefPtr<QuotaObject> quotaObject;
 
     if (mPrivateBrowsingId == 0) {
-      MOZ_ASSERT(!mDatabaseFilePath.IsEmpty());
-
-      QuotaManager* quotaManager = QuotaManager::Get();
-      MOZ_ASSERT(quotaManager);
-
-      quotaObject = quotaManager->GetQuotaObject(PERSISTENCE_TYPE_DEFAULT,
-                                                 mGroup,
-                                                 mOrigin,
-                                                 mDatabaseFilePath,
-                                                 mUsage);
+      quotaObject = GetQuotaObject();
       MOZ_ASSERT(quotaObject);
     }
 
@@ -4944,6 +5385,49 @@ PreloadedOp::GetResponse(LSSimpleRequestResponse& aResponse)
   aResponse = preloadedResponse;
 }
 
+/*******************************************************************************
+ * ArchivedOriginInfo
+ ******************************************************************************/
+
+// static
+ArchivedOriginInfo*
+ArchivedOriginInfo::Create(nsIPrincipal* aPrincipal)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+
+  nsCString originAttrSuffix;
+  nsCString originKey;
+  nsresult rv = GenerateOriginKey(aPrincipal, originAttrSuffix, originKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return nullptr;
+  }
+
+  return new ArchivedOriginInfo(originAttrSuffix, originKey);
+}
+
+nsresult
+ArchivedOriginInfo::BindToStatement(mozIStorageStatement* aStatement) const
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aStatement);
+
+  nsresult rv =
+    aStatement->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
+                                     mOriginNoSuffix);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aStatement->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
+                                        mOriginSuffix);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * QuotaClient
  ******************************************************************************/
@@ -5081,27 +5565,8 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
       return rv;
     }
 
-    nsCOMPtr<mozIStorageStatement> stmt;
-    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
-      "SELECT sum(length(key) + length(value)) "
-      "FROM data"
-    ), getter_AddRefs(stmt));
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    bool hasResult;
-    rv = stmt->ExecuteStep(&hasResult);
-    if (NS_WARN_IF(NS_FAILED(rv))) {
-      return rv;
-    }
-
-    if (NS_WARN_IF(!hasResult)) {
-      return NS_ERROR_FAILURE;
-    }
-
     int64_t usage;
-    rv = stmt->GetInt64(0, &usage);
+    rv = GetUsage(connection, /* aArchivedOriginInfo */ nullptr, &usage);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -5213,6 +5678,8 @@ QuotaClient::ReleaseIOThreadObjects()
   AssertIsOnIOThread();
 
   gUsages = nullptr;
+
+  gArchivedOrigins = nullptr;
 }
 
 void
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 6af0533497fb..b01e475a9d63 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -631,7 +631,13 @@ LSObject::DoRequestSynchronously(const LSRequestParams& aParams,
   }
 
   if (aResponse.type() == LSRequestResponse::Tnsresult) {
-    return aResponse.get_nsresult();
+    nsresult errorCode = aResponse.get_nsresult();
+
+    if (errorCode == NS_ERROR_FILE_NO_DEVICE_SPACE) {
+      errorCode = NS_ERROR_DOM_QUOTA_EXCEEDED_ERR;
+    }
+
+    return errorCode;
   }
 
   return NS_OK;
diff --git a/dom/localstorage/test/unit/archive_profile.zip b/dom/localstorage/test/unit/archive_profile.zip
new file mode 100644
index 0000000000000000000000000000000000000000..71b2d1e5f90de4334169d5ec4c1c385615358a02
GIT binary patch
literal 1162
zcmWIWW@Zs#U|`^2*c4mtv;0t{qA`%iz{J47#~{N{T#{dun4YRvT$qztk{TMq$-sR6
z)|Mz&x93r%72FJrEMFNJ7{L0u`M#Xg@eK7it$)f_C(s}*At@mxr63?60SH)=+}LUw
zI5|Ethq2kt<lQ5%b#a0K6GH&dYA%pv3_9#Te83DS!GW+^A8ctzY*;i1qge_v0pfU5
zpb7sL@mVAT9nR3e$RLSiLQb)6Vo`ENW*Le<(tyT+Fq)BXFP_bl4i#Z|u<`furcI|7
z-4>j^Ey%dDL$_%)*PREhPqbDDc)NORJ@!uade)o5=j~R~KJ%95UlH{`X#7BkdzRS2
zO$xR1OWeQTv3x$yx_n>#pNyzhi__haQAdAY)DPS&Su1-yP{j25W3MmMw|~4^{^(fR
zpUG)c?(Et8wd~X9{EMq1?2!Wk3;PGZMZQV?om)HaTI{!<YtMap&kVn1pC5J?nEg?%
z6Lou;6gVw3uI2oCnY!z*=Y4r}@Z-hDj{0_1|CW7MsoSsg@0-QGkkaQ$i=#u2mV|^a
z+_Qb<%l6IHA9uZb_j7r#^!B{<=M7d|x|q2mq*gN{kFVfT?T;1ZJH7un&i=9F`;#Yg
z<7V%WdEUKY=KdeWUzdN`vM*xq?ydW#9(z@~rE2f3zc=dHvA7sXsNsL_=hbIlNB^z+
z-X3Y^H?2~vD|6bEt9#BJe{Ay7X8kJHo0(g`{eSwkV21AH#K@<vR-B%;^Y^VeynyHE
z#m(ox@wq)b-K=x>=c225Z?`z@{+j&wSDy9Vdv#Ikjrl%TicSl)pJSWh-(|zOygPCi
zk9c)C-?wj8k<XitoV%ZzR%z$e?SD=FYQ)~ou>7$7b#G3yBDo9;v*CMfY3}O#xo18V
zZaT9$z-RN4HP1@z<d%CruME{a<+pmPy-eEal8CU%iGEH&=a=ihHn_2K+wFV%es0lo
zE}U_G^@cUS9=>s`4tnBg5qsuR>W&)=i#JSv`tyB&HzSiAGp<Z00e8ugMi3J{3$jA8
zAX>78n29Sl!ORq3IN(@~FcX-!fTm&1R|pqk<}8?Lz>L<o7s)g%`3z`0mOK{V&B_My
P8Z!_w0t=90W)KemBEOv8

literal 0
HcmV?d00001

diff --git a/dom/localstorage/test/unit/head.js b/dom/localstorage/test/unit/head.js
index 15320313f2eb..4e7c32a78793 100644
--- a/dom/localstorage/test/unit/head.js
+++ b/dom/localstorage/test/unit/head.js
@@ -106,6 +106,81 @@ function resetOrigin(principal, callback)
   return request;
 }
 
+function installPackage(packageName)
+{
+  let directoryService = Cc["@mozilla.org/file/directory_service;1"]
+                         .getService(Ci.nsIProperties);
+
+  let currentDir = directoryService.get("CurWorkD", Ci.nsIFile);
+
+  let packageFile = currentDir.clone();
+  packageFile.append(packageName + ".zip");
+
+  let zipReader = Cc["@mozilla.org/libjar/zip-reader;1"]
+                  .createInstance(Ci.nsIZipReader);
+  zipReader.open(packageFile);
+
+  let entryNames = [];
+  let entries = zipReader.findEntries(null);
+  while (entries.hasMore()) {
+    let entry = entries.getNext();
+    entryNames.push(entry);
+  }
+  entryNames.sort();
+
+  for (let entryName of entryNames) {
+    let zipentry = zipReader.getEntry(entryName);
+
+    let file = getRelativeFile(entryName);
+
+    if (zipentry.isDirectory) {
+      file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+    } else {
+      let istream = zipReader.getInputStream(entryName);
+
+      var ostream = Cc["@mozilla.org/network/file-output-stream;1"]
+                    .createInstance(Ci.nsIFileOutputStream);
+      ostream.init(file, -1, parseInt("0644", 8), 0);
+
+      let bostream = Cc['@mozilla.org/network/buffered-output-stream;1']
+                     .createInstance(Ci.nsIBufferedOutputStream);
+      bostream.init(ostream, 32768);
+
+      bostream.writeFrom(istream, istream.available());
+
+      istream.close();
+      bostream.close();
+    }
+  }
+
+  zipReader.close();
+}
+
+function getProfileDir()
+{
+  let directoryService =
+    Cc["@mozilla.org/file/directory_service;1"].getService(Ci.nsIProperties);
+
+  return directoryService.get("ProfD", Ci.nsIFile);
+}
+
+// Given a "/"-delimited path relative to the profile directory,
+// return an nsIFile representing the path.  This does not test
+// for the existence of the file or parent directories.
+// It is safe even on Windows where the directory separator is not "/",
+// but make sure you're not passing in a "\"-delimited path.
+function getRelativeFile(relativePath)
+{
+  let profileDir = getProfileDir();
+
+  let file = profileDir.clone();
+  relativePath.split('/').forEach(function(component) {
+    file.append(component);
+  });
+
+  return file;
+}
+
 function repeatChar(count, ch) {
   if (count == 0) {
     return "";
@@ -123,10 +198,13 @@ function repeatChar(count, ch) {
   return result + result.substring(0, count - result.length);
 }
 
-function getPrincipal(url)
+function getPrincipal(url, attrs)
 {
   let uri = Services.io.newURI(url);
-  return Services.scriptSecurityManager.createCodebasePrincipal(uri, {});
+  if (!attrs) {
+    attrs = {};
+  }
+  return Services.scriptSecurityManager.createCodebasePrincipal(uri, attrs);
 }
 
 function getCurrentPrincipal()
diff --git a/dom/localstorage/test/unit/migration_profile.zip b/dom/localstorage/test/unit/migration_profile.zip
new file mode 100644
index 0000000000000000000000000000000000000000..ff1cd89602076ad5bd332b0da22aee4e498c6648
GIT binary patch
literal 1727
zcmaKs2~ZPv6vj6YP=sL^N(AKyIviqSIBK<oGb&Kz3R*x>AO-@42t>kZLjvNhh@1sW
zIEpaHT}V($Vgw<OAg2X61mskLTn*t6fo^oBgJ^v_Z+3Tf_WR!a=l4B~og`EV0D$d)
zQee2fkT0O>NdUl2C;-R-YJhhz{#*#&%Lj+_C%gP214tGuG29=39U37EK%gQB0DS!W
z>Pd89zkvd43MNZWuFvi~s_%GtFDd4!r7Sd*yn~YckXfI)S1K6aD6_DH#PjY-J~G>N
zwpGs0dg_A_<nbdDOlfbVhbt6i8FSS&u^b4tgw=W17`fNFx!DCso!tM2@BKIYTrD<w
zT+r96nV~PtZ`^5ikY+XekJAXs2`SV{cNy$jKGp}#nc_zBPsG<UJ@*u)XbAgYCAIhT
z+k+T&n$)W(_G8<`xCKj<w~zyuY+Ijs&^#3m?YVs;ln5jE=?}Ff9S$q%QALk3UH(e&
z4Sb0-SW7Ph2D3%JJ0=x)KWRIROXRgXv_h)-O9}>x4q0<qwAkU>#j3HFi&o%Th4z+;
zD+Y0C(bNgVux!h1J<2tCRKC9Ph)Ui)E#s$W%HFGn7gvWN2RIc8`Bs_551b;~MXW#7
z_9Z-NxFK7LNll|Of@_=D1!!F5eYi12X8KCQ^o#fvJsXlvDNRdcm=<=c*NKxs=RO~B
zehtLbP^E3Qp(6*{C157G1K|YYXyrA<k>{|-i4eHWC2n?3S8b+^KiiBye^>rM#__nW
zp6)bWM;o-0;U>W4+?Vv3lvE^~Ins0T2&ZsWZFC>9&#1GTRe73qv&3^~u5J1{T}e8?
z7Ut#Q;lWx}h*f&Az9Oot^ptmQY_xiDZNI7`yk$t)YC+>S!dz7A2wHHbdj2kaCxqa5
z%Fo$x&an<g+E{xy)D&TkIz=oxDoWZYnCJ9s(*VqWxa|gUJI~*RH3H!SK;+~>+{ht8
z!RP$&Nb<!Cgb@5k;20om;@$$Bto#BGVe9g9vp(Me;cVcG#bJGXNEi$jEC|k8ni738
z+O$c|Ea5s!ftia{kl5s`4myt!lYR&RU4du_@EzzJj>nxNk;vi!KC`a|t;M~?e!2r4
z)bFFRjil>f0^xV`L9QT>-krO(AJ*k(YaZw2;P&rkbZR|t2^3UyWS}ZM5~s`CXQP+Z
zC^CT*xz@AQBNYV<;_!}6ss*om)s(tEOY9vok2FOlz1`QWdvR9))B-BHG#9BPg~pVX
zro0YP4jn!2ab|Jd!R&n099f8)7c5~D+zyM6?i;nft(7P;34b};)oP)hrJ*B4z~S*u
zOIf=5iyMW0JhUl&r&ap6Z8(1F_u+{dgTDS=YQ^C@RpC$6%;N^_k9o}=dwBxR$?QQi
zQz#@?orYNaMA2A7Yyp43f>svxW`-?LU*s=qI+W&>!9?S-?*ggw5fN+2Z@GpYGjo$O
zDV3?AhWvb9@Y;up)u@b2_RL_At5@E<ARaqEHPYDd%pY7I->RqmW0hQYvcdLeqo%|L
z0*adM;9l8V+-0HHOxSQ<&3bQhy4(1Mur>U9;uL$$fD-1awc;A)t|JUyCAHTyX-|eI
zwk0*&*U<i^kxOa)Smuf~bA_ubV)~XTUrV|W;yqF2nKI>C9r&oK`&~#~Iie!zpay<&
zyuQUTh2WOHSku*b9&ronjG&(WP_;bCSvUOFwBR?^kLK<H@^yU|Gnpd?)j7{~N_sDJ
z3$l{O85;#r$#}XoF_f*`ok@q;K1rdc@>s9T)fSL@ZBg`k;h12V%fDXK;!jD83;gv}
z35V17?#gVkRahc77r}mCzNU+@gGgvd{@<=v`8b;RK%-;CkDqUDsZX1G3&ooizl`H2
zMX^?NOb{5At>FYUzY@&NnqLNOv*skI`PJBg!e5CT#!gCFJd7lGZGiVeDe>FCnorEJ

literal 0
HcmV?d00001

diff --git a/dom/localstorage/test/unit/test_archive.js b/dom/localstorage/test/unit/test_archive.js
new file mode 100644
index 000000000000..02615c22968b
--- /dev/null
+++ b/dom/localstorage/test/unit/test_archive.js
@@ -0,0 +1,80 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const lsArchiveFile = "storage/ls-archive.sqlite";
+
+  const principalInfo = {
+    url: "http://example.com",
+    attrs: {}
+  };
+
+  function checkStorage()
+  {
+    let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
+    let storage = getLocalStorage(principal);
+    try {
+      storage.open();
+      ok(true, "Did not throw");
+    } catch(ex) {
+      ok(false, "Should not have thrown");
+    }
+  }
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+
+  // Profile 1 - Archive file is a directory.
+  info("Clearing");
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  let archiveFile = getRelativeFile(lsArchiveFile);
+
+  archiveFile.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+
+  checkStorage();
+
+  // Profile 2 - Corrupted archive file.
+  info("Clearing");
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  let ostream = Cc["@mozilla.org/network/file-output-stream;1"]
+                .createInstance(Ci.nsIFileOutputStream);
+  ostream.init(archiveFile, -1, parseInt("0644", 8), 0);
+  ostream.write("foobar", 6);
+  ostream.close();
+
+  checkStorage();
+
+  // Profile 3 - Nonupdateable archive file.
+  info("Clearing");
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  info("Installing package");
+
+  // The profile contains storage.sqlite and storage/ls-archive.sqlite
+  // storage/ls-archive.sqlite was taken from FF 54 to force an upgrade.
+  // There's just one record in the webappsstore2 table. The record was
+  // modified by renaming the origin attribute userContextId to userContextKey.
+  // This triggers an error during the upgrade.
+  installPackage("archive_profile");
+
+  let fileSize = archiveFile.fileSize;
+  ok(fileSize > 0, "archive file size is greater than zero");
+
+  checkStorage();
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_migration.js b/dom/localstorage/test/unit/test_migration.js
new file mode 100644
index 000000000000..35c31ce67e47
--- /dev/null
+++ b/dom/localstorage/test/unit/test_migration.js
@@ -0,0 +1,57 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const principalInfos = [
+    { url: "http://localhost", attrs: {} },
+    { url: "http://www.mozilla.org", attrs: {} },
+    { url: "http://example.com", attrs: {} },
+    { url: "http://example.org", attrs: { userContextId: 5 } }
+  ];
+
+  const data = {
+    key: "foo",
+    value: "bar"
+  };
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+
+  info("Clearing");
+
+  clear(continueToNextStepSync);
+  yield undefined;
+
+  info("Installing package");
+
+  // The profile contains storage.sqlite and webappsstore.sqlite. The file
+  // create_db.js in the package was run locally, specifically it was
+  // temporarily added to xpcshell.ini and then executed:
+  // mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
+  installPackage("migration_profile");
+
+  info("Getting storages");
+
+  let storages = [];
+  for (let i = 0; i < principalInfos.length; i++) {
+    let principalInfo = principalInfos[i];
+    let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
+    let storage = getLocalStorage(principal);
+    storages.push(storage);
+  }
+
+  info("Verifying data");
+
+  for (let i = 0; i < storages.length; i++) {
+    let value = storages[i].getItem(data.key + i);
+    is(value, data.value + i, "Correct value");
+  }
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/xpcshell.ini b/dom/localstorage/test/unit/xpcshell.ini
index 4dd70fa4f0c6..fbac5f3fa335 100644
--- a/dom/localstorage/test/unit/xpcshell.ini
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -4,6 +4,11 @@
 
 [DEFAULT]
 head = head.js
+support-files =
+  archive_profile.zip
+  migration_profile.zip
 
+[test_archive.js]
 [test_eviction.js]
 [test_groupLimit.js]
+[test_migration.js]
diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 58763e733330..b56e47f8fbc0 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -40,6 +40,7 @@
 #include "mozilla/dom/quota/PQuotaUsageRequestParent.h"
 #include "mozilla/dom/simpledb/ActorsParent.h"
 #include "mozilla/dom/StorageActivityService.h"
+#include "mozilla/dom/StorageDBUpdater.h"
 #include "mozilla/ipc/BackgroundParent.h"
 #include "mozilla/ipc/BackgroundUtils.h"
 #include "mozilla/IntegerRange.h"
@@ -211,6 +212,7 @@ enum AppId {
 #define METADATA_V2_FILE_NAME ".metadata-v2"
 #define METADATA_V2_TMP_FILE_NAME ".metadata-v2-tmp"
 
+#define WEB_APPS_STORE_FILE_NAME "webappsstore.sqlite"
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
 #define LS_ARCHIVE_TMP_FILE_NAME "ls-archive-tmp.sqlite"
 
@@ -264,6 +266,65 @@ CreateTables(mozIStorageConnection* aConnection)
   return NS_OK;
 }
 
+nsresult
+CreateWebAppsStoreConnection(nsIFile* aWebAppsStoreFile,
+                             mozIStorageService* aStorageService,
+                             mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aWebAppsStoreFile);
+  MOZ_ASSERT(aStorageService);
+  MOZ_ASSERT(aConnection);
+
+  // Check if the old database exists at all.
+  bool exists;
+  nsresult rv = aWebAppsStoreFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!exists) {
+    // webappsstore.sqlite doesn't exist, return a null connection.
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+
+  bool isDirectory;
+  rv = aWebAppsStoreFile->IsDirectory(&isDirectory);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (isDirectory) {
+    QM_WARNING("webappsstore.sqlite is not a file!");
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = aStorageService->OpenUnsharedDatabase(aWebAppsStoreFile,
+                                             getter_AddRefs(connection));
+  if (rv == NS_ERROR_FILE_CORRUPTED) {
+    // Don't throw an error, leave a corrupted webappsstore database as it is.
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StorageDBUpdater::Update(connection);
+  if (NS_FAILED(rv)) {
+    // Don't throw an error, leave a non-updateable webappsstore database as
+    // it is.
+    *aConnection = nullptr;
+    return NS_OK;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
 /******************************************************************************
  * Quota manager class declarations
  ******************************************************************************/
@@ -4837,10 +4898,7 @@ nsresult
 QuotaManager::MaybeRemoveLocalStorageData()
 {
   AssertIsOnIOThread();
-
-  if (CachedNextGenLocalStorageEnabled()) {
-    return NS_OK;
-  }
+  MOZ_ASSERT(!CachedNextGenLocalStorageEnabled());
 
   // Cleanup the tmp file first, if there's any.
   nsCOMPtr<nsIFile> lsArchiveTmpFile;
@@ -5037,6 +5095,202 @@ QuotaManager::MaybeRemoveLocalStorageDirectories()
   return NS_OK;
 }
 
+nsresult
+QuotaManager::MaybeCreateLocalStorageArchive()
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(CachedNextGenLocalStorageEnabled());
+
+  // Check if the archive was already successfully created.
+  nsCOMPtr<nsIFile> lsArchiveFile;
+  nsresult rv = NS_NewLocalFile(mStoragePath,
+                                false,
+                                getter_AddRefs(lsArchiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = lsArchiveFile->Append(NS_LITERAL_STRING(LS_ARCHIVE_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+  rv = lsArchiveFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (exists) {
+    // ls-archive.sqlite already exists, nothing to create.
+    return NS_OK;
+  }
+
+  // Get the storage service first, we will need it at multiple places.
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // Get the web apps store file.
+  nsCOMPtr<nsIFile> webAppsStoreFile;
+  rv = NS_NewLocalFile(mBasePath, false, getter_AddRefs(webAppsStoreFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = webAppsStoreFile->Append(NS_LITERAL_STRING(WEB_APPS_STORE_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // Now check if the web apps store is useable.
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = CreateWebAppsStoreConnection(webAppsStoreFile,
+                                    ss,
+                                    getter_AddRefs(connection));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (connection) {
+    // Find out the journal mode.
+    nsCOMPtr<mozIStorageStatement> stmt;
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA journal_mode;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    bool hasResult;
+    rv = stmt->ExecuteStep(&hasResult);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(hasResult);
+
+    nsCString journalMode;
+    rv = stmt->GetUTF8String(0, journalMode);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = stmt->Finalize();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (journalMode.EqualsLiteral("wal")) {
+      // We don't copy the WAL file, so make sure the old database is fully
+      // checkpointed.
+      rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA wal_checkpoint(TRUNCATE);"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    // Explicitely close the connection before the old database is copied.
+    rv = connection->Close();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    // Copy the old database. The database is copied from
+    // <profile>/webappsstore.sqlite to
+    // <profile>/storage/ls-archive-tmp.sqlite
+    // We use a "-tmp" postfix since we are not done yet.
+    nsCOMPtr<nsIFile> storageDir;
+    rv = NS_NewLocalFile(mStoragePath, false, getter_AddRefs(storageDir));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = webAppsStoreFile->CopyTo(storageDir,
+                                  NS_LITERAL_STRING(LS_ARCHIVE_TMP_FILE_NAME));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsCOMPtr<nsIFile> lsArchiveTmpFile;
+    rv = NS_NewLocalFile(mStoragePath, false, getter_AddRefs(lsArchiveTmpFile));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = lsArchiveTmpFile->Append(NS_LITERAL_STRING(LS_ARCHIVE_TMP_FILE_NAME));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (journalMode.EqualsLiteral("wal")) {
+      nsCOMPtr<mozIStorageConnection> lsArchiveTmpConnection;
+      rv = ss->OpenUnsharedDatabase(lsArchiveTmpFile,
+                                    getter_AddRefs(lsArchiveTmpConnection));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      // The archive will only be used for lazy data migration. There won't be
+      // any concurrent readers and writers that could benefit from Write-Ahead
+      // Logging. So switch to a standard rollback journal. The standard
+      // rollback journal also provides atomicity across multiple attached
+      // databases which is import for the lazy data migration to work safely.
+      rv = lsArchiveTmpConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+        "PRAGMA journal_mode = DELETE;"
+      ));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      // The connection will be now implicitely closed (it's always safer to
+      // close database connection before we manipulate underlying file)
+    }
+
+    // Finally, rename ls-archive-tmp.sqlite to ls-archive.sqlite
+    rv = lsArchiveTmpFile->MoveTo(nullptr,
+                                  NS_LITERAL_STRING(LS_ARCHIVE_FILE_NAME));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else {
+    // If webappsstore database is not useable, just create an empty archive.
+
+    // Ensure the storage directory actually exists.
+    nsCOMPtr<nsIFile> storageDirectory;
+    rv = NS_NewLocalFile(GetStoragePath(),
+                         false,
+                         getter_AddRefs(storageDirectory));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    bool dummy;
+    rv = EnsureDirectory(storageDirectory, &dummy);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsCOMPtr<mozIStorageConnection> lsArchiveConnection;
+    rv = ss->OpenUnsharedDatabase(lsArchiveFile,
+                                  getter_AddRefs(lsArchiveConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = StorageDBUpdater::Update(lsArchiveConnection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  return NS_OK;
+}
+
 #ifdef DEBUG
 
 void
@@ -5216,7 +5470,11 @@ QuotaManager::EnsureStorageIsInitialized()
     }
   }
 
-  rv = MaybeRemoveLocalStorageData();
+  if (CachedNextGenLocalStorageEnabled()) {
+    rv = MaybeCreateLocalStorageArchive();
+  } else {
+    rv = MaybeRemoveLocalStorageData();
+  }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index 498379bb3769..fc37d039967e 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -505,6 +505,9 @@ private:
   nsresult
   MaybeRemoveLocalStorageDirectories();
 
+  nsresult
+  MaybeCreateLocalStorageArchive();
+
   nsresult
   InitializeRepository(PersistenceType aPersistenceType);
 
diff --git a/dom/quota/test/unit/createLocalStorage_profile.zip b/dom/quota/test/unit/createLocalStorage_profile.zip
new file mode 100644
index 0000000000000000000000000000000000000000..d5958dbd5946fef2835373ad735f9992dcaf0cbb
GIT binary patch
literal 1006
zcmWIWW@Zs#U|`^2*c4mtv;0t{qA`%iz{J47#~{N{T#{dun4YRvT$qztk{TMq$-sR6
z)|Mz&x93r%72FJrEMFNJ7{L0u`M#Xg@eK7it$)f_C(s}*At@mxr63?60SH)=+}LUw
zI5|Ethq2kt<lQ5%b#a0K6GH&p>O0IoeEu!svq%P7%h15cAPls+JT)n?pr9D!0u=id
z!o#9L7|s5-7tiKNhl(&f*!X*S)2364ZVS%d7G&Jnq1&{Y>&^q$Ct526yj?xE9(yNy
zJ?l;3^L8s~pLt93uZa2|G=3n&JxlE1CWYGhCGOwvSU#U;UB0jWPexR$#p&+IsH49x
z>IZI?td%_;C}Mj3vDcUB+dp0{e{?ME&*ZczclK=lTK4I4{>4=h_Ml+H1Ag#Z<eTK*
zxwZ4I#eVy__S~oU%<x<G`C)f~*&pRPQMZ>#fzv|cTF$SRsk{Dq-j_!QKVE$7sBdTW
zZ`pU1y8TN3zFF)GDSfWAI6CxbNl5s@J=<5lY~Nh{ao4+dKbQAPZ_itQ-eAS0i<vt@
zYBe+R_zEu7{#a4I)BBI(>>o?MKY21YZuSnD=iM7-?*CEzb@`Vq`y%%4-nwt<u~(&A
zs`lRcd!wEmkIR8l4gY&TuRi-a`fuI$_DDOwX_aDKnbW3R-E;2vW0RLQ>sPtn%-s6z
z|I@DpGjuN}Mm}}5;`Fqgzi-Xq1w2PDZa)8w&+XyqW}Uk~7hTPJyTxhu*W|~)@~rRP
ztBYE1%=fucbXutW9NP^4E*s9}-I2R^#H-8szJ0TbeBONI-2K$FN;|J^|7-GBBldQN
z<%jLBdvltVfMdVcmgcU$pL^z0;ifa21AI0wS@W#aPHws9^U6@&Q+}(r+RLP!E{O=M
zoapBibbh)1Yl9m*x81(C@8=dh=fWB1S8rJJ>){*6>Yyi{7O`h8rS7<~uz17tr$65Z
zcr!A|G2_ZC5^zT?X#_FRvkogH>!2lDWHT{y0m96N#%iFMSn~wXU@Z9|z?+o~<P0Vt
LTnnVNnL#`N=MRHW

literal 0
HcmV?d00001

diff --git a/dom/quota/test/unit/head.js b/dom/quota/test/unit/head.js
index 55c810d92e26..7c0848bb1d43 100644
--- a/dom/quota/test/unit/head.js
+++ b/dom/quota/test/unit/head.js
@@ -223,7 +223,9 @@ function installPackage(packageName)
     let file = getRelativeFile(entryName);
 
     if (zipentry.isDirectory) {
-      file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+      if (!file.exists()) {
+        file.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+      }
     } else {
       let istream = zipReader.getInputStream(entryName);
 
diff --git a/dom/quota/test/unit/test_createLocalStorage.js b/dom/quota/test/unit/test_createLocalStorage.js
new file mode 100644
index 000000000000..c25df7100656
--- /dev/null
+++ b/dom/quota/test/unit/test_createLocalStorage.js
@@ -0,0 +1,161 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+async function testSteps()
+{
+  const webAppsStoreFile = "webappsstore.sqlite";
+  const lsArchiveFile = "storage/ls-archive.sqlite";
+  const lsArchiveTmpFile = "storage/ls-archive-tmp.sqlite";
+
+  function checkArchiveFileNotExists()
+  {
+    info("Checking archive tmp file");
+
+    let archiveTmpFile = getRelativeFile(lsArchiveTmpFile);
+
+    let exists = archiveTmpFile.exists();
+    ok(!exists, "archive tmp file doesn't exist");
+
+    info("Checking archive file");
+
+    let archiveFile = getRelativeFile(lsArchiveFile);
+
+    exists = archiveFile.exists();
+    ok(!exists, "archive file doesn't exist");
+  }
+
+  function checkArchiveFileExists()
+  {
+    info("Checking archive tmp file");
+
+    let archiveTmpFile = getRelativeFile(lsArchiveTmpFile);
+
+    let exists = archiveTmpFile.exists();
+    ok(!exists, "archive tmp file doesn't exist");
+
+    info("Checking archive file");
+
+    let archiveFile = getRelativeFile(lsArchiveFile);
+
+    exists = archiveFile.exists();
+    ok(exists, "archive file does exist");
+
+    info("Checking archive file size");
+
+    let fileSize = archiveFile.fileSize;
+    ok(fileSize > 0, "archive file size is greater than zero");
+  }
+
+  info("Setting pref");
+
+  SpecialPowers.setBoolPref("dom.storage.next_gen", true);
+
+  // Profile 1 - Nonexistent apps store file.
+  info("Clearing");
+
+  let request = clear();
+  await requestFinished(request);
+
+  let appsStoreFile = getRelativeFile(webAppsStoreFile);
+
+  let exists = appsStoreFile.exists();
+  ok(!exists, "apps store file doesn't exist");
+
+  checkArchiveFileNotExists();
+
+  try {
+    request = init();
+    await requestFinished(request);
+
+    ok(true, "Should not have thrown");
+  } catch(ex) {
+    ok(false, "Should not have thrown");
+  }
+
+  checkArchiveFileExists();
+
+  // Profile 2 - apps store file is a directory.
+  info("Clearing");
+
+  request = clear();
+  await requestFinished(request);
+
+  appsStoreFile.create(Ci.nsIFile.DIRECTORY_TYPE, parseInt("0755", 8));
+
+  checkArchiveFileNotExists();
+
+  try {
+    request = init();
+    await requestFinished(request);
+
+    ok(true, "Should not have thrown");
+  } catch(ex) {
+    ok(false, "Should not have thrown");
+  }
+
+  checkArchiveFileExists();
+
+  appsStoreFile.remove(true);
+
+  // Profile 3 - Corrupted apps store file.
+  info("Clearing");
+
+  request = clear();
+  await requestFinished(request);
+
+  let ostream = Cc["@mozilla.org/network/file-output-stream;1"]
+                .createInstance(Ci.nsIFileOutputStream);
+  ostream.init(appsStoreFile, -1, parseInt("0644", 8), 0);
+  ostream.write("foobar", 6);
+  ostream.close();
+
+  checkArchiveFileNotExists();
+
+  try {
+    request = init();
+    await requestFinished(request);
+
+    ok(true, "Should not have thrown");
+  } catch(ex) {
+    ok(false, "Should not have thrown");
+  }
+
+  checkArchiveFileExists();
+
+  appsStoreFile.remove(false);
+
+  // Profile 4 - Nonupdateable apps store file.
+  info("Clearing");
+
+  request = clear();
+  await requestFinished(request);
+
+  info("Installing package");
+
+  // The profile contains storage.sqlite and webappsstore.sqlite
+  // webappstore.sqlite was taken from FF 54 to force an upgrade.
+  // There's just one record in the webappsstore2 table. The record was
+  // modified by renaming the origin attribute userContextId to userContextKey.
+  // This triggers an error during the upgrade.
+  installPackage("createLocalStorage_profile");
+
+  let fileSize = appsStoreFile.fileSize;
+  ok(fileSize > 0, "apps store file size is greater than zero");
+
+  checkArchiveFileNotExists();
+
+  try {
+    request = init();
+    await requestFinished(request);
+
+    ok(true, "Should not have thrown");
+  } catch(ex) {
+    ok(false, "Should not have thrown");
+  }
+
+  checkArchiveFileExists();
+
+  appsStoreFile.remove(false);
+}
diff --git a/dom/quota/test/unit/xpcshell.ini b/dom/quota/test/unit/xpcshell.ini
index 6104caae4659..6499a01a3428 100644
--- a/dom/quota/test/unit/xpcshell.ini
+++ b/dom/quota/test/unit/xpcshell.ini
@@ -6,6 +6,7 @@
 head = head.js
 support-files =
   basics_profile.zip
+  createLocalStorage_profile.zip
   defaultStorageUpgrade_profile.zip
   getUsage_profile.zip
   idbSubdirUpgrade1_profile.zip
@@ -22,6 +23,7 @@ support-files =
 
 [test_basics.js]
 [test_bad_origin_directory.js]
+[test_createLocalStorage.js]
 [test_defaultStorageUpgrade.js]
 [test_getUsage.js]
 [test_idbSubdirUpgrade.js]
diff --git a/dom/storage/moz.build b/dom/storage/moz.build
index c1173af7d02c..345a018c96df 100644
--- a/dom/storage/moz.build
+++ b/dom/storage/moz.build
@@ -14,6 +14,7 @@ EXPORTS.mozilla.dom += [
     'SessionStorageManager.h',
     'Storage.h',
     'StorageActivityService.h',
+    'StorageDBUpdater.h',
     'StorageIPC.h',
     'StorageNotifierService.h',
     'StorageObserver.h',

From 1812608353e249efe88253e846aa755d0efa85e2 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:41 +0100
Subject: [PATCH 41/78] Bug 1286798 - Part 27: Share database actors; r=asuth

If a database actor already exists for given origin, reuse it instead of creating a new one. This improves memory footprint a bit and also eliminates some round trips to the parent process.
---
 dom/localstorage/ActorsChild.cpp            |  63 ----
 dom/localstorage/ActorsChild.h              |  40 ---
 dom/localstorage/ActorsParent.cpp           | 334 +++++++-------------
 dom/localstorage/ActorsParent.h             |  20 +-
 dom/localstorage/LSDatabase.cpp             |  51 ++-
 dom/localstorage/LSDatabase.h               |  16 +-
 dom/localstorage/LSObject.cpp               |  63 ++--
 dom/localstorage/LSObject.h                 |  11 -
 dom/localstorage/PBackgroundLSDatabase.ipdl |  10 +-
 dom/localstorage/PBackgroundLSObject.ipdl   |  27 --
 dom/localstorage/moz.build                  |   1 -
 ipc/glue/BackgroundChildImpl.cpp            |  16 +-
 ipc/glue/BackgroundChildImpl.h              |  12 +-
 ipc/glue/BackgroundParentImpl.cpp           |  36 +--
 ipc/glue/BackgroundParentImpl.h             |  21 +-
 ipc/glue/PBackground.ipdl                   |  10 +-
 16 files changed, 248 insertions(+), 483 deletions(-)
 delete mode 100644 dom/localstorage/PBackgroundLSObject.ipdl

diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index c0e7a68c331a..26645ac3bcea 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -14,69 +14,6 @@
 namespace mozilla {
 namespace dom {
 
-/*******************************************************************************
- * LSObjectChild
- ******************************************************************************/
-
-LSObjectChild::LSObjectChild(LSObject* aObject)
-  : mObject(aObject)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(aObject);
-  aObject->AssertIsOnOwningThread();
-
-  MOZ_COUNT_CTOR(LSObjectChild);
-}
-
-LSObjectChild::~LSObjectChild()
-{
-  AssertIsOnOwningThread();
-
-  MOZ_COUNT_DTOR(LSObjectChild);
-}
-
-void
-LSObjectChild::SendDeleteMeInternal()
-{
-  AssertIsOnOwningThread();
-
-  if (mObject) {
-    mObject->ClearActor();
-    mObject = nullptr;
-
-    MOZ_ALWAYS_TRUE(PBackgroundLSObjectChild::SendDeleteMe());
-  }
-}
-
-void
-LSObjectChild::ActorDestroy(ActorDestroyReason aWhy)
-{
-  AssertIsOnOwningThread();
-
-  if (mObject) {
-    mObject->ClearActor();
-#ifdef DEBUG
-    mObject = nullptr;
-#endif
-  }
-}
-
-LSObjectChild::PBackgroundLSDatabaseChild*
-LSObjectChild::AllocPBackgroundLSDatabaseChild(const uint64_t& aDatastoreId)
-{
-  MOZ_CRASH("PBackgroundLSDatabaseChild actor should be manually constructed!");
-}
-
-bool
-LSObjectChild::DeallocPBackgroundLSDatabaseChild(
-                                             PBackgroundLSDatabaseChild* aActor)
-{
-  MOZ_ASSERT(aActor);
-
-  delete aActor;
-  return true;
-}
-
 /*******************************************************************************
  * LSDatabaseChild
  ******************************************************************************/
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 100234ccb384..97986e88e969 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -8,7 +8,6 @@
 #define mozilla_dom_localstorage_ActorsChild_h
 
 #include "mozilla/dom/PBackgroundLSDatabaseChild.h"
-#include "mozilla/dom/PBackgroundLSObjectChild.h"
 #include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestChild.h"
@@ -30,45 +29,6 @@ class LSObserver;
 class LSRequestChildCallback;
 class LSSimpleRequestChildCallback;
 
-class LSObjectChild final
-  : public PBackgroundLSObjectChild
-{
-  friend class mozilla::ipc::BackgroundChildImpl;
-  friend class LSObject;
-
-  LSObject* mObject;
-
-  NS_DECL_OWNINGTHREAD
-
-public:
-  void
-  AssertIsOnOwningThread() const
-  {
-    NS_ASSERT_OWNINGTHREAD(LSObjectChild);
-  }
-
-private:
-  // Only created by LSObject.
-  explicit LSObjectChild(LSObject* aObject);
-
-  // Only destroyed by mozilla::ipc::BackgroundChildImpl.
-  ~LSObjectChild();
-
-  void
-  SendDeleteMeInternal();
-
-  // IPDL methods are only called by IPDL.
-  void
-  ActorDestroy(ActorDestroyReason aWhy) override;
-
-  PBackgroundLSDatabaseChild*
-  AllocPBackgroundLSDatabaseChild(const uint64_t& aDatastoreId) override;
-
-  bool
-  DeallocPBackgroundLSDatabaseChild(PBackgroundLSDatabaseChild* aActor)
-                                    override;
-};
-
 class LSDatabaseChild final
   : public PBackgroundLSDatabaseChild
 {
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 5da3b1266961..d6d0005e79f3 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -15,7 +15,6 @@
 #include "mozilla/Preferences.h"
 #include "mozilla/Unused.h"
 #include "mozilla/dom/PBackgroundLSDatabaseParent.h"
-#include "mozilla/dom/PBackgroundLSObjectParent.h"
 #include "mozilla/dom/PBackgroundLSObserverParent.h"
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
@@ -1114,17 +1113,20 @@ public:
 
   void
   SetItem(Database* aDatabase,
+          const nsString& aDocumentURI,
           const nsString& aKey,
           const nsString& aValue,
           LSWriteOpResponse& aResponse);
 
   void
   RemoveItem(Database* aDatabase,
+             const nsString& aDocumentURI,
              const nsString& aKey,
              LSWriteOpResponse& aResponse);
 
   void
   Clear(Database* aDatabase,
+        const nsString& aDocumentURI,
         LSWriteOpResponse& aResponse);
 
   void
@@ -1150,6 +1152,7 @@ private:
 
   void
   NotifyObservers(Database* aDatabase,
+                  const nsString& aDocumentURI,
                   const nsString& aKey,
                   const nsString& aOldValue,
                   const nsString& aNewValue);
@@ -1262,19 +1265,27 @@ private:
  * Actor class declarations
  ******************************************************************************/
 
-class Object final
-  : public PBackgroundLSObjectParent
+class Database final
+  : public PBackgroundLSDatabaseParent
 {
+  RefPtr<Datastore> mDatastore;
   const PrincipalInfo mPrincipalInfo;
-  const nsString mDocumentURI;
+  // Strings share buffers if possible, so it's not a problem to duplicate the
+  // origin here.
+  nsCString mOrigin;
   uint32_t mPrivateBrowsingId;
+  bool mAllowedToClose;
   bool mActorDestroyed;
+  bool mRequestedAllowToClose;
+#ifdef DEBUG
+  bool mActorWasAlive;
+#endif
 
 public:
-  // Created in AllocPBackgroundLSObjectParent.
-  Object(const PrincipalInfo& aPrincipalInfo,
-         const nsAString& aDocumentURI,
-         uint32_t aPrivateBrowsingId);
+  // Created in AllocPBackgroundLSDatabaseParent.
+  Database(const PrincipalInfo& aPrincipalInfo,
+           const nsACString& aOrigin,
+           uint32_t aPrivateBrowsingId);
 
   const PrincipalInfo&
   GetPrincipalInfo() const
@@ -1288,65 +1299,6 @@ public:
     return mPrivateBrowsingId;
   }
 
-  const nsString&
-  DocumentURI() const
-  {
-    return mDocumentURI;
-  }
-
-  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Object)
-
-private:
-  // Reference counted.
-  ~Object();
-
-  // IPDL methods are only called by IPDL.
-  void
-  ActorDestroy(ActorDestroyReason aWhy) override;
-
-  mozilla::ipc::IPCResult
-  RecvDeleteMe() override;
-
-  PBackgroundLSDatabaseParent*
-  AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId) override;
-
-  mozilla::ipc::IPCResult
-  RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
-                                       const uint64_t& aDatastoreId) override;
-
-  bool
-  DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
-                                     override;
-};
-
-class Database final
-  : public PBackgroundLSDatabaseParent
-{
-  RefPtr<Object> mObject;
-  RefPtr<Datastore> mDatastore;
-  // Strings share buffers if possible, so it's not a problem to duplicate the
-  // origin here.
-  nsCString mOrigin;
-  bool mAllowedToClose;
-  bool mActorDestroyed;
-  bool mRequestedAllowToClose;
-#ifdef DEBUG
-  bool mActorWasAlive;
-#endif
-
-public:
-  // Created in AllocPBackgroundLSDatabaseParent.
-  Database(Object* aObject,
-           const nsACString& aOrigin);
-
-  Object*
-  GetObject() const
-  {
-    AssertIsOnBackgroundThread();
-
-    return mObject;
-  }
-
   const nsCString&
   Origin() const
   {
@@ -1391,16 +1343,19 @@ private:
   RecvGetKeys(nsTArray<nsString>* aKeys) override;
 
   mozilla::ipc::IPCResult
-  RecvSetItem(const nsString& aKey,
+  RecvSetItem(const nsString& aDocumentURI,
+              const nsString& aKey,
               const nsString& aValue,
               LSWriteOpResponse* aResponse) override;
 
   mozilla::ipc::IPCResult
-  RecvRemoveItem(const nsString& aKey,
+  RecvRemoveItem(const nsString& aDocumentURI,
+                 const nsString& aKey,
                  LSWriteOpResponse* aResponse) override;
 
   mozilla::ipc::IPCResult
-  RecvClear(LSWriteOpResponse* aResponse) override;
+  RecvClear(const nsString& aDocumentURI,
+            LSWriteOpResponse* aResponse) override;
 };
 
 class Observer final
@@ -1421,6 +1376,7 @@ public:
 
   void
   Observe(Database* aDatabase,
+          const nsString& aDocumentURI,
           const nsString& aKey,
           const nsString& aOldValue,
           const nsString& aNewValue);
@@ -2189,10 +2145,10 @@ GetUsage(mozIStorageConnection* aConnection,
  * Exported functions
  ******************************************************************************/
 
-PBackgroundLSObjectParent*
-AllocPBackgroundLSObjectParent(const PrincipalInfo& aPrincipalInfo,
-                               const nsString& aDocumentURI,
-                               const uint32_t& aPrivateBrowsingId)
+PBackgroundLSDatabaseParent*
+AllocPBackgroundLSDatabaseParent(const PrincipalInfo& aPrincipalInfo,
+                                 const uint32_t& aPrivateBrowsingId,
+                                 const uint64_t& aDatastoreId)
 {
   AssertIsOnBackgroundThread();
 
@@ -2200,34 +2156,73 @@ AllocPBackgroundLSObjectParent(const PrincipalInfo& aPrincipalInfo,
     return nullptr;
   }
 
-  RefPtr<Object> object =
-    new Object(aPrincipalInfo, aDocumentURI, aPrivateBrowsingId);
+  if (NS_WARN_IF(!gPreparedDatastores)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  PreparedDatastore* preparedDatastore = gPreparedDatastores->Get(aDatastoreId);
+  if (NS_WARN_IF(!preparedDatastore)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  // If we ever decide to return null from this point on, we need to make sure
+  // that the datastore is closed and the prepared datastore is removed from the
+  // gPreparedDatastores hashtable.
+  // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
+  // once we return a valid actor in this method.
+
+  RefPtr<Database> database = new Database(aPrincipalInfo,
+                                           preparedDatastore->Origin(),
+                                           aPrivateBrowsingId);
 
   // Transfer ownership to IPDL.
-  return object.forget().take();
+  return database.forget().take();
 }
 
 bool
-RecvPBackgroundLSObjectConstructor(PBackgroundLSObjectParent* aActor,
-                                   const PrincipalInfo& aPrincipalInfo,
-                                   const nsString& aDocumentURI,
-                                   const uint32_t& aPrivateBrowsingId)
+RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                     const PrincipalInfo& aPrincipalInfo,
+                                     const uint32_t& aPrivateBrowsingId,
+                                     const uint64_t& aDatastoreId)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
+  MOZ_ASSERT(gPreparedDatastores);
+  MOZ_ASSERT(gPreparedDatastores->Get(aDatastoreId));
   MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
 
+  // The actor is now completely built (it has a manager, channel and it's
+  // registered as a subprotocol).
+  // ActorDestroy will be called if we fail here.
+
+  nsAutoPtr<PreparedDatastore> preparedDatastore;
+  gPreparedDatastores->Remove(aDatastoreId, &preparedDatastore);
+  MOZ_ASSERT(preparedDatastore);
+
+  auto* database = static_cast<Database*>(aActor);
+
+  database->SetActorAlive(preparedDatastore->GetDatastore());
+
+  // It's possible that AbortOperations was called before the database actor
+  // was created and became live. Let the child know that the database in no
+  // longer valid.
+  if (preparedDatastore->IsInvalidated()) {
+    database->RequestAllowToClose();
+  }
+
   return true;
 }
 
 bool
-DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor)
+DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
   // Transfer ownership back from IPDL.
-  RefPtr<Object> actor = dont_AddRef(static_cast<Object*>(aActor));
+  RefPtr<Database> actor = dont_AddRef(static_cast<Database*>(aActor));
 
   return true;
 }
@@ -3178,6 +3173,7 @@ Datastore::GetItem(const nsString& aKey, nsString& aValue) const
 
 void
 Datastore::SetItem(Database* aDatabase,
+                   const nsString& aDocumentURI,
                    const nsString& aKey,
                    const nsString& aValue,
                    LSWriteOpResponse& aResponse)
@@ -3213,7 +3209,7 @@ Datastore::SetItem(Database* aDatabase,
 
     mValues.Put(aKey, aValue);
 
-    NotifyObservers(aDatabase, aKey, oldValue, aValue);
+    NotifyObservers(aDatabase, aDocumentURI, aKey, oldValue, aValue);
 
     if (IsPersistent()) {
       EnsureTransaction();
@@ -3231,6 +3227,7 @@ Datastore::SetItem(Database* aDatabase,
 
 void
 Datastore::RemoveItem(Database* aDatabase,
+                      const nsString& aDocumentURI,
                       const nsString& aKey,
                       LSWriteOpResponse& aResponse)
 {
@@ -3254,7 +3251,7 @@ Datastore::RemoveItem(Database* aDatabase,
 
     mValues.Remove(aKey);
 
-    NotifyObservers(aDatabase, aKey, oldValue, VoidString());
+    NotifyObservers(aDatabase, aDocumentURI, aKey, oldValue, VoidString());
 
     if (IsPersistent()) {
       EnsureTransaction();
@@ -3272,6 +3269,7 @@ Datastore::RemoveItem(Database* aDatabase,
 
 void
 Datastore::Clear(Database* aDatabase,
+                 const nsString& aDocumentURI,
                  LSWriteOpResponse& aResponse)
 {
   AssertIsOnBackgroundThread();
@@ -3289,7 +3287,11 @@ Datastore::Clear(Database* aDatabase,
     mValues.Clear();
 
     if (aDatabase) {
-      NotifyObservers(aDatabase, VoidString(), VoidString(), VoidString());
+      NotifyObservers(aDatabase,
+                      aDocumentURI,
+                      VoidString(),
+                      VoidString(),
+                      VoidString());
     }
 
     if (IsPersistent()) {
@@ -3413,6 +3415,7 @@ Datastore::UpdateUsage(int64_t aDelta)
 
 void
 Datastore::NotifyObservers(Database* aDatabase,
+                           const nsString& aDocumentURI,
                            const nsString& aKey,
                            const nsString& aOldValue,
                            const nsString& aNewValue)
@@ -3431,11 +3434,11 @@ Datastore::NotifyObservers(Database* aDatabase,
 
   MOZ_ASSERT(array);
 
-  PBackgroundParent* databaseBackgroundActor = aDatabase->Manager()->Manager();
+  PBackgroundParent* databaseBackgroundActor = aDatabase->Manager();
 
   for (Observer* observer : *array) {
     if (observer->Manager() != databaseBackgroundActor) {
-      observer->Observe(aDatabase, aKey, aOldValue, aNewValue);
+      observer->Observe(aDatabase, aDocumentURI, aKey, aOldValue, aNewValue);
     }
   }
 }
@@ -3507,134 +3510,16 @@ PreparedDatastore::TimerCallback(nsITimer* aTimer, void* aClosure)
   self->Destroy();
 }
 
-/*******************************************************************************
- * Object
- ******************************************************************************/
-
-Object::Object(const PrincipalInfo& aPrincipalInfo,
-               const nsAString& aDocumentURI,
-               uint32_t aPrivateBrowsingId)
-  : mPrincipalInfo(aPrincipalInfo)
-  , mDocumentURI(aDocumentURI)
-  , mPrivateBrowsingId(aPrivateBrowsingId)
-  , mActorDestroyed(false)
-{
-  AssertIsOnBackgroundThread();
-}
-
-Object::~Object()
-{
-  MOZ_ASSERT(mActorDestroyed);
-}
-
-void
-Object::ActorDestroy(ActorDestroyReason aWhy)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(!mActorDestroyed);
-
-  mActorDestroyed = true;
-}
-
-mozilla::ipc::IPCResult
-Object::RecvDeleteMe()
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(!mActorDestroyed);
-
-  IProtocol* mgr = Manager();
-  if (!PBackgroundLSObjectParent::Send__delete__(this)) {
-    return IPC_FAIL_NO_REASON(mgr);
-  }
-  return IPC_OK();
-}
-
-PBackgroundLSDatabaseParent*
-Object::AllocPBackgroundLSDatabaseParent(const uint64_t& aDatastoreId)
-{
-  AssertIsOnBackgroundThread();
-
-  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread())) {
-    return nullptr;
-  }
-
-  if (NS_WARN_IF(!gPreparedDatastores)) {
-    ASSERT_UNLESS_FUZZING();
-    return nullptr;
-  }
-
-  PreparedDatastore* preparedDatastore = gPreparedDatastores->Get(aDatastoreId);
-  if (NS_WARN_IF(!preparedDatastore)) {
-    ASSERT_UNLESS_FUZZING();
-    return nullptr;
-  }
-
-  // If we ever decide to return null from this point on, we need to make sure
-  // that the datastore is closed and the prepared datastore is removed from the
-  // gPreparedDatastores hashtable.
-  // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
-  // once we return a valid actor in this method.
-
-  RefPtr<Database> database = new Database(this,
-                                           preparedDatastore->Origin());
-
-  // Transfer ownership to IPDL.
-  return database.forget().take();
-}
-
-mozilla::ipc::IPCResult
-Object::RecvPBackgroundLSDatabaseConstructor(
-                                            PBackgroundLSDatabaseParent* aActor,
-                                            const uint64_t& aDatastoreId)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aActor);
-  MOZ_ASSERT(gPreparedDatastores);
-  MOZ_ASSERT(gPreparedDatastores->Get(aDatastoreId));
-  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
-
-  // The actor is now completely built (it has a manager, channel and it's
-  // registered as a subprotocol).
-  // ActorDestroy will be called if we fail here.
-
-  nsAutoPtr<PreparedDatastore> preparedDatastore;
-  gPreparedDatastores->Remove(aDatastoreId, &preparedDatastore);
-  MOZ_ASSERT(preparedDatastore);
-
-  auto* database = static_cast<Database*>(aActor);
-
-  database->SetActorAlive(preparedDatastore->GetDatastore());
-
-  // It's possible that AbortOperations was called before the database actor
-  // was created and became live. Let the child know that the database in no
-  // longer valid.
-  if (preparedDatastore->IsInvalidated()) {
-    database->RequestAllowToClose();
-  }
-
-  return IPC_OK();
-}
-
-bool
-Object::DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aActor);
-
-  // Transfer ownership back from IPDL.
-  RefPtr<Database> actor = dont_AddRef(static_cast<Database*>(aActor));
-
-  return true;
-}
-
 /*******************************************************************************
  * Database
  ******************************************************************************/
 
-Database::Database(Object* aObject,
-                   const nsACString& aOrigin)
-  : mObject(aObject)
+Database::Database(const PrincipalInfo& aPrincipalInfo,
+                   const nsACString& aOrigin,
+                   uint32_t aPrivateBrowsingId)
+  : mPrincipalInfo(aPrincipalInfo)
   , mOrigin(aOrigin)
+  , mPrivateBrowsingId(aPrivateBrowsingId)
   , mAllowedToClose(false)
   , mActorDestroyed(false)
   , mRequestedAllowToClose(false)
@@ -3807,7 +3692,8 @@ Database::RecvGetItem(const nsString& aKey, nsString* aValue)
 }
 
 mozilla::ipc::IPCResult
-Database::RecvSetItem(const nsString& aKey,
+Database::RecvSetItem(const nsString& aDocumentURI,
+                      const nsString& aKey,
                       const nsString& aValue,
                       LSWriteOpResponse* aResponse)
 {
@@ -3820,13 +3706,14 @@ Database::RecvSetItem(const nsString& aKey,
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->SetItem(this, aKey, aValue, *aResponse);
+  mDatastore->SetItem(this, aDocumentURI, aKey, aValue, *aResponse);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvRemoveItem(const nsString& aKey,
+Database::RecvRemoveItem(const nsString& aDocumentURI,
+                         const nsString& aKey,
                          LSWriteOpResponse* aResponse)
 {
   AssertIsOnBackgroundThread();
@@ -3838,13 +3725,14 @@ Database::RecvRemoveItem(const nsString& aKey,
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->RemoveItem(this, aKey, *aResponse);
+  mDatastore->RemoveItem(this, aDocumentURI, aKey, *aResponse);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvClear(LSWriteOpResponse* aResponse)
+Database::RecvClear(const nsString& aDocumentURI,
+                    LSWriteOpResponse* aResponse)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aResponse);
@@ -3855,7 +3743,7 @@ Database::RecvClear(LSWriteOpResponse* aResponse)
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->Clear(this, *aResponse);
+  mDatastore->Clear(this, aDocumentURI, *aResponse);
 
   return IPC_OK();
 }
@@ -3895,6 +3783,7 @@ Observer::~Observer()
 
 void
 Observer::Observe(Database* aDatabase,
+                  const nsString& aDocumentURI,
                   const nsString& aKey,
                   const nsString& aOldValue,
                   const nsString& aNewValue)
@@ -3902,12 +3791,9 @@ Observer::Observe(Database* aDatabase,
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
 
-  Object* object = aDatabase->GetObject();
-  MOZ_ASSERT(object);
-
-  Unused << SendObserve(object->GetPrincipalInfo(),
-                        object->PrivateBrowsingId(),
-                        object->DocumentURI(),
+  Unused << SendObserve(aDatabase->GetPrincipalInfo(),
+                        aDatabase->PrivateBrowsingId(),
+                        aDocumentURI,
                         aKey,
                         aOldValue,
                         aNewValue);
@@ -5827,7 +5713,7 @@ ClearPrivateBrowsingRunnable::Run()
 
       if (datastore->PrivateBrowsingId()) {
         LSWriteOpResponse dummy;
-        datastore->Clear(nullptr, dummy);
+        datastore->Clear(nullptr, EmptyString(), dummy);
       }
     }
   }
diff --git a/dom/localstorage/ActorsParent.h b/dom/localstorage/ActorsParent.h
index 006ba0efaa01..b59a2ee138ac 100644
--- a/dom/localstorage/ActorsParent.h
+++ b/dom/localstorage/ActorsParent.h
@@ -20,7 +20,7 @@ namespace dom {
 
 class LSRequestParams;
 class LSSimpleRequestParams;
-class PBackgroundLSObjectParent;
+class PBackgroundLSDatabaseParent;
 class PBackgroundLSObserverParent;
 class PBackgroundLSRequestParent;
 class PBackgroundLSSimpleRequestParent;
@@ -31,21 +31,21 @@ class Client;
 
 } // namespace quota
 
-PBackgroundLSObjectParent*
-AllocPBackgroundLSObjectParent(
+PBackgroundLSDatabaseParent*
+AllocPBackgroundLSDatabaseParent(
                               const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
-                              const nsString& aDocumentURI,
-                              const uint32_t& aPrivateBrowsingId);
+                              const uint32_t& aPrivateBrowsingId,
+                              const uint64_t& aDatastoreId);
 
 bool
-RecvPBackgroundLSObjectConstructor(
-                              PBackgroundLSObjectParent* aActor,
+RecvPBackgroundLSDatabaseConstructor(
+                              PBackgroundLSDatabaseParent* aActor,
                               const mozilla::ipc::PrincipalInfo& aPrincipalInfo,
-                              const nsString& aDocumentURI,
-                              const uint32_t& aPrivateBrowsingId);
+                              const uint32_t& aPrivateBrowsingId,
+                              const uint64_t& aDatastoreId);
 
 bool
-DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor);
+DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor);
 
 PBackgroundLSObserverParent*
 AllocPBackgroundLSObserverParent(const uint64_t& aObserverId);
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index 96cbff118684..d41b724f9917 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -9,11 +9,27 @@
 namespace mozilla {
 namespace dom {
 
-LSDatabase::LSDatabase()
+namespace {
+
+typedef nsDataHashtable<nsCStringHashKey, LSDatabase*> LSDatabaseHashtable;
+
+StaticAutoPtr<LSDatabaseHashtable> gLSDatabases;
+
+} // namespace
+
+LSDatabase::LSDatabase(const nsACString& aOrigin)
   : mActor(nullptr)
+  , mOrigin(aOrigin)
   , mAllowedToClose(false)
 {
   AssertIsOnOwningThread();
+
+  if (!gLSDatabases) {
+    gLSDatabases = new LSDatabaseHashtable();
+  }
+
+  MOZ_ASSERT(!gLSDatabases->Get(mOrigin));
+  gLSDatabases->Put(mOrigin, this);
 }
 
 LSDatabase::~LSDatabase()
@@ -30,6 +46,13 @@ LSDatabase::~LSDatabase()
   }
 }
 
+// static
+LSDatabase*
+LSDatabase::Get(const nsACString& aOrigin)
+{
+  return gLSDatabases ? gLSDatabases->Get(aOrigin) : nullptr;
+}
+
 void
 LSDatabase::SetActor(LSDatabaseChild* aActor)
 {
@@ -51,6 +74,14 @@ LSDatabase::AllowToClose()
   if (mActor) {
     mActor->SendAllowToClose();
   }
+
+  MOZ_ASSERT(gLSDatabases);
+  MOZ_ASSERT(gLSDatabases->Get(mOrigin));
+  gLSDatabases->Remove(mOrigin);
+
+  if (!gLSDatabases->Count()) {
+    gLSDatabases = nullptr;
+  }
 }
 
 nsresult
@@ -120,7 +151,8 @@ LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
 }
 
 nsresult
-LSDatabase::SetItem(const nsAString& aKey,
+LSDatabase::SetItem(const nsAString& aDocumentURI,
+                    const nsAString& aKey,
                     const nsAString& aValue,
                     LSWriteOpResponse& aResponse)
 {
@@ -129,7 +161,8 @@ LSDatabase::SetItem(const nsAString& aKey,
   MOZ_ASSERT(!mAllowedToClose);
 
   LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aKey),
+  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aDocumentURI),
+                                      nsString(aKey),
                                       nsString(aValue),
                                       &response))) {
     return NS_ERROR_FAILURE;
@@ -140,7 +173,8 @@ LSDatabase::SetItem(const nsAString& aKey,
 }
 
 nsresult
-LSDatabase::RemoveItem(const nsAString& aKey,
+LSDatabase::RemoveItem(const nsAString& aDocumentURI,
+                       const nsAString& aKey,
                        LSWriteOpResponse& aResponse)
 {
   AssertIsOnOwningThread();
@@ -148,7 +182,9 @@ LSDatabase::RemoveItem(const nsAString& aKey,
   MOZ_ASSERT(!mAllowedToClose);
 
   LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aKey), &response))) {
+  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aDocumentURI),
+                                         nsString(aKey),
+                                         &response))) {
     return NS_ERROR_FAILURE;
   }
 
@@ -157,14 +193,15 @@ LSDatabase::RemoveItem(const nsAString& aKey,
 }
 
 nsresult
-LSDatabase::Clear(LSWriteOpResponse& aResponse)
+LSDatabase::Clear(const nsAString& aDocumentURI,
+                  LSWriteOpResponse& aResponse)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
   LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendClear(&response))) {
+  if (NS_WARN_IF(!mActor->SendClear(nsString(aDocumentURI), &response))) {
     return NS_ERROR_FAILURE;
   }
 
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index c0c391c3bc1e..919c66ec427e 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -17,10 +17,15 @@ class LSDatabase final
 {
   LSDatabaseChild* mActor;
 
+  const nsCString mOrigin;
+
   bool mAllowedToClose;
 
 public:
-  LSDatabase();
+  explicit LSDatabase(const nsACString& aOrigin);
+
+  static LSDatabase*
+  Get(const nsACString& aOrigin);
 
   NS_INLINE_DECL_REFCOUNTING(LSDatabase)
 
@@ -68,16 +73,19 @@ public:
   GetKeys(nsTArray<nsString>& aKeys);
 
   nsresult
-  SetItem(const nsAString& aKey,
+  SetItem(const nsAString& aDocumentURI,
+          const nsAString& aKey,
           const nsAString& aValue,
           LSWriteOpResponse& aResponse);
 
   nsresult
-  RemoveItem(const nsAString& aKey,
+  RemoveItem(const nsAString& aDocumentURI,
+             const nsAString& aKey,
              LSWriteOpResponse& aResponse);
 
   nsresult
-  Clear(LSWriteOpResponse& aResponse);
+  Clear(const nsAString& aDocumentURI,
+        LSWriteOpResponse& aResponse);
 
 private:
   ~LSDatabase();
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index b01e475a9d63..e0fc0fb8f747 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -127,9 +127,7 @@ private:
 LSObject::LSObject(nsPIDOMWindowInner* aWindow,
                    nsIPrincipal* aPrincipal)
   : Storage(aWindow, aPrincipal)
-  , mActor(nullptr)
   , mPrivateBrowsingId(0)
-  , mActorFailed(false)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(NextGenLocalStorageEnabled());
@@ -138,14 +136,8 @@ LSObject::LSObject(nsPIDOMWindowInner* aWindow,
 LSObject::~LSObject()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT_IF(mActorFailed, !mActor);
 
   DropObserver();
-
-  if (mActor) {
-    mActor->SendDeleteMeInternal();
-    MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
-  }
 }
 
 // static
@@ -466,7 +458,7 @@ LSObject::SetItem(const nsAString& aKey,
   }
 
   LSWriteOpResponse response;
-  rv = mDatabase->SetItem(aKey, aValue, response);
+  rv = mDatabase->SetItem(mDocumentURI, aKey, aValue, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
@@ -503,7 +495,7 @@ LSObject::RemoveItem(const nsAString& aKey,
   }
 
   LSWriteOpResponse response;
-  rv = mDatabase->RemoveItem(aKey, response);
+  rv = mDatabase->RemoveItem(mDocumentURI, aKey, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
@@ -539,7 +531,7 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
   }
 
   LSWriteOpResponse response;
-  rv = mDatabase->Clear(response);
+  rv = mDatabase->Clear(mDocumentURI, response);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
@@ -652,33 +644,20 @@ LSObject::EnsureDatabase()
     return NS_OK;
   }
 
-  mDatabase = nullptr;
+  mDatabase = LSDatabase::Get(mOrigin);
 
-  if (mActorFailed) {
-    return NS_ERROR_FAILURE;
+  if (mDatabase) {
+    MOZ_ASSERT(!mDatabase->IsAllowedToClose());
+    return NS_OK;
   }
 
-  if (!mActor) {
-    PBackgroundChild* backgroundActor =
-      BackgroundChild::GetOrCreateForCurrentThread();
-    if (NS_WARN_IF(!backgroundActor)) {
-      return NS_ERROR_FAILURE;
-    }
-
-    LSObjectChild* actor = new LSObjectChild(this);
-
-    mActor =
-      static_cast<LSObjectChild*>(
-        backgroundActor->SendPBackgroundLSObjectConstructor(
-                                                           actor,
-                                                           *mPrincipalInfo,
-                                                           mDocumentURI,
-                                                           mPrivateBrowsingId));
-
-    if (NS_WARN_IF(!mActor)) {
-      mActorFailed = true;
-      return NS_ERROR_FAILURE;
-    }
+  // We don't need this yet, but once the request successfully finishes, it's
+  // too late to initialize PBackground child on the owning thread, because
+  // it can fail and parent would keep an extra strong ref to the datastore.
+  PBackgroundChild* backgroundActor =
+    BackgroundChild::GetOrCreateForCurrentThread();
+  if (NS_WARN_IF(!backgroundActor)) {
+    return NS_ERROR_FAILURE;
   }
 
   LSRequestPrepareDatastoreParams params;
@@ -709,12 +688,15 @@ LSObject::EnsureDatabase()
   // Note that we now can't error out, otherwise parent will keep an extra
   // strong reference to the datastore.
 
-  RefPtr<LSDatabase> database = new LSDatabase();
+  RefPtr<LSDatabase> database = new LSDatabase(mOrigin);
 
   LSDatabaseChild* actor = new LSDatabaseChild(database);
 
   MOZ_ALWAYS_TRUE(
-    mActor->SendPBackgroundLSDatabaseConstructor(actor, datastoreId));
+    backgroundActor->SendPBackgroundLSDatabaseConstructor(actor,
+                                                          *mPrincipalInfo,
+                                                          mPrivateBrowsingId,
+                                                          datastoreId));
 
   database->SetActor(actor);
 
@@ -728,12 +710,7 @@ LSObject::DropDatabase()
 {
   AssertIsOnOwningThread();
 
-  if (mDatabase) {
-    if (!mDatabase->IsAllowedToClose()) {
-      mDatabase->AllowToClose();
-    }
-    mDatabase = nullptr;
-  }
+  mDatabase = nullptr;
 }
 
 nsresult
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index 3ed1d1867f51..8eb42248792a 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -47,12 +47,9 @@ class LSObject final
   RefPtr<LSDatabase> mDatabase;
   RefPtr<LSObserver> mObserver;
 
-  LSObjectChild* mActor;
-
   uint32_t mPrivateBrowsingId;
   nsCString mOrigin;
   nsString mDocumentURI;
-  bool mActorFailed;
 
 public:
   static nsresult
@@ -83,14 +80,6 @@ public:
     NS_ASSERT_OWNINGTHREAD(LSObject);
   }
 
-  void
-  ClearActor()
-  {
-    AssertIsOnOwningThread();
-
-    mActor = nullptr;
-  }
-
   LSRequestChild*
   StartRequest(nsIEventTarget* aMainEventTarget,
                const LSRequestParams& aParams,
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 78d77c7e172d..1bdce2ded6db 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -2,7 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this file,
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
-include protocol PBackgroundLSObject;
+include protocol PBackground;
 
 namespace mozilla {
 namespace dom {
@@ -21,7 +21,7 @@ union LSWriteOpResponse
 
 sync protocol PBackgroundLSDatabase
 {
-  manager PBackgroundLSObject;
+  manager PBackground;
 
 parent:
   // The DeleteMe message is used to avoid a race condition between the parent
@@ -49,13 +49,13 @@ parent:
   sync GetKeys()
     returns (nsString[] keys);
 
-  sync SetItem(nsString key, nsString value)
+  sync SetItem(nsString documentURI, nsString key, nsString value)
     returns (LSWriteOpResponse response);
 
-  sync RemoveItem(nsString key)
+  sync RemoveItem(nsString documentURI, nsString key)
     returns (LSWriteOpResponse response);
 
-  sync Clear()
+  sync Clear(nsString documentURI)
     returns (LSWriteOpResponse response);
 
 child:
diff --git a/dom/localstorage/PBackgroundLSObject.ipdl b/dom/localstorage/PBackgroundLSObject.ipdl
deleted file mode 100644
index 291e8acb29b2..000000000000
--- a/dom/localstorage/PBackgroundLSObject.ipdl
+++ /dev/null
@@ -1,27 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-include protocol PBackground;
-include protocol PBackgroundLSDatabase;
-
-namespace mozilla {
-namespace dom {
-
-sync protocol PBackgroundLSObject
-{
-  manager PBackground;
-
-  manages PBackgroundLSDatabase;
-
-parent:
-  async DeleteMe();
-
-  async PBackgroundLSDatabase(uint64_t datastoreId);
-
-child:
-  async __delete__();
-};
-
-} // namespace dom
-} // namespace mozilla
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index 528c057258e6..dd83a5cbf846 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -38,7 +38,6 @@ UNIFIED_SOURCES += [
 
 IPDL_SOURCES += [
     'PBackgroundLSDatabase.ipdl',
-    'PBackgroundLSObject.ipdl',
     'PBackgroundLSObserver.ipdl',
     'PBackgroundLSRequest.ipdl',
     'PBackgroundLSSharedTypes.ipdlh',
diff --git a/ipc/glue/BackgroundChildImpl.cpp b/ipc/glue/BackgroundChildImpl.cpp
index 75c802601f1e..0215c7e32fc3 100644
--- a/ipc/glue/BackgroundChildImpl.cpp
+++ b/ipc/glue/BackgroundChildImpl.cpp
@@ -16,7 +16,7 @@
 #include "mozilla/Assertions.h"
 #include "mozilla/SchedulerGroup.h"
 #include "mozilla/dom/ClientManagerActors.h"
-#include "mozilla/dom/PBackgroundLSObjectChild.h"
+#include "mozilla/dom/PBackgroundLSDatabaseChild.h"
 #include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestChild.h"
@@ -245,18 +245,18 @@ BackgroundChildImpl::DeallocPBackgroundSDBConnectionChild(
   return true;
 }
 
-BackgroundChildImpl::PBackgroundLSObjectChild*
-BackgroundChildImpl::AllocPBackgroundLSObjectChild(
+BackgroundChildImpl::PBackgroundLSDatabaseChild*
+BackgroundChildImpl::AllocPBackgroundLSDatabaseChild(
                                             const PrincipalInfo& aPrincipalInfo,
-                                            const nsString& aDocumentURI,
-                                            const uint32_t& aPrivateBrowsingId)
+                                            const uint32_t& aPrivateBrowsingId,
+                                            const uint64_t& aDatastoreId)
 {
-  MOZ_CRASH("PBackgroundLSObjectChild actor should be manually constructed!");
+  MOZ_CRASH("PBackgroundLSDatabaseChild actor should be manually constructed!");
 }
 
 bool
-BackgroundChildImpl::DeallocPBackgroundLSObjectChild(
-                                               PBackgroundLSObjectChild* aActor)
+BackgroundChildImpl::DeallocPBackgroundLSDatabaseChild(
+                                             PBackgroundLSDatabaseChild* aActor)
 {
   MOZ_ASSERT(aActor);
 
diff --git a/ipc/glue/BackgroundChildImpl.h b/ipc/glue/BackgroundChildImpl.h
index ccb42555623e..aeec24bf688e 100644
--- a/ipc/glue/BackgroundChildImpl.h
+++ b/ipc/glue/BackgroundChildImpl.h
@@ -78,14 +78,14 @@ protected:
   DeallocPBackgroundSDBConnectionChild(PBackgroundSDBConnectionChild* aActor)
                                        override;
 
-  virtual PBackgroundLSObjectChild*
-  AllocPBackgroundLSObjectChild(const PrincipalInfo& aPrincipalInfo,
-                                const nsString& aDocumentURI,
-                                const uint32_t& aPrivateBrowsingId) override;
+  virtual PBackgroundLSDatabaseChild*
+  AllocPBackgroundLSDatabaseChild(const PrincipalInfo& aPrincipalInfo,
+                                  const uint32_t& aPrivateBrowsingId,
+                                  const uint64_t& aDatastoreId) override;
 
   virtual bool
-  DeallocPBackgroundLSObjectChild(PBackgroundLSObjectChild* aActor)
-                                  override;
+  DeallocPBackgroundLSDatabaseChild(PBackgroundLSDatabaseChild* aActor)
+                                    override;
 
   virtual PBackgroundLSObserverChild*
   AllocPBackgroundLSObserverChild(const uint64_t& aObserverId) override;
diff --git a/ipc/glue/BackgroundParentImpl.cpp b/ipc/glue/BackgroundParentImpl.cpp
index ad88df7e83d3..70efd0873e60 100644
--- a/ipc/glue/BackgroundParentImpl.cpp
+++ b/ipc/glue/BackgroundParentImpl.cpp
@@ -294,49 +294,49 @@ BackgroundParentImpl::DeallocPBackgroundSDBConnectionParent(
   return mozilla::dom::DeallocPBackgroundSDBConnectionParent(aActor);
 }
 
-BackgroundParentImpl::PBackgroundLSObjectParent*
-BackgroundParentImpl::AllocPBackgroundLSObjectParent(
+BackgroundParentImpl::PBackgroundLSDatabaseParent*
+BackgroundParentImpl::AllocPBackgroundLSDatabaseParent(
                                             const PrincipalInfo& aPrincipalInfo,
-                                            const nsString& aDocumentURI,
-                                            const uint32_t& aPrivateBrowsingId)
+                                            const uint32_t& aPrivateBrowsingId,
+                                            const uint64_t& aDatastoreId)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
 
-  return mozilla::dom::AllocPBackgroundLSObjectParent(aPrincipalInfo,
-                                                      aDocumentURI,
-                                                      aPrivateBrowsingId);
+  return mozilla::dom::AllocPBackgroundLSDatabaseParent(aPrincipalInfo,
+                                                        aPrivateBrowsingId,
+                                                        aDatastoreId);
 }
 
 mozilla::ipc::IPCResult
-BackgroundParentImpl::RecvPBackgroundLSObjectConstructor(
-                                            PBackgroundLSObjectParent* aActor,
+BackgroundParentImpl::RecvPBackgroundLSDatabaseConstructor(
+                                            PBackgroundLSDatabaseParent* aActor,
                                             const PrincipalInfo& aPrincipalInfo,
-                                            const nsString& aDocumentURI,
-                                            const uint32_t& aPrivateBrowsingId)
+                                            const uint32_t& aPrivateBrowsingId,
+                                            const uint64_t& aDatastoreId)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
-  if (!mozilla::dom::RecvPBackgroundLSObjectConstructor(aActor,
-                                                        aPrincipalInfo,
-                                                        aDocumentURI,
-                                                        aPrivateBrowsingId)) {
+  if (!mozilla::dom::RecvPBackgroundLSDatabaseConstructor(aActor,
+                                                          aPrincipalInfo,
+                                                          aPrivateBrowsingId,
+                                                          aDatastoreId)) {
     return IPC_FAIL_NO_REASON(this);
   }
   return IPC_OK();
 }
 
 bool
-BackgroundParentImpl::DeallocPBackgroundLSObjectParent(
-                                              PBackgroundLSObjectParent* aActor)
+BackgroundParentImpl::DeallocPBackgroundLSDatabaseParent(
+                                            PBackgroundLSDatabaseParent* aActor)
 {
   AssertIsInMainProcess();
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aActor);
 
-  return mozilla::dom::DeallocPBackgroundLSObjectParent(aActor);
+  return mozilla::dom::DeallocPBackgroundLSDatabaseParent(aActor);
 }
 
 BackgroundParentImpl::PBackgroundLSObserverParent*
diff --git a/ipc/glue/BackgroundParentImpl.h b/ipc/glue/BackgroundParentImpl.h
index c8c8b4b67251..03b0524e08a5 100644
--- a/ipc/glue/BackgroundParentImpl.h
+++ b/ipc/glue/BackgroundParentImpl.h
@@ -77,21 +77,20 @@ protected:
   DeallocPBackgroundSDBConnectionParent(PBackgroundSDBConnectionParent* aActor)
                                         override;
 
-  virtual PBackgroundLSObjectParent*
-  AllocPBackgroundLSObjectParent(const PrincipalInfo& aPrincipalInfo,
-                                 const nsString& aDocumentURI,
-                                 const uint32_t& aPrivateBrowsingId) override;
+  virtual PBackgroundLSDatabaseParent*
+  AllocPBackgroundLSDatabaseParent(const PrincipalInfo& aPrincipalInfo,
+                                   const uint32_t& aPrivateBrowsingId,
+                                   const uint64_t& aDatastoreId) override;
 
   virtual mozilla::ipc::IPCResult
-  RecvPBackgroundLSObjectConstructor(PBackgroundLSObjectParent* aActor,
-                                     const PrincipalInfo& aPrincipalInfo,
-                                     const nsString& aDocumentURI,
-                                     const uint32_t& aPrivateBrowsingId)
-                                     override;
+  RecvPBackgroundLSDatabaseConstructor(PBackgroundLSDatabaseParent* aActor,
+                                       const PrincipalInfo& aPrincipalInfo,
+                                       const uint32_t& aPrivateBrowsingId,
+                                       const uint64_t& aDatastoreId) override;
 
   virtual bool
-  DeallocPBackgroundLSObjectParent(PBackgroundLSObjectParent* aActor)
-                                   override;
+  DeallocPBackgroundLSDatabaseParent(PBackgroundLSDatabaseParent* aActor)
+                                     override;
 
   virtual PBackgroundLSObserverParent*
   AllocPBackgroundLSObserverParent(const uint64_t& aObserverId) override;
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index a1f03a747747..14be7cd8f466 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -6,7 +6,7 @@ include protocol PAsmJSCacheEntry;
 include protocol PBackgroundIDBFactory;
 include protocol PBackgroundIndexedDBUtils;
 include protocol PBackgroundSDBConnection;
-include protocol PBackgroundLSObject;
+include protocol PBackgroundLSDatabase;
 include protocol PBackgroundLSObserver;
 include protocol PBackgroundLSRequest;
 include protocol PBackgroundLSSimpleRequest;
@@ -77,7 +77,7 @@ sync protocol PBackground
   manages PBackgroundIDBFactory;
   manages PBackgroundIndexedDBUtils;
   manages PBackgroundSDBConnection;
-  manages PBackgroundLSObject;
+  manages PBackgroundLSDatabase;
   manages PBackgroundLSObserver;
   manages PBackgroundLSRequest;
   manages PBackgroundLSSimpleRequest;
@@ -128,9 +128,9 @@ parent:
 
   async PBackgroundSDBConnection(PrincipalInfo principalInfo);
 
-  async PBackgroundLSObject(PrincipalInfo principalInfo,
-                            nsString documentURI,
-                            uint32_t privateBrowsingId);
+  async PBackgroundLSDatabase(PrincipalInfo principalInfo,
+                              uint32_t privateBrowsingId,
+                              uint64_t datastoreId);
 
   async PBackgroundLSObserver(uint64_t observerId);
 

From 54be9b73071df14bbcfadf690f1a4741f246de22 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:44 +0100
Subject: [PATCH 42/78] Bug 1286798 - Part 28: Add more
 QuotaClient::IsShuttingDownOnBackgroundThread() and MayProceed() checks;
 r=asuth

The shutdown and actor destroyed flag is now checked after each dispatch.
---
 dom/localstorage/ActorsParent.cpp | 199 ++++++++++++++++++++----------
 1 file changed, 133 insertions(+), 66 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index d6d0005e79f3..c3ef7a48e271 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1604,9 +1604,15 @@ private:
   nsresult
   CheckExistingOperations();
 
+  nsresult
+  CheckClosingDatastoreInternal();
+
   nsresult
   CheckClosingDatastore();
 
+  nsresult
+  BeginDatastorePreparationInternal();
+
   nsresult
   BeginDatastorePreparation();
 
@@ -1616,7 +1622,7 @@ private:
   nsresult
   OpenDirectory();
 
-  nsresult
+  void
   SendToIOThread();
 
   nsresult
@@ -1640,6 +1646,12 @@ private:
   nsresult
   BeginLoadData();
 
+  void
+  FinishNesting();
+
+  nsresult
+  FinishNestingOnNonOwningThread();
+
   nsresult
   NestedRun() override;
 
@@ -3879,16 +3891,19 @@ LSRequestBase::SendReadyMessage()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::SendingReadyMessage);
 
-  if (!MayProceed()) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
     MaybeSetFailureCode(NS_ERROR_FAILURE);
+  }
 
-    Cleanup();
-
-    mState = State::Completed;
-  } else {
+  if (MayProceed()) {
     Unused << SendReady();
 
     mState = State::WaitingForFinish;
+  } else {
+    Cleanup();
+
+    mState = State::Completed;
   }
 }
 
@@ -3898,9 +3913,12 @@ LSRequestBase::SendResults()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::SendingResults);
 
-  if (!MayProceed()) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
     MaybeSetFailureCode(NS_ERROR_FAILURE);
-  } else {
+  }
+
+  if (MayProceed()) {
     LSRequestResponse response;
 
     if (NS_SUCCEEDED(ResultCode())) {
@@ -3990,7 +4008,12 @@ LSRequestBase::RecvFinish()
   MOZ_ASSERT(mState == State::WaitingForFinish);
 
   mState = State::SendingResults;
-  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToCurrentThread(this));
+
+  // This LSRequestBase can only be held alive by the IPDL. Run() can end up
+  // with clearing that last reference. So we need to add a self reference here.
+  RefPtr<LSRequestBase> kungFuDeathGrip = this;
+
+  MOZ_ALWAYS_SUCCEEDS(this->Run());
 
   return IPC_OK();
 }
@@ -4135,7 +4158,7 @@ PrepareDatastoreOp::CheckExistingOperations()
     }
   }
 
-  nsresult rv = CheckClosingDatastore();
+  nsresult rv = CheckClosingDatastoreInternal();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -4155,6 +4178,23 @@ PrepareDatastoreOp::CheckClosingDatastore()
     return NS_ERROR_FAILURE;
   }
 
+  nsresult rv = CheckClosingDatastoreInternal();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::CheckClosingDatastoreInternal()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::CheckClosingDatastore);
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+  MOZ_ASSERT(MayProceed());
+
   mNestedState = NestedState::PreparationPending;
 
   RefPtr<Datastore> datastore;
@@ -4166,7 +4206,7 @@ PrepareDatastoreOp::CheckClosingDatastore()
     return NS_OK;
   }
 
-  nsresult rv = BeginDatastorePreparation();
+  nsresult rv = BeginDatastorePreparationInternal();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -4181,15 +4221,34 @@ PrepareDatastoreOp::BeginDatastorePreparation()
   MOZ_ASSERT(mState == State::Nesting);
   MOZ_ASSERT(mNestedState == NestedState::PreparationPending);
 
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsresult rv = BeginDatastorePreparationInternal();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+PrepareDatastoreOp::BeginDatastorePreparationInternal()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::Nesting);
+  MOZ_ASSERT(mNestedState == NestedState::PreparationPending);
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+  MOZ_ASSERT(MayProceed());
+
   if (gDatastores && (mDatastore = gDatastores->Get(mOrigin))) {
     MOZ_ASSERT(!mDatastore->IsClosed());
 
     mDatastore->NoteLivePrepareDatastoreOp(this);
 
-    mState = State::SendingReadyMessage;
-    mNestedState = NestedState::AfterNesting;
-
-    Unused << this->Run();
+    FinishNesting();
 
     return NS_OK;
   }
@@ -4243,6 +4302,7 @@ PrepareDatastoreOp::OpenDirectory()
   MOZ_ASSERT(!mOrigin.IsEmpty());
   MOZ_ASSERT(!mDirectoryLock);
   MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+  MOZ_ASSERT(MayProceed());
   MOZ_ASSERT(QuotaManager::Get());
 
   mNestedState = NestedState::DirectoryOpenPending;
@@ -4258,17 +4318,14 @@ PrepareDatastoreOp::OpenDirectory()
   return NS_OK;
 }
 
-nsresult
+void
 PrepareDatastoreOp::SendToIOThread()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::Nesting);
   MOZ_ASSERT(mNestedState == NestedState::DirectoryOpenPending);
-
-  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
-      !MayProceed()) {
-    return NS_ERROR_FAILURE;
-  }
+  MOZ_ASSERT(!QuotaClient::IsShuttingDownOnBackgroundThread());
+  MOZ_ASSERT(MayProceed());
 
   // Skip all disk related stuff and transition to SendingReadyMessage if we
   // are preparing a datastore for private browsing.
@@ -4280,12 +4337,9 @@ PrepareDatastoreOp::SendToIOThread()
   // Any following LocalStorage API call will trigger preparation of a new
   // (empty) datastore.
   if (mPrivateBrowsingId) {
-    mState = State::SendingReadyMessage;
-    mNestedState = NestedState::AfterNesting;
+    FinishNesting();
 
-    Unused << this->Run();
-
-    return NS_OK;
+    return;
   }
 
   QuotaManager* quotaManager = QuotaManager::Get();
@@ -4294,12 +4348,8 @@ PrepareDatastoreOp::SendToIOThread()
   // Must set this before dispatching otherwise we will race with the IO thread.
   mNestedState = NestedState::DatabaseWorkOpen;
 
-  nsresult rv = quotaManager->IOThread()->Dispatch(this, NS_DISPATCH_NORMAL);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
+  MOZ_ALWAYS_SUCCEEDS(
+    quotaManager->IOThread()->Dispatch(this, NS_DISPATCH_NORMAL));
 }
 
 nsresult
@@ -4517,12 +4567,7 @@ PrepareDatastoreOp::DatabaseNotAvailable()
 
   mDatabaseNotAvailable = true;
 
-  // Must set this before dispatching otherwise we will race with the owning
-  // thread.
-  mState = State::SendingReadyMessage;
-  mNestedState = NestedState::AfterNesting;
-
-  nsresult rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
+  nsresult rv = FinishNestingOnNonOwningThread();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -4672,6 +4717,40 @@ PrepareDatastoreOp::BeginLoadData()
   return NS_OK;
 }
 
+void
+PrepareDatastoreOp::FinishNesting()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mState == State::Nesting);
+
+  // The caller holds a strong reference to us, no need for a self reference
+  // before calling Run().
+
+  mState = State::SendingReadyMessage;
+  mNestedState = NestedState::AfterNesting;
+
+  MOZ_ALWAYS_SUCCEEDS(Run());
+}
+
+nsresult
+PrepareDatastoreOp::FinishNestingOnNonOwningThread()
+{
+  MOZ_ASSERT(!IsOnOwningThread());
+  MOZ_ASSERT(mState == State::Nesting);
+
+  // Must set mState before dispatching otherwise we will race with the owning
+  // thread.
+  mState = State::SendingReadyMessage;
+  mNestedState = NestedState::AfterNesting;
+
+  nsresult rv = OwningEventTarget()->Dispatch(this, NS_DISPATCH_NORMAL);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 nsresult
 PrepareDatastoreOp::NestedRun()
 {
@@ -4892,22 +4971,18 @@ PrepareDatastoreOp::DirectoryLockAcquired(DirectoryLock* aLock)
   MOZ_ASSERT(mNestedState == NestedState::DirectoryOpenPending);
   MOZ_ASSERT(!mDirectoryLock);
 
-  mDirectoryLock = aLock;
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
+    MaybeSetFailureCode(NS_ERROR_FAILURE);
 
-  nsresult rv = SendToIOThread();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    MaybeSetFailureCode(rv);
-
-    // The caller holds a strong reference to us, no need for a self reference
-    // before calling Run().
-
-    mState = State::SendingReadyMessage;
-    mNestedState = NestedState::AfterNesting;
-
-    MOZ_ALWAYS_SUCCEEDS(Run());
+    FinishNesting();
 
     return;
   }
+
+  mDirectoryLock = aLock;
+
+  SendToIOThread();
 }
 
 void
@@ -4920,13 +4995,7 @@ PrepareDatastoreOp::DirectoryLockFailed()
 
   MaybeSetFailureCode(NS_ERROR_FAILURE);
 
-  // The caller holds a strong reference to us, no need for a self reference
-  // before calling Run().
-
-  mState = State::SendingReadyMessage;
-  mNestedState = NestedState::AfterNesting;
-
-  MOZ_ALWAYS_SUCCEEDS(Run());
+  FinishNesting();
 }
 
 nsresult
@@ -4991,10 +5060,7 @@ LoadDataOp::OnSuccess()
                NestedState::DatabaseWorkLoadData);
   MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
 
-  mPrepareDatastoreOp->mState = State::SendingReadyMessage;
-  mPrepareDatastoreOp->mNestedState = NestedState::AfterNesting;
-
-  MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
+  mPrepareDatastoreOp->FinishNesting();
 }
 
 void
@@ -5009,10 +5075,8 @@ LoadDataOp::OnFailure(nsresult aResultCode)
   MOZ_ASSERT(mPrepareDatastoreOp->mLoadDataOp == this);
 
   mPrepareDatastoreOp->SetFailureCode(aResultCode);
-  mPrepareDatastoreOp->mState = State::SendingReadyMessage;
-  mPrepareDatastoreOp->mNestedState = NestedState::AfterNesting;
 
-  MOZ_ALWAYS_SUCCEEDS(mPrepareDatastoreOp->Run());
+  mPrepareDatastoreOp->FinishNesting();
 }
 
 void
@@ -5135,9 +5199,12 @@ LSSimpleRequestBase::SendResults()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mState == State::SendingResults);
 
-  if (!MayProceed()) {
+  if (NS_WARN_IF(QuotaClient::IsShuttingDownOnBackgroundThread()) ||
+      !MayProceed()) {
     MaybeSetFailureCode(NS_ERROR_FAILURE);
-  } else {
+  }
+
+  if (MayProceed()) {
     LSSimpleRequestResponse response;
 
     if (NS_SUCCEEDED(ResultCode())) {

From c4f55013cf613c38216b4e023e6a0b54af41b13d Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:47 +0100
Subject: [PATCH 43/78] Bug 1286798 - Part 29: Implement implicit snapshotting
 of databases; r=asuth,mccr8

This improves performance a lot in cases when multiple operations are invoked by a single JS function (number of sync IPC calls is reduced to a minimum). It also improves correctness since changes are not visible to other content processes until a JS function finishes.
The patch implements core infrastructure, all items are sent to content when a snapshot is initialized and everything is fully working. However, sending of all items at once is not optimal for bigger databases. Support for lazy loading of items is implemented in a following patch.
---
 dom/localstorage/ActorsChild.cpp              |  67 +-
 dom/localstorage/ActorsChild.h                |  43 +
 dom/localstorage/ActorsParent.cpp             | 740 ++++++++++++------
 dom/localstorage/LSDatabase.cpp               | 356 ++++++---
 dom/localstorage/LSDatabase.h                 |  47 +-
 dom/localstorage/LSObject.cpp                 |  66 +-
 dom/localstorage/LSObject.h                   |  12 +-
 dom/localstorage/LSSnapshot.cpp               | 320 ++++++++
 dom/localstorage/LSSnapshot.h                 |  96 +++
 dom/localstorage/LocalStorageCommon.h         |  35 +
 dom/localstorage/PBackgroundLSDatabase.ipdl   |  38 +-
 dom/localstorage/PBackgroundLSSnapshot.ipdl   |  57 ++
 dom/localstorage/moz.build                    |   2 +
 dom/localstorage/test/unit/head.js            |   7 +
 dom/localstorage/test/unit/test_groupLimit.js |   5 +
 ipc/ipdl/sync-messages.ini                    |  14 +-
 16 files changed, 1425 insertions(+), 480 deletions(-)
 create mode 100644 dom/localstorage/LSSnapshot.cpp
 create mode 100644 dom/localstorage/LSSnapshot.h
 create mode 100644 dom/localstorage/PBackgroundLSSnapshot.ipdl

diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 26645ac3bcea..852bfa8d2019 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -10,6 +10,7 @@
 #include "LSDatabase.h"
 #include "LSObject.h"
 #include "LSObserver.h"
+#include "LSSnapshot.h"
 
 namespace mozilla {
 namespace dom {
@@ -66,7 +67,7 @@ LSDatabaseChild::RecvRequestAllowToClose()
   AssertIsOnOwningThread();
 
   if (mDatabase) {
-    mDatabase->AllowToClose();
+    mDatabase->RequestAllowToClose();
 
     // TODO: A new datastore will be prepared at first LocalStorage API
     //       synchronous call. It would be better to start preparing a new
@@ -77,6 +78,24 @@ LSDatabaseChild::RecvRequestAllowToClose()
   return IPC_OK();
 }
 
+PBackgroundLSSnapshotChild*
+LSDatabaseChild::AllocPBackgroundLSSnapshotChild(const nsString& aDocumentURI,
+                                                 const int64_t& aRequestedSize,
+                                                 LSSnapshotInitInfo* aInitInfo)
+{
+  MOZ_CRASH("PBackgroundLSSnapshotChild actor should be manually constructed!");
+}
+
+bool
+LSDatabaseChild::DeallocPBackgroundLSSnapshotChild(
+                                             PBackgroundLSSnapshotChild* aActor)
+{
+  MOZ_ASSERT(aActor);
+
+  delete aActor;
+  return true;
+}
+
 /*******************************************************************************
  * LSObserverChild
  ******************************************************************************/
@@ -251,5 +270,51 @@ LSSimpleRequestChild::Recv__delete__(const LSSimpleRequestResponse& aResponse)
   return IPC_OK();
 }
 
+/*******************************************************************************
+ * LSSnapshotChild
+ ******************************************************************************/
+
+LSSnapshotChild::LSSnapshotChild(LSSnapshot* aSnapshot)
+  : mSnapshot(aSnapshot)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aSnapshot);
+
+  MOZ_COUNT_CTOR(LSSnapshotChild);
+}
+
+LSSnapshotChild::~LSSnapshotChild()
+{
+  AssertIsOnOwningThread();
+
+  MOZ_COUNT_DTOR(LSSnapshotChild);
+}
+
+void
+LSSnapshotChild::SendDeleteMeInternal()
+{
+  AssertIsOnOwningThread();
+
+  if (mSnapshot) {
+    mSnapshot->ClearActor();
+    mSnapshot = nullptr;
+
+    MOZ_ALWAYS_TRUE(PBackgroundLSSnapshotChild::SendDeleteMe());
+  }
+}
+
+void
+LSSnapshotChild::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnOwningThread();
+
+  if (mSnapshot) {
+    mSnapshot->ClearActor();
+#ifdef DEBUG
+    mSnapshot = nullptr;
+#endif
+  }
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 97986e88e969..ab4e43cea1ce 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -11,6 +11,7 @@
 #include "mozilla/dom/PBackgroundLSObserverChild.h"
 #include "mozilla/dom/PBackgroundLSRequestChild.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestChild.h"
+#include "mozilla/dom/PBackgroundLSSnapshotChild.h"
 
 namespace mozilla {
 
@@ -28,6 +29,7 @@ class LSObject;
 class LSObserver;
 class LSRequestChildCallback;
 class LSSimpleRequestChildCallback;
+class LSSnapshot;
 
 class LSDatabaseChild final
   : public PBackgroundLSDatabaseChild
@@ -63,6 +65,15 @@ private:
 
   mozilla::ipc::IPCResult
   RecvRequestAllowToClose() override;
+
+  PBackgroundLSSnapshotChild*
+  AllocPBackgroundLSSnapshotChild(const nsString& aDocumentURI,
+                                  const int64_t& aRequestedSize,
+                                  LSSnapshotInitInfo* aInitInfo) override;
+
+  bool
+  DeallocPBackgroundLSSnapshotChild(PBackgroundLSSnapshotChild* aActor)
+                                    override;
 };
 
 class LSObserverChild final
@@ -203,6 +214,38 @@ protected:
   { }
 };
 
+class LSSnapshotChild final
+  : public PBackgroundLSSnapshotChild
+{
+  friend class LSDatabase;
+  friend class LSSnapshot;
+
+  LSSnapshot* mSnapshot;
+
+  NS_DECL_OWNINGTHREAD
+
+public:
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSSnapshotChild);
+  }
+
+private:
+  // Only created by LSDatabase.
+  explicit LSSnapshotChild(LSSnapshot* aSnapshot);
+
+  // Only destroyed by LSDatabaseChild.
+  ~LSSnapshotChild();
+
+  void
+  SendDeleteMeInternal();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
+};
+
 } // namespace dom
 } // namespace mozilla
 
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index c3ef7a48e271..9c81bcd36ca1 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -19,6 +19,7 @@
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
 #include "mozilla/dom/PBackgroundLSSharedTypes.h"
 #include "mozilla/dom/PBackgroundLSSimpleRequestParent.h"
+#include "mozilla/dom/PBackgroundLSSnapshotParent.h"
 #include "mozilla/dom/StorageDBUpdater.h"
 #include "mozilla/dom/StorageUtils.h"
 #include "mozilla/dom/quota/QuotaManager.h"
@@ -61,6 +62,7 @@ class ConnectionThread;
 class Database;
 class PrepareDatastoreOp;
 class PreparedDatastore;
+class Snapshot;
 
 /*******************************************************************************
  * Constants
@@ -1026,11 +1028,17 @@ class Datastore final
   nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
   nsTHashtable<nsPtrHashKey<PreparedDatastore>> mPreparedDatastores;
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
+  nsTHashtable<nsPtrHashKey<Database>> mActiveDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
+  nsTArray<int64_t> mPendingUsageDeltas;
   const nsCString mOrigin;
   const uint32_t mPrivateBrowsingId;
   int64_t mUsage;
+  int64_t mUpdateBatchUsage;
   bool mClosed;
+#ifdef DEBUG
+  bool mInUpdateBatch;
+#endif
 
 public:
   // Created by PrepareDatastoreOp.
@@ -1060,6 +1068,13 @@ public:
     return mPrivateBrowsingId == 0;
   }
 
+  int64_t
+  Usage() const
+  {
+    AssertIsOnBackgroundThread();
+    return mUsage;
+  }
+
   void
   Close();
 
@@ -1102,11 +1117,14 @@ public:
   HasLiveDatabases() const;
 #endif
 
-  uint32_t
-  GetLength() const;
+  void
+  NoteActiveDatabase(Database* aDatabase);
 
   void
-  GetKey(uint32_t aIndex, nsString& aKey) const;
+  NoteInactiveDatabase(Database* aDatabase);
+
+  void
+  GetItemInfos(nsTArray<LSItemInfo>* aItemInfos);
 
   void
   GetItem(const nsString& aKey, nsString& aValue) const;
@@ -1115,22 +1133,30 @@ public:
   SetItem(Database* aDatabase,
           const nsString& aDocumentURI,
           const nsString& aKey,
-          const nsString& aValue,
-          LSWriteOpResponse& aResponse);
+          const nsString& aOldValue,
+          const nsString& aValue);
 
   void
   RemoveItem(Database* aDatabase,
              const nsString& aDocumentURI,
              const nsString& aKey,
-             LSWriteOpResponse& aResponse);
+             const nsString& aOldValue);
 
   void
   Clear(Database* aDatabase,
-        const nsString& aDocumentURI,
-        LSWriteOpResponse& aResponse);
+        const nsString& aDocumentURI);
 
   void
-  GetKeys(nsTArray<nsString>& aKeys) const;
+  PrivateBrowsingClear();
+
+  void
+  BeginUpdateBatch(int64_t aSnapshotInitialUsage);
+
+  void
+  EndUpdateBatch(int64_t aSnapshotPeakUsage);
+
+  bool
+  UpdateUsage(int64_t aDelta);
 
   NS_INLINE_DECL_REFCOUNTING(Datastore)
 
@@ -1147,9 +1173,6 @@ private:
   void
   CleanupMetadata();
 
-  bool
-  UpdateUsage(int64_t aDelta);
-
   void
   NotifyObservers(Database* aDatabase,
                   const nsString& aDocumentURI,
@@ -1269,6 +1292,7 @@ class Database final
   : public PBackgroundLSDatabaseParent
 {
   RefPtr<Datastore> mDatastore;
+  Snapshot* mSnapshot;
   const PrincipalInfo mPrincipalInfo;
   // Strings share buffers if possible, so it's not a problem to duplicate the
   // origin here.
@@ -1287,6 +1311,13 @@ public:
            const nsACString& aOrigin,
            uint32_t aPrivateBrowsingId);
 
+  Datastore*
+  GetDatastore() const
+  {
+    AssertIsOnBackgroundThread();
+    return mDatastore;
+  }
+
   const PrincipalInfo&
   GetPrincipalInfo() const
   {
@@ -1308,6 +1339,12 @@ public:
   void
   SetActorAlive(Datastore* aDatastore);
 
+  void
+  RegisterSnapshot(Snapshot* aSnapshot);
+
+  void
+  UnregisterSnapshot(Snapshot* aSnapshot);
+
   void
   RequestAllowToClose();
 
@@ -1330,32 +1367,72 @@ private:
   mozilla::ipc::IPCResult
   RecvAllowToClose() override;
 
-  mozilla::ipc::IPCResult
-  RecvGetLength(uint32_t* aLength) override;
+  PBackgroundLSSnapshotParent*
+  AllocPBackgroundLSSnapshotParent(const nsString& aDocumentURI,
+                                   const int64_t& aRequestedSize,
+                                   LSSnapshotInitInfo* aInitInfo) override;
 
   mozilla::ipc::IPCResult
-  RecvGetKey(const uint32_t& aIndex, nsString* aKey) override;
+  RecvPBackgroundLSSnapshotConstructor(PBackgroundLSSnapshotParent* aActor,
+                                       const nsString& aDocumentURI,
+                                       const int64_t& aRequestedSize,
+                                       LSSnapshotInitInfo* aInitInfo) override;
+
+  bool
+  DeallocPBackgroundLSSnapshotParent(PBackgroundLSSnapshotParent* aActor)
+                                     override;
+};
+
+class Snapshot final
+  : public PBackgroundLSSnapshotParent
+{
+  RefPtr<Database> mDatabase;
+  RefPtr<Datastore> mDatastore;
+  nsString mDocumentURI;
+  int64_t mInitialUsage;
+  int64_t mPeakUsage;
+  bool mActorDestroyed;
+  bool mFinishReceived;
+
+public:
+  // Created in AllocPBackgroundLSSnapshotParent.
+  Snapshot(Database* aDatabase,
+           const nsAString& aDocumentURI);
+
+  void
+  SetUsage(int64_t aInitialUsage,
+           int64_t aPeakUsage)
+  {
+    AssertIsOnBackgroundThread();
+    MOZ_ASSERT(aInitialUsage >= 0);
+    MOZ_ASSERT(aPeakUsage >= aInitialUsage);
+    MOZ_ASSERT(mInitialUsage == -1);
+    MOZ_ASSERT(mPeakUsage == -1);
+
+    mInitialUsage = aInitialUsage;
+    mPeakUsage = aPeakUsage;
+  }
+
+  NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Snapshot)
+
+private:
+  // Reference counted.
+  ~Snapshot();
+
+  // IPDL methods are only called by IPDL.
+  void
+  ActorDestroy(ActorDestroyReason aWhy) override;
 
   mozilla::ipc::IPCResult
-  RecvGetItem(const nsString& aKey, nsString* aValue) override;
+  RecvDeleteMe() override;
 
   mozilla::ipc::IPCResult
-  RecvGetKeys(nsTArray<nsString>* aKeys) override;
+  RecvFinish(const LSSnapshotFinishInfo& aFinishInfo) override;
 
   mozilla::ipc::IPCResult
-  RecvSetItem(const nsString& aDocumentURI,
-              const nsString& aKey,
-              const nsString& aValue,
-              LSWriteOpResponse* aResponse) override;
-
-  mozilla::ipc::IPCResult
-  RecvRemoveItem(const nsString& aDocumentURI,
-                 const nsString& aKey,
-                 LSWriteOpResponse* aResponse) override;
-
-  mozilla::ipc::IPCResult
-  RecvClear(const nsString& aDocumentURI,
-            LSWriteOpResponse* aResponse) override;
+  RecvIncreasePeakUsage(const int64_t& aRequestedSize,
+                        const int64_t& aMinSize,
+                        int64_t* aSize) override;
 };
 
 class Observer final
@@ -2981,7 +3058,11 @@ Datastore::Datastore(const nsACString& aOrigin,
   , mOrigin(aOrigin)
   , mPrivateBrowsingId(aPrivateBrowsingId)
   , mUsage(aUsage)
+  , mUpdateBatchUsage(-1)
   , mClosed(false)
+#ifdef DEBUG
+  , mInUpdateBatch(false)
+#endif
 {
   AssertIsOnBackgroundThread();
 
@@ -3129,6 +3210,7 @@ Datastore::NoteFinishedDatabase(Database* aDatabase)
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(mDatabases.GetEntry(aDatabase));
+  MOZ_ASSERT(!mActiveDatabases.GetEntry(aDatabase));
   MOZ_ASSERT(mDirectoryLock);
   MOZ_ASSERT(!mClosed);
 
@@ -3147,28 +3229,55 @@ Datastore::HasLiveDatabases() const
 }
 #endif
 
-uint32_t
-Datastore::GetLength() const
+void
+Datastore::NoteActiveDatabase(Database* aDatabase)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+  MOZ_ASSERT(mDatabases.GetEntry(aDatabase));
+  MOZ_ASSERT(!mActiveDatabases.GetEntry(aDatabase));
   MOZ_ASSERT(!mClosed);
 
-  return mValues.Count();
+  mActiveDatabases.PutEntry(aDatabase);
 }
 
 void
-Datastore::GetKey(uint32_t aIndex, nsString& aKey) const
+Datastore::NoteInactiveDatabase(Database* aDatabase)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+  MOZ_ASSERT(mDatabases.GetEntry(aDatabase));
+  MOZ_ASSERT(mActiveDatabases.GetEntry(aDatabase));
   MOZ_ASSERT(!mClosed);
 
-  aKey.SetIsVoid(true);
-  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
-    if (aIndex == 0) {
-      aKey = iter.Key();
-      return;
+  mActiveDatabases.RemoveEntry(aDatabase);
+
+  if (!mActiveDatabases.Count() &&
+      mPendingUsageDeltas.Length()) {
+    int64_t finalDelta = 0;
+
+    for (auto delta : mPendingUsageDeltas) {
+      finalDelta += delta;
     }
-    aIndex--;
+
+    MOZ_ASSERT(finalDelta <= 0);
+
+    if (finalDelta != 0) {
+      DebugOnly<bool> ok = UpdateUsage(finalDelta);
+      MOZ_ASSERT(ok);
+    }
+
+    mPendingUsageDeltas.Clear();
+  }
+}
+
+void
+Datastore::GetItemInfos(nsTArray<LSItemInfo>* aItemInfos)
+{
+  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+    LSItemInfo* itemInfo = aItemInfos->AppendElement();
+    itemInfo->key() = iter.Key();
+    itemInfo->value() = iter.Data();
   }
 }
 
@@ -3187,42 +3296,29 @@ void
 Datastore::SetItem(Database* aDatabase,
                    const nsString& aDocumentURI,
                    const nsString& aKey,
-                   const nsString& aValue,
-                   LSWriteOpResponse& aResponse)
+                   const nsString& aOldValue,
+                   const nsString& aValue)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(mInUpdateBatch);
 
   nsString oldValue;
-  if (!mValues.Get(aKey, &oldValue)) {
-    oldValue.SetIsVoid(true);
-  }
+  GetItem(aKey, oldValue);
 
-  bool changed;
-  if (oldValue == aValue && oldValue.IsVoid() == aValue.IsVoid()) {
-    changed = false;
-  } else {
-    changed = true;
-
-    int64_t delta = 0;
+  if (oldValue != aValue || oldValue.IsVoid() != aValue.IsVoid()) {
+    int64_t delta = static_cast<int64_t>(aValue.Length()) -
+                    static_cast<int64_t>(oldValue.Length());
 
     if (oldValue.IsVoid()) {
       delta += static_cast<int64_t>(aKey.Length());
     }
 
-    delta += static_cast<int64_t>(aValue.Length()) -
-             static_cast<int64_t>(oldValue.Length());
-
-    if (!UpdateUsage(delta)) {
-      aResponse = NS_ERROR_FILE_NO_DEVICE_SPACE;
-      return;
-    }
+    mUpdateBatchUsage += delta;
 
     mValues.Put(aKey, aValue);
 
-    NotifyObservers(aDatabase, aDocumentURI, aKey, oldValue, aValue);
-
     if (IsPersistent()) {
       EnsureTransaction();
 
@@ -3231,40 +3327,31 @@ Datastore::SetItem(Database* aDatabase,
     }
   }
 
-  LSNotifyInfo info;
-  info.changed() = changed;
-  info.oldValue() = oldValue;
-  aResponse = info;
+  NotifyObservers(aDatabase, aDocumentURI, aKey, aOldValue, aValue);
 }
 
 void
 Datastore::RemoveItem(Database* aDatabase,
                       const nsString& aDocumentURI,
                       const nsString& aKey,
-                      LSWriteOpResponse& aResponse)
+                      const nsString& aOldValue)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(mInUpdateBatch);
 
-  bool changed;
   nsString oldValue;
-  if (!mValues.Get(aKey, &oldValue)) {
-    oldValue.SetIsVoid(true);
-    changed = false;
-  } else {
-    changed = true;
+  GetItem(aKey, oldValue);
 
+  if (!oldValue.IsVoid()) {
     int64_t delta = -(static_cast<int64_t>(aKey.Length()) +
                       static_cast<int64_t>(oldValue.Length()));
 
-    DebugOnly<bool> ok = UpdateUsage(delta);
-    MOZ_ASSERT(ok);
+    mUpdateBatchUsage += delta;
 
     mValues.Remove(aKey);
 
-    NotifyObservers(aDatabase, aDocumentURI, aKey, oldValue, VoidString());
-
     if (IsPersistent()) {
       EnsureTransaction();
 
@@ -3273,39 +3360,31 @@ Datastore::RemoveItem(Database* aDatabase,
     }
   }
 
-  LSNotifyInfo info;
-  info.changed() = changed;
-  info.oldValue() = oldValue;
-  aResponse = info;
+  NotifyObservers(aDatabase, aDocumentURI, aKey, aOldValue, VoidString());
 }
 
 void
 Datastore::Clear(Database* aDatabase,
-                 const nsString& aDocumentURI,
-                 LSWriteOpResponse& aResponse)
+                 const nsString& aDocumentURI)
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
   MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(mInUpdateBatch);
 
-  bool changed;
-  if (!mValues.Count()) {
-    changed = false;
-  } else {
-    changed = true;
+  if (mValues.Count()) {
+    for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+      const nsAString& key = iter.Key();
+      const nsAString& value = iter.Data();
 
-    DebugOnly<bool> ok = UpdateUsage(-mUsage);
-    MOZ_ASSERT(ok);
+      int64_t delta = -(static_cast<int64_t>(key.Length()) +
+                        static_cast<int64_t>(value.Length()));
+
+      mUpdateBatchUsage += delta;
+    }
 
     mValues.Clear();
 
-    if (aDatabase) {
-      NotifyObservers(aDatabase,
-                      aDocumentURI,
-                      VoidString(),
-                      VoidString(),
-                      VoidString());
-    }
-
     if (IsPersistent()) {
       EnsureTransaction();
 
@@ -3314,22 +3393,120 @@ Datastore::Clear(Database* aDatabase,
     }
   }
 
-  LSNotifyInfo info;
-  info.changed() = changed;
-  aResponse = info;
+  NotifyObservers(aDatabase,
+                  aDocumentURI,
+                  VoidString(),
+                  VoidString(),
+                  VoidString());
 }
 
 void
-Datastore::GetKeys(nsTArray<nsString>& aKeys) const
+Datastore::PrivateBrowsingClear()
 {
   AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mPrivateBrowsingId);
   MOZ_ASSERT(!mClosed);
 
-  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
-    aKeys.AppendElement(iter.Key());
+  if (mValues.Count()) {
+    DebugOnly<bool> ok = UpdateUsage(-mUsage);
+    MOZ_ASSERT(ok);
+
+    mValues.Clear();
   }
 }
 
+void
+Datastore::BeginUpdateBatch(int64_t aSnapshotInitialUsage)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aSnapshotInitialUsage >= 0);
+  MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(mUpdateBatchUsage == -1);
+  MOZ_ASSERT(!mInUpdateBatch);
+
+  mUpdateBatchUsage = aSnapshotInitialUsage;
+
+#ifdef DEBUG
+  mInUpdateBatch = true;
+#endif
+}
+
+void
+Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aSnapshotPeakUsage >= 0);
+  MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(mInUpdateBatch);
+
+  int64_t delta = mUpdateBatchUsage - aSnapshotPeakUsage;
+
+  if (mActiveDatabases.Count()) {
+    // We can't apply deltas while other databases are still active.
+    // The final delta must be zero or negative, but individual deltas can be
+    // positive. A positive delta can't be applied asynchronously since there's
+    // no way to fire the quota exceeded error event.
+
+    mPendingUsageDeltas.AppendElement(delta);
+  } else {
+    MOZ_ASSERT(delta <= 0);
+    if (delta != 0) {
+      DebugOnly<bool> ok = UpdateUsage(delta);
+      MOZ_ASSERT(ok);
+    }
+  }
+
+  mUpdateBatchUsage = -1;
+
+#ifdef DEBUG
+  mInUpdateBatch = false;
+#endif
+}
+
+bool
+Datastore::UpdateUsage(int64_t aDelta)
+{
+  AssertIsOnBackgroundThread();
+
+  // Check internal LocalStorage origin limit.
+  int64_t newUsage = mUsage + aDelta;
+  if (newUsage > gOriginLimitKB * 1024) {
+    return false;
+  }
+
+  // Check QuotaManager limits (group and global limit).
+  if (IsPersistent()) {
+    MOZ_ASSERT(mQuotaObject);
+
+    if (!mQuotaObject->MaybeUpdateSize(newUsage, /* aTruncate */ true)) {
+      return false;
+    }
+
+  }
+
+  // Quota checks passed, set new usage.
+
+  mUsage = newUsage;
+
+  if (IsPersistent()) {
+    RefPtr<Runnable> runnable = NS_NewRunnableFunction(
+      "Datastore::UpdateUsage",
+      [origin = mOrigin, newUsage] () {
+        MOZ_ASSERT(gUsages);
+        MOZ_ASSERT(gUsages->Contains(origin));
+        gUsages->Put(origin, newUsage);
+      });
+
+    QuotaManager* quotaManager = QuotaManager::Get();
+    MOZ_ASSERT(quotaManager);
+
+    MOZ_ALWAYS_SUCCEEDS(
+      quotaManager->IOThread()->Dispatch(runnable, NS_DISPATCH_NORMAL));
+  }
+
+  return true;
+}
+
 void
 Datastore::MaybeClose()
 {
@@ -3381,50 +3558,6 @@ Datastore::CleanupMetadata()
   }
 }
 
-bool
-Datastore::UpdateUsage(int64_t aDelta)
-{
-  AssertIsOnBackgroundThread();
-
-  // Check internal LocalStorage origin limit.
-  int64_t newUsage = mUsage + aDelta;
-  if (newUsage > gOriginLimitKB * 1024) {
-    return false;
-  }
-
-  // Check QuotaManager limits (group and global limit).
-  if (IsPersistent()) {
-    MOZ_ASSERT(mQuotaObject);
-
-    if (!mQuotaObject->MaybeUpdateSize(newUsage, /* aTruncate */ true)) {
-      return false;
-    }
-
-  }
-
-  // Quota checks passed, set new usage.
-
-  mUsage = newUsage;
-
-  if (IsPersistent()) {
-    RefPtr<Runnable> runnable = NS_NewRunnableFunction(
-      "Datastore::UpdateUsage",
-      [origin = mOrigin, newUsage] () {
-        MOZ_ASSERT(gUsages);
-        MOZ_ASSERT(gUsages->Contains(origin));
-        gUsages->Put(origin, newUsage);
-      });
-
-    QuotaManager* quotaManager = QuotaManager::Get();
-    MOZ_ASSERT(quotaManager);
-
-    MOZ_ALWAYS_SUCCEEDS(
-      quotaManager->IOThread()->Dispatch(runnable, NS_DISPATCH_NORMAL));
-  }
-
-  return true;
-}
-
 void
 Datastore::NotifyObservers(Database* aDatabase,
                            const nsString& aDocumentURI,
@@ -3529,7 +3662,8 @@ PreparedDatastore::TimerCallback(nsITimer* aTimer, void* aClosure)
 Database::Database(const PrincipalInfo& aPrincipalInfo,
                    const nsACString& aOrigin,
                    uint32_t aPrivateBrowsingId)
-  : mPrincipalInfo(aPrincipalInfo)
+  : mSnapshot(nullptr)
+  , mPrincipalInfo(aPrincipalInfo)
   , mOrigin(aOrigin)
   , mPrivateBrowsingId(aPrivateBrowsingId)
   , mAllowedToClose(false)
@@ -3570,6 +3704,31 @@ Database::SetActorAlive(Datastore* aDatastore)
   gLiveDatabases->AppendElement(this);
 }
 
+void
+Database::RegisterSnapshot(Snapshot* aSnapshot)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aSnapshot);
+  MOZ_ASSERT(!mSnapshot);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  // Only one snapshot at a time is currently supported.
+  mSnapshot = aSnapshot;
+
+  mDatastore->NoteActiveDatabase(this);
+}
+
+void
+Database::UnregisterSnapshot(Snapshot* aSnapshot)
+{
+  MOZ_ASSERT(aSnapshot);
+  MOZ_ASSERT(mSnapshot == aSnapshot);
+
+  mSnapshot = nullptr;
+
+  mDatastore->NoteInactiveDatabase(this);
+}
+
 void
 Database::RequestAllowToClose()
 {
@@ -3652,128 +3811,226 @@ Database::RecvAllowToClose()
   return IPC_OK();
 }
 
-mozilla::ipc::IPCResult
-Database::RecvGetLength(uint32_t* aLength)
+PBackgroundLSSnapshotParent*
+Database::AllocPBackgroundLSSnapshotParent(const nsString& aDocumentURI,
+                                           const int64_t& aRequestedSize,
+                                           LSSnapshotInitInfo* aInitInfo)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aLength);
-  MOZ_ASSERT(mDatastore);
+
+  if (NS_WARN_IF(aRequestedSize < 0)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
 
   if (NS_WARN_IF(mAllowedToClose)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  RefPtr<Snapshot> snapshot = new Snapshot(this, aDocumentURI);
+
+  // Transfer ownership to IPDL.
+  return snapshot.forget().take();
+}
+
+mozilla::ipc::IPCResult
+Database::RecvPBackgroundLSSnapshotConstructor(
+                                            PBackgroundLSSnapshotParent* aActor,
+                                            const nsString& aDocumentURI,
+                                            const int64_t& aRequestedSize,
+                                            LSSnapshotInitInfo* aInitInfo)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aRequestedSize >= 0);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  auto* snapshot = static_cast<Snapshot*>(aActor);
+
+  // TODO: This can be optimized depending on which operation triggers snapshot
+  //       creation. For example clear() doesn't need to receive items at all.
+  nsTArray<LSItemInfo> itemInfos;
+  mDatastore->GetItemInfos(&itemInfos);
+
+  int64_t initialUsage = mDatastore->Usage();
+
+  int64_t peakUsage = initialUsage;
+  if (aRequestedSize && mDatastore->UpdateUsage(aRequestedSize)) {
+    peakUsage += aRequestedSize;
+  }
+
+  snapshot->SetUsage(initialUsage, peakUsage);
+
+  RegisterSnapshot(snapshot);
+
+  aInitInfo->itemInfos() = std::move(itemInfos);
+  aInitInfo->initialUsage() = initialUsage;
+  aInitInfo->peakUsage() = peakUsage;
+
+  return IPC_OK();
+}
+
+bool
+Database::DeallocPBackgroundLSSnapshotParent(
+                                            PBackgroundLSSnapshotParent* aActor)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aActor);
+
+  // Transfer ownership back from IPDL.
+  RefPtr<Snapshot> actor = dont_AddRef(static_cast<Snapshot*>(aActor));
+
+  return true;
+}
+
+/*******************************************************************************
+ * Snapshot
+ ******************************************************************************/
+
+Snapshot::Snapshot(Database* aDatabase,
+                   const nsAString& aDocumentURI)
+  : mDatabase(aDatabase)
+  , mDatastore(aDatabase->GetDatastore())
+  , mDocumentURI(aDocumentURI)
+  , mInitialUsage(-1)
+  , mPeakUsage(-1)
+  , mActorDestroyed(false)
+  , mFinishReceived(false)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+}
+
+Snapshot::~Snapshot()
+{
+  MOZ_ASSERT(mActorDestroyed);
+  MOZ_ASSERT(mFinishReceived);
+}
+
+void
+Snapshot::ActorDestroy(ActorDestroyReason aWhy)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  mActorDestroyed = true;
+
+  if (!mFinishReceived) {
+    mDatabase->UnregisterSnapshot(this);
+
+    mDatastore->BeginUpdateBatch(mInitialUsage);
+
+    mDatastore->EndUpdateBatch(mPeakUsage);
+  }
+}
+
+mozilla::ipc::IPCResult
+Snapshot::RecvDeleteMe()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mActorDestroyed);
+
+  IProtocol* mgr = Manager();
+  if (!PBackgroundLSSnapshotParent::Send__delete__(this)) {
+    return IPC_FAIL_NO_REASON(mgr);
+  }
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Snapshot::RecvFinish(const LSSnapshotFinishInfo& aFinishInfo)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mInitialUsage >= 0);
+  MOZ_ASSERT(mPeakUsage >= mInitialUsage);
+
+  if (NS_WARN_IF(mFinishReceived)) {
     ASSERT_UNLESS_FUZZING();
     return IPC_FAIL_NO_REASON(this);
   }
 
-  *aLength = mDatastore->GetLength();
+  mFinishReceived = true;
+
+  mDatabase->UnregisterSnapshot(this);
+
+  mDatastore->BeginUpdateBatch(mInitialUsage);
+
+  const nsTArray<LSWriteInfo>& writeInfos = aFinishInfo.writeInfos();
+  for (uint32_t index = 0; index < writeInfos.Length(); index++) {
+    const LSWriteInfo& writeInfo = writeInfos[index];
+    switch (writeInfo.type()) {
+      case LSWriteInfo::TLSSetItemInfo: {
+        const LSSetItemInfo& info = writeInfo.get_LSSetItemInfo();
+
+        mDatastore->SetItem(mDatabase,
+                            mDocumentURI,
+                            info.key(),
+                            info.oldValue(),
+                            info.value());
+
+        break;
+      }
+
+      case LSWriteInfo::TLSRemoveItemInfo: {
+        const LSRemoveItemInfo& info = writeInfo.get_LSRemoveItemInfo();
+
+        mDatastore->RemoveItem(mDatabase,
+                               mDocumentURI,
+                               info.key(),
+                               info.oldValue());
+
+        break;
+      }
+
+      case LSWriteInfo::TLSClearInfo: {
+        mDatastore->Clear(mDatabase, mDocumentURI);
+
+        break;
+      }
+
+      default:
+        MOZ_CRASH("Should never get here!");
+    }
+  }
+
+  mDatastore->EndUpdateBatch(mPeakUsage);
 
   return IPC_OK();
 }
 
 mozilla::ipc::IPCResult
-Database::RecvGetKey(const uint32_t& aIndex, nsString* aKey)
+Snapshot::RecvIncreasePeakUsage(const int64_t& aRequestedSize,
+                                const int64_t& aMinSize,
+                                int64_t* aSize)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aKey);
-  MOZ_ASSERT(mDatastore);
+  MOZ_ASSERT(aSize);
 
-  if (NS_WARN_IF(mAllowedToClose)) {
+  if (NS_WARN_IF(aRequestedSize <= 0)) {
     ASSERT_UNLESS_FUZZING();
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->GetKey(aIndex, *aKey);
-
-  return IPC_OK();
-}
-
-mozilla::ipc::IPCResult
-Database::RecvGetItem(const nsString& aKey, nsString* aValue)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aValue);
-  MOZ_ASSERT(mDatastore);
-
-  if (NS_WARN_IF(mAllowedToClose)) {
+  if (NS_WARN_IF(aMinSize <= 0)) {
     ASSERT_UNLESS_FUZZING();
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->GetItem(aKey, *aValue);
-
-  return IPC_OK();
-}
-
-mozilla::ipc::IPCResult
-Database::RecvSetItem(const nsString& aDocumentURI,
-                      const nsString& aKey,
-                      const nsString& aValue,
-                      LSWriteOpResponse* aResponse)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aResponse);
-  MOZ_ASSERT(mDatastore);
-
-  if (NS_WARN_IF(mAllowedToClose)) {
+  if (NS_WARN_IF(mFinishReceived)) {
     ASSERT_UNLESS_FUZZING();
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mDatastore->SetItem(this, aDocumentURI, aKey, aValue, *aResponse);
-
-  return IPC_OK();
-}
-
-mozilla::ipc::IPCResult
-Database::RecvRemoveItem(const nsString& aDocumentURI,
-                         const nsString& aKey,
-                         LSWriteOpResponse* aResponse)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aResponse);
-  MOZ_ASSERT(mDatastore);
-
-  if (NS_WARN_IF(mAllowedToClose)) {
-    ASSERT_UNLESS_FUZZING();
-    return IPC_FAIL_NO_REASON(this);
+  if (mDatastore->UpdateUsage(aRequestedSize)) {
+    mPeakUsage += aRequestedSize;
+    *aSize = aRequestedSize;
+  } else if (mDatastore->UpdateUsage(aMinSize)) {
+    mPeakUsage += aMinSize;
+    *aSize = aMinSize;
+  } else {
+    *aSize = 0;
   }
 
-  mDatastore->RemoveItem(this, aDocumentURI, aKey, *aResponse);
-
-  return IPC_OK();
-}
-
-mozilla::ipc::IPCResult
-Database::RecvClear(const nsString& aDocumentURI,
-                    LSWriteOpResponse* aResponse)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aResponse);
-  MOZ_ASSERT(mDatastore);
-
-  if (NS_WARN_IF(mAllowedToClose)) {
-    ASSERT_UNLESS_FUZZING();
-    return IPC_FAIL_NO_REASON(this);
-  }
-
-  mDatastore->Clear(this, aDocumentURI, *aResponse);
-
-  return IPC_OK();
-}
-
-mozilla::ipc::IPCResult
-Database::RecvGetKeys(nsTArray<nsString>* aKeys)
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aKeys);
-  MOZ_ASSERT(mDatastore);
-
-  if (NS_WARN_IF(mAllowedToClose)) {
-    ASSERT_UNLESS_FUZZING();
-    return IPC_FAIL_NO_REASON(this);
-  }
-
-  mDatastore->GetKeys(*aKeys);
-
   return IPC_OK();
 }
 
@@ -5779,8 +6036,7 @@ ClearPrivateBrowsingRunnable::Run()
       MOZ_ASSERT(datastore);
 
       if (datastore->PrivateBrowsingId()) {
-        LSWriteOpResponse dummy;
-        datastore->Clear(nullptr, EmptyString(), dummy);
+        datastore->PrivateBrowsingClear();
       }
     }
   }
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index d41b724f9917..1bf0ae1e2052 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -19,8 +19,10 @@ StaticAutoPtr<LSDatabaseHashtable> gLSDatabases;
 
 LSDatabase::LSDatabase(const nsACString& aOrigin)
   : mActor(nullptr)
+  , mSnapshot(nullptr)
   , mOrigin(aOrigin)
   , mAllowedToClose(false)
+  , mRequestedAllowToClose(false)
 {
   AssertIsOnOwningThread();
 
@@ -35,6 +37,7 @@ LSDatabase::LSDatabase(const nsACString& aOrigin)
 LSDatabase::~LSDatabase()
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(!mSnapshot);
 
   if (!mAllowedToClose) {
     AllowToClose();
@@ -63,11 +66,239 @@ LSDatabase::SetActor(LSDatabaseChild* aActor)
   mActor = aActor;
 }
 
+void
+LSDatabase::RequestAllowToClose()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!mRequestedAllowToClose);
+
+  mRequestedAllowToClose = true;
+
+  if (!mSnapshot) {
+    AllowToClose();
+  }
+}
+
+void
+LSDatabase::NoteFinishedSnapshot(LSSnapshot* aSnapshot)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aSnapshot == mSnapshot);
+
+  mSnapshot = nullptr;
+
+  if (mRequestedAllowToClose) {
+    AllowToClose();
+  }
+}
+
+nsresult
+LSDatabase::GetLength(LSObject* aObject,
+                      uint32_t* aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->GetLength(aResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetKey(LSObject* aObject,
+                   uint32_t aIndex,
+                   nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->GetKey(aIndex, aResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetItem(LSObject* aObject,
+                    const nsAString& aKey,
+                    nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->GetItem(aKey, aResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::GetKeys(LSObject* aObject,
+                    nsTArray<nsString>& aKeys)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->GetKeys(aKeys);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::SetItem(LSObject* aObject,
+                    const nsAString& aKey,
+                    const nsAString& aValue,
+                    LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ true);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->SetItem(aKey, aValue, aNotifyInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::RemoveItem(LSObject* aObject,
+                       const nsAString& aKey,
+                       LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->RemoveItem(aKey, aNotifyInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::Clear(LSObject* aObject,
+                  LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = mSnapshot->Clear(aNotifyInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::EnsureSnapshot(LSObject* aObject,
+                           bool aRequestedBySetItem)
+{
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  if (mSnapshot) {
+    return NS_OK;
+  }
+
+  RefPtr<LSSnapshot> snapshot = new LSSnapshot(this);
+
+  LSSnapshotChild* actor = new LSSnapshotChild(snapshot);
+
+  int64_t requestedSize = aRequestedBySetItem ? 4096 : 0;
+
+  LSSnapshotInitInfo initInfo;
+  bool ok =
+    mActor->SendPBackgroundLSSnapshotConstructor(actor,
+                                                 aObject->DocumentURI(),
+                                                 requestedSize,
+                                                 &initInfo);
+  if (NS_WARN_IF(!ok)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  snapshot->SetActor(actor);
+
+  // This add refs snapshot.
+  nsresult rv = snapshot->Init(initInfo);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  // This is cleared in LSSnapshot::Run() before the snapshot is destroyed.
+  mSnapshot = snapshot;
+
+  return NS_OK;
+}
+
 void
 LSDatabase::AllowToClose()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!mAllowedToClose);
+  MOZ_ASSERT(!mSnapshot);
 
   mAllowedToClose = true;
 
@@ -84,130 +315,5 @@ LSDatabase::AllowToClose()
   }
 }
 
-nsresult
-LSDatabase::GetLength(uint32_t* aResult)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  uint32_t result;
-  if (NS_WARN_IF(!mActor->SendGetLength(&result))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  *aResult = result;
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::GetKey(uint32_t aIndex,
-                   nsAString& aResult)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  nsString result;
-  if (NS_WARN_IF(!mActor->SendGetKey(aIndex, &result))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aResult = result;
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::GetItem(const nsAString& aKey,
-                    nsAString& aResult)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  nsString result;
-  if (NS_WARN_IF(!mActor->SendGetItem(nsString(aKey), &result))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aResult = result;
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::GetKeys(nsTArray<nsString>& aKeys)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  nsTArray<nsString> result;
-  if (NS_WARN_IF(!mActor->SendGetKeys(&result))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aKeys.SwapElements(result);
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::SetItem(const nsAString& aDocumentURI,
-                    const nsAString& aKey,
-                    const nsAString& aValue,
-                    LSWriteOpResponse& aResponse)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendSetItem(nsString(aDocumentURI),
-                                      nsString(aKey),
-                                      nsString(aValue),
-                                      &response))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aResponse = response;
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::RemoveItem(const nsAString& aDocumentURI,
-                       const nsAString& aKey,
-                       LSWriteOpResponse& aResponse)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendRemoveItem(nsString(aDocumentURI),
-                                         nsString(aKey),
-                                         &response))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aResponse = response;
-  return NS_OK;
-}
-
-nsresult
-LSDatabase::Clear(const nsAString& aDocumentURI,
-                  LSWriteOpResponse& aResponse)
-{
-  AssertIsOnOwningThread();
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mAllowedToClose);
-
-  LSWriteOpResponse response;
-  if (NS_WARN_IF(!mActor->SendClear(nsString(aDocumentURI), &response))) {
-    return NS_ERROR_FAILURE;
-  }
-
-  aResponse = response;
-  return NS_OK;
-}
-
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index 919c66ec427e..ed875322a4cc 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -11,15 +11,18 @@ namespace mozilla {
 namespace dom {
 
 class LSDatabaseChild;
-class LSWriteOpResponse;
+class LSSnapshot;
 
 class LSDatabase final
 {
   LSDatabaseChild* mActor;
 
+  LSSnapshot* mSnapshot;
+
   const nsCString mOrigin;
 
   bool mAllowedToClose;
+  bool mRequestedAllowToClose;
 
 public:
   explicit LSDatabase(const nsACString& aOrigin);
@@ -47,9 +50,6 @@ public:
     mActor = nullptr;
   }
 
-  void
-  AllowToClose();
-
   bool
   IsAllowedToClose() const
   {
@@ -58,37 +58,54 @@ public:
     return mAllowedToClose;
   }
 
-  nsresult
-  GetLength(uint32_t* aResult);
+  void
+  RequestAllowToClose();
+
+  void
+  NoteFinishedSnapshot(LSSnapshot* aSnapshot);
 
   nsresult
-  GetKey(uint32_t aIndex,
+  GetLength(LSObject* aObject,
+            uint32_t* aResult);
+
+  nsresult
+  GetKey(LSObject* aObject,
+         uint32_t aIndex,
          nsAString& aResult);
 
   nsresult
-  GetItem(const nsAString& aKey,
+  GetItem(LSObject* aObject,
+          const nsAString& aKey,
           nsAString& aResult);
 
   nsresult
-  GetKeys(nsTArray<nsString>& aKeys);
+  GetKeys(LSObject* aObject,
+          nsTArray<nsString>& aKeys);
 
   nsresult
-  SetItem(const nsAString& aDocumentURI,
+  SetItem(LSObject* aObject,
           const nsAString& aKey,
           const nsAString& aValue,
-          LSWriteOpResponse& aResponse);
+          LSNotifyInfo& aNotifyInfo);
 
   nsresult
-  RemoveItem(const nsAString& aDocumentURI,
+  RemoveItem(LSObject* aObject,
              const nsAString& aKey,
-             LSWriteOpResponse& aResponse);
+             LSNotifyInfo& aNotifyInfo);
 
   nsresult
-  Clear(const nsAString& aDocumentURI,
-        LSWriteOpResponse& aResponse);
+  Clear(LSObject* aObject,
+        LSNotifyInfo& aNotifyInfo);
 
 private:
   ~LSDatabase();
+
+  nsresult
+  EnsureSnapshot(LSObject* aObject,
+                 bool aRequestedBySetItem);
+
+  void
+  AllowToClose();
 };
 
 } // namespace dom
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index e0fc0fb8f747..40f95ea10979 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -349,7 +349,7 @@ LSObject::GetLength(nsIPrincipal& aSubjectPrincipal,
   }
 
   uint32_t result;
-  rv = mDatabase->GetLength(&result);
+  rv = mDatabase->GetLength(this, &result);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return 0;
@@ -378,7 +378,7 @@ LSObject::Key(uint32_t aIndex,
   }
 
   nsString result;
-  rv = mDatabase->GetKey(aIndex, result);
+  rv = mDatabase->GetKey(this, aIndex, result);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
@@ -407,7 +407,7 @@ LSObject::GetItem(const nsAString& aKey,
   }
 
   nsString result;
-  rv = mDatabase->GetItem(aKey, result);
+  rv = mDatabase->GetItem(this, aKey, result);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
@@ -432,7 +432,7 @@ LSObject::GetSupportedNames(nsTArray<nsString>& aNames)
     return;
   }
 
-  rv = mDatabase->GetKeys(aNames);
+  rv = mDatabase->GetKeys(this, aNames);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
   }
@@ -457,16 +457,12 @@ LSObject::SetItem(const nsAString& aKey,
     return;
   }
 
-  LSWriteOpResponse response;
-  rv = mDatabase->SetItem(mDocumentURI, aKey, aValue, response);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    aError.Throw(rv);
-    return;
-  }
-
   LSNotifyInfo info;
-  rv = GetInfoFromResponse(response, info);
-  if (NS_FAILED(rv)) {
+  rv = mDatabase->SetItem(this, aKey, aValue, info);
+  if (rv == NS_ERROR_FILE_NO_DEVICE_SPACE) {
+    rv = NS_ERROR_DOM_QUOTA_EXCEEDED_ERR;
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
@@ -494,16 +490,9 @@ LSObject::RemoveItem(const nsAString& aKey,
     return;
   }
 
-  LSWriteOpResponse response;
-  rv = mDatabase->RemoveItem(mDocumentURI, aKey, response);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    aError.Throw(rv);
-    return;
-  }
-
   LSNotifyInfo info;
-  rv = GetInfoFromResponse(response, info);
-  if (NS_FAILED(rv)) {
+  rv = mDatabase->RemoveItem(this, aKey, info);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
@@ -530,16 +519,9 @@ LSObject::Clear(nsIPrincipal& aSubjectPrincipal,
     return;
   }
 
-  LSWriteOpResponse response;
-  rv = mDatabase->Clear(mDocumentURI, response);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    aError.Throw(rv);
-    return;
-  }
-
   LSNotifyInfo info;
-  rv = GetInfoFromResponse(response, info);
-  if (NS_FAILED(rv)) {
+  rv = mDatabase->Clear(this, info);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
     aError.Throw(rv);
     return;
   }
@@ -780,28 +762,6 @@ LSObject::DropObserver()
   }
 }
 
-nsresult
-LSObject::GetInfoFromResponse(const LSWriteOpResponse& aResponse,
-                              LSNotifyInfo& aInfo)
-{
-  AssertIsOnOwningThread();
-
-  if (aResponse.type() == LSWriteOpResponse::Tnsresult) {
-    nsresult errorCode = aResponse.get_nsresult();
-
-    if (errorCode == NS_ERROR_FILE_NO_DEVICE_SPACE) {
-      errorCode = NS_ERROR_DOM_QUOTA_EXCEEDED_ERR;
-    }
-
-    return errorCode;
-  }
-
-  MOZ_ASSERT(aResponse.type() == LSWriteOpResponse::TLSNotifyInfo);
-
-  aInfo = aResponse.get_LSNotifyInfo();
-  return NS_OK;
-}
-
 void
 LSObject::OnChange(const nsAString& aKey,
                    const nsAString& aOldValue,
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index 8eb42248792a..caee2f634bc0 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -26,14 +26,12 @@ class PrincipalInfo;
 namespace dom {
 
 class LSDatabase;
-class LSNotifyInfo;
 class LSObjectChild;
 class LSObserver;
 class LSRequestChild;
 class LSRequestChildCallback;
 class LSRequestParams;
 class LSRequestResponse;
-class LSWriteOpResponse;
 
 class LSObject final
   : public Storage
@@ -80,6 +78,12 @@ public:
     NS_ASSERT_OWNINGTHREAD(LSObject);
   }
 
+  const nsString&
+  DocumentURI() const
+  {
+    return mDocumentURI;
+  }
+
   LSRequestChild*
   StartRequest(nsIEventTarget* aMainEventTarget,
                const LSRequestParams& aParams,
@@ -162,10 +166,6 @@ private:
   void
   DropObserver();
 
-  nsresult
-  GetInfoFromResponse(const LSWriteOpResponse& aResponse,
-                      LSNotifyInfo& aInfo);
-
   void
   OnChange(const nsAString& aKey,
            const nsAString& aOldValue,
diff --git a/dom/localstorage/LSSnapshot.cpp b/dom/localstorage/LSSnapshot.cpp
new file mode 100644
index 000000000000..d16e3358f59c
--- /dev/null
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -0,0 +1,320 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "LSSnapshot.h"
+
+#include "nsContentUtils.h"
+
+namespace mozilla {
+namespace dom {
+
+LSSnapshot::LSSnapshot(LSDatabase* aDatabase)
+  : mDatabase(aDatabase)
+  , mActor(nullptr)
+  , mExactUsage(0)
+  , mPeakUsage(0)
+#ifdef DEBUG
+  , mInitialized(false)
+  , mSentFinish(false)
+#endif
+{
+  AssertIsOnOwningThread();
+}
+
+LSSnapshot::~LSSnapshot()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT_IF(mInitialized, mSentFinish);
+
+  if (mActor) {
+    mActor->SendDeleteMeInternal();
+    MOZ_ASSERT(!mActor, "SendDeleteMeInternal should have cleared!");
+  }
+}
+
+void
+LSSnapshot::SetActor(LSSnapshotChild* aActor)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aActor);
+  MOZ_ASSERT(!mActor);
+
+  mActor = aActor;
+}
+
+nsresult
+LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  const nsTArray<LSItemInfo>& itemInfos = aInitInfo.itemInfos();
+  for (uint32_t i = 0; i < itemInfos.Length(); i++) {
+    const LSItemInfo& itemInfo = itemInfos[i];
+    mValues.Put(itemInfo.key(), itemInfo.value());
+  }
+
+  mExactUsage = aInitInfo.initialUsage();
+  mPeakUsage = aInitInfo.peakUsage();
+
+  nsCOMPtr<nsIRunnable> runnable = this;
+  nsContentUtils::RunInStableState(runnable.forget());
+
+#ifdef DEBUG
+  mInitialized = true;
+#endif
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::GetLength(uint32_t* aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  *aResult = mValues.Count();
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::GetKey(uint32_t aIndex,
+                   nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  aResult.SetIsVoid(true);
+  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+    if (aIndex == 0) {
+      aResult = iter.Key();
+      return NS_OK;
+    }
+    aIndex--;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::GetItem(const nsAString& aKey,
+                    nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  nsString result;
+  if (!mValues.Get(aKey, &result)) {
+    result.SetIsVoid(true);
+  }
+
+  aResult = result;
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::GetKeys(nsTArray<nsString>& aKeys)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+    aKeys.AppendElement(iter.Key());
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::SetItem(const nsAString& aKey,
+                    const nsAString& aValue,
+                    LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  nsString oldValue;
+  nsresult rv = GetItem(aKey, oldValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool changed;
+  if (oldValue == aValue && oldValue.IsVoid() == aValue.IsVoid()) {
+    changed = false;
+  } else {
+    changed = true;
+
+    int64_t delta = static_cast<int64_t>(aValue.Length()) -
+                    static_cast<int64_t>(oldValue.Length());
+
+    if (oldValue.IsVoid()) {
+      delta += static_cast<int64_t>(aKey.Length());
+    }
+
+    rv = UpdateUsage(delta);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    mValues.Put(aKey, nsString(aValue));
+
+    LSSetItemInfo setItemInfo;
+    setItemInfo.key() = aKey;
+    setItemInfo.oldValue() = oldValue;
+    setItemInfo.value() = aValue;
+
+    mWriteInfos.AppendElement(std::move(setItemInfo));
+  }
+
+  aNotifyInfo.changed() = changed;
+  aNotifyInfo.oldValue() = oldValue;
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::RemoveItem(const nsAString& aKey,
+                       LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  nsString oldValue;
+  nsresult rv = GetItem(aKey, oldValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool changed;
+  if (oldValue.IsVoid()) {
+    changed = false;
+  } else {
+    changed = true;
+
+    int64_t delta = -(static_cast<int64_t>(aKey.Length()) +
+                      static_cast<int64_t>(oldValue.Length()));
+
+    DebugOnly<nsresult> rv = UpdateUsage(delta);
+    MOZ_ASSERT(NS_SUCCEEDED(rv));
+
+    mValues.Remove(aKey);
+
+    LSRemoveItemInfo removeItemInfo;
+    removeItemInfo.key() = aKey;
+    removeItemInfo.oldValue() = oldValue;
+
+    mWriteInfos.AppendElement(std::move(removeItemInfo));
+  }
+
+  aNotifyInfo.changed() = changed;
+  aNotifyInfo.oldValue() = oldValue;
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::Clear(LSNotifyInfo& aNotifyInfo)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  bool changed;
+  if (!mValues.Count()) {
+    changed = false;
+  } else {
+    changed = true;
+
+    DebugOnly<nsresult> rv = UpdateUsage(-mExactUsage);
+    MOZ_ASSERT(NS_SUCCEEDED(rv));
+
+    mValues.Clear();
+
+    LSClearInfo clearInfo;
+
+    mWriteInfos.AppendElement(std::move(clearInfo));
+  }
+
+  aNotifyInfo.changed() = changed;
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::UpdateUsage(int64_t aDelta)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mPeakUsage >= mExactUsage);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  int64_t newExactUsage = mExactUsage + aDelta;
+  if (newExactUsage > mPeakUsage) {
+    int64_t minSize = newExactUsage - mPeakUsage;
+    int64_t requestedSize = minSize + 4096;
+    int64_t size;
+    if (NS_WARN_IF(!mActor->SendIncreasePeakUsage(requestedSize,
+                                                  minSize,
+                                                  &size))) {
+      return NS_ERROR_FAILURE;
+    }
+
+    MOZ_ASSERT(size >= 0);
+
+    if (size == 0) {
+      return NS_ERROR_FILE_NO_DEVICE_SPACE;
+    }
+
+    mPeakUsage += size;
+  }
+
+  mExactUsage = newExactUsage;
+  return NS_OK;
+}
+
+NS_IMPL_ISUPPORTS(LSSnapshot, nsIRunnable)
+
+NS_IMETHODIMP
+LSSnapshot::Run()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mSentFinish);
+
+  MOZ_ALWAYS_TRUE(mActor->SendFinish(mWriteInfos));
+
+#ifdef DEBUG
+  mSentFinish = true;
+#endif
+
+  mDatabase->NoteFinishedSnapshot(this);
+
+  return NS_OK;
+}
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
new file mode 100644
index 000000000000..27daa7b25fc0
--- /dev/null
+++ b/dom/localstorage/LSSnapshot.h
@@ -0,0 +1,96 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_LSSnapshot_h
+#define mozilla_dom_localstorage_LSSnapshot_h
+
+namespace mozilla {
+namespace dom {
+
+class LSSnapshotChild;
+
+class LSSnapshot final
+  : public nsIRunnable
+{
+  RefPtr<LSDatabase> mDatabase;
+
+  LSSnapshotChild* mActor;
+
+  nsDataHashtable<nsStringHashKey, nsString> mValues;
+  nsTArray<LSWriteInfo> mWriteInfos;
+
+  int64_t mExactUsage;
+  int64_t mPeakUsage;
+
+#ifdef DEBUG
+  bool mInitialized;
+  bool mSentFinish;
+#endif
+
+public:
+  explicit LSSnapshot(LSDatabase* aDatabase);
+
+  void
+  AssertIsOnOwningThread() const
+  {
+    NS_ASSERT_OWNINGTHREAD(LSSnapshot);
+  }
+
+  void
+  SetActor(LSSnapshotChild* aActor);
+
+  void
+  ClearActor()
+  {
+    AssertIsOnOwningThread();
+    MOZ_ASSERT(mActor);
+
+    mActor = nullptr;
+  }
+
+  nsresult
+  Init(const LSSnapshotInitInfo& aInitInfo);
+
+  nsresult
+  GetLength(uint32_t* aResult);
+
+  nsresult
+  GetKey(uint32_t aIndex,
+         nsAString& aResult);
+
+  nsresult
+  GetItem(const nsAString& aKey,
+          nsAString& aResult);
+
+  nsresult
+  GetKeys(nsTArray<nsString>& aKeys);
+
+  nsresult
+  SetItem(const nsAString& aKey,
+          const nsAString& aValue,
+          LSNotifyInfo& aNotifyInfo);
+
+  nsresult
+  RemoveItem(const nsAString& aKey,
+             LSNotifyInfo& aNotifyInfo);
+
+  nsresult
+  Clear(LSNotifyInfo& aNotifyInfo);
+
+private:
+  ~LSSnapshot();
+
+  nsresult
+  UpdateUsage(int64_t aDelta);
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_NSIRUNNABLE
+};
+
+} // namespace dom
+} // namespace mozilla
+
+#endif // mozilla_dom_localstorage_LSSnapshot_h
diff --git a/dom/localstorage/LocalStorageCommon.h b/dom/localstorage/LocalStorageCommon.h
index 7c31b7a56cf6..62d8b64834d0 100644
--- a/dom/localstorage/LocalStorageCommon.h
+++ b/dom/localstorage/LocalStorageCommon.h
@@ -186,6 +186,41 @@ namespace dom {
 
 extern const char16_t* kLocalStorageType;
 
+class MOZ_STACK_CLASS LSNotifyInfo
+{
+  bool mChanged;
+  nsString mOldValue;
+
+public:
+  LSNotifyInfo()
+    : mChanged(false)
+  { }
+
+  bool
+  changed() const
+  {
+    return mChanged;
+  }
+
+  bool&
+  changed()
+  {
+    return mChanged;
+  }
+
+  const nsString&
+  oldValue() const
+  {
+    return mOldValue;
+  }
+
+  nsString&
+  oldValue()
+  {
+    return mOldValue;
+  }
+};
+
 bool
 NextGenLocalStorageEnabled();
 
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 1bdce2ded6db..aa27a4939ab0 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -3,25 +3,28 @@
  * You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 include protocol PBackground;
+include protocol PBackgroundLSSnapshot;
 
 namespace mozilla {
 namespace dom {
 
-struct LSNotifyInfo
+struct LSItemInfo
 {
-  bool changed;
-  nsString oldValue;
+  nsString key;
+  nsString value;
 };
 
-union LSWriteOpResponse
+struct LSSnapshotInitInfo
 {
-  nsresult;
-  LSNotifyInfo;
+  LSItemInfo[] itemInfos;
+  int64_t initialUsage;
+  int64_t peakUsage;
 };
 
 sync protocol PBackgroundLSDatabase
 {
   manager PBackground;
+  manages PBackgroundLSSnapshot;
 
 parent:
   // The DeleteMe message is used to avoid a race condition between the parent
@@ -37,26 +40,9 @@ parent:
 
   async AllowToClose();
 
-  sync GetLength()
-    returns (uint32_t length);
-
-  sync GetKey(uint32_t index)
-    returns (nsString key);
-
-  sync GetItem(nsString key)
-    returns (nsString value);
-
-  sync GetKeys()
-    returns (nsString[] keys);
-
-  sync SetItem(nsString documentURI, nsString key, nsString value)
-    returns (LSWriteOpResponse response);
-
-  sync RemoveItem(nsString documentURI, nsString key)
-    returns (LSWriteOpResponse response);
-
-  sync Clear(nsString documentURI)
-    returns (LSWriteOpResponse response);
+  sync PBackgroundLSSnapshot(nsString documentURI,
+                             int64_t requestedSize)
+    returns (LSSnapshotInitInfo initInfo);
 
 child:
   async __delete__();
diff --git a/dom/localstorage/PBackgroundLSSnapshot.ipdl b/dom/localstorage/PBackgroundLSSnapshot.ipdl
new file mode 100644
index 000000000000..022e453943a2
--- /dev/null
+++ b/dom/localstorage/PBackgroundLSSnapshot.ipdl
@@ -0,0 +1,57 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+include protocol PBackground;
+include protocol PBackgroundLSDatabase;
+
+namespace mozilla {
+namespace dom {
+
+struct LSSetItemInfo
+{
+  nsString key;
+  nsString oldValue;
+  nsString value;
+};
+
+struct LSRemoveItemInfo
+{
+  nsString key;
+  nsString oldValue;
+};
+
+struct LSClearInfo
+{
+};
+
+union LSWriteInfo
+{
+  LSSetItemInfo;
+  LSRemoveItemInfo;
+  LSClearInfo;
+};
+
+struct LSSnapshotFinishInfo
+{
+  LSWriteInfo[] writeInfos;
+};
+
+sync protocol PBackgroundLSSnapshot
+{
+  manager PBackgroundLSDatabase;
+
+parent:
+  async DeleteMe();
+
+  async Finish(LSSnapshotFinishInfo finishInfo);
+
+  sync IncreasePeakUsage(int64_t requestedSize, int64_t minSize)
+    returns (int64_t size);
+
+child:
+  async __delete__();
+};
+
+} // namespace dom
+} // namespace mozilla
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index dd83a5cbf846..6cd6e8b4c0a9 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -33,6 +33,7 @@ UNIFIED_SOURCES += [
     'LSDatabase.cpp',
     'LSObject.cpp',
     'LSObserver.cpp',
+    'LSSnapshot.cpp',
     'ReportInternalError.cpp',
 ]
 
@@ -42,6 +43,7 @@ IPDL_SOURCES += [
     'PBackgroundLSRequest.ipdl',
     'PBackgroundLSSharedTypes.ipdlh',
     'PBackgroundLSSimpleRequest.ipdl',
+    'PBackgroundLSSnapshot.ipdl',
 ]
 
 include('/ipc/chromium/chromium-config.mozbuild')
diff --git a/dom/localstorage/test/unit/head.js b/dom/localstorage/test/unit/head.js
index 4e7c32a78793..9d9e9cdbe713 100644
--- a/dom/localstorage/test/unit/head.js
+++ b/dom/localstorage/test/unit/head.js
@@ -43,6 +43,13 @@ function finishTest()
   })
 }
 
+function continueToNextStep()
+{
+  executeSoon(function() {
+    testGenerator.next();
+  });
+}
+
 function continueToNextStepSync()
 {
   testGenerator.next();
diff --git a/dom/localstorage/test/unit/test_groupLimit.js b/dom/localstorage/test/unit/test_groupLimit.js
index f408a549e0e6..c9c140d7cc11 100644
--- a/dom/localstorage/test/unit/test_groupLimit.js
+++ b/dom/localstorage/test/unit/test_groupLimit.js
@@ -67,6 +67,11 @@ function* testSteps()
 
   storages[0].clear();
 
+  // Let the internal snapshot finish (usage is not descreased until all
+  // snapshots finish)..
+  continueToNextStep();
+  yield undefined;
+
   info("Verifying more data can be written");
 
   for (let i = 0; i < urls.length; i++) {
diff --git a/ipc/ipdl/sync-messages.ini b/ipc/ipdl/sync-messages.ini
index 07afb207e63b..4589561d3fdc 100644
--- a/ipc/ipdl/sync-messages.ini
+++ b/ipc/ipdl/sync-messages.ini
@@ -922,19 +922,9 @@ description =
 description =
 [PBackgroundStorage::Preload]
 description =
-[PBackgroundLSDatabase::GetLength]
+[PBackgroundLSDatabase::PBackgroundLSSnapshot]
 description =
-[PBackgroundLSDatabase::GetKey]
-description =
-[PBackgroundLSDatabase::GetItem]
-description =
-[PBackgroundLSDatabase::GetKeys]
-description =
-[PBackgroundLSDatabase::SetItem]
-description =
-[PBackgroundLSDatabase::RemoveItem]
-description =
-[PBackgroundLSDatabase::Clear]
+[PBackgroundLSSnapshot::IncreasePeakUsage]
 description =
 [PRemoteSpellcheckEngine::Check]
 description =

From 34cd5113f112ae7f70fc2e5d263cf7183f8018c8 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:51 +0100
Subject: [PATCH 44/78] Bug 1286798 - Part 30: Preserve key order when sending
 items to a content process; r=asuth

Keys needs to be redundantly stored in an array, so we can construct identical hashtable of values in a content process.
---
 dom/localstorage/ActorsParent.cpp             | 60 +++++++++++++++++--
 .../test/unit/test_snapshotting.js            | 50 ++++++++++++++++
 dom/localstorage/test/unit/xpcshell.ini       |  1 +
 3 files changed, 105 insertions(+), 6 deletions(-)
 create mode 100644 dom/localstorage/test/unit/test_snapshotting.js

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 9c81bcd36ca1..2fbc66a40555 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1030,6 +1030,9 @@ class Datastore final
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsTHashtable<nsPtrHashKey<Database>> mActiveDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
+  nsDataHashtable<nsStringHashKey, uint32_t> mUpdateBatchRemovals;
+  nsTArray<nsString> mUpdateBatchAppends;
+  nsTArray<nsString> mKeys;
   nsTArray<int64_t> mPendingUsageDeltas;
   const nsCString mOrigin;
   const uint32_t mPrivateBrowsingId;
@@ -1048,7 +1051,8 @@ public:
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
             already_AddRefed<QuotaObject>&& aQuotaObject,
-            nsDataHashtable<nsStringHashKey, nsString>& aValues);
+            nsDataHashtable<nsStringHashKey, nsString>& aValues,
+            nsTArray<nsString>& aKeys);
 
   const nsCString&
   Origin() const
@@ -1618,6 +1622,7 @@ class PrepareDatastoreOp
   nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   LoadDataOp* mLoadDataOp;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
+  nsTArray<nsString> mKeys;
   const LSRequestPrepareDatastoreParams mParams;
   nsCString mSuffix;
   nsCString mGroup;
@@ -3051,7 +3056,8 @@ Datastore::Datastore(const nsACString& aOrigin,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
                      already_AddRefed<QuotaObject>&& aQuotaObject,
-                     nsDataHashtable<nsStringHashKey, nsString>& aValues)
+                     nsDataHashtable<nsStringHashKey, nsString>& aValues,
+                     nsTArray<nsString>& aKeys)
   : mDirectoryLock(std::move(aDirectoryLock))
   , mConnection(std::move(aConnection))
   , mQuotaObject(std::move(aQuotaObject))
@@ -3067,6 +3073,7 @@ Datastore::Datastore(const nsACString& aOrigin,
   AssertIsOnBackgroundThread();
 
   mValues.SwapElements(aValues);
+  mKeys.SwapElements(aKeys);
 }
 
 Datastore::~Datastore()
@@ -3274,10 +3281,14 @@ Datastore::NoteInactiveDatabase(Database* aDatabase)
 void
 Datastore::GetItemInfos(nsTArray<LSItemInfo>* aItemInfos)
 {
-  for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+  for (auto key : mKeys) {
+    nsString value;
+    DebugOnly<bool> hasValue = mValues.Get(key, &value);
+    MOZ_ASSERT(hasValue);
+
     LSItemInfo* itemInfo = aItemInfos->AppendElement();
-    itemInfo->key() = iter.Key();
-    itemInfo->value() = iter.Data();
+    itemInfo->key() = key;
+    itemInfo->value() = value;
   }
 }
 
@@ -3313,6 +3324,8 @@ Datastore::SetItem(Database* aDatabase,
 
     if (oldValue.IsVoid()) {
       delta += static_cast<int64_t>(aKey.Length());
+
+      mUpdateBatchAppends.AppendElement(aKey);
     }
 
     mUpdateBatchUsage += delta;
@@ -3345,6 +3358,13 @@ Datastore::RemoveItem(Database* aDatabase,
   GetItem(aKey, oldValue);
 
   if (!oldValue.IsVoid()) {
+    auto entry = mUpdateBatchRemovals.LookupForAdd(aKey);
+    if (entry) {
+      entry.Data()++;
+    } else {
+      entry.OrInsert([]() { return 1; });
+    }
+
     int64_t delta = -(static_cast<int64_t>(aKey.Length()) +
                       static_cast<int64_t>(oldValue.Length()));
 
@@ -3373,6 +3393,9 @@ Datastore::Clear(Database* aDatabase,
   MOZ_ASSERT(mInUpdateBatch);
 
   if (mValues.Count()) {
+    mUpdateBatchRemovals.Clear();
+    mUpdateBatchAppends.Clear();
+
     for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
       const nsAString& key = iter.Key();
       const nsAString& value = iter.Data();
@@ -3384,6 +3407,7 @@ Datastore::Clear(Database* aDatabase,
     }
 
     mValues.Clear();
+    mKeys.Clear();
 
     if (IsPersistent()) {
       EnsureTransaction();
@@ -3412,6 +3436,7 @@ Datastore::PrivateBrowsingClear()
     MOZ_ASSERT(ok);
 
     mValues.Clear();
+    mKeys.Clear();
   }
 }
 
@@ -3439,6 +3464,27 @@ Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(mInUpdateBatch);
 
+  if (mUpdateBatchAppends.Length()) {
+    mKeys.AppendElements(std::move(mUpdateBatchAppends));
+  }
+
+  if (mUpdateBatchRemovals.Count()) {
+    RefPtr<Datastore> self = this;
+
+    mKeys.RemoveElementsBy([self](const nsString& aKey) {
+      if (auto entry = self->mUpdateBatchRemovals.Lookup(aKey)) {
+        if (--entry.Data() == 0) {
+          entry.Remove();
+        }
+        return true;
+      }
+      return false;
+    });
+  }
+
+  MOZ_ASSERT(mUpdateBatchAppends.Length() == 0);
+  MOZ_ASSERT(mUpdateBatchRemovals.Count() == 0);
+
   int64_t delta = mUpdateBatchUsage - aSnapshotPeakUsage;
 
   if (mActiveDatabases.Count()) {
@@ -5085,7 +5131,8 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
                                mDirectoryLock.forget(),
                                mConnection.forget(),
                                quotaObject.forget(),
-                               mValues);
+                               mValues,
+                               mKeys);
 
     mDatastore->NoteLivePrepareDatastoreOp(this);
 
@@ -5295,6 +5342,7 @@ LoadDataOp::DoDatastoreWork()
     }
 
     mPrepareDatastoreOp->mValues.Put(key, value);
+    mPrepareDatastoreOp->mKeys.AppendElement(key);
 #ifdef DEBUG
     mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
 #endif
diff --git a/dom/localstorage/test/unit/test_snapshotting.js b/dom/localstorage/test/unit/test_snapshotting.js
new file mode 100644
index 000000000000..e9d7bdacd384
--- /dev/null
+++ b/dom/localstorage/test/unit/test_snapshotting.js
@@ -0,0 +1,50 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const url = "http://example.com";
+
+  const items = [
+    { key: "key1", value: "value1" },
+    { key: "key2", value: "value2" },
+    { key: "key3", value: "value3" },
+    { key: "key4", value: "value4" },
+    { key: "key5", value: "value5" }
+  ];
+
+  info("Getting storage");
+
+  let storage = getLocalStorage(getPrincipal(url));
+
+  info("Adding data");
+
+  for (let item of items) {
+    storage.setItem(item.key, item.value);
+  }
+
+  info("Saving key order");
+
+  let keys = [];
+  for (let i = 0; i < items.length; i++) {
+    keys.push(storage.key(i));
+  }
+
+  // Let the internal snapshot finish by returning to the event loop.
+  continueToNextStep();
+  yield undefined;
+
+  is(storage.length, items.length, "Correct length");
+
+  info("Verifying key order");
+
+  for (let i = 0; i < items.length; i++) {
+    is(storage.key(i), keys[i], "Correct key");
+  }
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/xpcshell.ini b/dom/localstorage/test/unit/xpcshell.ini
index fbac5f3fa335..fb4ca0de3f04 100644
--- a/dom/localstorage/test/unit/xpcshell.ini
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -12,3 +12,4 @@ support-files =
 [test_eviction.js]
 [test_groupLimit.js]
 [test_migration.js]
+[test_snapshotting.js]

From d87888fe25b5efdb4e67d963b467b118fc083ca1 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:54 +0100
Subject: [PATCH 45/78] Bug 1286798 - Part 31: Support for lazy loading of
 items; r=asuth,mrbkap,mccr8

There's now an upper limit for snapshot prefilling. The value is configurable and is currently set to 4096 bytes.
Snapshots can operate in multiple modes depending on if all items have been loaded or all keys have been received. This should provide the best performance for each specific state.
This patch also adds support for creating explicit snapshots which can be used for testing.
---
 dom/localstorage/ActorsParent.cpp             | 341 +++++++++++++++++-
 dom/localstorage/LSDatabase.cpp               |  50 ++-
 dom/localstorage/LSDatabase.h                 |   9 +-
 dom/localstorage/LSObject.cpp                 |  92 +++++
 dom/localstorage/LSObject.h                   |  13 +
 dom/localstorage/LSSnapshot.cpp               | 256 ++++++++++++-
 dom/localstorage/LSSnapshot.h                 |  33 +-
 dom/localstorage/PBackgroundLSDatabase.ipdl   |   1 +
 dom/localstorage/PBackgroundLSSnapshot.ipdl   |  13 +
 .../test/unit/test_snapshotting.js            | 235 ++++++++++--
 dom/storage/Storage.h                         |   8 +
 dom/webidl/Storage.webidl                     |   6 +
 ipc/ipdl/sync-messages.ini                    |   6 +
 modules/libpref/init/all.js                   |   1 +
 14 files changed, 1014 insertions(+), 50 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 2fbc66a40555..daafa2a46fa8 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -118,7 +118,9 @@ const uint32_t kAutoCommitTimeoutMs = 5000;
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
+const uint32_t kDefaultSnapshotPrefill = 4096;
 const char kDefaultQuotaPref[] = "dom.storage.default_quota";
+const char kSnapshotPrefillPref[] = "dom.storage.snapshot_prefill";
 
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
 
@@ -1127,12 +1129,19 @@ public:
   void
   NoteInactiveDatabase(Database* aDatabase);
 
+  uint32_t
+  GetLength() const;
+
   void
-  GetItemInfos(nsTArray<LSItemInfo>* aItemInfos);
+  GetItemInfos(nsTHashtable<nsStringHashKey>& aLoadedItems,
+               nsTArray<LSItemInfo>& aItemInfos);
 
   void
   GetItem(const nsString& aKey, nsString& aValue) const;
 
+  void
+  GetKeys(nsTArray<nsString>& aKeys) const;
+
   void
   SetItem(Database* aDatabase,
           const nsString& aDocumentURI,
@@ -1177,6 +1186,12 @@ private:
   void
   CleanupMetadata();
 
+  void
+  NotifySnapshots(Database* aDatabase,
+                  const nsAString& aKey,
+                  const nsAString& aOldValue,
+                  bool aAffectsOrder);
+
   void
   NotifyObservers(Database* aDatabase,
                   const nsString& aDocumentURI,
@@ -1349,6 +1364,13 @@ public:
   void
   UnregisterSnapshot(Snapshot* aSnapshot);
 
+  Snapshot*
+  GetSnapshot() const
+  {
+    AssertIsOnBackgroundThread();
+    return mSnapshot;
+  }
+
   void
   RequestAllowToClose();
 
@@ -1392,11 +1414,20 @@ class Snapshot final
 {
   RefPtr<Database> mDatabase;
   RefPtr<Datastore> mDatastore;
+  nsTHashtable<nsStringHashKey> mLoadedItems;
+  nsTHashtable<nsStringHashKey> mUnknownItems;
+  nsDataHashtable<nsStringHashKey, nsString> mValues;
+  nsTArray<nsString> mKeys;
   nsString mDocumentURI;
+  uint32_t mTotalLength;
   int64_t mInitialUsage;
   int64_t mPeakUsage;
+  bool mSavedKeys;
   bool mActorDestroyed;
   bool mFinishReceived;
+  bool mLoadedReceived;
+  bool mLoadedAllItems;
+  bool mLoadKeysReceived;
 
 public:
   // Created in AllocPBackgroundLSSnapshotParent.
@@ -1404,19 +1435,36 @@ public:
            const nsAString& aDocumentURI);
 
   void
-  SetUsage(int64_t aInitialUsage,
-           int64_t aPeakUsage)
+  Init(nsTHashtable<nsStringHashKey>& aLoadedItems,
+       uint32_t aTotalLength,
+       int64_t aInitialUsage,
+       int64_t aPeakUsage,
+       bool aFullPrefill)
   {
     AssertIsOnBackgroundThread();
     MOZ_ASSERT(aInitialUsage >= 0);
     MOZ_ASSERT(aPeakUsage >= aInitialUsage);
+    MOZ_ASSERT_IF(aFullPrefill, aLoadedItems.Count() == 0);
+    MOZ_ASSERT(mTotalLength == 0);
     MOZ_ASSERT(mInitialUsage == -1);
     MOZ_ASSERT(mPeakUsage == -1);
 
+    mLoadedItems.SwapElements(aLoadedItems);
+    mTotalLength = aTotalLength;
     mInitialUsage = aInitialUsage;
     mPeakUsage = aPeakUsage;
+    if (aFullPrefill) {
+      mLoadedReceived = true;
+      mLoadedAllItems = true;
+      mLoadKeysReceived = true;
+    }
   }
 
+  void
+  SaveItem(const nsAString& aKey,
+           const nsAString& aOldValue,
+           bool aAffectsOrder);
+
   NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Snapshot)
 
 private:
@@ -1433,10 +1481,23 @@ private:
   mozilla::ipc::IPCResult
   RecvFinish(const LSSnapshotFinishInfo& aFinishInfo) override;
 
+  mozilla::ipc::IPCResult
+  RecvLoaded() override;
+
+  mozilla::ipc::IPCResult
+  RecvLoadItem(const nsString& aKey,
+               nsString* aValue) override;
+
+  mozilla::ipc::IPCResult
+  RecvLoadKeys(nsTArray<nsString>* aKeys) override;
+
   mozilla::ipc::IPCResult
   RecvIncreasePeakUsage(const int64_t& aRequestedSize,
                         const int64_t& aMinSize,
                         int64_t* aSize) override;
+
+  mozilla::ipc::IPCResult
+  RecvPing() override;
 };
 
 class Observer final
@@ -2085,6 +2146,7 @@ typedef nsClassHashtable<nsCStringHashKey, nsTArray<Observer*>>
 StaticAutoPtr<ObserverHashtable> gObservers;
 
 Atomic<uint32_t, Relaxed> gOriginLimitKB(kDefaultOriginLimitKB);
+Atomic<int32_t, Relaxed> gSnapshotPrefill(kDefaultSnapshotPrefill);
 
 typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
 
@@ -2233,6 +2295,24 @@ GetUsage(mozIStorageConnection* aConnection,
   return NS_OK;
 }
 
+void
+SnapshotPrefillPrefChangedCallback(const char* aPrefName, void* aClosure)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!strcmp(aPrefName, kSnapshotPrefillPref));
+  MOZ_ASSERT(!aClosure);
+
+  int32_t snapshotPrefill =
+    Preferences::GetInt(aPrefName, kDefaultSnapshotPrefill);
+
+  // The magic -1 is for use only by tests.
+  if (snapshotPrefill == -1) {
+    snapshotPrefill = INT32_MAX;
+  }
+
+  gSnapshotPrefill = snapshotPrefill;
+}
+
 } // namespace
 
 /*******************************************************************************
@@ -3278,18 +3358,43 @@ Datastore::NoteInactiveDatabase(Database* aDatabase)
   }
 }
 
-void
-Datastore::GetItemInfos(nsTArray<LSItemInfo>* aItemInfos)
+uint32_t
+Datastore::GetLength() const
 {
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
+
+  return mValues.Count();
+}
+
+void
+Datastore::GetItemInfos(nsTHashtable<nsStringHashKey>& aLoadedItems,
+                        nsTArray<LSItemInfo>& aItemInfos)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
+
+  int64_t size = 0;
   for (auto key : mKeys) {
     nsString value;
     DebugOnly<bool> hasValue = mValues.Get(key, &value);
     MOZ_ASSERT(hasValue);
 
-    LSItemInfo* itemInfo = aItemInfos->AppendElement();
+    size += static_cast<int64_t>(key.Length()) +
+            static_cast<int64_t>(value.Length());
+
+    if (size > gSnapshotPrefill) {
+      return;
+    }
+
+    aLoadedItems.PutEntry(key);
+
+    LSItemInfo* itemInfo = aItemInfos.AppendElement();
     itemInfo->key() = key;
     itemInfo->value() = value;
   }
+
+  aLoadedItems.Clear();
 }
 
 void
@@ -3303,6 +3408,15 @@ Datastore::GetItem(const nsString& aKey, nsString& aValue) const
   }
 }
 
+void
+Datastore::GetKeys(nsTArray<nsString>& aKeys) const
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(!mClosed);
+
+  aKeys.AppendElements(mKeys);
+}
+
 void
 Datastore::SetItem(Database* aDatabase,
                    const nsString& aDocumentURI,
@@ -3319,17 +3433,25 @@ Datastore::SetItem(Database* aDatabase,
   GetItem(aKey, oldValue);
 
   if (oldValue != aValue || oldValue.IsVoid() != aValue.IsVoid()) {
+    bool affectsOrder;
+
     int64_t delta = static_cast<int64_t>(aValue.Length()) -
                     static_cast<int64_t>(oldValue.Length());
 
     if (oldValue.IsVoid()) {
+      affectsOrder = true;
+
       delta += static_cast<int64_t>(aKey.Length());
 
       mUpdateBatchAppends.AppendElement(aKey);
+    } else {
+      affectsOrder = false;
     }
 
     mUpdateBatchUsage += delta;
 
+    NotifySnapshots(aDatabase, aKey, oldValue, affectsOrder);
+
     mValues.Put(aKey, aValue);
 
     if (IsPersistent()) {
@@ -3370,6 +3492,8 @@ Datastore::RemoveItem(Database* aDatabase,
 
     mUpdateBatchUsage += delta;
 
+    NotifySnapshots(aDatabase, aKey, oldValue, /* aAffectsOrder */ true);
+
     mValues.Remove(aKey);
 
     if (IsPersistent()) {
@@ -3404,6 +3528,8 @@ Datastore::Clear(Database* aDatabase,
                         static_cast<int64_t>(value.Length()));
 
       mUpdateBatchUsage += delta;
+
+      NotifySnapshots(aDatabase, key, value, /* aAffectsOrder */ true);
     }
 
     mValues.Clear();
@@ -3604,6 +3730,28 @@ Datastore::CleanupMetadata()
   }
 }
 
+void
+Datastore::NotifySnapshots(Database* aDatabase,
+                           const nsAString& aKey,
+                           const nsAString& aOldValue,
+                           bool aAffectsOrder)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aDatabase);
+
+  for (auto iter = mDatabases.ConstIter(); !iter.Done(); iter.Next()) {
+    Database* database = iter.Get()->GetKey();
+    if (database == aDatabase) {
+      continue;
+    }
+
+    Snapshot* snapshot = database->GetSnapshot();
+    if (snapshot) {
+      snapshot->SaveItem(aKey, aOldValue, aAffectsOrder);
+    }
+  }
+}
+
 void
 Datastore::NotifyObservers(Database* aDatabase,
                            const nsString& aDocumentURI,
@@ -3895,8 +4043,11 @@ Database::RecvPBackgroundLSSnapshotConstructor(
 
   // TODO: This can be optimized depending on which operation triggers snapshot
   //       creation. For example clear() doesn't need to receive items at all.
+  nsTHashtable<nsStringHashKey> loadedItems;
   nsTArray<LSItemInfo> itemInfos;
-  mDatastore->GetItemInfos(&itemInfos);
+  mDatastore->GetItemInfos(loadedItems, itemInfos);
+
+  uint32_t totalLength = mDatastore->GetLength();
 
   int64_t initialUsage = mDatastore->Usage();
 
@@ -3905,11 +4056,16 @@ Database::RecvPBackgroundLSSnapshotConstructor(
     peakUsage += aRequestedSize;
   }
 
-  snapshot->SetUsage(initialUsage, peakUsage);
+  snapshot->Init(loadedItems,
+                 totalLength,
+                 initialUsage,
+                 peakUsage,
+                 /* aFullPrefill */ itemInfos.Length() == totalLength);
 
   RegisterSnapshot(snapshot);
 
   aInitInfo->itemInfos() = std::move(itemInfos);
+  aInitInfo->totalLength() = totalLength;
   aInitInfo->initialUsage() = initialUsage;
   aInitInfo->peakUsage() = peakUsage;
 
@@ -3938,10 +4094,15 @@ Snapshot::Snapshot(Database* aDatabase,
   : mDatabase(aDatabase)
   , mDatastore(aDatabase->GetDatastore())
   , mDocumentURI(aDocumentURI)
+  , mTotalLength(0)
   , mInitialUsage(-1)
   , mPeakUsage(-1)
+  , mSavedKeys(false)
   , mActorDestroyed(false)
   , mFinishReceived(false)
+  , mLoadedReceived(false)
+  , mLoadedAllItems(false)
+  , mLoadKeysReceived(false)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
@@ -3953,6 +4114,30 @@ Snapshot::~Snapshot()
   MOZ_ASSERT(mFinishReceived);
 }
 
+void
+Snapshot::SaveItem(const nsAString& aKey,
+                   const nsAString& aOldValue,
+                   bool aAffectsOrder)
+{
+  AssertIsOnBackgroundThread();
+
+  if (mLoadedAllItems) {
+    return;
+  }
+
+  if (!mLoadedItems.GetEntry(aKey) && !mUnknownItems.GetEntry(aKey)) {
+    nsString oldValue(aOldValue);
+    mValues.LookupForAdd(aKey).OrInsert([oldValue]() {
+      return oldValue;
+    });
+  }
+
+  if (aAffectsOrder && !mSavedKeys && !mLoadKeysReceived) {
+    mDatastore->GetKeys(mKeys);
+    mSavedKeys = true;
+  }
+}
+
 void
 Snapshot::ActorDestroy(ActorDestroyReason aWhy)
 {
@@ -4044,6 +4229,132 @@ Snapshot::RecvFinish(const LSSnapshotFinishInfo& aFinishInfo)
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+Snapshot::RecvLoaded()
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(mFinishReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadedReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadedAllItems)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadKeysReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  mLoadedReceived = true;
+
+  mLoadedItems.Clear();
+  mUnknownItems.Clear();
+  mValues.Clear();
+  mKeys.Clear();
+  mLoadedAllItems = true;
+  mLoadKeysReceived = true;
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Snapshot::RecvLoadItem(const nsString& aKey,
+                       nsString* aValue)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aValue);
+  MOZ_ASSERT(mDatastore);
+
+  if (NS_WARN_IF(mFinishReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadedReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadedAllItems)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (mLoadedItems.GetEntry(aKey) || mUnknownItems.GetEntry(aKey)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (auto entry = mValues.Lookup(aKey)) {
+    *aValue = entry.Data();
+    entry.Remove();
+  } else {
+    mDatastore->GetItem(aKey, *aValue);
+  }
+
+  if (aValue->IsVoid()) {
+    mUnknownItems.PutEntry(aKey);
+  } else {
+    mLoadedItems.PutEntry(aKey);
+
+    if (mLoadedItems.Count() == mTotalLength) {
+      mLoadedItems.Clear();
+      mUnknownItems.Clear();
+#ifdef DEBUG
+      for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+        MOZ_ASSERT(iter.Data().IsVoid());
+      }
+#endif
+      mValues.Clear();
+      mLoadedAllItems = true;
+    }
+  }
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Snapshot::RecvLoadKeys(nsTArray<nsString>* aKeys)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aKeys);
+  MOZ_ASSERT(mDatastore);
+
+  if (NS_WARN_IF(mFinishReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadedReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  if (NS_WARN_IF(mLoadKeysReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  mLoadKeysReceived = true;
+
+  if (mSavedKeys) {
+    aKeys->AppendElements(std::move(mKeys));
+  } else {
+    mDatastore->GetKeys(*aKeys);
+  }
+
+  return IPC_OK();
+}
+
 mozilla::ipc::IPCResult
 Snapshot::RecvIncreasePeakUsage(const int64_t& aRequestedSize,
                                 const int64_t& aMinSize,
@@ -4080,6 +4391,17 @@ Snapshot::RecvIncreasePeakUsage(const int64_t& aRequestedSize,
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+Snapshot::RecvPing()
+{
+  AssertIsOnBackgroundThread();
+
+  // Do nothing here. This is purely a sync message allowing the child to
+  // confirm that the actor has received previous async message.
+
+  return IPC_OK();
+}
+
 /*******************************************************************************
  * Observer
  ******************************************************************************/
@@ -5743,6 +6065,9 @@ QuotaClient::RegisterObservers(nsIEventTarget* aBackgroundEventTarget)
       NS_WARNING("Unable to respond to default quota pref changes!");
     }
 
+    Preferences::RegisterCallbackAndCall(SnapshotPrefillPrefChangedCallback,
+                                         kSnapshotPrefillPref);
+
     sObserversRegistered = true;
   }
 
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index 1bf0ae1e2052..6ce932607964 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -251,12 +251,58 @@ LSDatabase::Clear(LSObject* aObject,
   return NS_OK;
 }
 
+nsresult
+LSDatabase::BeginExplicitSnapshot(LSObject* aObject)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  if (mSnapshot) {
+    return NS_ERROR_ALREADY_INITIALIZED;
+  }
+
+  nsresult rv = EnsureSnapshot(aObject,
+                               /* aRequestedBySetItem */ false,
+                               /* aExplicit */ true);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSDatabase::EndExplicitSnapshot(LSObject* aObject)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(aObject);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(!mAllowedToClose);
+
+  if (!mSnapshot) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  MOZ_ASSERT(mSnapshot->Explicit());
+
+  nsresult rv = mSnapshot->Finish();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 nsresult
 LSDatabase::EnsureSnapshot(LSObject* aObject,
-                           bool aRequestedBySetItem)
+                           bool aRequestedBySetItem,
+                           bool aExplicit)
 {
   MOZ_ASSERT(aObject);
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT_IF(mSnapshot, !aExplicit);
   MOZ_ASSERT(!mAllowedToClose);
 
   if (mSnapshot) {
@@ -282,7 +328,7 @@ LSDatabase::EnsureSnapshot(LSObject* aObject,
   snapshot->SetActor(actor);
 
   // This add refs snapshot.
-  nsresult rv = snapshot->Init(initInfo);
+  nsresult rv = snapshot->Init(initInfo, aExplicit);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index ed875322a4cc..2854cb70814c 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -97,12 +97,19 @@ public:
   Clear(LSObject* aObject,
         LSNotifyInfo& aNotifyInfo);
 
+  nsresult
+  BeginExplicitSnapshot(LSObject* aObject);
+
+  nsresult
+  EndExplicitSnapshot(LSObject* aObject);
+
 private:
   ~LSDatabase();
 
   nsresult
   EnsureSnapshot(LSObject* aObject,
-                 bool aRequestedBySetItem);
+                 bool aRequestedBySetItem,
+                 bool aExplicit = false);
 
   void
   AllowToClose();
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index 40f95ea10979..b56e0ace20ca 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -128,6 +128,7 @@ LSObject::LSObject(nsPIDOMWindowInner* aWindow,
                    nsIPrincipal* aPrincipal)
   : Storage(aWindow, aPrincipal)
   , mPrivateBrowsingId(0)
+  , mInExplicitSnapshot(false)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(NextGenLocalStorageEnabled());
@@ -563,6 +564,60 @@ LSObject::Close(nsIPrincipal& aSubjectPrincipal,
   DropDatabase();
 }
 
+void
+LSObject::BeginExplicitSnapshot(nsIPrincipal& aSubjectPrincipal,
+                                ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
+  if (mInExplicitSnapshot) {
+    aError.Throw(NS_ERROR_ALREADY_INITIALIZED);
+    return;
+  }
+
+  nsresult rv = EnsureDatabase();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  rv = mDatabase->BeginExplicitSnapshot(this);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+
+  mInExplicitSnapshot = true;
+}
+
+void
+LSObject::EndExplicitSnapshot(nsIPrincipal& aSubjectPrincipal,
+                              ErrorResult& aError)
+{
+  AssertIsOnOwningThread();
+
+  if (!CanUseStorage(aSubjectPrincipal)) {
+    aError.Throw(NS_ERROR_DOM_SECURITY_ERR);
+    return;
+  }
+
+  if (!mInExplicitSnapshot) {
+    aError.Throw(NS_ERROR_NOT_INITIALIZED);
+    return;
+  }
+
+  nsresult rv = EndExplicitSnapshotInternal();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    aError.Throw(rv);
+    return;
+  }
+}
+
 NS_IMPL_ADDREF_INHERITED(LSObject, Storage)
 NS_IMPL_RELEASE_INHERITED(LSObject, Storage)
 
@@ -692,6 +747,11 @@ LSObject::DropDatabase()
 {
   AssertIsOnOwningThread();
 
+  if (mInExplicitSnapshot) {
+    nsresult rv = EndExplicitSnapshotInternal();
+    Unused << NS_WARN_IF(NS_FAILED(rv));
+  }
+
   mDatabase = nullptr;
 }
 
@@ -780,6 +840,38 @@ LSObject::OnChange(const nsAString& aKey,
                /* aImmediateDispatch */ false);
 }
 
+nsresult
+LSObject::EndExplicitSnapshotInternal()
+{
+  AssertIsOnOwningThread();
+
+  // Can be only called if the mInExplicitSnapshot flag is true.
+  // An explicit snapshot must have been created.
+  MOZ_ASSERT(mInExplicitSnapshot);
+
+  // If an explicit snapshot have been created then mDatabase must be not null.
+  // DropDatabase could be called in the meatime, but that would set
+  // mInExplicitSnapshot to false. EnsureDatabase could be called in the
+  // meantime too, but that can't set mDatabase to null or to a new value. See
+  // the comment below.
+  MOZ_ASSERT(mDatabase);
+
+  // Existence of a snapshot prevents the database from allowing to close. See
+  // LSDatabase::RequestAllowToClose and LSDatabase::NoteFinishedSnapshot.
+  // If the database is not allowed to close then mDatabase could not have been
+  // nulled out or set to a new value. See EnsureDatabase.
+  MOZ_ASSERT(!mDatabase->IsAllowedToClose());
+
+  nsresult rv = mDatabase->EndExplicitSnapshot(this);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  mInExplicitSnapshot = false;
+
+  return NS_OK;
+}
+
 void
 LSObject::LastRelease()
 {
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index caee2f634bc0..f1b3cee37f38 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -49,6 +49,8 @@ class LSObject final
   nsCString mOrigin;
   nsString mDocumentURI;
 
+  bool mInExplicitSnapshot;
+
 public:
   static nsresult
   CreateForWindow(nsPIDOMWindowInner* aWindow,
@@ -141,6 +143,14 @@ public:
   Close(nsIPrincipal& aSubjectPrincipal,
         ErrorResult& aError) override;
 
+  void
+  BeginExplicitSnapshot(nsIPrincipal& aSubjectPrincipal,
+                        ErrorResult& aError) override;
+
+  void
+  EndExplicitSnapshot(nsIPrincipal& aSubjectPrincipal,
+                      ErrorResult& aError) override;
+
   NS_DECL_ISUPPORTS_INHERITED
   NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(LSObject, Storage)
 
@@ -171,6 +181,9 @@ private:
            const nsAString& aOldValue,
            const nsAString& aNewValue);
 
+  nsresult
+  EndExplicitSnapshotInternal();
+
   // Storage overrides.
   void
   LastRelease() override;
diff --git a/dom/localstorage/LSSnapshot.cpp b/dom/localstorage/LSSnapshot.cpp
index d16e3358f59c..2996cd5ba81c 100644
--- a/dom/localstorage/LSSnapshot.cpp
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -14,8 +14,12 @@ namespace dom {
 LSSnapshot::LSSnapshot(LSDatabase* aDatabase)
   : mDatabase(aDatabase)
   , mActor(nullptr)
+  , mInitLength(0)
+  , mLength(0)
   , mExactUsage(0)
   , mPeakUsage(0)
+  , mLoadState(LoadState::Initial)
+  , mExplicit(false)
 #ifdef DEBUG
   , mInitialized(false)
   , mSentFinish(false)
@@ -47,24 +51,41 @@ LSSnapshot::SetActor(LSSnapshotChild* aActor)
 }
 
 nsresult
-LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo)
+LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo,
+                 bool aExplicit)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mLoadState == LoadState::Initial);
   MOZ_ASSERT(!mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
   const nsTArray<LSItemInfo>& itemInfos = aInitInfo.itemInfos();
   for (uint32_t i = 0; i < itemInfos.Length(); i++) {
     const LSItemInfo& itemInfo = itemInfos[i];
+    mLoadedItems.PutEntry(itemInfo.key());
     mValues.Put(itemInfo.key(), itemInfo.value());
   }
 
+  if (itemInfos.Length() == aInitInfo.totalLength()) {
+    mLoadState = LoadState::AllOrderedItems;
+  } else {
+    mLoadState = LoadState::Partial;
+    mInitLength = aInitInfo.totalLength();
+    mLength = mInitLength;
+  }
+
   mExactUsage = aInitInfo.initialUsage();
   mPeakUsage = aInitInfo.peakUsage();
 
-  nsCOMPtr<nsIRunnable> runnable = this;
-  nsContentUtils::RunInStableState(runnable.forget());
+  mExplicit = aExplicit;
+
+  if (mExplicit) {
+    mSelfRef = this;
+  } else {
+    nsCOMPtr<nsIRunnable> runnable = this;
+    nsContentUtils::RunInStableState(runnable.forget());
+  }
 
 #ifdef DEBUG
   mInitialized = true;
@@ -81,7 +102,11 @@ LSSnapshot::GetLength(uint32_t* aResult)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
-  *aResult = mValues.Count();
+  if (mLoadState == LoadState::Partial) {
+    *aResult = mLength;
+  } else {
+    *aResult = mValues.Count();
+  }
 
   return NS_OK;
 }
@@ -95,6 +120,11 @@ LSSnapshot::GetKey(uint32_t aIndex,
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  nsresult rv = EnsureAllKeys();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   aResult.SetIsVoid(true);
   for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
     if (aIndex == 0) {
@@ -117,8 +147,74 @@ LSSnapshot::GetItem(const nsAString& aKey,
   MOZ_ASSERT(!mSentFinish);
 
   nsString result;
-  if (!mValues.Get(aKey, &result)) {
-    result.SetIsVoid(true);
+
+  switch (mLoadState) {
+    case LoadState::Partial: {
+      if (mValues.Get(aKey, &result)) {
+        MOZ_ASSERT(!result.IsVoid());
+      } else if (mLoadedItems.GetEntry(aKey) || mUnknownItems.GetEntry(aKey)) {
+        result.SetIsVoid(true);
+      } else {
+        if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
+          return NS_ERROR_FAILURE;
+        }
+
+        if (result.IsVoid()) {
+          mUnknownItems.PutEntry(aKey);
+        } else {
+          mLoadedItems.PutEntry(aKey);
+          mValues.Put(aKey, result);
+
+          if (mLoadedItems.Count() == mInitLength) {
+            mLoadedItems.Clear();
+            mUnknownItems.Clear();
+            mLength = 0;
+            mLoadState = LoadState::AllUnorderedItems;
+          }
+        }
+      }
+
+      break;
+    }
+
+    case LoadState::AllOrderedKeys: {
+      if (mValues.Get(aKey, &result)) {
+        if (result.IsVoid()) {
+          if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
+            return NS_ERROR_FAILURE;
+          }
+
+          MOZ_ASSERT(!result.IsVoid());
+
+          mLoadedItems.PutEntry(aKey);
+          mValues.Put(aKey, result);
+
+          if (mLoadedItems.Count() == mInitLength) {
+            mLoadedItems.Clear();
+            MOZ_ASSERT(mLength == 0);
+            mLoadState = LoadState::AllOrderedItems;
+          }
+        }
+      } else {
+        result.SetIsVoid(true);
+      }
+
+      break;
+    }
+
+    case LoadState::AllUnorderedItems:
+    case LoadState::AllOrderedItems: {
+      if (mValues.Get(aKey, &result)) {
+        MOZ_ASSERT(!result.IsVoid());
+      } else {
+        result.SetIsVoid(true);
+      }
+
+      break;
+    }
+
+    default:
+      MOZ_CRASH("Bad state!");
   }
 
   aResult = result;
@@ -133,6 +229,11 @@ LSSnapshot::GetKeys(nsTArray<nsString>& aKeys)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  nsresult rv = EnsureAllKeys();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
     aKeys.AppendElement(iter.Key());
   }
@@ -176,6 +277,10 @@ LSSnapshot::SetItem(const nsAString& aKey,
 
     mValues.Put(aKey, nsString(aValue));
 
+    if (oldValue.IsVoid() && mLoadState == LoadState::Partial) {
+      mLength++;
+    }
+
     LSSetItemInfo setItemInfo;
     setItemInfo.key() = aKey;
     setItemInfo.oldValue() = oldValue;
@@ -219,6 +324,10 @@ LSSnapshot::RemoveItem(const nsAString& aKey,
 
     mValues.Remove(aKey);
 
+    if (mLoadState == LoadState::Partial) {
+      mLength--;
+    }
+
     LSRemoveItemInfo removeItemInfo;
     removeItemInfo.key() = aKey;
     removeItemInfo.oldValue() = oldValue;
@@ -240,8 +349,23 @@ LSSnapshot::Clear(LSNotifyInfo& aNotifyInfo)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  uint32_t length;
+  if (mLoadState == LoadState::Partial) {
+    length = mLength;
+    MOZ_ASSERT(length);
+
+    MOZ_ALWAYS_TRUE(mActor->SendLoaded());
+
+    mLoadedItems.Clear();
+    mUnknownItems.Clear();
+    mLength = 0;
+    mLoadState = LoadState::AllOrderedItems;
+  } else {
+    length = mValues.Count();
+  }
+
   bool changed;
-  if (!mValues.Count()) {
+  if (!length) {
     changed = false;
   } else {
     changed = true;
@@ -261,6 +385,112 @@ LSSnapshot::Clear(LSNotifyInfo& aNotifyInfo)
   return NS_OK;
 }
 
+nsresult
+LSSnapshot::Finish()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  MOZ_ALWAYS_TRUE(mActor->SendFinish(mWriteInfos));
+
+#ifdef DEBUG
+  mSentFinish = true;
+#endif
+
+  if (mExplicit) {
+    if (NS_WARN_IF(!mActor->SendPing())) {
+      return NS_ERROR_FAILURE;
+    }
+  }
+
+  mDatabase->NoteFinishedSnapshot(this);
+
+  if (mExplicit) {
+    mSelfRef = nullptr;
+  } else {
+    MOZ_ASSERT(!mSelfRef);
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::EnsureAllKeys()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+  MOZ_ASSERT(mLoadState != LoadState::Initial);
+
+  if (mLoadState == LoadState::AllOrderedKeys ||
+      mLoadState == LoadState::AllOrderedItems) {
+    return NS_OK;
+  }
+
+  nsTArray<nsString> keys;
+  if (NS_WARN_IF(!mActor->SendLoadKeys(&keys))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsDataHashtable<nsStringHashKey, nsString> newValues;
+
+  for (auto key : keys) {
+    newValues.Put(key, VoidString());
+  }
+
+  for (uint32_t index = 0; index < mWriteInfos.Length(); index++) {
+    const LSWriteInfo& writeInfo = mWriteInfos[index];
+
+    switch (writeInfo.type()) {
+      case LSWriteInfo::TLSSetItemInfo: {
+        newValues.Put(writeInfo.get_LSSetItemInfo().key(), VoidString());
+        break;
+      }
+      case LSWriteInfo::TLSRemoveItemInfo: {
+        newValues.Remove(writeInfo.get_LSRemoveItemInfo().key());
+        break;
+      }
+      case LSWriteInfo::TLSClearInfo: {
+        newValues.Clear();
+        break;
+      }
+
+      default:
+        MOZ_CRASH("Should never get here!");
+    }
+  }
+
+  MOZ_ASSERT_IF(mLoadState == LoadState::AllUnorderedItems,
+                newValues.Count() == mValues.Count());
+
+  for (auto iter = newValues.Iter(); !iter.Done(); iter.Next()) {
+    nsString value;
+    if (mValues.Get(iter.Key(), &value)) {
+      iter.Data() = value;
+    }
+  }
+
+  mValues.SwapElements(newValues);
+
+  if (mLoadState == LoadState::Partial) {
+    mUnknownItems.Clear();
+    mLength = 0;
+    mLoadState = LoadState::AllOrderedKeys;
+  } else {
+    MOZ_ASSERT(mLoadState == LoadState::AllUnorderedItems);
+
+    MOZ_ASSERT(mUnknownItems.Count() == 0);
+    MOZ_ASSERT(mLength == 0);
+    mLoadState = LoadState::AllOrderedItems;
+  }
+
+  return NS_OK;
+}
+
 nsresult
 LSSnapshot::UpdateUsage(int64_t aDelta)
 {
@@ -301,17 +531,9 @@ NS_IMETHODIMP
 LSSnapshot::Run()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mDatabase);
-  MOZ_ASSERT(mActor);
-  MOZ_ASSERT(!mSentFinish);
+  MOZ_ASSERT(!mExplicit);
 
-  MOZ_ALWAYS_TRUE(mActor->SendFinish(mWriteInfos));
-
-#ifdef DEBUG
-  mSentFinish = true;
-#endif
-
-  mDatabase->NoteFinishedSnapshot(this);
+  MOZ_ALWAYS_SUCCEEDS(Finish());
 
   return NS_OK;
 }
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
index 27daa7b25fc0..84d574ec1c46 100644
--- a/dom/localstorage/LSSnapshot.h
+++ b/dom/localstorage/LSSnapshot.h
@@ -15,16 +15,34 @@ class LSSnapshotChild;
 class LSSnapshot final
   : public nsIRunnable
 {
+  enum class LoadState
+  {
+    Initial,
+    Partial,
+    AllOrderedKeys,
+    AllUnorderedItems,
+    AllOrderedItems
+  };
+
+  RefPtr<LSSnapshot> mSelfRef;
+
   RefPtr<LSDatabase> mDatabase;
 
   LSSnapshotChild* mActor;
 
+  nsTHashtable<nsStringHashKey> mLoadedItems;
+  nsTHashtable<nsStringHashKey> mUnknownItems;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   nsTArray<LSWriteInfo> mWriteInfos;
 
+  uint32_t mInitLength;
+  uint32_t mLength;
   int64_t mExactUsage;
   int64_t mPeakUsage;
 
+  LoadState mLoadState;
+  bool mExplicit;
+
 #ifdef DEBUG
   bool mInitialized;
   bool mSentFinish;
@@ -51,8 +69,15 @@ public:
     mActor = nullptr;
   }
 
+  bool
+  Explicit() const
+  {
+    return mExplicit;
+  }
+
   nsresult
-  Init(const LSSnapshotInitInfo& aInitInfo);
+  Init(const LSSnapshotInitInfo& aInitInfo,
+       bool aExplicit);
 
   nsresult
   GetLength(uint32_t* aResult);
@@ -80,9 +105,15 @@ public:
   nsresult
   Clear(LSNotifyInfo& aNotifyInfo);
 
+  nsresult
+  Finish();
+
 private:
   ~LSSnapshot();
 
+  nsresult
+  EnsureAllKeys();
+
   nsresult
   UpdateUsage(int64_t aDelta);
 
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index aa27a4939ab0..0c0bd9c7bf30 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -17,6 +17,7 @@ struct LSItemInfo
 struct LSSnapshotInitInfo
 {
   LSItemInfo[] itemInfos;
+  uint32_t totalLength;
   int64_t initialUsage;
   int64_t peakUsage;
 };
diff --git a/dom/localstorage/PBackgroundLSSnapshot.ipdl b/dom/localstorage/PBackgroundLSSnapshot.ipdl
index 022e453943a2..75fa738563fd 100644
--- a/dom/localstorage/PBackgroundLSSnapshot.ipdl
+++ b/dom/localstorage/PBackgroundLSSnapshot.ipdl
@@ -46,9 +46,22 @@ parent:
 
   async Finish(LSSnapshotFinishInfo finishInfo);
 
+  async Loaded();
+
+  sync LoadItem(nsString key)
+    returns (nsString value);
+
+  sync LoadKeys()
+    returns (nsString[] keys);
+
   sync IncreasePeakUsage(int64_t requestedSize, int64_t minSize)
     returns (int64_t size);
 
+  // A synchronous ping to the parent actor to confirm that the parent actor
+  // has received previous async message. This should only be used by the
+  // snapshotting code to end an explicit snapshot.
+  sync Ping();
+
 child:
   async __delete__();
 };
diff --git a/dom/localstorage/test/unit/test_snapshotting.js b/dom/localstorage/test/unit/test_snapshotting.js
index e9d7bdacd384..4fbd1bfab0a8 100644
--- a/dom/localstorage/test/unit/test_snapshotting.js
+++ b/dom/localstorage/test/unit/test_snapshotting.js
@@ -14,36 +14,229 @@ function* testSteps()
     { key: "key2", value: "value2" },
     { key: "key3", value: "value3" },
     { key: "key4", value: "value4" },
-    { key: "key5", value: "value5" }
+    { key: "key5", value: "value5" },
+    { key: "key6", value: "value6" },
+    { key: "key7", value: "value7" },
+    { key: "key8", value: "value8" },
+    { key: "key9", value: "value9" },
+    { key: "key10", value: "value10" }
   ];
 
-  info("Getting storage");
-
-  let storage = getLocalStorage(getPrincipal(url));
-
-  info("Adding data");
-
-  for (let item of items) {
-    storage.setItem(item.key, item.value);
+  function getPartialPrefill()
+  {
+    let size = 0;
+    for (let i = 0; i < items.length / 2; i++) {
+      let item = items[i];
+      size += item.key.length + item.value.length;
+    }
+    return size;
   }
 
-  info("Saving key order");
+  const prefillValues = [
+    0,                   // no prefill
+    getPartialPrefill(), // partial prefill
+    -1,                  // full prefill
+  ];
 
-  let keys = [];
-  for (let i = 0; i < items.length; i++) {
-    keys.push(storage.key(i));
-  }
+  for (let prefillValue of prefillValues) {
+    info("Setting prefill value");
 
-  // Let the internal snapshot finish by returning to the event loop.
-  continueToNextStep();
-  yield undefined;
+    Services.prefs.setIntPref("dom.storage.snapshot_prefill", prefillValue);
 
-  is(storage.length, items.length, "Correct length");
+    info("Getting storage");
 
-  info("Verifying key order");
+    let storage = getLocalStorage(getPrincipal(url));
 
-  for (let i = 0; i < items.length; i++) {
-    is(storage.key(i), keys[i], "Correct key");
+    // 1st snapshot
+
+    info("Adding data");
+
+    for (let item of items) {
+      storage.setItem(item.key, item.value);
+    }
+
+    info("Saving key order");
+
+    // This forces GetKeys to be called internally.
+    let savedKeys = Object.keys(storage);
+
+    // GetKey should match GetKeys
+    for (let i = 0; i < savedKeys.length; i++) {
+      is(storage.key(i), savedKeys[i], "Correct key");
+    }
+
+    info("Returning to event loop");
+
+    // Returning to event loop forces the internal snapshot to finish.
+    continueToNextStep();
+    yield undefined;
+
+    // 2nd snapshot
+
+    info("Verifying length");
+
+    is(storage.length, items.length, "Correct length");
+
+    info("Verifying key order");
+
+    let keys = Object.keys(storage);
+
+    is(keys.length, savedKeys.length);
+
+    for (let i = 0; i < keys.length; i++) {
+      is(keys[i], savedKeys[i], "Correct key");
+    }
+
+    info("Verifying values");
+
+    for (let item of items) {
+      is(storage.getItem(item.key), item.value, "Correct value");
+    }
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
+
+    // 3rd snapshot
+
+    // Force key2 to load.
+    storage.getItem("key2");
+
+    // Fill out write infos a bit.
+    storage.removeItem("key5");
+    storage.setItem("key5", "value5");
+    storage.removeItem("key5");
+    storage.setItem("key11", "value11");
+    storage.setItem("key5", "value5");
+
+    items.push({ key: "key11", value: "value11" });
+
+    info("Verifying length");
+
+    is(storage.length, items.length, "Correct length");
+
+    // This forces to get all keys from the parent and then apply write infos
+    // on already cached values.
+    savedKeys = Object.keys(storage);
+
+    info("Verifying values");
+
+    for (let item of items) {
+      is(storage.getItem(item.key), item.value, "Correct value");
+    }
+
+    storage.removeItem("key11");
+
+    items.pop();
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
+
+    // 4th snapshot
+
+    // Force loading of all items.
+    info("Verifying length");
+
+    is(storage.length, items.length, "Correct length");
+
+    info("Verifying values");
+
+    for (let item of items) {
+      is(storage.getItem(item.key), item.value, "Correct value");
+    }
+
+    is(storage.getItem("key11"), null, "Correct value");
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
+
+    // 5th snapshot
+
+    // Force loading of all keys.
+    info("Saving key order");
+
+    savedKeys = Object.keys(storage);
+
+    // Force loading of all items.
+    info("Verifying length");
+
+    is(storage.length, items.length, "Correct length");
+
+    info("Verifying values");
+
+    for (let item of items) {
+      is(storage.getItem(item.key), item.value, "Correct value");
+    }
+
+    is(storage.getItem("key11"), null, "Correct value");
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
+
+    // 6th snapshot
+    info("Verifying unknown item");
+
+    is(storage.getItem("key11"), null, "Correct value");
+
+    info("Verifying unknown item again");
+
+    is(storage.getItem("key11"), null, "Correct value");
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
+
+    // 7th snapshot
+
+    // Save actual key order.
+    info("Saving key order");
+
+    savedKeys = Object.keys(storage);
+
+    continueToNextStep();
+    yield undefined;
+
+    // 8th snapshot
+
+    // Force loading of all items, but in reverse order.
+    info("Getting values");
+
+    for (let i = items.length - 1; i >= 0; i--) {
+      let item = items[i];
+      storage.getItem(item.key);
+    }
+
+    info("Verifying key order");
+
+    keys = Object.keys(storage);
+
+    is(keys.length, savedKeys.length);
+
+    for (let i = 0; i < keys.length; i++) {
+      is(keys[i], savedKeys[i], "Correct key");
+    }
+
+    continueToNextStep();
+    yield undefined;
+
+    // 9th snapshot
+
+    info("Clearing");
+
+    storage.clear();
+
+    info("Returning to event loop");
+
+    continueToNextStep();
+    yield undefined;
   }
 
   finishTest();
diff --git a/dom/storage/Storage.h b/dom/storage/Storage.h
index 00e470d5d30a..3fb549cfa239 100644
--- a/dom/storage/Storage.h
+++ b/dom/storage/Storage.h
@@ -118,6 +118,14 @@ public:
   Close(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
   { }
 
+  virtual void
+  BeginExplicitSnapshot(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
+  { }
+
+  virtual void
+  EndExplicitSnapshot(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
+  { }
+
   // aStorage can be null if this method is called by LocalStorageCacheChild.
   //
   // aImmediateDispatch is for use by child IPC code (LocalStorageCacheChild)
diff --git a/dom/webidl/Storage.webidl b/dom/webidl/Storage.webidl
index dd5a31ba287f..0496341b86d2 100644
--- a/dom/webidl/Storage.webidl
+++ b/dom/webidl/Storage.webidl
@@ -41,4 +41,10 @@ partial interface Storage {
 
   [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
   void close();
+
+  [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
+  void beginExplicitSnapshot();
+
+  [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
+  void endExplicitSnapshot();
 };
diff --git a/ipc/ipdl/sync-messages.ini b/ipc/ipdl/sync-messages.ini
index 4589561d3fdc..fdb1a188bfc2 100644
--- a/ipc/ipdl/sync-messages.ini
+++ b/ipc/ipdl/sync-messages.ini
@@ -924,8 +924,14 @@ description =
 description =
 [PBackgroundLSDatabase::PBackgroundLSSnapshot]
 description =
+[PBackgroundLSSnapshot::LoadItem]
+description =
+[PBackgroundLSSnapshot::LoadKeys]
+description =
 [PBackgroundLSSnapshot::IncreasePeakUsage]
 description =
+[PBackgroundLSSnapshot::Ping]
+description = See corresponding comment in PBackgroundLSSnapshot.ipdl
 [PRemoteSpellcheckEngine::Check]
 description =
 [PRemoteSpellcheckEngine::CheckAndSuggest]
diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index e37b8722e9a8..822ccef2c014 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1292,6 +1292,7 @@ pref("dom.storage.next_gen", true);
 pref("dom.storage.next_gen", false);
 #endif
 pref("dom.storage.default_quota",      5120);
+pref("dom.storage.snapshot_prefill", 4096);
 pref("dom.storage.testing", false);
 
 pref("dom.send_after_paint_to_content", false);

From 794118139550934bcad1b72962ce785486e9ac90 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:48:57 +0100
Subject: [PATCH 46/78] Bug 1286798 - Part 32: Add a test for snapshotting
 verification in multi-e10s setup; r=asuth

---
 dom/tests/browser/browser.ini                 |   4 +
 .../browser/browser_localStorage_e10s.js      |  86 +---
 .../browser_localStorage_snapshotting_e10s.js | 371 ++++++++++++++++++
 dom/tests/browser/helper_localStorage_e10s.js |  72 ++++
 .../page_localstorage_snapshotting_e10s.html  |  55 +++
 5 files changed, 514 insertions(+), 74 deletions(-)
 create mode 100644 dom/tests/browser/browser_localStorage_snapshotting_e10s.js
 create mode 100644 dom/tests/browser/helper_localStorage_e10s.js
 create mode 100644 dom/tests/browser/page_localstorage_snapshotting_e10s.html

diff --git a/dom/tests/browser/browser.ini b/dom/tests/browser/browser.ini
index 26ed1132a8a0..0384564616ca 100644
--- a/dom/tests/browser/browser.ini
+++ b/dom/tests/browser/browser.ini
@@ -4,6 +4,7 @@ support-files =
   browser_frame_elements.html
   page_privatestorageevent.html
   page_localstorage_e10s.html
+  page_localstorage_snapshotting_e10s.html
   position.html
   test-console-api.html
   test_bug1004814.html
@@ -16,6 +17,7 @@ support-files =
   test_largeAllocation2.html^headers^
   test_largeAllocationFormSubmit.sjs
   helper_largeAllocation.js
+  helper_localStorage_e10s.js
   !/dom/tests/mochitest/geolocation/network_geolocation.sjs
 
 [browser_allocateGigabyte.js]
@@ -56,6 +58,8 @@ skip-if = !e10s || (os == "win" && processor == "x86") || (verify && debug && (o
 [browser_localStorage_e10s.js]
 skip-if = !e10s || verify # This is a test of e10s functionality.
 [browser_localStorage_privatestorageevent.js]
+[browser_localStorage_snapshotting_e10s.js]
+skip-if = !e10s # This is a test of e10s functionality.
 [browser_persist_cookies.js]
 support-files =
   set-samesite-cookies-and-redirect.sjs
diff --git a/dom/tests/browser/browser_localStorage_e10s.js b/dom/tests/browser/browser_localStorage_e10s.js
index 49e564f90267..44b6f1a80fd2 100644
--- a/dom/tests/browser/browser_localStorage_e10s.js
+++ b/dom/tests/browser/browser_localStorage_e10s.js
@@ -2,74 +2,9 @@ const HELPER_PAGE_URL =
   "http://example.com/browser/dom/tests/browser/page_localstorage_e10s.html";
 const HELPER_PAGE_ORIGIN = "http://example.com/";
 
-// Simple tab wrapper abstracting our messaging mechanism;
-class KnownTab {
-  constructor(name, tab) {
-    this.name = name;
-    this.tab = tab;
-  }
-
-  cleanup() {
-    this.tab = null;
-  }
-}
-
-// Simple data structure class to help us track opened tabs and their pids.
-class KnownTabs {
-  constructor() {
-    this.byPid = new Map();
-    this.byName = new Map();
-  }
-
-  cleanup() {
-    this.byPid = null;
-    this.byName = null;
-  }
-}
-
-/**
- * Open our helper page in a tab in its own content process, asserting that it
- * really is in its own process.  We initially load and wait for about:blank to
- * load, and only then loadURI to our actual page.  This is to ensure that
- * LocalStorageManager has had an opportunity to be created and populate
- * mOriginsHavingData.
- *
- * (nsGlobalWindow will reliably create LocalStorageManager as a side-effect of
- * the unconditional call to nsGlobalWindow::PreloadLocalStorage.  This will
- * reliably create the StorageDBChild instance, and its corresponding
- * StorageDBParent will send the set of origins when it is constructed.)
- */
-async function openTestTabInOwnProcess(name, knownTabs) {
-  let realUrl = HELPER_PAGE_URL + '?' + encodeURIComponent(name);
-  // Load and wait for about:blank.
-  let tab = await BrowserTestUtils.openNewForegroundTab({
-    gBrowser, opening: 'about:blank', forceNewProcess: true
-  });
-  let pid = tab.linkedBrowser.frameLoader.tabParent.osPid;
-  ok(!knownTabs.byName.has(name), "tab needs its own name: " + name);
-  ok(!knownTabs.byPid.has(pid), "tab needs to be in its own process: " + pid);
-
-  let knownTab = new KnownTab(name, tab);
-  knownTabs.byPid.set(pid, knownTab);
-  knownTabs.byName.set(name, knownTab);
-
-  // Now trigger the actual load of our page.
-  BrowserTestUtils.loadURI(tab.linkedBrowser, realUrl);
-  await BrowserTestUtils.browserLoaded(tab.linkedBrowser);
-  is(tab.linkedBrowser.frameLoader.tabParent.osPid, pid, "still same pid");
-  return knownTab;
-}
-
-/**
- * Close all the tabs we opened.
- */
-async function cleanupTabs(knownTabs) {
-  for (let knownTab of knownTabs.byName.values()) {
-    BrowserTestUtils.removeTab(knownTab.tab);
-    knownTab.cleanup();
-  }
-  knownTabs.cleanup();
-}
+let testDir = gTestPath.substr(0, gTestPath.lastIndexOf("/"));
+Services.scriptloader.loadSubScript(testDir + "/helper_localStorage_e10s.js",
+                                    this);
 
 /**
  * Wait for a LocalStorage flush to occur.  This notification can occur as a
@@ -340,10 +275,13 @@ add_task(async function() {
 
   // - Open tabs.  Don't configure any of them yet.
   const knownTabs = new KnownTabs();
-  const writerTab = await openTestTabInOwnProcess("writer", knownTabs);
-  const listenerTab = await openTestTabInOwnProcess("listener", knownTabs);
-  const readerTab = await openTestTabInOwnProcess("reader", knownTabs);
-  const lateWriteThenListenTab = await openTestTabInOwnProcess(
+  const writerTab = await openTestTabInOwnProcess(HELPER_PAGE_URL, "writer",
+    knownTabs);
+  const listenerTab = await openTestTabInOwnProcess(HELPER_PAGE_URL, "listener",
+    knownTabs);
+  const readerTab = await openTestTabInOwnProcess(HELPER_PAGE_URL, "reader",
+    knownTabs);
+  const lateWriteThenListenTab = await openTestTabInOwnProcess(HELPER_PAGE_URL,
     "lateWriteThenListen", knownTabs);
 
   // Sanity check that preloading did not occur in the tabs.
@@ -484,8 +422,8 @@ add_task(async function() {
 
   // - Open a fresh tab and make sure it sees the precache/preload
   info("late open preload check");
-  const lateOpenSeesPreload =
-    await openTestTabInOwnProcess("lateOpenSeesPreload", knownTabs);
+  const lateOpenSeesPreload = await openTestTabInOwnProcess(HELPER_PAGE_URL,
+    "lateOpenSeesPreload", knownTabs);
   await verifyTabPreload(lateOpenSeesPreload, true);
 
   // - Clean up.
diff --git a/dom/tests/browser/browser_localStorage_snapshotting_e10s.js b/dom/tests/browser/browser_localStorage_snapshotting_e10s.js
new file mode 100644
index 000000000000..c1a6ac28ffa4
--- /dev/null
+++ b/dom/tests/browser/browser_localStorage_snapshotting_e10s.js
@@ -0,0 +1,371 @@
+const HELPER_PAGE_URL =
+  "http://example.com/browser/dom/tests/browser/page_localstorage_snapshotting_e10s.html";
+const HELPER_PAGE_ORIGIN = "http://example.com/";
+
+let testDir = gTestPath.substr(0, gTestPath.lastIndexOf("/"));
+Services.scriptloader.loadSubScript(testDir + "/helper_localStorage_e10s.js",
+                                    this);
+
+function clearOrigin() {
+  let principal =
+    Services.scriptSecurityManager.createCodebasePrincipalFromOrigin(
+      HELPER_PAGE_ORIGIN);
+  let request =
+    Services.qms.clearStoragesForPrincipal(principal, "default", "ls");
+  let promise = new Promise(resolve => {
+    request.callback = () => {
+      resolve();
+    };
+  });
+  return promise;
+}
+
+async function applyMutations(knownTab, mutations) {
+  await ContentTask.spawn(
+    knownTab.tab.linkedBrowser,
+    mutations,
+    function(mutations) {
+      return content.wrappedJSObject.applyMutations(Cu.cloneInto(mutations,
+                                                                 content));
+    });
+}
+
+async function verifyState(knownTab, expectedState) {
+  let actualState = await ContentTask.spawn(
+    knownTab.tab.linkedBrowser,
+    {},
+    function() {
+      return content.wrappedJSObject.getState();
+    });
+
+  for (let [expectedKey, expectedValue] of Object.entries(expectedState)) {
+    ok(actualState.hasOwnProperty(expectedKey), "key present: " + expectedKey);
+    is(actualState[expectedKey], expectedValue, "value correct");
+  }
+  for (let actualKey of Object.keys(actualState)) {
+    if (!expectedState.hasOwnProperty(actualKey)) {
+      ok(false, "actual state has key it shouldn't have: " + actualKey);
+    }
+  }
+}
+
+async function getKeys(knownTab) {
+  let keys = await ContentTask.spawn(
+    knownTab.tab.linkedBrowser,
+    null,
+    function() {
+      return content.wrappedJSObject.getKeys();
+    });
+  return keys;
+}
+
+async function beginExplicitSnapshot(knownTab) {
+  await ContentTask.spawn(
+    knownTab.tab.linkedBrowser,
+    null,
+    function() {
+      return content.wrappedJSObject.beginExplicitSnapshot();
+    });
+}
+
+async function endExplicitSnapshot(knownTab) {
+  await ContentTask.spawn(
+    knownTab.tab.linkedBrowser,
+    null,
+    function() {
+      return content.wrappedJSObject.endExplicitSnapshot();
+    });
+}
+
+// We spin up a ton of child processes.
+requestLongerTimeout(4);
+
+/**
+ * Verify snapshotting of our localStorage implementation in multi-e10s setup.
+ */
+add_task(async function() {
+  if (!Services.lsm.nextGenLocalStorageEnabled) {
+    ok(true, "Test ignored when the next gen local storage is not enabled.");
+    return;
+  }
+
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      // Enable LocalStorage's testing API so we can explicitly create
+      // snapshots when needed.
+      ["dom.storage.testing", true],
+    ]
+  });
+
+  // Ensure that there is no localstorage data by forcing the origin to be
+  // cleared prior to the start of our test..
+  await clearOrigin();
+
+  // - Open tabs.  Don't configure any of them yet.
+  const knownTabs = new KnownTabs();
+  const writerTab1 = await openTestTabInOwnProcess(HELPER_PAGE_URL, "writer1",
+    knownTabs);
+  const writerTab2 = await openTestTabInOwnProcess(HELPER_PAGE_URL, "writer2",
+    knownTabs);
+  const readerTab1 = await openTestTabInOwnProcess(HELPER_PAGE_URL, "reader1",
+    knownTabs);
+  const readerTab2 = await openTestTabInOwnProcess(HELPER_PAGE_URL, "reader2",
+    knownTabs);
+
+  const initialMutations = [
+    [null, null],
+    ["key1", "initial1"],
+    ["key2", "initial2"],
+    ["key3", "initial3"],
+    ["key5", "initial5"],
+    ["key6", "initial6"],
+    ["key7", "initial7"],
+    ["key8", "initial8"]
+  ];
+
+  const initialState = {
+    key1: "initial1",
+    key2: "initial2",
+    key3: "initial3",
+    key5: "initial5",
+    key6: "initial6",
+    key7: "initial7",
+    key8: "initial8"
+  };
+
+  function getPartialPrefill()
+  {
+    let size = 0;
+    let entries = Object.entries(initialState);
+    for (let i = 0; i < entries.length / 2; i++) {
+      let entry = entries[i];
+      size += entry[0].length + entry[1].length;
+    }
+    return size;
+  }
+
+  const prefillValues = [
+    0,                   // no prefill
+    getPartialPrefill(), // partial prefill
+    -1                   // full prefill
+  ];
+
+  for (let prefillValue of prefillValues) {
+    info("Setting prefill value");
+
+    await SpecialPowers.pushPrefEnv({
+      set: [
+        ["dom.storage.snapshot_prefill", prefillValue]
+      ]
+    });
+
+    info("Stage 1");
+
+    const setRemoveMutations1 = [
+      ["key0", "setRemove10"],
+      ["key1", "setRemove11"],
+      ["key2", null],
+      ["key3", "setRemove13"],
+      ["key4", "setRemove14"],
+      ["key5", "setRemove15"],
+      ["key6", "setRemove16"],
+      ["key7", "setRemove17"],
+      ["key8", null],
+      ["key9", "setRemove19"]
+    ];
+
+    const setRemoveState1 = {
+      key0: "setRemove10",
+      key1: "setRemove11",
+      key3: "setRemove13",
+      key4: "setRemove14",
+      key5: "setRemove15",
+      key6: "setRemove16",
+      key7: "setRemove17",
+      key9: "setRemove19"
+    };
+
+    const setRemoveMutations2 = [
+      ["key0", "setRemove20"],
+      ["key1", null],
+      ["key2", "setRemove22"],
+      ["key3", "setRemove23"],
+      ["key4", "setRemove24"],
+      ["key5", "setRemove25"],
+      ["key6", "setRemove26"],
+      ["key7", null],
+      ["key8", "setRemove28"],
+      ["key9", "setRemove29"]
+    ];
+
+    const setRemoveState2 = {
+      key0: "setRemove20",
+      key2: "setRemove22",
+      key3: "setRemove23",
+      key4: "setRemove24",
+      key5: "setRemove25",
+      key6: "setRemove26",
+      key8: "setRemove28",
+      key9: "setRemove29"
+    };
+
+    // Apply initial mutations using an explicit snapshot. The explicit
+    // snapshot here ensures that the parent process have received the changes.
+    await beginExplicitSnapshot(writerTab1);
+    await applyMutations(writerTab1, initialMutations);
+    await endExplicitSnapshot(writerTab1);
+
+    // Begin explicit snapshots in all tabs except readerTab2. All these tabs
+    // should see the initial state regardless what other tabs are doing.
+    await beginExplicitSnapshot(writerTab1);
+    await beginExplicitSnapshot(writerTab2);
+    await beginExplicitSnapshot(readerTab1);
+
+    // Apply first array of set/remove mutations in writerTab1 and end the
+    // explicit snapshot. This will trigger saving of values in other active
+    // snapshots.
+    await applyMutations(writerTab1, setRemoveMutations1);
+    await endExplicitSnapshot(writerTab1);
+
+    // Begin an explicit snapshot in readerTab2. writerTab1 already ended its
+    // explicit snapshot, so readerTab2 should see mutations done by
+    // writerTab1.
+    await beginExplicitSnapshot(readerTab2);
+
+    // Apply second array of set/remove mutations in writerTab2 and end the
+    // explicit snapshot. This will trigger saving of values in other active
+    // snapshots, but only if they haven't been saved already.
+    await applyMutations(writerTab2, setRemoveMutations2);
+    await endExplicitSnapshot(writerTab2);
+
+    // Verify state in readerTab1, it should match the initial state.
+    await verifyState(readerTab1, initialState);
+    await endExplicitSnapshot(readerTab1);
+
+    // Verify state in readerTab2, it should match the state after the first
+    // array of set/remove mutatations have been applied and "commited".
+    await verifyState(readerTab2, setRemoveState1);
+    await endExplicitSnapshot(readerTab2);
+
+    // Verify final state, it should match the state after the second array of
+    // set/remove mutation have been applied and "commited". An explicit
+    // snapshot is used.
+    await beginExplicitSnapshot(readerTab1);
+    await verifyState(readerTab1, setRemoveState2);
+    await endExplicitSnapshot(readerTab1);
+
+    info("Stage 2");
+
+    const setRemoveClearMutations1 = [
+      ["key0", "setRemoveClear10"],
+      ["key1", null],
+      [null, null]
+    ];
+
+    const setRemoveClearState1 = {
+    };
+
+    const setRemoveClearMutations2 = [
+      ["key8", null],
+      ["key9", "setRemoveClear29"],
+      [null, null]
+    ];
+
+    const setRemoveClearState2 = {
+    };
+
+    // This is very similar to previous stage except that in addition to
+    // set/remove, the clear operation is involved too.
+    await beginExplicitSnapshot(writerTab1);
+    await applyMutations(writerTab1, initialMutations);
+    await endExplicitSnapshot(writerTab1);
+
+    await beginExplicitSnapshot(writerTab1);
+    await beginExplicitSnapshot(writerTab2);
+    await beginExplicitSnapshot(readerTab1);
+
+    await applyMutations(writerTab1, setRemoveClearMutations1);
+    await endExplicitSnapshot(writerTab1);
+
+    await beginExplicitSnapshot(readerTab2);
+
+    await applyMutations(writerTab2, setRemoveClearMutations2);
+    await endExplicitSnapshot(writerTab2);
+
+    await verifyState(readerTab1, initialState);
+    await endExplicitSnapshot(readerTab1);
+
+    await verifyState(readerTab2, setRemoveClearState1);
+    await endExplicitSnapshot(readerTab2);
+
+    await beginExplicitSnapshot(readerTab1);
+    await verifyState(readerTab1, setRemoveClearState2);
+    await endExplicitSnapshot(readerTab1);
+
+    info("Stage 3");
+
+    const changeOrderMutations = [
+      ["key1", null],
+      ["key2", null],
+      ["key3", null],
+      ["key5", null],
+      ["key6", null],
+      ["key7", null],
+      ["key8", null],
+      ["key8", "initial8"],
+      ["key7", "initial7"],
+      ["key6", "initial6"],
+      ["key5", "initial5"],
+      ["key3", "initial3"],
+      ["key2", "initial2"],
+      ["key1", "initial1"]
+    ];
+
+    // Apply initial mutations using an explicit snapshot. The explicit
+    // snapshot here ensures that the parent process have received the changes.
+    await beginExplicitSnapshot(writerTab1);
+    await applyMutations(writerTab1, initialMutations);
+    await endExplicitSnapshot(writerTab1);
+
+    // Begin explicit snapshots in all tabs except writerTab2 which is not used
+    // in this stage. All these tabs should see the initial order regardless
+    // what other tabs are doing.
+    await beginExplicitSnapshot(readerTab1);
+    await beginExplicitSnapshot(writerTab1);
+    await beginExplicitSnapshot(readerTab2);
+
+    // Get all keys in readerTab1 and end the explicit snapshot. No mutations
+    // have been applied yet.
+    let tab1Keys = await getKeys(readerTab1);
+    await endExplicitSnapshot(readerTab1);
+
+    // Apply mutations that change the order of keys and end the explicit
+    // snapshot. The state is unchanged. This will trigger saving of key order
+    // in other active snapshots, but only if the order hasn't been saved
+    // already.
+    await applyMutations(writerTab1, changeOrderMutations);
+    await endExplicitSnapshot(writerTab1);
+
+    // Get all keys in readerTab2 and end the explicit snapshot. Change order
+    // mutations have been applied, but the order should stay unchanged.
+    let tab2Keys = await getKeys(readerTab2);
+    await endExplicitSnapshot(readerTab2);
+
+    // Verify the key order is the same.
+    is(tab2Keys.length, tab1Keys.length, "Correct keys length");
+    for (let i = 0; i < tab2Keys.length; i++) {
+      is(tab2Keys[i], tab1Keys[i], "Correct key");
+    }
+
+    // Verify final state, it should match the initial state since applied
+    // mutations only changed the key order. An explicit snapshot is used.
+    await beginExplicitSnapshot(readerTab1);
+    await verifyState(readerTab1, initialState);
+    await endExplicitSnapshot(readerTab1);
+  }
+
+  // - Clean up.
+  await cleanupTabs(knownTabs);
+
+  clearOrigin();
+});
diff --git a/dom/tests/browser/helper_localStorage_e10s.js b/dom/tests/browser/helper_localStorage_e10s.js
new file mode 100644
index 000000000000..307d0e84c9c9
--- /dev/null
+++ b/dom/tests/browser/helper_localStorage_e10s.js
@@ -0,0 +1,72 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+// Simple tab wrapper abstracting our messaging mechanism;
+class KnownTab {
+  constructor(name, tab) {
+    this.name = name;
+    this.tab = tab;
+  }
+
+  cleanup() {
+    this.tab = null;
+  }
+}
+
+// Simple data structure class to help us track opened tabs and their pids.
+class KnownTabs {
+  constructor() {
+    this.byPid = new Map();
+    this.byName = new Map();
+  }
+
+  cleanup() {
+    this.byPid = null;
+    this.byName = null;
+  }
+}
+
+/**
+ * Open our helper page in a tab in its own content process, asserting that it
+ * really is in its own process.  We initially load and wait for about:blank to
+ * load, and only then loadURI to our actual page.  This is to ensure that
+ * LocalStorageManager has had an opportunity to be created and populate
+ * mOriginsHavingData.
+ *
+ * (nsGlobalWindow will reliably create LocalStorageManager as a side-effect of
+ * the unconditional call to nsGlobalWindow::PreloadLocalStorage.  This will
+ * reliably create the StorageDBChild instance, and its corresponding
+ * StorageDBParent will send the set of origins when it is constructed.)
+ */
+async function openTestTabInOwnProcess(helperPageUrl, name, knownTabs) {
+  let realUrl = helperPageUrl + '?' + encodeURIComponent(name);
+  // Load and wait for about:blank.
+  let tab = await BrowserTestUtils.openNewForegroundTab({
+    gBrowser, opening: 'about:blank', forceNewProcess: true
+  });
+  let pid = tab.linkedBrowser.frameLoader.tabParent.osPid;
+  ok(!knownTabs.byName.has(name), "tab needs its own name: " + name);
+  ok(!knownTabs.byPid.has(pid), "tab needs to be in its own process: " + pid);
+
+  let knownTab = new KnownTab(name, tab);
+  knownTabs.byPid.set(pid, knownTab);
+  knownTabs.byName.set(name, knownTab);
+
+  // Now trigger the actual load of our page.
+  BrowserTestUtils.loadURI(tab.linkedBrowser, realUrl);
+  await BrowserTestUtils.browserLoaded(tab.linkedBrowser);
+  is(tab.linkedBrowser.frameLoader.tabParent.osPid, pid, "still same pid");
+  return knownTab;
+}
+
+/**
+ * Close all the tabs we opened.
+ */
+async function cleanupTabs(knownTabs) {
+  for (let knownTab of knownTabs.byName.values()) {
+    BrowserTestUtils.removeTab(knownTab.tab);
+    knownTab.cleanup();
+  }
+  knownTabs.cleanup();
+}
diff --git a/dom/tests/browser/page_localstorage_snapshotting_e10s.html b/dom/tests/browser/page_localstorage_snapshotting_e10s.html
new file mode 100644
index 000000000000..ae8b685d02ed
--- /dev/null
+++ b/dom/tests/browser/page_localstorage_snapshotting_e10s.html
@@ -0,0 +1,55 @@
+<!doctype html>
+<html>
+<head>
+  <meta charset="utf-8">
+<script>
+/**
+ * Helper page used by browser_localStorage_snapshotting.js.
+ *
+ * We expose methods to be invoked by ContentTask.spawn() calls.
+ *
+ **/
+var pageName = document.location.search.substring(1);
+window.addEventListener(
+  "load",
+  () => { document.getElementById("pageNameH").textContent = pageName; });
+
+function applyMutations(mutations) {
+  mutations.forEach(function([key, value]) {
+    if (key !== null) {
+      if (value === null) {
+        localStorage.removeItem(key);
+      } else {
+        localStorage.setItem(key, value);
+      }
+    } else {
+      localStorage.clear();
+    }
+  });
+}
+
+function getState() {
+  let state = {};
+  let length = localStorage.length;
+  for (let index = 0; index < length; index++) {
+    let key = localStorage.key(index);
+    state[key] = localStorage.getItem(key);
+  }
+  return state;
+}
+
+function getKeys() {
+  return Object.keys(localStorage);
+}
+
+function beginExplicitSnapshot() {
+  localStorage.beginExplicitSnapshot();
+}
+
+function endExplicitSnapshot() {
+  localStorage.endExplicitSnapshot();
+}
+</script>
+</head>
+<body><h2 id="pageNameH"></h2></body>
+</html>

From 06d1c0193235634c9f26349b391dce9686d49f0f Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:01 +0100
Subject: [PATCH 47/78] Bug 1286798 - Part 33: Restrict localStorage from being
 available on some pages; r=asuth

This matches the old implementation. localStorage shouldn't be available on some pages, for example about:home.
---
 dom/localstorage/LSObject.cpp | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index b56e0ace20ca..abd842cdbb31 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -165,8 +165,20 @@ LSObject::CreateForWindow(nsPIDOMWindowInner* aWindow,
     return NS_ERROR_NOT_AVAILABLE;
   }
 
+  // localStorage is not available on some pages on purpose, for example
+  // about:home. Match the old implementation by using GenerateOriginKey
+  // for the check.
+  nsCString dummyOriginAttrSuffix;
+  nsCString dummyOriginKey;
+  nsresult rv = GenerateOriginKey(principal,
+                                  dummyOriginAttrSuffix,
+                                  dummyOriginKey);
+  if (NS_FAILED(rv)) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   nsAutoPtr<PrincipalInfo> principalInfo(new PrincipalInfo());
-  nsresult rv = PrincipalToPrincipalInfo(principal, principalInfo);
+  rv = PrincipalToPrincipalInfo(principal, principalInfo);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -215,8 +227,17 @@ LSObject::CreateForPrincipal(nsPIDOMWindowInner* aWindow,
   MOZ_ASSERT(aPrincipal);
   MOZ_ASSERT(aObject);
 
+  nsCString dummyOriginAttrSuffix;
+  nsCString dummyOriginKey;
+  nsresult rv = GenerateOriginKey(aPrincipal,
+                                  dummyOriginAttrSuffix,
+                                  dummyOriginKey);
+  if (NS_FAILED(rv)) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
   nsAutoPtr<PrincipalInfo> principalInfo(new PrincipalInfo());
-  nsresult rv = PrincipalToPrincipalInfo(aPrincipal, principalInfo);
+  rv = PrincipalToPrincipalInfo(aPrincipal, principalInfo);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }

From 4afa59b9c8fa1a702d50829a0a7c7888da62963b Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:04 +0100
Subject: [PATCH 48/78] Bug 1286798 - Part 34: Queue database operations until
 update batch ends; r=asuth

This avoids dispatching to the connection thread for every database operation and paves a way for database shadowing.
---
 dom/localstorage/ActorsParent.cpp | 511 ++++++++++++++----------------
 1 file changed, 238 insertions(+), 273 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index daafa2a46fa8..35a59eeb62a9 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -113,8 +113,6 @@ static_assert(kSQLiteGrowthIncrement >= 0 &&
 #define DATA_FILE_NAME "data.sqlite"
 #define JOURNAL_FILE_NAME "data.sqlite-journal"
 
-const uint32_t kAutoCommitTimeoutMs = 5000;
-
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
@@ -780,17 +778,23 @@ public:
   class CachedStatement;
 
 private:
-  class BeginOp;
-  class CommitOp;
+  class WriteInfo;
+  class SetItemInfo;
+  class RemoveItemInfo;
+  class ClearInfo;
+  class EndUpdateBatchOp;
   class CloseOp;
 
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
+  nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
   const nsCString mOrigin;
   const nsString mFilePath;
-  bool mInTransaction;
+#ifdef DEBUG
+  bool mInUpdateBatch;
+#endif
 
 public:
   NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Connection)
@@ -803,33 +807,32 @@ public:
 
   // Methods which can only be called on the owning thread.
 
-  bool
-  InTransaction()
-  {
-    AssertIsOnOwningThread();
-    return mInTransaction;
-  }
-
   // This method is used to asynchronously execute a connection datastore
   // operation on the connection thread.
   void
   Dispatch(ConnectionDatastoreOperationBase* aOp);
 
-  // This method is used to asynchronously start a transaction on the connection
-  // thread.
-  void
-  Begin(bool aReadonly);
-
-  // This method is used to asynchronously end a transaction on the connection
-  // thread.
-  void
-  Commit();
-
   // This method is used to asynchronously close the storage connection on the
   // connection thread.
   void
   Close(nsIRunnable* aCallback);
 
+  void
+  SetItem(const nsString& aKey,
+          const nsString& aValue);
+
+  void
+  RemoveItem(const nsString& aKey);
+
+  void
+  Clear();
+
+  void
+  BeginUpdateBatch();
+
+  void
+  EndUpdateBatch();
+
   // Methods which can only be called on the connection thread.
 
   nsresult
@@ -885,31 +888,73 @@ private:
   CachedStatement& operator=(const CachedStatement&) = delete;
 };
 
-class Connection::BeginOp final
-  : public ConnectionDatastoreOperationBase
+class Connection::WriteInfo
 {
-  const bool mReadonly;
+public:
+  virtual nsresult
+  Perform(Connection* aConnection) = 0;
+
+  virtual ~WriteInfo() = default;
+};
+
+class Connection::SetItemInfo final
+  : public WriteInfo
+{
+  nsString mKey;
+  nsString mValue;
 
 public:
-  BeginOp(Connection* aConnection,
-          bool aReadonly)
-    : ConnectionDatastoreOperationBase(aConnection)
-    , mReadonly(aReadonly)
+  SetItemInfo(const nsAString& aKey,
+              const nsAString& aValue)
+    : mKey(aKey)
+    , mValue(aValue)
   { }
 
 private:
   nsresult
-  DoDatastoreWork() override;
+  Perform(Connection* aConnection) override;
 };
 
-class Connection::CommitOp final
-  : public ConnectionDatastoreOperationBase
+class Connection::RemoveItemInfo final
+  : public WriteInfo
+{
+  nsString mKey;
+
+public:
+  explicit RemoveItemInfo(const nsAString& aKey)
+    : mKey(aKey)
+  { }
+
+private:
+  nsresult
+  Perform(Connection* aConnection) override;
+};
+
+class Connection::ClearInfo final
+  : public WriteInfo
 {
 public:
-  explicit CommitOp(Connection* aConnection)
-    : ConnectionDatastoreOperationBase(aConnection)
+  ClearInfo()
   { }
 
+private:
+  nsresult
+  Perform(Connection* aConnection) override;
+};
+
+class Connection::EndUpdateBatchOp final
+  : public ConnectionDatastoreOperationBase
+{
+  nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
+
+public:
+  explicit EndUpdateBatchOp(Connection* aConnection,
+                            nsTArray<nsAutoPtr<WriteInfo>>& aWriteInfos)
+    : ConnectionDatastoreOperationBase(aConnection)
+  {
+    mWriteInfos.SwapElements(aWriteInfos);
+  }
+
 private:
   nsresult
   DoDatastoreWork() override;
@@ -970,62 +1015,11 @@ private:
   ~ConnectionThread();
 };
 
-class SetItemOp final
-  : public ConnectionDatastoreOperationBase
-{
-  nsString mKey;
-  nsString mValue;
-
-public:
-  SetItemOp(Connection* aConnection,
-            const nsAString& aKey,
-            const nsAString& aValue)
-    : ConnectionDatastoreOperationBase(aConnection)
-    , mKey(aKey)
-    , mValue(aValue)
-  { }
-
-private:
-  nsresult
-  DoDatastoreWork() override;
-};
-
-class RemoveItemOp final
-  : public ConnectionDatastoreOperationBase
-{
-  nsString mKey;
-
-public:
-  RemoveItemOp(Connection* aConnection,
-               const nsAString& aKey)
-    : ConnectionDatastoreOperationBase(aConnection)
-    , mKey(aKey)
-  { }
-
-private:
-  nsresult
-  DoDatastoreWork() override;
-};
-
-class ClearOp final
-  : public ConnectionDatastoreOperationBase
-{
-public:
-  explicit ClearOp(Connection* aConnection)
-    : ConnectionDatastoreOperationBase(aConnection)
-  { }
-
-private:
-  nsresult
-  DoDatastoreWork() override;
-};
-
 class Datastore final
 {
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
   RefPtr<QuotaObject> mQuotaObject;
-  nsCOMPtr<nsITimer> mAutoCommitTimer;
   nsCOMPtr<nsIRunnable> mCompleteCallback;
   nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
   nsTHashtable<nsPtrHashKey<PreparedDatastore>> mPreparedDatastores;
@@ -1198,12 +1192,6 @@ private:
                   const nsString& aKey,
                   const nsString& aOldValue,
                   const nsString& aNewValue);
-
-  void
-  EnsureTransaction();
-
-  static void
-  AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure);
 };
 
 class PreparedDatastore
@@ -2739,7 +2727,9 @@ Connection::Connection(ConnectionThread* aConnectionThread,
   : mConnectionThread(aConnectionThread)
   , mOrigin(aOrigin)
   , mFilePath(aFilePath)
-  , mInTransaction(false)
+#ifdef DEBUG
+  , mInUpdateBatch(false)
+#endif
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!aOrigin.IsEmpty());
@@ -2751,7 +2741,7 @@ Connection::~Connection()
   AssertIsOnOwningThread();
   MOZ_ASSERT(!mStorageConnection);
   MOZ_ASSERT(!mCachedStatements.Count());
-  MOZ_ASSERT(!mInTransaction);
+  MOZ_ASSERT(!mInUpdateBatch);
 }
 
 void
@@ -2764,30 +2754,6 @@ Connection::Dispatch(ConnectionDatastoreOperationBase* aOp)
                                                            NS_DISPATCH_NORMAL));
 }
 
-void
-Connection::Begin(bool aReadonly)
-{
-  AssertIsOnOwningThread();
-
-  RefPtr<BeginOp> op = new BeginOp(this, aReadonly);
-
-  Dispatch(op);
-
-  mInTransaction = true;
-}
-
-void
-Connection::Commit()
-{
-  AssertIsOnOwningThread();
-
-  RefPtr<CommitOp> op = new CommitOp(this);
-
-  Dispatch(op);
-
-  mInTransaction = false;
-}
-
 void
 Connection::Close(nsIRunnable* aCallback)
 {
@@ -2799,6 +2765,65 @@ Connection::Close(nsIRunnable* aCallback)
   Dispatch(op);
 }
 
+void
+Connection::SetItem(const nsString& aKey,
+                    const nsString& aValue)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mInUpdateBatch);
+
+  nsAutoPtr<WriteInfo> writeInfo(new SetItemInfo(aKey, aValue));
+  mWriteInfos.AppendElement(writeInfo.forget());
+}
+
+void
+Connection::RemoveItem(const nsString& aKey)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mInUpdateBatch);
+
+  nsAutoPtr<WriteInfo> writeInfo(new RemoveItemInfo(aKey));
+  mWriteInfos.AppendElement(writeInfo.forget());
+}
+
+void
+Connection::Clear()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mInUpdateBatch);
+
+  nsAutoPtr<WriteInfo> writeInfo(new ClearInfo());
+  mWriteInfos.AppendElement(writeInfo.forget());
+}
+
+void
+Connection::BeginUpdateBatch()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(!mInUpdateBatch);
+
+#ifdef DEBUG
+  mInUpdateBatch = true;
+#endif
+}
+
+void
+Connection::EndUpdateBatch()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mInUpdateBatch);
+
+  if (!mWriteInfos.IsEmpty()) {
+    RefPtr<EndUpdateBatchOp> op = new EndUpdateBatchOp(this, mWriteInfos);
+
+    Dispatch(op);
+  }
+
+#ifdef DEBUG
+  mInUpdateBatch = false;
+#endif
+}
+
 nsresult
 Connection::EnsureStorageConnection()
 {
@@ -2911,19 +2936,26 @@ CachedStatement::Assign(Connection* aConnection,
 
 nsresult
 Connection::
-BeginOp::DoDatastoreWork()
+SetItemInfo::Perform(Connection* aConnection)
 {
   AssertIsOnConnectionThread();
-  MOZ_ASSERT(mConnection);
+  MOZ_ASSERT(aConnection);
 
-  CachedStatement stmt;
-  nsresult rv;
-  if (mReadonly) {
-    rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN;"), &stmt);
-  } else {
-    rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
-                                         &stmt);
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO data (key, value) "
+    "VALUES(:key, :value)"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
   }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -2938,14 +2970,81 @@ BeginOp::DoDatastoreWork()
 
 nsresult
 Connection::
-CommitOp::DoDatastoreWork()
+RemoveItemInfo::Perform(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data "
+      "WHERE key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+Connection::
+ClearInfo::Perform(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+Connection::
+EndUpdateBatchOp::DoDatastoreWork()
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mConnection);
 
   CachedStatement stmt;
   nsresult rv =
-    mConnection->GetCachedStatement(NS_LITERAL_CSTRING("COMMIT;"), &stmt);
+    mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
+                                    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  for (auto writeInfo : mWriteInfos) {
+    writeInfo->Perform(mConnection);
+  }
+
+  rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("COMMIT;"), &stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3043,89 +3142,6 @@ ConnectionThread::Shutdown()
   mThread->Shutdown();
 }
 
-nsresult
-SetItemOp::DoDatastoreWork()
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(mConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "INSERT OR REPLACE INTO data (key, value) "
-    "VALUES(:key, :value)"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
-nsresult
-RemoveItemOp::DoDatastoreWork()
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(mConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM data "
-      "WHERE key = :key;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
-nsresult
-ClearOp::DoDatastoreWork()
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(mConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM data;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
 /*******************************************************************************
  * Datastore
  ******************************************************************************/
@@ -3176,15 +3192,6 @@ Datastore::Close()
     MOZ_ASSERT(mConnection);
     MOZ_ASSERT(mQuotaObject);
 
-    if (mConnection->InTransaction()) {
-      MOZ_ASSERT(mAutoCommitTimer);
-      MOZ_ALWAYS_SUCCEEDS(mAutoCommitTimer->Cancel());
-
-      mConnection->Commit();
-
-      mAutoCommitTimer = nullptr;
-    }
-
     // We can't release the directory lock and unregister itself from the
     // hashtable until the connection is fully closed.
     nsCOMPtr<nsIRunnable> callback =
@@ -3455,10 +3462,7 @@ Datastore::SetItem(Database* aDatabase,
     mValues.Put(aKey, aValue);
 
     if (IsPersistent()) {
-      EnsureTransaction();
-
-      RefPtr<SetItemOp> op = new SetItemOp(mConnection, aKey, aValue);
-      mConnection->Dispatch(op);
+      mConnection->SetItem(aKey, aValue);
     }
   }
 
@@ -3497,10 +3501,7 @@ Datastore::RemoveItem(Database* aDatabase,
     mValues.Remove(aKey);
 
     if (IsPersistent()) {
-      EnsureTransaction();
-
-      RefPtr<RemoveItemOp> op = new RemoveItemOp(mConnection, aKey);
-      mConnection->Dispatch(op);
+      mConnection->RemoveItem(aKey);
     }
   }
 
@@ -3536,10 +3537,7 @@ Datastore::Clear(Database* aDatabase,
     mKeys.Clear();
 
     if (IsPersistent()) {
-      EnsureTransaction();
-
-      RefPtr<ClearOp> op = new ClearOp(mConnection);
-      mConnection->Dispatch(op);
+      mConnection->Clear();
     }
   }
 
@@ -3577,6 +3575,10 @@ Datastore::BeginUpdateBatch(int64_t aSnapshotInitialUsage)
 
   mUpdateBatchUsage = aSnapshotInitialUsage;
 
+  if (IsPersistent()) {
+    mConnection->BeginUpdateBatch();
+  }
+
 #ifdef DEBUG
   mInUpdateBatch = true;
 #endif
@@ -3630,6 +3632,10 @@ Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
 
   mUpdateBatchUsage = -1;
 
+  if (IsPersistent()) {
+    mConnection->EndUpdateBatch();
+  }
+
 #ifdef DEBUG
   mInUpdateBatch = false;
 #endif
@@ -3782,45 +3788,6 @@ Datastore::NotifyObservers(Database* aDatabase,
   }
 }
 
-void
-Datastore::EnsureTransaction()
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(mConnection);
-
-  if (!mConnection->InTransaction()) {
-    mConnection->Begin(/* aReadonly */ false);
-
-    if (!mAutoCommitTimer) {
-      mAutoCommitTimer = NS_NewTimer();
-      MOZ_ASSERT(mAutoCommitTimer);
-    }
-
-    MOZ_ALWAYS_SUCCEEDS(
-      mAutoCommitTimer->InitWithNamedFuncCallback(
-                                          AutoCommitTimerCallback,
-                                          this,
-                                          kAutoCommitTimeoutMs,
-                                          nsITimer::TYPE_ONE_SHOT,
-                                          "Database::AutoCommitTimerCallback"));
-  }
-}
-
-// static
-void
-Datastore::AutoCommitTimerCallback(nsITimer* aTimer, void* aClosure)
-{
-  MOZ_ASSERT(aClosure);
-
-  auto* self = static_cast<Datastore*>(aClosure);
-  MOZ_ASSERT(self);
-
-  MOZ_ASSERT(self->mConnection);
-  MOZ_ASSERT(self->mConnection->InTransaction());
-
-  self->mConnection->Commit();
-}
-
 /*******************************************************************************
  * PreparedDatastore
  ******************************************************************************/
@@ -5322,8 +5289,6 @@ PrepareDatastoreOp::BeginLoadData()
                                                     mDatabaseFilePath);
   MOZ_ASSERT(mConnection);
 
-  MOZ_ASSERT(!mConnection->InTransaction());
-
   // Must set this before dispatching otherwise we will race with the
   // connection thread.
   mNestedState = NestedState::DatabaseWorkLoadData;

From ec6ab2e194140dad6f57f6bb3b1ea9c4289a941f Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:07 +0100
Subject: [PATCH 49/78] Bug 1286798 - Part 35: Implement database shadowing;
 r=asuth,janv

This adds synchronization to the global database used by previous local storage implementation.

This patch was enhanced by asuth to bind attached database path.
Places has shown that argument binding is necessary.  (Profiles may include usernames in their path which can have cool emoji and such.)
---
 dom/localstorage/ActorsParent.cpp             | 433 +++++++++++++++++-
 .../test/unit/test_databaseShadowing1.js      |  54 +++
 .../test/unit/test_databaseShadowing2.js      |  46 ++
 dom/localstorage/test/unit/xpcshell.ini       |   4 +
 dom/storage/StorageDBThread.cpp               |  60 +--
 dom/storage/StorageUtils.cpp                  |  56 +++
 dom/storage/StorageUtils.h                    |   4 +
 7 files changed, 585 insertions(+), 72 deletions(-)
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing1.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing2.js

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 35a59eeb62a9..73add17db06b 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -123,6 +123,12 @@ const char kSnapshotPrefillPref[] = "dom.storage.snapshot_prefill";
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
 
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
+#define WEB_APPS_STORE_FILE_NAME "webappsstore.sqlite"
+
+// Shadow database Write Ahead Log's maximum size is 512KB
+const uint32_t kShadowMaxWALSize = 512 * 1024;
+
+const uint32_t kShadowJournalSizeLimit = kShadowMaxWALSize * 3;
 
 bool
 IsOnConnectionThread();
@@ -624,6 +630,260 @@ DetachArchiveDatabase(mozIStorageConnection* aConnection)
   return NS_OK;
 }
 
+nsresult
+GetShadowFile(const nsAString& aBasePath,
+              nsIFile** aArchiveFile)
+{
+  MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aArchiveFile);
+
+  nsCOMPtr<nsIFile> archiveFile;
+  nsresult rv = NS_NewLocalFile(aBasePath,
+                                false,
+                                getter_AddRefs(archiveFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = archiveFile->Append(NS_LITERAL_STRING(WEB_APPS_STORE_FILE_NAME));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  archiveFile.forget(aArchiveFile);
+  return NS_OK;
+}
+
+nsresult
+SetShadowJournalMode(mozIStorageConnection* aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+
+  // Try enabling WAL mode. This can fail in various circumstances so we have to
+  // check the results here.
+  NS_NAMED_LITERAL_CSTRING(journalModeQueryStart, "PRAGMA journal_mode = ");
+  NS_NAMED_LITERAL_CSTRING(journalModeWAL, "wal");
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  nsresult rv =
+    aConnection->CreateStatement(journalModeQueryStart + journalModeWAL,
+                                 getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool hasResult;
+  rv = stmt->ExecuteStep(&hasResult);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(hasResult);
+
+  nsCString journalMode;
+  rv = stmt->GetUTF8String(0, journalMode);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (journalMode.Equals(journalModeWAL)) {
+    // WAL mode successfully enabled. Set limits on its size here.
+
+    // Set the threshold for auto-checkpointing the WAL. We don't want giant
+    // logs slowing down us.
+    rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA page_size;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    bool hasResult;
+    rv = stmt->ExecuteStep(&hasResult);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(hasResult);
+
+    int32_t pageSize;
+    rv = stmt->GetInt32(0, &pageSize);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    MOZ_ASSERT(pageSize >= 512 && pageSize <= 65536);
+
+    nsAutoCString pageCount;
+    pageCount.AppendInt(static_cast<int32_t>(kShadowMaxWALSize / pageSize));
+
+    rv = aConnection->ExecuteSimpleSQL(
+      NS_LITERAL_CSTRING("PRAGMA wal_autocheckpoint = ") + pageCount);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    // Set the maximum WAL log size to reduce footprint on mobile (large empty
+    // WAL files will be truncated)
+    nsAutoCString sizeLimit;
+    sizeLimit.AppendInt(kShadowJournalSizeLimit);
+
+    rv = aConnection->ExecuteSimpleSQL(
+      NS_LITERAL_CSTRING("PRAGMA journal_size_limit = ") + sizeLimit);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else {
+    rv = aConnection->ExecuteSimpleSQL(journalModeQueryStart +
+                                       NS_LITERAL_CSTRING("truncate"));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  return NS_OK;
+}
+
+nsresult
+CreateShadowStorageConnection(const nsAString& aBasePath,
+                              mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> shadowFile;
+  nsresult rv = GetShadowFile(aBasePath, getter_AddRefs(shadowFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+  if (rv == NS_ERROR_FILE_CORRUPTED) {
+    rv = shadowFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = SetShadowJournalMode(connection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = StorageDBUpdater::Update(connection);
+  if (NS_FAILED(rv)) {
+    rv = connection->Close();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = shadowFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = SetShadowJournalMode(connection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = StorageDBUpdater::Update(connection);
+  }
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
+nsresult
+AttachShadowDatabase(const nsAString& aBasePath,
+                     mozIStorageConnection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> shadowFile;
+  nsresult rv = GetShadowFile(aBasePath, getter_AddRefs(shadowFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+#ifdef DEBUG
+  bool exists;
+  rv = shadowFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  MOZ_ASSERT(exists);
+#endif
+
+  nsString path;
+  rv = shadowFile->GetPath(path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = aConnection->CreateStatement(
+    NS_LITERAL_CSTRING("ATTACH DATABASE :path AS shadow;"),
+    getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("path"), path);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+DetachShadowDatabase(mozIStorageConnection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv = aConnection->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
+    "DETACH DATABASE shadow"
+  ));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * Non-actor class declarations
  ******************************************************************************/
@@ -787,6 +1047,7 @@ private:
 
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
+  nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
@@ -805,6 +1066,12 @@ public:
     NS_ASSERT_OWNINGTHREAD(Connection);
   }
 
+  ArchivedOriginInfo*
+  GetArchivedOriginInfo() const
+  {
+    return mArchivedOriginInfo;
+  }
+
   // Methods which can only be called on the owning thread.
 
   // This method is used to asynchronously execute a connection datastore
@@ -858,7 +1125,8 @@ private:
   // Only created by ConnectionThread.
   Connection(ConnectionThread* aConnectionThread,
              const nsACString& aOrigin,
-             const nsAString& aFilePath);
+             const nsAString& aFilePath,
+             nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
 
   ~Connection();
 };
@@ -874,6 +1142,8 @@ public:
   CachedStatement();
   ~CachedStatement();
 
+  operator mozIStorageStatement*() const;
+
   mozIStorageStatement*
   operator->() const MOZ_NO_ADDREF_RELEASE_ON_RETURN;
 
@@ -1004,7 +1274,8 @@ public:
 
   already_AddRefed<Connection>
   CreateConnection(const nsACString& aOrigin,
-                   const nsAString& aFilePath);
+                   const nsAString& aFilePath,
+                   nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
 
   void
   Shutdown();
@@ -2145,6 +2416,9 @@ typedef nsTHashtable<nsCStringHashKey> ArchivedOriginHashtable;
 
 StaticAutoPtr<ArchivedOriginHashtable> gArchivedOrigins;
 
+// Can only be touched on the Quota Manager I/O thread.
+bool gInitializedShadowStorage = false;
+
 bool
 IsOnConnectionThread()
 {
@@ -2723,8 +2997,10 @@ ConnectionDatastoreOperationBase::Run()
 
 Connection::Connection(ConnectionThread* aConnectionThread,
                        const nsACString& aOrigin,
-                       const nsAString& aFilePath)
+                       const nsAString& aFilePath,
+                       nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
   : mConnectionThread(aConnectionThread)
+  , mArchivedOriginInfo(std::move(aArchivedOriginInfo))
   , mOrigin(aOrigin)
   , mFilePath(aFilePath)
 #ifdef DEBUG
@@ -2908,6 +3184,14 @@ CachedStatement::~CachedStatement()
   MOZ_COUNT_DTOR(Connection::CachedStatement);
 }
 
+Connection::
+CachedStatement::operator mozIStorageStatement*() const
+{
+  AssertIsOnConnectionThread();
+
+  return mStatement;
+}
+
 mozIStorageStatement*
 Connection::
 CachedStatement::operator->() const
@@ -2965,6 +3249,46 @@ SetItemInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO shadow.webappsstore2 "
+      "(originAttributes, originKey, scope, key, value) "
+      "VALUES (:originAttributes, :originKey, :scope, :key, :value) "),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  ArchivedOriginInfo* archivedOriginInfo = aConnection->GetArchivedOriginInfo();
+
+  rv = archivedOriginInfo->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCString scope = Scheme0Scope(archivedOriginInfo->OriginSuffix(),
+                                 archivedOriginInfo->OriginNoSuffix());
+
+  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
+                                  scope);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
@@ -2994,6 +3318,31 @@ RemoveItemInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey "
+      "AND key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
@@ -3017,6 +3366,25 @@ ClearInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
@@ -3027,10 +3395,22 @@ EndUpdateBatchOp::DoDatastoreWork()
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mConnection);
 
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsCOMPtr<mozIStorageConnection> storageConnection =
+    mConnection->StorageConnection();
+  MOZ_ASSERT(storageConnection);
+
+  nsresult rv = AttachShadowDatabase(quotaManager->GetBasePath(),
+                                     storageConnection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   CachedStatement stmt;
-  nsresult rv =
-    mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
-                                    &stmt);
+  rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("BEGIN IMMEDIATE;"),
+                                       &stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3054,6 +3434,11 @@ EndUpdateBatchOp::DoDatastoreWork()
     return rv;
   }
 
+  rv = DetachShadowDatabase(storageConnection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
   return NS_OK;
 }
 
@@ -3120,14 +3505,17 @@ ConnectionThread::AssertIsOnConnectionThread()
 }
 
 already_AddRefed<Connection>
-ConnectionThread::CreateConnection(const nsACString& aOrigin,
-                                   const nsAString& aFilePath)
+ConnectionThread::CreateConnection(
+                            const nsACString& aOrigin,
+                            const nsAString& aFilePath,
+                            nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!aOrigin.IsEmpty());
   MOZ_ASSERT(!mConnections.GetWeak(aOrigin));
 
-  RefPtr<Connection> connection = new Connection(this, aOrigin, aFilePath);
+  RefPtr<Connection> connection =
+    new Connection(this, aOrigin, aFilePath, std::move(aArchivedOriginInfo));
   mConnections.Put(aOrigin, connection);
 
   return connection.forget();
@@ -5134,10 +5522,25 @@ PrepareDatastoreOp::DatabaseWork()
     gUsages->Put(mOrigin, newUsage);
   }
 
-  // Must close connection before dispatching otherwise we might race with the
-  // connection thread which needs to open the same database.
+  nsCOMPtr<mozIStorageConnection> shadowConnection;
+  if (!gInitializedShadowStorage) {
+    rv = CreateShadowStorageConnection(quotaManager->GetBasePath(),
+                                       getter_AddRefs(shadowConnection));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    gInitializedShadowStorage = true;
+  }
+
+  // Must close connections before dispatching otherwise we might race with the
+  // connection thread which needs to open the same databases.
   MOZ_ALWAYS_SUCCEEDS(connection->Close());
 
+  if (shadowConnection) {
+    MOZ_ALWAYS_SUCCEEDS(shadowConnection->Close());
+  }
+
   // Must set this before dispatching otherwise we will race with the owning
   // thread.
   mNestedState = NestedState::BeginLoadData;
@@ -5285,8 +5688,10 @@ PrepareDatastoreOp::BeginLoadData()
     gConnectionThread = new ConnectionThread();
   }
 
-  mConnection = gConnectionThread->CreateConnection(mOrigin,
-                                                    mDatabaseFilePath);
+  mConnection =
+    gConnectionThread->CreateConnection(mOrigin,
+                                        mDatabaseFilePath,
+                                        std::move(mArchivedOriginInfo));
   MOZ_ASSERT(mConnection);
 
   // Must set this before dispatching otherwise we will race with the
@@ -5954,7 +6359,7 @@ ArchivedOriginInfo::Create(nsIPrincipal* aPrincipal)
 nsresult
 ArchivedOriginInfo::BindToStatement(mozIStorageStatement* aStatement) const
 {
-  AssertIsOnIOThread();
+  MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
   MOZ_ASSERT(aStatement);
 
   nsresult rv =
diff --git a/dom/localstorage/test/unit/test_databaseShadowing1.js b/dom/localstorage/test/unit/test_databaseShadowing1.js
new file mode 100644
index 000000000000..3f053d43b7e1
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing1.js
@@ -0,0 +1,54 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const url = "http://example.com";
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+
+  let principal = getPrincipal(url);
+
+  info("Getting storage");
+
+  let storage = getLocalStorage(principal);
+
+  info("Adding data");
+
+  storage.setItem("key0", "value0");
+  storage.clear();
+  storage.setItem("key1", "value1");
+  storage.removeItem("key1");
+  storage.setItem("key2", "value2");
+
+  info("Closing storage");
+
+  storage.close();
+
+  resetOrigin(principal, continueToNextStepSync);
+  yield undefined;
+
+  info("Verifying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  let shadowDatabase = profileDir.clone();
+  shadowDatabase.append("webappsstore.sqlite");
+
+  let exists = shadowDatabase.exists();
+  ok(exists, "Shadow database does exist");
+
+  info("Copying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  shadowDatabase.copyTo(currentDir, "");
+
+  // The shadow database is now prepared for test_databaseShadowing2.js
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing2.js b/dom/localstorage/test/unit/test_databaseShadowing2.js
new file mode 100644
index 000000000000..d00c1ec724f1
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing2.js
@@ -0,0 +1,46 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+function* testSteps()
+{
+  const url = "http://example.com";
+
+  // The shadow database is prepared in test_databaseShadowing1.js
+
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", false);
+
+  info("Verifying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  let shadowDatabase = currentDir.clone();
+  shadowDatabase.append("webappsstore.sqlite");
+
+  let exists = shadowDatabase.exists();
+  if (!exists) {
+    finishTest();
+    return;
+  }
+
+  info("Copying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  shadowDatabase.copyTo(profileDir, "");
+
+  info("Getting storage");
+
+  let storage = getLocalStorage(getPrincipal(url));
+
+  info("Verifying data");
+
+  ok(storage.getItem("key0") == null, "Correct value");
+  ok(storage.getItem("key1") == null, "Correct value");
+  ok(storage.getItem("key2") == "value2", "Correct value");
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/xpcshell.ini b/dom/localstorage/test/unit/xpcshell.ini
index fb4ca0de3f04..ba2bfd3ac7a1 100644
--- a/dom/localstorage/test/unit/xpcshell.ini
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -9,6 +9,10 @@ support-files =
   migration_profile.zip
 
 [test_archive.js]
+[test_databaseShadowing1.js]
+run-sequentially = test_databaseShadowing2.js depends on a file produced by this test
+[test_databaseShadowing2.js]
+run-sequentially = this test depends on a file produced by test_databaseShadowing1.js
 [test_eviction.js]
 [test_groupLimit.js]
 [test_migration.js]
diff --git a/dom/storage/StorageDBThread.cpp b/dom/storage/StorageDBThread.cpp
index f956bd0ba36e..2e38a7c7116c 100644
--- a/dom/storage/StorageDBThread.cpp
+++ b/dom/storage/StorageDBThread.cpp
@@ -58,63 +58,6 @@ StorageDBThread* sStorageThread = nullptr;
 // False until we shut the storage thread down.
 bool sStorageThreadDown = false;
 
-// This is only a compatibility code for schema version 0.  Returns the 'scope'
-// key in the schema version 0 format for the scope column.
-nsCString
-Scheme0Scope(LocalStorageCacheBridge* aCache)
-{
-  nsCString result;
-
-  nsCString suffix = aCache->OriginSuffix();
-
-  OriginAttributes oa;
-  if (!suffix.IsEmpty()) {
-    DebugOnly<bool> success = oa.PopulateFromSuffix(suffix);
-    MOZ_ASSERT(success);
-  }
-
-  if (oa.mAppId != nsIScriptSecurityManager::NO_APP_ID ||
-      oa.mInIsolatedMozBrowser) {
-    result.AppendInt(oa.mAppId);
-    result.Append(':');
-    result.Append(oa.mInIsolatedMozBrowser ? 't' : 'f');
-    result.Append(':');
-  }
-
-  // If there is more than just appid and/or inbrowser stored in origin
-  // attributes, put it to the schema 0 scope as well.  We must do that
-  // to keep the scope column unique (same resolution as schema 1 has
-  // with originAttributes and originKey columns) so that switch between
-  // schema 1 and 0 always works in both ways.
-  nsAutoCString remaining;
-  oa.mAppId = 0;
-  oa.mInIsolatedMozBrowser = false;
-  oa.CreateSuffix(remaining);
-  if (!remaining.IsEmpty()) {
-    MOZ_ASSERT(!suffix.IsEmpty());
-
-    if (result.IsEmpty()) {
-      // Must contain the old prefix, otherwise we won't search for the whole
-      // origin attributes suffix.
-      result.AppendLiteral("0:f:");
-    }
-
-    // Append the whole origin attributes suffix despite we have already stored
-    // appid and inbrowser.  We are only looking for it when the scope string
-    // starts with "$appid:$inbrowser:" (with whatever valid values).
-    //
-    // The OriginAttributes suffix is a string in a form like:
-    // "^addonId=101&userContextId=5" and it's ensured it always starts with '^'
-    // and never contains ':'.  See OriginAttributes::CreateSuffix.
-    result.Append(suffix);
-    result.Append(':');
-  }
-
-  result.Append(aCache->OriginNoSuffix());
-
-  return result;
-}
-
 } // anon
 
 // XXX Fix me!
@@ -1156,7 +1099,8 @@ StorageDBThread::DBOperation::Perform(StorageDBThread* aThread)
     NS_ENSURE_SUCCESS(rv, rv);
     // Filling the 'scope' column just for downgrade compatibility reasons
     rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
-                                    Scheme0Scope(mCache));
+                                    Scheme0Scope(mCache->OriginSuffix(),
+                                                 mCache->OriginNoSuffix()));
     NS_ENSURE_SUCCESS(rv, rv);
     rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"),
                                 mKey);
diff --git a/dom/storage/StorageUtils.cpp b/dom/storage/StorageUtils.cpp
index 076f49e650b5..a239de4357d1 100644
--- a/dom/storage/StorageUtils.cpp
+++ b/dom/storage/StorageUtils.cpp
@@ -113,6 +113,62 @@ CreateReversedDomain(const nsACString& aAsciiDomain,
   return NS_OK;
 }
 
+// This is only a compatibility code for schema version 0.  Returns the 'scope'
+// key in the schema version 0 format for the scope column.
+nsCString
+Scheme0Scope(const nsACString& aOriginSuffix,
+             const nsACString& aOriginNoSuffix)
+{
+  nsCString result;
+
+  OriginAttributes oa;
+  if (!aOriginSuffix.IsEmpty()) {
+    DebugOnly<bool> success = oa.PopulateFromSuffix(aOriginSuffix);
+    MOZ_ASSERT(success);
+  }
+
+  if (oa.mAppId != nsIScriptSecurityManager::NO_APP_ID ||
+      oa.mInIsolatedMozBrowser) {
+    result.AppendInt(oa.mAppId);
+    result.Append(':');
+    result.Append(oa.mInIsolatedMozBrowser ? 't' : 'f');
+    result.Append(':');
+  }
+
+  // If there is more than just appid and/or inbrowser stored in origin
+  // attributes, put it to the schema 0 scope as well.  We must do that
+  // to keep the scope column unique (same resolution as schema 1 has
+  // with originAttributes and originKey columns) so that switch between
+  // schema 1 and 0 always works in both ways.
+  nsAutoCString remaining;
+  oa.mAppId = 0;
+  oa.mInIsolatedMozBrowser = false;
+  oa.CreateSuffix(remaining);
+  if (!remaining.IsEmpty()) {
+    MOZ_ASSERT(!aOriginSuffix.IsEmpty());
+
+    if (result.IsEmpty()) {
+      // Must contain the old prefix, otherwise we won't search for the whole
+      // origin attributes suffix.
+      result.AppendLiteral("0:f:");
+    }
+
+    // Append the whole origin attributes suffix despite we have already stored
+    // appid and inbrowser.  We are only looking for it when the scope string
+    // starts with "$appid:$inbrowser:" (with whatever valid values).
+    //
+    // The OriginAttributes suffix is a string in a form like:
+    // "^addonId=101&userContextId=5" and it's ensured it always starts with '^'
+    // and never contains ':'.  See OriginAttributes::CreateSuffix.
+    result.Append(aOriginSuffix);
+    result.Append(':');
+  }
+
+  result.Append(aOriginNoSuffix);
+
+  return result;
+}
+
 } // StorageUtils namespace
 } // dom namespace
 } // mozilla namespace
diff --git a/dom/storage/StorageUtils.h b/dom/storage/StorageUtils.h
index a48eed332855..75b85d17cb4f 100644
--- a/dom/storage/StorageUtils.h
+++ b/dom/storage/StorageUtils.h
@@ -28,6 +28,10 @@ nsresult
 CreateReversedDomain(const nsACString& aAsciiDomain,
                      nsACString& aKey);
 
+nsCString
+Scheme0Scope(const nsACString& aOriginSuffix,
+             const nsACString& aOriginNoSuffix);
+
 } // StorageUtils namespace
 } // dom namespace
 } // mozilla namespace

From 5809f14bbdf247dae4c2b7db2b3a0881a14cf3cb Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:10 +0100
Subject: [PATCH 50/78] Bug 1286798 - Part 36: Allow snapshot initialization to
 a specific load state; r=asuth

Before this patch, it was only possible to initialize a snapshot to the Partial state or AllOrderedItems state. Now there's a third state AllOrderedKeys.
This improves performance by eliminating sync calls to parent process when we know nothing about a key in content process (in that case we have to use a sync call to the parent process to see if there's a value for it).
With this patch we always try to send all keys to content when a snapshot is being initialized. For this to work efficiently, we cache the size of all keys.
Having cached size of all keys also allows us to just iterate the mValues hashtable when the size of keys is bigger than snapshot prefill threshold (instead of iterating over the mKeys array and joining with mValues for each particular key).
There's some additional cleanup in snapshot info construction and Datastore::SetItem/RemoveItem/Clear methods.
---
 dom/localstorage/ActorsParent.cpp           | 219 ++++++++++++--------
 dom/localstorage/LSSnapshot.cpp             |  53 +++--
 dom/localstorage/LSSnapshot.h               |  10 +-
 dom/localstorage/PBackgroundLSDatabase.ipdl |   6 +
 dom/localstorage/SerializationHelpers.h     |  25 +++
 dom/localstorage/moz.build                  |   2 +
 6 files changed, 209 insertions(+), 106 deletions(-)
 create mode 100644 dom/localstorage/SerializationHelpers.h

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 73add17db06b..42f5c4a59ced 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1305,6 +1305,7 @@ class Datastore final
   const uint32_t mPrivateBrowsingId;
   int64_t mUsage;
   int64_t mUpdateBatchUsage;
+  int64_t mSizeOfKeys;
   bool mClosed;
 #ifdef DEBUG
   bool mInUpdateBatch;
@@ -1315,6 +1316,7 @@ public:
   Datastore(const nsACString& aOrigin,
             uint32_t aPrivateBrowsingId,
             int64_t aUsage,
+            int64_t aSizeOfKeys,
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
             already_AddRefed<QuotaObject>&& aQuotaObject,
@@ -1339,13 +1341,6 @@ public:
     return mPrivateBrowsingId == 0;
   }
 
-  int64_t
-  Usage() const
-  {
-    AssertIsOnBackgroundThread();
-    return mUsage;
-  }
-
   void
   Close();
 
@@ -1394,12 +1389,14 @@ public:
   void
   NoteInactiveDatabase(Database* aDatabase);
 
-  uint32_t
-  GetLength() const;
-
   void
-  GetItemInfos(nsTHashtable<nsStringHashKey>& aLoadedItems,
-               nsTArray<LSItemInfo>& aItemInfos);
+  GetSnapshotInitInfo(int64_t aRequestedSize,
+                      nsTHashtable<nsStringHashKey>& aLoadedItems,
+                      nsTArray<LSItemInfo>& aItemInfos,
+                      uint32_t& aTotalLength,
+                      int64_t& aInitialUsage,
+                      int64_t& aPeakUsage,
+                      LSSnapshot::LoadState& aLoadState);
 
   void
   GetItem(const nsString& aKey, nsString& aValue) const;
@@ -1698,12 +1695,13 @@ public:
        uint32_t aTotalLength,
        int64_t aInitialUsage,
        int64_t aPeakUsage,
-       bool aFullPrefill)
+       LSSnapshot::LoadState aLoadState)
   {
     AssertIsOnBackgroundThread();
     MOZ_ASSERT(aInitialUsage >= 0);
     MOZ_ASSERT(aPeakUsage >= aInitialUsage);
-    MOZ_ASSERT_IF(aFullPrefill, aLoadedItems.Count() == 0);
+    MOZ_ASSERT_IF(aLoadState == LSSnapshot::LoadState::AllOrderedItems,
+                  aLoadedItems.Count() == 0);
     MOZ_ASSERT(mTotalLength == 0);
     MOZ_ASSERT(mInitialUsage == -1);
     MOZ_ASSERT(mPeakUsage == -1);
@@ -1712,7 +1710,9 @@ public:
     mTotalLength = aTotalLength;
     mInitialUsage = aInitialUsage;
     mPeakUsage = aPeakUsage;
-    if (aFullPrefill) {
+    if (aLoadState == LSSnapshot::LoadState::AllOrderedKeys) {
+      mLoadKeysReceived = true;
+    } else if (aLoadState == LSSnapshot::LoadState::AllOrderedItems) {
       mLoadedReceived = true;
       mLoadedAllItems = true;
       mLoadKeysReceived = true;
@@ -1951,6 +1951,7 @@ class PrepareDatastoreOp
   nsString mDatabaseFilePath;
   uint32_t mPrivateBrowsingId;
   int64_t mUsage;
+  int64_t mSizeOfKeys;
   NestedState mNestedState;
   bool mDatabaseNotAvailable;
   bool mRequestedDirectoryLock;
@@ -3537,6 +3538,7 @@ ConnectionThread::Shutdown()
 Datastore::Datastore(const nsACString& aOrigin,
                      uint32_t aPrivateBrowsingId,
                      int64_t aUsage,
+                     int64_t aSizeOfKeys,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
                      already_AddRefed<QuotaObject>&& aQuotaObject,
@@ -3549,6 +3551,7 @@ Datastore::Datastore(const nsACString& aOrigin,
   , mPrivateBrowsingId(aPrivateBrowsingId)
   , mUsage(aUsage)
   , mUpdateBatchUsage(-1)
+  , mSizeOfKeys(aSizeOfKeys)
   , mClosed(false)
 #ifdef DEBUG
   , mInUpdateBatch(false)
@@ -3753,43 +3756,85 @@ Datastore::NoteInactiveDatabase(Database* aDatabase)
   }
 }
 
-uint32_t
-Datastore::GetLength() const
-{
-  AssertIsOnBackgroundThread();
-  MOZ_ASSERT(!mClosed);
-
-  return mValues.Count();
-}
-
 void
-Datastore::GetItemInfos(nsTHashtable<nsStringHashKey>& aLoadedItems,
-                        nsTArray<LSItemInfo>& aItemInfos)
+Datastore::GetSnapshotInitInfo(int64_t aRequestedSize,
+                               nsTHashtable<nsStringHashKey>& aLoadedItems,
+                               nsTArray<LSItemInfo>& aItemInfos,
+                               uint32_t& aTotalLength,
+                               int64_t& aInitialUsage,
+                               int64_t& aPeakUsage,
+                               LSSnapshot::LoadState& aLoadState)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!mClosed);
 
+#ifdef DEBUG
+  int64_t sizeOfKeys = 0;
+  for (auto key : mKeys) {
+    sizeOfKeys += static_cast<int64_t>(key.Length());
+  }
+  MOZ_ASSERT(mSizeOfKeys == sizeOfKeys);
+#endif
+
   int64_t size = 0;
-  for (auto key : mKeys) {
+  if (mSizeOfKeys <= gSnapshotPrefill) {
     nsString value;
-    DebugOnly<bool> hasValue = mValues.Get(key, &value);
-    MOZ_ASSERT(hasValue);
+    for (auto key : mKeys) {
+      if (!value.IsVoid()) {
+        DebugOnly<bool> hasValue = mValues.Get(key, &value);
+        MOZ_ASSERT(hasValue);
 
-    size += static_cast<int64_t>(key.Length()) +
-            static_cast<int64_t>(value.Length());
+        size += static_cast<int64_t>(key.Length()) +
+                static_cast<int64_t>(value.Length());
 
-    if (size > gSnapshotPrefill) {
-      return;
+        if (size > gSnapshotPrefill) {
+          value.SetIsVoid(true);
+        } else {
+          aLoadedItems.PutEntry(key);
+        }
+      }
+
+      LSItemInfo* itemInfo = aItemInfos.AppendElement();
+      itemInfo->key() = key;
+      itemInfo->value() = value;
     }
 
-    aLoadedItems.PutEntry(key);
+    if (value.IsVoid()) {
+      aLoadState = LSSnapshot::LoadState::AllOrderedKeys;
+    } else {
+      aLoadedItems.Clear();
+      aLoadState = LSSnapshot::LoadState::AllOrderedItems;
+    }
+  } else {
+    for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
+      const nsAString& key = iter.Key();
+      const nsString& value = iter.Data();
 
-    LSItemInfo* itemInfo = aItemInfos.AppendElement();
-    itemInfo->key() = key;
-    itemInfo->value() = value;
+      size += static_cast<int64_t>(key.Length()) +
+              static_cast<int64_t>(value.Length());
+
+      if (size > gSnapshotPrefill) {
+        break;
+      }
+
+      aLoadedItems.PutEntry(key);
+
+      LSItemInfo* itemInfo = aItemInfos.AppendElement();
+      itemInfo->key() = iter.Key();
+      itemInfo->value() = iter.Data();
+    }
+
+    MOZ_ASSERT(aItemInfos.Length() < mKeys.Length());
+    aLoadState = LSSnapshot::LoadState::Partial;
   }
 
-  aLoadedItems.Clear();
+  aTotalLength = mValues.Count();
+
+  aInitialUsage = mUsage;
+  aPeakUsage = aInitialUsage;
+  if (aRequestedSize && UpdateUsage(aRequestedSize)) {
+    aPeakUsage += aRequestedSize;
+  }
 }
 
 void
@@ -3828,27 +3873,24 @@ Datastore::SetItem(Database* aDatabase,
   GetItem(aKey, oldValue);
 
   if (oldValue != aValue || oldValue.IsVoid() != aValue.IsVoid()) {
-    bool affectsOrder;
+    bool isNewItem = oldValue.IsVoid();
 
-    int64_t delta = static_cast<int64_t>(aValue.Length()) -
-                    static_cast<int64_t>(oldValue.Length());
-
-    if (oldValue.IsVoid()) {
-      affectsOrder = true;
-
-      delta += static_cast<int64_t>(aKey.Length());
-
-      mUpdateBatchAppends.AppendElement(aKey);
-    } else {
-      affectsOrder = false;
-    }
-
-    mUpdateBatchUsage += delta;
-
-    NotifySnapshots(aDatabase, aKey, oldValue, affectsOrder);
+    NotifySnapshots(aDatabase, aKey, oldValue, /* affectsOrder */ isNewItem);
 
     mValues.Put(aKey, aValue);
 
+    if (isNewItem) {
+      mUpdateBatchAppends.AppendElement(aKey);
+
+      mUpdateBatchUsage += static_cast<int64_t>(aKey.Length()) +
+                           static_cast<int64_t>(aValue.Length());
+
+      mSizeOfKeys += static_cast<int64_t>(aKey.Length());
+    } else {
+      mUpdateBatchUsage += static_cast<int64_t>(aValue.Length()) -
+                           static_cast<int64_t>(oldValue.Length());
+    }
+
     if (IsPersistent()) {
       mConnection->SetItem(aKey, aValue);
     }
@@ -3872,6 +3914,10 @@ Datastore::RemoveItem(Database* aDatabase,
   GetItem(aKey, oldValue);
 
   if (!oldValue.IsVoid()) {
+    NotifySnapshots(aDatabase, aKey, oldValue, /* aAffectsOrder */ true);
+
+    mValues.Remove(aKey);
+
     auto entry = mUpdateBatchRemovals.LookupForAdd(aKey);
     if (entry) {
       entry.Data()++;
@@ -3879,14 +3925,10 @@ Datastore::RemoveItem(Database* aDatabase,
       entry.OrInsert([]() { return 1; });
     }
 
-    int64_t delta = -(static_cast<int64_t>(aKey.Length()) +
-                      static_cast<int64_t>(oldValue.Length()));
+    mUpdateBatchUsage -= (static_cast<int64_t>(aKey.Length()) +
+                          static_cast<int64_t>(oldValue.Length()));
 
-    mUpdateBatchUsage += delta;
-
-    NotifySnapshots(aDatabase, aKey, oldValue, /* aAffectsOrder */ true);
-
-    mValues.Remove(aKey);
+    mSizeOfKeys -= static_cast<int64_t>(aKey.Length());
 
     if (IsPersistent()) {
       mConnection->RemoveItem(aKey);
@@ -3906,24 +3948,28 @@ Datastore::Clear(Database* aDatabase,
   MOZ_ASSERT(mInUpdateBatch);
 
   if (mValues.Count()) {
-    mUpdateBatchRemovals.Clear();
-    mUpdateBatchAppends.Clear();
-
+    int64_t updateBatchUsage = mUpdateBatchUsage;
     for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
       const nsAString& key = iter.Key();
       const nsAString& value = iter.Data();
 
-      int64_t delta = -(static_cast<int64_t>(key.Length()) +
-                        static_cast<int64_t>(value.Length()));
-
-      mUpdateBatchUsage += delta;
+      updateBatchUsage -= (static_cast<int64_t>(key.Length()) +
+                           static_cast<int64_t>(value.Length()));
 
       NotifySnapshots(aDatabase, key, value, /* aAffectsOrder */ true);
     }
 
     mValues.Clear();
+
+    mUpdateBatchRemovals.Clear();
+    mUpdateBatchAppends.Clear();
+
     mKeys.Clear();
 
+    mUpdateBatchUsage = updateBatchUsage;
+
+    mSizeOfKeys = 0;
+
     if (IsPersistent()) {
       mConnection->Clear();
     }
@@ -3948,7 +3994,10 @@ Datastore::PrivateBrowsingClear()
     MOZ_ASSERT(ok);
 
     mValues.Clear();
+
     mKeys.Clear();
+
+    mSizeOfKeys = 0;
   }
 }
 
@@ -4392,6 +4441,7 @@ Database::RecvPBackgroundLSSnapshotConstructor(
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aRequestedSize >= 0);
+  MOZ_ASSERT(aInitInfo);
   MOZ_ASSERT(!mAllowedToClose);
 
   auto* snapshot = static_cast<Snapshot*>(aActor);
@@ -4400,22 +4450,19 @@ Database::RecvPBackgroundLSSnapshotConstructor(
   //       creation. For example clear() doesn't need to receive items at all.
   nsTHashtable<nsStringHashKey> loadedItems;
   nsTArray<LSItemInfo> itemInfos;
-  mDatastore->GetItemInfos(loadedItems, itemInfos);
+  uint32_t totalLength;
+  int64_t initialUsage;
+  int64_t peakUsage;
+  LSSnapshot::LoadState loadState;
+  mDatastore->GetSnapshotInitInfo(aRequestedSize,
+                                  loadedItems,
+                                  itemInfos,
+                                  totalLength,
+                                  initialUsage,
+                                  peakUsage,
+                                  loadState);
 
-  uint32_t totalLength = mDatastore->GetLength();
-
-  int64_t initialUsage = mDatastore->Usage();
-
-  int64_t peakUsage = initialUsage;
-  if (aRequestedSize && mDatastore->UpdateUsage(aRequestedSize)) {
-    peakUsage += aRequestedSize;
-  }
-
-  snapshot->Init(loadedItems,
-                 totalLength,
-                 initialUsage,
-                 peakUsage,
-                 /* aFullPrefill */ itemInfos.Length() == totalLength);
+  snapshot->Init(loadedItems, totalLength, initialUsage, peakUsage, loadState);
 
   RegisterSnapshot(snapshot);
 
@@ -4423,6 +4470,7 @@ Database::RecvPBackgroundLSSnapshotConstructor(
   aInitInfo->totalLength() = totalLength;
   aInitInfo->initialUsage() = initialUsage;
   aInitInfo->peakUsage() = peakUsage;
+  aInitInfo->loadState() = loadState;
 
   return IPC_OK();
 }
@@ -5010,6 +5058,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mPrivateBrowsingId(0)
   , mUsage(0)
+  , mSizeOfKeys(0)
   , mNestedState(NestedState::BeforeNesting)
   , mDatabaseNotAvailable(false)
   , mRequestedDirectoryLock(false)
@@ -5820,6 +5869,7 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
     mDatastore = new Datastore(mOrigin,
                                mPrivateBrowsingId,
                                mUsage,
+                               mSizeOfKeys,
                                mDirectoryLock.forget(),
                                mConnection.forget(),
                                quotaObject.forget(),
@@ -6035,6 +6085,7 @@ LoadDataOp::DoDatastoreWork()
 
     mPrepareDatastoreOp->mValues.Put(key, value);
     mPrepareDatastoreOp->mKeys.AppendElement(key);
+    mPrepareDatastoreOp->mSizeOfKeys += key.Length();
 #ifdef DEBUG
     mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
 #endif
diff --git a/dom/localstorage/LSSnapshot.cpp b/dom/localstorage/LSSnapshot.cpp
index 2996cd5ba81c..670dc82d342e 100644
--- a/dom/localstorage/LSSnapshot.cpp
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -60,33 +60,44 @@ LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo,
   MOZ_ASSERT(!mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
-  const nsTArray<LSItemInfo>& itemInfos = aInitInfo.itemInfos();
-  for (uint32_t i = 0; i < itemInfos.Length(); i++) {
-    const LSItemInfo& itemInfo = itemInfos[i];
-    mLoadedItems.PutEntry(itemInfo.key());
-    mValues.Put(itemInfo.key(), itemInfo.value());
-  }
-
-  if (itemInfos.Length() == aInitInfo.totalLength()) {
-    mLoadState = LoadState::AllOrderedItems;
-  } else {
-    mLoadState = LoadState::Partial;
-    mInitLength = aInitInfo.totalLength();
-    mLength = mInitLength;
-  }
-
-  mExactUsage = aInitInfo.initialUsage();
-  mPeakUsage = aInitInfo.peakUsage();
-
-  mExplicit = aExplicit;
-
-  if (mExplicit) {
+  if (aExplicit) {
     mSelfRef = this;
   } else {
     nsCOMPtr<nsIRunnable> runnable = this;
     nsContentUtils::RunInStableState(runnable.forget());
   }
 
+  LoadState loadState = aInitInfo.loadState();
+
+  const nsTArray<LSItemInfo>& itemInfos = aInitInfo.itemInfos();
+  for (uint32_t i = 0; i < itemInfos.Length(); i++) {
+    const LSItemInfo& itemInfo = itemInfos[i];
+
+    const nsString& value = itemInfo.value();
+
+    if (loadState != LoadState::AllOrderedItems && !value.IsVoid()) {
+      mLoadedItems.PutEntry(itemInfo.key());
+    }
+
+    mValues.Put(itemInfo.key(), value);
+  }
+
+  if (loadState == LoadState::Partial) {
+    mInitLength = aInitInfo.totalLength();
+    mLength = mInitLength;
+  } else if (loadState == LoadState::AllOrderedKeys) {
+    mInitLength = aInitInfo.totalLength();
+  } else {
+    MOZ_ASSERT(loadState == LoadState::AllOrderedItems);
+  }
+
+  mExactUsage = aInitInfo.initialUsage();
+  mPeakUsage = aInitInfo.peakUsage();
+
+  mLoadState = aInitInfo.loadState();
+
+  mExplicit = aExplicit;
+
 #ifdef DEBUG
   mInitialized = true;
 #endif
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
index 84d574ec1c46..5eda4ac3b756 100644
--- a/dom/localstorage/LSSnapshot.h
+++ b/dom/localstorage/LSSnapshot.h
@@ -10,20 +10,27 @@
 namespace mozilla {
 namespace dom {
 
+class LSDatabase;
+class LSNotifyInfo;
 class LSSnapshotChild;
+class LSSnapshotInitInfo;
+class LSWriteInfo;
 
 class LSSnapshot final
   : public nsIRunnable
 {
+public:
   enum class LoadState
   {
     Initial,
     Partial,
     AllOrderedKeys,
     AllUnorderedItems,
-    AllOrderedItems
+    AllOrderedItems,
+    EndGuard
   };
 
+private:
   RefPtr<LSSnapshot> mSelfRef;
 
   RefPtr<LSDatabase> mDatabase;
@@ -41,6 +48,7 @@ class LSSnapshot final
   int64_t mPeakUsage;
 
   LoadState mLoadState;
+
   bool mExplicit;
 
 #ifdef DEBUG
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 0c0bd9c7bf30..e802597e41d4 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -5,6 +5,11 @@
 include protocol PBackground;
 include protocol PBackgroundLSSnapshot;
 
+include "mozilla/dom/localstorage/SerializationHelpers.h";
+
+using mozilla::dom::LSSnapshot::LoadState
+  from "mozilla/dom/LSSnapshot.h";
+
 namespace mozilla {
 namespace dom {
 
@@ -20,6 +25,7 @@ struct LSSnapshotInitInfo
   uint32_t totalLength;
   int64_t initialUsage;
   int64_t peakUsage;
+  LoadState loadState;
 };
 
 sync protocol PBackgroundLSDatabase
diff --git a/dom/localstorage/SerializationHelpers.h b/dom/localstorage/SerializationHelpers.h
new file mode 100644
index 000000000000..d13586a17d5c
--- /dev/null
+++ b/dom/localstorage/SerializationHelpers.h
@@ -0,0 +1,25 @@
+/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* vim: set ts=8 sts=2 et sw=2 tw=80: */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#ifndef mozilla_dom_localstorage_SerializationHelpers_h
+#define mozilla_dom_localstorage_SerializationHelpers_h
+
+#include "ipc/IPCMessageUtils.h"
+
+#include "mozilla/dom/LSSnapshot.h"
+
+namespace IPC {
+
+template <>
+struct ParamTraits<mozilla::dom::LSSnapshot::LoadState> :
+  public ContiguousEnumSerializer<mozilla::dom::LSSnapshot::LoadState,
+                                  mozilla::dom::LSSnapshot::LoadState::Initial,
+                                  mozilla::dom::LSSnapshot::LoadState::EndGuard>
+{ };
+
+} // namespace IPC
+
+#endif // mozilla_dom_localstorage_SerializationHelpers_h
diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index 6cd6e8b4c0a9..b3c05070b7ae 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -16,6 +16,7 @@ XPIDL_MODULE = 'dom_localstorage'
 
 EXPORTS.mozilla.dom.localstorage += [
     'ActorsParent.h',
+    'SerializationHelpers.h',
 ]
 
 EXPORTS.mozilla.dom += [
@@ -23,6 +24,7 @@ EXPORTS.mozilla.dom += [
     'LocalStorageManager2.h',
     'LSObject.h',
     'LSObserver.h',
+    'LSSnapshot.h',
 ]
 
 UNIFIED_SOURCES += [

From 05662ca09efb222606f3b07bec28144b9b70f755 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:14 +0100
Subject: [PATCH 51/78] Bug 1286798 - Part 37: Always preallocate quota when
 initializing a snapshot; r=asuth

Besides always preallocating quota we now also preallocate more. This mitigates number of additional sync calls.
---
 dom/localstorage/ActorsChild.cpp            |  2 +
 dom/localstorage/ActorsChild.h              |  2 +
 dom/localstorage/ActorsParent.cpp           | 76 +++++++++++++++------
 dom/localstorage/LSDatabase.cpp             | 25 +++----
 dom/localstorage/LSDatabase.h               |  1 -
 dom/localstorage/PBackgroundLSDatabase.ipdl |  4 +-
 6 files changed, 72 insertions(+), 38 deletions(-)

diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 852bfa8d2019..926711a53ab4 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -80,7 +80,9 @@ LSDatabaseChild::RecvRequestAllowToClose()
 
 PBackgroundLSSnapshotChild*
 LSDatabaseChild::AllocPBackgroundLSSnapshotChild(const nsString& aDocumentURI,
+                                                 const bool& aIncreasePeakUsage,
                                                  const int64_t& aRequestedSize,
+                                                 const int64_t& aMinSize,
                                                  LSSnapshotInitInfo* aInitInfo)
 {
   MOZ_CRASH("PBackgroundLSSnapshotChild actor should be manually constructed!");
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index ab4e43cea1ce..700e61d94f37 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -68,7 +68,9 @@ private:
 
   PBackgroundLSSnapshotChild*
   AllocPBackgroundLSSnapshotChild(const nsString& aDocumentURI,
+                                  const bool& aIncreasePeakUsage,
                                   const int64_t& aRequestedSize,
+                                  const int64_t& aMinSize,
                                   LSSnapshotInitInfo* aInitInfo) override;
 
   bool
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 42f5c4a59ced..57b3dfa17a4d 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1390,8 +1390,7 @@ public:
   NoteInactiveDatabase(Database* aDatabase);
 
   void
-  GetSnapshotInitInfo(int64_t aRequestedSize,
-                      nsTHashtable<nsStringHashKey>& aLoadedItems,
+  GetSnapshotInitInfo(nsTHashtable<nsStringHashKey>& aLoadedItems,
                       nsTArray<LSItemInfo>& aItemInfos,
                       uint32_t& aTotalLength,
                       int64_t& aInitialUsage,
@@ -1430,8 +1429,9 @@ public:
   void
   EndUpdateBatch(int64_t aSnapshotPeakUsage);
 
-  bool
-  UpdateUsage(int64_t aDelta);
+  int64_t
+  RequestUpdateUsage(int64_t aRequestedSize,
+                     int64_t aMinSize);
 
   NS_INLINE_DECL_REFCOUNTING(Datastore)
 
@@ -1439,6 +1439,9 @@ private:
   // Reference counted.
   ~Datastore();
 
+  bool
+  UpdateUsage(int64_t aDelta);
+
   void
   MaybeClose();
 
@@ -1651,13 +1654,17 @@ private:
 
   PBackgroundLSSnapshotParent*
   AllocPBackgroundLSSnapshotParent(const nsString& aDocumentURI,
+                                   const bool& aIncreasePeakUsage,
                                    const int64_t& aRequestedSize,
+                                   const int64_t& aMinSize,
                                    LSSnapshotInitInfo* aInitInfo) override;
 
   mozilla::ipc::IPCResult
   RecvPBackgroundLSSnapshotConstructor(PBackgroundLSSnapshotParent* aActor,
                                        const nsString& aDocumentURI,
+                                       const bool& aIncreasePeakUsage,
                                        const int64_t& aRequestedSize,
+                                       const int64_t& aMinSize,
                                        LSSnapshotInitInfo* aInitInfo) override;
 
   bool
@@ -3757,8 +3764,7 @@ Datastore::NoteInactiveDatabase(Database* aDatabase)
 }
 
 void
-Datastore::GetSnapshotInitInfo(int64_t aRequestedSize,
-                               nsTHashtable<nsStringHashKey>& aLoadedItems,
+Datastore::GetSnapshotInitInfo(nsTHashtable<nsStringHashKey>& aLoadedItems,
                                nsTArray<LSItemInfo>& aItemInfos,
                                uint32_t& aTotalLength,
                                int64_t& aInitialUsage,
@@ -3832,9 +3838,6 @@ Datastore::GetSnapshotInitInfo(int64_t aRequestedSize,
 
   aInitialUsage = mUsage;
   aPeakUsage = aInitialUsage;
-  if (aRequestedSize && UpdateUsage(aRequestedSize)) {
-    aPeakUsage += aRequestedSize;
-  }
 }
 
 void
@@ -4078,6 +4081,25 @@ Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
 #endif
 }
 
+int64_t
+Datastore::RequestUpdateUsage(int64_t aRequestedSize,
+                              int64_t aMinSize)
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(aRequestedSize > 0);
+  MOZ_ASSERT(aMinSize > 0);
+
+  if (UpdateUsage(aRequestedSize)) {
+    return aRequestedSize;
+  }
+
+  if (UpdateUsage(aMinSize)) {
+    return aMinSize;
+  }
+
+  return 0;
+}
+
 bool
 Datastore::UpdateUsage(int64_t aDelta)
 {
@@ -4411,12 +4433,19 @@ Database::RecvAllowToClose()
 
 PBackgroundLSSnapshotParent*
 Database::AllocPBackgroundLSSnapshotParent(const nsString& aDocumentURI,
+                                           const bool& aIncreasePeakUsage,
                                            const int64_t& aRequestedSize,
+                                           const int64_t& aMinSize,
                                            LSSnapshotInitInfo* aInitInfo)
 {
   AssertIsOnBackgroundThread();
 
-  if (NS_WARN_IF(aRequestedSize < 0)) {
+  if (NS_WARN_IF(aIncreasePeakUsage && aRequestedSize <= 0)) {
+    ASSERT_UNLESS_FUZZING();
+    return nullptr;
+  }
+
+  if (NS_WARN_IF(aIncreasePeakUsage && aMinSize <= 0)) {
     ASSERT_UNLESS_FUZZING();
     return nullptr;
   }
@@ -4436,11 +4465,14 @@ mozilla::ipc::IPCResult
 Database::RecvPBackgroundLSSnapshotConstructor(
                                             PBackgroundLSSnapshotParent* aActor,
                                             const nsString& aDocumentURI,
+                                            const bool& aIncreasePeakUsage,
                                             const int64_t& aRequestedSize,
+                                            const int64_t& aMinSize,
                                             LSSnapshotInitInfo* aInitInfo)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aRequestedSize >= 0);
+  MOZ_ASSERT_IF(aIncreasePeakUsage, aRequestedSize > 0);
+  MOZ_ASSERT_IF(aIncreasePeakUsage, aMinSize > 0);
   MOZ_ASSERT(aInitInfo);
   MOZ_ASSERT(!mAllowedToClose);
 
@@ -4454,14 +4486,18 @@ Database::RecvPBackgroundLSSnapshotConstructor(
   int64_t initialUsage;
   int64_t peakUsage;
   LSSnapshot::LoadState loadState;
-  mDatastore->GetSnapshotInitInfo(aRequestedSize,
-                                  loadedItems,
+  mDatastore->GetSnapshotInitInfo(loadedItems,
                                   itemInfos,
                                   totalLength,
                                   initialUsage,
                                   peakUsage,
                                   loadState);
 
+  if (aIncreasePeakUsage) {
+    int64_t size = mDatastore->RequestUpdateUsage(aRequestedSize, aMinSize);
+    peakUsage += size;
+  }
+
   snapshot->Init(loadedItems, totalLength, initialUsage, peakUsage, loadState);
 
   RegisterSnapshot(snapshot);
@@ -4781,15 +4817,11 @@ Snapshot::RecvIncreasePeakUsage(const int64_t& aRequestedSize,
     return IPC_FAIL_NO_REASON(this);
   }
 
-  if (mDatastore->UpdateUsage(aRequestedSize)) {
-    mPeakUsage += aRequestedSize;
-    *aSize = aRequestedSize;
-  } else if (mDatastore->UpdateUsage(aMinSize)) {
-    mPeakUsage += aMinSize;
-    *aSize = aMinSize;
-  } else {
-    *aSize = 0;
-  }
+  int64_t size = mDatastore->RequestUpdateUsage(aRequestedSize, aMinSize);
+
+  mPeakUsage += size;
+
+  *aSize = size;
 
   return IPC_OK();
 }
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index 6ce932607964..e643f8ef18f7 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -101,7 +101,7 @@ LSDatabase::GetLength(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -124,7 +124,7 @@ LSDatabase::GetKey(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -147,7 +147,7 @@ LSDatabase::GetItem(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -169,7 +169,7 @@ LSDatabase::GetKeys(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -193,7 +193,7 @@ LSDatabase::SetItem(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ true);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -216,7 +216,7 @@ LSDatabase::RemoveItem(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -238,7 +238,7 @@ LSDatabase::Clear(LSObject* aObject,
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(!mAllowedToClose);
 
-  nsresult rv = EnsureSnapshot(aObject, /* aRequestedBySetItem */ false);
+  nsresult rv = EnsureSnapshot(aObject);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -263,9 +263,7 @@ LSDatabase::BeginExplicitSnapshot(LSObject* aObject)
     return NS_ERROR_ALREADY_INITIALIZED;
   }
 
-  nsresult rv = EnsureSnapshot(aObject,
-                               /* aRequestedBySetItem */ false,
-                               /* aExplicit */ true);
+  nsresult rv = EnsureSnapshot(aObject, /* aExplicit */ true);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -297,7 +295,6 @@ LSDatabase::EndExplicitSnapshot(LSObject* aObject)
 
 nsresult
 LSDatabase::EnsureSnapshot(LSObject* aObject,
-                           bool aRequestedBySetItem,
                            bool aExplicit)
 {
   MOZ_ASSERT(aObject);
@@ -313,13 +310,13 @@ LSDatabase::EnsureSnapshot(LSObject* aObject,
 
   LSSnapshotChild* actor = new LSSnapshotChild(snapshot);
 
-  int64_t requestedSize = aRequestedBySetItem ? 4096 : 0;
-
   LSSnapshotInitInfo initInfo;
   bool ok =
     mActor->SendPBackgroundLSSnapshotConstructor(actor,
                                                  aObject->DocumentURI(),
-                                                 requestedSize,
+                                                 /* increasePeakUsage */ true,
+                                                 /* requestedSize */ 131072,
+                                                 /* minSize */ 4096,
                                                  &initInfo);
   if (NS_WARN_IF(!ok)) {
     return NS_ERROR_FAILURE;
diff --git a/dom/localstorage/LSDatabase.h b/dom/localstorage/LSDatabase.h
index 2854cb70814c..ec7435b789fc 100644
--- a/dom/localstorage/LSDatabase.h
+++ b/dom/localstorage/LSDatabase.h
@@ -108,7 +108,6 @@ private:
 
   nsresult
   EnsureSnapshot(LSObject* aObject,
-                 bool aRequestedBySetItem,
                  bool aExplicit = false);
 
   void
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index e802597e41d4..843ea3bd91aa 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -48,7 +48,9 @@ parent:
   async AllowToClose();
 
   sync PBackgroundLSSnapshot(nsString documentURI,
-                             int64_t requestedSize)
+                             bool increasePeakUsage,
+                             int64_t requestedSize,
+                             int64_t minSize)
     returns (LSSnapshotInitInfo initInfo);
 
 child:

From f2c157765252e822ad79292c42fdade0df00189e Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:17 +0100
Subject: [PATCH 52/78] Bug 1286798 - Part 38: Cache items in an array; r=asuth

Items are now cached also in an array (besides a hashtable). This gives us very fast snapshot initizilization for datastores that fit into the prefill limit. String buffers are reference counted, so memory footprint is only affected by the size of nsString.
This patch also introduces a WriteOptimizer which is an abstraction for collecting, coalescing and applying write operations.
---
 dom/localstorage/ActorsParent.cpp | 386 ++++++++++++++++++++++++------
 1 file changed, 313 insertions(+), 73 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 57b3dfa17a4d..af3d2c7a4c64 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -888,6 +888,128 @@ DetachShadowDatabase(mozIStorageConnection* aConnection)
  * Non-actor class declarations
  ******************************************************************************/
 
+class WriteOptimizer final
+{
+  class WriteInfo;
+  class AddItemInfo;
+  class UpdateItemInfo;
+  class RemoveItemInfo;
+
+  nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
+  bool mClearedWriteInfos;
+
+public:
+  WriteOptimizer()
+    : mClearedWriteInfos(false)
+  { }
+
+  void
+  AddItem(const nsString& aKey,
+          const nsString& aValue);
+
+  void
+  UpdateItem(const nsString& aKey,
+             const nsString& aValue);
+
+  void
+  RemoveItem(const nsString& aKey);
+
+  void
+  Clear();
+
+  void
+  ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems);
+};
+
+class WriteOptimizer::WriteInfo
+{
+public:
+  enum Type {
+    AddItem = 0,
+    UpdateItem,
+    RemoveItem
+  };
+
+  virtual Type
+  GetType() = 0;
+
+  virtual ~WriteInfo() = default;
+};
+
+class WriteOptimizer::AddItemInfo
+  : public WriteInfo
+{
+  nsString mKey;
+  nsString mValue;
+
+public:
+  AddItemInfo(const nsAString& aKey,
+              const nsAString& aValue)
+    : mKey(aKey)
+    , mValue(aValue)
+  { }
+
+  const nsAString&
+  GetKey() const
+  {
+    return mKey;
+  }
+
+  const nsAString&
+  GetValue() const
+  {
+    return mValue;
+  }
+
+private:
+  Type
+  GetType() override
+  {
+    return AddItem;
+  }
+};
+
+class WriteOptimizer::UpdateItemInfo final
+  : public AddItemInfo
+{
+public:
+  UpdateItemInfo(const nsAString& aKey,
+                 const nsAString& aValue)
+    : AddItemInfo(aKey, aValue)
+  { }
+
+private:
+  Type
+  GetType() override
+  {
+    return UpdateItem;
+  }
+};
+
+class WriteOptimizer::RemoveItemInfo final
+  : public WriteInfo
+{
+  nsString mKey;
+
+public:
+  explicit RemoveItemInfo(const nsAString& aKey)
+    : mKey(aKey)
+  { }
+
+  const nsAString&
+  GetKey() const
+  {
+    return mKey;
+  }
+
+private:
+  Type
+  GetType() override
+  {
+    return RemoveItem;
+  }
+};
+
 class DatastoreOperationBase
   : public Runnable
 {
@@ -1297,15 +1419,15 @@ class Datastore final
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
   nsTHashtable<nsPtrHashKey<Database>> mActiveDatabases;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
-  nsDataHashtable<nsStringHashKey, uint32_t> mUpdateBatchRemovals;
-  nsTArray<nsString> mUpdateBatchAppends;
-  nsTArray<nsString> mKeys;
+  nsTArray<LSItemInfo> mOrderedItems;
   nsTArray<int64_t> mPendingUsageDeltas;
+  WriteOptimizer mWriteOptimizer;
   const nsCString mOrigin;
   const uint32_t mPrivateBrowsingId;
   int64_t mUsage;
   int64_t mUpdateBatchUsage;
   int64_t mSizeOfKeys;
+  int64_t mSizeOfItems;
   bool mClosed;
 #ifdef DEBUG
   bool mInUpdateBatch;
@@ -1317,11 +1439,12 @@ public:
             uint32_t aPrivateBrowsingId,
             int64_t aUsage,
             int64_t aSizeOfKeys,
+            int64_t aSizeOfItems,
             already_AddRefed<DirectoryLock>&& aDirectoryLock,
             already_AddRefed<Connection>&& aConnection,
             already_AddRefed<QuotaObject>&& aQuotaObject,
             nsDataHashtable<nsStringHashKey, nsString>& aValues,
-            nsTArray<nsString>& aKeys);
+            nsTArray<LSItemInfo>& aOrderedItems);
 
   const nsCString&
   Origin() const
@@ -1949,7 +2072,7 @@ class PrepareDatastoreOp
   nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   LoadDataOp* mLoadDataOp;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
-  nsTArray<nsString> mKeys;
+  nsTArray<LSItemInfo> mOrderedItems;
   const LSRequestPrepareDatastoreParams mParams;
   nsCString mSuffix;
   nsCString mGroup;
@@ -1959,6 +2082,7 @@ class PrepareDatastoreOp
   uint32_t mPrivateBrowsingId;
   int64_t mUsage;
   int64_t mSizeOfKeys;
+  int64_t mSizeOfItems;
   NestedState mNestedState;
   bool mDatabaseNotAvailable;
   bool mRequestedDirectoryLock;
@@ -2896,6 +3020,122 @@ CreateQuotaClient()
 
 } // namespace localstorage
 
+/*******************************************************************************
+ * WriteOptimizer
+ ******************************************************************************/
+
+void
+WriteOptimizer::AddItem(const nsString& aKey,
+                        const nsString& aValue)
+{
+  AssertIsOnBackgroundThread();
+
+  WriteInfo* existingWriteInfo;
+  nsAutoPtr<WriteInfo> newWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::RemoveItem) {
+    newWriteInfo = new UpdateItemInfo(aKey, aValue);
+  } else {
+    newWriteInfo = new AddItemInfo(aKey, aValue);
+  }
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
+}
+
+void
+WriteOptimizer::UpdateItem(const nsString& aKey,
+                           const nsString& aValue)
+{
+  AssertIsOnBackgroundThread();
+
+  WriteInfo* existingWriteInfo;
+  nsAutoPtr<WriteInfo> newWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::AddItem) {
+    newWriteInfo = new AddItemInfo(aKey, aValue);
+  } else {
+    newWriteInfo = new UpdateItemInfo(aKey, aValue);
+  }
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
+}
+
+void
+WriteOptimizer::RemoveItem(const nsString& aKey)
+{
+  AssertIsOnBackgroundThread();
+
+  WriteInfo* existingWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::AddItem) {
+    mWriteInfos.Remove(aKey);
+    return;
+  }
+
+  nsAutoPtr<WriteInfo> newWriteInfo(new RemoveItemInfo(aKey));
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
+}
+
+void
+WriteOptimizer::Clear()
+{
+  AssertIsOnBackgroundThread();
+
+  mWriteInfos.Clear();
+  mClearedWriteInfos = true;
+}
+
+void
+WriteOptimizer::ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems)
+{
+  if (mClearedWriteInfos) {
+    aOrderedItems.Clear();
+    mClearedWriteInfos = false;
+  }
+
+  for (int32_t index = aOrderedItems.Length() - 1;
+       index >= 0;
+       index--) {
+    LSItemInfo& item = aOrderedItems[index];
+
+    if (auto entry = mWriteInfos.Lookup(item.key())) {
+      WriteInfo* writeInfo = entry.Data();
+
+      switch (writeInfo->GetType()) {
+        case WriteInfo::RemoveItem:
+          aOrderedItems.RemoveElementAt(index);
+          entry.Remove();
+          break;
+
+        case WriteInfo::UpdateItem: {
+          auto updateItemInfo = static_cast<UpdateItemInfo*>(writeInfo);
+          item.value() = updateItemInfo->GetValue();
+          entry.Remove();
+          break;
+        }
+
+        case WriteInfo::AddItem:
+          break;
+
+        default:
+          MOZ_CRASH("Bad type!");
+      }
+    }
+  }
+
+  for (auto iter = mWriteInfos.ConstIter(); !iter.Done(); iter.Next()) {
+    WriteInfo* writeInfo = iter.Data();
+
+    MOZ_ASSERT(writeInfo->GetType() == WriteInfo::AddItem);
+
+    auto addItemInfo = static_cast<AddItemInfo*>(writeInfo);
+
+    LSItemInfo* itemInfo = aOrderedItems.AppendElement();
+    itemInfo->key() = addItemInfo->GetKey();
+    itemInfo->value() = addItemInfo->GetValue();
+  }
+
+  mWriteInfos.Clear();
+}
+
 /*******************************************************************************
  * DatastoreOperationBase
  ******************************************************************************/
@@ -3546,11 +3786,12 @@ Datastore::Datastore(const nsACString& aOrigin,
                      uint32_t aPrivateBrowsingId,
                      int64_t aUsage,
                      int64_t aSizeOfKeys,
+                     int64_t aSizeOfItems,
                      already_AddRefed<DirectoryLock>&& aDirectoryLock,
                      already_AddRefed<Connection>&& aConnection,
                      already_AddRefed<QuotaObject>&& aQuotaObject,
                      nsDataHashtable<nsStringHashKey, nsString>& aValues,
-                     nsTArray<nsString>& aKeys)
+                     nsTArray<LSItemInfo>& aOrderedItems)
   : mDirectoryLock(std::move(aDirectoryLock))
   , mConnection(std::move(aConnection))
   , mQuotaObject(std::move(aQuotaObject))
@@ -3559,6 +3800,7 @@ Datastore::Datastore(const nsACString& aOrigin,
   , mUsage(aUsage)
   , mUpdateBatchUsage(-1)
   , mSizeOfKeys(aSizeOfKeys)
+  , mSizeOfItems(aSizeOfItems)
   , mClosed(false)
 #ifdef DEBUG
   , mInUpdateBatch(false)
@@ -3567,7 +3809,7 @@ Datastore::Datastore(const nsACString& aOrigin,
   AssertIsOnBackgroundThread();
 
   mValues.SwapElements(aValues);
-  mKeys.SwapElements(aKeys);
+  mOrderedItems.SwapElements(aOrderedItems);
 }
 
 Datastore::~Datastore()
@@ -3773,43 +4015,47 @@ Datastore::GetSnapshotInitInfo(nsTHashtable<nsStringHashKey>& aLoadedItems,
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(!mInUpdateBatch);
 
 #ifdef DEBUG
   int64_t sizeOfKeys = 0;
-  for (auto key : mKeys) {
-    sizeOfKeys += static_cast<int64_t>(key.Length());
+  int64_t sizeOfItems = 0;
+  for (auto item : mOrderedItems) {
+    int64_t sizeOfKey = static_cast<int64_t>(item.key().Length());
+    sizeOfKeys += sizeOfKey;
+    sizeOfItems += sizeOfKey + static_cast<int64_t>(item.value().Length());
   }
   MOZ_ASSERT(mSizeOfKeys == sizeOfKeys);
+  MOZ_ASSERT(mSizeOfItems == sizeOfItems);
 #endif
 
   int64_t size = 0;
   if (mSizeOfKeys <= gSnapshotPrefill) {
-    nsString value;
-    for (auto key : mKeys) {
-      if (!value.IsVoid()) {
-        DebugOnly<bool> hasValue = mValues.Get(key, &value);
-        MOZ_ASSERT(hasValue);
+    if (mSizeOfItems <= gSnapshotPrefill) {
+      aItemInfos.AppendElements(mOrderedItems);
+      aLoadState = LSSnapshot::LoadState::AllOrderedItems;
+    } else {
+      nsString value;
+      for (auto item : mOrderedItems) {
+        if (!value.IsVoid()) {
+          value = item.value();
 
-        size += static_cast<int64_t>(key.Length()) +
-                static_cast<int64_t>(value.Length());
+          size += static_cast<int64_t>(item.key().Length()) +
+                  static_cast<int64_t>(value.Length());
 
-        if (size > gSnapshotPrefill) {
-          value.SetIsVoid(true);
-        } else {
-          aLoadedItems.PutEntry(key);
+          if (size <= gSnapshotPrefill) {
+            aLoadedItems.PutEntry(item.key());
+          } else {
+            value.SetIsVoid(true);
+          }
         }
+
+        LSItemInfo* itemInfo = aItemInfos.AppendElement();
+        itemInfo->key() = item.key();
+        itemInfo->value() = value;
       }
 
-      LSItemInfo* itemInfo = aItemInfos.AppendElement();
-      itemInfo->key() = key;
-      itemInfo->value() = value;
-    }
-
-    if (value.IsVoid()) {
       aLoadState = LSSnapshot::LoadState::AllOrderedKeys;
-    } else {
-      aLoadedItems.Clear();
-      aLoadState = LSSnapshot::LoadState::AllOrderedItems;
     }
   } else {
     for (auto iter = mValues.ConstIter(); !iter.Done(); iter.Next()) {
@@ -3830,7 +4076,7 @@ Datastore::GetSnapshotInitInfo(nsTHashtable<nsStringHashKey>& aLoadedItems,
       itemInfo->value() = iter.Data();
     }
 
-    MOZ_ASSERT(aItemInfos.Length() < mKeys.Length());
+    MOZ_ASSERT(aItemInfos.Length() < mOrderedItems.Length());
     aLoadState = LSSnapshot::LoadState::Partial;
   }
 
@@ -3857,7 +4103,9 @@ Datastore::GetKeys(nsTArray<nsString>& aKeys) const
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!mClosed);
 
-  aKeys.AppendElements(mKeys);
+  for (auto item : mOrderedItems) {
+    aKeys.AppendElement(item.key());
+  }
 }
 
 void
@@ -3883,15 +4131,24 @@ Datastore::SetItem(Database* aDatabase,
     mValues.Put(aKey, aValue);
 
     if (isNewItem) {
-      mUpdateBatchAppends.AppendElement(aKey);
+      mWriteOptimizer.AddItem(aKey, aValue);
 
-      mUpdateBatchUsage += static_cast<int64_t>(aKey.Length()) +
-                           static_cast<int64_t>(aValue.Length());
+      int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
+      int64_t sizeOfItem = sizeOfKey + static_cast<int64_t>(aValue.Length());
 
-      mSizeOfKeys += static_cast<int64_t>(aKey.Length());
+      mUpdateBatchUsage += sizeOfItem;
+
+      mSizeOfKeys += sizeOfKey;
+      mSizeOfItems += sizeOfItem;
     } else {
-      mUpdateBatchUsage += static_cast<int64_t>(aValue.Length()) -
-                           static_cast<int64_t>(oldValue.Length());
+      mWriteOptimizer.UpdateItem(aKey, aValue);
+
+      int64_t delta = static_cast<int64_t>(aValue.Length()) -
+                      static_cast<int64_t>(oldValue.Length());
+
+      mUpdateBatchUsage += delta;
+
+      mSizeOfItems += delta;
     }
 
     if (IsPersistent()) {
@@ -3921,17 +4178,15 @@ Datastore::RemoveItem(Database* aDatabase,
 
     mValues.Remove(aKey);
 
-    auto entry = mUpdateBatchRemovals.LookupForAdd(aKey);
-    if (entry) {
-      entry.Data()++;
-    } else {
-      entry.OrInsert([]() { return 1; });
-    }
+    mWriteOptimizer.RemoveItem(aKey);
 
-    mUpdateBatchUsage -= (static_cast<int64_t>(aKey.Length()) +
-                          static_cast<int64_t>(oldValue.Length()));
+    int64_t sizeOfKey = static_cast<int64_t>(aKey.Length());
+    int64_t sizeOfItem = sizeOfKey + static_cast<int64_t>(oldValue.Length());
 
-    mSizeOfKeys -= static_cast<int64_t>(aKey.Length());
+    mUpdateBatchUsage -= sizeOfItem;
+
+    mSizeOfKeys -= sizeOfKey;
+    mSizeOfItems -= sizeOfItem;
 
     if (IsPersistent()) {
       mConnection->RemoveItem(aKey);
@@ -3964,14 +4219,12 @@ Datastore::Clear(Database* aDatabase,
 
     mValues.Clear();
 
-    mUpdateBatchRemovals.Clear();
-    mUpdateBatchAppends.Clear();
-
-    mKeys.Clear();
+    mWriteOptimizer.Clear();
 
     mUpdateBatchUsage = updateBatchUsage;
 
     mSizeOfKeys = 0;
+    mSizeOfItems = 0;
 
     if (IsPersistent()) {
       mConnection->Clear();
@@ -3998,9 +4251,10 @@ Datastore::PrivateBrowsingClear()
 
     mValues.Clear();
 
-    mKeys.Clear();
+    mOrderedItems.Clear();
 
     mSizeOfKeys = 0;
+    mSizeOfItems = 0;
   }
 }
 
@@ -4032,26 +4286,7 @@ Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(mInUpdateBatch);
 
-  if (mUpdateBatchAppends.Length()) {
-    mKeys.AppendElements(std::move(mUpdateBatchAppends));
-  }
-
-  if (mUpdateBatchRemovals.Count()) {
-    RefPtr<Datastore> self = this;
-
-    mKeys.RemoveElementsBy([self](const nsString& aKey) {
-      if (auto entry = self->mUpdateBatchRemovals.Lookup(aKey)) {
-        if (--entry.Data() == 0) {
-          entry.Remove();
-        }
-        return true;
-      }
-      return false;
-    });
-  }
-
-  MOZ_ASSERT(mUpdateBatchAppends.Length() == 0);
-  MOZ_ASSERT(mUpdateBatchRemovals.Count() == 0);
+  mWriteOptimizer.ApplyWrites(mOrderedItems);
 
   int64_t delta = mUpdateBatchUsage - aSnapshotPeakUsage;
 
@@ -5091,6 +5326,7 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
   , mPrivateBrowsingId(0)
   , mUsage(0)
   , mSizeOfKeys(0)
+  , mSizeOfItems(0)
   , mNestedState(NestedState::BeforeNesting)
   , mDatabaseNotAvailable(false)
   , mRequestedDirectoryLock(false)
@@ -5902,11 +6138,12 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
                                mPrivateBrowsingId,
                                mUsage,
                                mSizeOfKeys,
+                               mSizeOfItems,
                                mDirectoryLock.forget(),
                                mConnection.forget(),
                                quotaObject.forget(),
                                mValues,
-                               mKeys);
+                               mOrderedItems);
 
     mDatastore->NoteLivePrepareDatastoreOp(this);
 
@@ -6116,8 +6353,11 @@ LoadDataOp::DoDatastoreWork()
     }
 
     mPrepareDatastoreOp->mValues.Put(key, value);
-    mPrepareDatastoreOp->mKeys.AppendElement(key);
+    auto item = mPrepareDatastoreOp->mOrderedItems.AppendElement();
+    item->key() = key;
+    item->value() = value;
     mPrepareDatastoreOp->mSizeOfKeys += key.Length();
+    mPrepareDatastoreOp->mSizeOfItems += key.Length() + value.Length();
 #ifdef DEBUG
     mPrepareDatastoreOp->mDEBUGUsage += key.Length() + value.Length();
 #endif

From 8d2b3b583dc20aadd12c5fb3ac0f76b78548f20d Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:20 +0100
Subject: [PATCH 53/78] Bug 1286798 - Part 39: Reduce number of hash lookups;
 r=asuth

---
 dom/localstorage/LSSnapshot.cpp | 211 +++++++++++++++++++++-----------
 dom/localstorage/LSSnapshot.h   |   5 +
 2 files changed, 142 insertions(+), 74 deletions(-)

diff --git a/dom/localstorage/LSSnapshot.cpp b/dom/localstorage/LSSnapshot.cpp
index 670dc82d342e..62ebb6f8966d 100644
--- a/dom/localstorage/LSSnapshot.cpp
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -158,74 +158,9 @@ LSSnapshot::GetItem(const nsAString& aKey,
   MOZ_ASSERT(!mSentFinish);
 
   nsString result;
-
-  switch (mLoadState) {
-    case LoadState::Partial: {
-      if (mValues.Get(aKey, &result)) {
-        MOZ_ASSERT(!result.IsVoid());
-      } else if (mLoadedItems.GetEntry(aKey) || mUnknownItems.GetEntry(aKey)) {
-        result.SetIsVoid(true);
-      } else {
-        if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
-          return NS_ERROR_FAILURE;
-        }
-
-        if (result.IsVoid()) {
-          mUnknownItems.PutEntry(aKey);
-        } else {
-          mLoadedItems.PutEntry(aKey);
-          mValues.Put(aKey, result);
-
-          if (mLoadedItems.Count() == mInitLength) {
-            mLoadedItems.Clear();
-            mUnknownItems.Clear();
-            mLength = 0;
-            mLoadState = LoadState::AllUnorderedItems;
-          }
-        }
-      }
-
-      break;
-    }
-
-    case LoadState::AllOrderedKeys: {
-      if (mValues.Get(aKey, &result)) {
-        if (result.IsVoid()) {
-          if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
-            return NS_ERROR_FAILURE;
-          }
-
-          MOZ_ASSERT(!result.IsVoid());
-
-          mLoadedItems.PutEntry(aKey);
-          mValues.Put(aKey, result);
-
-          if (mLoadedItems.Count() == mInitLength) {
-            mLoadedItems.Clear();
-            MOZ_ASSERT(mLength == 0);
-            mLoadState = LoadState::AllOrderedItems;
-          }
-        }
-      } else {
-        result.SetIsVoid(true);
-      }
-
-      break;
-    }
-
-    case LoadState::AllUnorderedItems:
-    case LoadState::AllOrderedItems: {
-      if (mValues.Get(aKey, &result)) {
-        MOZ_ASSERT(!result.IsVoid());
-      } else {
-        result.SetIsVoid(true);
-      }
-
-      break;
-    }
-
-    default:
-      MOZ_CRASH("Bad state!");
+  nsresult rv = GetItemInternal(aKey, Optional<nsString>(), result);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
   }
 
   aResult = result;
@@ -263,7 +198,8 @@ LSSnapshot::SetItem(const nsAString& aKey,
   MOZ_ASSERT(!mSentFinish);
 
   nsString oldValue;
-  nsresult rv = GetItem(aKey, oldValue);
+  nsresult rv =
+    GetItemInternal(aKey, Optional<nsString>(nsString(aValue)), oldValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -283,11 +219,14 @@ LSSnapshot::SetItem(const nsAString& aKey,
 
     rv = UpdateUsage(delta);
     if (NS_WARN_IF(NS_FAILED(rv))) {
+      if (oldValue.IsVoid()) {
+        mValues.Remove(aKey);
+      } else {
+        mValues.Put(aKey, oldValue);
+      }
       return rv;
     }
 
-    mValues.Put(aKey, nsString(aValue));
-
     if (oldValue.IsVoid() && mLoadState == LoadState::Partial) {
       mLength++;
     }
@@ -316,7 +255,8 @@ LSSnapshot::RemoveItem(const nsAString& aKey,
   MOZ_ASSERT(!mSentFinish);
 
   nsString oldValue;
-  nsresult rv = GetItem(aKey, oldValue);
+  nsresult rv =
+    GetItemInternal(aKey, Optional<nsString>(VoidString()), oldValue);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -333,8 +273,6 @@ LSSnapshot::RemoveItem(const nsAString& aKey,
     DebugOnly<nsresult> rv = UpdateUsage(delta);
     MOZ_ASSERT(NS_SUCCEEDED(rv));
 
-    mValues.Remove(aKey);
-
     if (mLoadState == LoadState::Partial) {
       mLength--;
     }
@@ -428,6 +366,131 @@ LSSnapshot::Finish()
   return NS_OK;
 }
 
+nsresult
+LSSnapshot::GetItemInternal(const nsAString& aKey,
+                            const Optional<nsString>& aValue,
+                            nsAString& aResult)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  nsString result;
+
+  switch (mLoadState) {
+    case LoadState::Partial: {
+      if (mValues.Get(aKey, &result)) {
+        MOZ_ASSERT(!result.IsVoid());
+      } else if (mLoadedItems.GetEntry(aKey) || mUnknownItems.GetEntry(aKey)) {
+        result.SetIsVoid(true);
+      } else {
+        if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
+          return NS_ERROR_FAILURE;
+        }
+
+        if (result.IsVoid()) {
+          mUnknownItems.PutEntry(aKey);
+        } else {
+          mLoadedItems.PutEntry(aKey);
+          mValues.Put(aKey, result);
+
+          if (mLoadedItems.Count() == mInitLength) {
+            mLoadedItems.Clear();
+            mUnknownItems.Clear();
+            mLength = 0;
+            mLoadState = LoadState::AllUnorderedItems;
+          }
+        }
+      }
+
+      if (aValue.WasPassed()) {
+        const nsString& value = aValue.Value();
+        if (!value.IsVoid()) {
+          mValues.Put(aKey, value);
+        } else if (!result.IsVoid()) {
+          mValues.Remove(aKey);
+        }
+      }
+
+      break;
+    }
+
+    case LoadState::AllOrderedKeys: {
+      if (mValues.Get(aKey, &result)) {
+        if (result.IsVoid()) {
+          if (NS_WARN_IF(!mActor->SendLoadItem(nsString(aKey), &result))) {
+            return NS_ERROR_FAILURE;
+          }
+
+          MOZ_ASSERT(!result.IsVoid());
+
+          mLoadedItems.PutEntry(aKey);
+          mValues.Put(aKey, result);
+
+          if (mLoadedItems.Count() == mInitLength) {
+            mLoadedItems.Clear();
+            MOZ_ASSERT(mLength == 0);
+            mLoadState = LoadState::AllOrderedItems;
+          }
+        }
+      } else {
+        result.SetIsVoid(true);
+      }
+
+      if (aValue.WasPassed()) {
+        const nsString& value = aValue.Value();
+        if (!value.IsVoid()) {
+          mValues.Put(aKey, value);
+        } else if (!result.IsVoid()) {
+          mValues.Remove(aKey);
+        }
+      }
+
+      break;
+    }
+
+    case LoadState::AllUnorderedItems:
+    case LoadState::AllOrderedItems: {
+      if (aValue.WasPassed()) {
+        const nsString& value = aValue.Value();
+        if (!value.IsVoid()) {
+          auto entry = mValues.LookupForAdd(aKey);
+          if (entry) {
+            result = entry.Data();
+            entry.Data() = value;
+          } else {
+            result.SetIsVoid(true);
+            entry.OrInsert([value]() { return value; });
+          }
+        } else {
+          if (auto entry = mValues.Lookup(aKey)) {
+            result = entry.Data();
+            MOZ_ASSERT(!result.IsVoid());
+            entry.Remove();
+          } else {
+            result.SetIsVoid(true);
+          }
+        }
+      } else {
+        if (mValues.Get(aKey, &result)) {
+          MOZ_ASSERT(!result.IsVoid());
+        } else {
+          result.SetIsVoid(true);
+        }
+      }
+
+      break;
+    }
+
+    default:
+      MOZ_CRASH("Bad state!");
+  }
+
+  aResult = result;
+  return NS_OK;
+}
+
 nsresult
 LSSnapshot::EnsureAllKeys()
 {
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
index 5eda4ac3b756..99757407b0b7 100644
--- a/dom/localstorage/LSSnapshot.h
+++ b/dom/localstorage/LSSnapshot.h
@@ -119,6 +119,11 @@ public:
 private:
   ~LSSnapshot();
 
+  nsresult
+  GetItemInternal(const nsAString& aKey,
+                  const Optional<nsString>& aValue,
+                  nsAString& aResult);
+
   nsresult
   EnsureAllKeys();
 

From 1d777b512bf850df4747d1324d43b4eb40c3325c Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:24 +0100
Subject: [PATCH 54/78] Bug 1286798 - Part 40: Increase initial snapshot
 prefill to 16KB; r=asuth

The number has been set by running tp6 tests on all platforms.
---
 modules/libpref/init/all.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index 822ccef2c014..10a85120de55 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1292,7 +1292,7 @@ pref("dom.storage.next_gen", true);
 pref("dom.storage.next_gen", false);
 #endif
 pref("dom.storage.default_quota",      5120);
-pref("dom.storage.snapshot_prefill", 4096);
+pref("dom.storage.snapshot_prefill", 16384);
 pref("dom.storage.testing", false);
 
 pref("dom.send_after_paint_to_content", false);

From 150cf2ce62dd33a2b147a5b3e110ad258b44ffe3 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:27 +0100
Subject: [PATCH 55/78] Bug 1286798 - Part 41: Implement
 QuotaClient::AbortOperationsForProcess; r=asuth

Needed for snapshot reusing.
---
 dom/localstorage/ActorsParent.cpp | 62 ++++++++++++++++++++++++++++---
 1 file changed, 56 insertions(+), 6 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index af3d2c7a4c64..290d83f2c921 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -14,6 +14,7 @@
 #include "mozStorageHelper.h"
 #include "mozilla/Preferences.h"
 #include "mozilla/Unused.h"
+#include "mozilla/dom/ContentParent.h"
 #include "mozilla/dom/PBackgroundLSDatabaseParent.h"
 #include "mozilla/dom/PBackgroundLSObserverParent.h"
 #include "mozilla/dom/PBackgroundLSRequestParent.h"
@@ -1592,6 +1593,7 @@ class PreparedDatastore
 {
   RefPtr<Datastore> mDatastore;
   nsCOMPtr<nsITimer> mTimer;
+  const Maybe<ContentParentId> mContentParentId;
   // Strings share buffers if possible, so it's not a problem to duplicate the
   // origin here.
   const nsCString mOrigin;
@@ -1601,11 +1603,13 @@ class PreparedDatastore
 
 public:
   PreparedDatastore(Datastore* aDatastore,
+                    const Maybe<ContentParentId>& aContentParentId,
                     const nsACString& aOrigin,
                     uint64_t aDatastoreId,
                     bool aForPreload)
     : mDatastore(aDatastore)
     , mTimer(NS_NewTimer())
+    , mContentParentId(aContentParentId)
     , mOrigin(aOrigin)
     , mDatastoreId(aDatastoreId)
     , mForPreload(aForPreload)
@@ -1644,6 +1648,12 @@ public:
     return mDatastore;
   }
 
+  const Maybe<ContentParentId>&
+  GetContentParentId() const
+  {
+    return mContentParentId;
+  }
+
   const nsCString&
   Origin() const
   {
@@ -1695,6 +1705,7 @@ class Database final
   RefPtr<Datastore> mDatastore;
   Snapshot* mSnapshot;
   const PrincipalInfo mPrincipalInfo;
+  const Maybe<ContentParentId> mContentParentId;
   // Strings share buffers if possible, so it's not a problem to duplicate the
   // origin here.
   nsCString mOrigin;
@@ -1709,6 +1720,7 @@ class Database final
 public:
   // Created in AllocPBackgroundLSDatabaseParent.
   Database(const PrincipalInfo& aPrincipalInfo,
+           const Maybe<ContentParentId>& aContentParentId,
            const nsACString& aOrigin,
            uint32_t aPrivateBrowsingId);
 
@@ -1725,6 +1737,12 @@ public:
     return mPrincipalInfo;
   }
 
+  bool
+  IsOwnedByProcess(ContentParentId aContentParentId) const
+  {
+    return mContentParentId && mContentParentId.value() == aContentParentId;
+  }
+
   uint32_t
   PrivateBrowsingId() const
   {
@@ -2065,6 +2083,7 @@ class PrepareDatastoreOp
   };
 
   nsCOMPtr<nsIEventTarget> mMainEventTarget;
+  RefPtr<ContentParent> mContentParent;
   RefPtr<PrepareDatastoreOp> mDelayedOp;
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
@@ -2074,6 +2093,7 @@ class PrepareDatastoreOp
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   nsTArray<LSItemInfo> mOrderedItems;
   const LSRequestPrepareDatastoreParams mParams;
+  Maybe<ContentParentId> mContentParentId;
   nsCString mSuffix;
   nsCString mGroup;
   nsCString mMainThreadOrigin;
@@ -2094,6 +2114,7 @@ class PrepareDatastoreOp
 
 public:
   PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
+                     already_AddRefed<ContentParent> aContentParent,
                      const LSRequestParams& aParams);
 
   bool
@@ -2741,9 +2762,11 @@ AllocPBackgroundLSDatabaseParent(const PrincipalInfo& aPrincipalInfo,
   // We also assume that IPDL must call RecvPBackgroundLSDatabaseConstructor
   // once we return a valid actor in this method.
 
-  RefPtr<Database> database = new Database(aPrincipalInfo,
-                                           preparedDatastore->Origin(),
-                                           aPrivateBrowsingId);
+  RefPtr<Database> database =
+    new Database(aPrincipalInfo,
+                 preparedDatastore->GetContentParentId(),
+                 preparedDatastore->Origin(),
+                 aPrivateBrowsingId);
 
   // Transfer ownership to IPDL.
   return database.forget().take();
@@ -2885,8 +2908,13 @@ AllocPBackgroundLSRequestParent(PBackgroundParent* aBackgroundActor,
 
   switch (aParams.type()) {
     case LSRequestParams::TLSRequestPrepareDatastoreParams: {
+      RefPtr<ContentParent> contentParent =
+        BackgroundParent::GetContentParent(aBackgroundActor);
+
       RefPtr<PrepareDatastoreOp> prepareDatastoreOp =
-        new PrepareDatastoreOp(mainEventTarget, aParams);
+        new PrepareDatastoreOp(mainEventTarget,
+                               contentParent.forget(),
+                               aParams);
 
       if (!gPrepareDatastoreOps) {
         gPrepareDatastoreOps = new PrepareDatastoreOpArray();
@@ -4515,10 +4543,12 @@ PreparedDatastore::TimerCallback(nsITimer* aTimer, void* aClosure)
  ******************************************************************************/
 
 Database::Database(const PrincipalInfo& aPrincipalInfo,
+                   const Maybe<ContentParentId>& aContentParentId,
                    const nsACString& aOrigin,
                    uint32_t aPrivateBrowsingId)
   : mSnapshot(nullptr)
   , mPrincipalInfo(aPrincipalInfo)
+  , mContentParentId(aContentParentId)
   , mOrigin(aOrigin)
   , mPrivateBrowsingId(aPrivateBrowsingId)
   , mAllowedToClose(false)
@@ -5317,10 +5347,13 @@ LSRequestBase::RecvFinish()
  * PrepareDatastoreOp
  ******************************************************************************/
 
-PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
-                                       const LSRequestParams& aParams)
+PrepareDatastoreOp::PrepareDatastoreOp(
+                                 nsIEventTarget* aMainEventTarget,
+                                 already_AddRefed<ContentParent> aContentParent,
+                                 const LSRequestParams& aParams)
   : LSRequestBase(aMainEventTarget)
   , mMainEventTarget(aMainEventTarget)
+  , mContentParent(std::move(aContentParent))
   , mLoadDataOp(nullptr)
   , mParams(aParams.get_LSRequestPrepareDatastoreParams())
   , mPrivateBrowsingId(0)
@@ -5337,6 +5370,10 @@ PrepareDatastoreOp::PrepareDatastoreOp(nsIEventTarget* aMainEventTarget,
 {
   MOZ_ASSERT(aParams.type() ==
                LSRequestParams::TLSRequestPrepareDatastoreParams);
+
+  if (mContentParent) {
+    mContentParentId = Some(mContentParent->ChildID());
+  }
 }
 
 PrepareDatastoreOp::~PrepareDatastoreOp()
@@ -5354,6 +5391,10 @@ PrepareDatastoreOp::Open()
   MOZ_ASSERT(mState == State::Opening);
   MOZ_ASSERT(mNestedState == NestedState::BeforeNesting);
 
+  // Swap this to the stack now to ensure that we release it on this thread.
+  RefPtr<ContentParent> contentParent;
+  mContentParent.swap(contentParent);
+
   if (NS_WARN_IF(QuotaClient::IsShuttingDownOnNonBackgroundThread()) ||
       !MayProceedOnNonOwningThread()) {
     return NS_ERROR_FAILURE;
@@ -6159,6 +6200,7 @@ PrepareDatastoreOp::GetResponse(LSRequestResponse& aResponse)
 
   nsAutoPtr<PreparedDatastore> preparedDatastore(
     new PreparedDatastore(mDatastore,
+                          mContentParentId,
                           mOrigin,
                           datastoreId,
                           /* aForPreload */ !mParams.createIfNotExists()));
@@ -7029,6 +7071,14 @@ void
 QuotaClient::AbortOperationsForProcess(ContentParentId aContentParentId)
 {
   AssertIsOnBackgroundThread();
+
+  if (gLiveDatabases) {
+    for (Database* database : *gLiveDatabases) {
+      if (database->IsOwnedByProcess(aContentParentId)) {
+        database->RequestAllowToClose();
+      }
+    }
+  }
 }
 
 void

From 3177670987363a25a0324f2d5100f9a3d8cb1c7b Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:31 +0100
Subject: [PATCH 56/78] Bug 1286798 - Part 42: Implement snapshot reusing;
 r=asuth

This improves performance by keeping snapshots around for some time if there are no changes done by other processes. If a snapshot is not destroyed immediately after getting into the stable state then there's a chance that it won't have to be synchronously created again when a new opeartion is requested.
---
 .../browser_ext_browsingData_localStorage.js  |   3 +
 dom/localstorage/ActorsChild.cpp              |  14 ++
 dom/localstorage/ActorsChild.h                |   3 +
 dom/localstorage/ActorsParent.cpp             | 152 ++++++++----
 dom/localstorage/LSDatabase.cpp               |   6 +-
 dom/localstorage/LSSnapshot.cpp               | 217 ++++++++++++++++--
 dom/localstorage/LSSnapshot.h                 |  28 ++-
 dom/localstorage/PBackgroundLSSnapshot.ipdl   |  11 +-
 dom/localstorage/test/unit/test_eviction.js   |   3 +-
 dom/localstorage/test/unit/test_groupLimit.js |   4 +
 .../test/unit/test_snapshotting.js            |   4 +
 modules/libpref/init/all.js                   |   1 +
 12 files changed, 372 insertions(+), 74 deletions(-)

diff --git a/browser/components/extensions/test/browser/browser_ext_browsingData_localStorage.js b/browser/components/extensions/test/browser/browser_ext_browsingData_localStorage.js
index 2a2425849030..d072153f807c 100644
--- a/browser/components/extensions/test/browser/browser_ext_browsingData_localStorage.js
+++ b/browser/components/extensions/test/browser/browser_ext_browsingData_localStorage.js
@@ -60,6 +60,9 @@ add_task(async function testLocalStorage() {
     await browser.browsingData.remove({}, {localStorage: true});
     await sendMessageToTabs(tabs, "checkLocalStorageCleared");
 
+    // Cleanup (checkLocalStorageCleared creates empty LS databases).
+    await browser.browsingData.removeLocalStorage({});
+
     browser.tabs.remove(tabs.map(tab => tab.id));
 
     browser.test.notifyPass("done");
diff --git a/dom/localstorage/ActorsChild.cpp b/dom/localstorage/ActorsChild.cpp
index 926711a53ab4..39eaf0499db3 100644
--- a/dom/localstorage/ActorsChild.cpp
+++ b/dom/localstorage/ActorsChild.cpp
@@ -318,5 +318,19 @@ LSSnapshotChild::ActorDestroy(ActorDestroyReason aWhy)
   }
 }
 
+mozilla::ipc::IPCResult
+LSSnapshotChild::RecvMarkDirty()
+{
+  AssertIsOnOwningThread();
+
+  if (!mSnapshot) {
+    return IPC_OK();
+  }
+
+  mSnapshot->MarkDirty();
+
+  return IPC_OK();
+}
+
 } // namespace dom
 } // namespace mozilla
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 700e61d94f37..0acbae7db39b 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -246,6 +246,9 @@ private:
   // IPDL methods are only called by IPDL.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
+
+  mozilla::ipc::IPCResult
+  RecvMarkDirty() override;
 };
 
 } // namespace dom
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 290d83f2c921..9ffba95b41d4 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1550,7 +1550,7 @@ public:
   void
   BeginUpdateBatch(int64_t aSnapshotInitialUsage);
 
-  void
+  int64_t
   EndUpdateBatch(int64_t aSnapshotPeakUsage);
 
   int64_t
@@ -1581,6 +1581,9 @@ private:
                   const nsAString& aOldValue,
                   bool aAffectsOrder);
 
+  void
+  MarkSnapshotsDirty();
+
   void
   NotifyObservers(Database* aDatabase,
                   const nsString& aDocumentURI,
@@ -1824,7 +1827,7 @@ class Snapshot final
   nsTArray<nsString> mKeys;
   nsString mDocumentURI;
   uint32_t mTotalLength;
-  int64_t mInitialUsage;
+  int64_t mUsage;
   int64_t mPeakUsage;
   bool mSavedKeys;
   bool mActorDestroyed;
@@ -1832,6 +1835,7 @@ class Snapshot final
   bool mLoadedReceived;
   bool mLoadedAllItems;
   bool mLoadKeysReceived;
+  bool mSentMarkDirty;
 
 public:
   // Created in AllocPBackgroundLSSnapshotParent.
@@ -1851,12 +1855,12 @@ public:
     MOZ_ASSERT_IF(aLoadState == LSSnapshot::LoadState::AllOrderedItems,
                   aLoadedItems.Count() == 0);
     MOZ_ASSERT(mTotalLength == 0);
-    MOZ_ASSERT(mInitialUsage == -1);
+    MOZ_ASSERT(mUsage == -1);
     MOZ_ASSERT(mPeakUsage == -1);
 
     mLoadedItems.SwapElements(aLoadedItems);
     mTotalLength = aTotalLength;
-    mInitialUsage = aInitialUsage;
+    mUsage = aInitialUsage;
     mPeakUsage = aPeakUsage;
     if (aLoadState == LSSnapshot::LoadState::AllOrderedKeys) {
       mLoadKeysReceived = true;
@@ -1872,12 +1876,18 @@ public:
            const nsAString& aOldValue,
            bool aAffectsOrder);
 
+  void
+  MarkDirty();
+
   NS_INLINE_DECL_REFCOUNTING(mozilla::dom::Snapshot)
 
 private:
   // Reference counted.
   ~Snapshot();
 
+  void
+  Finish();
+
   // IPDL methods are only called by IPDL.
   void
   ActorDestroy(ActorDestroyReason aWhy) override;
@@ -1886,7 +1896,10 @@ private:
   RecvDeleteMe() override;
 
   mozilla::ipc::IPCResult
-  RecvFinish(const LSSnapshotFinishInfo& aFinishInfo) override;
+  RecvCheckpoint(nsTArray<LSWriteInfo>&& aWriteInfos) override;
+
+  mozilla::ipc::IPCResult
+  RecvFinish() override;
 
   mozilla::ipc::IPCResult
   RecvLoaded() override;
@@ -4272,15 +4285,18 @@ Datastore::PrivateBrowsingClear()
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(mPrivateBrowsingId);
   MOZ_ASSERT(!mClosed);
+  MOZ_ASSERT(!mInUpdateBatch);
 
   if (mValues.Count()) {
-    DebugOnly<bool> ok = UpdateUsage(-mUsage);
-    MOZ_ASSERT(ok);
+    MarkSnapshotsDirty();
 
     mValues.Clear();
 
     mOrderedItems.Clear();
 
+    DebugOnly<bool> ok = UpdateUsage(-mSizeOfItems);
+    MOZ_ASSERT(ok);
+
     mSizeOfKeys = 0;
     mSizeOfItems = 0;
   }
@@ -4306,33 +4322,35 @@ Datastore::BeginUpdateBatch(int64_t aSnapshotInitialUsage)
 #endif
 }
 
-void
+int64_t
 Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(aSnapshotPeakUsage >= 0);
   MOZ_ASSERT(!mClosed);
   MOZ_ASSERT(mInUpdateBatch);
 
   mWriteOptimizer.ApplyWrites(mOrderedItems);
 
-  int64_t delta = mUpdateBatchUsage - aSnapshotPeakUsage;
+  if (aSnapshotPeakUsage >= 0) {
+    int64_t delta = mUpdateBatchUsage - aSnapshotPeakUsage;
 
-  if (mActiveDatabases.Count()) {
-    // We can't apply deltas while other databases are still active.
-    // The final delta must be zero or negative, but individual deltas can be
-    // positive. A positive delta can't be applied asynchronously since there's
-    // no way to fire the quota exceeded error event.
+    if (mActiveDatabases.Count()) {
+      // We can't apply deltas while other databases are still active.
+      // The final delta must be zero or negative, but individual deltas can
+      // be positive. A positive delta can't be applied asynchronously since
+      // there's no way to fire the quota exceeded error event.
 
-    mPendingUsageDeltas.AppendElement(delta);
-  } else {
-    MOZ_ASSERT(delta <= 0);
-    if (delta != 0) {
-      DebugOnly<bool> ok = UpdateUsage(delta);
-      MOZ_ASSERT(ok);
+      mPendingUsageDeltas.AppendElement(delta);
+    } else {
+      MOZ_ASSERT(delta <= 0);
+      if (delta != 0) {
+        DebugOnly<bool> ok = UpdateUsage(delta);
+        MOZ_ASSERT(ok);
+      }
     }
   }
 
+  int64_t result = mUpdateBatchUsage;
   mUpdateBatchUsage = -1;
 
   if (IsPersistent()) {
@@ -4342,6 +4360,8 @@ Datastore::EndUpdateBatch(int64_t aSnapshotPeakUsage)
 #ifdef DEBUG
   mInUpdateBatch = false;
 #endif
+
+  return result;
 }
 
 int64_t
@@ -4480,6 +4500,21 @@ Datastore::NotifySnapshots(Database* aDatabase,
   }
 }
 
+void
+Datastore::MarkSnapshotsDirty()
+{
+  AssertIsOnBackgroundThread();
+
+  for (auto iter = mDatabases.ConstIter(); !iter.Done(); iter.Next()) {
+    Database* database = iter.Get()->GetKey();
+
+    Snapshot* snapshot = database->GetSnapshot();
+    if (snapshot) {
+      snapshot->MarkDirty();
+    }
+  }
+}
+
 void
 Datastore::NotifyObservers(Database* aDatabase,
                            const nsString& aDocumentURI,
@@ -4799,7 +4834,7 @@ Snapshot::Snapshot(Database* aDatabase,
   , mDatastore(aDatabase->GetDatastore())
   , mDocumentURI(aDocumentURI)
   , mTotalLength(0)
-  , mInitialUsage(-1)
+  , mUsage(-1)
   , mPeakUsage(-1)
   , mSavedKeys(false)
   , mActorDestroyed(false)
@@ -4807,6 +4842,7 @@ Snapshot::Snapshot(Database* aDatabase,
   , mLoadedReceived(false)
   , mLoadedAllItems(false)
   , mLoadKeysReceived(false)
+  , mSentMarkDirty(false)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(aDatabase);
@@ -4825,6 +4861,8 @@ Snapshot::SaveItem(const nsAString& aKey,
 {
   AssertIsOnBackgroundThread();
 
+  MarkDirty();
+
   if (mLoadedAllItems) {
     return;
   }
@@ -4842,6 +4880,34 @@ Snapshot::SaveItem(const nsAString& aKey,
   }
 }
 
+void
+Snapshot::MarkDirty()
+{
+  AssertIsOnBackgroundThread();
+
+  if (!mSentMarkDirty) {
+    Unused << SendMarkDirty();
+    mSentMarkDirty = true;
+  }
+}
+
+void
+Snapshot::Finish()
+{
+  AssertIsOnBackgroundThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(mDatastore);
+  MOZ_ASSERT(!mFinishReceived);
+
+  mDatastore->BeginUpdateBatch(mUsage);
+
+  mDatastore->EndUpdateBatch(mPeakUsage);
+
+  mDatabase->UnregisterSnapshot(this);
+
+  mFinishReceived = true;
+}
+
 void
 Snapshot::ActorDestroy(ActorDestroyReason aWhy)
 {
@@ -4851,11 +4917,7 @@ Snapshot::ActorDestroy(ActorDestroyReason aWhy)
   mActorDestroyed = true;
 
   if (!mFinishReceived) {
-    mDatabase->UnregisterSnapshot(this);
-
-    mDatastore->BeginUpdateBatch(mInitialUsage);
-
-    mDatastore->EndUpdateBatch(mPeakUsage);
+    Finish();
   }
 }
 
@@ -4873,26 +4935,21 @@ Snapshot::RecvDeleteMe()
 }
 
 mozilla::ipc::IPCResult
-Snapshot::RecvFinish(const LSSnapshotFinishInfo& aFinishInfo)
+Snapshot::RecvCheckpoint(nsTArray<LSWriteInfo>&& aWriteInfos)
 {
   AssertIsOnBackgroundThread();
-  MOZ_ASSERT(mInitialUsage >= 0);
-  MOZ_ASSERT(mPeakUsage >= mInitialUsage);
+  MOZ_ASSERT(mUsage >= 0);
+  MOZ_ASSERT(mPeakUsage >= mUsage);
 
-  if (NS_WARN_IF(mFinishReceived)) {
+  if (NS_WARN_IF(aWriteInfos.IsEmpty())) {
     ASSERT_UNLESS_FUZZING();
     return IPC_FAIL_NO_REASON(this);
   }
 
-  mFinishReceived = true;
+  mDatastore->BeginUpdateBatch(mUsage);
 
-  mDatabase->UnregisterSnapshot(this);
-
-  mDatastore->BeginUpdateBatch(mInitialUsage);
-
-  const nsTArray<LSWriteInfo>& writeInfos = aFinishInfo.writeInfos();
-  for (uint32_t index = 0; index < writeInfos.Length(); index++) {
-    const LSWriteInfo& writeInfo = writeInfos[index];
+  for (uint32_t index = 0; index < aWriteInfos.Length(); index++) {
+    const LSWriteInfo& writeInfo = aWriteInfos[index];
     switch (writeInfo.type()) {
       case LSWriteInfo::TLSSetItemInfo: {
         const LSSetItemInfo& info = writeInfo.get_LSSetItemInfo();
@@ -4928,7 +4985,22 @@ Snapshot::RecvFinish(const LSSnapshotFinishInfo& aFinishInfo)
     }
   }
 
-  mDatastore->EndUpdateBatch(mPeakUsage);
+  mUsage = mDatastore->EndUpdateBatch(-1);
+
+  return IPC_OK();
+}
+
+mozilla::ipc::IPCResult
+Snapshot::RecvFinish()
+{
+  AssertIsOnBackgroundThread();
+
+  if (NS_WARN_IF(mFinishReceived)) {
+    ASSERT_UNLESS_FUZZING();
+    return IPC_FAIL_NO_REASON(this);
+  }
+
+  Finish();
 
   return IPC_OK();
 }
diff --git a/dom/localstorage/LSDatabase.cpp b/dom/localstorage/LSDatabase.cpp
index e643f8ef18f7..6464dca1736b 100644
--- a/dom/localstorage/LSDatabase.cpp
+++ b/dom/localstorage/LSDatabase.cpp
@@ -74,7 +74,9 @@ LSDatabase::RequestAllowToClose()
 
   mRequestedAllowToClose = true;
 
-  if (!mSnapshot) {
+  if (mSnapshot) {
+    mSnapshot->MarkDirty();
+  } else {
     AllowToClose();
   }
 }
@@ -285,7 +287,7 @@ LSDatabase::EndExplicitSnapshot(LSObject* aObject)
 
   MOZ_ASSERT(mSnapshot->Explicit());
 
-  nsresult rv = mSnapshot->Finish();
+  nsresult rv = mSnapshot->End();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
diff --git a/dom/localstorage/LSSnapshot.cpp b/dom/localstorage/LSSnapshot.cpp
index 62ebb6f8966d..882b68fde129 100644
--- a/dom/localstorage/LSSnapshot.cpp
+++ b/dom/localstorage/LSSnapshot.cpp
@@ -11,6 +11,12 @@
 namespace mozilla {
 namespace dom {
 
+namespace {
+
+const uint32_t kSnapshotTimeoutMs = 20000;
+
+} // namespace
+
 LSSnapshot::LSSnapshot(LSDatabase* aDatabase)
   : mDatabase(aDatabase)
   , mActor(nullptr)
@@ -20,6 +26,9 @@ LSSnapshot::LSSnapshot(LSDatabase* aDatabase)
   , mPeakUsage(0)
   , mLoadState(LoadState::Initial)
   , mExplicit(false)
+  , mHasPendingStableStateCallback(false)
+  , mHasPendingTimerCallback(false)
+  , mDirty(false)
 #ifdef DEBUG
   , mInitialized(false)
   , mSentFinish(false)
@@ -32,6 +41,8 @@ LSSnapshot::~LSSnapshot()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(!mHasPendingStableStateCallback);
+  MOZ_ASSERT(!mHasPendingTimerCallback);
   MOZ_ASSERT_IF(mInitialized, mSentFinish);
 
   if (mActor) {
@@ -55,17 +66,13 @@ LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo,
                  bool aExplicit)
 {
   AssertIsOnOwningThread();
+  MOZ_ASSERT(!mSelfRef);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(mLoadState == LoadState::Initial);
   MOZ_ASSERT(!mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
-  if (aExplicit) {
-    mSelfRef = this;
-  } else {
-    nsCOMPtr<nsIRunnable> runnable = this;
-    nsContentUtils::RunInStableState(runnable.forget());
-  }
+  mSelfRef = this;
 
   LoadState loadState = aInitInfo.loadState();
 
@@ -102,6 +109,13 @@ LSSnapshot::Init(const LSSnapshotInitInfo& aInitInfo,
   mInitialized = true;
 #endif
 
+  if (!mExplicit) {
+    mTimer = NS_NewTimer();
+    MOZ_ASSERT(mTimer);
+
+    ScheduleStableStateCallback();
+  }
+
   return NS_OK;
 }
 
@@ -113,6 +127,8 @@ LSSnapshot::GetLength(uint32_t* aResult)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   if (mLoadState == LoadState::Partial) {
     *aResult = mLength;
   } else {
@@ -131,6 +147,8 @@ LSSnapshot::GetKey(uint32_t aIndex,
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   nsresult rv = EnsureAllKeys();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -157,6 +175,8 @@ LSSnapshot::GetItem(const nsAString& aKey,
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   nsString result;
   nsresult rv = GetItemInternal(aKey, Optional<nsString>(), result);
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -175,6 +195,8 @@ LSSnapshot::GetKeys(nsTArray<nsString>& aKeys)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   nsresult rv = EnsureAllKeys();
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -197,6 +219,8 @@ LSSnapshot::SetItem(const nsAString& aKey,
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   nsString oldValue;
   nsresult rv =
     GetItemInternal(aKey, Optional<nsString>(nsString(aValue)), oldValue);
@@ -254,6 +278,8 @@ LSSnapshot::RemoveItem(const nsAString& aKey,
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   nsString oldValue;
   nsresult rv =
     GetItemInternal(aKey, Optional<nsString>(VoidString()), oldValue);
@@ -298,6 +324,8 @@ LSSnapshot::Clear(LSNotifyInfo& aNotifyInfo)
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
+  MaybeScheduleStableStateCallback();
+
   uint32_t length;
   if (mLoadState == LoadState::Partial) {
     length = mLength;
@@ -334,38 +362,89 @@ LSSnapshot::Clear(LSNotifyInfo& aNotifyInfo)
   return NS_OK;
 }
 
-nsresult
-LSSnapshot::Finish()
+void
+LSSnapshot::MarkDirty()
 {
   AssertIsOnOwningThread();
-  MOZ_ASSERT(mDatabase);
   MOZ_ASSERT(mActor);
   MOZ_ASSERT(mInitialized);
   MOZ_ASSERT(!mSentFinish);
 
-  MOZ_ALWAYS_TRUE(mActor->SendFinish(mWriteInfos));
-
-#ifdef DEBUG
-  mSentFinish = true;
-#endif
-
-  if (mExplicit) {
-    if (NS_WARN_IF(!mActor->SendPing())) {
-      return NS_ERROR_FAILURE;
-    }
+  if (mDirty) {
+    return;
   }
 
-  mDatabase->NoteFinishedSnapshot(this);
+  mDirty = true;
 
-  if (mExplicit) {
-    mSelfRef = nullptr;
+  if (!mExplicit && !mHasPendingStableStateCallback) {
+    CancelTimer();
+
+    MOZ_ALWAYS_SUCCEEDS(Checkpoint());
+
+    MOZ_ALWAYS_SUCCEEDS(Finish());
   } else {
-    MOZ_ASSERT(!mSelfRef);
+    MOZ_ASSERT(!mHasPendingTimerCallback);
+  }
+}
+
+nsresult
+LSSnapshot::End()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mExplicit);
+  MOZ_ASSERT(!mHasPendingStableStateCallback);
+  MOZ_ASSERT(!mHasPendingTimerCallback);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  nsresult rv = Checkpoint();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  RefPtr<LSSnapshot> kungFuDeathGrip = this;
+
+  rv = Finish();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!mActor->SendPing())) {
+    return NS_ERROR_FAILURE;
   }
 
   return NS_OK;
 }
 
+void
+LSSnapshot::ScheduleStableStateCallback()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mTimer);
+  MOZ_ASSERT(!mExplicit);
+  MOZ_ASSERT(!mHasPendingStableStateCallback);
+
+  CancelTimer();
+
+  nsCOMPtr<nsIRunnable> runnable = this;
+  nsContentUtils::RunInStableState(runnable.forget());
+
+  mHasPendingStableStateCallback = true;
+}
+
+void
+LSSnapshot::MaybeScheduleStableStateCallback()
+{
+  AssertIsOnOwningThread();
+
+  if (!mExplicit && !mHasPendingStableStateCallback) {
+    ScheduleStableStateCallback();
+  } else {
+    MOZ_ASSERT(!mHasPendingTimerCallback);
+  }
+}
+
 nsresult
 LSSnapshot::GetItemInternal(const nsAString& aKey,
                             const Optional<nsString>& aValue,
@@ -599,6 +678,77 @@ LSSnapshot::UpdateUsage(int64_t aDelta)
   return NS_OK;
 }
 
+nsresult
+LSSnapshot::Checkpoint()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  if (!mWriteInfos.IsEmpty()) {
+    MOZ_ALWAYS_TRUE(mActor->SendCheckpoint(mWriteInfos));
+
+    mWriteInfos.Clear();
+  }
+
+  return NS_OK;
+}
+
+nsresult
+LSSnapshot::Finish()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mDatabase);
+  MOZ_ASSERT(mActor);
+  MOZ_ASSERT(mInitialized);
+  MOZ_ASSERT(!mSentFinish);
+
+  MOZ_ALWAYS_TRUE(mActor->SendFinish());
+
+  mDatabase->NoteFinishedSnapshot(this);
+
+#ifdef DEBUG
+  mSentFinish = true;
+#endif
+
+  // Clear the self reference added in Init method.
+  MOZ_ASSERT(mSelfRef);
+  mSelfRef = nullptr;
+
+  return NS_OK;
+}
+
+void
+LSSnapshot::CancelTimer()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mTimer);
+
+  if (mHasPendingTimerCallback) {
+    MOZ_ALWAYS_SUCCEEDS(mTimer->Cancel());
+    mHasPendingTimerCallback = false;
+  }
+}
+
+// static
+void
+LSSnapshot::TimerCallback(nsITimer* aTimer, void* aClosure)
+{
+  MOZ_ASSERT(aTimer);
+
+  auto* self = static_cast<LSSnapshot*>(aClosure);
+  MOZ_ASSERT(self);
+  MOZ_ASSERT(self->mTimer);
+  MOZ_ASSERT(SameCOMIdentity(self->mTimer, aTimer));
+  MOZ_ASSERT(!self->mHasPendingStableStateCallback);
+  MOZ_ASSERT(self->mHasPendingTimerCallback);
+
+  self->mHasPendingTimerCallback = false;
+
+  MOZ_ALWAYS_SUCCEEDS(self->Finish());
+}
+
 NS_IMPL_ISUPPORTS(LSSnapshot, nsIRunnable)
 
 NS_IMETHODIMP
@@ -606,8 +756,27 @@ LSSnapshot::Run()
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!mExplicit);
+  MOZ_ASSERT(mHasPendingStableStateCallback);
+  MOZ_ASSERT(!mHasPendingTimerCallback);
 
-  MOZ_ALWAYS_SUCCEEDS(Finish());
+  mHasPendingStableStateCallback = false;
+
+  MOZ_ALWAYS_SUCCEEDS(Checkpoint());
+
+  if (mDirty || !Preferences::GetBool("dom.storage.snapshot_reusing")) {
+    MOZ_ALWAYS_SUCCEEDS(Finish());
+  } else if (!mExplicit) {
+    MOZ_ASSERT(mTimer);
+
+    MOZ_ALWAYS_SUCCEEDS(
+      mTimer->InitWithNamedFuncCallback(TimerCallback,
+                                        this,
+                                        kSnapshotTimeoutMs,
+                                        nsITimer::TYPE_ONE_SHOT,
+                                        "LSSnapshot::TimerCallback"));
+
+    mHasPendingTimerCallback = true;
+  }
 
   return NS_OK;
 }
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
index 99757407b0b7..5505700df244 100644
--- a/dom/localstorage/LSSnapshot.h
+++ b/dom/localstorage/LSSnapshot.h
@@ -35,6 +35,8 @@ private:
 
   RefPtr<LSDatabase> mDatabase;
 
+  nsCOMPtr<nsITimer> mTimer;
+
   LSSnapshotChild* mActor;
 
   nsTHashtable<nsStringHashKey> mLoadedItems;
@@ -50,6 +52,9 @@ private:
   LoadState mLoadState;
 
   bool mExplicit;
+  bool mHasPendingStableStateCallback;
+  bool mHasPendingTimerCallback;
+  bool mDirty;
 
 #ifdef DEBUG
   bool mInitialized;
@@ -113,12 +118,21 @@ public:
   nsresult
   Clear(LSNotifyInfo& aNotifyInfo);
 
+  void
+  MarkDirty();
+
   nsresult
-  Finish();
+  End();
 
 private:
   ~LSSnapshot();
 
+  void
+  ScheduleStableStateCallback();
+
+  void
+  MaybeScheduleStableStateCallback();
+
   nsresult
   GetItemInternal(const nsAString& aKey,
                   const Optional<nsString>& aValue,
@@ -130,6 +144,18 @@ private:
   nsresult
   UpdateUsage(int64_t aDelta);
 
+  nsresult
+  Checkpoint();
+
+  nsresult
+  Finish();
+
+  void
+  CancelTimer();
+
+  static void
+  TimerCallback(nsITimer* aTimer, void* aClosure);
+
   NS_DECL_ISUPPORTS
   NS_DECL_NSIRUNNABLE
 };
diff --git a/dom/localstorage/PBackgroundLSSnapshot.ipdl b/dom/localstorage/PBackgroundLSSnapshot.ipdl
index 75fa738563fd..316f741a0b93 100644
--- a/dom/localstorage/PBackgroundLSSnapshot.ipdl
+++ b/dom/localstorage/PBackgroundLSSnapshot.ipdl
@@ -32,11 +32,6 @@ union LSWriteInfo
   LSClearInfo;
 };
 
-struct LSSnapshotFinishInfo
-{
-  LSWriteInfo[] writeInfos;
-};
-
 sync protocol PBackgroundLSSnapshot
 {
   manager PBackgroundLSDatabase;
@@ -44,7 +39,9 @@ sync protocol PBackgroundLSSnapshot
 parent:
   async DeleteMe();
 
-  async Finish(LSSnapshotFinishInfo finishInfo);
+  async Checkpoint(LSWriteInfo[] writeInfos);
+
+  async Finish();
 
   async Loaded();
 
@@ -63,6 +60,8 @@ parent:
   sync Ping();
 
 child:
+  async MarkDirty();
+
   async __delete__();
 };
 
diff --git a/dom/localstorage/test/unit/test_eviction.js b/dom/localstorage/test/unit/test_eviction.js
index 0af547fd47e0..a33210411009 100644
--- a/dom/localstorage/test/unit/test_eviction.js
+++ b/dom/localstorage/test/unit/test_eviction.js
@@ -19,9 +19,10 @@ function* testSteps()
     return "http://example" + index + ".com";
   }
 
-  info("Setting pref");
+  info("Setting prefs");
 
   Services.prefs.setBoolPref("dom.storage.next_gen", true);
+  Services.prefs.setBoolPref("dom.storage.snapshot_reusing", false);
 
   info("Setting limits");
 
diff --git a/dom/localstorage/test/unit/test_groupLimit.js b/dom/localstorage/test/unit/test_groupLimit.js
index c9c140d7cc11..568486f6e914 100644
--- a/dom/localstorage/test/unit/test_groupLimit.js
+++ b/dom/localstorage/test/unit/test_groupLimit.js
@@ -26,6 +26,10 @@ function* testSteps()
   data.value = repeatChar(data.sizeKB * 1024 - data.key.length, ".");
   data.urlCount = groupLimitKB / data.sizeKB;
 
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.snapshot_reusing", false);
+
   info("Setting limits");
 
   setGlobalLimit(globalLimitKB);
diff --git a/dom/localstorage/test/unit/test_snapshotting.js b/dom/localstorage/test/unit/test_snapshotting.js
index 4fbd1bfab0a8..250d56b598f7 100644
--- a/dom/localstorage/test/unit/test_snapshotting.js
+++ b/dom/localstorage/test/unit/test_snapshotting.js
@@ -38,6 +38,10 @@ function* testSteps()
     -1,                  // full prefill
   ];
 
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.snapshot_reusing", false);
+
   for (let prefillValue of prefillValues) {
     info("Setting prefill value");
 
diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index 10a85120de55..54d43cc63846 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1293,6 +1293,7 @@ pref("dom.storage.next_gen", false);
 #endif
 pref("dom.storage.default_quota",      5120);
 pref("dom.storage.snapshot_prefill", 16384);
+pref("dom.storage.snapshot_reusing", true);
 pref("dom.storage.testing", false);
 
 pref("dom.send_after_paint_to_content", false);

From 1e027be676cf74d152a91a3f5ee8b73d2b5610ac Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:34 +0100
Subject: [PATCH 57/78] Bug 1286798 - Part 43: Coalesce database operations
 before they are applied to disk; r=asuth

This avoids persistence to disk in many cases since sites aften do setItem/removeItem for the same key in one JS function.
---
 dom/localstorage/ActorsParent.cpp | 147 +++++++++++++++++++++++++-----
 1 file changed, 124 insertions(+), 23 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 9ffba95b41d4..cb02276c1e46 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -1162,7 +1162,8 @@ public:
 
 private:
   class WriteInfo;
-  class SetItemInfo;
+  class AddItemInfo;
+  class UpdateItemInfo;
   class RemoveItemInfo;
   class ClearInfo;
   class EndUpdateBatchOp;
@@ -1171,7 +1172,8 @@ private:
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
   nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
-  nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
+  nsAutoPtr<WriteInfo> mClearInfo;
+  nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
   const nsCString mOrigin;
@@ -1208,9 +1210,13 @@ public:
   Close(nsIRunnable* aCallback);
 
   void
-  SetItem(const nsString& aKey,
+  AddItem(const nsString& aKey,
           const nsString& aValue);
 
+  void
+  UpdateItem(const nsString& aKey,
+             const nsString& aValue);
+
   void
   RemoveItem(const nsString& aKey);
 
@@ -1284,30 +1290,66 @@ private:
 class Connection::WriteInfo
 {
 public:
+  enum Type {
+    AddItem = 0,
+    UpdateItem,
+    RemoveItem,
+    Clear,
+  };
+
+  virtual Type
+  GetType() = 0;
+
   virtual nsresult
   Perform(Connection* aConnection) = 0;
 
   virtual ~WriteInfo() = default;
 };
 
-class Connection::SetItemInfo final
+class Connection::AddItemInfo
   : public WriteInfo
 {
   nsString mKey;
   nsString mValue;
 
 public:
-  SetItemInfo(const nsAString& aKey,
+  AddItemInfo(const nsAString& aKey,
               const nsAString& aValue)
     : mKey(aKey)
     , mValue(aValue)
   { }
 
 private:
+  Type
+  GetType() override
+  {
+    return AddItem;
+  }
+
   nsresult
   Perform(Connection* aConnection) override;
 };
 
+class Connection::UpdateItemInfo final
+  : public AddItemInfo
+{
+  nsString mKey;
+  nsString mValue;
+
+public:
+  UpdateItemInfo(const nsAString& aKey,
+                 const nsAString& aValue)
+    : AddItemInfo(aKey, aValue)
+  { }
+
+private:
+  Type
+  GetType() override
+  {
+    return UpdateItem;
+  }
+};
+
 class Connection::RemoveItemInfo final
   : public WriteInfo
 {
@@ -1319,6 +1361,12 @@ public:
   { }
 
 private:
+Type
+  GetType() override
+  {
+    return RemoveItem;
+  }
+
   nsresult
   Perform(Connection* aConnection) override;
 };
@@ -1331,6 +1379,12 @@ public:
   { }
 
 private:
+  Type
+  GetType() override
+  {
+    return Clear;
+  }
+
   nsresult
   Perform(Connection* aConnection) override;
 };
@@ -1338,12 +1392,15 @@ private:
 class Connection::EndUpdateBatchOp final
   : public ConnectionDatastoreOperationBase
 {
-  nsTArray<nsAutoPtr<WriteInfo>> mWriteInfos;
+  nsAutoPtr<WriteInfo> mClearInfo;
+  nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
 
 public:
-  explicit EndUpdateBatchOp(Connection* aConnection,
-                            nsTArray<nsAutoPtr<WriteInfo>>& aWriteInfos)
+  EndUpdateBatchOp(Connection* aConnection,
+                   nsAutoPtr<WriteInfo>&& aClearInfo,
+                   nsClassHashtable<nsStringHashKey, WriteInfo>& aWriteInfos)
     : ConnectionDatastoreOperationBase(aConnection)
+    , mClearInfo(std::move(aClearInfo))
   {
     mWriteInfos.SwapElements(aWriteInfos);
   }
@@ -3331,14 +3388,41 @@ Connection::Close(nsIRunnable* aCallback)
 }
 
 void
-Connection::SetItem(const nsString& aKey,
+Connection::AddItem(const nsString& aKey,
                     const nsString& aValue)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  nsAutoPtr<WriteInfo> writeInfo(new SetItemInfo(aKey, aValue));
-  mWriteInfos.AppendElement(writeInfo.forget());
+  WriteInfo* existingWriteInfo;
+  nsAutoPtr<WriteInfo> newWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::RemoveItem) {
+    newWriteInfo = new UpdateItemInfo(aKey, aValue);
+  } else {
+    newWriteInfo = new AddItemInfo(aKey, aValue);
+  }
+
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
+}
+
+void
+Connection::UpdateItem(const nsString& aKey,
+                       const nsString& aValue)
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mInUpdateBatch);
+
+  WriteInfo* existingWriteInfo;
+  nsAutoPtr<WriteInfo> newWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::AddItem) {
+    newWriteInfo = new AddItemInfo(aKey, aValue);
+  } else {
+    newWriteInfo = new UpdateItemInfo(aKey, aValue);
+  }
+
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
 }
 
 void
@@ -3347,8 +3431,15 @@ Connection::RemoveItem(const nsString& aKey)
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  nsAutoPtr<WriteInfo> writeInfo(new RemoveItemInfo(aKey));
-  mWriteInfos.AppendElement(writeInfo.forget());
+  WriteInfo* existingWriteInfo;
+  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
+      existingWriteInfo->GetType() == WriteInfo::AddItem) {
+    mWriteInfos.Remove(aKey);
+    return;
+  }
+
+  nsAutoPtr<WriteInfo> newWriteInfo(new RemoveItemInfo(aKey));
+  mWriteInfos.Put(aKey, newWriteInfo.forget());
 }
 
 void
@@ -3357,8 +3448,11 @@ Connection::Clear()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  nsAutoPtr<WriteInfo> writeInfo(new ClearInfo());
-  mWriteInfos.AppendElement(writeInfo.forget());
+  mWriteInfos.Clear();
+
+  if (!mClearInfo) {
+    mClearInfo = new ClearInfo();
+  }
 }
 
 void
@@ -3378,11 +3472,10 @@ Connection::EndUpdateBatch()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  if (!mWriteInfos.IsEmpty()) {
-    RefPtr<EndUpdateBatchOp> op = new EndUpdateBatchOp(this, mWriteInfos);
+  RefPtr<EndUpdateBatchOp> op =
+    new EndUpdateBatchOp(this, std::move(mClearInfo), mWriteInfos);
 
-    Dispatch(op);
-  }
+  Dispatch(op);
 
 #ifdef DEBUG
   mInUpdateBatch = false;
@@ -3509,7 +3602,7 @@ CachedStatement::Assign(Connection* aConnection,
 
 nsresult
 Connection::
-SetItemInfo::Perform(Connection* aConnection)
+AddItemInfo::Perform(Connection* aConnection)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -3709,8 +3802,12 @@ EndUpdateBatchOp::DoDatastoreWork()
     return rv;
   }
 
-  for (auto writeInfo : mWriteInfos) {
-    writeInfo->Perform(mConnection);
+  if (mClearInfo) {
+    mClearInfo->Perform(mConnection);
+  }
+
+  for (auto iter = mWriteInfos.ConstIter(); !iter.Done(); iter.Next()) {
+    iter.Data()->Perform(mConnection);
   }
 
   rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("COMMIT;"), &stmt);
@@ -4193,7 +4290,11 @@ Datastore::SetItem(Database* aDatabase,
     }
 
     if (IsPersistent()) {
-      mConnection->SetItem(aKey, aValue);
+      if (oldValue.IsVoid()) {
+        mConnection->AddItem(aKey, aValue);
+      } else {
+        mConnection->UpdateItem(aKey, aValue);
+      }
     }
   }
 

From cf023d79b4e41c40888cde6a0d0e16a57b29aea9 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:37 +0100
Subject: [PATCH 58/78] Bug 1286798 - Part 44: Switch Connection to use
 WriteOptimizer too; r=asuth

This eliminates some code duplication.
---
 dom/localstorage/ActorsParent.cpp | 590 +++++++++++++-----------------
 1 file changed, 256 insertions(+), 334 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index cb02276c1e46..21188051994f 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -895,15 +895,23 @@ class WriteOptimizer final
   class AddItemInfo;
   class UpdateItemInfo;
   class RemoveItemInfo;
+  class ClearInfo;
 
+  nsAutoPtr<WriteInfo> mClearInfo;
   nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
-  bool mClearedWriteInfos;
 
 public:
   WriteOptimizer()
-    : mClearedWriteInfos(false)
   { }
 
+  WriteOptimizer(WriteOptimizer&& aWriteOptimizer)
+    : mClearInfo(std::move(aWriteOptimizer.mClearInfo))
+  {
+    MOZ_ASSERT(&aWriteOptimizer != this);
+
+    mWriteInfos.SwapElements(aWriteOptimizer.mWriteInfos);
+  }
+
   void
   AddItem(const nsString& aKey,
           const nsString& aValue);
@@ -920,6 +928,9 @@ public:
 
   void
   ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems);
+
+  nsresult
+  PerformWrites(Connection* aConnection);
 };
 
 class WriteOptimizer::WriteInfo
@@ -928,12 +939,16 @@ public:
   enum Type {
     AddItem = 0,
     UpdateItem,
-    RemoveItem
+    RemoveItem,
+    Clear
   };
 
   virtual Type
   GetType() = 0;
 
+  virtual nsresult
+  Perform(Connection* aConnection) = 0;
+
   virtual ~WriteInfo() = default;
 };
 
@@ -968,6 +983,9 @@ private:
   {
     return AddItem;
   }
+
+  nsresult
+  Perform(Connection* aConnection) override;
 };
 
 class WriteOptimizer::UpdateItemInfo final
@@ -1009,6 +1027,27 @@ private:
   {
     return RemoveItem;
   }
+
+  nsresult
+  Perform(Connection* aConnection) override;
+};
+
+class WriteOptimizer::ClearInfo final
+  : public WriteInfo
+{
+public:
+  ClearInfo()
+  { }
+
+private:
+  Type
+  GetType() override
+  {
+    return Clear;
+  }
+
+  nsresult
+  Perform(Connection* aConnection) override;
 };
 
 class DatastoreOperationBase
@@ -1161,21 +1200,15 @@ public:
   class CachedStatement;
 
 private:
-  class WriteInfo;
-  class AddItemInfo;
-  class UpdateItemInfo;
-  class RemoveItemInfo;
-  class ClearInfo;
   class EndUpdateBatchOp;
   class CloseOp;
 
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
   nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
-  nsAutoPtr<WriteInfo> mClearInfo;
-  nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
+  WriteOptimizer mWriteOptimizer;
   const nsCString mOrigin;
   const nsString mFilePath;
 #ifdef DEBUG
@@ -1287,123 +1320,17 @@ private:
   CachedStatement& operator=(const CachedStatement&) = delete;
 };
 
-class Connection::WriteInfo
-{
-public:
-  enum Type {
-    AddItem = 0,
-    UpdateItem,
-    RemoveItem,
-    Clear,
-  };
-
-  virtual Type
-  GetType() = 0;
-
-  virtual nsresult
-  Perform(Connection* aConnection) = 0;
-
-  virtual ~WriteInfo() = default;
-};
-
-class Connection::AddItemInfo
-  : public WriteInfo
-{
-  nsString mKey;
-  nsString mValue;
-
-public:
-  AddItemInfo(const nsAString& aKey,
-              const nsAString& aValue)
-    : mKey(aKey)
-    , mValue(aValue)
-  { }
-
-private:
-  Type
-  GetType() override
-  {
-    return AddItem;
-  }
-
-  nsresult
-  Perform(Connection* aConnection) override;
-};
-
-class Connection::UpdateItemInfo final
-  : public AddItemInfo
-{
-  nsString mKey;
-  nsString mValue;
-
-public:
-  UpdateItemInfo(const nsAString& aKey,
-                 const nsAString& aValue)
-    : AddItemInfo(aKey, aValue)
-  { }
-
-private:
-  Type
-  GetType() override
-  {
-    return UpdateItem;
-  }
-};
-
-class Connection::RemoveItemInfo final
-  : public WriteInfo
-{
-  nsString mKey;
-
-public:
-  explicit RemoveItemInfo(const nsAString& aKey)
-    : mKey(aKey)
-  { }
-
-private:
-Type
-  GetType() override
-  {
-    return RemoveItem;
-  }
-
-  nsresult
-  Perform(Connection* aConnection) override;
-};
-
-class Connection::ClearInfo final
-  : public WriteInfo
-{
-public:
-  ClearInfo()
-  { }
-
-private:
-  Type
-  GetType() override
-  {
-    return Clear;
-  }
-
-  nsresult
-  Perform(Connection* aConnection) override;
-};
-
 class Connection::EndUpdateBatchOp final
   : public ConnectionDatastoreOperationBase
 {
-  nsAutoPtr<WriteInfo> mClearInfo;
-  nsClassHashtable<nsStringHashKey, WriteInfo> mWriteInfos;
+  WriteOptimizer mWriteOptimizer;
 
 public:
   EndUpdateBatchOp(Connection* aConnection,
-                   nsAutoPtr<WriteInfo>&& aClearInfo,
-                   nsClassHashtable<nsStringHashKey, WriteInfo>& aWriteInfos)
+                   WriteOptimizer&& aWriteOptimizer)
     : ConnectionDatastoreOperationBase(aConnection)
-    , mClearInfo(std::move(aClearInfo))
-  {
-    mWriteInfos.SwapElements(aWriteInfos);
-  }
+    , mWriteOptimizer(std::move(aWriteOptimizer))
+  { }
 
 private:
   nsresult
@@ -3178,15 +3105,18 @@ WriteOptimizer::Clear()
   AssertIsOnBackgroundThread();
 
   mWriteInfos.Clear();
-  mClearedWriteInfos = true;
+
+  if (!mClearInfo) {
+    mClearInfo = new ClearInfo();
+  }
 }
 
 void
 WriteOptimizer::ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems)
 {
-  if (mClearedWriteInfos) {
+  if (mClearInfo) {
     aOrderedItems.Clear();
-    mClearedWriteInfos = false;
+    mClearInfo = nullptr;
   }
 
   for (int32_t index = aOrderedItems.Length() - 1;
@@ -3234,6 +3164,201 @@ WriteOptimizer::ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems)
   mWriteInfos.Clear();
 }
 
+nsresult
+WriteOptimizer::PerformWrites(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  nsresult rv;
+
+  if (mClearInfo) {
+    rv = mClearInfo->Perform(aConnection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  for (auto iter = mWriteInfos.ConstIter(); !iter.Done(); iter.Next()) {
+    rv = iter.Data()->Perform(aConnection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  return NS_OK;
+}
+
+nsresult
+WriteOptimizer::
+AddItemInfo::Perform(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO data (key, value) "
+    "VALUES(:key, :value)"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "INSERT OR REPLACE INTO shadow.webappsstore2 "
+      "(originAttributes, originKey, scope, key, value) "
+      "VALUES (:originAttributes, :originKey, :scope, :key, :value) "),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  ArchivedOriginInfo* archivedOriginInfo = aConnection->GetArchivedOriginInfo();
+
+  rv = archivedOriginInfo->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCString scope = Scheme0Scope(archivedOriginInfo->OriginSuffix(),
+                                 archivedOriginInfo->OriginNoSuffix());
+
+  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
+                                  scope);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+WriteOptimizer::
+RemoveItemInfo::Perform(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data "
+      "WHERE key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey "
+      "AND key = :key;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
+nsresult
+WriteOptimizer::
+ClearInfo::Perform(Connection* aConnection)
+{
+  AssertIsOnConnectionThread();
+  MOZ_ASSERT(aConnection);
+
+  Connection::CachedStatement stmt;
+  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM data;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM shadow.webappsstore2 "
+      "WHERE originAttributes = :originAttributes "
+      "AND originKey = :originKey;"),
+    &stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 /*******************************************************************************
  * DatastoreOperationBase
  ******************************************************************************/
@@ -3394,16 +3519,7 @@ Connection::AddItem(const nsString& aKey,
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  WriteInfo* existingWriteInfo;
-  nsAutoPtr<WriteInfo> newWriteInfo;
-  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
-      existingWriteInfo->GetType() == WriteInfo::RemoveItem) {
-    newWriteInfo = new UpdateItemInfo(aKey, aValue);
-  } else {
-    newWriteInfo = new AddItemInfo(aKey, aValue);
-  }
-
-  mWriteInfos.Put(aKey, newWriteInfo.forget());
+  mWriteOptimizer.AddItem(aKey, aValue);
 }
 
 void
@@ -3413,16 +3529,7 @@ Connection::UpdateItem(const nsString& aKey,
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  WriteInfo* existingWriteInfo;
-  nsAutoPtr<WriteInfo> newWriteInfo;
-  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
-      existingWriteInfo->GetType() == WriteInfo::AddItem) {
-    newWriteInfo = new AddItemInfo(aKey, aValue);
-  } else {
-    newWriteInfo = new UpdateItemInfo(aKey, aValue);
-  }
-
-  mWriteInfos.Put(aKey, newWriteInfo.forget());
+  mWriteOptimizer.UpdateItem(aKey, aValue);
 }
 
 void
@@ -3431,15 +3538,7 @@ Connection::RemoveItem(const nsString& aKey)
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  WriteInfo* existingWriteInfo;
-  if (mWriteInfos.Get(aKey, &existingWriteInfo) &&
-      existingWriteInfo->GetType() == WriteInfo::AddItem) {
-    mWriteInfos.Remove(aKey);
-    return;
-  }
-
-  nsAutoPtr<WriteInfo> newWriteInfo(new RemoveItemInfo(aKey));
-  mWriteInfos.Put(aKey, newWriteInfo.forget());
+  mWriteOptimizer.RemoveItem(aKey);
 }
 
 void
@@ -3448,11 +3547,7 @@ Connection::Clear()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  mWriteInfos.Clear();
-
-  if (!mClearInfo) {
-    mClearInfo = new ClearInfo();
-  }
+  mWriteOptimizer.Clear();
 }
 
 void
@@ -3473,7 +3568,7 @@ Connection::EndUpdateBatch()
   MOZ_ASSERT(mInUpdateBatch);
 
   RefPtr<EndUpdateBatchOp> op =
-    new EndUpdateBatchOp(this, std::move(mClearInfo), mWriteInfos);
+    new EndUpdateBatchOp(this, std::move(mWriteOptimizer));
 
   Dispatch(op);
 
@@ -3600,176 +3695,6 @@ CachedStatement::Assign(Connection* aConnection,
   }
 }
 
-nsresult
-Connection::
-AddItemInfo::Perform(Connection* aConnection)
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(aConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "INSERT OR REPLACE INTO data (key, value) "
-    "VALUES(:key, :value)"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "INSERT OR REPLACE INTO shadow.webappsstore2 "
-      "(originAttributes, originKey, scope, key, value) "
-      "VALUES (:originAttributes, :originKey, :scope, :key, :value) "),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  ArchivedOriginInfo* archivedOriginInfo = aConnection->GetArchivedOriginInfo();
-
-  rv = archivedOriginInfo->BindToStatement(stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  nsCString scope = Scheme0Scope(archivedOriginInfo->OriginSuffix(),
-                                 archivedOriginInfo->OriginNoSuffix());
-
-  rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
-                                  scope);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("value"), mValue);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
-nsresult
-Connection::
-RemoveItemInfo::Perform(Connection* aConnection)
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(aConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM data "
-      "WHERE key = :key;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM shadow.webappsstore2 "
-      "WHERE originAttributes = :originAttributes "
-      "AND originKey = :originKey "
-      "AND key = :key;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->BindStringByName(NS_LITERAL_CSTRING("key"), mKey);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
-nsresult
-Connection::
-ClearInfo::Perform(Connection* aConnection)
-{
-  AssertIsOnConnectionThread();
-  MOZ_ASSERT(aConnection);
-
-  Connection::CachedStatement stmt;
-  nsresult rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM data;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
-    "DELETE FROM shadow.webappsstore2 "
-      "WHERE originAttributes = :originAttributes "
-      "AND originKey = :originKey;"),
-    &stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  rv = stmt->Execute();
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
-  }
-
-  return NS_OK;
-}
-
 nsresult
 Connection::
 EndUpdateBatchOp::DoDatastoreWork()
@@ -3802,12 +3727,9 @@ EndUpdateBatchOp::DoDatastoreWork()
     return rv;
   }
 
-  if (mClearInfo) {
-    mClearInfo->Perform(mConnection);
-  }
-
-  for (auto iter = mWriteInfos.ConstIter(); !iter.Done(); iter.Next()) {
-    iter.Data()->Perform(mConnection);
+  rv = mWriteOptimizer.PerformWrites(mConnection);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
   }
 
   rv = mConnection->GetCachedStatement(NS_LITERAL_CSTRING("COMMIT;"), &stmt);

From 3f301fd6c01cc82cfe62ef3d140bcf43cbc06368 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:40 +0100
Subject: [PATCH 59/78] Bug 1286798 - Part 45: Delay flushing to disk using a
 timer; r=asuth

This improves performance even more by grouping database operations from multiple checkpoints and possibly from multiple processes.
---
 dom/localstorage/ActorsParent.cpp | 102 +++++++++++++++++++++++++++---
 1 file changed, 93 insertions(+), 9 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 21188051994f..454f56e17eb7 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -114,6 +114,8 @@ static_assert(kSQLiteGrowthIncrement >= 0 &&
 #define DATA_FILE_NAME "data.sqlite"
 #define JOURNAL_FILE_NAME "data.sqlite-journal"
 
+const uint32_t kFlushTimeoutMs = 5000;
+
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
@@ -907,6 +909,7 @@ public:
   WriteOptimizer(WriteOptimizer&& aWriteOptimizer)
     : mClearInfo(std::move(aWriteOptimizer.mClearInfo))
   {
+    AssertIsOnBackgroundThread();
     MOZ_ASSERT(&aWriteOptimizer != this);
 
     mWriteInfos.SwapElements(aWriteOptimizer.mWriteInfos);
@@ -926,6 +929,14 @@ public:
   void
   Clear();
 
+  bool
+  HasWrites() const
+  {
+    AssertIsOnBackgroundThread();
+
+    return mClearInfo || !mWriteInfos.IsEmpty();
+  }
+
   void
   ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems);
 
@@ -1200,10 +1211,11 @@ public:
   class CachedStatement;
 
 private:
-  class EndUpdateBatchOp;
+  class FlushOp;
   class CloseOp;
 
   RefPtr<ConnectionThread> mConnectionThread;
+  nsCOMPtr<nsITimer> mFlushTimer;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
   nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
@@ -1211,6 +1223,7 @@ private:
   WriteOptimizer mWriteOptimizer;
   const nsCString mOrigin;
   const nsString mFilePath;
+  bool mFlushScheduled;
 #ifdef DEBUG
   bool mInUpdateBatch;
 #endif
@@ -1291,6 +1304,15 @@ private:
              nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
 
   ~Connection();
+
+  void
+  ScheduleFlush();
+
+  void
+  Flush();
+
+  static void
+  FlushTimerCallback(nsITimer* aTimer, void* aClosure);
 };
 
 class Connection::CachedStatement final
@@ -1320,14 +1342,14 @@ private:
   CachedStatement& operator=(const CachedStatement&) = delete;
 };
 
-class Connection::EndUpdateBatchOp final
+class Connection::FlushOp final
   : public ConnectionDatastoreOperationBase
 {
   WriteOptimizer mWriteOptimizer;
 
 public:
-  EndUpdateBatchOp(Connection* aConnection,
-                   WriteOptimizer&& aWriteOptimizer)
+  FlushOp(Connection* aConnection,
+          WriteOptimizer&& aWriteOptimizer)
     : ConnectionDatastoreOperationBase(aConnection)
     , mWriteOptimizer(std::move(aWriteOptimizer))
   { }
@@ -3114,6 +3136,8 @@ WriteOptimizer::Clear()
 void
 WriteOptimizer::ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems)
 {
+  AssertIsOnBackgroundThread();
+
   if (mClearInfo) {
     aOrderedItems.Clear();
     mClearInfo = nullptr;
@@ -3474,6 +3498,7 @@ Connection::Connection(ConnectionThread* aConnectionThread,
   , mArchivedOriginInfo(std::move(aArchivedOriginInfo))
   , mOrigin(aOrigin)
   , mFilePath(aFilePath)
+  , mFlushScheduled(false)
 #ifdef DEBUG
   , mInUpdateBatch(false)
 #endif
@@ -3489,6 +3514,7 @@ Connection::~Connection()
   MOZ_ASSERT(!mStorageConnection);
   MOZ_ASSERT(!mCachedStatements.Count());
   MOZ_ASSERT(!mInUpdateBatch);
+  MOZ_ASSERT(!mFlushScheduled);
 }
 
 void
@@ -3507,6 +3533,15 @@ Connection::Close(nsIRunnable* aCallback)
   AssertIsOnOwningThread();
   MOZ_ASSERT(aCallback);
 
+  if (mFlushScheduled) {
+    MOZ_ASSERT(mFlushTimer);
+    MOZ_ALWAYS_SUCCEEDS(mFlushTimer->Cancel());
+
+    Flush();
+
+    mFlushTimer = nullptr;
+  }
+
   RefPtr<CloseOp> op = new CloseOp(this, aCallback);
 
   Dispatch(op);
@@ -3567,10 +3602,9 @@ Connection::EndUpdateBatch()
   AssertIsOnOwningThread();
   MOZ_ASSERT(mInUpdateBatch);
 
-  RefPtr<EndUpdateBatchOp> op =
-    new EndUpdateBatchOp(this, std::move(mWriteOptimizer));
-
-  Dispatch(op);
+  if (mWriteOptimizer.HasWrites() && !mFlushScheduled)  {
+    ScheduleFlush();
+  }
 
 #ifdef DEBUG
   mInUpdateBatch = false;
@@ -3645,6 +3679,56 @@ Connection::GetCachedStatement(const nsACString& aQuery,
   return NS_OK;
 }
 
+void
+Connection::ScheduleFlush()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mWriteOptimizer.HasWrites());
+  MOZ_ASSERT(!mFlushScheduled);
+
+  if (!mFlushTimer) {
+    mFlushTimer = NS_NewTimer();
+    MOZ_ASSERT(mFlushTimer);
+  }
+
+  MOZ_ALWAYS_SUCCEEDS(
+    mFlushTimer->InitWithNamedFuncCallback(FlushTimerCallback,
+                                           this,
+                                           kFlushTimeoutMs,
+                                           nsITimer::TYPE_ONE_SHOT,
+                                           "Connection::FlushTimerCallback"));
+
+  mFlushScheduled = true;
+}
+
+void
+Connection::Flush()
+{
+  AssertIsOnOwningThread();
+  MOZ_ASSERT(mFlushScheduled);
+
+  if (mWriteOptimizer.HasWrites()) {
+    RefPtr<FlushOp> op = new FlushOp(this, std::move(mWriteOptimizer));
+
+    Dispatch(op);
+  }
+
+  mFlushScheduled = false;
+}
+
+// static
+void
+Connection::FlushTimerCallback(nsITimer* aTimer, void* aClosure)
+{
+  MOZ_ASSERT(aClosure);
+
+  auto* self = static_cast<Connection*>(aClosure);
+  MOZ_ASSERT(self);
+  MOZ_ASSERT(self->mFlushScheduled);
+
+  self->Flush();
+}
+
 Connection::
 CachedStatement::CachedStatement()
 {
@@ -3697,7 +3781,7 @@ CachedStatement::Assign(Connection* aConnection,
 
 nsresult
 Connection::
-EndUpdateBatchOp::DoDatastoreWork()
+FlushOp::DoDatastoreWork()
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(mConnection);

From 03fc857fab5990b79c5a95d34e186dc22443a83a Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:43 +0100
Subject: [PATCH 60/78] Bug 1286798 - Part 46: Add a pref for database
 shadowing; r=asuth

---
 dom/localstorage/ActorsParent.cpp | 86 ++++++++++++++++++++++---------
 modules/libpref/init/all.js       |  1 +
 2 files changed, 64 insertions(+), 23 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 454f56e17eb7..94e8ff4048dc 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -119,8 +119,10 @@ const uint32_t kFlushTimeoutMs = 5000;
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
+const uint32_t kDefaultShadowWrites = true;
 const uint32_t kDefaultSnapshotPrefill = 4096;
 const char kDefaultQuotaPref[] = "dom.storage.default_quota";
+const char kShadowWritesPref[] = "dom.storage.shadow_writes";
 const char kSnapshotPrefillPref[] = "dom.storage.snapshot_prefill";
 
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
@@ -941,7 +943,7 @@ public:
   ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems);
 
   nsresult
-  PerformWrites(Connection* aConnection);
+  PerformWrites(Connection* aConnection, bool aShadowWrites);
 };
 
 class WriteOptimizer::WriteInfo
@@ -958,7 +960,7 @@ public:
   GetType() = 0;
 
   virtual nsresult
-  Perform(Connection* aConnection) = 0;
+  Perform(Connection* aConnection, bool aShadowWrites) = 0;
 
   virtual ~WriteInfo() = default;
 };
@@ -996,7 +998,7 @@ private:
   }
 
   nsresult
-  Perform(Connection* aConnection) override;
+  Perform(Connection* aConnection, bool aShadowWrites) override;
 };
 
 class WriteOptimizer::UpdateItemInfo final
@@ -1040,7 +1042,7 @@ private:
   }
 
   nsresult
-  Perform(Connection* aConnection) override;
+  Perform(Connection* aConnection, bool aShadowWrites) override;
 };
 
 class WriteOptimizer::ClearInfo final
@@ -1058,7 +1060,7 @@ private:
   }
 
   nsresult
-  Perform(Connection* aConnection) override;
+  Perform(Connection* aConnection, bool aShadowWrites) override;
 };
 
 class DatastoreOperationBase
@@ -1346,13 +1348,11 @@ class Connection::FlushOp final
   : public ConnectionDatastoreOperationBase
 {
   WriteOptimizer mWriteOptimizer;
+  bool mShadowWrites;
 
 public:
   FlushOp(Connection* aConnection,
-          WriteOptimizer&& aWriteOptimizer)
-    : ConnectionDatastoreOperationBase(aConnection)
-    , mWriteOptimizer(std::move(aWriteOptimizer))
-  { }
+          WriteOptimizer&& aWriteOptimizer);
 
 private:
   nsresult
@@ -2577,6 +2577,7 @@ typedef nsClassHashtable<nsCStringHashKey, nsTArray<Observer*>>
 StaticAutoPtr<ObserverHashtable> gObservers;
 
 Atomic<uint32_t, Relaxed> gOriginLimitKB(kDefaultOriginLimitKB);
+Atomic<bool> gShadowWrites(kDefaultShadowWrites);
 Atomic<int32_t, Relaxed> gSnapshotPrefill(kDefaultSnapshotPrefill);
 
 typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
@@ -2729,6 +2730,16 @@ GetUsage(mozIStorageConnection* aConnection,
   return NS_OK;
 }
 
+void
+ShadowWritesPrefChangedCallback(const char* aPrefName, void* aClosure)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(!strcmp(aPrefName, kShadowWritesPref));
+  MOZ_ASSERT(!aClosure);
+
+  gShadowWrites = Preferences::GetBool(aPrefName, kDefaultShadowWrites);
+}
+
 void
 SnapshotPrefillPrefChangedCallback(const char* aPrefName, void* aClosure)
 {
@@ -3189,7 +3200,7 @@ WriteOptimizer::ApplyWrites(nsTArray<LSItemInfo>& aOrderedItems)
 }
 
 nsresult
-WriteOptimizer::PerformWrites(Connection* aConnection)
+WriteOptimizer::PerformWrites(Connection* aConnection, bool aShadowWrites)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -3197,14 +3208,14 @@ WriteOptimizer::PerformWrites(Connection* aConnection)
   nsresult rv;
 
   if (mClearInfo) {
-    rv = mClearInfo->Perform(aConnection);
+    rv = mClearInfo->Perform(aConnection, aShadowWrites);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
   }
 
   for (auto iter = mWriteInfos.ConstIter(); !iter.Done(); iter.Next()) {
-    rv = iter.Data()->Perform(aConnection);
+    rv = iter.Data()->Perform(aConnection, aShadowWrites);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -3215,7 +3226,7 @@ WriteOptimizer::PerformWrites(Connection* aConnection)
 
 nsresult
 WriteOptimizer::
-AddItemInfo::Perform(Connection* aConnection)
+AddItemInfo::Perform(Connection* aConnection, bool aShadowWrites)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -3244,6 +3255,10 @@ AddItemInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  if (!aShadowWrites) {
+    return NS_OK;
+  }
+
   rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
     "INSERT OR REPLACE INTO shadow.webappsstore2 "
       "(originAttributes, originKey, scope, key, value) "
@@ -3289,7 +3304,7 @@ AddItemInfo::Perform(Connection* aConnection)
 
 nsresult
 WriteOptimizer::
-RemoveItemInfo::Perform(Connection* aConnection)
+RemoveItemInfo::Perform(Connection* aConnection, bool aShadowWrites)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -3313,6 +3328,10 @@ RemoveItemInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  if (!aShadowWrites) {
+    return NS_OK;
+  }
+
   rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
     "DELETE FROM shadow.webappsstore2 "
       "WHERE originAttributes = :originAttributes "
@@ -3343,7 +3362,7 @@ RemoveItemInfo::Perform(Connection* aConnection)
 
 nsresult
 WriteOptimizer::
-ClearInfo::Perform(Connection* aConnection)
+ClearInfo::Perform(Connection* aConnection, bool aShadowWrites)
 {
   AssertIsOnConnectionThread();
   MOZ_ASSERT(aConnection);
@@ -3361,6 +3380,10 @@ ClearInfo::Perform(Connection* aConnection)
     return rv;
   }
 
+  if (!aShadowWrites) {
+    return NS_OK;
+  }
+
   rv = aConnection->GetCachedStatement(NS_LITERAL_CSTRING(
     "DELETE FROM shadow.webappsstore2 "
       "WHERE originAttributes = :originAttributes "
@@ -3779,6 +3802,15 @@ CachedStatement::Assign(Connection* aConnection,
   }
 }
 
+Connection::
+FlushOp::FlushOp(Connection* aConnection,
+                 WriteOptimizer&& aWriteOptimizer)
+  : ConnectionDatastoreOperationBase(aConnection)
+  , mWriteOptimizer(std::move(aWriteOptimizer))
+  , mShadowWrites(gShadowWrites)
+{
+}
+
 nsresult
 Connection::
 FlushOp::DoDatastoreWork()
@@ -3793,10 +3825,13 @@ FlushOp::DoDatastoreWork()
     mConnection->StorageConnection();
   MOZ_ASSERT(storageConnection);
 
-  nsresult rv = AttachShadowDatabase(quotaManager->GetBasePath(),
-                                     storageConnection);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  nsresult rv;
+
+  if (mShadowWrites) {
+    rv = AttachShadowDatabase(quotaManager->GetBasePath(), storageConnection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
   }
 
   CachedStatement stmt;
@@ -3811,7 +3846,7 @@ FlushOp::DoDatastoreWork()
     return rv;
   }
 
-  rv = mWriteOptimizer.PerformWrites(mConnection);
+  rv = mWriteOptimizer.PerformWrites(mConnection, mShadowWrites);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3826,9 +3861,11 @@ FlushOp::DoDatastoreWork()
     return rv;
   }
 
-  rv = DetachShadowDatabase(storageConnection);
-  if (NS_WARN_IF(NS_FAILED(rv))) {
-    return rv;
+  if (mShadowWrites) {
+    rv = DetachShadowDatabase(storageConnection);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
   }
 
   return NS_OK;
@@ -6979,6 +7016,9 @@ QuotaClient::RegisterObservers(nsIEventTarget* aBackgroundEventTarget)
       NS_WARNING("Unable to respond to default quota pref changes!");
     }
 
+    Preferences::RegisterCallbackAndCall(ShadowWritesPrefChangedCallback,
+                                         kShadowWritesPref);
+
     Preferences::RegisterCallbackAndCall(SnapshotPrefillPrefChangedCallback,
                                          kSnapshotPrefillPref);
 
diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index 54d43cc63846..db646db121db 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1292,6 +1292,7 @@ pref("dom.storage.next_gen", true);
 pref("dom.storage.next_gen", false);
 #endif
 pref("dom.storage.default_quota",      5120);
+pref("dom.storage.shadow_writes", true);
 pref("dom.storage.snapshot_prefill", 16384);
 pref("dom.storage.snapshot_reusing", true);
 pref("dom.storage.testing", false);

From 9c1d177a11338e3cb940ae484b534e575de500ac Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:46 +0100
Subject: [PATCH 61/78] Bug 1286798 - Part 47: Add AboutToClearOrigins() method
 to the quota client interface; r=asuth

---
 dom/quota/ActorsParent.cpp | 55 +++++++++++++++++++++++++++++++++++---
 dom/quota/Client.h         | 10 +++++++
 dom/quota/QuotaManager.h   |  5 ++++
 3 files changed, 66 insertions(+), 4 deletions(-)

diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index b56e47f8fbc0..876c2272e8cb 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -5800,6 +5800,35 @@ QuotaManager::EnsureOriginDirectory(nsIFile* aDirectory,
   return NS_OK;
 }
 
+nsresult
+QuotaManager::AboutToClearOrigins(
+                              const Nullable<PersistenceType>& aPersistenceType,
+                              const OriginScope& aOriginScope,
+                              const Nullable<Client::Type>& aClientType)
+{
+  AssertIsOnIOThread();
+
+  nsresult rv;
+
+  if (aClientType.IsNull()) {
+    for (uint32_t index = 0; index < uint32_t(Client::TypeMax()); index++) {
+      rv = mClients[index]->AboutToClearOrigins(aPersistenceType,
+                                                aOriginScope);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+  } else {
+    rv = mClients[aClientType.Value()]->AboutToClearOrigins(aPersistenceType,
+                                                            aOriginScope);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  return NS_OK;
+}
+
 void
 QuotaManager::OriginClearCompleted(PersistenceType aPersistenceType,
                                    const nsACString& aOrigin,
@@ -7876,10 +7905,18 @@ ResetOrClearOp::DeleteFiles(QuotaManager* aQuotaManager)
   AssertIsOnIOThread();
   MOZ_ASSERT(aQuotaManager);
 
+  nsresult rv =
+    aQuotaManager->AboutToClearOrigins(Nullable<PersistenceType>(),
+                                       OriginScope::FromNull(),
+                                       Nullable<Client::Type>());
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
 
   nsCOMPtr<nsIFile> directory;
-  nsresult rv = NS_NewLocalFile(aQuotaManager->GetStoragePath(), false,
-                                getter_AddRefs(directory));
+  rv = NS_NewLocalFile(aQuotaManager->GetStoragePath(),
+                       false,
+                       getter_AddRefs(directory));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
   }
@@ -7949,9 +7986,19 @@ ClearRequestBase::DeleteFiles(QuotaManager* aQuotaManager,
   AssertIsOnIOThread();
   MOZ_ASSERT(aQuotaManager);
 
+  nsresult rv =
+    aQuotaManager->AboutToClearOrigins(
+                                    Nullable<PersistenceType>(aPersistenceType),
+                                    mOriginScope,
+                                    mClientType);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return;
+  }
+
   nsCOMPtr<nsIFile> directory;
-  nsresult rv = NS_NewLocalFile(aQuotaManager->GetStoragePath(aPersistenceType),
-                                false, getter_AddRefs(directory));
+  rv = NS_NewLocalFile(aQuotaManager->GetStoragePath(aPersistenceType),
+                       false,
+                       getter_AddRefs(directory));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return;
   }
diff --git a/dom/quota/Client.h b/dom/quota/Client.h
index c701e0d2bf94..cfa32095f795 100644
--- a/dom/quota/Client.h
+++ b/dom/quota/Client.h
@@ -25,6 +25,7 @@ class nsIRunnable;
 
 BEGIN_QUOTA_NAMESPACE
 
+class OriginScope;
 class QuotaManager;
 class UsageInfo;
 
@@ -167,6 +168,15 @@ public:
                     const AtomicBool& aCanceled,
                     UsageInfo* aUsageInfo) = 0;
 
+  // This method is called when origins are about to be cleared
+  // (except the case when clearing is triggered by the origin eviction).
+  virtual nsresult
+  AboutToClearOrigins(const Nullable<PersistenceType>& aPersistenceType,
+                      const OriginScope& aOriginScope)
+  {
+    return NS_OK;
+  }
+
   virtual void
   OnOriginClearCompleted(PersistenceType aPersistenceType,
                          const nsACString& aOrigin) = 0;
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index fc37d039967e..39e9ac7d3dc1 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -311,6 +311,11 @@ public:
                         bool aCreateIfNotExists,
                         bool* aCreated);
 
+  nsresult
+  AboutToClearOrigins(const Nullable<PersistenceType>& aPersistenceType,
+                      const OriginScope& aOriginScope,
+                      const Nullable<Client::Type>& aClientType);
+
   void
   OriginClearCompleted(PersistenceType aPersistenceType,
                        const nsACString& aOrigin,

From 6714997188e6756ab2f16b1f926ac4d5df61ee4d Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:49 +0100
Subject: [PATCH 62/78] Bug 1286798 - Part 48: Add ParseOrigin() method to the
 quota manager; r=asuth

---
 dom/quota/ActorsParent.cpp | 25 +++++++++++++++++++++++++
 dom/quota/QuotaManager.h   | 11 +++++++++++
 2 files changed, 36 insertions(+)

diff --git a/dom/quota/ActorsParent.cpp b/dom/quota/ActorsParent.cpp
index 876c2272e8cb..782f10c1b7a8 100644
--- a/dom/quota/ActorsParent.cpp
+++ b/dom/quota/ActorsParent.cpp
@@ -6102,6 +6102,31 @@ QuotaManager::AreOriginsEqualOnDisk(nsACString& aOrigin1,
   return origin1Sanitized == origin2Sanitized;
 }
 
+// static
+bool
+QuotaManager::ParseOrigin(const nsACString& aOrigin,
+                          nsCString& aSpec,
+                          OriginAttributes* aAttrs)
+{
+  MOZ_ASSERT(aAttrs);
+
+  if (aOrigin.Equals(kChromeOrigin)) {
+    aSpec = kChromeOrigin;
+    return true;
+  }
+
+  nsCString sanitizedOrigin(aOrigin);
+  SanitizeOriginString(sanitizedOrigin);
+
+  OriginParser::ResultType result =
+    OriginParser::ParseOrigin(sanitizedOrigin, aSpec, aAttrs);
+  if (NS_WARN_IF(result != OriginParser::ValidOrigin)) {
+    return false;
+  }
+
+  return true;
+}
+
 uint64_t
 QuotaManager::LockedCollectOriginsForEviction(
                                   uint64_t aMinSizeToBeFreed,
diff --git a/dom/quota/QuotaManager.h b/dom/quota/QuotaManager.h
index 39e9ac7d3dc1..d99f9a908138 100644
--- a/dom/quota/QuotaManager.h
+++ b/dom/quota/QuotaManager.h
@@ -32,6 +32,12 @@ class nsIURI;
 class nsPIDOMWindowOuter;
 class nsIRunnable;
 
+namespace mozilla {
+
+class OriginAttributes;
+
+} // namespace mozilla
+
 BEGIN_QUOTA_NAMESPACE
 
 class DirectoryLockImpl;
@@ -431,6 +437,11 @@ public:
   AreOriginsEqualOnDisk(nsACString& aOrigin1,
                         nsACString& aOrigin2);
 
+  static bool
+  ParseOrigin(const nsACString& aOrigin,
+              nsCString& aSpec,
+              OriginAttributes* aAttrs);
+
 private:
   QuotaManager();
 

From d4d5f86d8218f214b42ca653e7266380e48b63d8 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:52 +0100
Subject: [PATCH 63/78] Bug 1286798 - Part 49: Add clearStoragesForPattern()
 method to the quota manager service; r=asuth

---
 dom/quota/QuotaManagerService.cpp    | 36 +++++++++++++++++++++-------
 dom/quota/nsIQuotaManagerService.idl | 17 +++++++++++++
 2 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/dom/quota/QuotaManagerService.cpp b/dom/quota/QuotaManagerService.cpp
index 66f52c5ca245..663176641ee4 100644
--- a/dom/quota/QuotaManagerService.cpp
+++ b/dom/quota/QuotaManagerService.cpp
@@ -654,6 +654,30 @@ QuotaManagerService::Clear(nsIQuotaRequest** _retval)
   return NS_OK;
 }
 
+NS_IMETHODIMP
+QuotaManagerService::ClearStoragesForOriginAttributesPattern(
+                                                      const nsAString& aPattern,
+                                                      nsIQuotaRequest** _retval)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  RefPtr<Request> request = new Request();
+
+  ClearDataParams params;
+
+  params.pattern() = aPattern;
+
+  nsAutoPtr<PendingRequestInfo> info(new RequestInfo(request, params));
+
+  nsresult rv = InitiateRequest(info);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  request.forget(_retval);
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 QuotaManagerService::ClearStoragesForPrincipal(nsIPrincipal* aPrincipal,
                                                const nsACString& aPersistenceType,
@@ -846,14 +870,10 @@ QuotaManagerService::Observe(nsISupports* aSubject,
   }
 
   if (!strcmp(aTopic, "clear-origin-attributes-data")) {
-    RefPtr<Request> request = new Request();
-
-    ClearDataParams params;
-    params.pattern() = nsDependentString(aData);
-
-    nsAutoPtr<PendingRequestInfo> info(new RequestInfo(request, params));
-
-    nsresult rv = InitiateRequest(info);
+    nsCOMPtr<nsIQuotaRequest> request;
+    nsresult rv =
+      ClearStoragesForOriginAttributesPattern(nsDependentString(aData),
+                                              getter_AddRefs(request));
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
diff --git a/dom/quota/nsIQuotaManagerService.idl b/dom/quota/nsIQuotaManagerService.idl
index b2098384e66f..d972297ba998 100644
--- a/dom/quota/nsIQuotaManagerService.idl
+++ b/dom/quota/nsIQuotaManagerService.idl
@@ -96,6 +96,23 @@ interface nsIQuotaManagerService : nsISupports
   [must_use] nsIQuotaRequest
   clear();
 
+  /**
+   * Removes all storages stored for the given pattern. The files may not be
+   * deleted immediately depending on prohibitive concurrent operations.
+   * In terms of locks, it will get an exclusive multi directory lock for given
+   * pattern.  For example, given pattern {"appId":1007} and set of 3 origins
+   * ["http://www.mozilla.org^appId=1007", "http://www.example.org^appId=1007",
+   * "http://www.example.org^appId=1008"], the method will only lock 2 origins
+   * ["http://www.mozilla.org^appId=1007", "http://www.example.org^appId=1007"].
+   *
+   * @param aPattern
+   *        A pattern for the origins whose storages are to be cleared.
+   *        Currently this is expected to be a JSON representation of the
+   *        OriginAttributesPatternDictionary defined in ChromeUtils.webidl.
+   */
+  [must_use] nsIQuotaRequest
+  clearStoragesForOriginAttributesPattern(in AString aPattern);
+
   /**
    * Removes all storages stored for the given principal. The files may not be
    * deleted immediately depending on prohibitive concurrent operations.

From ccca35f7f399838db0a5e16445fd46530301d6ae Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:55 +0100
Subject: [PATCH 64/78] Bug 1286798 - Part 50: Add support for clearing of the
 archive and shadow database; r=asuth

---
 dom/localstorage/ActorsParent.cpp | 1050 +++++++++++++++++++++++++++--
 1 file changed, 981 insertions(+), 69 deletions(-)

diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 94e8ff4048dc..506fb5255fcf 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -9,6 +9,7 @@
 #include "LocalStorageCommon.h"
 #include "LSObject.h"
 #include "mozIStorageConnection.h"
+#include "mozIStorageFunction.h"
 #include "mozIStorageService.h"
 #include "mozStorageCID.h"
 #include "mozStorageHelper.h"
@@ -23,6 +24,7 @@
 #include "mozilla/dom/PBackgroundLSSnapshotParent.h"
 #include "mozilla/dom/StorageDBUpdater.h"
 #include "mozilla/dom/StorageUtils.h"
+#include "mozilla/dom/quota/OriginScope.h"
 #include "mozilla/dom/quota/QuotaManager.h"
 #include "mozilla/dom/quota/QuotaObject.h"
 #include "mozilla/dom/quota/UsageInfo.h"
@@ -33,6 +35,7 @@
 #include "nsDataHashtable.h"
 #include "nsInterfaceHashtable.h"
 #include "nsISimpleEnumerator.h"
+#include "nsNetUtil.h"
 #include "nsRefPtrHashtable.h"
 #include "ReportInternalError.h"
 
@@ -57,14 +60,19 @@ using namespace mozilla::ipc;
 
 namespace {
 
-class ArchivedOriginInfo;
+struct ArchivedOriginInfo;
+class ArchivedOriginScope;
 class Connection;
 class ConnectionThread;
 class Database;
 class PrepareDatastoreOp;
 class PreparedDatastore;
+class QuotaClient;
 class Snapshot;
 
+typedef nsClassHashtable<nsCStringHashKey, ArchivedOriginInfo>
+  ArchivedOriginHashtable;
+
 /*******************************************************************************
  * Constants
  ******************************************************************************/
@@ -154,6 +162,13 @@ MakeSchemaVersion(uint32_t aMajorSchemaVersion,
 }
 #endif
 
+nsCString
+GetArchivedOriginHashKey(const nsACString& aOriginSuffix,
+                         const nsACString& aOriginNoSuffix)
+{
+  return aOriginSuffix + NS_LITERAL_CSTRING(":") + aOriginNoSuffix;
+}
+
 nsresult
 CreateTables(mozIStorageConnection* aConnection)
 {
@@ -822,6 +837,46 @@ CreateShadowStorageConnection(const nsAString& aBasePath,
   return NS_OK;
 }
 
+nsresult
+GetShadowStorageConnection(const nsAString& aBasePath,
+                           mozIStorageConnection** aConnection)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(!aBasePath.IsEmpty());
+  MOZ_ASSERT(aConnection);
+
+  nsCOMPtr<nsIFile> shadowFile;
+  nsresult rv = GetShadowFile(aBasePath, getter_AddRefs(shadowFile));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  bool exists;
+  rv = shadowFile->Exists(&exists);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (NS_WARN_IF(!exists)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  nsCOMPtr<mozIStorageService> ss =
+    do_GetService(MOZ_STORAGE_SERVICE_CONTRACTID, &rv);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  nsCOMPtr<mozIStorageConnection> connection;
+  rv = ss->OpenUnsharedDatabase(shadowFile, getter_AddRefs(connection));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  connection.forget(aConnection);
+  return NS_OK;
+}
+
 nsresult
 AttachShadowDatabase(const nsAString& aBasePath,
                      mozIStorageConnection* aConnection)
@@ -1219,7 +1274,7 @@ private:
   RefPtr<ConnectionThread> mConnectionThread;
   nsCOMPtr<nsITimer> mFlushTimer;
   nsCOMPtr<mozIStorageConnection> mStorageConnection;
-  nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
+  nsAutoPtr<ArchivedOriginScope> mArchivedOriginScope;
   nsInterfaceHashtable<nsCStringHashKey, mozIStorageStatement>
     mCachedStatements;
   WriteOptimizer mWriteOptimizer;
@@ -1239,10 +1294,10 @@ public:
     NS_ASSERT_OWNINGTHREAD(Connection);
   }
 
-  ArchivedOriginInfo*
-  GetArchivedOriginInfo() const
+  ArchivedOriginScope*
+  GetArchivedOriginScope() const
   {
-    return mArchivedOriginInfo;
+    return mArchivedOriginScope;
   }
 
   // Methods which can only be called on the owning thread.
@@ -1303,7 +1358,7 @@ private:
   Connection(ConnectionThread* aConnectionThread,
              const nsACString& aOrigin,
              const nsAString& aFilePath,
-             nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
+             nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope);
 
   ~Connection();
 
@@ -1347,6 +1402,7 @@ private:
 class Connection::FlushOp final
   : public ConnectionDatastoreOperationBase
 {
+  RefPtr<QuotaClient> mQuotaClient;
   WriteOptimizer mWriteOptimizer;
   bool mShadowWrites;
 
@@ -1404,7 +1460,7 @@ public:
   already_AddRefed<Connection>
   CreateConnection(const nsACString& aOrigin,
                    const nsAString& aFilePath,
-                   nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo);
+                   nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope);
 
   void
   Shutdown();
@@ -2107,7 +2163,7 @@ class PrepareDatastoreOp
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
   RefPtr<Datastore> mDatastore;
-  nsAutoPtr<ArchivedOriginInfo> mArchivedOriginInfo;
+  nsAutoPtr<ArchivedOriginScope> mArchivedOriginScope;
   LoadDataOp* mLoadDataOp;
   nsDataHashtable<nsStringHashKey, nsString> mValues;
   nsTArray<LSItemInfo> mOrderedItems;
@@ -2372,42 +2428,218 @@ private:
  * Other class declarations
  ******************************************************************************/
 
-class ArchivedOriginInfo
+struct ArchivedOriginInfo
 {
-  nsCString mOriginSuffix;
+  OriginAttributes mOriginAttributes;
   nsCString mOriginNoSuffix;
 
-public:
-  static ArchivedOriginInfo*
-  Create(nsIPrincipal* aPrincipal);
+  ArchivedOriginInfo(const OriginAttributes& aOriginAttributes,
+                     const nsACString& aOriginNoSuffix)
+    : mOriginAttributes(aOriginAttributes)
+    , mOriginNoSuffix(aOriginNoSuffix)
+  { }
+};
 
-  const nsCString&
+class ArchivedOriginScope
+{
+  struct Origin
+  {
+    nsCString mOriginSuffix;
+    nsCString mOriginNoSuffix;
+
+    Origin(const nsACString& aOriginSuffix,
+           const nsACString& aOriginNoSuffix)
+      : mOriginSuffix(aOriginSuffix)
+      , mOriginNoSuffix(aOriginNoSuffix)
+    { }
+
+    const nsACString&
+    OriginSuffix() const
+    {
+      return mOriginSuffix;
+    }
+
+    const nsACString&
+    OriginNoSuffix() const
+    {
+      return mOriginNoSuffix;
+    }
+  };
+
+  struct Prefix
+  {
+    nsCString mOriginNoSuffix;
+
+    explicit Prefix(const nsACString& aOriginNoSuffix)
+      : mOriginNoSuffix(aOriginNoSuffix)
+    { }
+
+    const nsACString&
+    OriginNoSuffix() const
+    {
+      return mOriginNoSuffix;
+    }
+  };
+
+  struct Pattern
+  {
+    UniquePtr<OriginAttributesPattern> mPattern;
+
+    explicit Pattern(const OriginAttributesPattern& aPattern)
+      : mPattern(MakeUnique<OriginAttributesPattern>(aPattern))
+    { }
+
+    Pattern(const Pattern& aOther)
+      : mPattern(MakeUnique<OriginAttributesPattern>(*aOther.mPattern))
+    { }
+
+    Pattern(Pattern&& aOther) = default;
+
+    const OriginAttributesPattern&
+    GetPattern() const
+    {
+      MOZ_ASSERT(mPattern);
+      return *mPattern;
+    }
+  };
+
+  struct Null
+  { };
+
+  using DataType = Variant<Origin, Pattern, Prefix, Null>;
+
+  DataType mData;
+
+public:
+  static ArchivedOriginScope*
+  CreateFromOrigin(nsIPrincipal* aPrincipal);
+
+  static ArchivedOriginScope*
+  CreateFromPrefix(nsIPrincipal* aPrincipal);
+
+  static ArchivedOriginScope*
+  CreateFromPattern(const OriginAttributesPattern& aPattern);
+
+  static ArchivedOriginScope*
+  CreateFromNull();
+
+  bool
+  IsOrigin() const
+  {
+    return mData.is<Origin>();
+  }
+
+  bool
+  IsPrefix() const
+  {
+    return mData.is<Prefix>();
+  }
+
+  bool
+  IsPattern() const
+  {
+    return mData.is<Pattern>();
+  }
+
+  bool
+  IsNull() const
+  {
+    return mData.is<Null>();
+  }
+
+  const nsACString&
   OriginSuffix() const
   {
-    return mOriginSuffix;
+    MOZ_ASSERT(IsOrigin());
+
+    return mData.as<Origin>().OriginSuffix();
   }
 
-  const nsCString&
+  const nsACString&
   OriginNoSuffix() const
   {
-    return mOriginNoSuffix;
+    MOZ_ASSERT(IsOrigin() || IsPrefix());
+
+    if (IsOrigin()) {
+      return mData.as<Origin>().OriginNoSuffix();
+    }
+    return mData.as<Prefix>().OriginNoSuffix();
   }
 
-  const nsCString
-  Origin() const
+  const OriginAttributesPattern&
+  GetPattern() const
   {
-    return mOriginSuffix + NS_LITERAL_CSTRING(":") + mOriginNoSuffix;
+    MOZ_ASSERT(IsPattern());
+
+    return mData.as<Pattern>().GetPattern();
   }
 
+  void
+  GetBindingClause(nsACString& aBindingClause) const;
+
   nsresult
   BindToStatement(mozIStorageStatement* aStatement) const;
 
+  bool
+  HasMatches(ArchivedOriginHashtable* aHashtable) const;
+
+  void
+  RemoveMatches(ArchivedOriginHashtable* aHashtable) const;
+
 private:
-  ArchivedOriginInfo(const nsACString& aOriginSuffix,
-                     const nsACString& aOriginNoSuffix)
-    : mOriginSuffix(aOriginSuffix)
-    , mOriginNoSuffix(aOriginNoSuffix)
+  // Move constructors
+  explicit ArchivedOriginScope(const Origin&& aOrigin)
+    : mData(aOrigin)
   { }
+
+  explicit ArchivedOriginScope(const Pattern&& aPattern)
+    : mData(aPattern)
+  { }
+
+  explicit ArchivedOriginScope(const Prefix&& aPrefix)
+    : mData(aPrefix)
+  { }
+
+  explicit ArchivedOriginScope(const Null&& aNull)
+    : mData(aNull)
+  { }
+};
+
+class ArchivedOriginScopeHelper
+  : public Runnable
+{
+  Monitor mMonitor;
+  const OriginAttributes mAttrs;
+  const nsCString mSpec;
+  nsAutoPtr<ArchivedOriginScope> mArchivedOriginScope;
+  nsresult mMainThreadResultCode;
+  bool mWaiting;
+  bool mPrefix;
+
+public:
+  ArchivedOriginScopeHelper(const nsACString& aSpec,
+                            const OriginAttributes& aAttrs,
+                            bool aPrefix)
+    : Runnable("dom::localstorage::ArchivedOriginScopeHelper")
+    , mMonitor("ArchivedOriginScopeHelper::mMonitor")
+    , mAttrs(aAttrs)
+    , mSpec(aSpec)
+    , mMainThreadResultCode(NS_OK)
+    , mWaiting(true)
+    , mPrefix(aPrefix)
+  {
+    AssertIsOnIOThread();
+  }
+
+  nsresult
+  BlockAndReturnArchivedOriginScope(
+                          nsAutoPtr<ArchivedOriginScope>& aArchivedOriginScope);
+
+private:
+  nsresult
+  RunOnMainThread();
+
+  NS_DECL_NSIRUNNABLE
 };
 
 class QuotaClient final
@@ -2415,15 +2647,25 @@ class QuotaClient final
 {
   class ClearPrivateBrowsingRunnable;
   class Observer;
+  class MatchFunction;
 
   static QuotaClient* sInstance;
   static bool sObserversRegistered;
 
+  Mutex mShadowDatabaseMutex;
   bool mShutdownRequested;
 
 public:
   QuotaClient();
 
+  static QuotaClient*
+  GetInstance()
+  {
+    AssertIsOnBackgroundThread();
+
+    return sInstance;
+  }
+
   static bool
   IsShuttingDownOnBackgroundThread()
   {
@@ -2447,6 +2689,14 @@ public:
   static nsresult
   RegisterObservers(nsIEventTarget* aBackgroundEventTarget);
 
+  mozilla::Mutex&
+  ShadowDatabaseMutex()
+  {
+    MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
+
+    return mShadowDatabaseMutex;
+  }
+
   bool
   IsShuttingDown() const
   {
@@ -2474,6 +2724,10 @@ public:
                     const AtomicBool& aCanceled,
                     UsageInfo* aUsageInfo) override;
 
+  nsresult
+  AboutToClearOrigins(const Nullable<PersistenceType>& aPersistenceType,
+                      const OriginScope& aOriginScope) override;
+
   void
   OnOriginClearCompleted(PersistenceType aPersistenceType,
                          const nsACString& aOrigin)
@@ -2499,6 +2753,16 @@ public:
 
 private:
   ~QuotaClient() override;
+
+  nsresult
+  CreateArchivedOriginScope(
+                          const OriginScope& aOriginScope,
+                          nsAutoPtr<ArchivedOriginScope>& aArchivedOriginScope);
+
+  nsresult
+  PerformDelete(mozIStorageConnection* aConnection,
+                const nsACString& aSchemaName,
+                ArchivedOriginScope* aArchivedOriginScope) const;
 };
 
 class QuotaClient::ClearPrivateBrowsingRunnable final
@@ -2540,6 +2804,23 @@ private:
   NS_DECL_NSIOBSERVER
 };
 
+class QuotaClient::MatchFunction final
+  : public mozIStorageFunction
+{
+  OriginAttributesPattern mPattern;
+
+public:
+  explicit MatchFunction(const OriginAttributesPattern& aPattern)
+    : mPattern(aPattern)
+  { }
+
+private:
+  ~MatchFunction() = default;
+
+  NS_DECL_ISUPPORTS
+  NS_DECL_MOZISTORAGEFUNCTION
+};
+
 /*******************************************************************************
  * Globals
  ******************************************************************************/
@@ -2585,8 +2866,6 @@ typedef nsDataHashtable<nsCStringHashKey, int64_t> UsageHashtable;
 // Can only be touched on the Quota Manager I/O thread.
 StaticAutoPtr<UsageHashtable> gUsages;
 
-typedef nsTHashtable<nsCStringHashKey> ArchivedOriginHashtable;
-
 StaticAutoPtr<ArchivedOriginHashtable> gArchivedOrigins;
 
 // Can only be touched on the Quota Manager I/O thread.
@@ -2648,7 +2927,7 @@ LoadArchivedOrigins()
 
   nsCOMPtr<mozIStorageStatement> stmt;
   rv = connection->CreateStatement(NS_LITERAL_CSTRING(
-    "SELECT DISTINCT originAttributes || ':' || originKey "
+    "SELECT DISTINCT originAttributes, originKey "
       "FROM webappsstore2;"
   ), getter_AddRefs(stmt));
   if (NS_WARN_IF(NS_FAILED(rv))) {
@@ -2660,13 +2939,29 @@ LoadArchivedOrigins()
 
   bool hasResult;
   while (NS_SUCCEEDED(rv = stmt->ExecuteStep(&hasResult)) && hasResult) {
-    nsCString origin;
-    rv = stmt->GetUTF8String(0, origin);
+    nsCString originSuffix;
+    rv = stmt->GetUTF8String(0, originSuffix);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
 
-    archivedOrigins->PutEntry(origin);
+    nsCString originNoSuffix;
+    rv = stmt->GetUTF8String(1, originNoSuffix);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    nsCString hashKey = GetArchivedOriginHashKey(originSuffix, originNoSuffix);
+
+    OriginAttributes originAttributes;
+    if (NS_WARN_IF(!originAttributes.PopulateFromSuffix(originSuffix))) {
+      return NS_ERROR_FAILURE;
+    }
+
+    nsAutoPtr<ArchivedOriginInfo> archivedOriginInfo(
+      new ArchivedOriginInfo(originAttributes, originNoSuffix));
+
+    archivedOrigins->Put(hashKey, archivedOriginInfo.forget());
   }
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
@@ -2678,7 +2973,7 @@ LoadArchivedOrigins()
 
 nsresult
 GetUsage(mozIStorageConnection* aConnection,
-         ArchivedOriginInfo* aArchivedOriginInfo,
+         ArchivedOriginScope* aArchivedOriginScope,
          int64_t* aUsage)
 {
   AssertIsOnIOThread();
@@ -2688,7 +2983,7 @@ GetUsage(mozIStorageConnection* aConnection,
   nsresult rv;
 
   nsCOMPtr<mozIStorageStatement> stmt;
-  if (aArchivedOriginInfo) {
+  if (aArchivedOriginScope) {
     rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
       "SELECT sum(length(key) + length(value)) "
       "FROM webappsstore2 "
@@ -2699,7 +2994,7 @@ GetUsage(mozIStorageConnection* aConnection,
       return rv;
     }
 
-    rv = aArchivedOriginInfo->BindToStatement(stmt);
+    rv = aArchivedOriginScope->BindToStatement(stmt);
   } else {
     rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
       "SELECT sum(length(key) + length(value)) "
@@ -3268,15 +3563,16 @@ AddItemInfo::Perform(Connection* aConnection, bool aShadowWrites)
     return rv;
   }
 
-  ArchivedOriginInfo* archivedOriginInfo = aConnection->GetArchivedOriginInfo();
+  ArchivedOriginScope* archivedOriginScope =
+    aConnection->GetArchivedOriginScope();
 
-  rv = archivedOriginInfo->BindToStatement(stmt);
+  rv = archivedOriginScope->BindToStatement(stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  nsCString scope = Scheme0Scope(archivedOriginInfo->OriginSuffix(),
-                                 archivedOriginInfo->OriginNoSuffix());
+  nsCString scope = Scheme0Scope(archivedOriginScope->OriginSuffix(),
+                                 archivedOriginScope->OriginNoSuffix());
 
   rv = stmt->BindUTF8StringByName(NS_LITERAL_CSTRING("scope"),
                                   scope);
@@ -3342,7 +3638,7 @@ RemoveItemInfo::Perform(Connection* aConnection, bool aShadowWrites)
     return rv;
   }
 
-  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  rv = aConnection->GetArchivedOriginScope()->BindToStatement(stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3393,7 +3689,7 @@ ClearInfo::Perform(Connection* aConnection, bool aShadowWrites)
     return rv;
   }
 
-  rv = aConnection->GetArchivedOriginInfo()->BindToStatement(stmt);
+  rv = aConnection->GetArchivedOriginScope()->BindToStatement(stmt);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
@@ -3516,9 +3812,9 @@ ConnectionDatastoreOperationBase::Run()
 Connection::Connection(ConnectionThread* aConnectionThread,
                        const nsACString& aOrigin,
                        const nsAString& aFilePath,
-                       nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
+                       nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope)
   : mConnectionThread(aConnectionThread)
-  , mArchivedOriginInfo(std::move(aArchivedOriginInfo))
+  , mArchivedOriginScope(std::move(aArchivedOriginScope))
   , mOrigin(aOrigin)
   , mFilePath(aFilePath)
   , mFlushScheduled(false)
@@ -3806,9 +4102,11 @@ Connection::
 FlushOp::FlushOp(Connection* aConnection,
                  WriteOptimizer&& aWriteOptimizer)
   : ConnectionDatastoreOperationBase(aConnection)
+  , mQuotaClient(QuotaClient::GetInstance())
   , mWriteOptimizer(std::move(aWriteOptimizer))
   , mShadowWrites(gShadowWrites)
 {
+  MOZ_ASSERT(mQuotaClient);
 }
 
 nsresult
@@ -3827,7 +4125,13 @@ FlushOp::DoDatastoreWork()
 
   nsresult rv;
 
+  Maybe<MutexAutoLock> shadowDatabaseLock;
+
   if (mShadowWrites) {
+    MOZ_ASSERT(mQuotaClient);
+
+    shadowDatabaseLock.emplace(mQuotaClient->ShadowDatabaseMutex());
+
     rv = AttachShadowDatabase(quotaManager->GetBasePath(), storageConnection);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
@@ -3935,16 +4239,16 @@ ConnectionThread::AssertIsOnConnectionThread()
 
 already_AddRefed<Connection>
 ConnectionThread::CreateConnection(
-                            const nsACString& aOrigin,
-                            const nsAString& aFilePath,
-                            nsAutoPtr<ArchivedOriginInfo>&& aArchivedOriginInfo)
+                          const nsACString& aOrigin,
+                          const nsAString& aFilePath,
+                          nsAutoPtr<ArchivedOriginScope>&& aArchivedOriginScope)
 {
   AssertIsOnOwningThread();
   MOZ_ASSERT(!aOrigin.IsEmpty());
   MOZ_ASSERT(!mConnections.GetWeak(aOrigin));
 
   RefPtr<Connection> connection =
-    new Connection(this, aOrigin, aFilePath, std::move(aArchivedOriginInfo));
+    new Connection(this, aOrigin, aFilePath, std::move(aArchivedOriginScope));
   mConnections.Put(aOrigin, connection);
 
   return connection.forget();
@@ -5643,8 +5947,8 @@ PrepareDatastoreOp::Open()
       return rv;
     }
 
-    mArchivedOriginInfo = ArchivedOriginInfo::Create(principal);
-    if (NS_WARN_IF(!mArchivedOriginInfo)) {
+    mArchivedOriginScope = ArchivedOriginScope::CreateFromOrigin(principal);
+    if (NS_WARN_IF(!mArchivedOriginScope)) {
       return NS_ERROR_FAILURE;
     }
   }
@@ -5910,7 +6214,7 @@ nsresult
 PrepareDatastoreOp::DatabaseWork()
 {
   AssertIsOnIOThread();
-  MOZ_ASSERT(mArchivedOriginInfo);
+  MOZ_ASSERT(mArchivedOriginScope);
   MOZ_ASSERT(mState == State::Nesting);
   MOZ_ASSERT(mNestedState == NestedState::DatabaseWorkOpen);
 
@@ -5932,8 +6236,7 @@ PrepareDatastoreOp::DatabaseWork()
     MOZ_ASSERT(gArchivedOrigins);
   }
 
-  bool hasDataForMigration =
-    gArchivedOrigins->GetEntry(mArchivedOriginInfo->Origin());
+  bool hasDataForMigration = mArchivedOriginScope->HasMatches(gArchivedOrigins);
 
   bool createIfNotExists = mParams.createIfNotExists() || hasDataForMigration;
 
@@ -6018,7 +6321,7 @@ PrepareDatastoreOp::DatabaseWork()
     }
 
     int64_t newUsage;
-    rv = GetUsage(connection, mArchivedOriginInfo, &newUsage);
+    rv = GetUsage(connection, mArchivedOriginScope, &newUsage);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -6046,7 +6349,7 @@ PrepareDatastoreOp::DatabaseWork()
       return rv;
     }
 
-    rv = mArchivedOriginInfo->BindToStatement(stmt);
+    rv = mArchivedOriginScope->BindToStatement(stmt);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -6065,7 +6368,7 @@ PrepareDatastoreOp::DatabaseWork()
       return rv;
     }
 
-    rv = mArchivedOriginInfo->BindToStatement(stmt);
+    rv = mArchivedOriginScope->BindToStatement(stmt);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -6086,8 +6389,8 @@ PrepareDatastoreOp::DatabaseWork()
     }
 
     MOZ_ASSERT(gArchivedOrigins);
-    MOZ_ASSERT(gArchivedOrigins->GetEntry(mArchivedOriginInfo->Origin()));
-    gArchivedOrigins->RemoveEntry(mArchivedOriginInfo->Origin());
+    MOZ_ASSERT(mArchivedOriginScope->HasMatches(gArchivedOrigins));
+    mArchivedOriginScope->RemoveMatches(gArchivedOrigins);
 
     mUsage = newUsage;
 
@@ -6265,7 +6568,7 @@ PrepareDatastoreOp::BeginLoadData()
   mConnection =
     gConnectionThread->CreateConnection(mOrigin,
                                         mDatabaseFilePath,
-                                        std::move(mArchivedOriginInfo));
+                                        std::move(mArchivedOriginScope));
   MOZ_ASSERT(mConnection);
 
   // Must set this before dispatching otherwise we will race with the
@@ -6917,12 +7220,12 @@ PreloadedOp::GetResponse(LSSimpleRequestResponse& aResponse)
 }
 
 /*******************************************************************************
- * ArchivedOriginInfo
+ * ArchivedOriginScope
  ******************************************************************************/
 
 // static
-ArchivedOriginInfo*
-ArchivedOriginInfo::Create(nsIPrincipal* aPrincipal)
+ArchivedOriginScope*
+ArchivedOriginScope::CreateFromOrigin(nsIPrincipal* aPrincipal)
 {
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(aPrincipal);
@@ -6934,28 +7237,334 @@ ArchivedOriginInfo::Create(nsIPrincipal* aPrincipal)
     return nullptr;
   }
 
-  return new ArchivedOriginInfo(originAttrSuffix, originKey);
+  return new ArchivedOriginScope(std::move(Origin(originAttrSuffix,
+                                                  originKey)));
+}
+
+// static
+ArchivedOriginScope*
+ArchivedOriginScope::CreateFromPrefix(nsIPrincipal* aPrincipal)
+{
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aPrincipal);
+
+  nsCString originAttrSuffix;
+  nsCString originKey;
+  nsresult rv = GenerateOriginKey(aPrincipal, originAttrSuffix, originKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return nullptr;
+  }
+
+  return new ArchivedOriginScope(std::move(Prefix(originKey)));
+}
+
+// static
+ArchivedOriginScope*
+ArchivedOriginScope::CreateFromPattern(const OriginAttributesPattern& aPattern)
+{
+  return new ArchivedOriginScope(std::move(Pattern(aPattern)));
+}
+
+// static
+ArchivedOriginScope*
+ArchivedOriginScope::CreateFromNull()
+{
+  return new ArchivedOriginScope(std::move(Null()));
+}
+
+void
+ArchivedOriginScope::GetBindingClause(nsACString& aBindingClause) const
+{
+  struct Matcher
+  {
+    nsACString* mBindingClause;
+
+    explicit Matcher(nsACString* aBindingClause)
+      : mBindingClause(aBindingClause)
+    { }
+
+    void
+    match(const Origin& aOrigin) {
+      *mBindingClause = NS_LITERAL_CSTRING(
+        " WHERE originKey = :originKey "
+        "AND originAttributes = :originAttributes"
+      );
+    }
+
+    void
+    match(const Prefix& aPrefix) {
+      *mBindingClause = NS_LITERAL_CSTRING(
+        " WHERE originKey = :originKey"
+      );
+    }
+
+    void
+    match(const Pattern& aPattern) {
+      *mBindingClause = NS_LITERAL_CSTRING(
+        " WHERE originAttributes MATCH :originAttributesPattern"
+      );
+    }
+
+    void
+    match(const Null& aNull) {
+      *mBindingClause = EmptyCString();
+    }
+  };
+
+  mData.match(Matcher(&aBindingClause));
 }
 
 nsresult
-ArchivedOriginInfo::BindToStatement(mozIStorageStatement* aStatement) const
+ArchivedOriginScope::BindToStatement(mozIStorageStatement* aStmt) const
 {
   MOZ_ASSERT(IsOnIOThread() || IsOnConnectionThread());
-  MOZ_ASSERT(aStatement);
+  MOZ_ASSERT(aStmt);
 
-  nsresult rv =
-    aStatement->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
-                                     mOriginNoSuffix);
+  struct Matcher
+  {
+    mozIStorageStatement* mStmt;
+
+    explicit Matcher(mozIStorageStatement* aStmt)
+      : mStmt(aStmt)
+    { }
+
+    nsresult
+    match(const Origin& aOrigin) {
+      nsresult rv = mStmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
+                                                aOrigin.OriginNoSuffix());
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      rv = mStmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
+                                       aOrigin.OriginSuffix());
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      return NS_OK;
+    }
+
+    nsresult
+    match(const Prefix& aPrefix) {
+      nsresult rv = mStmt->BindUTF8StringByName(NS_LITERAL_CSTRING("originKey"),
+                                                aPrefix.OriginNoSuffix());
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      return NS_OK;
+    }
+
+    nsresult
+    match(const Pattern& aPattern) {
+      nsresult rv = mStmt->BindUTF8StringByName(
+                                  NS_LITERAL_CSTRING("originAttributesPattern"),
+                                  NS_LITERAL_CSTRING("pattern1"));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      return NS_OK;
+    }
+
+    nsresult
+    match(const Null& aNull) {
+      return NS_OK;
+    }
+  };
+
+  nsresult rv = mData.match(Matcher(aStmt));
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
-  rv = aStatement->BindUTF8StringByName(NS_LITERAL_CSTRING("originAttributes"),
-                                        mOriginSuffix);
+  return NS_OK;
+}
+
+bool
+ArchivedOriginScope::HasMatches(ArchivedOriginHashtable* aHashtable) const
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aHashtable);
+
+  struct Matcher
+  {
+    ArchivedOriginHashtable* mHashtable;
+
+    explicit Matcher(ArchivedOriginHashtable* aHashtable)
+      : mHashtable(aHashtable)
+    { }
+
+    bool
+    match(const Origin& aOrigin) {
+      nsCString hashKey = GetArchivedOriginHashKey(aOrigin.OriginSuffix(),
+                                                   aOrigin.OriginNoSuffix());
+
+      ArchivedOriginInfo* archivedOriginInfo;
+      return mHashtable->Get(hashKey, &archivedOriginInfo);
+    }
+
+    bool
+    match(const Prefix& aPrefix) {
+      for (auto iter = mHashtable->ConstIter(); !iter.Done(); iter.Next()) {
+        ArchivedOriginInfo* archivedOriginInfo = iter.Data();
+
+        if (archivedOriginInfo->mOriginNoSuffix == aPrefix.OriginNoSuffix()) {
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    bool
+    match(const Pattern& aPattern) {
+      for (auto iter = mHashtable->ConstIter(); !iter.Done(); iter.Next()) {
+        ArchivedOriginInfo* archivedOriginInfo = iter.Data();
+
+        if (aPattern.GetPattern().Matches(
+                                       archivedOriginInfo->mOriginAttributes)) {
+          return true;
+        }
+      }
+
+      return false;
+    }
+
+    bool
+    match(const Null& aNull) {
+      return mHashtable->Count();
+    }
+  };
+
+  return mData.match(Matcher(aHashtable));
+}
+
+void
+ArchivedOriginScope::RemoveMatches(ArchivedOriginHashtable* aHashtable) const
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aHashtable);
+
+  struct Matcher
+  {
+    ArchivedOriginHashtable* mHashtable;
+
+    explicit Matcher(ArchivedOriginHashtable* aHashtable)
+      : mHashtable(aHashtable)
+    { }
+
+    void
+    match(const Origin& aOrigin) {
+      nsCString hashKey = GetArchivedOriginHashKey(aOrigin.OriginSuffix(),
+                                                   aOrigin.OriginNoSuffix());
+
+      mHashtable->Remove(hashKey);
+    }
+
+    void
+    match(const Prefix& aPrefix) {
+      for (auto iter = mHashtable->Iter(); !iter.Done(); iter.Next()) {
+        ArchivedOriginInfo* archivedOriginInfo = iter.Data();
+
+        if (archivedOriginInfo->mOriginNoSuffix == aPrefix.OriginNoSuffix()) {
+          iter.Remove();
+        }
+      }
+    }
+
+    void
+    match(const Pattern& aPattern) {
+      for (auto iter = mHashtable->Iter(); !iter.Done(); iter.Next()) {
+        ArchivedOriginInfo* archivedOriginInfo = iter.Data();
+
+        if (aPattern.GetPattern().Matches(
+                                       archivedOriginInfo->mOriginAttributes)) {
+          iter.Remove();
+        }
+      }
+    }
+
+    void
+    match(const Null& aNull) {
+      mHashtable->Clear();
+    }
+  };
+
+  mData.match(Matcher(aHashtable));
+}
+
+/*******************************************************************************
+ * ArchivedOriginScopeHelper
+ ******************************************************************************/
+
+nsresult
+ArchivedOriginScopeHelper::BlockAndReturnArchivedOriginScope(
+                           nsAutoPtr<ArchivedOriginScope>& aArchivedOriginScope)
+{
+  AssertIsOnIOThread();
+
+  MOZ_ALWAYS_SUCCEEDS(NS_DispatchToMainThread(this));
+
+  mozilla::MonitorAutoLock lock(mMonitor);
+  while (mWaiting) {
+    lock.Wait();
+  }
+
+  if (NS_WARN_IF(NS_FAILED(mMainThreadResultCode))) {
+    return mMainThreadResultCode;
+  }
+
+  aArchivedOriginScope = std::move(mArchivedOriginScope);
+  return NS_OK;
+}
+
+nsresult
+ArchivedOriginScopeHelper::RunOnMainThread()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  nsCOMPtr<nsIURI> uri;
+  nsresult rv = NS_NewURI(getter_AddRefs(uri), mSpec);
   if (NS_WARN_IF(NS_FAILED(rv))) {
     return rv;
   }
 
+  nsCOMPtr<nsIPrincipal> principal =
+    BasePrincipal::CreateCodebasePrincipal(uri, mAttrs);
+  if (NS_WARN_IF(!principal)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  if (mPrefix) {
+    mArchivedOriginScope = ArchivedOriginScope::CreateFromPrefix(principal);
+  } else {
+    mArchivedOriginScope = ArchivedOriginScope::CreateFromOrigin(principal);
+  }
+  if (NS_WARN_IF(!mArchivedOriginScope)) {
+    return NS_ERROR_FAILURE;
+  }
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+ArchivedOriginScopeHelper::Run()
+{
+  MOZ_ASSERT(NS_IsMainThread());
+
+  nsresult rv = RunOnMainThread();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    mMainThreadResultCode = rv;
+  }
+
+  mozilla::MonitorAutoLock lock(mMonitor);
+  MOZ_ASSERT(mWaiting);
+
+  mWaiting = false;
+  lock.Notify();
+
   return NS_OK;
 }
 
@@ -6967,7 +7576,8 @@ QuotaClient* QuotaClient::sInstance = nullptr;
 bool QuotaClient::sObserversRegistered = false;
 
 QuotaClient::QuotaClient()
-  : mShutdownRequested(false)
+  : mShadowDatabaseMutex("LocalStorage mShadowDatabaseMutex")
+  , mShutdownRequested(false)
 {
   AssertIsOnBackgroundThread();
   MOZ_ASSERT(!sInstance, "We expect this to be a singleton!");
@@ -7103,7 +7713,7 @@ QuotaClient::InitOrigin(PersistenceType aPersistenceType,
     }
 
     int64_t usage;
-    rv = GetUsage(connection, /* aArchivedOriginInfo */ nullptr, &usage);
+    rv = GetUsage(connection, /* aArchivedOriginScope */ nullptr, &usage);
     if (NS_WARN_IF(NS_FAILED(rv))) {
       return rv;
     }
@@ -7194,6 +7804,177 @@ QuotaClient::GetUsageForOrigin(PersistenceType aPersistenceType,
   return NS_OK;
 }
 
+nsresult
+QuotaClient::AboutToClearOrigins(
+                              const Nullable<PersistenceType>& aPersistenceType,
+                              const OriginScope& aOriginScope)
+{
+  AssertIsOnIOThread();
+
+  // This method is not called when the clearing is triggered by the eviction
+  // process. It's on purpose to avoid a problem with the origin access time
+  // which can be described as follows:
+  // When there's a storage pressure condition and quota manager starts
+  // collecting origins for eviction, there can be an origin that hasn't been
+  // touched for long time. However, the old implementation of local storage
+  // could have touched the origin only recently and the new implementation
+  // hasn't had a chance to create a new per origin database for it yet (the
+  // data is still in the archive database), so the origin access time hasn't
+  // been updated either. In the end, the origin would be evicted despite the
+  // fact that there was recent local storage activity.
+  // So this method clears the archived data and shadow database entries for
+  // given origin scope, but only if it's a privacy-related origin clearing.
+
+  if (!aPersistenceType.IsNull() &&
+      aPersistenceType.Value() != PERSISTENCE_TYPE_DEFAULT) {
+    return NS_OK;
+  }
+
+  bool shadowWrites = gShadowWrites;
+
+  nsAutoPtr<ArchivedOriginScope> archivedOriginScope;
+  nsresult rv = CreateArchivedOriginScope(aOriginScope, archivedOriginScope);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  if (!gArchivedOrigins) {
+    rv = LoadArchivedOrigins();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+    MOZ_ASSERT(gArchivedOrigins);
+  }
+
+  bool hasDataForRemoval = archivedOriginScope->HasMatches(gArchivedOrigins);
+
+  QuotaManager* quotaManager = QuotaManager::Get();
+  MOZ_ASSERT(quotaManager);
+
+  nsString basePath = quotaManager->GetBasePath();
+
+  {
+    MutexAutoLock shadowDatabaseLock(mShadowDatabaseMutex);
+
+    nsCOMPtr<mozIStorageConnection> connection;
+    if (gInitializedShadowStorage) {
+      rv = GetShadowStorageConnection(basePath, getter_AddRefs(connection));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    } else {
+      rv = CreateShadowStorageConnection(basePath, getter_AddRefs(connection));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      gInitializedShadowStorage = true;
+    }
+
+    if (hasDataForRemoval) {
+      rv = AttachArchiveDatabase(quotaManager->GetStoragePath(), connection);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    if (archivedOriginScope->IsPattern()) {
+      nsCOMPtr<mozIStorageFunction> function(
+        new MatchFunction(archivedOriginScope->GetPattern()));
+
+      rv = connection->CreateFunction(NS_LITERAL_CSTRING("match"), 2, function);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    nsCOMPtr<mozIStorageStatement> stmt;
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "BEGIN IMMEDIATE;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = stmt->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    if (shadowWrites) {
+      rv = PerformDelete(connection,
+                         NS_LITERAL_CSTRING("main"),
+                         archivedOriginScope);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    if (hasDataForRemoval) {
+      rv = PerformDelete(connection,
+                         NS_LITERAL_CSTRING("archive"),
+                         archivedOriginScope);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    rv = connection->CreateStatement(NS_LITERAL_CSTRING(
+      "COMMIT;"
+    ), getter_AddRefs(stmt));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = stmt->Execute();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    stmt = nullptr;
+
+    if (archivedOriginScope->IsPattern()) {
+      rv = connection->RemoveFunction(NS_LITERAL_CSTRING("match"));
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+    }
+
+    if (hasDataForRemoval) {
+      rv = DetachArchiveDatabase(connection);
+      if (NS_WARN_IF(NS_FAILED(rv))) {
+        return rv;
+      }
+
+      MOZ_ASSERT(gArchivedOrigins);
+      MOZ_ASSERT(archivedOriginScope->HasMatches(gArchivedOrigins));
+      archivedOriginScope->RemoveMatches(gArchivedOrigins);
+    }
+
+    rv = connection->Close();
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  }
+
+  if (aOriginScope.IsNull()) {
+    nsCOMPtr<nsIFile> shadowFile;
+    rv = GetShadowFile(basePath, getter_AddRefs(shadowFile));
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    rv = shadowFile->Remove(false);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+
+    gInitializedShadowStorage = false;
+  }
+
+  return NS_OK;
+}
+
 void
 QuotaClient::OnOriginClearCompleted(PersistenceType aPersistenceType,
                                     const nsACString& aOrigin)
@@ -7216,6 +7997,9 @@ QuotaClient::ReleaseIOThreadObjects()
 
   gUsages = nullptr;
 
+  // Delete archived origins hashtable since QuotaManager clears the whole
+  // storage directory including ls-archive.sqlite.
+
   gArchivedOrigins = nullptr;
 }
 
@@ -7359,6 +8143,100 @@ QuotaClient::ShutdownWorkThreads()
   }
 }
 
+nsresult
+QuotaClient::CreateArchivedOriginScope(
+                           const OriginScope& aOriginScope,
+                           nsAutoPtr<ArchivedOriginScope>& aArchivedOriginScope)
+{
+  AssertIsOnIOThread();
+
+  nsresult rv;
+
+  nsAutoPtr<ArchivedOriginScope> archivedOriginScope;
+
+  if (aOriginScope.IsOrigin()) {
+    nsCString spec;
+    OriginAttributes attrs;
+    if (NS_WARN_IF(!QuotaManager::ParseOrigin(aOriginScope.GetOrigin(),
+                                              spec,
+                                              &attrs))) {
+      return NS_ERROR_FAILURE;
+    }
+
+    RefPtr<ArchivedOriginScopeHelper> helper =
+      new ArchivedOriginScopeHelper(spec, attrs, /* aPrefix */ false);
+
+    rv = helper->BlockAndReturnArchivedOriginScope(archivedOriginScope);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else if (aOriginScope.IsPrefix()) {
+    nsCString spec;
+    OriginAttributes attrs;
+    if (NS_WARN_IF(!QuotaManager::ParseOrigin(aOriginScope.GetOriginNoSuffix(),
+                                              spec,
+                                              &attrs))) {
+      return NS_ERROR_FAILURE;
+    }
+
+    RefPtr<ArchivedOriginScopeHelper> helper =
+      new ArchivedOriginScopeHelper(spec, attrs, /* aPrefix */ true);
+
+    rv = helper->BlockAndReturnArchivedOriginScope(archivedOriginScope);
+    if (NS_WARN_IF(NS_FAILED(rv))) {
+      return rv;
+    }
+  } else if (aOriginScope.IsPattern()) {
+    archivedOriginScope =
+      ArchivedOriginScope::CreateFromPattern(aOriginScope.GetPattern());
+  } else {
+    MOZ_ASSERT(aOriginScope.IsNull());
+
+    archivedOriginScope = ArchivedOriginScope::CreateFromNull();
+  }
+
+  MOZ_ASSERT(archivedOriginScope);
+
+  aArchivedOriginScope = std::move(archivedOriginScope);
+  return NS_OK;
+}
+
+nsresult
+QuotaClient::PerformDelete(mozIStorageConnection* aConnection,
+                           const nsACString& aSchemaName,
+                           ArchivedOriginScope* aArchivedOriginScope) const
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aConnection);
+  MOZ_ASSERT(aArchivedOriginScope);
+
+  nsresult rv;
+
+  nsCString bindingClause;
+  aArchivedOriginScope->GetBindingClause(bindingClause);
+
+  nsCOMPtr<mozIStorageStatement> stmt;
+  rv = aConnection->CreateStatement(NS_LITERAL_CSTRING(
+    "DELETE FROM ") + aSchemaName + NS_LITERAL_CSTRING(".webappsstore2") +
+      bindingClause + NS_LITERAL_CSTRING(";"),
+    getter_AddRefs(stmt));
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = aArchivedOriginScope->BindToStatement(stmt);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  rv = stmt->Execute();
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 QuotaClient::
 ClearPrivateBrowsingRunnable::Run()
@@ -7403,5 +8281,39 @@ Observer::Observe(nsISupports* aSubject,
   return NS_OK;
 }
 
+NS_IMPL_ISUPPORTS(QuotaClient::MatchFunction, mozIStorageFunction)
+
+NS_IMETHODIMP
+QuotaClient::
+MatchFunction::OnFunctionCall(mozIStorageValueArray* aFunctionArguments,
+                              nsIVariant** aResult)
+{
+  AssertIsOnIOThread();
+  MOZ_ASSERT(aFunctionArguments);
+  MOZ_ASSERT(aResult);
+
+  nsCString suffix;
+  nsresult rv = aFunctionArguments->GetUTF8String(1, suffix);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  OriginAttributes oa;
+  if (NS_WARN_IF(!oa.PopulateFromSuffix(suffix))) {
+    return NS_ERROR_FAILURE;
+  }
+
+  bool result = mPattern.Matches(oa);
+
+  RefPtr<nsVariant> outVar(new nsVariant());
+  rv = outVar->SetAsBool(result);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return rv;
+  }
+
+  outVar.forget(aResult);
+  return NS_OK;
+}
+
 } // namespace dom
 } // namespace mozilla

From 4220031bba21787f0c40054d469a9b5b3b1eb92e Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:49:58 +0100
Subject: [PATCH 65/78] Bug 1286798 - Part 51: Add tests for archive and shadow
 database clearing; r=asuth

---
 dom/localstorage/moz.build                    |   4 +
 .../test/unit/databaseShadowing-shared.js     | 114 ++++++++++++++++++
 dom/localstorage/test/unit/head.js            |  40 ++++++
 .../test/unit/migration_profile.zip           | Bin 1727 -> 2026 bytes
 .../test/unit/test_databaseShadowing1.js      |  43 ++-----
 .../test/unit/test_databaseShadowing2.js      |  34 +-----
 .../test_databaseShadowing_clearOrigin1.js    |  34 ++++++
 .../test_databaseShadowing_clearOrigin2.js    |  24 ++++
 ...atabaseShadowing_clearOriginsByPattern1.js |  34 ++++++
 ...atabaseShadowing_clearOriginsByPattern2.js |  25 ++++
 ...databaseShadowing_clearOriginsByPrefix1.js |  32 +++++
 ...databaseShadowing_clearOriginsByPrefix2.js |  25 ++++
 dom/localstorage/test/unit/test_migration.js  |  99 +++++++++++++--
 dom/localstorage/test/unit/xpcshell.ini       |  12 ++
 14 files changed, 444 insertions(+), 76 deletions(-)
 create mode 100644 dom/localstorage/test/unit/databaseShadowing-shared.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
 create mode 100644 dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js

diff --git a/dom/localstorage/moz.build b/dom/localstorage/moz.build
index b3c05070b7ae..048e52e247c5 100644
--- a/dom/localstorage/moz.build
+++ b/dom/localstorage/moz.build
@@ -8,6 +8,10 @@ XPCSHELL_TESTS_MANIFESTS += [
     'test/unit/xpcshell.ini'
 ]
 
+TEST_HARNESS_FILES.xpcshell.dom.localstorage.test.unit += [
+    'test/unit/databaseShadowing-shared.js',
+]
+
 XPIDL_SOURCES += [
     'nsILocalStorageManager.idl',
 ]
diff --git a/dom/localstorage/test/unit/databaseShadowing-shared.js b/dom/localstorage/test/unit/databaseShadowing-shared.js
new file mode 100644
index 000000000000..82b2cbb43dc8
--- /dev/null
+++ b/dom/localstorage/test/unit/databaseShadowing-shared.js
@@ -0,0 +1,114 @@
+const principalInfos = [
+  { url: "http://example.com", attrs: {} },
+
+  { url: "http://origin.test", attrs: {} },
+
+  { url: "http://prefix.test", attrs: {} },
+  { url: "http://prefix.test", attrs: { userContextId: 10 } },
+
+  { url: "http://pattern.test", attrs: { userContextId: 15 } },
+  { url: "http://pattern.test:8080", attrs: { userContextId: 15 } },
+  { url: "https://pattern.test", attrs: { userContextId: 15 } },
+];
+
+function enableNextGenLocalStorage()
+{
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+}
+
+function disableNextGenLocalStorage()
+{
+  info("Setting pref");
+
+  Services.prefs.setBoolPref("dom.storage.next_gen", false);
+}
+
+function storeData()
+{
+  for (let i = 0; i < principalInfos.length; i++) {
+    let principalInfo = principalInfos[i];
+    let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
+
+    info("Getting storage");
+
+    let storage = getLocalStorage(principal);
+
+    info("Adding data");
+
+    storage.setItem("key0", "value0");
+    storage.clear();
+    storage.setItem("key1", "value1");
+    storage.removeItem("key1");
+    storage.setItem("key2", "value2");
+
+    info("Closing storage");
+
+    storage.close();
+  }
+}
+
+function exportShadowDatabase(name)
+{
+  info("Verifying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  let shadowDatabase = profileDir.clone();
+  shadowDatabase.append("webappsstore.sqlite");
+
+  let exists = shadowDatabase.exists();
+  ok(exists, "Shadow database does exist");
+
+  info("Copying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  shadowDatabase.copyTo(currentDir, name);
+}
+
+function importShadowDatabase(name)
+{
+  info("Verifying shadow database");
+
+  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
+  let shadowDatabase = currentDir.clone();
+  shadowDatabase.append(name);
+
+  let exists = shadowDatabase.exists();
+  if (!exists) {
+    return false;
+  }
+
+  info("Copying shadow database");
+
+  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
+  shadowDatabase.copyTo(profileDir, "webappsstore.sqlite");
+
+  return true;
+}
+
+function verifyData(clearedOrigins)
+{
+  for (let i = 0; i < principalInfos.length; i++) {
+    let principalInfo = principalInfos[i];
+    let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
+
+    info("Getting storage");
+
+    let storage = getLocalStorage(principal);
+
+    info("Verifying data");
+
+    if (clearedOrigins.includes(i)) {
+      ok(storage.getItem("key2") == null, "Correct value");
+    } else {
+      ok(storage.getItem("key0") == null, "Correct value");
+      ok(storage.getItem("key1") == null, "Correct value");
+      ok(storage.getItem("key2") == "value2", "Correct value");
+    }
+
+    info("Closing storage");
+
+    storage.close();
+  }
+}
diff --git a/dom/localstorage/test/unit/head.js b/dom/localstorage/test/unit/head.js
index 9d9e9cdbe713..f3134d6a4dca 100644
--- a/dom/localstorage/test/unit/head.js
+++ b/dom/localstorage/test/unit/head.js
@@ -104,6 +104,39 @@ function clear(callback)
   return request;
 }
 
+function clearOriginsByPattern(pattern, callback)
+{
+  let request = Services.qms.clearStoragesForOriginAttributesPattern(pattern);
+  request.callback = callback;
+
+  return request;
+}
+
+function clearOriginsByPrefix(principal, persistence, callback)
+{
+  let request =
+    Services.qms.clearStoragesForPrincipal(principal, persistence, null, true);
+  request.callback = callback;
+
+  return request;
+}
+
+function clearOrigin(principal, persistence, callback)
+{
+  let request = Services.qms.clearStoragesForPrincipal(principal, persistence);
+  request.callback = callback;
+
+  return request;
+}
+
+function reset(callback)
+{
+  let request = Services.qms.reset();
+  request.callback = callback;
+
+  return request;
+}
+
 function resetOrigin(principal, callback)
 {
   let request =
@@ -227,3 +260,10 @@ function getLocalStorage(principal)
 
   return Services.domStorageManager.createStorage(null, principal, "");
 }
+
+function loadSubscript(path)
+{
+  let file = do_get_file(path, false);
+  let uri = Services.io.newFileURI(file);
+  Services.scriptloader.loadSubScript(uri.spec);
+}
diff --git a/dom/localstorage/test/unit/migration_profile.zip b/dom/localstorage/test/unit/migration_profile.zip
index ff1cd89602076ad5bd332b0da22aee4e498c6648..19dc3d480537b14daef5f22fa74d6114d23b91f9 100644
GIT binary patch
delta 1641
zcmV-v2A2814eAenP)h>@6aWAK2mth58%<W-@-tup008y{000aC8~|f-Wnpw>Uu0q~
zYI9Xo2>=7U$b(z8$b(yTcnbgl1oZ&`00a~O005m-(N5bi6n)QET;(Z1<&=j>s0>Yr
zNvP5$p>=ygJQ(LT@#ff(ebZ7<|9#hK(o!g6;wkd=ImhRJUY|HWP5>_L3Y2SBUY8Yt
z4p3mBl_V^H98iUD)7~4c;H#^pwt^`YQ8USN<x!GepmKHXY!J?udDCHw4w^rrcloIp
z=gD!NBv+MpHU1)Pg3xv2JY}iJ;99Sc)k=?f?TXf5$Y<ymT44~sL!BO-C&@y3APjVk
z7QH0r;i6lA&j_24MZ~<dinO!G&+Ew(yH)Ispz+!&-AFUFa~GI?%Ub|X(0Vh0v~S5c
zB~9f5rLZ0y0xXwHQM*TNjEw!h1;YnfHwFcd<~>MM8G$77VG=O`{gtzX4>T<%@CnwD
ze@MXbjq!cNcAp8n32!{k^}|8<f1_WzI~~7muVE5@`d9v5N%S4QqcAxepN&5t6F%bP
z?mXG%rjR5#El-{DA6&8S+_`jf(n6XRxpFUk%7t~u8<Gem(zlqW8Ru1&nC(UIh>;sn
zgT#h4AF_?TcEaNF*N^EwM{r-9^?}qi!nA$NruyYzHFmw;32Y&7(bZt*ASkaJnxSH6
zwE8Z8EjQpr_s8RvEAU<hycy^jgZjDl_B>MS?M{v#gnkP9Qp0F$Ys_fpX9Q{5DXrr`
zH8+r@*GQyo31LGC-Mg^qV<^SH`YzJu{6J)JCr)*I&Y#aO9sps`mQ?Z9$;r{Pi|D18
z_Rrf3xB9McbvRr!ZK$$-_0s%hQAF=COug?bLl(DE2h0t7e6R>eQ^Gona)L7nC&2UX
zR$U_FdgoxStq#>s4u5<o*U4X#Zvuo01b-M!9Kb!1Xake|0wRBS4uD&D4uD&Acnbgl
z1oZ&`00a~O008aX&u$x46bE42shu{+ON#{(vJg^Rsg3Q#lccVKC?$)Cs;H1kl@N_4
z?j*Hpdn|i`-HO`niVZITFM?RHVaX$~X3GkR$sZM`PO7j1!S_kFe9t|<duB$OwLiai
zr<FwI-MHTglkzHZ=0zr#%Y0ZaXEK>w`kBl8<zRhbYMni<y-6;!`2LIH!H@B8nMptW
z0+TTX9De`+;9uZnHaGwF^~>3pH(T9D(J1b>_FCQTC(+aG!}T!Q346W4Ac_0Y(%kLO
zHg0Wgl<(i&ymx=2ym|NIjR)n|6Xj3umQSRv9Ch4El74GvKZyp{jx(P`Pp_5J+|`$F
zW#`|%oL<BIB;097C(<Vtx#J7iy7keWjgwQ#GdE+(TN@9yre+<l2hF$_9shpvs`^#f
z-ao2dEv8Rp;p;y#lb<KwPPWFskAE0{Grsctr{_Cq5tA_m9De`+`2T1w6e>6ON~MSU
zgQ$Ny?k3SF*?hEK3z9*UEOz@*l7zj*#^WUE4I1_Oa<#GBnCWet(YzbSH+I7Qjpn6u
z4{5Ab*Q)0oa@d>>SxbkkZC@-@R-5M?@;64L16I=kt6$C+DnU3qpn8}ZwRWTao6oN{
zPU}pMUrCQ&S$`=Mia}5+oz`<kE=Y61N`5NWZSA%C@yXn4wKP{xbM-GS6s`x`Gq)4$
zrM*$R*9k|_o9-rPoYt5gzMLMuyfSxmQJuIs({oxa%`K(5r4Mq2;`(~2bZ~aqZnvJr
zoyFmB_^R)$dYZ4L`Px#pP^{Gs9!C7Q9X8|k)T4O4k|SoSX{Nf6z6J~9Zf5f1<on6i
zB$)g<$&Y^-e>d(<2LJTy36n7e9De`+@ZYtRuiPwUAMOvL{_VJ%M5AQ$(RwXN22rxu
z?MF!x_KyCce$c4bmv`g%#=^NHjkW6G-`cNTs^%-J=Z&O+)r*yUC72CV4+EptZqz@~
zzcT-BzL@r(u^kkqw!5vpRzIHJuICr?*Jq~%dudzL?sdXZG~K>@;pntZ+(tZYdui^y
zd~v;$JxGV`cI#Q(SsV_B(=D~!JNaU*bSW9ckK180ZclGk{{&D=0RkQa6aWAK2mth5
z8%<W-@-tup008y{lf(rj1iZ+DTa!Ts8<PkI9s#D4Ck8bj1b-M!9Kb!1XafKM0Du7i
n695<h0000000000fPo+blY9mm1b7aBTa%gw9|qM100000o9i9P

delta 1359
zcmZ|PeKeB`90%}cHWs>F$F(SIa)xx*P^xQQSK%nH6-s%DMzhQ^jPC5I^j;DVE?x6-
zi?h66Z{^xr7Sks4(j_M3wM^db%uB*`yMNq!?m735@1MVOe&>5W=PUb`{|)GQ6pqjW
z0ALfKMP<0k#WI$u5&+yr0KjHI4+x1OosA|1o(sl@)4flq0&vb^wcq27#KfrsFoXgI
z0AH`KA16}#%r&@^D0OyPV_xSWGxw{8v@0(h)DbasHD=zEn#K%6<tS3K>ii;}B)SiO
zYP;=>WOJAkIQeA{?AcRm&$8Zle;)+F;mUQN)CwTV0VN2j-{aTd>w7dp`S`A1LLYt*
zKf>~eV=`&4LAIH^uz9PkCFnTwhsQ9%124yBc@K0g9hM^UCx!9iW62FQ0XvG*(Q+xO
zwBdnSdt|l1fOVa~d*+gwH1DAE5w`o1i{zz07@z^}-*NX=3<X6FGaGD!4#XDs=n_Y2
zynjmxrM|_RuV$4419^&2wFwQ;4@PcdN(Jq15?F0tDW|`9zmt#)CJx;#(M`Nsyo|vX
zxjJYrn<r&nW{u;9)LZYGGH+}l6q;EL>l8e|TD>@3{#ln%QWuNw=U1i_I_8u-_K0s+
zaKT^ocBVXUx}{#`nUTq^j%s+t;}C<Z9%8JRs#DjRrrsnkn>y2s%RsEcA~W_*uLnPy
zEqvYY^&YrV&r)$l665>Zl~C4?`Wa-a%i61&!>>`#Qehb9OTxVTu7(`vaGtGr?*5kD
z*+-JPdb%@39c_rtYF}A!{zLe=3AiR1Jbkd|_(6Wrir&ahymU`zH@E5(_jYN(;B4E}
zO}3UwgbOOr-`}6RqLHZeW^Gv^s0!HP)!gj(=Eg2vcTDS`w&OhdC-Q89WSA(sS2uSb
zvlT{mKN;rbKI<+((biX=48Dr9Bb=lZA5uW;oH>4<5eWQ4ZaWp)htz-sO6S$|H}s8C
zpuN#bkR{v@tlKd5SN`K{s)HEWcTNXMGjX$qi94jQYdCyJ=XRqfg2Ft5BclA^UE8ZW
zv5&o}vf7SpLZyG|R7LyD<t06)DwVlea;9#$l2c6?QtM>di@H~ASZgzs-a)%~8$9%J
zXN$?jZ4n52gyPa{ycR~8=viKt{ytJWX5@(f>4h~n+w%#tbb0WcY%wUs_W;y^K!d;<
zQbJK!Gv3a0N$mA=(Z+Hd29xZum}_FTuwE1<BHFOGI%bWzFi4X>4~<WoOZ$3Rl?U$C
zGM?+%B@MV94xBms_85kr(}QbaGHE`>O^KxO;?br=j=0|*ERX##&6DXbh?fl9$_mO+
ziZS(1RMuQv+-llKp+(2^>;!f?y(%NdLR=_{TK!VFl8~Lln;wYt2`reCB?rw-4mUTw
z42OMv^hMujmC%oFzUk$N4W)@pU}d@aRrQv1*_*bI*S%J9KRci5wz?&kFutcu@>b26
zu|C*kpIAR*dDIH6z5bQaM6_lb)Z$tX{tnX1z`mfGWv7~Dp^2g<v|RfJbRjxqyfz?x
z(x;C4w6^<Gw4ef430a~^6Jw37?&)OTtnU`;yPD7A?gV+^Sf{?!E=};)EdDN#yU)%q
zVv9g}F{ftOQlJ0YxU}~|w=6epta_c3kVaxVQDS)7-8pQO%ky-0hKT#lPH!G>=t5vO
z%132OLh;Sw)^KKO67|P-rF_2h)3up2$JkWiBOLE@#j1(tQJ4}M{(sx10|3y13L3_N
z$VfD_7m0>NLdTH%{_QXYsbw&+F@_9M@L3v-PLGa^BH`&5FOZ{2-Y1j+_+LJuCL~6s
KNO@yGfj<E<)0ImA

diff --git a/dom/localstorage/test/unit/test_databaseShadowing1.js b/dom/localstorage/test/unit/test_databaseShadowing1.js
index 3f053d43b7e1..6901ca95bf28 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing1.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing1.js
@@ -5,48 +5,21 @@
 
 var testGenerator = testSteps();
 
+loadSubscript("databaseShadowing-shared.js");
+
 function* testSteps()
 {
-  const url = "http://example.com";
+  enableNextGenLocalStorage();
 
-  info("Setting pref");
+  storeData();
 
-  Services.prefs.setBoolPref("dom.storage.next_gen", true);
+  verifyData([]);
 
-  let principal = getPrincipal(url);
-
-  info("Getting storage");
-
-  let storage = getLocalStorage(principal);
-
-  info("Adding data");
-
-  storage.setItem("key0", "value0");
-  storage.clear();
-  storage.setItem("key1", "value1");
-  storage.removeItem("key1");
-  storage.setItem("key2", "value2");
-
-  info("Closing storage");
-
-  storage.close();
-
-  resetOrigin(principal, continueToNextStepSync);
+  // Wait for all database connections to close.
+  reset(continueToNextStepSync);
   yield undefined;
 
-  info("Verifying shadow database");
-
-  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
-  let shadowDatabase = profileDir.clone();
-  shadowDatabase.append("webappsstore.sqlite");
-
-  let exists = shadowDatabase.exists();
-  ok(exists, "Shadow database does exist");
-
-  info("Copying shadow database");
-
-  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
-  shadowDatabase.copyTo(currentDir, "");
+  exportShadowDatabase("shadowdb.sqlite");
 
   // The shadow database is now prepared for test_databaseShadowing2.js
 
diff --git a/dom/localstorage/test/unit/test_databaseShadowing2.js b/dom/localstorage/test/unit/test_databaseShadowing2.js
index d00c1ec724f1..308cec2e3bb2 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing2.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing2.js
@@ -5,42 +5,20 @@
 
 var testGenerator = testSteps();
 
+loadSubscript("databaseShadowing-shared.js");
+
 function* testSteps()
 {
-  const url = "http://example.com";
+  // The shadow database was prepared in test_databaseShadowing1.js
 
-  // The shadow database is prepared in test_databaseShadowing1.js
+  disableNextGenLocalStorage();
 
-  info("Setting pref");
-
-  Services.prefs.setBoolPref("dom.storage.next_gen", false);
-
-  info("Verifying shadow database");
-
-  let currentDir = Services.dirsvc.get("CurWorkD", Ci.nsIFile);
-  let shadowDatabase = currentDir.clone();
-  shadowDatabase.append("webappsstore.sqlite");
-
-  let exists = shadowDatabase.exists();
-  if (!exists) {
+  if (!importShadowDatabase("shadowdb.sqlite")) {
     finishTest();
     return;
   }
 
-  info("Copying shadow database");
-
-  let profileDir = Services.dirsvc.get("ProfD", Ci.nsIFile);
-  shadowDatabase.copyTo(profileDir, "");
-
-  info("Getting storage");
-
-  let storage = getLocalStorage(getPrincipal(url));
-
-  info("Verifying data");
-
-  ok(storage.getItem("key0") == null, "Correct value");
-  ok(storage.getItem("key1") == null, "Correct value");
-  ok(storage.getItem("key2") == "value2", "Correct value");
+  verifyData([]);
 
   finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
new file mode 100644
index 000000000000..be5f758088a0
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
@@ -0,0 +1,34 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  enableNextGenLocalStorage();
+
+  storeData();
+
+  verifyData([]);
+
+  let principal = getPrincipal("http://origin.test", {});
+  clearOrigin(principal, "default", continueToNextStepSync);
+  yield undefined;
+
+  verifyData([1]);
+
+  // Wait for all database connections to close.
+  reset(continueToNextStepSync);
+  yield undefined;
+
+  exportShadowDatabase("shadowdb_clearedOrigin.sqlite");
+
+  // The shadow database is now prepared for
+  // test_databaseShadowing_clearOrigin2.js
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
new file mode 100644
index 000000000000..320b0a8b50b9
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
@@ -0,0 +1,24 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  // The shadow database was prepared in test_databaseShadowing_clearOrigin1.js
+
+  disableNextGenLocalStorage();
+
+  if (!importShadowDatabase("shadowdb-clearedOrigin.sqlite")) {
+    finishTest();
+    return;
+  }
+
+  verifyData([1]);
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
new file mode 100644
index 000000000000..b8b3445f2031
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
@@ -0,0 +1,34 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  enableNextGenLocalStorage();
+
+  storeData();
+
+  verifyData([]);
+
+  clearOriginsByPattern(JSON.stringify({ userContextId: 15 }),
+                        continueToNextStepSync);
+  yield undefined;
+
+  verifyData([4,5,6]);
+
+  // Wait for all database connections to close.
+  reset(continueToNextStepSync);
+  yield undefined;
+
+  exportShadowDatabase("shadowdb-clearedOriginsByPattern.sqlite");
+
+  // The shadow database is now prepared for
+  // test_databaseShadowing_clearOriginsByPattern2.js
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
new file mode 100644
index 000000000000..6fa8a8fd9142
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
@@ -0,0 +1,25 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  // The shadow database was prepared in
+  // test_databaseShadowing_clearOriginsByPattern1.js
+
+  disableNextGenLocalStorage();
+
+  if (!importShadowDatabase("shadowdb-clearedOriginsByPattern.sqlite")) {
+    finishTest();
+    return;
+  }
+
+  verifyData([4,5,6]);
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
new file mode 100644
index 000000000000..e9a0b413991b
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
@@ -0,0 +1,32 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  enableNextGenLocalStorage();
+
+  storeData();
+
+  verifyData([]);
+
+  let principal = getPrincipal("http://prefix.test", {});
+  clearOriginsByPrefix(principal, "default", continueToNextStepSync);
+  yield undefined;
+
+  // Wait for all database connections to close.
+  reset(continueToNextStepSync);
+  yield undefined;
+
+  exportShadowDatabase("shadowdb-clearedOriginsByPrefix.sqlite");
+
+  // The shadow database is now prepared for
+  // test_databaseShadowing_clearOriginsByPrefix2.js
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js
new file mode 100644
index 000000000000..763da6c886a9
--- /dev/null
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js
@@ -0,0 +1,25 @@
+/**
+ * Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+var testGenerator = testSteps();
+
+loadSubscript("databaseShadowing-shared.js");
+
+function* testSteps()
+{
+  // The shadow database was prepared in
+  // test_databaseShadowing_clearOriginsByPrefix1.js
+
+  disableNextGenLocalStorage();
+
+  if (!importShadowDatabase("shadowdb-clearedOriginsByPrefix.sqlite")) {
+    finishTest();
+    return;
+  }
+
+  verifyData([2,3]);
+
+  finishTest();
+}
diff --git a/dom/localstorage/test/unit/test_migration.js b/dom/localstorage/test/unit/test_migration.js
index 35c31ce67e47..b5ed386e43b1 100644
--- a/dom/localstorage/test/unit/test_migration.js
+++ b/dom/localstorage/test/unit/test_migration.js
@@ -11,7 +11,16 @@ function* testSteps()
     { url: "http://localhost", attrs: {} },
     { url: "http://www.mozilla.org", attrs: {} },
     { url: "http://example.com", attrs: {} },
-    { url: "http://example.org", attrs: { userContextId: 5 } }
+    { url: "http://example.org", attrs: { userContextId: 5 } },
+
+    { url: "http://origin.test", attrs: {} },
+
+    { url: "http://prefix.test", attrs: {} },
+    { url: "http://prefix.test", attrs: { userContextId: 10 } },
+
+    { url: "http://pattern.test", attrs: { userContextId: 15 } },
+    { url: "http://pattern.test:8080", attrs: { userContextId: 15 } },
+    { url: "https://pattern.test", attrs: { userContextId: 15 } },
   ];
 
   const data = {
@@ -19,10 +28,35 @@ function* testSteps()
     value: "bar"
   };
 
+  function verifyData(clearedOrigins) {
+    info("Getting storages");
+
+    let storages = [];
+    for (let i = 0; i < principalInfos.length; i++) {
+      let principalInfo = principalInfos[i];
+      let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
+      let storage = getLocalStorage(principal);
+      storages.push(storage);
+    }
+
+    info("Verifying data");
+
+    for (let i = 0; i < storages.length; i++) {
+      let value = storages[i].getItem(data.key + i);
+      if (clearedOrigins.includes(i)) {
+        is(value, null, "Correct value");
+      } else {
+        is(value, data.value + i, "Correct value");
+      }
+    }
+  }
+
   info("Setting pref");
 
   Services.prefs.setBoolPref("dom.storage.next_gen", true);
 
+  info("Stage 1 - Testing archived data migration");
+
   info("Clearing");
 
   clear(continueToNextStepSync);
@@ -36,21 +70,60 @@ function* testSteps()
   // mach xpcshell-test --interactive dom/localstorage/test/unit/create_db.js
   installPackage("migration_profile");
 
-  info("Getting storages");
+  verifyData([]);
 
-  let storages = [];
-  for (let i = 0; i < principalInfos.length; i++) {
-    let principalInfo = principalInfos[i];
-    let principal = getPrincipal(principalInfo.url, principalInfo.attrs);
-    let storage = getLocalStorage(principal);
-    storages.push(storage);
-  }
+  info("Stage 2 - Testing archived data clearing");
 
-  info("Verifying data");
+  for (let type of ["origin", "prefix", "pattern"]) {
+    info("Clearing");
 
-  for (let i = 0; i < storages.length; i++) {
-    let value = storages[i].getItem(data.key + i);
-    is(value, data.value + i, "Correct value");
+    clear(continueToNextStepSync);
+    yield undefined;
+
+    info("Installing package");
+
+    // See the comment for the first installPackage() call.
+    installPackage("migration_profile");
+
+    let clearedOrigins = [];
+
+    switch (type) {
+      case "origin": {
+        let principal = getPrincipal("http://origin.test", {});
+        clearOrigin(principal, "default", continueToNextStepSync);
+        yield undefined;
+
+        clearedOrigins.push(4);
+
+        break;
+      }
+
+      case "prefix": {
+        let principal = getPrincipal("http://prefix.test", {});
+        clearOriginsByPrefix(principal, "default", continueToNextStepSync);
+        yield undefined;
+
+        clearedOrigins.push(5, 6);
+
+        break;
+      }
+
+      case "pattern": {
+        clearOriginsByPattern(JSON.stringify({ userContextId: 15 }),
+                              continueToNextStepSync);
+        yield undefined;
+
+        clearedOrigins.push(7, 8, 9);
+
+        break;
+      }
+
+      default: {
+        throw("Unknown type: " + type);
+      }
+    }
+
+    verifyData(clearedOrigins);
   }
 
   finishTest();
diff --git a/dom/localstorage/test/unit/xpcshell.ini b/dom/localstorage/test/unit/xpcshell.ini
index ba2bfd3ac7a1..0ba82e9019cd 100644
--- a/dom/localstorage/test/unit/xpcshell.ini
+++ b/dom/localstorage/test/unit/xpcshell.ini
@@ -13,6 +13,18 @@ support-files =
 run-sequentially = test_databaseShadowing2.js depends on a file produced by this test
 [test_databaseShadowing2.js]
 run-sequentially = this test depends on a file produced by test_databaseShadowing1.js
+[test_databaseShadowing_clearOrigin1.js]
+run-sequentially = test_databaseShadowing_clearOrigin2.js depends on a file produced by this test
+[test_databaseShadowing_clearOrigin2.js]
+run-sequentially = this test depends on a file produced by test_databaseShadowing_clearOrigin1.js
+[test_databaseShadowing_clearOriginsByPattern1.js]
+run-sequentially = test_databaseShadowing_clearOriginsByPattern2.js depends on a file produced by this test
+[test_databaseShadowing_clearOriginsByPattern2.js]
+run-sequentially = this test depends on a file produced by test_databaseShadowing_clearOriginsByPattern1.js
+[test_databaseShadowing_clearOriginsByPrefix1.js]
+run-sequentially = test_databaseShadowing_clearOriginsByPrefix2.js depends on a file produced by this test
+[test_databaseShadowing_clearOriginsByPrefix2.js]
+run-sequentially = this test depends on a file produced by test_databaseShadowing_clearOriginsByPrefix1.js
 [test_eviction.js]
 [test_groupLimit.js]
 [test_migration.js]

From 35d317c9198681c7ae8064ed653f6a94a10e3b56 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:50:01 +0100
Subject: [PATCH 66/78] Bug 1286798 - Part 52: Rework tests to use async
 functions; r=asuth

---
 dom/localstorage/test/unit/head.js            | 66 ++++++++++---------
 dom/localstorage/test/unit/test_archive.js    | 18 ++---
 .../test/unit/test_databaseShadowing1.js      | 10 +--
 .../test/unit/test_databaseShadowing2.js      |  7 +-
 .../test_databaseShadowing_clearOrigin1.js    | 14 ++--
 .../test_databaseShadowing_clearOrigin2.js    |  7 +-
 ...atabaseShadowing_clearOriginsByPattern1.js | 15 ++---
 ...atabaseShadowing_clearOriginsByPattern2.js |  7 +-
 ...databaseShadowing_clearOriginsByPrefix1.js | 14 ++--
 ...databaseShadowing_clearOriginsByPrefix2.js |  7 +-
 dom/localstorage/test/unit/test_eviction.js   | 22 +++----
 dom/localstorage/test/unit/test_groupLimit.js | 13 ++--
 dom/localstorage/test/unit/test_migration.js  | 27 ++++----
 .../test/unit/test_snapshotting.js            | 33 +++-------
 14 files changed, 97 insertions(+), 163 deletions(-)

diff --git a/dom/localstorage/test/unit/head.js b/dom/localstorage/test/unit/head.js
index f3134d6a4dca..61f69ab2daa6 100644
--- a/dom/localstorage/test/unit/head.js
+++ b/dom/localstorage/test/unit/head.js
@@ -29,32 +29,28 @@ if (!this.runTest) {
 
     enableTesting();
 
-    do_test_pending();
-    testGenerator.next();
+    Assert.ok(typeof testSteps === "function",
+              "There should be a testSteps function");
+    Assert.ok(testSteps.constructor.name === "AsyncFunction",
+              "testSteps should be an async function");
+
+    registerCleanupFunction(resetTesting);
+
+    add_task(testSteps);
+
+    // Since we defined run_test, we must invoke run_next_test() to start the
+    // async test.
+    run_next_test();
   }
 }
 
-function finishTest()
+function returnToEventLoop()
 {
-  resetTesting();
-
-  executeSoon(function() {
-    do_test_finished();
-  })
-}
-
-function continueToNextStep()
-{
-  executeSoon(function() {
-    testGenerator.next();
+  return new Promise(function(resolve) {
+    executeSoon(resolve);
   });
 }
 
-function continueToNextStepSync()
-{
-  testGenerator.next();
-}
-
 function enableTesting()
 {
   Services.prefs.setBoolPref("dom.storage.testing", true);
@@ -88,43 +84,38 @@ function resetOriginLimit()
   Services.prefs.clearUserPref("dom.storage.default_quota");
 }
 
-function getOriginUsage(principal, callback)
+function getOriginUsage(principal)
 {
-  let request = Services.qms.getUsageForPrincipal(principal, callback);
-  request.callback = callback;
+  let request = Services.qms.getUsageForPrincipal(principal, function() { });
 
   return request;
 }
 
-function clear(callback)
+function clear()
 {
   let request = Services.qms.clear();
-  request.callback = callback;
 
   return request;
 }
 
-function clearOriginsByPattern(pattern, callback)
+function clearOriginsByPattern(pattern)
 {
   let request = Services.qms.clearStoragesForOriginAttributesPattern(pattern);
-  request.callback = callback;
 
   return request;
 }
 
-function clearOriginsByPrefix(principal, persistence, callback)
+function clearOriginsByPrefix(principal, persistence)
 {
   let request =
     Services.qms.clearStoragesForPrincipal(principal, persistence, null, true);
-  request.callback = callback;
 
   return request;
 }
 
-function clearOrigin(principal, persistence, callback)
+function clearOrigin(principal, persistence)
 {
   let request = Services.qms.clearStoragesForPrincipal(principal, persistence);
-  request.callback = callback;
 
   return request;
 }
@@ -137,11 +128,10 @@ function reset(callback)
   return request;
 }
 
-function resetOrigin(principal, callback)
+function resetOrigin(principal)
 {
   let request =
     Services.qms.resetStoragesForPrincipal(principal, "default", "ls");
-  request.callback = callback;
 
   return request;
 }
@@ -261,6 +251,18 @@ function getLocalStorage(principal)
   return Services.domStorageManager.createStorage(null, principal, "");
 }
 
+function requestFinished(request) {
+  return new Promise(function(resolve, reject) {
+    request.callback = function(request) {
+      if (request.resultCode == Cr.NS_OK) {
+        resolve(request.result);
+      } else {
+        reject(request.resultCode);
+      }
+    }
+  });
+}
+
 function loadSubscript(path)
 {
   let file = do_get_file(path, false);
diff --git a/dom/localstorage/test/unit/test_archive.js b/dom/localstorage/test/unit/test_archive.js
index 02615c22968b..272d677b1064 100644
--- a/dom/localstorage/test/unit/test_archive.js
+++ b/dom/localstorage/test/unit/test_archive.js
@@ -3,9 +3,7 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
-function* testSteps()
+async function testSteps()
 {
   const lsArchiveFile = "storage/ls-archive.sqlite";
 
@@ -33,8 +31,8 @@ function* testSteps()
   // Profile 1 - Archive file is a directory.
   info("Clearing");
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  let request = clear();
+  await requestFinished(request);
 
   let archiveFile = getRelativeFile(lsArchiveFile);
 
@@ -45,8 +43,8 @@ function* testSteps()
   // Profile 2 - Corrupted archive file.
   info("Clearing");
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  request = clear();
+  await requestFinished(request);
 
   let ostream = Cc["@mozilla.org/network/file-output-stream;1"]
                 .createInstance(Ci.nsIFileOutputStream);
@@ -59,8 +57,8 @@ function* testSteps()
   // Profile 3 - Nonupdateable archive file.
   info("Clearing");
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  request = clear();
+  await requestFinished(request);
 
   info("Installing package");
 
@@ -75,6 +73,4 @@ function* testSteps()
   ok(fileSize > 0, "archive file size is greater than zero");
 
   checkStorage();
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing1.js b/dom/localstorage/test/unit/test_databaseShadowing1.js
index 6901ca95bf28..5533941a7489 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing1.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing1.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   enableNextGenLocalStorage();
 
@@ -16,12 +14,10 @@ function* testSteps()
   verifyData([]);
 
   // Wait for all database connections to close.
-  reset(continueToNextStepSync);
-  yield undefined;
+  let request = reset();
+  await requestFinished(request);
 
   exportShadowDatabase("shadowdb.sqlite");
 
   // The shadow database is now prepared for test_databaseShadowing2.js
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing2.js b/dom/localstorage/test/unit/test_databaseShadowing2.js
index 308cec2e3bb2..f0f5f7e7ae38 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing2.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing2.js
@@ -3,22 +3,17 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   // The shadow database was prepared in test_databaseShadowing1.js
 
   disableNextGenLocalStorage();
 
   if (!importShadowDatabase("shadowdb.sqlite")) {
-    finishTest();
     return;
   }
 
   verifyData([]);
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
index be5f758088a0..536fe0332d6f 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin1.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   enableNextGenLocalStorage();
 
@@ -16,19 +14,17 @@ function* testSteps()
   verifyData([]);
 
   let principal = getPrincipal("http://origin.test", {});
-  clearOrigin(principal, "default", continueToNextStepSync);
-  yield undefined;
+  let request = clearOrigin(principal, "default");
+  await requestFinished(request);
 
   verifyData([1]);
 
   // Wait for all database connections to close.
-  reset(continueToNextStepSync);
-  yield undefined;
+  request = reset();
+  await requestFinished(request);
 
   exportShadowDatabase("shadowdb_clearedOrigin.sqlite");
 
   // The shadow database is now prepared for
   // test_databaseShadowing_clearOrigin2.js
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
index 320b0a8b50b9..43f1d8b9c3ad 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOrigin2.js
@@ -3,22 +3,17 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   // The shadow database was prepared in test_databaseShadowing_clearOrigin1.js
 
   disableNextGenLocalStorage();
 
   if (!importShadowDatabase("shadowdb-clearedOrigin.sqlite")) {
-    finishTest();
     return;
   }
 
   verifyData([1]);
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
index b8b3445f2031..12ca2b770ffb 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern1.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   enableNextGenLocalStorage();
 
@@ -15,20 +13,17 @@ function* testSteps()
 
   verifyData([]);
 
-  clearOriginsByPattern(JSON.stringify({ userContextId: 15 }),
-                        continueToNextStepSync);
-  yield undefined;
+  let request = clearOriginsByPattern(JSON.stringify({ userContextId: 15 }));
+  await requestFinished(request);
 
   verifyData([4,5,6]);
 
   // Wait for all database connections to close.
-  reset(continueToNextStepSync);
-  yield undefined;
+  request = reset();
+  await requestFinished(request);
 
   exportShadowDatabase("shadowdb-clearedOriginsByPattern.sqlite");
 
   // The shadow database is now prepared for
   // test_databaseShadowing_clearOriginsByPattern2.js
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
index 6fa8a8fd9142..337eea04a5aa 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPattern2.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   // The shadow database was prepared in
   // test_databaseShadowing_clearOriginsByPattern1.js
@@ -15,11 +13,8 @@ function* testSteps()
   disableNextGenLocalStorage();
 
   if (!importShadowDatabase("shadowdb-clearedOriginsByPattern.sqlite")) {
-    finishTest();
     return;
   }
 
   verifyData([4,5,6]);
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
index e9a0b413991b..d2a84a245c19 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix1.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   enableNextGenLocalStorage();
 
@@ -16,17 +14,15 @@ function* testSteps()
   verifyData([]);
 
   let principal = getPrincipal("http://prefix.test", {});
-  clearOriginsByPrefix(principal, "default", continueToNextStepSync);
-  yield undefined;
+  let request = clearOriginsByPrefix(principal, "default");
+  await requestFinished(request);
 
   // Wait for all database connections to close.
-  reset(continueToNextStepSync);
-  yield undefined;
+  request = reset();
+  await requestFinished(request);
 
   exportShadowDatabase("shadowdb-clearedOriginsByPrefix.sqlite");
 
   // The shadow database is now prepared for
   // test_databaseShadowing_clearOriginsByPrefix2.js
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js
index 763da6c886a9..8d334c7fa0fb 100644
--- a/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js
+++ b/dom/localstorage/test/unit/test_databaseShadowing_clearOriginsByPrefix2.js
@@ -3,11 +3,9 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
 loadSubscript("databaseShadowing-shared.js");
 
-function* testSteps()
+async function testSteps()
 {
   // The shadow database was prepared in
   // test_databaseShadowing_clearOriginsByPrefix1.js
@@ -15,11 +13,8 @@ function* testSteps()
   disableNextGenLocalStorage();
 
   if (!importShadowDatabase("shadowdb-clearedOriginsByPrefix.sqlite")) {
-    finishTest();
     return;
   }
 
   verifyData([2,3]);
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_eviction.js b/dom/localstorage/test/unit/test_eviction.js
index a33210411009..5eec9c130d1d 100644
--- a/dom/localstorage/test/unit/test_eviction.js
+++ b/dom/localstorage/test/unit/test_eviction.js
@@ -3,9 +3,7 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
-function* testSteps()
+async function testSteps()
 {
   const globalLimitKB = 5 * 1024;
 
@@ -28,8 +26,8 @@ function* testSteps()
 
   setGlobalLimit(globalLimitKB);
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  let request = clear();
+  await requestFinished(request);
 
   info("Getting storages");
 
@@ -65,13 +63,13 @@ function* testSteps()
 
   let principal = getPrincipal("http://example0.com");
 
-  resetOrigin(principal, continueToNextStepSync);
-  yield undefined;
+  request = resetOrigin(principal);
+  await requestFinished(request);
 
   info("Getting usage for first origin");
 
-  let request = getOriginUsage(principal, continueToNextStepSync);
-  yield undefined;
+  request = getOriginUsage(principal);
+  await requestFinished(request);
 
   is(request.result.usage, data.sizeKB * 1024, "Correct usage");
 
@@ -83,10 +81,8 @@ function* testSteps()
 
   info("Getting usage for first origin");
 
-  request = getOriginUsage(principal, continueToNextStepSync);
-  yield undefined;
+  request = getOriginUsage(principal);
+  await requestFinished(request);
 
   is(request.result.usage, 0, "Zero usage");
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_groupLimit.js b/dom/localstorage/test/unit/test_groupLimit.js
index 568486f6e914..14b8aa5103b6 100644
--- a/dom/localstorage/test/unit/test_groupLimit.js
+++ b/dom/localstorage/test/unit/test_groupLimit.js
@@ -3,9 +3,7 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
-function* testSteps()
+async function testSteps()
 {
   const groupLimitKB = 10 * 1024;
 
@@ -34,8 +32,8 @@ function* testSteps()
 
   setGlobalLimit(globalLimitKB);
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  let request = clear();
+  await requestFinished(request);
 
   setOriginLimit(originLimit);
 
@@ -73,14 +71,11 @@ function* testSteps()
 
   // Let the internal snapshot finish (usage is not descreased until all
   // snapshots finish)..
-  continueToNextStep();
-  yield undefined;
+  await returnToEventLoop();
 
   info("Verifying more data can be written");
 
   for (let i = 0; i < urls.length; i++) {
     storages[i].setItem("B", "");
   }
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_migration.js b/dom/localstorage/test/unit/test_migration.js
index b5ed386e43b1..03ffaf91554c 100644
--- a/dom/localstorage/test/unit/test_migration.js
+++ b/dom/localstorage/test/unit/test_migration.js
@@ -3,9 +3,7 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
-function* testSteps()
+async function testSteps()
 {
   const principalInfos = [
     { url: "http://localhost", attrs: {} },
@@ -59,8 +57,8 @@ function* testSteps()
 
   info("Clearing");
 
-  clear(continueToNextStepSync);
-  yield undefined;
+  let request = clear();
+  await requestFinished(request);
 
   info("Installing package");
 
@@ -77,8 +75,8 @@ function* testSteps()
   for (let type of ["origin", "prefix", "pattern"]) {
     info("Clearing");
 
-    clear(continueToNextStepSync);
-    yield undefined;
+    request = clear();
+    await requestFinished(request);
 
     info("Installing package");
 
@@ -90,8 +88,8 @@ function* testSteps()
     switch (type) {
       case "origin": {
         let principal = getPrincipal("http://origin.test", {});
-        clearOrigin(principal, "default", continueToNextStepSync);
-        yield undefined;
+        request = clearOrigin(principal, "default");
+        await requestFinished(request);
 
         clearedOrigins.push(4);
 
@@ -100,8 +98,8 @@ function* testSteps()
 
       case "prefix": {
         let principal = getPrincipal("http://prefix.test", {});
-        clearOriginsByPrefix(principal, "default", continueToNextStepSync);
-        yield undefined;
+        request = clearOriginsByPrefix(principal, "default");
+        await requestFinished(request);
 
         clearedOrigins.push(5, 6);
 
@@ -109,9 +107,8 @@ function* testSteps()
       }
 
       case "pattern": {
-        clearOriginsByPattern(JSON.stringify({ userContextId: 15 }),
-                              continueToNextStepSync);
-        yield undefined;
+        request = clearOriginsByPattern(JSON.stringify({ userContextId: 15 }));
+        await requestFinished(request);
 
         clearedOrigins.push(7, 8, 9);
 
@@ -125,6 +122,4 @@ function* testSteps()
 
     verifyData(clearedOrigins);
   }
-
-  finishTest();
 }
diff --git a/dom/localstorage/test/unit/test_snapshotting.js b/dom/localstorage/test/unit/test_snapshotting.js
index 250d56b598f7..b9c735f35ce7 100644
--- a/dom/localstorage/test/unit/test_snapshotting.js
+++ b/dom/localstorage/test/unit/test_snapshotting.js
@@ -3,9 +3,7 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-var testGenerator = testSteps();
-
-function* testSteps()
+async function testSteps()
 {
   const url = "http://example.com";
 
@@ -72,8 +70,7 @@ function* testSteps()
     info("Returning to event loop");
 
     // Returning to event loop forces the internal snapshot to finish.
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 2nd snapshot
 
@@ -99,8 +96,7 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 3rd snapshot
 
@@ -136,8 +132,7 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 4th snapshot
 
@@ -156,8 +151,7 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 5th snapshot
 
@@ -181,8 +175,7 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 6th snapshot
     info("Verifying unknown item");
@@ -195,8 +188,7 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 7th snapshot
 
@@ -205,8 +197,7 @@ function* testSteps()
 
     savedKeys = Object.keys(storage);
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 8th snapshot
 
@@ -228,8 +219,7 @@ function* testSteps()
       is(keys[i], savedKeys[i], "Correct key");
     }
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
 
     // 9th snapshot
 
@@ -239,9 +229,6 @@ function* testSteps()
 
     info("Returning to event loop");
 
-    continueToNextStep();
-    yield undefined;
+    await returnToEventLoop();
   }
-
-  finishTest();
 }

From 6c6e230a770294859c36749920456461409a0782 Mon Sep 17 00:00:00 2001
From: Andrew Sutherland <asutherland@asutherland.org>
Date: Mon, 5 Nov 2018 14:04:39 -0500
Subject: [PATCH 67/78] Bug 1286798 - Part 53: Review code comments;
 r=janv,mrbkap,mccr8

---
 .../storage/nsIDOMStorageManager.idl          |   5 +
 dom/localstorage/ActorsChild.h                |  56 +++++
 dom/localstorage/ActorsParent.cpp             | 220 ++++++++++++++++++
 dom/localstorage/LSObject.cpp                 |  67 ++++++
 dom/localstorage/LSObject.h                   |  67 ++++++
 dom/localstorage/LSObserver.h                 |  18 ++
 dom/localstorage/LSSnapshot.h                 |  38 +++
 dom/localstorage/LocalStorageCommon.h         |  14 ++
 dom/localstorage/LocalStorageManager2.cpp     |   8 +
 dom/localstorage/LocalStorageManager2.h       |  23 ++
 dom/localstorage/PBackgroundLSDatabase.ipdl   |  85 +++++++
 dom/localstorage/PBackgroundLSObserver.ipdl   |  26 +++
 dom/localstorage/PBackgroundLSRequest.ipdl    |  39 ++++
 .../PBackgroundLSSimpleRequest.ipdl           |  20 ++
 dom/localstorage/PBackgroundLSSnapshot.ipdl   |  45 ++++
 dom/localstorage/nsILocalStorageManager.idl   |  14 ++
 dom/storage/Storage.h                         |  16 +-
 dom/storage/StorageNotifierService.h          |  16 ++
 dom/webidl/Storage.webidl                     |  25 +-
 ipc/glue/PBackground.ipdl                     |  11 +
 ipc/ipdl/sync-messages.ini                    |   8 +-
 21 files changed, 815 insertions(+), 6 deletions(-)

diff --git a/dom/interfaces/storage/nsIDOMStorageManager.idl b/dom/interfaces/storage/nsIDOMStorageManager.idl
index aad0aa2ba9da..894995ec9964 100644
--- a/dom/interfaces/storage/nsIDOMStorageManager.idl
+++ b/dom/interfaces/storage/nsIDOMStorageManager.idl
@@ -49,6 +49,11 @@ interface nsIDOMStorageManager : nsISupports
                         in AString aDocumentURI,
                         [optional] in bool aPrivate);
   /**
+   * DEPRECATED.  The only good reason to use this was if you were writing a
+   * test and wanted to hackily determine if a preload happened.  That's now
+   * covered by `nsILocalStorageManager.isPreloaded` and you should use that if
+   * that's what you want.  If LSNG is in use, this will throw.
+   *
    * Returns instance of DOM storage object for given principal.
    * If there is no storage managed for the scope, then null is returned and
    * no object is created.  Otherwise, an object (new) for the existing storage
diff --git a/dom/localstorage/ActorsChild.h b/dom/localstorage/ActorsChild.h
index 0acbae7db39b..a8dbbace27b8 100644
--- a/dom/localstorage/ActorsChild.h
+++ b/dom/localstorage/ActorsChild.h
@@ -31,6 +31,24 @@ class LSRequestChildCallback;
 class LSSimpleRequestChildCallback;
 class LSSnapshot;
 
+/**
+ * Minimal glue actor with standard IPC-managed new/delete existence that exists
+ * primarily to track the continued existence of the LSDatabase in the child.
+ * Most of the interesting bits happen via PBackgroundLSSnapshot.
+ *
+ * Mutual raw pointers are maintained between LSDatabase and this class that are
+ * cleared at either (expected) when the child starts the deletion process
+ * (SendDeleteMeInternal) or unexpected actor death (ActorDestroy).
+ *
+ * See `PBackgroundLSDatabase.ipdl` for more information.
+ *
+ *
+ * ## Low-Level Lifecycle ##
+ * - Created by LSObject::EnsureDatabase if it had to create a database.
+ * - Deletion begun by LSDatabase's destructor invoking SendDeleteMeInternal
+ *   which will result in the parent sending __delete__ which destroys the
+ *   actor.
+ */
 class LSDatabaseChild final
   : public PBackgroundLSDatabaseChild
 {
@@ -78,6 +96,14 @@ private:
                                     override;
 };
 
+/**
+ * Minimal IPC-managed (new/delete) actor that exists to receive and relay
+ * "storage" events from changes to LocalStorage that take place in other
+ * processes as their Snapshots are checkpointed to the canonical Datastore in
+ * the parent process.
+ *
+ * See `PBackgroundLSObserver.ipdl` for more info.
+ */
 class LSObserverChild final
   : public PBackgroundLSObserverChild
 {
@@ -119,6 +145,17 @@ private:
               const nsString& aNewValue) override;
 };
 
+/**
+ * Minimal glue IPC-managed (new/delete) actor that is used by LSObject and its
+ * RequestHelper to perform synchronous requests on top of an asynchronous
+ * protocol.
+ *
+ * Takes an `LSReuestChildCallback` to be invoked when a response is received
+ * via __delete__.
+ *
+ * See `PBackgroundLSRequest.ipdl`, `LSObject`, and `RequestHelper` for more
+ * info.
+ */
 class LSRequestChild final
   : public PBackgroundLSRequestChild
 {
@@ -172,6 +209,15 @@ protected:
   { }
 };
 
+/**
+ * Minimal glue IPC-managed (new/delete) actor used by `LocalStorageManager2` to
+ * issue asynchronous requests in an asynchronous fashion.
+ *
+ * Takes an `LSSimpleRequestChildCallback` to be invoked when a response is
+ * received via __delete__.
+ *
+ * See `PBackgroundLSSimpleRequest.ipdl` for more info.
+ */
 class LSSimpleRequestChild final
   : public PBackgroundLSSimpleRequestChild
 {
@@ -216,6 +262,16 @@ protected:
   { }
 };
 
+/**
+ * Minimal IPC-managed (new/delete) actor that lasts as long as its owning
+ * LSSnapshot.
+ *
+ * Mutual raw pointers are maintained between LSSnapshot and this class that are
+ * cleared at either (expected) when the child starts the deletion process
+ * (SendDeleteMeInternal) or unexpected actor death (ActorDestroy).
+ *
+ * See `PBackgroundLSSnapshot.ipdl` and `LSSnapshot` for more info.
+ */
 class LSSnapshotChild final
   : public PBackgroundLSSnapshotChild
 {
diff --git a/dom/localstorage/ActorsParent.cpp b/dom/localstorage/ActorsParent.cpp
index 506fb5255fcf..042658ca077b 100644
--- a/dom/localstorage/ActorsParent.cpp
+++ b/dom/localstorage/ActorsParent.cpp
@@ -119,9 +119,25 @@ static_assert(kSQLiteGrowthIncrement >= 0 &&
               kSQLiteGrowthIncrement < uint32_t(INT32_MAX),
               "Must be 0 (disabled) or a positive multiple of the page size!");
 
+/**
+ * The database name for LocalStorage data in a per-origin directory.
+ */
 #define DATA_FILE_NAME "data.sqlite"
+/**
+ * The journal corresponding to DATA_FILE_NAME.  (We don't use WAL mode.)
+ */
 #define JOURNAL_FILE_NAME "data.sqlite-journal"
 
+/**
+ * How long between the first moment we know we have data to be written on a
+ * `Connection` and when we should actually perform the write.  This helps
+ * limit disk churn under silly usage patterns and is historically consistent
+ * with the previous, legacy implementation.
+ *
+ * Note that flushing happens downstream of Snapshot checkpointing and its
+ * batch mechanism which helps avoid wasteful IPC in the case of silly content
+ * code.
+ */
 const uint32_t kFlushTimeoutMs = 5000;
 
 const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
@@ -129,13 +145,55 @@ const char kPrivateBrowsingObserverTopic[] = "last-pb-context-exited";
 const uint32_t kDefaultOriginLimitKB = 5 * 1024;
 const uint32_t kDefaultShadowWrites = true;
 const uint32_t kDefaultSnapshotPrefill = 4096;
+/**
+ * LocalStorage data limit as determined by summing up the lengths of all string
+ * keys and values.  This is consistent with the legacy implementation and other
+ * browser engines.  This value should really only ever change in unit testing
+ * where being able to lower it makes it easier for us to test certain edge
+ * cases.
+ */
 const char kDefaultQuotaPref[] = "dom.storage.default_quota";
+/**
+ * Should all mutations also be reflected in the "shadow" database, which is
+ * the legacy webappsstore.sqlite database.  When this is enabled, users can
+ * downgrade their version of Firefox and/or otherwise fall back to the legacy
+ * implementation without loss of data.  (Older versions of Firefox will
+ * recognize the presence of ls-archive.sqlite and purge it and the other
+ * LocalStorage directories so privacy is maintained.)
+ */
 const char kShadowWritesPref[] = "dom.storage.shadow_writes";
+/**
+ * Byte budget for sending data down to the LSSnapshot instance when it is first
+ * created.  If there is less data than this (measured by tallying the string
+ * length of the keys and values), all data is sent, otherwise partial data is
+ * sent.  See `Snapshot`.
+ */
 const char kSnapshotPrefillPref[] = "dom.storage.snapshot_prefill";
 
+/**
+ * The amount of time a PreparedDatastore instance should stick around after a
+ * preload is triggered in order to give time for the page to use LocalStorage
+ * without triggering worst-case synchronous jank.
+ */
 const uint32_t kPreparedDatastoreTimeoutMs = 20000;
 
+/**
+ * Cold storage for LocalStorage data extracted from webappsstore.sqlite at
+ * LSNG first-run that has not yet been migrated to its own per-origin directory
+ * by use.
+ *
+ * In other words, at first run, LSNG copies the contents of webappsstore.sqlite
+ * into this database.  As requests are made for that LocalStorage data, the
+ * contents are removed from this database and placed into per-origin QM
+ * storage.  So the contents of this database are always old, unused
+ * LocalStorage data that we can potentially get rid of at some point in the
+ * future.
+ */
 #define LS_ARCHIVE_FILE_NAME "ls-archive.sqlite"
+/**
+ * The legacy LocalStorage database.  Its contents are maintained as our
+ * "shadow" database so that LSNG can be disabled without loss of user data.
+ */
 #define WEB_APPS_STORE_FILE_NAME "webappsstore.sqlite"
 
 // Shadow database Write Ahead Log's maximum size is 512KB
@@ -948,6 +1006,16 @@ DetachShadowDatabase(mozIStorageConnection* aConnection)
  * Non-actor class declarations
  ******************************************************************************/
 
+/**
+ * Coalescing manipulation queue used by `Connection` and `DataStore`.  Used by
+ * `Connection` to buffer and coalesce manipulations applied to the Datastore
+ * in batches by Snapshot Checkpointing until flushed to disk.  Used by
+ * `Datastore` to update `DataStore::mOrderedItems` efficiently/for code
+ * simplification.  (DataStore does not actually depend on the coalescing, as
+ * mutations are applied atomically when a Snapshot Checkpoints, and with
+ * `Datastore::mValues` being updated at the same time the mutations are applied
+ * to Datastore's mWriteOptimizer.)
+ */
 class WriteOptimizer final
 {
   class WriteInfo;
@@ -1001,6 +1069,12 @@ public:
   PerformWrites(Connection* aConnection, bool aShadowWrites);
 };
 
+/**
+ * Base class for specific mutations.  Each subclass knows how to `Perform` the
+ * manipulation against a `Connection` and the "shadow" database (legacy
+ * webappsstore.sqlite database that exists so LSNG can be disabled/safely
+ * downgraded from.)
+ */
 class WriteOptimizer::WriteInfo
 {
 public:
@@ -1020,6 +1094,9 @@ public:
   virtual ~WriteInfo() = default;
 };
 
+/**
+ * SetItem mutation where the key did not previously exist.
+ */
 class WriteOptimizer::AddItemInfo
   : public WriteInfo
 {
@@ -1056,6 +1133,9 @@ private:
   Perform(Connection* aConnection, bool aShadowWrites) override;
 };
 
+/**
+ * SetItem mutation where the key already existed.
+ */
 class WriteOptimizer::UpdateItemInfo final
   : public AddItemInfo
 {
@@ -1100,6 +1180,9 @@ private:
   Perform(Connection* aConnection, bool aShadowWrites) override;
 };
 
+/**
+ * Clear mutation.
+ */
 class WriteOptimizer::ClearInfo final
   : public WriteInfo
 {
@@ -1300,6 +1383,7 @@ public:
     return mArchivedOriginScope;
   }
 
+  //////////////////////////////////////////////////////////////////////////////
   // Methods which can only be called on the owning thread.
 
   // This method is used to asynchronously execute a connection datastore
@@ -1332,6 +1416,7 @@ public:
   void
   EndUpdateBatch();
 
+  //////////////////////////////////////////////////////////////////////////////
   // Methods which can only be called on the connection thread.
 
   nsresult
@@ -1471,17 +1556,54 @@ private:
   ~ConnectionThread();
 };
 
+/**
+ * Canonical state of Storage for an origin, containing all keys and their
+ * values in the parent process.  Specifically, this is the state that will
+ * be handed out to freshly created Snapshots and that will be persisted to disk
+ * when the Connection's flush completes.  State is mutated in batches as
+ * Snapshot instances Checkpoint their mutations locally accumulated in the
+ * child LSSnapshots.
+ */
 class Datastore final
 {
   RefPtr<DirectoryLock> mDirectoryLock;
   RefPtr<Connection> mConnection;
   RefPtr<QuotaObject> mQuotaObject;
   nsCOMPtr<nsIRunnable> mCompleteCallback;
+  /**
+   * PrepareDatastoreOps register themselves with the Datastore at
+   * and unregister in PrepareDatastoreOp::Cleanup.
+   */
   nsTHashtable<nsPtrHashKey<PrepareDatastoreOp>> mPrepareDatastoreOps;
+  /**
+   * PreparedDatastore instances register themselves with their associated
+   * Datastore at construction time and unregister at destruction time.  They
+   * hang around for kPreparedDatastoreTimeoutMs in order to keep the Datastore
+   * from closing itself via MaybeClose(), thereby giving the document enough
+   * time to load and access LocalStorage.
+   */
   nsTHashtable<nsPtrHashKey<PreparedDatastore>> mPreparedDatastores;
+  /**
+   * A database is live (and in this hashtable) if it has a live LSDatabase
+   * actor.  There is at most one Database per origin per content process.  Each
+   * Database corresponds to an LSDatabase in its associated content process.
+   */
   nsTHashtable<nsPtrHashKey<Database>> mDatabases;
+  /**
+   * A database is active if it has a non-null `mSnapshot`.  As long as there
+   * are any active databases final deltas can't be calculated and
+   * `UpdateUsage()` can't be invoked.
+   */
   nsTHashtable<nsPtrHashKey<Database>> mActiveDatabases;
+  /**
+   * Non-authoritative hashtable representation of mOrderedItems for efficient
+   * lookup.
+   */
   nsDataHashtable<nsStringHashKey, nsString> mValues;
+  /**
+   * The authoritative ordered state of the Datastore; mValue also exists as an
+   * unordered hashtable for efficient lookup.
+   */
   nsTArray<LSItemInfo> mOrderedItems;
   nsTArray<int64_t> mPendingUsageDeltas;
   WriteOptimizer mWriteOptimizer;
@@ -1524,6 +1646,9 @@ public:
   bool
   IsPersistent() const
   {
+    // Private-browsing is forbidden from touching disk, but
+    // StorageAccess::eSessionScoped is allowed to touch disk because
+    // QuotaManager's storage for such origins is wiped at shutdown.
     return mPrivateBrowsingId == 0;
   }
 
@@ -1589,6 +1714,15 @@ public:
   void
   GetKeys(nsTArray<nsString>& aKeys) const;
 
+  //////////////////////////////////////////////////////////////////////////////
+  // Mutation Methods
+  //
+  // These are only called during Snapshot::RecvCheckpoint
+
+  /**
+   * Used by Snapshot::RecvCheckpoint to set a key/value pair as part of a an
+   * explicit batch.
+   */
   void
   SetItem(Database* aDatabase,
           const nsString& aDocumentURI,
@@ -1878,24 +2012,103 @@ private:
                                      override;
 };
 
+/**
+ * Attempts to capture the state of the underlying Datastore at the time of its
+ * creation so run-to-completion semantics can be honored.
+ *
+ * Rather than simply duplicate the contents of `DataStore::mValues` and
+ * `Datastore::mOrderedItems` at the time of their creation, the Snapshot tracks
+ * mutations to the Datastore as they happen, saving off the state of values as
+ * they existed when the Snapshot was created.  In other words, given an initial
+ * Datastore state of { foo: 'bar', bar: 'baz' }, the Snapshot won't store those
+ * values until it hears via `SaveItem` that "foo" is being over-written.  At
+ * that time, it will save off foo='bar' in mValues.
+ *
+ * ## Quota Allocation ##
+ *
+ * ## States ##
+ *
+ */
 class Snapshot final
   : public PBackgroundLSSnapshotParent
 {
+  /**
+   * The Database that owns this snapshot.  There is a 1:1 relationship between
+   * snapshots and databases.
+   */
   RefPtr<Database> mDatabase;
   RefPtr<Datastore> mDatastore;
+  /**
+   * The set of keys for which values have been sent to the child LSSnapshot.
+   * Cleared once all values have been sent as indicated by
+   * mLoadedItems.Count()==mTotalLength and therefore mLoadedAllItems should be
+   * true.  No requests should be received for keys already in this set, and
+   * this is enforced by fatal IPC error (unless fuzzing).
+   */
   nsTHashtable<nsStringHashKey> mLoadedItems;
+  /**
+   * The set of keys for which a RecvLoadItem request was received but there
+   * was no such key, and so null was returned.  The child LSSnapshot will also
+   * cache these values, so redundant requests are also handled with fatal
+   * process termination just like for mLoadedItems.  Also cleared when
+   * mLoadedAllItems becomes true because then the child can infer that all
+   * other values must be null.  (Note: this could also be done when
+   * mLoadKeysReceived is true as a further optimization, but is not.)
+   */
   nsTHashtable<nsStringHashKey> mUnknownItems;
+  /**
+   * Values that have changed in mDatastore as reported by SaveItem
+   * notifications that are not yet known to the child LSSnapshot.
+   *
+   * The naive way to snapshot the state of mDatastore would be to duplicate its
+   * internal mValues at the time of our creation, but that is wasteful if few
+   * changes are made to the Datastore's state.  So we only track values that
+   * are changed/evicted from the Datastore as they happen, as reported to us by
+   * SaveItem notifications.
+   */
   nsDataHashtable<nsStringHashKey, nsString> mValues;
+  /**
+   * Latched state of mDatastore's keys during a SaveItem notification with
+   * aAffectsOrder=true.  The ordered keys needed to be saved off so that a
+   * consistent ordering could be presented to the child LSSnapshot when it asks
+   * for them via RecvLoadKeys.
+   */
   nsTArray<nsString> mKeys;
   nsString mDocumentURI;
+  /**
+   * The number of key/value pairs that were present in the Datastore at the
+   * time the snapshot was created.  Once we have sent this many values to the
+   * child LSSnapshot, we can infer that it has received all of the keys/values
+   * and set mLoadedAllItems to true and clear mLoadedItems and mUnknownItems.
+   * Note that knowing the keys/values is not the same as knowing their ordering
+   * and so mKeys may be retained.
+   */
   uint32_t mTotalLength;
   int64_t mUsage;
   int64_t mPeakUsage;
+  /**
+   * True if SaveItem has saved mDatastore's keys into mKeys because a SaveItem
+   * notification with aAffectsOrder=true was received.
+   */
   bool mSavedKeys;
   bool mActorDestroyed;
   bool mFinishReceived;
   bool mLoadedReceived;
+  /**
+   * True if LSSnapshot's mLoadState should be LoadState::AllOrderedItems or
+   * LoadState::AllUnorderedItems.  It will be AllOrderedItems if the initial
+   * snapshot contained all the data or if the state was AllOrderedKeys and
+   * successive RecvLoadItem requests have resulted in the LSSnapshot being told
+   * all of the key/value pairs.  It will be AllUnorderedItems if the state was
+   * LoadState::Partial and successive RecvLoadItem requests got all the
+   * keys/values but the key ordering was not retrieved.
+   */
   bool mLoadedAllItems;
+  /**
+   * True if LSSnapshot's mLoadState should be LoadState::AllOrderedItems or
+   * AllOrderedKeys.  This can occur because of the initial snapshot, or because
+   * a RecvLoadKeys request was received.
+   */
   bool mLoadKeysReceived;
   bool mSentMarkDirty;
 
@@ -1933,6 +2146,11 @@ public:
     }
   }
 
+  /**
+   * Called via NotifySnapshots by Datastore whenever it is updating its
+   * internal state so that snapshots can save off the state of a value at the
+   * time of their creation.
+   */
   void
   SaveItem(const nsAString& aKey,
            const nsAString& aOldValue,
@@ -4984,6 +5202,8 @@ Datastore::NotifyObservers(Database* aDatabase,
 
   MOZ_ASSERT(array);
 
+  // We do not want to send information about events back to the content process
+  // that caused the change.
   PBackgroundParent* databaseBackgroundActor = aDatabase->Manager();
 
   for (Observer* observer : *array) {
diff --git a/dom/localstorage/LSObject.cpp b/dom/localstorage/LSObject.cpp
index abd842cdbb31..3203818ee599 100644
--- a/dom/localstorage/LSObject.cpp
+++ b/dom/localstorage/LSObject.cpp
@@ -28,26 +28,85 @@ class RequestHelper;
 StaticMutex gRequestHelperMutex;
 RequestHelper* gRequestHelper = nullptr;
 
+/**
+ * Main-thread helper that implements the blocking logic required by
+ * LocalStorage's synchronous semantics.  StartAndReturnResponse pushes an
+ * event queue which is a new event target and spins its nested event loop until
+ * a result is received or an abort is necessary due to a PContent-managed sync
+ * IPC message being received.  Note that because the event queue is its own
+ * event target, there is no re-entrancy.  Normal main-thread runnables will not
+ * get a chance to run.  See StartAndReturnResponse() for info on this choice.
+ *
+ * The normal life-cycle of this method looks like:
+ * - Main Thread: LSObject::DoRequestSynchronously creates a RequestHelper and
+ *   invokes StartAndReturnResponse().  It pushes the event queue and Dispatches
+ *   the RequestHelper to the DOM File Thread.
+ * - DOM File Thread: RequestHelper::Run is called, invoking Start() which
+ *   invokes LSObject::StartRequest, which gets-or-creates the PBackground actor
+ *   if necessary (which may dispatch a runnable to the nested event queue on
+ *   the main thread), sends LSRequest constructor which is provided with a
+ *   callback reference to the RequestHelper. State advances to ResponsePending.
+ * - DOM File Thread:: LSRequestChild::Recv__delete__ is received, which invokes
+ *   RequestHelepr::OnResponse, advancing the state to Finishing and dispatching
+ *   RequestHelper to its own nested event target.
+ * - Main Thread: RequestHelper::Run is called, invoking Finish() which advances
+ *   the state to Complete and sets mWaiting to false, allowing the nested event
+ *   loop being spun by StartAndReturnResponse to cease spinning and return the
+ *   received response.
+ *
+ * See LocalStorageCommon.h for high-level context and method comments for
+ * low-level details.
+ */
 class RequestHelper final
   : public Runnable
   , public LSRequestChildCallback
 {
   enum class State
   {
+    /**
+     * The RequestHelper has been created and dispatched to the DOM File Thread.
+     */
     Initial,
+    /**
+     * Start() has been invoked on the DOM File Thread and
+     * LSObject::StartRequest has been invoked from there, sending an IPC
+     * message to PBackground to service the request.  We stay in this state
+     * until a response is received.
+     */
     ResponsePending,
+    /**
+     * A response has been received and RequestHelper has been dispatched back
+     * to the nested event loop to call Finish().
+     */
     Finishing,
+    /**
+     * Finish() has been called on the main thread.  The nested event loop will
+     * terminate imminently and the received response returned to the caller of
+     * StartAndReturnResponse.
+     */
     Complete
   };
 
+  // The object we are issuing a request on behalf of.  Present because of the
+  // need to invoke LSObject::StartRequest off the main thread.  Dropped on
+  // return to the main-thread in Finish().
   RefPtr<LSObject> mObject;
+  // The thread the RequestHelper was created on.  This should be the main
+  // thread.
   nsCOMPtr<nsIEventTarget> mOwningEventTarget;
+  // The pushed event queue that we use to spin the event loop without
+  // processing any of the events dispatched at the mOwningEventTarget (which
+  // would result in re-entrancy and violate LocalStorage semantics).
   nsCOMPtr<nsIEventTarget> mNestedEventTarget;
+  // The IPC actor handling the request with standard IPC allocation rules.
+  // Our reference is nulled in OnResponse which corresponds to the actor's
+  // __destroy__ method.
   LSRequestChild* mActor;
   const LSRequestParams mParams;
   LSRequestResponse mResponse;
   nsresult mResultCode;
   State mState;
+  // Control flag for the nested event loop; once set to false, the loop ends.
   bool mWaiting;
 
 public:
@@ -350,6 +409,14 @@ LSObject::GetOriginQuotaUsage() const
 {
   AssertIsOnOwningThread();
 
+  // It's not necessary to return an actual value here.  This method is
+  // implemented only because the SessionStore currently needs it to cap the
+  // amount of data it persists to disk (via nsIDOMWindowUtils.getStorageUsage).
+  // Any callers that want to know about storage usage should be asking
+  // QuotaManager directly.
+  //
+  // Note: This may change as LocalStorage is repurposed to be the new
+  // SessionStorage backend.
   return 0;
 }
 
diff --git a/dom/localstorage/LSObject.h b/dom/localstorage/LSObject.h
index f1b3cee37f38..b8311a036118 100644
--- a/dom/localstorage/LSObject.h
+++ b/dom/localstorage/LSObject.h
@@ -33,6 +33,28 @@ class LSRequestChildCallback;
 class LSRequestParams;
 class LSRequestResponse;
 
+/**
+ * Backs the WebIDL `Storage` binding; all content LocalStorage calls are
+ * handled by this class.
+ *
+ * ## Semantics under e10s / multi-process ##
+ *
+ * A snapshot mechanism used in conjuction with stable points ensures that JS
+ * run-to-completion semantics are experienced even if the same origin is
+ * concurrently accessing LocalStorage across multiple content processes.
+ *
+ * ### Snapshot Consistency ###
+ *
+ * An LSSnapshot is created locally whenever the contents of LocalStorage are
+ * about to be read or written (including length).  This synchronously
+ * establishes a corresponding Snapshot in PBackground in the parent process.
+ * An effort is made to send as much data from the parent process as possible,
+ * so sites using a small/reasonable amount of LocalStorage data will have it
+ * sent to the content process for immediate access.  Sites with greater
+ * LocalStorage usage may only have some of the information relayed.  In that
+ * case, the parent Snapshot will ensure that it retains the exact state of the
+ * parent Datastore at the moment the Snapshot was created.
+ */
 class LSObject final
   : public Storage
 {
@@ -52,10 +74,21 @@ class LSObject final
   bool mInExplicitSnapshot;
 
 public:
+  /**
+   * The normal creation path invoked by nsGlobalWindowInner.
+   */
   static nsresult
   CreateForWindow(nsPIDOMWindowInner* aWindow,
                   Storage** aStorage);
 
+  /**
+   * nsIDOMStorageManager creation path for use in testing logic.  Supports the
+   * system principal where CreateForWindow does not.  This is also why aPrivate
+   * exists separate from the principal; because the system principal can never
+   * be mutated to have a private browsing id even though it can be used in a
+   * window/document marked as private browsing.  That's a legacy issue that is
+   * being dealt with, but it's why it exists here.
+   */
   static nsresult
   CreateForPrincipal(nsPIDOMWindowInner* aWindow,
                      nsIPrincipal* aPrincipal,
@@ -71,6 +104,16 @@ public:
   static already_AddRefed<nsIEventTarget>
   GetSyncLoopEventTarget();
 
+  /**
+   * Helper invoked by ContentChild::OnChannelReceivedMessage when a sync IPC
+   * message is received.  This will be invoked on the IPC I/O thread and it's
+   * necessary to unblock the main thread when this happens to avoid the
+   * potential for browser deadlock.  This should only occur in (ugly) testing
+   * scenarios where CPOWs are in use.
+   *
+   * Cancellation will result in the underlying LSRequest being explicitly
+   * canceled, resulting in the parent sending an NS_ERROR_FAILURE result.
+   */
   static void
   CancelSyncLoop();
 
@@ -135,6 +178,8 @@ public:
   Clear(nsIPrincipal& aSubjectPrincipal,
         ErrorResult& aError) override;
 
+  //////////////////////////////////////////////////////////////////////////////
+  // Testing Methods: See Storage.h
   void
   Open(nsIPrincipal& aSubjectPrincipal,
        ErrorResult& aError) override;
@@ -151,6 +196,8 @@ public:
   EndExplicitSnapshot(nsIPrincipal& aSubjectPrincipal,
                       ErrorResult& aError) override;
 
+  //////////////////////////////////////////////////////////////////////////////
+
   NS_DECL_ISUPPORTS_INHERITED
   NS_DECL_CYCLE_COLLECTION_CLASS_INHERITED(LSObject, Storage)
 
@@ -170,12 +217,32 @@ private:
   void
   DropDatabase();
 
+  /**
+   * Invoked by nsGlobalWindowInner whenever a new "storage" event listener is
+   * added to the window in order to ensure that "storage" events are received
+   * from other processes.  (`LSObject::OnChange` directly invokes
+   * `Storage::NotifyChange` to notify in-process listeners.)
+   *
+   * If this is the first request in the process for an observer for this
+   * origin, this will trigger a RequestHelper-mediated synchronous LSRequest
+   * to prepare a new observer in the parent process and also construction of
+   * corresponding actors, which will result in the observer being fully
+   * registered in the parent process.
+   */
   nsresult
   EnsureObserver();
 
+  /**
+   * Invoked by nsGlobalWindowInner whenever its last "storage" event listener
+   * is removed.
+   */
   void
   DropObserver();
 
+  /**
+   * Internal helper method used by mutation methods that wraps the call to
+   * Storage::NotifyChange to generate same-process "storage" events.
+   */
   void
   OnChange(const nsAString& aKey,
            const nsAString& aOldValue,
diff --git a/dom/localstorage/LSObserver.h b/dom/localstorage/LSObserver.h
index 0266ceb31345..74a3d9aa43c1 100644
--- a/dom/localstorage/LSObserver.h
+++ b/dom/localstorage/LSObserver.h
@@ -12,6 +12,24 @@ namespace dom {
 
 class LSObserverChild;
 
+/**
+ * Effectively just a refcounted life-cycle management wrapper around
+ * LSObserverChild which exists to receive "storage" event information from
+ * other processes.  (Same-process events are handled within the process, see
+ * `LSObject::OnChange`.)
+ *
+ * ## Lifecycle ##
+ * - Created by LSObject::EnsureObserver via synchronous LSRequest idiom
+ *   whenever the first window's origin adds a "storage" event.  Placed in the
+ *   gLSObservers LSObserverHashtable for subsequent LSObject's via
+ *   LSObserver::Get lookup.
+ * - The LSObserverChild directly handles "Observe" messages, shunting them
+ *   directly to Storage::NotifyChange which does all the legwork of notifying
+ *   windows about "storage" events.
+ * - Destroyed when refcount goes to zero due to all owning LSObjects being
+ *   destroyed or having their `LSObject::DropObserver` methods invoked due to
+ *   the last "storage" event listener being removed from the owning window.
+ */
 class LSObserver final
 {
   friend class LSObject;
diff --git a/dom/localstorage/LSSnapshot.h b/dom/localstorage/LSSnapshot.h
index 5505700df244..25fa0f7f48a5 100644
--- a/dom/localstorage/LSSnapshot.h
+++ b/dom/localstorage/LSSnapshot.h
@@ -20,12 +20,50 @@ class LSSnapshot final
   : public nsIRunnable
 {
 public:
+  /**
+   * The LoadState expresses what subset of information a snapshot has from the
+   * authoritative Datastore in the parent process.  The initial snapshot is
+   * populated heuristically based on the size of the keys and size of the items
+   * (inclusive of the key value; item is key+value, not just value) of the
+   * entire datastore relative to the configured prefill limit (via pref
+   * "dom.storage.snapshot_prefill" exposed as gSnapshotPrefill in bytes).
+   *
+   * If there's less data than the limit, we send both keys and values and end
+   * up as AllOrderedItems.  If there's enough room for all the keys but not
+   * all the values, we end up as AllOrderedKeys with as many values present as
+   * would fit.  If there's not enough room for all the keys, then we end up as
+   * Partial with as many key-value pairs as will fit.
+   *
+   * The state AllUnorderedItems can only be reached by code getting items one
+   * by one.
+   */
   enum class LoadState
   {
+    /**
+     * Class constructed, Init(LSSnapshotInitInfo) has not been invoked yet.
+     */
     Initial,
+    /**
+     * Some keys and their values are known.
+     */
     Partial,
+    /**
+     * All the keys are known in order, but some values are unknown.
+     */
     AllOrderedKeys,
+    /**
+     * All keys and their values are known, but in an arbitrary order.
+     */
     AllUnorderedItems,
+    /**
+     * All keys and their values are known and are present in their canonical
+     * order.  This is everything, and is the preferred case.  The initial
+     * population will send this info when the size of all items is less than
+     * the prefill threshold.
+     *
+     * mValues will contain all keys and values, mLoadedItems and mUnknownItems
+     * are unused.
+     */
     AllOrderedItems,
     EndGuard
   };
diff --git a/dom/localstorage/LocalStorageCommon.h b/dom/localstorage/LocalStorageCommon.h
index 62d8b64834d0..4dffbbb5cf18 100644
--- a/dom/localstorage/LocalStorageCommon.h
+++ b/dom/localstorage/LocalStorageCommon.h
@@ -186,6 +186,13 @@ namespace dom {
 
 extern const char16_t* kLocalStorageType;
 
+/**
+ * Convenience data-structure to make it easier to track whether a value has
+ * changed and what its previous value was for notification purposes.  Instances
+ * are created on the stack by LSObject and passed to LSDatabase which in turn
+ * passes them onto LSSnapshot for final updating/population.  LSObject then
+ * generates an event, if appropriate.
+ */
 class MOZ_STACK_CLASS LSNotifyInfo
 {
   bool mChanged;
@@ -221,9 +228,16 @@ public:
   }
 };
 
+/**
+ * Main-thread-only check of LSNG being enabled, the value is latched once
+ * initialized so changing the preference during runtime has no effect.
+ */
 bool
 NextGenLocalStorageEnabled();
 
+/**
+ * Cached any-thread version of NextGenLocalStorageEnabled().
+ */
 bool
 CachedNextGenLocalStorageEnabled();
 
diff --git a/dom/localstorage/LocalStorageManager2.cpp b/dom/localstorage/LocalStorageManager2.cpp
index 278b74b289fd..6fe6785e88e5 100644
--- a/dom/localstorage/LocalStorageManager2.cpp
+++ b/dom/localstorage/LocalStorageManager2.cpp
@@ -133,6 +133,11 @@ LocalStorageManager2::PrecacheStorage(nsIPrincipal* aPrincipal,
   MOZ_ASSERT(aPrincipal);
   MOZ_ASSERT(_retval);
 
+  // This method was created as part of the e10s-ification of the old LS
+  // implementation to perform a preload in the content/current process.  That's
+  // not how things work in LSNG.  Instead everything happens in the parent
+  // process, triggered by the official preloading spot,
+  // ContentParent::AboutToLoadHttpFtpWyciwygDocumentForChild.
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
@@ -182,6 +187,8 @@ LocalStorageManager2::CloneStorage(Storage* aStorageToCloneFrom)
   MOZ_ASSERT(NS_IsMainThread());
   MOZ_ASSERT(aStorageToCloneFrom);
 
+  // Cloning is specific to sessionStorage; state is forked when a new tab is
+  // opened from an existing tab.
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
@@ -195,6 +202,7 @@ LocalStorageManager2::CheckStorage(nsIPrincipal* aPrincipal,
   MOZ_ASSERT(aStorage);
   MOZ_ASSERT(_retval);
 
+  // Only used by sessionStorage.
   return NS_ERROR_NOT_IMPLEMENTED;
 }
 
diff --git a/dom/localstorage/LocalStorageManager2.h b/dom/localstorage/LocalStorageManager2.h
index 6601466bd228..a20837d695b3 100644
--- a/dom/localstorage/LocalStorageManager2.h
+++ b/dom/localstorage/LocalStorageManager2.h
@@ -17,6 +17,19 @@ class LSRequestParams;
 class LSSimpleRequestParams;
 class Promise;
 
+/**
+ * Under LSNG this exposes nsILocalStorageManager::Preload to ContentParent to
+ * trigger preloading.  Otherwise, this is basically just a place for test logic
+ * that doesn't make sense to put directly on the Storage WebIDL interface.
+ *
+ * Previously, the nsIDOMStorageManager XPCOM interface was also used by
+ * nsGlobalWindowInner to interact with LocalStorage, but in these de-XPCOM
+ * days, we've moved to just directly reference the relevant concrete classes
+ * (ex: LSObject) directly.
+ *
+ * Note that testing methods are now also directly exposed on the Storage WebIDL
+ * interface for simplicity/sanity.
+ */
 class LocalStorageManager2 final
   : public nsIDOMStorageManager
   , public nsILocalStorageManager
@@ -31,10 +44,20 @@ public:
 private:
   ~LocalStorageManager2();
 
+  /**
+   * Helper to trigger an LSRequest and resolve/reject the provided promise when
+   * the result comes in.  This routine is notable because the LSRequest
+   * mechanism is normally used synchronously from content, but here it's
+   * exposed asynchronously.
+   */
   nsresult
   StartRequest(Promise* aPromise,
                const LSRequestParams& aParams);
 
+  /**
+   * Helper to trigger an LSSimpleRequst and resolve/reject the provided promise
+   * when the result comes in.
+   */
   nsresult
   StartSimpleRequest(Promise* aPromise,
                      const LSSimpleRequestParams& aParams);
diff --git a/dom/localstorage/PBackgroundLSDatabase.ipdl b/dom/localstorage/PBackgroundLSDatabase.ipdl
index 843ea3bd91aa..13765bc48c65 100644
--- a/dom/localstorage/PBackgroundLSDatabase.ipdl
+++ b/dom/localstorage/PBackgroundLSDatabase.ipdl
@@ -13,21 +13,67 @@ using mozilla::dom::LSSnapshot::LoadState
 namespace mozilla {
 namespace dom {
 
+/**
+ * LocalStorage key/value pair wire representations.  `value` may be void in
+ * cases where there is a value but it is not being sent for memory/bandwidth
+ * conservation purposes.  (It's not possible to have a null/undefined `value`
+ * as Storage is defined explicitly as a String store.)
+ */
 struct LSItemInfo
 {
   nsString key;
   nsString value;
 };
 
+/**
+ * Initial LSSnapshot state as produced by Datastore::GetSnapshotInitInfo.  See
+ * `LSSnapshot::LoadState` for more details about the possible states and a
+ * high level overview.
+ */
 struct LSSnapshotInitInfo
 {
+  /**
+   * As many key/value or key/void pairs as the snapshot prefill byte budget
+   * allowed.
+   */
   LSItemInfo[] itemInfos;
+  /**
+   * The total number of key/value pairs in LocalStorage for this origin at the
+   * time the snapshot was created.  (And the point of the snapshot is to
+   * conceptually freeze the state of the Datastore in time, so this value does
+   * not change despite what other LSDatabase objects get up to in other
+   * processes.)
+   */
   uint32_t totalLength;
+  /**
+   * The current amount of LocalStorage usage as measured by the summing the
+   * nsString Length() of both the key and the value over all stored pairs.
+   */
   int64_t initialUsage;
+  /**
+   * The amount of storage allowed to be used by the Snapshot without requesting
+   * more storage space via IncreasePeakUsage.  This is the `initialUsage` plus
+   * 0 or more bytes of space.  If space was available, the increase will be the
+   * `requestedSize` from the PBackgroundLSSnapshot constructor.  If the
+   * LocalStorage usage was already close to the limit, then the fallback is the
+   * `minSize` requested, or 0 if there wasn't space for that.
+   */
   int64_t peakUsage;
+  // See `LSSnapshot::LoadState` in `LSSnapshot.h`
   LoadState loadState;
 };
 
+/**
+ * This protocol is asynchronously created via constructor on PBackground but
+ * has synchronous semantics from the perspective of content on the main thread.
+ * The construction potentially involves waiting for disk I/O to load the
+ * LocalStorage data from disk as well as related QuotaManager checks, so async
+ * calls to PBackground are the only viable mechanism because blocking
+ * PBackground is not acceptable.  (Note that an attempt is made to minimize any
+ * I/O latency by triggering preloading from
+ * ContentParent::AboutToLoadHttpFtpWyciwygDocumentForChild, the central place
+ * for pre-loading.)
+ */
 sync protocol PBackgroundLSDatabase
 {
   manager PBackground;
@@ -45,8 +91,28 @@ parent:
   // and the parent then sends the __delete__ message to the child.
   async DeleteMe();
 
+  /**
+   * Sent in response to a `RequestAllowToClose` message once the snapshot
+   * cleanup has happened OR from LSDatabase's destructor if AllowToClose has
+   * not already been reported.
+   */
   async AllowToClose();
 
+  /**
+   * Invoked to create an LSSnapshot backed by a Snapshot in PBackground that
+   * presents an atomic and consistent view of the state of the authoritative
+   * Datastore state in the parent.
+   *
+   * This needs to be synchronous because LocalStorage's semantics are
+   * synchronous.  Note that the Datastore in the PBackground parent already
+   * has the answers to this request immediately available without needing to
+   * consult any other threads or perform any I/O.  Additionally, the response
+   * is explicitly bounded in size by the tunable snapshot prefill byte limit.
+   *
+   * @param increasePeakUsage
+   *   Whether the parent should attempt to pre-allocate some amount of quota
+   *   usage to the Snapshot.
+   */
   sync PBackgroundLSSnapshot(nsString documentURI,
                              bool increasePeakUsage,
                              int64_t requestedSize,
@@ -54,8 +120,27 @@ parent:
     returns (LSSnapshotInitInfo initInfo);
 
 child:
+  /**
+   * Only sent by the parent in response to the child's DeleteMe request.
+   */
   async __delete__();
 
+  /**
+   * Request to close the LSDatabase, checkpointing and finishing any
+   * outstanding snapshots so no state is lost.  This request is issued when
+   * QuotaManager is shutting down or is aborting operations for an origin or
+   * process.  Once the snapshot has cleaned up, AllowToClose will be sent to
+   * the parent.
+   *
+   * Note that the QuotaManager shutdown process is more likely to happen in
+   * unit tests where we explicitly reset the QuotaManager.  At runtime, we
+   * expect windows to be closed and content processes terminated well before
+   * QuotaManager shutdown would actually occur.
+   *
+   * Also, Operations are usually aborted for an origin due to privacy API's
+   * clearing data for an origin.  Operations are aborted for a process by
+   * ContentParent::ShutDownProcess.
+   */
   async RequestAllowToClose();
 };
 
diff --git a/dom/localstorage/PBackgroundLSObserver.ipdl b/dom/localstorage/PBackgroundLSObserver.ipdl
index 155f760588f2..56434075427d 100644
--- a/dom/localstorage/PBackgroundLSObserver.ipdl
+++ b/dom/localstorage/PBackgroundLSObserver.ipdl
@@ -9,16 +9,42 @@ include PBackgroundSharedTypes;
 namespace mozilla {
 namespace dom {
 
+/**
+ * The observer protocol sends "storage" event notifications for changes to
+ * LocalStorage that take place in other processes as their Snapshots are
+ * Checkpointed to the canonical Datastore in the parent process.  Same-process
+ * notifications are generated as mutations happen.
+ *
+ * Note that mutations are never generated for redundant mutations.  Setting the
+ * key "foo" to have value "bar" when it already has value "bar" will never
+ * result in a "storage" event.
+ */
 async protocol PBackgroundLSObserver
 {
   manager PBackground;
 
 parent:
+  /**
+   * Sent by the LSObserver's destructor when it's going away.  Any Observe
+   * messages received after this is sent will be ignored.  Which is fine,
+   * because there should be nothing around left to hear.  In the event a new
+   * page came into existence, its Observer creation will happen (effectively)
+   * synchronously.
+   */
   async DeleteMe();
 
 child:
+  /**
+   * Only sent by the parent in response to a deletion request.
+   */
   async __delete__();
 
+  /**
+   * Sent by the parent process as Snapshots from other processes are
+   * Checkpointed, applying their mutations.  The child actor currently directly
+   * shunts these to Storage::NotifyChange to generate "storage" events for
+   * immediate dispatch.
+   */
   async Observe(PrincipalInfo principalInfo,
                 uint32_t privateBrowsingId,
                 nsString documentURI,
diff --git a/dom/localstorage/PBackgroundLSRequest.ipdl b/dom/localstorage/PBackgroundLSRequest.ipdl
index 7432bfff6ecd..c30270be3b5c 100644
--- a/dom/localstorage/PBackgroundLSRequest.ipdl
+++ b/dom/localstorage/PBackgroundLSRequest.ipdl
@@ -26,6 +26,10 @@ struct LSRequestPrepareObserverResponse
   uint64_t observerId;
 };
 
+/**
+ * Discriminated union which can contain an error code (`nsresult`) or
+ * particular request response.
+ */
 union LSRequestResponse
 {
   nsresult;
@@ -33,6 +37,15 @@ union LSRequestResponse
   LSRequestPrepareObserverResponse;
 };
 
+/**
+ * An asynchronous protocol for issuing requests that are used in a synchronous
+ * fashion by LocalStorage via LSObject's RequestHelper mechanism.  This differs
+ * from LSSimpleRequest which is implemented and used asynchronously.
+ *
+ * See `PBackgroundLSSharedTypes.ipdlh` for more on the request types, the
+ * response types above for their corresponding responses, and `RequestHelper`
+ * for more on the usage and lifecycle of this mechanism.
+ */
 protocol PBackgroundLSRequest
 {
   manager PBackground;
@@ -50,13 +63,39 @@ parent:
   // because we are blocking the main thread in the content process.
   // The dead lock is prevented by canceling our nested event loop in the
   // content process when we receive a synchronous IPC message from the parent.
+  //
+  // Note that cancellation isn't instantaneous.  It's just an asynchronous flow
+  // that definitely doesn't involve the main thread in the parent process, so
+  // we're guaranteed to unblock the main-thread in the content process and
+  // allow the sync IPC to make progress.  When Cancel() is received by the
+  // parent, it will Send__delete__.  The child will either send Cancel or
+  // Finish, but not both.
   async Cancel();
 
+  /**
+   * Sent by the child in response to Ready, requesting that __delete__ be sent
+   * with the result.  The child will either send Finish or Cancel, but not
+   * both.  No further message will be sent from the child after invoking one.
+   */
   async Finish();
 
 child:
+  /**
+   * The deletion is sent with the result of the request directly in response to
+   * either Cancel or Finish.
+   */
   async __delete__(LSRequestResponse response);
 
+  /**
+   * Sent by the parent when it has completed whatever async stuff it needs to
+   * do and is ready to send the results.  It then awaits the Finish() call to
+   * send the results.  This may seem redundant, but it's not.  If the
+   * __delete__ was sent directly, it's possible there could be a race where
+   * Cancel() would be received by the parent after it had already sent
+   * __delete__.  (Which may no longer be fatal thanks to improvements to the
+   * IPC layer, but it would still lead to warnings, etc.  And we don't
+   * expect PBackground to be highly contended nor the DOM File thread.)
+   */
   async Ready();
 };
 
diff --git a/dom/localstorage/PBackgroundLSSimpleRequest.ipdl b/dom/localstorage/PBackgroundLSSimpleRequest.ipdl
index fc0c5132e921..c4bdd8c2771b 100644
--- a/dom/localstorage/PBackgroundLSSimpleRequest.ipdl
+++ b/dom/localstorage/PBackgroundLSSimpleRequest.ipdl
@@ -7,17 +7,37 @@ include protocol PBackground;
 namespace mozilla {
 namespace dom {
 
+/**
+ * Response to a `LSSimpleRequestPreloadedParams` request indicating whether the
+ * origin was preloaded.
+ */
 struct LSSimpleRequestPreloadedResponse
 {
   bool preloaded;
 };
 
+/**
+ * Discriminated union which can contain an error code (`nsresult`) or
+ * particular simple request response.
+ */
 union LSSimpleRequestResponse
 {
   nsresult;
   LSSimpleRequestPreloadedResponse;
 };
 
+/**
+ * Simple requests are async-only from both a protocol perspective and the
+ * manner in which they're used.  In comparison, PBackgroundLSRequests are
+ * async only from a protocol perspective; they are used synchronously from the
+ * main thread via LSObject's RequestHelper mechanism.  (With the caveat that
+ * nsILocalStorageManager does expose LSRequests asynchronously.)
+ *
+ * These requests use the common idiom where the arguments to the request are
+ * sent in the constructor and the result is sent in the __delete__ response.
+ * Request types are indicated by the Params variant used and those live in
+ * `PBackgroundLSSharedTypes.ipdlh`.
+ */
 protocol PBackgroundLSSimpleRequest
 {
   manager PBackground;
diff --git a/dom/localstorage/PBackgroundLSSnapshot.ipdl b/dom/localstorage/PBackgroundLSSnapshot.ipdl
index 316f741a0b93..5333fa4ea9ce 100644
--- a/dom/localstorage/PBackgroundLSSnapshot.ipdl
+++ b/dom/localstorage/PBackgroundLSSnapshot.ipdl
@@ -25,6 +25,9 @@ struct LSClearInfo
 {
 };
 
+/**
+ * Union of LocalStorage mutation types.
+ */
 union LSWriteInfo
 {
   LSSetItemInfo;
@@ -45,12 +48,42 @@ parent:
 
   async Loaded();
 
+  /**
+   * Invoked on demand to load an item that didn't fit into the initial
+   * snapshot prefill.
+   *
+   * This needs to be synchronous because LocalStorage's semantics are
+   * synchronous.  Note that the Snapshot in the PBackground parent already
+   * has the answers to this request immediately available without needing to
+   * consult any other threads or perform any I/O.
+   */
   sync LoadItem(nsString key)
     returns (nsString value);
 
+  /**
+   * Invoked on demand to load all keys in in their canonical order if they
+   * didn't fit into the initial snapshot prefill.
+   *
+   * This needs to be synchronous because LocalStorage's semantics are
+   * synchronous.  Note that the Snapshot in the PBackground parent already
+   * has the answers to this request immediately available without needing to
+   * consult any other threads or perform any I/O.
+   */
   sync LoadKeys()
     returns (nsString[] keys);
 
+  /**
+   * This needs to be synchronous because LocalStorage's semantics are
+   * synchronous.  Note that the Snapshot in the PBackground parent typically
+   * doesn't need to consult any other threads or perform any I/O to handle
+   * this request.  However, it has to call a quota manager method that can
+   * potentially do I/O directly on the PBackground thread.  It can only happen
+   * rarely in a storage pressure (low storage space) situation.  Specifically,
+   * after we get a list of origin directories for eviction, we will delete
+   * them directly on the PBackground thread.  This doesn't cause any
+   * performance problems, but avoiding I/O completely might need to be done as
+   * a futher optimization.
+   */
   sync IncreasePeakUsage(int64_t requestedSize, int64_t minSize)
     returns (int64_t size);
 
@@ -60,6 +93,18 @@ parent:
   sync Ping();
 
 child:
+  /**
+   * Compels the child LSSnapshot to Checkpoint() and Finish(), effectively
+   * compelling the snapshot to flush any issued mutations and close itself.
+   * The child LSSnapshot does that either immediately if it's just waiting
+   * to be reused or when it gets into a stable state.
+   *
+   * This message is expected to be sent in the following two cases only:
+   *   1. The state of the underlying Datastore starts to differ from the state
+   *      captured at the time of snapshot creation.
+   *   2. The last private browsing context exits.  And in that case we expect
+   *      all private browsing globals to already have been destroyed.
+   */
   async MarkDirty();
 
   async __delete__();
diff --git a/dom/localstorage/nsILocalStorageManager.idl b/dom/localstorage/nsILocalStorageManager.idl
index c3791d81f9b7..eb712fd75c3b 100644
--- a/dom/localstorage/nsILocalStorageManager.idl
+++ b/dom/localstorage/nsILocalStorageManager.idl
@@ -8,11 +8,25 @@
 
 interface nsIPrincipal;
 
+/**
+ * Methods specific to LocalStorage, see nsIDOMStorageManager for methods shared
+ * with SessionStorage.  Methods may migrate there as SessionStorage is
+ * overhauled.
+ */
 [scriptable, builtinclass, uuid(d4f534da-2744-4db3-8774-8b187c64ade9)]
 interface nsILocalStorageManager : nsISupports
 {
   readonly attribute boolean nextGenLocalStorageEnabled;
 
+  /**
+   * Trigger preload of LocalStorage for the given principal.  For use by
+   * ContentParent::AboutToLoadHttpFtpWyciwygDocumentForChild to maximize the
+   * amount of time we have to load the data off disk before the page might
+   * attempt to touch LocalStorage.
+   *
+   * This method will not create a QuotaManager-managed directory on disk if
+   * one does not already exist for the principal.
+   */
   [implicit_jscontext] nsISupports
   preload(in nsIPrincipal aPrincipal);
 
diff --git a/dom/storage/Storage.h b/dom/storage/Storage.h
index 3fb549cfa239..75180573b840 100644
--- a/dom/storage/Storage.h
+++ b/dom/storage/Storage.h
@@ -110,6 +110,13 @@ public:
 
   bool IsSessionOnly() const { return mIsSessionOnly; }
 
+  //////////////////////////////////////////////////////////////////////////////
+  // Testing Methods:
+  //
+  // These methods are exposed on the `Storage` WebIDL interface behind a
+  // preference for the benefit of automated-tests.  They are not exposed to
+  // content.  See `Storage.webidl` for more details.
+
   virtual void
   Open(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
   { }
@@ -126,11 +133,18 @@ public:
   EndExplicitSnapshot(nsIPrincipal& aSubjectPrincipal, ErrorResult& aRv)
   { }
 
+  //////////////////////////////////////////////////////////////////////////////
+
+  // Dispatch storage notification events on all impacted pages in the current
+  // process as well as for consumption by devtools.  Pages receive the
+  // notification via StorageNotifierService (not observers like in the past),
+  // while devtools does receive the notification via the observer service.
+  //
   // aStorage can be null if this method is called by LocalStorageCacheChild.
   //
   // aImmediateDispatch is for use by child IPC code (LocalStorageCacheChild)
   // so that PBackground ordering can be maintained.  Without this, the event
-  // would be/ enqueued and run in a future turn of the event loop, potentially
+  // would be enqueued and run in a future turn of the event loop, potentially
   // allowing other PBackground Recv* methods to trigger script that wants to
   // assume our localstorage changes have already been applied.  This is the
   // case for message manager messages which are used by ContentTask testing
diff --git a/dom/storage/StorageNotifierService.h b/dom/storage/StorageNotifierService.h
index 8762785f7014..d7c8034389ff 100644
--- a/dom/storage/StorageNotifierService.h
+++ b/dom/storage/StorageNotifierService.h
@@ -14,6 +14,13 @@ namespace dom {
 
 class StorageEvent;
 
+/**
+ * Enables the StorageNotifierService to check whether an observer is interested
+ * in receiving events for the given principal before calling the method, an
+ * optimization refactoring.
+ *
+ * Expected to only be implemented by nsGlobalWindowObserver or its succesor.
+ */
 class StorageNotificationObserver
 {
 public:
@@ -34,6 +41,15 @@ public:
   GetEventTarget() const = 0;
 };
 
+/**
+ * A specialized version of the observer service that uses the custom
+ * StorageNotificationObserver so that principal checks can happen in this class
+ * rather than in the nsIObserver::observe method where they used to happen.
+ *
+ * The only expected consumers are nsGlobalWindowInner instances via their
+ * nsGlobalWindowObserver helper that avoids being able to use the window as an
+ * nsIObserver.
+ */
 class StorageNotifierService final
 {
 public:
diff --git a/dom/webidl/Storage.webidl b/dom/webidl/Storage.webidl
index 0496341b86d2..3b35ada642d2 100644
--- a/dom/webidl/Storage.webidl
+++ b/dom/webidl/Storage.webidl
@@ -34,17 +34,40 @@ interface Storage {
   readonly attribute boolean isSessionOnly;
 };
 
-// Testing only.
+/**
+ * Testing methods that exist only for the benefit of automated glass-box
+ * testing.  Will never be exposed to content at large and unlikely to be useful
+ * in a WebDriver context.
+ */
 partial interface Storage {
+  /**
+   * Does a security-check and ensures the underlying database has been opened
+   * without actually calling any database methods.  (Read-only methods will
+   * have a similar effect but also impact the state of the snapshot.)
+   */
   [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
   void open();
 
+  /**
+   * Automatically ends any explicit snapshot and drops the reference to the
+   * underlying database, but does not otherwise perturb the database.
+   */
   [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
   void close();
 
+  /**
+   * Ensures the database has been opened and initiates an explicit snapshot.
+   * Snapshots are normally automatically ended and checkpointed back to the
+   * parent, but explicitly opened snapshots must be explicitly ended via
+   * `endExplicitSnapshot` or `close`.
+   */
   [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
   void beginExplicitSnapshot();
 
+  /**
+   * Ends the explicitly begun snapshot and retains the underlying database.
+   * Compare with `close` which also drops the reference to the database.
+   */
   [Throws, NeedsSubjectPrincipal, Pref="dom.storage.testing"]
   void endExplicitSnapshot();
 };
diff --git a/ipc/glue/PBackground.ipdl b/ipc/glue/PBackground.ipdl
index 14be7cd8f466..47bff68fdb02 100644
--- a/ipc/glue/PBackground.ipdl
+++ b/ipc/glue/PBackground.ipdl
@@ -134,8 +134,19 @@ parent:
 
   async PBackgroundLSObserver(uint64_t observerId);
 
+  /**
+   * Issue an asynchronous request that will be used in a synchronous fashion
+   * through complex machinations described in `PBackgroundLSRequest.ipdl` and
+   * `LSObject.h`.
+   */
   async PBackgroundLSRequest(LSRequestParams params);
 
+  /**
+   * Issues a simple, non-cancelable asynchronous request that's used in an
+   * asynchronous fashion by callers.  (LSRequest is not simple because it used
+   * in a synchronous fashion which leads to complexities regarding cancelation,
+   * see `PBackgroundLSRequest.ipdl` for details.)
+   */
   async PBackgroundLSSimpleRequest(LSSimpleRequestParams params);
 
   async PBackgroundLocalStorageCache(PrincipalInfo principalInfo,
diff --git a/ipc/ipdl/sync-messages.ini b/ipc/ipdl/sync-messages.ini
index fdb1a188bfc2..a753f6530e75 100644
--- a/ipc/ipdl/sync-messages.ini
+++ b/ipc/ipdl/sync-messages.ini
@@ -923,13 +923,13 @@ description =
 [PBackgroundStorage::Preload]
 description =
 [PBackgroundLSDatabase::PBackgroundLSSnapshot]
-description =
+description = See corresponding comment in PBackgroundLSDatabase.ipdl
 [PBackgroundLSSnapshot::LoadItem]
-description =
+description = See corresponding comment in PBackgroundLSSnapshot.ipdl
 [PBackgroundLSSnapshot::LoadKeys]
-description =
+description = See corresponding comment in PBackgroundLSSnapshot.ipdl
 [PBackgroundLSSnapshot::IncreasePeakUsage]
-description =
+description = See corresponding comment in PBackgroundLSSnapshot.ipdl
 [PBackgroundLSSnapshot::Ping]
 description = See corresponding comment in PBackgroundLSSnapshot.ipdl
 [PRemoteSpellcheckEngine::Check]

From 9aeecaa75e7e0b807c20d25a39b9ffcad0e57066 Mon Sep 17 00:00:00 2001
From: Jan Varga <jan.varga@gmail.com>
Date: Thu, 29 Nov 2018 21:50:07 +0100
Subject: [PATCH 68/78] Bug 1286798 - Part 54: Disable LSNG by default; r=asuth

---
 modules/libpref/init/all.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/modules/libpref/init/all.js b/modules/libpref/init/all.js
index db646db121db..e9707ec6da88 100644
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -1286,8 +1286,9 @@ pref("dom.serviceWorkers.disable_open_click_delay", 1000);
 
 pref("dom.storage.enabled", true);
 // Whether or not LSNG (Next Generation Local Storage) is enabled.
+// See bug 1510410 for enabling this on Nightly.
 #ifdef NIGHTLY_BUILD
-pref("dom.storage.next_gen", true);
+pref("dom.storage.next_gen", false);
 #else
 pref("dom.storage.next_gen", false);
 #endif

From 79614c052cd9cc8170fbb34ccca4608ebfb75d88 Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Wed, 28 Nov 2018 18:45:08 -0800
Subject: [PATCH 69/78] Bug 857456 Part 0: Clean up remaining tests using
 legacy extensions r=kmag

--HG--
extra : rebase_source : a36ea76dfff29438f24ea1467959595e5fa902ca
extra : histedit_source : 24dfb942cf4389cc1ebf4e147c649ab9099aa513
---
 .../content/test/webextensions/browser.ini    |   1 -
 ...er_extension_update_background_noprompt.js |   6 --
 .../test/webextensions/browser_legacy.xpi     | Bin 4705 -> 0 bytes
 .../browser_update_interactive_noprompt.js    |   5 -
 .../client/aboutdebugging/test/browser.ini    |   9 +-
 .../test/browser_addons_debug_info.js         |   3 +
 .../test/browser_addons_install.js            |   6 +-
 .../test/browser_addons_reload.js             |   5 +-
 .../test/browser_addons_remove.js             |   3 +
 devtools/client/shared/test/browser.ini       |   8 +-
 .../extensions/test/xpcshell/test_registry.js |  96 +++++-------------
 .../test/xpcshell/test_trash_directory.js     |  38 +------
 12 files changed, 58 insertions(+), 122 deletions(-)
 delete mode 100644 browser/base/content/test/webextensions/browser_legacy.xpi

diff --git a/browser/base/content/test/webextensions/browser.ini b/browser/base/content/test/webextensions/browser.ini
index 66d49a4fc761..c1e2838cc4de 100644
--- a/browser/base/content/test/webextensions/browser.ini
+++ b/browser/base/content/test/webextensions/browser.ini
@@ -2,7 +2,6 @@
 support-files =
   head.js
   file_install_extensions.html
-  browser_legacy.xpi
   browser_legacy_webext.xpi
   browser_webext_permissions.xpi
   browser_webext_nopermissions.xpi
diff --git a/browser/base/content/test/webextensions/browser_extension_update_background_noprompt.js b/browser/base/content/test/webextensions/browser_extension_update_background_noprompt.js
index dd813f012381..e9b1eb09feed 100644
--- a/browser/base/content/test/webextensions/browser_extension_update_background_noprompt.js
+++ b/browser/base/content/test/webextensions/browser_extension_update_background_noprompt.js
@@ -1,7 +1,6 @@
 const {AddonManagerPrivate} = ChromeUtils.import("resource://gre/modules/AddonManager.jsm", {});
 
 const ID_PERMS = "update_perms@tests.mozilla.org";
-const ID_LEGACY = "legacy_update@tests.mozilla.org";
 const ID_ORIGINS = "update_origins@tests.mozilla.org";
 
 function getBadgeStatus() {
@@ -80,11 +79,6 @@ async function testNoPrompt(origUrl, id) {
 add_task(() => testNoPrompt(`${BASE}/browser_webext_update_perms1.xpi`,
                             ID_PERMS));
 
-// Test that an update from a legacy extension to a webextension
-// doesn't show a prompt even when the webextension uses
-// promptable required permissions.
-add_task(() => testNoPrompt(`${BASE}/browser_legacy.xpi`, ID_LEGACY));
-
 // Test that an update that narrows origin permissions is just applied without
 // showing a notification promt
 add_task(() => testNoPrompt(`${BASE}/browser_webext_update_origins1.xpi`,
diff --git a/browser/base/content/test/webextensions/browser_legacy.xpi b/browser/base/content/test/webextensions/browser_legacy.xpi
deleted file mode 100644
index da4b3062796c180155819f83f6514ad6738827ab..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 4705
zcma)AbyO7IwjR0%B$S~<K)OLf8A3u(VrWE=6c}o#p&JQ-p+QOM?vRoi6c~CyL_k79
zx<Pt)c;8#^cdz$%*ShDd{m)r@e{1jaednyTH{=-(E;RrEAO!quf;^oTH9P)A3IG^W
z006&F)f9E)_*B%D1RP-AP<wkbeiv7>S$%7r#LZfD*7{M>F?oW|6sMGg%9^FhyK1=c
zl#!;7i6S_$<?>h*W#vrATitN6@YWSAVGxRSV`#1xZUNJ1UE{^p#S*4vvl5Q+8xdd0
zOnq=TNEe9o2C4@Hm#WQLVvj!_<0@|fgzA!D<6D5Ryd}ueK*T?Cu&|;#k~-I^74FF9
ztOGbl$J{t^NrO7O#2#vC)By@R>N}$elyV$oGGzHoo!CAhNe2k_7_jl##saC=cGlOK
zi{%dXS<WQlW>pl*MFS<1gjqgcCzXO|iRdokGOjX!G`K_Gjo<7j=*x)&$9A+*VMXEP
zlapolExiojGV=}KHp^hhXUND@Q7|bPkZDti_cf}i1?*H&tamY(x62rxnJd3+UweNp
zv#onw@4{k?e=3<;eE<ORgePLDV@uGhvoq`f7O%;IAjG$cs;@}1e&}~zr%DlY`V-p3
ztKzfCuuOGja@wq`HwC|vES`?d?VRxccwI!pNST`x1_nl4_r)7`CN69&xJ}HB3`_;7
zLhe)fqORlVWP=E=+c2hd6iL`-Sd5p}fe>a>$WD?#d_%e36QcH*t7-sUd7p~+H)z@D
zv$hn2g8Ji%jB)(1+D$U)%I$i;um>1g#I!C1aW;hbu1fwZ+7|y!qS6)BvbBKFFMP_f
z7d-d<`B$yQ4%44iu(cJSITDC^Dj>Or1?#fe$S*Y?p<;|BypxlWVkM3J+VDI3zog$}
zf5{65Bo(YT>bM%BS4Z4UDw8PQ9Zs0DTH@%@J#e{SySnY5gE9hX<}w~seY{Wj=m9^M
zB2Z#Z?`O54_09&To4%Oy4!iQYU*SYVbq{M3vz#_U1mW+#5ae9H8>+Nkq4nxvctS>s
zN@=I{CnJ9&2FHYUTUFXC#>K*ltolS3-Ti(kqo22Flf!$~w?&nX;BJ~8F(ahh8bTW$
z^x_g4w?CbTt2W9*8Q6&Dhjk0o?bkRsllNkVV++`5e#$Z^WNpnwqH7=0FGivxU8|1_
zbkwr%`gij#`D+`B_IPEcEd-lo(ttc04nFP=DDJ4$Ffx2!4|XV%*1+7NE=d>|!4fhy
zpGDd^p{=QWr`QY+%0f4xZx-4wboNE!qQt}(@lc8pBu2ZS`*oWtNWQYuw1$&HqMX*C
zj_0lUM}xMH1Zg|T1l?EbeGzc}RB&qCc=U*1+Y-p9)j?QedN1jPAz`y2TqJ)&?}8TE
zJ&>2_W9ws#^$mg<L^V+58R&o;YjJ~u<rV@|IghG^zB9qNMl1GkPe+`&YxlPZR|W~^
z1t&^bY}v~9Kr=%9_Ctf@wNjzG>7P99llM`vdmJpt34(^RJ5uv^gc~@`Y2`r|42pQX
z;|P7_F};M|VQVg)lh5#|{oN<`mqY1y9``w5yFb3`jBNjL>W)dI=!g*LM?_`W&ULqm
zC57i^DHw_-rmwEHgE0#jqjMw4pH0|s3T`4nzq_E^Xn$Y`6Iw)=HO*=y7Fr`&l`vz2
z$g8H>B*XC<e@L}4u|ixhL9c0Fr=b+;<Cz)RSV$$pnzEC)jNN)6xi2QpG-&89Uh~9%
zGCj$3?TvLq4R4Q*6#p|dsjosi5qthMO&(<DTVqI^U9~UCv&<9|6qdQ>-gH3}{Y!o=
zp)8*~TxUoIyccH9`Vsb)KZ7AYx`(;25ZDvL8L^e^vuLS|4xk|Jv~r)A*Rzi$QJ15)
zm5#hE>8nL%Ml3YIB3IMd*2R72&Nced@T1t*T|Gn7J)1wHgQ$HDQKB$Q(a{kl^SQe+
zS!H(SMWvaIX&8$5GF#&DRzcm;s3%WL@lL}<0@L=`ORAA*wu7>j547Y(vIodo>2SH2
z`KH5yJP-^{fQ<sxAfqoAF9YlC8syG+$_yDT%+N_}Mx~fL-{W2qe^o3$ZLwS4fW%G~
zL|mzf4K)fFajC7l?vcNc+$p*ZonFEs_NCA6wDQ-IOrpU(^Lfh*^psk=PU^t}e9@p=
z-MbgxNAL-8MxXh}i2W+tM2a~ixfiwITr0D7LeY<*$4r)|UL(5FTVI7`9WbVvQ<P{d
zy7#Aww+<~`$eQ2BSVY-H{u)N?Jd7DWzu-@=#VhupYDiXmOUh&-THYc2)*dn)jq#H`
zWin^;mIuQfd+(`8Vj-22?@T&W0=Qt~rxpW5lxuC5z=S?AduquDn?Yd{szndWngO~d
zN>DOWaa*wM>eo7DIXGJf=N@x1KPL&kn^MJ`Upx_i^S$Q}VTHSNPuyxrU3_w%%j^;-
zs~v9kets%rq0!_nmp<bP3$jeBW9RqOc)AF*{!ufzNiVY0DE8tBoX`S6S$gaPs}EOm
zRQ87ZBSTuy92Q3PxQaPNhtGL)YAW9kC}8t+t=howcj%s9kQ|47Bu{4YHy%&6xq5W(
zDD`4*fj_`WWFGK#pD{nQb7*%&{WyMA>(1uZN{6ZWw<>hbeh_8-rnGW=hIw;8;*oR+
ztmI|eYl&IQJ3gp_Y}%i8Nbu63qP5ItR%9<tqjxY<7*{iON>*9%xnUKfKwE+JRs}~u
zf#Uk&u`@?j%=oR~*nH)CbF1604wKq=N2aA6&GW~fGSHe*>U7LF0{A9rV$0NH_e=!s
z=ntq^>yC!p@p&|6NyIBEnMwtUq1!q~^1^{+-RACfqnP%u$L4r2gXy!Dd7}VP3jxnu
ziI@Do%pT$9jnhu-A-e2N4qhg|7*$=hoebUq=~<nuO4LYjf{=X6b)2Snf!IAb5g~;N
z_sm2A)TDC+@5pZrblo9!7wmBgT%w3#G{@}(6v=mYiHg=k)?zi~g}Nn7x0W}5ZAp$x
z)`)2EcOSxXmPIA;b^tkN;V!ofY#vgiQ6e$<j_5Qa2}M!~P0iDE!hrTf%g1mf8yok%
zYmxhqPE%QY^?>+NvO<4tf^O4AhFnL>+PAs&N7orjP!0kp$6XCB0pv;m#Z2t6{$=Go
z8z)N;54}^KNXxLVu)of3)3HHWoT4&_VBqTNnfkmpp476UY7s%UAvX|JbSjOn6aX2z
z#+_Q-(rYf+Unb*^^+1qU6;lBwc)28>+f3_Yfa79bI>~IMTyD=_**c~jv(*uMy!FPj
z9>}$1kKPR3bNEn^!~e=H7d=tBVQydvJ-O1|h*>>8r{cuyT7?Z^K5|?<&&dZkck5W_
z%(a5LlU<=>Z<#JMQUbsxWwcqsqrOQt(apl?S_ya~j~{_FYiF9nxb(YehrQ_>w7u%`
zra1A9^EJo<`tqaW+DQxEG)fiIj-aTbrl*vu7n4<ob!=y<8s}_e0|GhJ<wJM1MY6PJ
z>a&uU0+wDo%(lupaWRM`8l>?kb$X;4m?*GUwL+f)p9t78BijS-PJRg-NU)8~Oj}+_
zPp`Zltjn;Nu$uI1OxUM6)U0Vzkc7c|4>M*9SC|`%?Z)iSup($a>?XeZJa`!{S)$?S
z6M?R9CYD}(_Pu&=2$izj_B9=QFV>C*QD&K>@ZG~mByDR{ciXo6i{)*?Ph;=PY<0G>
zIOAvXM`(e$w)t|aH0=%=^EqeoC>aUe{Ha~h^!~m9nEc)G7-f9F%wB^%8Cad2@uoeA
z(<*yfz$Ov>@sb)9cJWd}(GM>{_sjI`H2guWgBI{96ke(I1*cn>1a``tg5B`2$#M{=
z&cs}Q((x+w5}t6?@?&fpP-zmA{jKA@8(?05h%9C0?g^0j;2Ci@lhcz(&sQ^38!wl)
z>Qg!PrCUE=H2EFK=-r9N#>%AN3kH=>h^X>5y7EAe)E`_*|Ipccn9tHa9j;yw1t0an
z23?1D@x<;z?aN2Gt!Cbid&vnQSsm3*JKZCHtmQHv8dDC1MD)Hf0oOI*PnZnvWN1kt
zFKdjfmep|;tEs?KhYS52Av9>V4+E#3z4P$ygVxi53uwa3@LExA6e_>Xpof^WsROUf
zW*f|K;3Ko$#(Vth$ID_h?#w<f^0&t*3SU7_E$zcoU!TglvuU)tI(TV&k(zp&)N&VX
zx#F0V^Ek`FrUvx}rtFj1qKcQa$ww_b{!|kBsc_J})J~|W+mA$5KN%=w-@-TaEpF(C
z{?S1LS(k?sQAG!1YHDsTMzVOQXtCU+`h^M23Gu<uJ_`*s7O-#JfUt4XIraVG=qo!P
zzBsc*8yQD37?+;W##Wj9YwFhTf&24ikHp33;BTcMj}~xTW}O3uH}$K&rkSvTLTN=1
zLK@kc`kVA_#$`pdU+bMh#s(7+lplL4IUJ>Co>(pR1g8q<?PeFQ;5EThqOt7<+GE*F
zy+|`&e2{5x-7uIm*}7lJ;GQLUV9^_){P`?&uszE6<N14K*qg+oVd?ed42ZJa0ylUN
z&3T)?RqrS`=QI&Q?&5!5uJy!ZkHS^t#ATo8vy+NHcZ{(L*fS`Pwa|cOg{sSDGYh8s
z);L)RI53Kmp}%{zW&-*DK55bu@>Hw;Ry+g?0BFYs07w9|0H~v@o0+{mzl)`nt_~gm
zCrq%=0PaZwz`{Mg@xs07qyTWO8TAlB+V)j0I(KU=Lm*m?cHDBmpC@lF4cKo#c6;ph
zlxTZvd2}K{WxvPywLi24ObnbW87ohqpFF8<9b(|{`#u{I6Yss$4$|fzY;f3Y$k8vs
zIarcVl8}`H=xRRpO9*4g`NA`awK_sa#eMswrk~*O8!|${7z!Helm6`<thk-xVVW|v
zMttUrQX%?AP2uJyO7@0(8S-*$JSAe8n^If_EM|&5>p{y!lZE^dIgN!&$G$Hky{BxN
zzl$G&=UuuxM7Y^*iTZ2=cZLk%kBH5b@`-atLHDmoT~VUQ4BIq3J=0j$nkw!*1Pk0s
zXPiTIbWr0<P+DutPhJkc4-Nt7l|Y29-<U5#-e|5<suNdfQj=&2TsTUuqHIn#|9W%H
z4*hN6wraakWMp-Cg;f||!?MJJew!0^*aLJG_(B-dGD+?n*+P&eIsX?So|PI;g(({n
zU)<bs*Nu_h5MmC4xw*Qzm^ty=y8eTaBXGf=jEqleDOYgz32>{ZX>#(b@N#qUjcRfZ
zbMmR61V+X+`v!4{e=e_aCjFF4B{n=aGJ+O(q3)-ic?o(Ul0GFUFox!S`LuW`r53tj
z_?oyl=Tt}5p*s)P@lhci;IHI<chy^+mUlaE5~IG+!oPOaW{yy+SFUdS4pxXEIg(&O
zK-iB9|Ia(|_fv`&%<2cD6<<j{tki1!X4I^>P*oN2&~4*a*9lVIfVFh&o4@7<r8V~8
zN)cRBlnzJ^O*@e9Hm!|UU2FnMXmAW&=~swb5E|584qjdBxF%$8ZtN(j=;Dk|Wr|5@
z?+t-7gy^4DP*^Jpy_+Rq1W!XU^xbOTjUCm-vt!l>Dk^IDS|zLx#oyOG-+2F2|G0L6
zyM?CXn6vr%Ao1~39KnmlaZ!|;_{rmP55mM!{{!-Wt+*?LWWx5QVy_#cy0L$oBe+_T
z46BYj<K^OeIzje~k9&A<q)toli^#$OTxbH0Qq|z&M<1&V4yfV?ms6J})I|?;kC$;D
z&Vsc`kRl1LL)xx54i3%1U<NXw<nR|(t6s|l>@>8YDFzD7ir(kR@7S%ZQwtTEr(K+}
zZpgsGrpEctLynvL{;>jY==qyeZ$=)!5t#lD;qRXR@3`N=`9YjyfdA$*_-DYslkZ=E
z%|O6^CFVbi{8Mwk5r3~hEIsmnF7hu0{)zA>hyNS@8$tPII`lV;LZ0E_|8~N=*)})V
Kv2^1G0Q?U&(uH^c

diff --git a/browser/base/content/test/webextensions/browser_update_interactive_noprompt.js b/browser/base/content/test/webextensions/browser_update_interactive_noprompt.js
index 46a0894d7db5..fe95a7fadbd5 100644
--- a/browser/base/content/test/webextensions/browser_update_interactive_noprompt.js
+++ b/browser/base/content/test/webextensions/browser_update_interactive_noprompt.js
@@ -51,11 +51,6 @@ async function testUpdateNoPrompt(filename, id,
   await addon.uninstall();
 }
 
-// Test that we don't see a prompt when updating from a legacy
-// extension to a webextension.
-add_task(() => testUpdateNoPrompt("browser_legacy.xpi",
-                                  "legacy_update@tests.mozilla.org", "1.1"));
-
 // Test that we don't see a prompt when no new promptable permissions
 // are added.
 add_task(() => testUpdateNoPrompt("browser_webext_update_perms1.xpi",
diff --git a/devtools/client/aboutdebugging/test/browser.ini b/devtools/client/aboutdebugging/test/browser.ini
index 8ee48029be5c..b3809f5bb445 100644
--- a/devtools/client/aboutdebugging/test/browser.ini
+++ b/devtools/client/aboutdebugging/test/browser.ini
@@ -23,6 +23,8 @@ support-files =
   !/devtools/client/shared/test/telemetry-test-helpers.js
 
 [browser_addons_debug_bootstrapped.js]
+# To be removed in bug 1497264
+skip-if = true
 [browser_addons_debug_info.js]
 [browser_addons_debug_webextension.js]
 tags = webextensions
@@ -34,11 +36,16 @@ tags = webextensions
 skip-if = (verify && debug) || (debug && os == "linux" && bits == 64) # verify: crashes on shutdown, timeouts linux debug Bug 1299001
 tags = webextensions
 [browser_addons_debugging_initial_state.js]
+# To be removed or updated in bug 1497264
+skip-if = true
 [browser_addons_install.js]
-skip-if = verify && debug
+# To be updated in bug 1497264 (was "verify && debug")
+skip-if = true
 [browser_addons_reload.js]
 [browser_addons_remove.js]
 [browser_addons_toggle_debug.js]
+# To be removed or updated in bug 1497264
+skip-if = true
 [browser_page_not_found.js]
 [browser_service_workers.js]
 [browser_service_workers_fetch_flag.js]
diff --git a/devtools/client/aboutdebugging/test/browser_addons_debug_info.js b/devtools/client/aboutdebugging/test/browser_addons_debug_info.js
index fb1814beda1f..eb43b72c5102 100644
--- a/devtools/client/aboutdebugging/test/browser_addons_debug_info.js
+++ b/devtools/client/aboutdebugging/test/browser_addons_debug_info.js
@@ -13,6 +13,8 @@ function testFilePath(container, expectedFilePath) {
   is(filePath.previousElementSibling.textContent, "Location", "file path has label");
 }
 
+// Remove in Bug 1497264
+/*
 add_task(async function testLegacyAddon() {
   const addonId = "test-devtools@mozilla.org";
   const addonName = "test-devtools";
@@ -32,6 +34,7 @@ add_task(async function testLegacyAddon() {
 
   await closeAboutDebugging(tab);
 });
+*/
 
 add_task(async function testWebExtension() {
   const addonId = "test-devtools-webextension-nobg@mozilla.org";
diff --git a/devtools/client/aboutdebugging/test/browser_addons_install.js b/devtools/client/aboutdebugging/test/browser_addons_install.js
index ab779210ff64..420e306ca381 100644
--- a/devtools/client/aboutdebugging/test/browser_addons_install.js
+++ b/devtools/client/aboutdebugging/test/browser_addons_install.js
@@ -7,9 +7,6 @@ loader.lazyImporter(this, "AddonTestUtils",
 
 AddonTestUtils.initMochitest(this);
 
-const ADDON_ID = "test-devtools@mozilla.org";
-const ADDON_NAME = "test-devtools";
-
 function mockFilePicker(window, file) {
   // Mock the file picker to select a test addon
   const MockFilePicker = SpecialPowers.MockFilePicker;
@@ -35,6 +32,9 @@ function promiseWriteWebManifestForExtension(manifest, dir) {
 }
 
 add_task(async function testLegacyInstallSuccess() {
+  const ADDON_ID = "test-devtools@mozilla.org";
+  const ADDON_NAME = "test-devtools";
+
   const { tab, document } = await openAboutDebugging("addons");
   await waitForInitialAddonList(document);
 
diff --git a/devtools/client/aboutdebugging/test/browser_addons_reload.js b/devtools/client/aboutdebugging/test/browser_addons_reload.js
index 16353be7d72f..a371a735dfde 100644
--- a/devtools/client/aboutdebugging/test/browser_addons_reload.js
+++ b/devtools/client/aboutdebugging/test/browser_addons_reload.js
@@ -3,7 +3,6 @@
 "use strict";
 
 const ADDON_ID = "test-devtools@mozilla.org";
-const ADDON_NAME = "test-devtools";
 
 const PACKAGED_ADDON_ID = "bug1273184@tests";
 const PACKAGED_ADDON_NAME = "bug 1273184";
@@ -60,7 +59,10 @@ class TempWebExt {
   }
 }
 
+// Remove in Bug 1497264
+/*
 add_task(async function reloadButtonReloadsAddon() {
+  const ADDON_NAME = "test-devtools";
   const { tab, document, window } = await openAboutDebugging("addons");
   const { AboutDebugging } = window;
   await waitForInitialAddonList(document);
@@ -96,6 +98,7 @@ add_task(async function reloadButtonReloadsAddon() {
   await tearDownAddon(AboutDebugging, reloadedAddon);
   await closeAboutDebugging(tab);
 });
+*/
 
 add_task(async function reloadButtonRefreshesMetadata() {
   const { tab, document, window } = await openAboutDebugging("addons");
diff --git a/devtools/client/aboutdebugging/test/browser_addons_remove.js b/devtools/client/aboutdebugging/test/browser_addons_remove.js
index 27ed891b7b5f..f04a4a15719e 100644
--- a/devtools/client/aboutdebugging/test/browser_addons_remove.js
+++ b/devtools/client/aboutdebugging/test/browser_addons_remove.js
@@ -12,6 +12,8 @@ function getRemoveButton(document, id) {
   return document.querySelector(`[data-addon-id="${id}"] .uninstall-button`);
 }
 
+// Remove in Bug 1497264
+/*
 add_task(async function removeLegacyExtension() {
   const addonID = "test-devtools@mozilla.org";
   const addonName = "test-devtools";
@@ -36,6 +38,7 @@ add_task(async function removeLegacyExtension() {
 
   await closeAboutDebugging(tab);
 });
+*/
 
 add_task(async function removeWebextension() {
   const addonID = "test-devtools-webextension@mozilla.org";
diff --git a/devtools/client/shared/test/browser.ini b/devtools/client/shared/test/browser.ini
index d5cb61629d95..0f7448c06b84 100644
--- a/devtools/client/shared/test/browser.ini
+++ b/devtools/client/shared/test/browser.ini
@@ -77,7 +77,9 @@ support-files =
 [browser_cubic-bezier-06.js]
 [browser_cubic-bezier-07.js]
 [browser_dbg_addon-console.js]
-skip-if = (e10s && debug || os == 'win' || verify)
+# To be removed or updated in bug 1497264
+# previously was: (e10s && debug || os == 'win' || verify)
+skip-if = true 
 tags = addons
 [browser_dbg_debugger-statement.js]
 skip-if = e10s && debug
@@ -245,7 +247,9 @@ skip-if = !e10s || os == "win" # RDM only works for remote tabs, Win: bug 140419
 skip-if = verify
 [browser_theme_switching.js]
 [browser_dbg_listaddons.js]
-skip-if = e10s && debug
+# To be removed or updated in bug 1497264
+# previously was: e10s && debug
+skip-if = true
 tags = addons
 [browser_dbg_listtabs-01.js]
 [browser_dbg_listtabs-02.js]
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_registry.js b/toolkit/mozapps/extensions/test/xpcshell/test_registry.js
index 7068dab6b071..8e580293f389 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_registry.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_registry.js
@@ -10,45 +10,20 @@ Services.prefs.setIntPref("extensions.enabledScopes",
                           AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_USER +
                           AddonManager.SCOPE_SYSTEM);
 
-var addon1 = {
-  id: "addon1@tests.mozilla.org",
-  version: "1.0",
-  name: "Test 1",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
-
-var addon2 = {
-  id: "addon2@tests.mozilla.org",
-  version: "2.0",
-  name: "Test 2",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "2",
-  }],
-};
-
-const IDS = ["addon1@tests.mozilla.org",
-             "addon2@tests.mozilla.org",
-             "addon3@tests.mozilla.org"];
-
-
-var addon1Dir;
-var addon2Dir;
-const addon3Dir = gProfD.clone();
-addon3Dir.append("addon3@tests.mozilla.org");
+const ID1 = "addon1@tests.mozilla.org";
+const ID2 = "addon2@tests.mozilla.org";
+let xpi1, xpi2;
 
 let registry;
 
 add_task(async function setup() {
-  addon1Dir = await promiseWriteInstallRDFForExtension(addon1, gProfD, "addon1");
-  addon2Dir = await promiseWriteInstallRDFForExtension(addon2, gProfD, "addon2");
+  xpi1 = await createTempWebExtensionFile({
+    manifest: {applications: {gecko: {id: ID1}}},
+  });
+
+  xpi2 = await createTempWebExtensionFile({
+    manifest: {applications: {gecko: {id: ID2}}},
+  });
 
   registry = new MockRegistry();
   registerCleanupFunction(() => {
@@ -59,18 +34,15 @@ add_task(async function setup() {
 // Tests whether basic registry install works
 add_task(async function test_1() {
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
-                   "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", addon1Dir.path);
+                    "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
+                    ID1, xpi1.path);
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon2@tests.mozilla.org", addon2Dir.path);
-  registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
-                    "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon3@tests.mozilla.org", addon3Dir.path);
+                    ID2, xpi2.path);
 
   await promiseStartupManager();
 
-  let [a1, a2, a3] = await AddonManager.getAddonsByIDs(IDS);
+  let [a1, a2] = await AddonManager.getAddonsByIDs([ID1, ID2]);
   notEqual(a1, null);
   ok(a1.isActive);
   ok(!hasFlag(a1.permissions, AddonManager.PERM_CAN_UNINSTALL));
@@ -80,48 +52,38 @@ add_task(async function test_1() {
   ok(a2.isActive);
   ok(!hasFlag(a2.permissions, AddonManager.PERM_CAN_UNINSTALL));
   equal(a2.scope, AddonManager.SCOPE_USER);
-
-  equal(a3, null);
 });
 
 // Tests whether uninstalling from the registry works
 add_task(async function test_2() {
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", null);
+                    ID1, null);
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon2@tests.mozilla.org", null);
-  registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
-                    "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon3@tests.mozilla.org", null);
+                    ID2, null);
 
   await promiseRestartManager();
 
-  let [a1, a2, a3] = await AddonManager.getAddonsByIDs(IDS);
+  let [a1, a2] = await AddonManager.getAddonsByIDs([ID1, ID2]);
   equal(a1, null);
   equal(a2, null);
-  equal(a3, null);
 });
 
 // Checks that the ID in the registry must match that in the install manifest
 add_task(async function test_3() {
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", addon2Dir.path);
+                    ID1, xpi2.path);
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon2@tests.mozilla.org", addon1Dir.path);
-  registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
-                    "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon3@tests.mozilla.org", addon3Dir.path);
+                    ID2, xpi1.path);
 
   await promiseRestartManager();
 
-  let [a1, a2, a3] = await AddonManager.getAddonsByIDs(IDS);
+  let [a1, a2] = await AddonManager.getAddonsByIDs([ID1, ID2]);
   equal(a1, null);
   equal(a2, null);
-  equal(a3, null);
 });
 
 // Tests whether an extension's ID can change without its directory changing
@@ -131,34 +93,30 @@ add_task(async function test_4() {
 
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", null);
+                    ID1, null);
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon2@tests.mozilla.org", null);
-  registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
-                    "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon3@tests.mozilla.org", null);
+                    ID2, null);
 
   await promiseRestartManager();
 
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", addon1Dir.path);
+                    ID1, xpi1.path);
 
   await promiseShutdownManager();
 
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_LOCAL_MACHINE,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon1@tests.mozilla.org", null);
+                    ID1, null);
   registry.setValue(Ci.nsIWindowsRegKey.ROOT_KEY_CURRENT_USER,
                     "SOFTWARE\\Mozilla\\XPCShell\\Extensions",
-                    "addon2@tests.mozilla.org", addon1Dir.path);
-  await promiseWriteInstallRDFForExtension(addon2, gProfD, "addon1");
+                    ID2, xpi1.path);
+  xpi2.copyTo(xpi1.parent, xpi1.leafName);
 
   await promiseStartupManager();
 
-  let [a1, a2, a3] = await AddonManager.getAddonsByIDs(IDS);
+  let [a1, a2] = await AddonManager.getAddonsByIDs([ID1, ID2]);
   equal(a1, null);
   notEqual(a2, null);
-  equal(a3, null);
 });
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_trash_directory.js b/toolkit/mozapps/extensions/test/xpcshell/test_trash_directory.js
index ecaf8f501809..42bccfd055ba 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_trash_directory.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_trash_directory.js
@@ -2,22 +2,12 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-const ADDONS = {
-  test_bootstrap1_1: {
-    "install.rdf": {
-      "id": "bootstrap1@tests.mozilla.org",
-      "name": "Test Bootstrap 1",
-    },
-    "bootstrap.js": BOOTSTRAP_MONITOR_BOOTSTRAP_JS,
-  },
-};
-
-add_task(async function setup() {
+// Test that an open file inside the trash directory does not cause
+// unrelated installs to break (see bug 1180901 for more background).
+add_task(async function test() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   await promiseStartupManager();
-});
 
-add_task(async function() {
   let profileDir = OS.Constants.Path.profileDir;
   let trashDir = OS.Path.join(profileDir, "extensions", "trash");
   let testFile = OS.Path.join(trashDir, "test.txt");
@@ -34,33 +24,13 @@ add_task(async function() {
   let fileExists = await OS.File.exists(testFile);
   ok(fileExists, "test.txt should have been created in " + trashDir);
 
-  let promiseInstallStatus = new Promise((resolve, reject) => {
-    let listener = {
-      onInstallFailed() {
-        AddonManager.removeInstallListener(listener);
-        reject("extension installation should not have failed");
-      },
-      onInstallEnded() {
-        AddonManager.removeInstallListener(listener);
-        ok(true, "extension installation should not have failed");
-        resolve();
-      },
-    };
-
-    AddonManager.addInstallListener(listener);
-  });
-
-  await AddonTestUtils.promiseInstallXPI(ADDONS.test_bootstrap1_1);
+  await promiseInstallWebExtension({});
 
   // The testFile should still exist at this point because we have not
   // yet closed the file handle and as a result, Windows cannot remove it.
   fileExists = await OS.File.exists(testFile);
   ok(fileExists, "test.txt should still exist");
 
-  // Wait for the AddonManager to tell us if the installation of the extension
-  // succeeded or not.
-  await promiseInstallStatus;
-
   // Cleanup
   await promiseShutdownManager();
   await file.close();

From 1f10472c91ba9dc189dbcfea2d566358bacff726 Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Wed, 21 Nov 2018 20:01:26 -0800
Subject: [PATCH 70/78] Bug 857456 Part 1: Get rid of internal/external type
 distinction in addons manager r=kmag

AddonInternal objects have a "type" field which is used inside the
addon manager, but AddonWrapper objects map the internal values to
different externally visible values.  The internal values generally
convey whether a particular addon uses webextension packaging
(ie manifest.json) or not.  This patch cleans that all up by adding a
new isWebExtension property and then just using the externally visible
values for type consistently.

--HG--
extra : rebase_source : 0fad9dc9a153f9e7b5edc245bb9b05617a835bd8
extra : histedit_source : 3bad57dd783a16adc0861b1eb71f9a33f5d8d9a3
---
 .../extensions/AddonManagerStartup.cpp        |   2 +-
 .../extensions/internal/XPIDatabase.jsm       | 104 +++---------------
 .../extensions/internal/XPIInstall.jsm        |  33 +++---
 .../extensions/internal/XPIProvider.jsm       |  47 ++++----
 4 files changed, 55 insertions(+), 131 deletions(-)

diff --git a/toolkit/mozapps/extensions/AddonManagerStartup.cpp b/toolkit/mozapps/extensions/AddonManagerStartup.cpp
index f46c4d02fe60..a0306f5b42e7 100644
--- a/toolkit/mozapps/extensions/AddonManagerStartup.cpp
+++ b/toolkit/mozapps/extensions/AddonManagerStartup.cpp
@@ -418,7 +418,7 @@ public:
 
   bool ShouldCheckStartupModifications()
   {
-    return Type().EqualsLiteral("webextension-langpack");
+    return Type().EqualsLiteral("locale");
   }
 
 
diff --git a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
index f4ac2ad02493..8ad029ec59f8 100644
--- a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
@@ -47,14 +47,12 @@ const {nsIBlocklistService} = Ci;
  *         BOOTSTRAP_REASONS,
  *         DB_SCHEMA,
  *         XPIStates,
- *         isWebExtension,
  */
 
 for (let sym of [
   "BOOTSTRAP_REASONS",
   "DB_SCHEMA",
   "XPIStates",
-  "isWebExtension",
 ]) {
   XPCOMUtils.defineLazyGetter(this, sym, () => XPIInternal[sym]);
 }
@@ -100,12 +98,11 @@ const PENDING_INSTALL_METADATA =
 const COMPATIBLE_BY_DEFAULT_TYPES = {
   extension: true,
   dictionary: true,
-  "webextension-dictionary": true,
 };
 
 // Properties to save in JSON file
 const PROP_JSON_FIELDS = ["id", "syncGUID", "version", "type",
-                          "updateURL", "optionsURL",
+                          "isWebExtension", "updateURL", "optionsURL",
                           "optionsType", "optionsBrowserStyle", "aboutURL",
                           "defaultLocale", "visible", "active", "userDisabled",
                           "appDisabled", "pendingUninstall", "installDate",
@@ -123,20 +120,10 @@ const LEGACY_TYPES = new Set([
   "extension",
 ]);
 
-// Some add-on types that we track internally are presented as other types
-// externally
-const TYPE_ALIASES = {
-  "webextension": "extension",
-  "webextension-dictionary": "dictionary",
-  "webextension-langpack": "locale",
-  "webextension-theme": "theme",
-};
-
 const SIGNED_TYPES = new Set([
   "extension",
-  "webextension",
-  "webextension-langpack",
-  "webextension-theme",
+  "locale",
+  "theme",
 ]);
 
 // Time to wait before async save of XPI JSON database, in milliseconds
@@ -197,46 +184,6 @@ async function getRepositoryAddon(aAddon) {
   return aAddon;
 }
 
-/**
- * Helper function that determines whether an addon of a certain type is a
- * theme.
- *
- * @param {string} type
- *        The add-on type to check.
- * @returns {boolean}
- */
-function isTheme(type) {
-  return type == "theme" || TYPE_ALIASES[type] == "theme";
-}
-
-/**
- * Converts a list of API types to a list of API types and any aliases for those
- * types.
- *
- * @param {Array<string>?} aTypes
- *        An array of types or null for all types
- * @returns {Set<string>?}
- *        An set of types or null for all types
- */
-function getAllAliasesForTypes(aTypes) {
-  if (!aTypes)
-    return null;
-
-  let types = new Set(aTypes);
-  for (let [alias, type] of Object.entries(TYPE_ALIASES)) {
-    // Add any alias for the internal type
-    if (types.has(type)) {
-      types.add(alias);
-    } else {
-      // If this internal type was explicitly requested and its external
-      // type wasn't, ignore it.
-      types.delete(alias);
-    }
-  }
-
-  return types;
-}
-
 /**
  * Copies properties from one object to another. If no target object is passed
  * a new object will be created and returned.
@@ -497,7 +444,7 @@ class AddonInternal {
     if (this.type in COMPATIBLE_BY_DEFAULT_TYPES &&
         !this.strictCompatibility &&
         (!AddonManager.strictCompatibility ||
-         this.type == "webextension-dictionary")) {
+         this.type == "dictionary")) {
 
       // The repository can specify compatibility overrides.
       // Note: For now, only blacklisting is supported by overrides.
@@ -729,14 +676,6 @@ AddonWrapper = class {
     return addon.installTelemetryInfo;
   }
 
-  get type() {
-    return XPIDatabase.getExternalType(addonFor(this).type);
-  }
-
-  get isWebExtension() {
-    return isWebExtension(addonFor(this).type);
-  }
-
   get temporarilyInstalled() {
     return addonFor(this).location.isTemporary;
   }
@@ -987,8 +926,8 @@ AddonWrapper = class {
 
     if (addon.inDatabase) {
       // When softDisabling a theme just enable the active theme
-      if (isTheme(addon.type) && val && !addon.userDisabled) {
-        if (isWebExtension(addon.type))
+      if (addon.type === "theme" && val && !addon.userDisabled) {
+        if (addon.isWebExtension)
           XPIDatabase.updateAddonDisabledState(addon, undefined, val);
       } else {
         XPIDatabase.updateAddonDisabledState(addon, undefined, val);
@@ -1111,7 +1050,8 @@ function defineAddonWrapperProperty(name, getter) {
   });
 }
 
-["id", "syncGUID", "version", "isCompatible", "isPlatformCompatible",
+["id", "syncGUID", "version", "type", "isWebExtension",
+ "isCompatible", "isPlatformCompatible",
  "providesUpdatesSecurely", "blocklistState", "appDisabled",
  "softDisabled", "skinnable", "foreignInstall",
  "strictCompatibility", "updateURL", "dependencies",
@@ -1622,10 +1562,10 @@ this.XPIDatabase = {
    */
   async addonChanged(aId, aType) {
     // We only care about themes in this provider
-    if (!isTheme(aType))
+    if (aType !== "theme")
       return;
 
-    let addons = this.getAddonsByType("webextension-theme");
+    let addons = this.getAddonsByType("theme");
     for (let theme of addons) {
       if (theme.visible && theme.id != aId)
         await this.updateAddonDisabledState(theme, true, undefined, true);
@@ -1642,21 +1582,6 @@ this.XPIDatabase = {
     }
   },
 
-  /**
-   * Converts an internal add-on type to the type presented through the API.
-   *
-   * @param {string} aType
-   *        The internal add-on type
-   * @returns {string}
-   *        An external add-on type
-   */
-  getExternalType(aType) {
-    if (aType in TYPE_ALIASES)
-      return TYPE_ALIASES[aType];
-    return aType;
-  },
-
-  isTheme,
   SIGNED_TYPES,
 
   /**
@@ -1830,7 +1755,7 @@ this.XPIDatabase = {
    * @returns {Addon[]}
    */
   async getAddonsByTypes(aTypes) {
-    let addons = await this.getVisibleAddons(getAllAliasesForTypes(aTypes));
+    let addons = await this.getVisibleAddons(aTypes ? new Set(aTypes) : null);
     return addons.map(a => a.wrapper);
   },
 
@@ -1845,7 +1770,7 @@ this.XPIDatabase = {
     if (!SIGNED_TYPES.has(aType))
       return false;
 
-    if (aType == "webextension-langpack") {
+    if (aType == "locale") {
       return AddonSettings.LANGPACKS_REQUIRE_SIGNING;
     }
 
@@ -1861,6 +1786,7 @@ this.XPIDatabase = {
    */
   isDisabledLegacy(addon) {
     return (!AddonSettings.ALLOW_LEGACY_EXTENSIONS &&
+            !addon.isWebExtension &&
             LEGACY_TYPES.has(addon.type) &&
 
             // Legacy add-ons are allowed in the system location.
@@ -2243,7 +2169,7 @@ this.XPIDatabase = {
     }
 
     // Notify any other providers that a new theme has been enabled
-    if (isTheme(aAddon.type)) {
+    if (aAddon.type === "theme") {
       if (!isDisabled) {
         AddonManagerPrivate.notifyAddonChanged(aAddon.id, aAddon.type);
         this.updateXPIStates(aAddon);
@@ -2895,7 +2821,7 @@ this.XPIDatabaseReconcile = {
     } else if (xpiState && xpiState.wasRestored) {
       isActive = xpiState.enabled;
 
-      if (currentAddon.type == "webextension-theme")
+      if (currentAddon.isWebExtension && currentAddon.type == "theme")
         currentAddon.userDisabled = !isActive;
 
       // If the add-on wasn't active and it isn't already disabled in some way
diff --git a/toolkit/mozapps/extensions/internal/XPIInstall.jsm b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
index 1191ba7704e5..bc0724b738ca 100644
--- a/toolkit/mozapps/extensions/internal/XPIInstall.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
@@ -87,7 +87,7 @@ const PREF_XPI_WHITELIST_REQUIRED     = "xpinstall.whitelist.required";
 
 const TOOLKIT_ID                      = "toolkit@mozilla.org";
 
-/* globals BOOTSTRAP_REASONS, KEY_APP_SYSTEM_ADDONS, KEY_APP_SYSTEM_DEFAULTS, PREF_BRANCH_INSTALLED_ADDON, PREF_SYSTEM_ADDON_SET, TEMPORARY_ADDON_SUFFIX, XPI_PERMISSION, XPIStates, isWebExtension, iterDirectory */
+/* globals BOOTSTRAP_REASONS, KEY_APP_SYSTEM_ADDONS, KEY_APP_SYSTEM_DEFAULTS, PREF_BRANCH_INSTALLED_ADDON, PREF_SYSTEM_ADDON_SET, TEMPORARY_ADDON_SUFFIX, XPI_PERMISSION, XPIStates, iterDirectory */
 const XPI_INTERNAL_SYMBOLS = [
   "BOOTSTRAP_REASONS",
   "KEY_APP_SYSTEM_ADDONS",
@@ -97,7 +97,6 @@ const XPI_INTERNAL_SYMBOLS = [
   "TEMPORARY_ADDON_SUFFIX",
   "XPI_PERMISSION",
   "XPIStates",
-  "isWebExtension",
   "iterDirectory",
 ];
 
@@ -105,10 +104,6 @@ for (let name of XPI_INTERNAL_SYMBOLS) {
   XPCOMUtils.defineLazyGetter(this, name, () => XPIInternal[name]);
 }
 
-function isTheme(type) {
-  return XPIDatabase.isTheme(type);
-}
-
 /**
  * Returns a nsIFile instance for the given path, relative to the given
  * base file, if provided.
@@ -183,7 +178,6 @@ const TYPES = {
 const COMPATIBLE_BY_DEFAULT_TYPES = {
   extension: true,
   dictionary: true,
-  "webextension-dictionary": true,
 };
 
 // This is a random number array that can be used as "salt" when generating
@@ -454,8 +448,8 @@ async function loadManifestFromWebManifest(aUri, aPackage) {
   let addon = new AddonInternal();
   addon.id = bss.id;
   addon.version = manifest.version;
-  addon.type = extension.type === "extension" ?
-               "webextension" : `webextension-${extension.type}`;
+  addon.type = extension.type === "langpack" ? "locale" : extension.type;
+  addon.isWebExtension = true;
   addon.strictCompatibility = true;
   addon.internalName = null;
   addon.updateURL = bss.update_url;
@@ -467,7 +461,7 @@ async function loadManifestFromWebManifest(aUri, aPackage) {
   addon.startupData = extension.startupData;
   addon.hidden = manifest.hidden;
 
-  if (isTheme(addon.type) && await aPackage.hasResource("preview.png")) {
+  if (addon.type === "theme" && await aPackage.hasResource("preview.png")) {
     addon.previewImage = "preview.png";
   }
 
@@ -625,6 +619,7 @@ async function loadManifestFromRDF(aUri, aData, aPackage) {
       }
     }
   }
+  addon.isWebExtension = false;
 
   if (!(addon.type in TYPES))
     throw new Error("Install manifest specifies unknown type: " + addon.type);
@@ -654,7 +649,7 @@ async function loadManifestFromRDF(aUri, aData, aPackage) {
     // Convert legacy dictionaries into a format the WebExtension
     // dictionary loader can process.
     if (addon.type === "dictionary") {
-      addon.type = "webextension-dictionary";
+      addon.isWebExtension = true;
       let dictionaries = {};
       await aPackage.iterFiles(({path}) => {
         let match = /^dictionaries\/([^\/]+)\.dic$/.exec(path);
@@ -1561,7 +1556,7 @@ class AddonInstall {
                                  `Refusing to upgrade addon ${this.existingAddon.id} to different ID ${this.addon.id}`]);
         }
 
-        if (isWebExtension(this.existingAddon.type) && !isWebExtension(this.addon.type)) {
+        if (this.existingAddon.isWebExtension && !this.addon.isWebExtension) {
           return Promise.reject([AddonManager.ERROR_UNEXPECTED_ADDON_TYPE,
                                  "WebExtensions may not be updated to other extension types"]);
         }
@@ -1790,7 +1785,7 @@ class AddonInstall {
         XPIDatabase.recordAddonTelemetry(this.addon);
 
         // Notify providers that a new theme has been enabled.
-        if (isTheme(this.addon.type) && this.addon.active)
+        if (this.addon.type === "theme" && this.addon.active)
           AddonManagerPrivate.notifyAddonChanged(this.addon.id, this.addon.type);
       };
 
@@ -2499,7 +2494,7 @@ AddonInstallWrapper.prototype = {
   },
 
   get type() {
-    return XPIDatabase.getExternalType(installFor(this).type);
+    return installFor(this).type;
   },
 
   get iconURL() {
@@ -2653,7 +2648,7 @@ UpdateChecker.prototype = {
     let AUC = AddonUpdateChecker;
     let ignoreMaxVersion = false;
     // Ignore strict compatibility for dictionaries by default.
-    let ignoreStrictCompat = (this.addon.type == "webextension-dictionary");
+    let ignoreStrictCompat = (this.addon.type == "dictionary");
     if (!AddonManager.checkCompatibility) {
       ignoreMaxVersion = true;
       ignoreStrictCompat = true;
@@ -3857,7 +3852,7 @@ var XPIInstall = {
     let results = [...this.installs];
     if (aTypes) {
       results = results.filter(install => {
-        return aTypes.includes(XPIDatabase.getExternalType(install.type));
+        return aTypes.includes(install.type);
       });
     }
 
@@ -3952,7 +3947,7 @@ var XPIInstall = {
     AddonManagerPrivate.callAddonListeners("onInstalled", addon.wrapper);
 
     // Notify providers that a new theme has been enabled.
-    if (isTheme(addon.type))
+    if (addon.type === "theme")
       AddonManagerPrivate.notifyAddonChanged(addon.id, addon.type, false);
 
     return addon.wrapper;
@@ -4069,7 +4064,7 @@ var XPIInstall = {
     }
 
     // Notify any other providers that a new theme has been enabled
-    if (isTheme(aAddon.type) && aAddon.active)
+    if (aAddon.type === "theme" && aAddon.active)
       AddonManagerPrivate.notifyAddonChanged(null, aAddon.type);
   },
 
@@ -4110,7 +4105,7 @@ var XPIInstall = {
     }
 
     // Notify any other providers that this theme is now enabled again.
-    if (isTheme(aAddon.type) && aAddon.active)
+    if (aAddon.type === "theme" && aAddon.active)
       AddonManagerPrivate.notifyAddonChanged(aAddon.id, aAddon.type, false);
   },
 
diff --git a/toolkit/mozapps/extensions/internal/XPIProvider.jsm b/toolkit/mozapps/extensions/internal/XPIProvider.jsm
index 22be52811fd3..0b25dcc963fa 100644
--- a/toolkit/mozapps/extensions/internal/XPIProvider.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIProvider.jsm
@@ -222,18 +222,6 @@ function getFile(path, base = null) {
   return file;
 }
 
-/**
- * Helper function that determines whether an addon of a certain type is a
- * WebExtension.
- *
- * @param {string} type
- *        The add-on type to check.
- * @returns {boolean}
- */
-function isWebExtension(type) {
-  return type == "webextension" || type == "webextension-theme";
-}
-
 /**
  * Returns true if the given file, based on its name, should be treated
  * as an XPI. If the file does not have an appropriate extension, it is
@@ -375,6 +363,7 @@ const JSON_FIELDS = Object.freeze([
   "dependencies",
   "enabled",
   "file",
+  "isWebExtension",
   "lastModifiedTime",
   "path",
   "runInSafeMode",
@@ -458,6 +447,7 @@ class XPIState {
     let json = {
       dependencies: this.dependencies,
       enabled: this.enabled,
+      isWebExtension: this.isWebExtension,
       lastModifiedTime: this.lastModifiedTime,
       path: this.relativePath,
       runInSafeMode: this.runInSafeMode,
@@ -531,6 +521,7 @@ class XPIState {
 
     this.version = aDBAddon.version;
     this.type = aDBAddon.type;
+    this.isWebExtension = aDBAddon.isWebExtension;
     if (aDBAddon.startupData) {
       this.startupData = aDBAddon.startupData;
     }
@@ -1521,7 +1512,8 @@ class BootstrapScope {
     if (Services.appinfo.inSafeMode && !runInSafeMode)
       return null;
 
-    if (addon.type == "extension" && aMethod == "startup") {
+    if (!addon.isWebExtension && addon.type == "extension" &&
+        aMethod == "startup") {
       logger.debug(`Registering manifest for ${this.file.path}`);
       Components.manager.addBootstrappedManifestLocation(this.file);
     }
@@ -1632,12 +1624,24 @@ class BootstrapScope {
 
     logger.debug(`Loading bootstrap scope from ${this.file.path}`);
 
-    if (isWebExtension(this.addon.type)) {
-      this.scope = Extension.getBootstrapScope(this.addon.id, this.file);
-    } else if (this.addon.type === "webextension-langpack") {
-      this.scope = Langpack.getBootstrapScope(this.addon.id, this.file);
-    } else if (this.addon.type === "webextension-dictionary") {
-      this.scope = Dictionary.getBootstrapScope(this.addon.id, this.file);
+    if (this.addon.isWebExtension) {
+      switch (this.addon.type) {
+        case "extension":
+        case "theme":
+          this.scope = Extension.getBootstrapScope(this.addon.id, this.file);
+          break;
+
+        case "locale":
+          this.scope = Langpack.getBootstrapScope(this.addon.id, this.file);
+          break;
+
+        case "dictionary":
+          this.scope = Dictionary.getBootstrapScope(this.addon.id, this.file);
+          break;
+
+        default:
+          throw new Error(`Unknown webextension type ${this.addon.type}`);
+      }
     } else {
       let uri = getURIForResourceInFile(this.file, "bootstrap.js").spec;
 
@@ -1833,7 +1837,7 @@ class BootstrapScope {
     let reason = XPIInstall.newVersionReason(this.addon.version, newAddon.version);
     let extraArgs = {oldVersion: this.addon.version, newVersion: newAddon.version};
 
-    let callUpdate = isWebExtension(this.addon.type) && isWebExtension(newAddon.type);
+    let callUpdate = this.addon.isWebExtension && newAddon.isWebExtension;
 
     await this._uninstall(reason, callUpdate, extraArgs);
 
@@ -2668,7 +2672,7 @@ var XPIProvider = {
         updateDate: addon.lastModifiedTime,
         scope,
         isSystem,
-        isWebExtension: isWebExtension(addon),
+        isWebExtension: addon.isWebExtension,
       });
     }
 
@@ -2745,7 +2749,6 @@ var XPIInternal = {
   awaitPromise,
   canRunInSafeMode,
   getURIForResourceInFile,
-  isWebExtension,
   isXPI,
   iterDirectory,
 };

From db881f0da08fe3f8e571f44579836afb119ff01f Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Tue, 20 Nov 2018 20:19:59 -0800
Subject: [PATCH 71/78] Bug 857456 Part 2: Separate handling for
 non-webextensions from the addons manager r=kmag

This patch adds a hook to the addon manager that can be used to teach it
how to load xpi-packaged addons that the addon manager doesn't already
know about.  Handing for install.rdf/bootstrap.js is (temporarily)
converted to use this hook rather than being built directly into the
addon manager.

--HG--
extra : rebase_source : bbbb6b9023b40a2985c585973a87f999d8f103b5
extra : histedit_source : 31115c34c25225a431c44ef246c24759235fe39b
---
 toolkit/mozapps/extensions/AddonManager.jsm   |  18 +
 .../extensions/internal/AddonTestUtils.jsm    |   3 +
 .../extensions/internal/BootstrapLoader.jsm   | 363 ++++++++++++++++++
 .../extensions/internal/XPIDatabase.jsm       |  43 ++-
 .../extensions/internal/XPIInstall.jsm        | 254 +-----------
 .../extensions/internal/XPIProvider.jsm       |  52 +--
 toolkit/mozapps/extensions/internal/moz.build |   1 +
 .../extensions/test/xpcshell/test_shutdown.js |   1 +
 8 files changed, 457 insertions(+), 278 deletions(-)
 create mode 100644 toolkit/mozapps/extensions/internal/BootstrapLoader.jsm

diff --git a/toolkit/mozapps/extensions/AddonManager.jsm b/toolkit/mozapps/extensions/AddonManager.jsm
index 3f0ab7affbd8..607d0d3f6bac 100644
--- a/toolkit/mozapps/extensions/AddonManager.jsm
+++ b/toolkit/mozapps/extensions/AddonManager.jsm
@@ -551,6 +551,7 @@ var AddonManagerInternal = {
   // Store telemetry details per addon provider
   telemetryDetails: {},
   upgradeListeners: new Map(),
+  externalExtensionLoaders: new Map(),
 
   recordTimestamp(name, value) {
     this.TelemetryTimestamps.add(name, value);
@@ -779,6 +780,10 @@ var AddonManagerInternal = {
         }
       }
 
+      // XXX temporary
+      ChromeUtils.import("resource://gre/modules/addons/BootstrapLoader.jsm");
+      AddonManager.addExternalExtensionLoader(BootstrapLoader);
+
       // Load any providers registered in the category manager
       for (let {entry, value: url} of Services.catMan.enumerateCategory(CATEGORY_PROVIDER_MODULE)) {
         try {
@@ -2032,6 +2037,10 @@ var AddonManagerInternal = {
     }
   },
 
+  addExternalExtensionLoader(loader) {
+    this.externalExtensionLoaders.set(loader.name, loader);
+  },
+
   /**
    * Installs a temporary add-on from a local file or directory.
    *
@@ -2941,6 +2950,10 @@ var AddonManagerPrivate = {
     return AddonManagerInternal.upgradeListeners.get(aId);
   },
 
+  get externalExtensionLoaders() {
+    return AddonManagerInternal.externalExtensionLoaders;
+  },
+
   /**
    * Predicate that returns true if we think the given extension ID
    * might have been generated by XPIProvider.
@@ -3353,6 +3366,11 @@ var AddonManager = {
   removeUpgradeListener(aInstanceID) {
     return AddonManagerInternal.removeUpgradeListener(aInstanceID);
   },
+
+  addExternalExtensionLoader(types, loader) {
+    return AddonManagerInternal.addExternalExtensionLoader(types, loader);
+  },
+
   addAddonListener(aListener) {
     AddonManagerInternal.addAddonListener(aListener);
   },
diff --git a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
index e811b727b0a9..7e9e05b2df04 100644
--- a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
+++ b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
@@ -843,6 +843,9 @@ var AddonTestUtils = {
     Cu.unload("resource://gre/modules/addons/XPIDatabase.jsm");
     Cu.unload("resource://gre/modules/addons/XPIInstall.jsm");
 
+    // XXX
+    Cu.unload("resource://gre/modules/addons/BootstrapLoader.jsm");
+
     let ExtensionScope = ChromeUtils.import("resource://gre/modules/Extension.jsm", null);
     ChromeUtils.defineModuleGetter(ExtensionScope, "XPIProvider",
                                    "resource://gre/modules/addons/XPIProvider.jsm");
diff --git a/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm b/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm
new file mode 100644
index 000000000000..d57e358dc5d2
--- /dev/null
+++ b/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm
@@ -0,0 +1,363 @@
+
+"use strict";
+
+var EXPORTED_SYMBOLS = ["BootstrapLoader"];
+
+ChromeUtils.import("resource://gre/modules/AddonManager.jsm");
+ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  AddonInternal: "resource://gre/modules/addons/XPIDatabase.jsm",
+  Blocklist: "resource://gre/modules/Blocklist.jsm",
+  ConsoleAPI: "resource://gre/modules/Console.jsm",
+  InstallRDF: "resource://gre/modules/addons/RDFManifestConverter.jsm",
+  Services: "resource://gre/modules/Services.jsm",
+});
+
+XPCOMUtils.defineLazyGetter(this, "BOOTSTRAP_REASONS", () => {
+  const {XPIProvider} = ChromeUtils.import("resource://gre/modules/addons/XPIProvider.jsm", {});
+  return XPIProvider.BOOTSTRAP_REASONS;
+});
+
+ChromeUtils.import("resource://gre/modules/Log.jsm");
+var logger = Log.repository.getLogger("addons.bootstrap");
+
+/**
+ * Valid IDs fit this pattern.
+ */
+var gIDTest = /^(\{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}|[a-z0-9-\._]*\@[a-z0-9-\._]+)$/i;
+
+// Properties that exist in the install manifest
+const PROP_METADATA      = ["id", "version", "type", "internalName", "updateURL",
+                            "optionsURL", "optionsType", "aboutURL", "iconURL"];
+const PROP_LOCALE_SINGLE = ["name", "description", "creator", "homepageURL"];
+const PROP_LOCALE_MULTI  = ["developers", "translators", "contributors"];
+
+// Map new string type identifiers to old style nsIUpdateItem types.
+// Retired values:
+// 32 = multipackage xpi file
+// 8 = locale
+// 256 = apiextension
+// 128 = experiment
+// theme = 4
+const TYPES = {
+  extension: 2,
+  dictionary: 64,
+};
+
+const COMPATIBLE_BY_DEFAULT_TYPES = {
+  extension: true,
+  dictionary: true,
+};
+
+const hasOwnProperty = Function.call.bind(Object.prototype.hasOwnProperty);
+
+function isXPI(filename) {
+  let ext = filename.slice(-4).toLowerCase();
+  return ext === ".xpi" || ext === ".zip";
+}
+
+/**
+ * Gets an nsIURI for a file within another file, either a directory or an XPI
+ * file. If aFile is a directory then this will return a file: URI, if it is an
+ * XPI file then it will return a jar: URI.
+ *
+ * @param {nsIFile} aFile
+ *        The file containing the resources, must be either a directory or an
+ *        XPI file
+ * @param {string} aPath
+ *        The path to find the resource at, "/" separated. If aPath is empty
+ *        then the uri to the root of the contained files will be returned
+ * @returns {nsIURI}
+ *        An nsIURI pointing at the resource
+ */
+function getURIForResourceInFile(aFile, aPath) {
+  if (!isXPI(aFile.leafName)) {
+    let resource = aFile.clone();
+    if (aPath)
+      aPath.split("/").forEach(part => resource.append(part));
+
+    return Services.io.newFileURI(resource);
+  }
+
+  return buildJarURI(aFile, aPath);
+}
+
+/**
+ * Creates a jar: URI for a file inside a ZIP file.
+ *
+ * @param {nsIFile} aJarfile
+ *        The ZIP file as an nsIFile
+ * @param {string} aPath
+ *        The path inside the ZIP file
+ * @returns {nsIURI}
+ *        An nsIURI for the file
+ */
+function buildJarURI(aJarfile, aPath) {
+  let uri = Services.io.newFileURI(aJarfile);
+  uri = "jar:" + uri.spec + "!/" + aPath;
+  return Services.io.newURI(uri);
+}
+
+var BootstrapLoader = {
+  name: "bootstrap",
+  manifestFile: "install.rdf",
+  async loadManifest(pkg) {
+    /**
+     * Reads locale properties from either the main install manifest root or
+     * an em:localized section in the install manifest.
+     *
+     * @param {Object} aSource
+     *        The resource to read the properties from.
+     * @param {boolean} isDefault
+     *        True if the locale is to be read from the main install manifest
+     *        root
+     * @param {string[]} aSeenLocales
+     *        An array of locale names already seen for this install manifest.
+     *        Any locale names seen as a part of this function will be added to
+     *        this array
+     * @returns {Object}
+     *        an object containing the locale properties
+     */
+    function readLocale(aSource, isDefault, aSeenLocales) {
+      let locale = {};
+      if (!isDefault) {
+        locale.locales = [];
+        for (let localeName of aSource.locales || []) {
+          if (!localeName) {
+            logger.warn("Ignoring empty locale in localized properties");
+            continue;
+          }
+          if (aSeenLocales.includes(localeName)) {
+            logger.warn("Ignoring duplicate locale in localized properties");
+            continue;
+          }
+          aSeenLocales.push(localeName);
+          locale.locales.push(localeName);
+        }
+
+        if (locale.locales.length == 0) {
+          logger.warn("Ignoring localized properties with no listed locales");
+          return null;
+        }
+      }
+
+      for (let prop of [...PROP_LOCALE_SINGLE, ...PROP_LOCALE_MULTI]) {
+        if (hasOwnProperty(aSource, prop)) {
+          locale[prop] = aSource[prop];
+        }
+      }
+
+      return locale;
+    }
+
+    let manifestData = await pkg.readString("install.rdf");
+    let manifest = InstallRDF.loadFromString(manifestData).decode();
+
+    let addon = new AddonInternal();
+    for (let prop of PROP_METADATA) {
+      if (hasOwnProperty(manifest, prop)) {
+        addon[prop] = manifest[prop];
+      }
+    }
+
+    if (!addon.type) {
+      addon.type = "extension";
+    } else {
+      let type = addon.type;
+      addon.type = null;
+      for (let name in TYPES) {
+        if (TYPES[name] == type) {
+          addon.type = name;
+          break;
+        }
+      }
+    }
+
+    if (!(addon.type in TYPES))
+      throw new Error("Install manifest specifies unknown type: " + addon.type);
+
+    if (!addon.id)
+      throw new Error("No ID in install manifest");
+    if (!gIDTest.test(addon.id))
+      throw new Error("Illegal add-on ID " + addon.id);
+    if (!addon.version)
+      throw new Error("No version in install manifest");
+
+    addon.strictCompatibility = (!(addon.type in COMPATIBLE_BY_DEFAULT_TYPES) ||
+                                 manifest.strictCompatibility == "true");
+
+    // Only read these properties for extensions.
+    if (addon.type == "extension") {
+      if (manifest.bootstrap != "true") {
+        throw new Error("Non-restartless extensions no longer supported");
+      }
+
+      if (addon.optionsType &&
+          addon.optionsType != AddonManager.OPTIONS_TYPE_INLINE_BROWSER &&
+          addon.optionsType != AddonManager.OPTIONS_TYPE_TAB) {
+            throw new Error("Install manifest specifies unknown optionsType: " + addon.optionsType);
+      }
+    } else {
+      // Convert legacy dictionaries into a format the WebExtension
+      // dictionary loader can process.
+      if (addon.type === "dictionary") {
+        addon.loader = null;
+        let dictionaries = {};
+        await pkg.iterFiles(({path}) => {
+          let match = /^dictionaries\/([^\/]+)\.dic$/.exec(path);
+          if (match) {
+            let lang = match[1].replace(/_/g, "-");
+            dictionaries[lang] = match[0];
+          }
+        });
+        addon.startupData = {dictionaries};
+      }
+
+      // Only extensions are allowed to provide an optionsURL, optionsType,
+      // optionsBrowserStyle, or aboutURL. For all other types they are silently ignored
+      addon.aboutURL = null;
+      addon.optionsBrowserStyle = null;
+      addon.optionsType = null;
+      addon.optionsURL = null;
+    }
+
+    addon.defaultLocale = readLocale(manifest, true);
+
+    let seenLocales = [];
+    addon.locales = [];
+    for (let localeData of manifest.localized || []) {
+      let locale = readLocale(localeData, false, seenLocales);
+      if (locale)
+        addon.locales.push(locale);
+    }
+
+    let dependencies = new Set(manifest.dependencies);
+    addon.dependencies = Object.freeze(Array.from(dependencies));
+
+    let seenApplications = [];
+    addon.targetApplications = [];
+    for (let targetApp of manifest.targetApplications || []) {
+      if (!targetApp.id || !targetApp.minVersion ||
+          !targetApp.maxVersion) {
+            logger.warn("Ignoring invalid targetApplication entry in install manifest");
+            continue;
+      }
+      if (seenApplications.includes(targetApp.id)) {
+        logger.warn("Ignoring duplicate targetApplication entry for " + targetApp.id +
+                    " in install manifest");
+        continue;
+      }
+      seenApplications.push(targetApp.id);
+      addon.targetApplications.push(targetApp);
+    }
+
+    // Note that we don't need to check for duplicate targetPlatform entries since
+    // the RDF service coalesces them for us.
+    addon.targetPlatforms = [];
+    for (let targetPlatform of manifest.targetPlatforms || []) {
+      let platform = {
+        os: null,
+        abi: null,
+      };
+
+      let pos = targetPlatform.indexOf("_");
+      if (pos != -1) {
+        platform.os = targetPlatform.substring(0, pos);
+        platform.abi = targetPlatform.substring(pos + 1);
+      } else {
+        platform.os = targetPlatform;
+      }
+
+      addon.targetPlatforms.push(platform);
+    }
+
+    addon.userDisabled = false;
+    addon.softDisabled = addon.blocklistState == Blocklist.STATE_SOFTBLOCKED;
+    addon.applyBackgroundUpdates = AddonManager.AUTOUPDATE_DEFAULT;
+
+    addon.userPermissions = null;
+
+    addon.icons = {};
+    if (await pkg.hasResource("icon.png")) {
+      addon.icons[32] = "icon.png";
+      addon.icons[48] = "icon.png";
+    }
+
+    if (await pkg.hasResource("icon64.png")) {
+      addon.icons[64] = "icon64.png";
+    }
+
+    return addon;
+  },
+
+  loadScope(addon, file) {
+    let uri = getURIForResourceInFile(file, "bootstrap.js").spec;
+    let principal = Services.scriptSecurityManager.getSystemPrincipal();
+
+    let sandbox = new Cu.Sandbox(principal, {
+      sandboxName: uri,
+      addonId: addon.id,
+      wantGlobalProperties: ["ChromeUtils"],
+      metadata: { addonID: addon.id, URI: uri },
+    });
+
+    try {
+      Object.assign(sandbox, BOOTSTRAP_REASONS);
+
+      XPCOMUtils.defineLazyGetter(sandbox, "console", () =>
+        new ConsoleAPI({ consoleID: `addon/${addon.id}` }));
+
+      Services.scriptloader.loadSubScript(uri, sandbox);
+    } catch (e) {
+      logger.warn(`Error loading bootstrap.js for ${addon.id}`, e);
+    }
+
+    function findMethod(name) {
+      if (sandbox.name) {
+        return sandbox.name;
+      }
+
+      try {
+        let method = Cu.evalInSandbox(name, sandbox);
+        return method;
+      } catch (err) { }
+
+      return () => {
+        logger.warn(`Add-on ${addon.id} is missing bootstrap method ${name}`);
+      };
+    }
+
+    let install = findMethod("install");
+    let uninstall = findMethod("uninstall");
+    let startup = findMethod("startup");
+    let shutdown = findMethod("shutdown");
+
+    return {
+      install: (...args) => install(...args),
+      uninstall: (...args) => uninstall(...args),
+
+      startup(...args) {
+        if (addon.type == "extension") {
+          logger.debug(`Registering manifest for ${file.path}\n`);
+          Components.manager.addBootstrappedManifestLocation(file);
+        }
+        return startup(...args);
+      },
+
+      shutdown(data, reason) {
+        try {
+          return shutdown(data, reason);
+        } catch (err) {
+          throw err;
+        } finally {
+          if (reason != BOOTSTRAP_REASONS.APP_SHUTDOWN) {
+            logger.debug(`Removing manifest for ${file.path}\n`);
+            Components.manager.removeBootstrappedManifestLocation(file);
+          }
+        }
+      },
+    };
+  },
+};
+
diff --git a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
index 8ad029ec59f8..51fae8a9d7d0 100644
--- a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
@@ -102,7 +102,7 @@ const COMPATIBLE_BY_DEFAULT_TYPES = {
 
 // Properties to save in JSON file
 const PROP_JSON_FIELDS = ["id", "syncGUID", "version", "type",
-                          "isWebExtension", "updateURL", "optionsURL",
+                          "loader", "updateURL", "optionsURL",
                           "optionsType", "optionsBrowserStyle", "aboutURL",
                           "defaultLocale", "visible", "active", "userDisabled",
                           "appDisabled", "pendingUninstall", "installDate",
@@ -281,6 +281,10 @@ class AddonInternal {
     this.inDatabase = true;
   }
 
+  get isWebExtension() {
+    return this.loader == null;
+  }
+
   get selectedLocale() {
     if (this._selectedLocale)
       return this._selectedLocale;
@@ -1318,7 +1322,42 @@ this.XPIDatabase = {
         throw error;
       }
 
-      if (inputAddons.schemaVersion != DB_SCHEMA) {
+      if (inputAddons.schemaVersion == 27) {
+        // Types were translated in bug 857456.
+        for (let addon of inputAddons.addons) {
+          switch (addon.type) {
+            case "extension":
+            case "dictionary":
+            case "locale":
+            case "theme":
+              addon.loader = "bootstrap";
+              break;
+
+            case "webbextension":
+              addon.type = "extension";
+              addon.loader = null;
+              break;
+
+            case "webextension-dictionary":
+              addon.type = "dictionary";
+              addon.loader = null;
+              break;
+
+            case "webextension-langpack":
+              addon.type = "locale";
+              addon.loader = null;
+              break;
+
+            case "webextension-theme":
+              addon.type = "theme";
+              addon.loader = null;
+              break;
+
+            default:
+              logger.warn(`Not converting unknown addon type ${addon.type}`);
+          }
+        }
+      } else if (inputAddons.schemaVersion != DB_SCHEMA) {
         // For now, we assume compatibility for JSON data with a
         // mismatched schema version, though we throw away any fields we
         // don't know about (bug 902956)
diff --git a/toolkit/mozapps/extensions/internal/XPIInstall.jsm b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
index bc0724b738ca..429ce9e5b53a 100644
--- a/toolkit/mozapps/extensions/internal/XPIInstall.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
@@ -43,7 +43,6 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   UpdateUtils: "resource://gre/modules/UpdateUtils.jsm",
 
   AddonInternal: "resource://gre/modules/addons/XPIDatabase.jsm",
-  InstallRDF: "resource://gre/modules/addons/RDFManifestConverter.jsm",
   XPIDatabase: "resource://gre/modules/addons/XPIDatabase.jsm",
   XPIInternal: "resource://gre/modules/addons/XPIProvider.jsm",
 });
@@ -75,8 +74,6 @@ XPCOMUtils.defineLazyServiceGetters(this, {
   gCertDB: ["@mozilla.org/security/x509certdb;1", "nsIX509CertDB"],
 });
 
-const hasOwnProperty = Function.call.bind(Object.prototype.hasOwnProperty);
-
 const PREF_INSTALL_REQUIRESECUREORIGIN = "extensions.install.requireSecureOrigin";
 const PREF_PENDING_OPERATIONS         = "extensions.pendingOperations";
 const PREF_SYSTEM_ADDON_UPDATE_URL    = "extensions.systemAddon.update.url";
@@ -157,24 +154,6 @@ const KEY_APP_PROFILE                 = "app-profile";
 const DIR_STAGE                       = "staged";
 const DIR_TRASH                       = "trash";
 
-// Properties that exist in the install manifest
-const PROP_METADATA      = ["id", "version", "type", "internalName", "updateURL",
-                            "optionsURL", "optionsType", "aboutURL", "iconURL"];
-const PROP_LOCALE_SINGLE = ["name", "description", "creator", "homepageURL"];
-const PROP_LOCALE_MULTI  = ["developers", "translators", "contributors"];
-
-// Map new string type identifiers to old style nsIUpdateItem types.
-// Retired values:
-// 32 = multipackage xpi file
-// 8 = locale
-// 256 = apiextension
-// 128 = experiment
-// theme = 4
-const TYPES = {
-  extension: 2,
-  dictionary: 64,
-};
-
 const COMPATIBLE_BY_DEFAULT_TYPES = {
   extension: true,
   dictionary: true,
@@ -406,7 +385,7 @@ function waitForAllPromises(promises) {
 }
 
 /**
- * Reads an AddonInternal object from a manifest stream.
+ * Reads an AddonInternal object from a webextension manifest.json
  *
  * @param {nsIURI} aUri
  *        A |file:| or |jar:| URL for the manifest
@@ -449,7 +428,7 @@ async function loadManifestFromWebManifest(aUri, aPackage) {
   addon.id = bss.id;
   addon.version = manifest.version;
   addon.type = extension.type === "langpack" ? "locale" : extension.type;
-  addon.isWebExtension = true;
+  addon.loader = null;
   addon.strictCompatibility = true;
   addon.internalName = null;
   addon.updateURL = bss.update_url;
@@ -536,200 +515,6 @@ async function loadManifestFromWebManifest(aUri, aPackage) {
   return addon;
 }
 
-/**
- * Reads an AddonInternal object from an RDF stream.
- *
- * @param {nsIURI} aUri
- *        The URI that the manifest is being read from
- * @param {string} aData
- *        The manifest text
- * @param {InstallPackage} aPackage
- *        An install package instance for the extension.
- * @returns {AddonInternal}
- * @throws if the install manifest in the RDF stream is corrupt or could not
- *         be read
- */
-async function loadManifestFromRDF(aUri, aData, aPackage) {
-  /**
-   * Reads locale properties from either the main install manifest root or
-   * an em:localized section in the install manifest.
-   *
-   * @param {Object} aSource
-   *        The resource to read the properties from.
-   * @param {boolean} isDefault
-   *        True if the locale is to be read from the main install manifest
-   *        root
-   * @param {string[]} aSeenLocales
-   *        An array of locale names already seen for this install manifest.
-   *        Any locale names seen as a part of this function will be added to
-   *        this array
-   * @returns {Object}
-   *        an object containing the locale properties
-   */
-  function readLocale(aSource, isDefault, aSeenLocales) {
-    let locale = {};
-    if (!isDefault) {
-      locale.locales = [];
-      for (let localeName of aSource.locales || []) {
-        if (!localeName) {
-          logger.warn("Ignoring empty locale in localized properties");
-          continue;
-        }
-        if (aSeenLocales.includes(localeName)) {
-          logger.warn("Ignoring duplicate locale in localized properties");
-          continue;
-        }
-        aSeenLocales.push(localeName);
-        locale.locales.push(localeName);
-      }
-
-      if (locale.locales.length == 0) {
-        logger.warn("Ignoring localized properties with no listed locales");
-        return null;
-      }
-    }
-
-    for (let prop of [...PROP_LOCALE_SINGLE, ...PROP_LOCALE_MULTI]) {
-      if (hasOwnProperty(aSource, prop)) {
-        locale[prop] = aSource[prop];
-      }
-    }
-
-    return locale;
-  }
-
-  let manifest = InstallRDF.loadFromString(aData).decode();
-
-  let addon = new AddonInternal();
-  for (let prop of PROP_METADATA) {
-    if (hasOwnProperty(manifest, prop)) {
-      addon[prop] = manifest[prop];
-    }
-  }
-
-  if (!addon.type) {
-    addon.type = "extension";
-  } else {
-    let type = addon.type;
-    addon.type = null;
-    for (let name in TYPES) {
-      if (TYPES[name] == type) {
-        addon.type = name;
-        break;
-      }
-    }
-  }
-  addon.isWebExtension = false;
-
-  if (!(addon.type in TYPES))
-    throw new Error("Install manifest specifies unknown type: " + addon.type);
-
-  if (!addon.id)
-    throw new Error("No ID in install manifest");
-  if (!gIDTest.test(addon.id))
-    throw new Error("Illegal add-on ID " + addon.id);
-  if (!addon.version)
-    throw new Error("No version in install manifest");
-
-  addon.strictCompatibility = (!(addon.type in COMPATIBLE_BY_DEFAULT_TYPES) ||
-                               manifest.strictCompatibility == "true");
-
-  // Only read these properties for extensions.
-  if (addon.type == "extension") {
-    if (manifest.bootstrap != "true") {
-      throw new Error("Non-restartless extensions no longer supported");
-    }
-
-    if (addon.optionsType &&
-        addon.optionsType != AddonManager.OPTIONS_TYPE_INLINE_BROWSER &&
-        addon.optionsType != AddonManager.OPTIONS_TYPE_TAB) {
-      throw new Error("Install manifest specifies unknown optionsType: " + addon.optionsType);
-    }
-  } else {
-    // Convert legacy dictionaries into a format the WebExtension
-    // dictionary loader can process.
-    if (addon.type === "dictionary") {
-      addon.isWebExtension = true;
-      let dictionaries = {};
-      await aPackage.iterFiles(({path}) => {
-        let match = /^dictionaries\/([^\/]+)\.dic$/.exec(path);
-        if (match) {
-          let lang = match[1].replace(/_/g, "-");
-          dictionaries[lang] = match[0];
-        }
-      });
-      addon.startupData = {dictionaries};
-    }
-
-    // Only extensions are allowed to provide an optionsURL, optionsType,
-    // optionsBrowserStyle, or aboutURL. For all other types they are silently ignored
-    addon.aboutURL = null;
-    addon.optionsBrowserStyle = null;
-    addon.optionsType = null;
-    addon.optionsURL = null;
-  }
-
-  addon.defaultLocale = readLocale(manifest, true);
-
-  let seenLocales = [];
-  addon.locales = [];
-  for (let localeData of manifest.localized || []) {
-    let locale = readLocale(localeData, false, seenLocales);
-    if (locale)
-      addon.locales.push(locale);
-  }
-
-  let dependencies = new Set(manifest.dependencies);
-  addon.dependencies = Object.freeze(Array.from(dependencies));
-
-  let seenApplications = [];
-  addon.targetApplications = [];
-  for (let targetApp of manifest.targetApplications || []) {
-    if (!targetApp.id || !targetApp.minVersion ||
-        !targetApp.maxVersion) {
-      logger.warn("Ignoring invalid targetApplication entry in install manifest");
-      continue;
-    }
-    if (seenApplications.includes(targetApp.id)) {
-      logger.warn("Ignoring duplicate targetApplication entry for " + targetApp.id +
-           " in install manifest");
-      continue;
-    }
-    seenApplications.push(targetApp.id);
-    addon.targetApplications.push(targetApp);
-  }
-
-  // Note that we don't need to check for duplicate targetPlatform entries since
-  // the RDF service coalesces them for us.
-  addon.targetPlatforms = [];
-  for (let targetPlatform of manifest.targetPlatforms || []) {
-    let platform = {
-      os: null,
-      abi: null,
-    };
-
-    let pos = targetPlatform.indexOf("_");
-    if (pos != -1) {
-      platform.os = targetPlatform.substring(0, pos);
-      platform.abi = targetPlatform.substring(pos + 1);
-    } else {
-      platform.os = targetPlatform;
-    }
-
-    addon.targetPlatforms.push(platform);
-  }
-
-  addon.userDisabled = false;
-  addon.softDisabled = addon.blocklistState == nsIBlocklistService.STATE_SOFTBLOCKED;
-  addon.applyBackgroundUpdates = AddonManager.AUTOUPDATE_DEFAULT;
-
-  // icons will be filled by the calling function
-  addon.icons = {};
-  addon.userPermissions = null;
-
-  return addon;
-}
-
 function defineSyncGUID(aAddon) {
   // Define .syncGUID as a lazy property which is also settable
   Object.defineProperty(aAddon, "syncGUID", {
@@ -760,32 +545,23 @@ function generateTemporaryInstallID(aFile) {
 }
 
 var loadManifest = async function(aPackage, aLocation, aOldAddon) {
-  async function loadFromRDF(aUri) {
-    let manifest = await aPackage.readString("install.rdf");
-    let addon = await loadManifestFromRDF(aUri, manifest, aPackage);
-
-    if (await aPackage.hasResource("icon.png")) {
-      addon.icons[32] = "icon.png";
-      addon.icons[48] = "icon.png";
+  let addon;
+  if (await aPackage.hasResource("manifest.json")) {
+    addon = await loadManifestFromWebManifest(aPackage.rootURI, aPackage);
+  } else {
+    for (let loader of AddonManagerPrivate.externalExtensionLoaders.values()) {
+      if (await aPackage.hasResource(loader.manifestFile)) {
+        addon = await loader.loadManifest(aPackage);
+        addon.loader = loader.name;
+        break;
+      }
     }
-
-    if (await aPackage.hasResource("icon64.png")) {
-      addon.icons[64] = "icon64.png";
-    }
-
-    return addon;
   }
 
-  let entry = await aPackage.getManifestFile();
-  if (!entry) {
+  if (!addon) {
     throw new Error(`File ${aPackage.filePath} does not contain a valid manifest`);
   }
 
-  let isWebExtension = entry == "manifest.json";
-  let addon = isWebExtension ?
-              await loadManifestFromWebManifest(aPackage.rootURI, aPackage) :
-              await loadFromRDF(aPackage.getURI("install.rdf"));
-
   addon._sourceBundle = aPackage.file;
   addon.location = aLocation;
 
@@ -795,11 +571,11 @@ var loadManifest = async function(aPackage, aLocation, aOldAddon) {
     addon.hidden = false;
   }
 
-  if (isWebExtension && !addon.id) {
+  if (!addon.id) {
     if (cert) {
       addon.id = cert.commonName;
       if (!gIDTest.test(addon.id)) {
-        throw new Error(`Webextension is signed with an invalid id (${addon.id})`);
+        throw new Error(`Extension is signed with an invalid id (${addon.id})`);
       }
     }
     if (!addon.id && aLocation.isTemporary) {
diff --git a/toolkit/mozapps/extensions/internal/XPIProvider.jsm b/toolkit/mozapps/extensions/internal/XPIProvider.jsm
index 0b25dcc963fa..541cb9f9a200 100644
--- a/toolkit/mozapps/extensions/internal/XPIProvider.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIProvider.jsm
@@ -33,7 +33,6 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   Langpack: "resource://gre/modules/Extension.jsm",
   FileUtils: "resource://gre/modules/FileUtils.jsm",
   OS: "resource://gre/modules/osfile.jsm",
-  ConsoleAPI: "resource://gre/modules/Console.jsm",
   JSONFile: "resource://gre/modules/JSONFile.jsm",
   TelemetrySession: "resource://gre/modules/TelemetrySession.jsm",
 
@@ -105,7 +104,7 @@ const XPI_PERMISSION                  = "install";
 
 const XPI_SIGNATURE_CHECK_PERIOD      = 24 * 60 * 60;
 
-const DB_SCHEMA = 27;
+const DB_SCHEMA = 28;
 
 const NOTIFICATION_TOOLBOX_CONNECTION_CHANGE      = "toolbox-connection-change";
 
@@ -363,7 +362,7 @@ const JSON_FIELDS = Object.freeze([
   "dependencies",
   "enabled",
   "file",
-  "isWebExtension",
+  "loader",
   "lastModifiedTime",
   "path",
   "runInSafeMode",
@@ -447,8 +446,8 @@ class XPIState {
     let json = {
       dependencies: this.dependencies,
       enabled: this.enabled,
-      isWebExtension: this.isWebExtension,
       lastModifiedTime: this.lastModifiedTime,
+      loader: this.loader,
       path: this.relativePath,
       runInSafeMode: this.runInSafeMode,
       signedState: this.signedState,
@@ -464,6 +463,10 @@ class XPIState {
     return json;
   }
 
+  get isWebExtension() {
+    return this.loader == null;
+  }
+
   /**
    * Update the last modified time for an add-on on disk.
    *
@@ -521,7 +524,8 @@ class XPIState {
 
     this.version = aDBAddon.version;
     this.type = aDBAddon.type;
-    this.isWebExtension = aDBAddon.isWebExtension;
+    this.loader = aDBAddon.loader;
+
     if (aDBAddon.startupData) {
       this.startupData = aDBAddon.startupData;
     }
@@ -1512,12 +1516,6 @@ class BootstrapScope {
     if (Services.appinfo.inSafeMode && !runInSafeMode)
       return null;
 
-    if (!addon.isWebExtension && addon.type == "extension" &&
-        aMethod == "startup") {
-      logger.debug(`Registering manifest for ${this.file.path}`);
-      Components.manager.addBootstrappedManifestLocation(this.file);
-    }
-
     try {
       if (!this.scope) {
         this.loadBootstrapScope(aReason);
@@ -1530,7 +1528,7 @@ class BootstrapScope {
       let method = undefined;
       let {scope} = this;
       try {
-        method = scope[aMethod] || Cu.evalInSandbox(`${aMethod};`, scope);
+        method = scope[aMethod];
       } catch (e) {
         // An exception will be caught if the expected method is not defined.
         // That will be logged below.
@@ -1589,12 +1587,6 @@ class BootstrapScope {
           XPIDatabase.updateAddonDisabledState(addon);
         }
       }
-
-      if (addon.type == "extension" && aMethod == "shutdown" &&
-          aReason != BOOTSTRAP_REASONS.APP_SHUTDOWN) {
-        logger.debug(`Removing manifest for ${this.file.path}`);
-        Components.manager.removeBootstrappedManifestLocation(this.file);
-      }
     }
   }
 
@@ -1643,26 +1635,12 @@ class BootstrapScope {
           throw new Error(`Unknown webextension type ${this.addon.type}`);
       }
     } else {
-      let uri = getURIForResourceInFile(this.file, "bootstrap.js").spec;
-
-      let principal = Services.scriptSecurityManager.getSystemPrincipal();
-      this.scope =
-        new Cu.Sandbox(principal, { sandboxName: uri,
-                                    addonId: this.addon.id,
-                                    wantGlobalProperties: ["ChromeUtils"],
-                                    metadata: { addonID: this.addon.id, URI: uri } });
-
-      try {
-        Object.assign(this.scope, BOOTSTRAP_REASONS);
-
-        XPCOMUtils.defineLazyGetter(
-          this.scope, "console",
-          () => new ConsoleAPI({ consoleID: `addon/${this.addon.id}` }));
-
-        Services.scriptloader.loadSubScript(uri, this.scope);
-      } catch (e) {
-        logger.warn(`Error loading bootstrap.js for ${this.addon.id}`, e);
+      let loader = AddonManagerPrivate.externalExtensionLoaders.get(this.addon.loader);
+      if (!loader) {
+        throw new Error(`Cannot find loader for ${this.addon.loader}`);
       }
+
+      this.scope = loader.loadScope(this.addon, this.file);
     }
 
     // Notify the BrowserToolboxProcess that a new addon has been loaded.
diff --git a/toolkit/mozapps/extensions/internal/moz.build b/toolkit/mozapps/extensions/internal/moz.build
index f0a27b669980..9c2f0483babe 100644
--- a/toolkit/mozapps/extensions/internal/moz.build
+++ b/toolkit/mozapps/extensions/internal/moz.build
@@ -8,6 +8,7 @@ EXTRA_JS_MODULES.addons += [
     'AddonRepository.jsm',
     'AddonSettings.jsm',
     'AddonUpdateChecker.jsm',
+    'BootstrapLoader.jsm',
     'Content.js',
     'GMPProvider.jsm',
     'LightweightThemeImageOptimizer.jsm',
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_shutdown.js b/toolkit/mozapps/extensions/test/xpcshell/test_shutdown.js
index df614d5b202f..609b304a9f00 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_shutdown.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_shutdown.js
@@ -10,6 +10,7 @@ const IGNORE = ["getPreferredIconURL", "escapeAddonURI",
                 "addAddonListener", "removeAddonListener",
                 "addInstallListener", "removeInstallListener",
                 "addManagerListener", "removeManagerListener",
+                "addExternalExtensionLoader",
                 "shutdown", "init",
                 "stateToString", "errorToString", "getUpgradeListener",
                 "addUpgradeListener", "removeUpgradeListener",

From d304bec8d7435b798f54cd61f7f8ef29bb908398 Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Sat, 24 Nov 2018 20:27:20 -0800
Subject: [PATCH 72/78] Bug 857456 Part 3: Convert (non-) strict compatibility
 tests to use an external extension loader r=kmag

The addon manager has a bunch of handling for "strict compatibility" --
an option for individual profiles and addons to control exactly how
compatibility is handled.  However, WebExtensions don't use or expose
this feature.  We want to retain this capability for other projects that
use the addon manager but since it cannot be tested with WebExtensions,
this patch converts its tests to use the new extension loading hook added
in the previous patch.

--HG--
extra : rebase_source : 023ef88b35d02f02976681894e9f052146f8b66c
extra : histedit_source : e6ff774d3639fe580f97bc7aa6df6f15a42d1a61
---
 .../extensions/internal/AddonTestUtils.jsm    |  12 +-
 .../extensions/internal/XPIDatabase.jsm       |  11 +-
 .../extensions/internal/XPIInstall.jsm        |   8 +-
 .../test/xpcshell/data/test_update.json       |  78 ----
 .../data/test_updatecompatmode_ignore.json    |  18 -
 .../data/test_updatecompatmode_normal.json    |  18 -
 .../data/test_updatecompatmode_strict.json    |  18 -
 .../extensions/test/xpcshell/head_compat.js   |  48 +++
 .../test/xpcshell/test_compatoverrides.js     | 382 ++++++++----------
 .../test_onPropertyChanged_appDisabled.js     |  71 ++--
 .../test/xpcshell/test_strictcompatibility.js | 174 +++-----
 .../test/xpcshell/test_update_compatmode.js   | 174 +++-----
 .../test/xpcshell/test_update_strictcompat.js | 266 +++++++-----
 .../extensions/test/xpcshell/test_upgrade.js  | 116 +++---
 .../test_webextension_install_syntax_error.js |   8 +-
 .../extensions/test/xpcshell/xpcshell.ini     |   7 +
 16 files changed, 563 insertions(+), 846 deletions(-)
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_ignore.json
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_normal.json
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_strict.json
 create mode 100644 toolkit/mozapps/extensions/test/xpcshell/head_compat.js

diff --git a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
index 7e9e05b2df04..4fcbe2781e51 100644
--- a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
+++ b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
@@ -672,9 +672,15 @@ var AddonTestUtils = {
         callback = callback.wrappedJSObject;
 
         try {
-          let manifestURI = this.getManifestURI(file);
-
-          let id = await this.getIDFromManifest(manifestURI);
+          let id;
+          try {
+            let manifestURI = this.getManifestURI(file);
+            id = await this.getIDFromManifest(manifestURI);
+          } catch (err) {
+            if (file.leafName.endsWith(".xpi")) {
+              id = file.leafName.slice(0, -4);
+            }
+          }
 
           let fakeCert = {commonName: id};
           if (this.usePrivilegedSignatures) {
diff --git a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
index 51fae8a9d7d0..d4e549a0ca08 100644
--- a/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIDatabase.jsm
@@ -95,11 +95,6 @@ const PENDING_INSTALL_METADATA =
      "updateDate", "applyBackgroundUpdates", "compatibilityOverrides",
      "installTelemetryInfo"];
 
-const COMPATIBLE_BY_DEFAULT_TYPES = {
-  extension: true,
-  dictionary: true,
-};
-
 // Properties to save in JSON file
 const PROP_JSON_FIELDS = ["id", "syncGUID", "version", "type",
                           "loader", "updateURL", "optionsURL",
@@ -445,10 +440,8 @@ class AddonInternal {
     // Only extensions and dictionaries can be compatible by default; themes
     // and language packs always use strict compatibility checking.
     // Dictionaries are compatible by default unless requested by the dictinary.
-    if (this.type in COMPATIBLE_BY_DEFAULT_TYPES &&
-        !this.strictCompatibility &&
-        (!AddonManager.strictCompatibility ||
-         this.type == "dictionary")) {
+    if (!this.strictCompatibility &&
+        (!AddonManager.strictCompatibility || this.type == "dictionary")) {
 
       // The repository can specify compatibility overrides.
       // Note: For now, only blacklisting is supported by overrides.
diff --git a/toolkit/mozapps/extensions/internal/XPIInstall.jsm b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
index 429ce9e5b53a..c3c0dd4b1223 100644
--- a/toolkit/mozapps/extensions/internal/XPIInstall.jsm
+++ b/toolkit/mozapps/extensions/internal/XPIInstall.jsm
@@ -154,11 +154,6 @@ const KEY_APP_PROFILE                 = "app-profile";
 const DIR_STAGE                       = "staged";
 const DIR_TRASH                       = "trash";
 
-const COMPATIBLE_BY_DEFAULT_TYPES = {
-  extension: true,
-  dictionary: true,
-};
-
 // This is a random number array that can be used as "salt" when generating
 // an automatic ID based on the directory path of an add-on. It will prevent
 // someone from creating an ID for a permanent add-on that could be replaced
@@ -2428,8 +2423,7 @@ UpdateChecker.prototype = {
     if (!AddonManager.checkCompatibility) {
       ignoreMaxVersion = true;
       ignoreStrictCompat = true;
-    } else if (this.addon.type in COMPATIBLE_BY_DEFAULT_TYPES &&
-               !AddonManager.strictCompatibility &&
+    } else if (!AddonManager.strictCompatibility &&
                !this.addon.strictCompatibility) {
       ignoreMaxVersion = true;
     }
diff --git a/toolkit/mozapps/extensions/test/xpcshell/data/test_update.json b/toolkit/mozapps/extensions/test/xpcshell/data/test_update.json
index 6b4c72e48a5a..9e3483d5209d 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/data/test_update.json
+++ b/toolkit/mozapps/extensions/test/xpcshell/data/test_update.json
@@ -119,84 +119,6 @@
       ]
     },
 
-    "addon9@tests.mozilla.org": {
-      "updates": [
-        {
-          "version": "2.0",
-          "update_link": "http://example.com/addons/test_update9_2.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "1",
-              "advisory_max_version": "1"
-            }
-          }
-        },
-        {
-          "_comment_": "Incompatible when strict compatibility is enabled",
-          "version": "3.0",
-          "update_link": "http://example.com/addons/test_update9_3.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "0.9",
-              "advisory_max_version": "0.9"
-            }
-          }
-        },
-        {
-          "_comment_": "Incompatible due to compatibility override",
-          "version": "4.0",
-          "update_link": "http://example.com/addons/test_update9_4.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "0.9",
-              "advisory_max_version": "0.9"
-            }
-          }
-        },
-        {
-          "_comment_": "Addon for future version of app",
-          "version": "4.0",
-          "update_link": "http://example.com/addons/test_update9_5.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "5",
-              "advisory_max_version": "6"
-            }
-          }
-        }
-      ]
-    },
-
-    "addon10@tests.mozilla.org": {
-      "updates": [
-        {
-          "version": "1.0",
-          "update_link": "http://example.com/addons/test_update10.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "0.1",
-              "advisory_max_version": "0.4"
-            }
-          }
-        }
-      ]
-    },
-
-    "addon11@tests.mozilla.org": {
-      "updates": [
-        {
-          "version": "2.0",
-          "update_link": "http://example.com/addons/test_update11.xpi",
-          "applications": {
-            "gecko": {
-              "strict_min_version": "0.1",
-              "strict_max_version": "0.2"
-            }
-          }
-        }
-      ]
-    },
-
     "addon12@tests.mozilla.org": {
       "updates": [
         {
diff --git a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_ignore.json b/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_ignore.json
deleted file mode 100644
index ac08c02f4e6d..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_ignore.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "addons": {
-    "compatmode-ignore@tests.mozilla.org": {
-      "updates": [
-        {
-          "applications": {
-            "gecko": {
-              "strict_min_version": "1",
-              "advisory_max_version": "2"
-            }
-          },
-          "version": "2.0",
-          "update_link": "https://example.com/addons/test1.xpi"
-        }
-      ]
-    }
-  }
-}
diff --git a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_normal.json b/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_normal.json
deleted file mode 100644
index ce43df1a4f8f..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_normal.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "addons": {
-    "compatmode-normal@tests.mozilla.org": {
-      "updates": [
-        {
-          "applications": {
-            "gecko": {
-              "strict_min_version": "1",
-              "advisory_max_version": "2"
-            }
-          },
-          "version": "2.0",
-          "update_link": "https://example.com/addons/test1.xpi"
-        }
-      ]
-    }
-  }
-}
diff --git a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_strict.json b/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_strict.json
deleted file mode 100644
index 7874865a4bf1..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/data/test_updatecompatmode_strict.json
+++ /dev/null
@@ -1,18 +0,0 @@
-{
-  "addons": {
-    "compatmode-strict@tests.mozilla.org": {
-      "updates": [
-        {
-          "applications": {
-            "gecko": {
-              "strict_min_version": "1",
-              "advisory_max_version": "2"
-            }
-          },
-          "version": "2.0",
-          "update_link": "https://example.com/addons/test1.xpi"
-        }
-      ]
-    }
-  }
-}
diff --git a/toolkit/mozapps/extensions/test/xpcshell/head_compat.js b/toolkit/mozapps/extensions/test/xpcshell/head_compat.js
new file mode 100644
index 000000000000..b8d1ac4ae25b
--- /dev/null
+++ b/toolkit/mozapps/extensions/test/xpcshell/head_compat.js
@@ -0,0 +1,48 @@
+//
+// This file provides helpers for tests of addons that use strictCompatibility.
+// Since WebExtensions cannot opt out of strictCompatibility, we add a
+// simple extension loader that lets tests directly set AddonInternal
+// properties (including strictCompatibility)
+//
+
+/* import-globals-from head_addons.js */
+
+const MANIFEST = "compat_manifest.json";
+
+AddonManager.addExternalExtensionLoader({
+  name: "compat-test",
+  manifestFile: MANIFEST,
+  async loadManifest(pkg) {
+    // XPIDatabase.jsm gets unloaded in AddonTestUtils when the
+    // addon manager is restarted.  Work around that by just importing
+    // it every time we need to create an AddonInternal.
+    ChromeUtils.import("resource://gre/modules/addons/XPIDatabase.jsm");
+    /* globals AddonInternal */
+    let addon = new AddonInternal();
+    let manifest = JSON.parse(await pkg.readString(MANIFEST));
+    Object.assign(addon, manifest);
+    return addon;
+  },
+  loadScope(addon, file) {
+    return {
+      install() {},
+      uninstall() {},
+      startup() {},
+      shutdonw() {},
+    };
+  },
+});
+
+const DEFAULTS = {
+  defaultLocale: {},
+  locales: [],
+  targetPlatforms: [],
+  type: "extension",
+  version: "1.0",
+};
+
+function createAddon(manifest) {
+  return AddonTestUtils.createTempXPIFile({
+    [MANIFEST]: Object.assign({}, DEFAULTS, manifest),
+  });
+}
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_compatoverrides.js b/toolkit/mozapps/extensions/test/xpcshell/test_compatoverrides.js
index 36cf82b576e2..1e9ec9eae9ca 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_compatoverrides.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_compatoverrides.js
@@ -7,23 +7,15 @@
 
 const PREF_GETADDONS_CACHE_ENABLED = "extensions.getAddons.cache.enabled";
 
-ChromeUtils.import("resource://testing-common/httpd.js");
-var gServer = new HttpServer();
-gServer.start(-1);
-gPort = gServer.identity.primaryPort;
-
-const PORT            = gPort;
-const BASE_URL        = "http://localhost:" + PORT;
+let testserver = createHttpServer({hosts: ["example.com"]});
 
 const GETADDONS_RESPONSE = {
   next: null,
   previous: null,
   results: [],
 };
-gServer.registerPathHandler("/addons.json", (request, response) => {
-  response.setHeader("content-type", "application/json");
-  response.write(JSON.stringify(GETADDONS_RESPONSE));
-});
+
+AddonTestUtils.registerJSON(testserver, "/addons.json", GETADDONS_RESPONSE);
 
 const COMPAT_RESPONSE = {
   next: null,
@@ -180,237 +172,179 @@ const COMPAT_RESPONSE = {
     },
   ],
 };
-gServer.registerPathHandler("/compat.json", (request, response) => {
-  response.setHeader("content-type", "application/json");
-  response.write(JSON.stringify(COMPAT_RESPONSE));
-});
+
+AddonTestUtils.registerJSON(testserver, "/compat.json", COMPAT_RESPONSE);
 
 Services.prefs.setBoolPref(PREF_EM_STRICT_COMPATIBILITY, false);
 Services.prefs.setBoolPref(PREF_GETADDONS_CACHE_ENABLED, true);
-Services.prefs.setCharPref(PREF_GETADDONS_BYIDS, `${BASE_URL}/addons.json`);
-Services.prefs.setCharPref(PREF_COMPAT_OVERRIDES, `${BASE_URL}/compat.json`);
+Services.prefs.setCharPref(PREF_GETADDONS_BYIDS, "http://example.com/addons.json");
+Services.prefs.setCharPref(PREF_COMPAT_OVERRIDES, "http://example.com/compat.json");
 
+const ADDONS = [
+  // Not hosted, no overrides
+  {
+    manifest:  {
+      id: "addon1@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: true,
+    overrides: 0,
+  },
 
-// Not hosted, no overrides
-var addon1 = {
-  id: "addon1@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 1",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, no overrides
+  {
+    manifest: {
+      id: "addon2@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: true,
+    overrides: 0,
+  },
 
-// Hosted, no overrides
-var addon2 = {
-  id: "addon2@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 2",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, matching override
+  {
+    manifest: {
+      id: "addon3@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: false,
+    overrides: 1,
+  },
 
-// Hosted, matching override
-var addon3 = {
-  id: "addon3@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 3",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, matching override,
+  // wouldn't be compatible if strict checking is enabled
+  {
+    manifest: {
+      id: "addon4@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "0.1",
+        maxVersion: "0.2",
+      }],
+    },
+    compatible: false,
+    overrides: 1,
+  },
 
-// Hosted, matching override, wouldn't be compatible if strict checking is enabled
-var addon4 = {
-  id: "addon4@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 4",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "0.1",
-    maxVersion: "0.2",
-  }],
-};
+  // Hosted, app ID doesn't match in override
+  {
+    manifest: {
+      id: "addon5@tests.mozilla.org",
+      version: "1.0",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: true,
+    overrides: 0,
+  },
 
-// Hosted, app ID doesn't match in override
-var addon5 = {
-  id: "addon5@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 5",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, addon version range doesn't match in override
+  {
+    manifest: {
+      id: "addon6@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: true,
+    overrides: 1,
+  },
 
-// Hosted, addon version range doesn't match in override
-var addon6 = {
-  id: "addon6@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 6",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, app version range doesn't match in override
+  {
+    manifest: {
+      id: "addon7@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: true,
+    overrides: 1,
+  },
 
-// Hosted, app version range doesn't match in override
-var addon7 = {
-  id: "addon7@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 7",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Hosted, multiple overrides
+  {
+    manifest: {
+      id: "addon8@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: false,
+    overrides: 3,
+  },
 
-// Hosted, multiple overrides
-var addon8 = {
-  id: "addon8@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 8",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
+  // Not hosted, matching override
+  {
+    manifest: {
+      id: "addon9@tests.mozilla.org",
+      targetApplications: [{
+        id: "xpcshell@tests.mozilla.org",
+        minVersion: "1",
+        maxVersion: "1",
+      }],
+    },
+    compatible: false,
+    overrides: 1,
+  },
+];
 
-// Not hosted, matching override
-var addon9 = {
-  id: "addon9@tests.mozilla.org",
-  version: "1.0",
-  name: "Test addon 9",
-  bootstrap: true,
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1",
-  }],
-};
-
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
-
-async function run_test() {
-  do_test_pending();
+add_task(async function run_tests() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "2");
 
-  await promiseWriteInstallRDFForExtension(addon1, profileDir);
-  await promiseWriteInstallRDFForExtension(addon2, profileDir);
-  await promiseWriteInstallRDFForExtension(addon3, profileDir);
-  await promiseWriteInstallRDFForExtension(addon4, profileDir);
-  await promiseWriteInstallRDFForExtension(addon5, profileDir);
-  await promiseWriteInstallRDFForExtension(addon6, profileDir);
-  await promiseWriteInstallRDFForExtension(addon7, profileDir);
-  await promiseWriteInstallRDFForExtension(addon8, profileDir);
-  await promiseWriteInstallRDFForExtension(addon9, profileDir);
-
+  for (let addon of ADDONS) {
+    let xpi = await createAddon(addon.manifest);
+    await manuallyInstall(xpi, AddonTestUtils.profileExtensions,
+                          addon.manifest.id);
+  }
   await promiseStartupManager();
 
-  AddonManagerInternal.backgroundUpdateCheck().then(run_test_1);
-}
+  await AddonManagerInternal.backgroundUpdateCheck();
 
-function end_test() {
-  gServer.stop(do_test_finished);
-}
+  async function check() {
+    for (let info of ADDONS) {
+      let {id} = info.manifest;
+      let addon = await promiseAddonByID(id);
+      Assert.notEqual(addon, null, `Found ${id}`);
+      let overrides = AddonRepository.getCompatibilityOverridesSync(id);
+      if (info.overrides === 0) {
+        Assert.equal(overrides, null, `Got no overrides for ${id}`);
+      } else {
+        Assert.notEqual(overrides, null, `Got overrides for ${id}`);
+        Assert.equal(overrides.length, info.overrides,
+                     `Got right number of overrides for ${id}`);
+      }
+      Assert.equal(addon.isCompatible, info.compatible,
+                   `Got expected compatibility for ${id}`);
+      Assert.equal(addon.appDisabled, !info.compatible,
+                   `Got expected appDisabled for ${id}`);
+    }
+  }
 
-async function check_compat_status(aCallback) {
-  let [a1, a2, a3, a4, a5, a6, a7, a8, a9] = await AddonManager.getAddonsByIDs(["addon1@tests.mozilla.org",
-                                                                                "addon2@tests.mozilla.org",
-                                                                                "addon3@tests.mozilla.org",
-                                                                                "addon4@tests.mozilla.org",
-                                                                                "addon5@tests.mozilla.org",
-                                                                                "addon6@tests.mozilla.org",
-                                                                                "addon7@tests.mozilla.org",
-                                                                                "addon8@tests.mozilla.org",
-                                                                                "addon9@tests.mozilla.org"]);
-  Assert.notEqual(a1, null);
-  Assert.equal(AddonRepository.getCompatibilityOverridesSync(a1.id), null);
-  Assert.ok(a1.isCompatible);
-  Assert.ok(!a1.appDisabled);
+  await check();
 
-  Assert.notEqual(a2, null);
-  Assert.equal(AddonRepository.getCompatibilityOverridesSync(a2.id), null);
-  Assert.ok(a2.isCompatible);
-  Assert.ok(!a2.appDisabled);
-
-  Assert.notEqual(a3, null);
-  let overrides = AddonRepository.getCompatibilityOverridesSync(a3.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 1);
-  Assert.ok(!a3.isCompatible);
-  Assert.ok(a3.appDisabled);
-
-  Assert.notEqual(a4, null);
-  overrides = AddonRepository.getCompatibilityOverridesSync(a4.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 1);
-  Assert.ok(!a4.isCompatible);
-  Assert.ok(a4.appDisabled);
-
-  Assert.notEqual(a5, null);
-  Assert.equal(AddonRepository.getCompatibilityOverridesSync(a5.id), null);
-  Assert.ok(a5.isCompatible);
-  Assert.ok(!a5.appDisabled);
-
-  Assert.notEqual(a6, null);
-  overrides = AddonRepository.getCompatibilityOverridesSync(a6.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 1);
-  Assert.ok(a6.isCompatible);
-  Assert.ok(!a6.appDisabled);
-
-  Assert.notEqual(a7, null);
-  overrides = AddonRepository.getCompatibilityOverridesSync(a7.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 1);
-  Assert.ok(a7.isCompatible);
-  Assert.ok(!a7.appDisabled);
-
-  Assert.notEqual(a8, null);
-  overrides = AddonRepository.getCompatibilityOverridesSync(a8.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 3);
-  Assert.ok(!a8.isCompatible);
-  Assert.ok(a8.appDisabled);
-
-  Assert.notEqual(a9, null);
-  overrides = AddonRepository.getCompatibilityOverridesSync(a9.id);
-  Assert.notEqual(overrides, null);
-  Assert.equal(overrides.length, 1);
-  Assert.ok(!a9.isCompatible);
-  Assert.ok(a9.appDisabled);
-
-  executeSoon(aCallback);
-}
-
-function run_test_1() {
-  info("Run test 1");
-  check_compat_status(run_test_2);
-}
-
-async function run_test_2() {
-  info("Run test 2");
   await promiseRestartManager();
-  check_compat_status(end_test);
-}
+
+  await check();
+});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_onPropertyChanged_appDisabled.js b/toolkit/mozapps/extensions/test/xpcshell/test_onPropertyChanged_appDisabled.js
index d000aaf0290e..bf80f9727333 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_onPropertyChanged_appDisabled.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_onPropertyChanged_appDisabled.js
@@ -1,64 +1,45 @@
 /* Any copyright is dedicated to the Public Domain.
    http://creativecommons.org/publicdomain/zero/1.0/ */
 
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
-
-async function run_test() {
-  do_test_pending();
+const ID = "addon1@tests.mozilla.org";
+add_task(async function run_test() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 
-  await promiseWriteInstallRDFForExtension({
-    id: "addon1@tests.mozilla.org",
-    version: "1.0",
-    name: "Test 1",
-    bootstrap: true,
+  let xpi = createAddon({
+    id: ID,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "0.1",
       maxVersion: "0.2",
     }],
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID);
 
+  AddonManager.strictCompatibility = false;
   await promiseStartupManager();
 
-  AddonManager.strictCompatibility = false;
+  let addon = await AddonManager.getAddonByID(ID);
+  Assert.notEqual(addon, null);
+  await addon.disable();
 
-  let aAddon = await AddonManager.getAddonByID("addon1@tests.mozilla.org");
-  Assert.notEqual(aAddon, null);
-  await aAddon.disable();
-  executeSoon(run_test_1);
-}
-
-async function run_test_1() {
-  await promiseRestartManager();
-  let aAddon = await AddonManager.getAddonByID("addon1@tests.mozilla.org");
-  Assert.notEqual(aAddon, null);
-  Assert.ok(aAddon.userDisabled);
-  Assert.ok(!aAddon.isActive);
-  Assert.ok(!aAddon.appDisabled);
-
-  prepare_test({
-    "addon1@tests.mozilla.org": [
-      ["onPropertyChanged", ["appDisabled"]],
-    ],
-  }, [], run_test_2);
+  Assert.ok(addon.userDisabled);
+  Assert.ok(!addon.isActive);
+  Assert.ok(!addon.appDisabled);
 
+  let promise = promiseAddonEvent("onPropertyChanged");
   AddonManager.strictCompatibility = true;
-}
+  let [, properties] = await promise;
 
-async function run_test_2() {
-  let aAddon = await AddonManager.getAddonByID("addon1@tests.mozilla.org");
-  Assert.notEqual(aAddon, null);
-  Assert.ok(aAddon.userDisabled);
-  Assert.ok(!aAddon.isActive);
-  Assert.ok(aAddon.appDisabled);
-
-  prepare_test({
-    "addon1@tests.mozilla.org": [
-      ["onPropertyChanged", ["appDisabled"]],
-    ],
-  }, [], callback_soon(do_test_finished));
+  Assert.deepEqual(properties, ["appDisabled"],
+                   "Got onPropertyChanged for appDisabled");
+  Assert.ok(addon.appDisabled);
 
+  promise = promiseAddonEvent("onPropertyChanged");
   AddonManager.strictCompatibility = false;
-}
+  [, properties] = await promise;
+
+  Assert.deepEqual(properties, ["appDisabled"],
+                   "Got onPropertyChanged for appDisabled");
+  Assert.ok(!addon.appDisabled);
+});
+
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_strictcompatibility.js b/toolkit/mozapps/extensions/test/xpcshell/test_strictcompatibility.js
index 8abe753216ad..8d6b53a07740 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_strictcompatibility.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_strictcompatibility.js
@@ -8,23 +8,17 @@
 
 // The `compatbile` array defines which of the tests below the add-on
 // should be compatible in. It's pretty gross.
-const ADDONS = {
+const ADDONS = [
   // Always compatible
-  "addon1@tests.mozilla.org": {
-    "install.rdf": {
+  {
+    manifest: {
       id: "addon1@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 1",
-      bootstrap: true,
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
         minVersion: "1",
         maxVersion: "1",
       }],
     },
-    expected: {
-      strictCompatibility: false,
-    },
     compatible: {
       nonStrict: true,
       strict: true,
@@ -32,21 +26,15 @@ const ADDONS = {
   },
 
   // Incompatible in strict compatibility mode
-  "addon2@tests.mozilla.org": {
-    "install.rdf": {
+  {
+    manifest: {
       id: "addon2@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 2",
-      bootstrap: true,
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
         minVersion: "0.7",
         maxVersion: "0.8",
       }],
     },
-    expected: {
-      strictCompatibility: false,
-    },
     compatible: {
       nonStrict: true,
       strict: false,
@@ -54,12 +42,9 @@ const ADDONS = {
   },
 
   // Opt-in to strict compatibility - always incompatible
-  "addon4@tests.mozilla.org": {
-    "install.rdf": {
-      id: "addon4@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 4",
-      bootstrap: true,
+  {
+    manifest: {
+      id: "addon3@tests.mozilla.org",
       strictCompatibility: true,
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
@@ -67,9 +52,6 @@ const ADDONS = {
         maxVersion: "0.9",
       }],
     },
-    expected: {
-      strictCompatibility: true,
-    },
     compatible: {
       nonStrict: false,
       strict: false,
@@ -78,89 +60,50 @@ const ADDONS = {
 
   // Addon from the future - would be marked as compatibile-by-default,
   // but minVersion is higher than the app version
-  "addon5@tests.mozilla.org": {
-    "install.rdf": {
-      id: "addon5@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 5",
-      bootstrap: true,
+  {
+    manifest: {
+      id: "addon4@tests.mozilla.org",
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
         minVersion: "3",
         maxVersion: "5",
       }],
     },
-    expected: {
-      strictCompatibility: false,
-    },
     compatible: {
       nonStrict: false,
       strict: false,
     },
   },
 
-  // Extremely old addon - maxVersion is less than the minimum compat version
-  // set in extensions.minCompatibleVersion
-  "addon6@tests.mozilla.org": {
-    "install.rdf": {
-      id: "addon6@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 6",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "0.1",
-        maxVersion: "0.2",
-      }],
-    },
-    expected: {
-      strictCompatibility: false,
-    },
-    compatible: {
-      nonStrict: true,
-      strict: false,
-    },
-  },
-
   // Dictionary - compatible even in strict compatibility mode
-  "addon7@tests.mozilla.org": {
-    "install.rdf": {
-      id: "addon7@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 7",
-      type: "64",
+  {
+    manifest: {
+      id: "addon5@tests.mozilla.org",
+      type: "dictionary",
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
         minVersion: "0.8",
         maxVersion: "0.9",
       }],
     },
-    expected: {
-      strictCompatibility: false,
-    },
     compatible: {
       nonStrict: true,
       strict: true,
     },
   },
-};
-
-const IDS = Object.keys(ADDONS);
-
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
+];
 
 async function checkCompatStatus(strict, index) {
   info(`Checking compat status for test ${index}\n`);
 
   equal(AddonManager.strictCompatibility, strict);
 
-  let addons = await getAddons(IDS);
-  for (let [id, addon] of Object.entries(ADDONS)) {
-    checkAddon(id, addons.get(id), {
-      ...addon.expected,
-      isCompatible: addon.compatible[index],
-      appDisabled: !addon.compatible[index],
+  for (let test of ADDONS) {
+    let {id} = test.manifest;
+    let addon = await promiseAddonByID(id);
+    checkAddon(id, addon, {
+      isCompatible: test.compatible[index],
+      appDisabled: !test.compatible[index],
     });
   }
 }
@@ -168,8 +111,9 @@ async function checkCompatStatus(strict, index) {
 add_task(async function setup() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 
-  for (let addon of Object.values(ADDONS)) {
-    await promiseWriteInstallRDFForExtension(addon["install.rdf"], profileDir);
+  for (let addon of ADDONS) {
+    let xpi = await createAddon(addon.manifest);
+    await manuallyInstall(xpi, AddonTestUtils.profileExtensions, addon.manifest.id);
   }
 
   await promiseStartupManager();
@@ -191,14 +135,11 @@ add_task(async function test_2() {
   await checkCompatStatus(true, "strict");
 });
 
-const CHECK_COMPAT_ADDONS = {
-  "cc-addon1@tests.mozilla.org": {
-    "install.rdf": {
-      // Cannot be enabled as it has no target app info for the applciation
+const CHECK_COMPAT_ADDONS = [
+  // Cannot be enabled as it has no target app info for the applciation
+  {
+    manifest: {
       id: "cc-addon1@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 1",
-      bootstrap: true,
       targetApplications: [{
         id: "unknown@tests.mozilla.org",
         minVersion: "1",
@@ -209,14 +150,12 @@ const CHECK_COMPAT_ADDONS = {
     canOverride: false,
   },
 
-  "cc-addon2@tests.mozilla.org": {
-    "install.rdf": {
-      // Always appears incompatible but can be enabled if compatibility checking is
-      // disabled
+
+  // Always appears incompatible but can be enabled if compatibility checking is
+  // disabled
+  {
+    manifest: {
       id: "cc-addon2@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 2",
-      bootstrap: true,
       targetApplications: [{
         id: "toolkit@mozilla.org",
         minVersion: "1",
@@ -227,13 +166,10 @@ const CHECK_COMPAT_ADDONS = {
     canOverride: true,
   },
 
-  "cc-addon4@tests.mozilla.org": {
-    "install.rdf": {
-      // Always compatible and enabled
-      id: "cc-addon4@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 4",
-      bootstrap: true,
+  // Always compatible and enabled
+  {
+    manifest: {
+      id: "cc-addon3@tests.mozilla.org",
       targetApplications: [{
         id: "toolkit@mozilla.org",
         minVersion: "1",
@@ -243,13 +179,10 @@ const CHECK_COMPAT_ADDONS = {
     compatible: true,
   },
 
-  "cc-addon5@tests.mozilla.org": {
-    "install.rdf": {
-      // Always compatible and enabled
-      id: "cc-addon5@tests.mozilla.org",
-      version: "1.0",
-      name: "Test 5",
-      bootstrap: true,
+  // Always compatible and enabled
+  {
+    manifest: {
+      id: "cc-addon4@tests.mozilla.org",
       targetApplications: [{
         id: "xpcshell@tests.mozilla.org",
         minVersion: "1",
@@ -258,17 +191,15 @@ const CHECK_COMPAT_ADDONS = {
     },
     compatible: true,
   },
-};
-
-const CHECK_COMPAT_IDS = Object.keys(CHECK_COMPAT_ADDONS);
+];
 
 async function checkCompatOverrides(overridden) {
-  let addons = await getAddons(CHECK_COMPAT_IDS);
-
-  for (let [id, addon] of Object.entries(CHECK_COMPAT_ADDONS)) {
-    checkAddon(id, addons.get(id), {
-      isCompatible: addon.compatible,
-      isActive: addon.compatible || (overridden && addon.canOverride),
+  for (let test of CHECK_COMPAT_ADDONS) {
+    let {id} = test.manifest;
+    let addon = await promiseAddonByID(id);
+    checkAddon(id, addon, {
+      isCompatible: test.compatible,
+      isActive: test.compatible || (overridden && test.canOverride),
     });
   }
 }
@@ -283,8 +214,10 @@ add_task(async function setupCheckCompat() {
   Object.assign(AddonTestUtils.appInfo,
                 {version: "2.2.3", platformVersion: "2"});
 
-  for (let addon of Object.values(CHECK_COMPAT_ADDONS)) {
-    await promiseWriteInstallRDFForExtension(addon["install.rdf"], profileDir);
+  for (let addon of CHECK_COMPAT_ADDONS) {
+    let {manifest} = addon;
+    let xpi = await createAddon(manifest);
+    await manuallyInstall(xpi, AddonTestUtils.profileExtensions, manifest.id);
   }
   await promiseRestartManager("2.2.3");
 });
@@ -329,3 +262,4 @@ add_task(async function test_compat_overrides_4() {
 
   await checkCompatOverrides(false);
 });
+
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_update_compatmode.js b/toolkit/mozapps/extensions/test/xpcshell/test_update_compatmode.js
index 07ba07830d4a..34e3024ae328 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_update_compatmode.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_update_compatmode.js
@@ -5,174 +5,90 @@
 // This verifies that add-on update check correctly fills in the
 // %COMPATIBILITY_MODE% token in the update URL.
 
+Cu.importGlobalProperties(["URLSearchParams"]);
 
 // The test extension uses an insecure update url.
 Services.prefs.setBoolPref(PREF_EM_CHECK_UPDATE_SECURITY, false);
 
-var testserver = AddonTestUtils.createHttpServer({hosts: ["example.com"]});
-testserver.registerDirectory("/data/", do_get_file("data"));
+let testserver = createHttpServer({hosts: ["example.com"]});
 
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
+let lastMode;
+testserver.registerPathHandler("/update.json", (request, response) => {
+  let params = new URLSearchParams(request.queryString);
+  lastMode = params.get("mode");
 
-async function run_test() {
-  do_test_pending();
+  response.setHeader("content-type", "application/json", true);
+  response.write(JSON.stringify({addons: {}}));
+});
+
+const ID_NORMAL = "compatmode@tests.mozilla.org";
+const ID_STRICT = "compatmode-strict@tests.mozilla.org";
+
+add_task(async function setup() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 
-  await promiseWriteInstallRDFForExtension({
-    id: "compatmode-normal@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/test_updatecompatmode_%COMPATIBILITY_MODE%.json",
+  let xpi = await createAddon({
+    id: ID_NORMAL,
+    updateURL: "http://example.com/update.json?mode=%COMPATIBILITY_MODE%",
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "1",
       maxVersion: "1",
     }],
-    name: "Test Addon - normal",
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID_NORMAL);
 
-  await promiseWriteInstallRDFForExtension({
-    id: "compatmode-strict@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/test_updatecompatmode_%COMPATIBILITY_MODE%.json",
-    targetApplications: [{
-      id: "xpcshell@tests.mozilla.org",
-      minVersion: "1",
-      maxVersion: "1",
-    }],
-    name: "Test Addon - strict",
-  }, profileDir);
-
-  await promiseWriteInstallRDFForExtension({
-    id: "compatmode-strict-optin@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/test_updatecompatmode_%COMPATIBILITY_MODE%.json",
-    targetApplications: [{
-      id: "xpcshell@tests.mozilla.org",
-      minVersion: "1",
-      maxVersion: "1",
-    }],
-    name: "Test Addon - strict opt-in",
+  xpi = await createAddon({
+    id: ID_STRICT,
+    updateURL: "http://example.com/update.json?mode=%COMPATIBILITY_MODE%",
     strictCompatibility: true,
-  }, profileDir);
-
-  await promiseWriteInstallRDFForExtension({
-    id: "compatmode-ignore@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/test_updatecompatmode_%COMPATIBILITY_MODE%.json",
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "1",
       maxVersion: "1",
     }],
-    name: "Test Addon - ignore",
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID_STRICT);
 
   await promiseStartupManager();
-  run_test_1();
-}
-
-function end_test() {
-  do_test_finished();
-}
-
+});
 
 // Strict compatibility checking disabled.
-async function run_test_1() {
-  info("Testing with strict compatibility checking disabled");
+add_task(async function test_strict_disabled() {
   Services.prefs.setBoolPref(PREF_EM_STRICT_COMPATIBILITY, false);
-  let addon = await AddonManager.getAddonByID("compatmode-normal@tests.mozilla.org");
+  let addon = await AddonManager.getAddonByID(ID_NORMAL);
   Assert.notEqual(addon, null);
-  addon.findUpdates({
-    onCompatibilityUpdateAvailable() {
-      do_throw("Should have not have seen compatibility information");
-    },
 
-    onNoUpdateAvailable() {
-      do_throw("Should have seen an available update");
-    },
-
-    onUpdateAvailable(unused, install) {
-      Assert.equal(install.version, "2.0");
-    },
-
-    onUpdateFinished() {
-      run_test_2();
-    },
-  }, AddonManager.UPDATE_WHEN_USER_REQUESTED);
-}
+  await promiseFindAddonUpdates(addon, AddonManager.UPDATE_WHEN_USER_REQUESTED);
+  Assert.equal(lastMode, "normal", "COMPATIBIILITY_MODE normal was set correctly");
+});
 
 // Strict compatibility checking enabled.
-async function run_test_2() {
-  info("Testing with strict compatibility checking enabled");
+add_task(async function test_strict_enabled() {
   Services.prefs.setBoolPref(PREF_EM_STRICT_COMPATIBILITY, true);
-  let addon = await AddonManager.getAddonByID("compatmode-strict@tests.mozilla.org");
+  let addon = await AddonManager.getAddonByID(ID_NORMAL);
   Assert.notEqual(addon, null);
-  addon.findUpdates({
-    onCompatibilityUpdateAvailable() {
-      do_throw("Should have not have seen compatibility information");
-    },
 
-    onNoUpdateAvailable() {
-      do_throw("Should have seen an available update");
-    },
-
-    onUpdateAvailable(unused, install) {
-      Assert.equal(install.version, "2.0");
-    },
-
-    onUpdateFinished() {
-      run_test_3();
-    },
-  }, AddonManager.UPDATE_WHEN_USER_REQUESTED);
-}
+  await promiseFindAddonUpdates(addon, AddonManager.UPDATE_WHEN_USER_REQUESTED);
+  Assert.equal(lastMode, "strict", "COMPATIBILITY_MODE strict was set correctly");
+});
 
 // Strict compatibility checking opt-in.
-async function run_test_3() {
-  info("Testing with strict compatibility disabled, but addon opt-in");
+add_task(async function test_strict_optin() {
   Services.prefs.setBoolPref(PREF_EM_STRICT_COMPATIBILITY, false);
-  let addon = await AddonManager.getAddonByID("compatmode-strict-optin@tests.mozilla.org");
+  let addon = await AddonManager.getAddonByID(ID_STRICT);
   Assert.notEqual(addon, null);
-  addon.findUpdates({
-    onCompatibilityUpdateAvailable() {
-      do_throw("Should have not have seen compatibility information");
-    },
 
-    onUpdateAvailable() {
-      do_throw("Should not have seen an available update");
-    },
-
-    onUpdateFinished() {
-      run_test_4();
-    },
-  }, AddonManager.UPDATE_WHEN_USER_REQUESTED);
-}
+  await promiseFindAddonUpdates(addon, AddonManager.UPDATE_WHEN_USER_REQUESTED);
+  Assert.equal(lastMode, "normal", "COMPATIBILITY_MODE is normal even for an addon with strictCompatibility");
+});
 
 // Compatibility checking disabled.
-async function run_test_4() {
-  info("Testing with all compatibility checking disabled");
+add_task(async function test_compat_disabled() {
   AddonManager.checkCompatibility = false;
-  let addon = await AddonManager.getAddonByID("compatmode-ignore@tests.mozilla.org");
+  let addon = await AddonManager.getAddonByID(ID_NORMAL);
   Assert.notEqual(addon, null);
-  addon.findUpdates({
-    onCompatibilityUpdateAvailable() {
-      do_throw("Should have not have seen compatibility information");
-    },
 
-    onNoUpdateAvailable() {
-      do_throw("Should have seen an available update");
-    },
-
-    onUpdateAvailable(unused, install) {
-      Assert.equal(install.version, "2.0");
-    },
-
-    onUpdateFinished() {
-      end_test();
-    },
-  }, AddonManager.UPDATE_WHEN_USER_REQUESTED);
-}
+  await promiseFindAddonUpdates(addon, AddonManager.UPDATE_WHEN_USER_REQUESTED);
+  Assert.equal(lastMode, "ignore", "COMPATIBILITY_MODE ignore was set correctly");
+});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_update_strictcompat.js b/toolkit/mozapps/extensions/test/xpcshell/test_update_strictcompat.js
index c1714ba6b29e..e0b087ae890c 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_update_strictcompat.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_update_strictcompat.js
@@ -2,9 +2,8 @@
  * http://creativecommons.org/publicdomain/zero/1.0/
  */
 
-// This verifies that add-on update checks work
-// This file is a placeholder for now, it holds test cases related to
-// strict compatibility to be moved or removed shortly.
+// This verifies that add-on update checks work in conjunction with
+// strict compatibility settings.
 
 const PREF_GETADDONS_CACHE_ENABLED = "extensions.getAddons.cache.enabled";
 
@@ -12,160 +11,205 @@ const PREF_GETADDONS_CACHE_ENABLED = "extensions.getAddons.cache.enabled";
 Services.prefs.setBoolPref(PREF_EM_CHECK_UPDATE_SECURITY, false);
 Services.prefs.setBoolPref(PREF_EM_STRICT_COMPATIBILITY, false);
 
-const updateFile = "test_update.json";
 const appId = "toolkit@mozilla.org";
 
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
-
-const ADDONS = {
-  test_update: {
-    id: "addon1@tests.mozilla.org",
-    version: "2.0",
-    name: "Test 1",
-  },
-  test_update8: {
-    id: "addon8@tests.mozilla.org",
-    version: "2.0",
-    name: "Test 8",
-  },
-  test_update12: {
-    id: "addon12@tests.mozilla.org",
-    version: "2.0",
-    name: "Test 12",
-  },
-  test_install2_1: {
-    id: "addon2@tests.mozilla.org",
-    version: "2.0",
-    name: "Real Test 2",
-  },
-  test_install2_2: {
-    id: "addon2@tests.mozilla.org",
-    version: "3.0",
-    name: "Real Test 3",
-  },
-};
-
-var testserver = createHttpServer({hosts: ["example.com"]});
+testserver = createHttpServer({hosts: ["example.com"]});
 testserver.registerDirectory("/data/", do_get_file("data"));
 
-const XPIS = {};
-
 add_task(async function setup() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
-
-  Services.locale.requestedLocales = ["fr-FR"];
-
-  for (let [name, info] of Object.entries(ADDONS)) {
-    XPIS[name] = createTempWebExtensionFile({
-      manifest: {
-        name: info.name,
-        version: info.version,
-        applications: {gecko: {id: info.id}},
-      },
-    });
-    testserver.registerFile(`/addons/${name}.xpi`, XPIS[name]);
-  }
-
   AddonTestUtils.updateReason = AddonManager.UPDATE_WHEN_USER_REQUESTED;
 
-  await promiseStartupManager();
+  Services.prefs.setCharPref(PREF_GETADDONS_BYIDS,
+                             "http://example.com/data/test_update_addons.json");
+  Services.prefs.setCharPref(PREF_COMPAT_OVERRIDES,
+                             "http://example.com/data/test_update_compat.json");
+  Services.prefs.setBoolPref(PREF_GETADDONS_CACHE_ENABLED, true);
 });
 
 // Test that the update check correctly observes the
 // extensions.strictCompatibility pref and compatibility overrides.
-add_task(async function test_17() {
-  await promiseInstallWebExtension({
-    manifest: {
-      name: "Test Addon 9",
-      version: "1.0",
-      applications: {
-        gecko: {
-          id: "addon9@tests.mozilla.org",
-          update_url: "http://example.com/data/" + updateFile,
-        },
+add_task(async function test_update_strict() {
+  const ID = "addon9@tests.mozilla.org";
+  let xpi = await createAddon({
+    id: ID,
+    updateURL: "http://example.com/update.json",
+    targetApplications: [{
+      id: "xpcshell@tests.mozilla.org",
+      minVersion: "0.1",
+      maxVersion: "0.2",
+    }],
+  });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID);
+
+  await promiseStartupManager();
+
+  await AddonRepository.backgroundUpdateCheck();
+
+  let UPDATE = {
+    addons: {
+      [ID]: {
+        updates: [
+          {
+            version: "2.0",
+            update_link: "http://example.com/addons/test_update9_2.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "1",
+                advisory_max_version: "1",
+              },
+            },
+          },
+
+          // Incompatible when strict compatibility is enabled
+          {
+            version: "3.0",
+            update_link: "http://example.com/addons/test_update9_3.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "0.9",
+                advisory_max_version: "0.9",
+              },
+            },
+          },
+
+          // Incompatible due to compatibility override
+          {
+            version: "4.0",
+            update_link: "http://example.com/addons/test_update9_4.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "0.9",
+                advisory_max_version: "0.9",
+              },
+            },
+          },
+
+          // Addon for future version of app
+          {
+            version: "4.0",
+            update_link: "http://example.com/addons/test_update9_5.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "5",
+                advisory_max_version: "6",
+              },
+            },
+          },
+        ],
       },
     },
-  });
+  };
 
-  let listener;
-  await new Promise(resolve => {
-    listener = {
-      onNewInstall(aInstall) {
-        equal(aInstall.existingAddon.id, "addon9@tests.mozilla.org",
-              "Saw unexpected onNewInstall for " + aInstall.existingAddon.id);
-        equal(aInstall.version, "3.0");
-      },
-      onDownloadFailed(aInstall) {
-        resolve();
-      },
-    };
-    AddonManager.addInstallListener(listener);
+  AddonTestUtils.registerJSON(testserver, "/update.json", UPDATE);
 
-    Services.prefs.setCharPref(PREF_GETADDONS_BYIDS,
-                               `http://example.com/data/test_update_addons.json`);
-    Services.prefs.setCharPref(PREF_COMPAT_OVERRIDES,
-                               `http://example.com/data/test_update_compat.json`);
-    Services.prefs.setBoolPref(PREF_GETADDONS_CACHE_ENABLED, true);
+  let addon = await AddonManager.getAddonByID(ID);
+  let {updateAvailable} = await promiseFindAddonUpdates(addon);
 
-    AddonManagerInternal.backgroundUpdateCheck();
-  });
+  Assert.notEqual(updateAvailable, null, "Got update");
+  Assert.equal(updateAvailable.version, "3.0", "The correct update was selected");
+  await addon.uninstall();
 
-  AddonManager.removeInstallListener(listener);
-
-  let a9 = await AddonManager.getAddonByID("addon9@tests.mozilla.org");
-  await a9.uninstall();
+  await promiseShutdownManager();
 });
 
 // Tests that compatibility updates are applied to addons when the updated
 // compatibility data wouldn't match with strict compatibility enabled.
-add_task(async function test_18() {
-  await promiseInstallXPI({
-    id: "addon10@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/" + updateFile,
+add_task(async function test_update_strict2() {
+  const ID = "addon10@tests.mozilla.org";
+  let xpi = createAddon({
+    id: ID,
+    updateURL: "http://example.com/update.json",
     targetApplications: [{
       id: appId,
       minVersion: "0.1",
       maxVersion: "0.2",
     }],
-    name: "Test Addon 10",
   });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID);
 
-  let a10 = await AddonManager.getAddonByID("addon10@tests.mozilla.org");
-  notEqual(a10, null);
+  await promiseStartupManager();
+  await AddonRepository.backgroundUpdateCheck();
 
-  let result = await AddonTestUtils.promiseFindAddonUpdates(a10);
+  const UPDATE = {
+    addons: {
+      [ID]: {
+        updates: [
+          {
+            version: "1.0",
+            update_link: "http://example.com/addons/test_update10.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "0.1",
+                advisory_max_version: "0.4",
+              },
+            },
+          },
+        ],
+      },
+    },
+  };
+
+  AddonTestUtils.registerJSON(testserver, "/update.json", UPDATE);
+
+  let addon = await AddonManager.getAddonByID(ID);
+  notEqual(addon, null);
+
+  let result = await promiseFindAddonUpdates(addon);
   ok(result.compatibilityUpdate, "Should have seen a compatibility update");
   ok(!result.updateAvailable, "Should not have seen a version update");
 
-  await a10.uninstall();
+  await addon.uninstall();
+  await promiseShutdownManager();
 });
 
 // Test that the update check correctly observes when an addon opts-in to
 // strict compatibility checking.
-add_task(async function test_19() {
-  await promiseInstallXPI({
-    id: "addon11@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    updateURL: "http://example.com/data/" + updateFile,
+add_task(async function test_update_strict_optin() {
+  const ID = "addon11@tests.mozilla.org";
+  let xpi = await createAddon({
+    id: ID,
+    updateURL: "http://example.com/update.json",
     targetApplications: [{
       id: appId,
       minVersion: "0.1",
       maxVersion: "0.2",
     }],
-    name: "Test Addon 11",
   });
+  await manuallyInstall(xpi, AddonTestUtils.profileExtensions, ID);
 
-  let a11 = await AddonManager.getAddonByID("addon11@tests.mozilla.org");
-  notEqual(a11, null);
+  await promiseStartupManager();
 
-  let result = await AddonTestUtils.promiseFindAddonUpdates(a11);
+  await AddonRepository.backgroundUpdateCheck();
+
+  const UPDATE = {
+    addons: {
+      [ID]: {
+        updates: [
+          {
+            version: "2.0",
+            update_link: "http://example.com/addons/test_update11.xpi",
+            applications: {
+              gecko: {
+                strict_min_version: "0.1",
+                strict_max_version: "0.2",
+              },
+            },
+          },
+        ],
+      },
+    },
+  };
+
+  AddonTestUtils.registerJSON(testserver, "/update.json", UPDATE);
+
+  let addon = await AddonManager.getAddonByID(ID);
+  notEqual(addon, null);
+
+  let result = await AddonTestUtils.promiseFindAddonUpdates(addon);
   ok(!result.compatibilityUpdate, "Should not have seen a compatibility update");
   ok(!result.updateAvailable, "Should not have seen a version update");
 
-  await a11.uninstall();
+  await addon.uninstall();
+  await promiseShutdownManager();
 });
-
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_upgrade.js b/toolkit/mozapps/extensions/test/xpcshell/test_upgrade.js
index df74ef1452b3..728d48ab0d67 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_upgrade.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_upgrade.js
@@ -21,75 +21,78 @@ globalDir.append("extensions");
 var gGlobalExisted = globalDir.exists();
 var gInstallTime = Date.now();
 
+const ID1 = "addon1@tests.mozilla.org";
+const ID2 = "addon2@tests.mozilla.org";
+const ID3 = "addon3@tests.mozilla.org";
+const ID4 = "addon4@tests.mozilla.org";
+const PATH4 = OS.Path.join(globalDir.path, `${ID4}.xpi`);
+
 add_task(async function setup() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
 
   // Will be compatible in the first version and incompatible in subsequent versions
-  await promiseWriteInstallRDFForExtension({
-    id: "addon1@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
+  let xpi = await createAddon({
+    id: ID1,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "1",
       maxVersion: "1",
     }],
-    name: "Test Addon 1",
     targetPlatforms: [
-      "XPCShell",
-      "WINNT_x86",
+      {os: "XPCShell"},
+      {os: "WINNT_x86"},
     ],
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, profileDir, ID1);
+
 
   // Works in all tested versions
-  await promiseWriteInstallRDFForExtension({
-    id: "addon2@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
+  xpi = await createAddon({
+    id: ID2,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "1",
       maxVersion: "2",
     }],
-    name: "Test Addon 2",
     targetPlatforms: [
-      "XPCShell_noarch-spidermonkey",
+      {
+        os: "XPCShell",
+        abi: "noarch-spidermonkey",
+      },
     ],
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, profileDir, ID2);
 
   // Will be disabled in the first version and enabled in the second.
-  await promiseWriteInstallRDFForExtension({
-    id: "addon3@tests.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
+  xpi = createAddon({
+    id: ID3,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "2",
       maxVersion: "2",
     }],
-    name: "Test Addon 3",
-  }, profileDir);
+  });
+  await manuallyInstall(xpi, profileDir, ID3);
 
   // Will be compatible in both versions but will change version in between
-  var dest = await promiseWriteInstallRDFForExtension({
-    id: "addon4@tests.mozilla.org",
+  xpi = await createAddon({
+    id: ID4,
     version: "1.0",
-    bootstrap: true,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "1",
       maxVersion: "1",
     }],
-    name: "Test Addon 4",
-  }, globalDir);
-  setExtensionModifiedTime(dest, gInstallTime);
+  });
+  await manuallyInstall(xpi, globalDir, ID4);
+  await promiseSetExtensionModifiedTime(PATH4, gInstallTime);
 });
 
 registerCleanupFunction(function end_test() {
   if (!gGlobalExisted) {
     globalDir.remove(true);
   } else {
-    globalDir.append(do_get_expected_addon_name("addon4@tests.mozilla.org"));
+    globalDir.append(do_get_expected_addon_name(ID4));
     globalDir.remove(true);
   }
 });
@@ -98,22 +101,20 @@ registerCleanupFunction(function end_test() {
 add_task(async function test_1() {
   await promiseStartupManager();
 
-  let [a1, a2, a3, a4] = await AddonManager.getAddonsByIDs(["addon1@tests.mozilla.org",
-                                                            "addon2@tests.mozilla.org",
-                                                            "addon3@tests.mozilla.org",
-                                                            "addon4@tests.mozilla.org"]);
-  Assert.notEqual(a1, null);
-  Assert.ok(isExtensionInBootstrappedList(profileDir, a1.id));
+  let [a1, a2, a3, a4] = await promiseAddonsByIDs([ID1, ID2, ID3, ID4]);
+  Assert.notEqual(a1, null, "Found extension 1");
+  Assert.equal(a1.isActive, true, "Extension 1 is active");
 
-  Assert.notEqual(a2, null);
-  Assert.ok(isExtensionInBootstrappedList(profileDir, a2.id));
+  Assert.notEqual(a2, null, "Found extension 2");
+  Assert.equal(a2.isActive, true, "Extension 2 is active");
 
-  Assert.notEqual(a3, null);
-  Assert.ok(!isExtensionInBootstrappedList(profileDir, a3.id));
+  Assert.notEqual(a3, null, "Found extension 3");
+  Assert.equal(a3.isActive, false, "Extension 3 is not active");
 
   Assert.notEqual(a4, null);
-  Assert.ok(isExtensionInBootstrappedList(globalDir, a4.id));
+  Assert.equal(a4.isActive, true);
   Assert.equal(a4.version, "1.0");
+
 });
 
 // Test that upgrading the application doesn't disable now incompatible add-ons
@@ -121,24 +122,20 @@ add_task(async function test_2() {
   await promiseShutdownManager();
 
   // Upgrade the extension
-  var dest = await promiseWriteInstallRDFForExtension({
-    id: "addon4@tests.mozilla.org",
+  let xpi = createAddon({
+    id: ID4,
     version: "2.0",
-    bootstrap: true,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "2",
       maxVersion: "2",
     }],
-    name: "Test Addon 4",
-  }, globalDir);
-  setExtensionModifiedTime(dest, gInstallTime);
+  });
+  await manuallyInstall(xpi, globalDir, ID4);
+  await promiseSetExtensionModifiedTime(PATH4, gInstallTime);
 
   await promiseStartupManager("2");
-  let [a1, a2, a3, a4] = await AddonManager.getAddonsByIDs(["addon1@tests.mozilla.org",
-                                                            "addon2@tests.mozilla.org",
-                                                            "addon3@tests.mozilla.org",
-                                                            "addon4@tests.mozilla.org"]);
+  let [a1, a2, a3, a4] = await promiseAddonsByIDs([ID1, ID2, ID3, ID4]);
   Assert.notEqual(a1, null);
   Assert.ok(isExtensionInBootstrappedList(profileDir, a1.id));
 
@@ -158,28 +155,24 @@ add_task(async function test_3() {
   await promiseShutdownManager();
 
   // Upgrade the extension
-  var dest = await promiseWriteInstallRDFForExtension({
-    id: "addon4@tests.mozilla.org",
+  let xpi = createAddon({
+    id: ID4,
     version: "3.0",
-    bootstrap: true,
     targetApplications: [{
       id: "xpcshell@tests.mozilla.org",
       minVersion: "3",
       maxVersion: "3",
     }],
-    name: "Test Addon 4",
-  }, globalDir);
-  setExtensionModifiedTime(dest, gInstallTime);
+  });
+  await manuallyInstall(xpi, globalDir, ID4);
+  await promiseSetExtensionModifiedTime(PATH4, gInstallTime);
 
-  // Simulates a simple Build ID change, the platform deletes extensions.ini
-  // whenever the application is changed.
+  // Simulates a simple Build ID change
   gAddonStartup.remove(true);
   await promiseStartupManager();
 
-  let [a1, a2, a3, a4] = await AddonManager.getAddonsByIDs(["addon1@tests.mozilla.org",
-                                                            "addon2@tests.mozilla.org",
-                                                            "addon3@tests.mozilla.org",
-                                                            "addon4@tests.mozilla.org"]);
+  let [a1, a2, a3, a4] = await promiseAddonsByIDs([ID1, ID2, ID3, ID4]);
+
   Assert.notEqual(a1, null);
   Assert.ok(isExtensionInBootstrappedList(profileDir, a1.id));
 
@@ -195,3 +188,4 @@ add_task(async function test_3() {
 
   await promiseShutdownManager();
 });
+
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_webextension_install_syntax_error.js b/toolkit/mozapps/extensions/test/xpcshell/test_webextension_install_syntax_error.js
index 369324bf057a..306b35739507 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/test_webextension_install_syntax_error.js
+++ b/toolkit/mozapps/extensions/test/xpcshell/test_webextension_install_syntax_error.js
@@ -7,11 +7,10 @@ add_task(async function setup() {
 
 add_task(async function install_xpi() {
 
-  // Data for WebExtension with syntax error
+  // WebExtension with a JSON syntax error in manifest.json
   let xpi1 = Extension.generateXPI({
     files: {
       "manifest.json": String.raw`{
-        // This is a manifest. Intentional syntax error in next line.
         "manifest_version: 2,
         "applications": {"gecko": {"id": "${ADDON_ID}"}},
         "name": "Temp WebExt with Error",
@@ -20,11 +19,10 @@ add_task(async function install_xpi() {
     },
   });
 
-  // Data for WebExtension without syntax error
+  // Valid WebExtension
   let xpi2 = Extension.generateXPI({
     files: {
       "manifest.json": String.raw`{
-        // This is a manifest.
         "manifest_version": 2,
         "applications": {"gecko": {"id": "${ADDON_ID}"}},
         "name": "Temp WebExt without Error",
@@ -41,7 +39,7 @@ add_task(async function install_xpi() {
   xpi2.moveTo(xpi1.parent, xpi1.leafName);
 
   let install2 = await AddonManager.getInstallForFile(xpi2);
-  Assert.notEqual(install2.error, AddonManager.ERROR_CORRUPT_FILE);
+  Assert.equal(install2.error, 0);
 
   xpi1.remove(false);
 });
diff --git a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
index 8a6067de0391..aa2c7b31a977 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
+++ b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
@@ -71,6 +71,7 @@ tags = blocklist
 [test_cacheflush.js]
 [test_childprocess.js]
 [test_compatoverrides.js]
+head = head_addons.js head_compat.js
 [test_corrupt.js]
 [test_crash_annotation_quoting.js]
 [test_db_path.js]
@@ -149,6 +150,7 @@ skip-if = true
 [test_no_addons.js]
 [test_nodisable_hidden.js]
 [test_onPropertyChanged_appDisabled.js]
+head = head_addons.js head_compat.js
 [test_overrideblocklist.js]
 run-sequentially = Uses global XCurProcD dir.
 tags = blocklist
@@ -200,6 +202,7 @@ tags = blocklist
 # Bug 676992: test consistently fails on Android
 fail-if = os == "android"
 [test_strictcompatibility.js]
+head = head_addons.js head_compat.js
 [test_syncGUID.js]
 [test_system_allowed.js]
 head = head_addons.js head_system_addons.js
@@ -245,9 +248,12 @@ skip-if = os == "win" # Bug 1358846
 skip-if = os == "android"
 [test_updateCancel.js]
 [test_update_compatmode.js]
+head = head_addons.js head_compat.js
 [test_update_ignorecompat.js]
 skip-if = true # Bug 676922 Bug 1437697
 [test_update_rdf.js]
+[test_update_strictcompat.js]
+head = head_addons.js head_compat.js
 [test_update_webextensions.js]
 tags = webextensions
 [test_updatecheck.js]
@@ -258,6 +264,7 @@ skip-if = os == "android"
 # Bug 676992: test consistently hangs on Android
 skip-if = os == "android"
 [test_upgrade.js]
+head = head_addons.js head_compat.js
 # Bug 676992: test consistently hangs on Android
 skip-if = os == "android"
 run-sequentially = Uses global XCurProcD dir.

From 59d8cce6420eb1469315399d5585b97556ecbdea Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Mon, 26 Nov 2018 13:44:32 -0800
Subject: [PATCH 73/78] Bug 857456 Part 4: Remove toolkit support for
 install.rdf r=kmag

--HG--
extra : rebase_source : b844d31e0e001376f1c25a9a555c8134d1e512ec
extra : histedit_source : 83870efa70d1a0dc34e9e7bc3015aaa8c736de9f
---
 toolkit/mozapps/extensions/AddonManager.jsm   |    4 -
 .../extensions/internal/AddonTestUtils.jsm    |    3 -
 .../extensions/internal/BootstrapLoader.jsm   |  363 -----
 .../internal/RDFManifestConverter.jsm         |   73 +-
 toolkit/mozapps/extensions/internal/moz.build |    1 -
 .../test/xpcshell/test_bootstrap.js           | 1183 -----------------
 .../test/xpcshell/test_bootstrap_const.js     |   27 -
 .../test/xpcshell/test_bootstrap_globals.js   |   66 -
 .../test_bootstrapped_chrome_manifest.js      |   50 -
 .../test/xpcshell/test_invalid_install_rdf.js |  113 --
 .../extensions/test/xpcshell/test_legacy.js   |  118 --
 .../extensions/test/xpcshell/test_manifest.js |  752 -----------
 .../test/xpcshell/test_manifest_locales.js    |  134 --
 .../extensions/test/xpcshell/xpcshell.ini     |   11 -
 14 files changed, 1 insertion(+), 2897 deletions(-)
 delete mode 100644 toolkit/mozapps/extensions/internal/BootstrapLoader.jsm
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_const.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_globals.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_bootstrapped_chrome_manifest.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_invalid_install_rdf.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_legacy.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_manifest.js
 delete mode 100644 toolkit/mozapps/extensions/test/xpcshell/test_manifest_locales.js

diff --git a/toolkit/mozapps/extensions/AddonManager.jsm b/toolkit/mozapps/extensions/AddonManager.jsm
index 607d0d3f6bac..60bbf764e19a 100644
--- a/toolkit/mozapps/extensions/AddonManager.jsm
+++ b/toolkit/mozapps/extensions/AddonManager.jsm
@@ -780,10 +780,6 @@ var AddonManagerInternal = {
         }
       }
 
-      // XXX temporary
-      ChromeUtils.import("resource://gre/modules/addons/BootstrapLoader.jsm");
-      AddonManager.addExternalExtensionLoader(BootstrapLoader);
-
       // Load any providers registered in the category manager
       for (let {entry, value: url} of Services.catMan.enumerateCategory(CATEGORY_PROVIDER_MODULE)) {
         try {
diff --git a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
index 4fcbe2781e51..7af1b528fb30 100644
--- a/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
+++ b/toolkit/mozapps/extensions/internal/AddonTestUtils.jsm
@@ -849,9 +849,6 @@ var AddonTestUtils = {
     Cu.unload("resource://gre/modules/addons/XPIDatabase.jsm");
     Cu.unload("resource://gre/modules/addons/XPIInstall.jsm");
 
-    // XXX
-    Cu.unload("resource://gre/modules/addons/BootstrapLoader.jsm");
-
     let ExtensionScope = ChromeUtils.import("resource://gre/modules/Extension.jsm", null);
     ChromeUtils.defineModuleGetter(ExtensionScope, "XPIProvider",
                                    "resource://gre/modules/addons/XPIProvider.jsm");
diff --git a/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm b/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm
deleted file mode 100644
index d57e358dc5d2..000000000000
--- a/toolkit/mozapps/extensions/internal/BootstrapLoader.jsm
+++ /dev/null
@@ -1,363 +0,0 @@
-
-"use strict";
-
-var EXPORTED_SYMBOLS = ["BootstrapLoader"];
-
-ChromeUtils.import("resource://gre/modules/AddonManager.jsm");
-ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
-
-XPCOMUtils.defineLazyModuleGetters(this, {
-  AddonInternal: "resource://gre/modules/addons/XPIDatabase.jsm",
-  Blocklist: "resource://gre/modules/Blocklist.jsm",
-  ConsoleAPI: "resource://gre/modules/Console.jsm",
-  InstallRDF: "resource://gre/modules/addons/RDFManifestConverter.jsm",
-  Services: "resource://gre/modules/Services.jsm",
-});
-
-XPCOMUtils.defineLazyGetter(this, "BOOTSTRAP_REASONS", () => {
-  const {XPIProvider} = ChromeUtils.import("resource://gre/modules/addons/XPIProvider.jsm", {});
-  return XPIProvider.BOOTSTRAP_REASONS;
-});
-
-ChromeUtils.import("resource://gre/modules/Log.jsm");
-var logger = Log.repository.getLogger("addons.bootstrap");
-
-/**
- * Valid IDs fit this pattern.
- */
-var gIDTest = /^(\{[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\}|[a-z0-9-\._]*\@[a-z0-9-\._]+)$/i;
-
-// Properties that exist in the install manifest
-const PROP_METADATA      = ["id", "version", "type", "internalName", "updateURL",
-                            "optionsURL", "optionsType", "aboutURL", "iconURL"];
-const PROP_LOCALE_SINGLE = ["name", "description", "creator", "homepageURL"];
-const PROP_LOCALE_MULTI  = ["developers", "translators", "contributors"];
-
-// Map new string type identifiers to old style nsIUpdateItem types.
-// Retired values:
-// 32 = multipackage xpi file
-// 8 = locale
-// 256 = apiextension
-// 128 = experiment
-// theme = 4
-const TYPES = {
-  extension: 2,
-  dictionary: 64,
-};
-
-const COMPATIBLE_BY_DEFAULT_TYPES = {
-  extension: true,
-  dictionary: true,
-};
-
-const hasOwnProperty = Function.call.bind(Object.prototype.hasOwnProperty);
-
-function isXPI(filename) {
-  let ext = filename.slice(-4).toLowerCase();
-  return ext === ".xpi" || ext === ".zip";
-}
-
-/**
- * Gets an nsIURI for a file within another file, either a directory or an XPI
- * file. If aFile is a directory then this will return a file: URI, if it is an
- * XPI file then it will return a jar: URI.
- *
- * @param {nsIFile} aFile
- *        The file containing the resources, must be either a directory or an
- *        XPI file
- * @param {string} aPath
- *        The path to find the resource at, "/" separated. If aPath is empty
- *        then the uri to the root of the contained files will be returned
- * @returns {nsIURI}
- *        An nsIURI pointing at the resource
- */
-function getURIForResourceInFile(aFile, aPath) {
-  if (!isXPI(aFile.leafName)) {
-    let resource = aFile.clone();
-    if (aPath)
-      aPath.split("/").forEach(part => resource.append(part));
-
-    return Services.io.newFileURI(resource);
-  }
-
-  return buildJarURI(aFile, aPath);
-}
-
-/**
- * Creates a jar: URI for a file inside a ZIP file.
- *
- * @param {nsIFile} aJarfile
- *        The ZIP file as an nsIFile
- * @param {string} aPath
- *        The path inside the ZIP file
- * @returns {nsIURI}
- *        An nsIURI for the file
- */
-function buildJarURI(aJarfile, aPath) {
-  let uri = Services.io.newFileURI(aJarfile);
-  uri = "jar:" + uri.spec + "!/" + aPath;
-  return Services.io.newURI(uri);
-}
-
-var BootstrapLoader = {
-  name: "bootstrap",
-  manifestFile: "install.rdf",
-  async loadManifest(pkg) {
-    /**
-     * Reads locale properties from either the main install manifest root or
-     * an em:localized section in the install manifest.
-     *
-     * @param {Object} aSource
-     *        The resource to read the properties from.
-     * @param {boolean} isDefault
-     *        True if the locale is to be read from the main install manifest
-     *        root
-     * @param {string[]} aSeenLocales
-     *        An array of locale names already seen for this install manifest.
-     *        Any locale names seen as a part of this function will be added to
-     *        this array
-     * @returns {Object}
-     *        an object containing the locale properties
-     */
-    function readLocale(aSource, isDefault, aSeenLocales) {
-      let locale = {};
-      if (!isDefault) {
-        locale.locales = [];
-        for (let localeName of aSource.locales || []) {
-          if (!localeName) {
-            logger.warn("Ignoring empty locale in localized properties");
-            continue;
-          }
-          if (aSeenLocales.includes(localeName)) {
-            logger.warn("Ignoring duplicate locale in localized properties");
-            continue;
-          }
-          aSeenLocales.push(localeName);
-          locale.locales.push(localeName);
-        }
-
-        if (locale.locales.length == 0) {
-          logger.warn("Ignoring localized properties with no listed locales");
-          return null;
-        }
-      }
-
-      for (let prop of [...PROP_LOCALE_SINGLE, ...PROP_LOCALE_MULTI]) {
-        if (hasOwnProperty(aSource, prop)) {
-          locale[prop] = aSource[prop];
-        }
-      }
-
-      return locale;
-    }
-
-    let manifestData = await pkg.readString("install.rdf");
-    let manifest = InstallRDF.loadFromString(manifestData).decode();
-
-    let addon = new AddonInternal();
-    for (let prop of PROP_METADATA) {
-      if (hasOwnProperty(manifest, prop)) {
-        addon[prop] = manifest[prop];
-      }
-    }
-
-    if (!addon.type) {
-      addon.type = "extension";
-    } else {
-      let type = addon.type;
-      addon.type = null;
-      for (let name in TYPES) {
-        if (TYPES[name] == type) {
-          addon.type = name;
-          break;
-        }
-      }
-    }
-
-    if (!(addon.type in TYPES))
-      throw new Error("Install manifest specifies unknown type: " + addon.type);
-
-    if (!addon.id)
-      throw new Error("No ID in install manifest");
-    if (!gIDTest.test(addon.id))
-      throw new Error("Illegal add-on ID " + addon.id);
-    if (!addon.version)
-      throw new Error("No version in install manifest");
-
-    addon.strictCompatibility = (!(addon.type in COMPATIBLE_BY_DEFAULT_TYPES) ||
-                                 manifest.strictCompatibility == "true");
-
-    // Only read these properties for extensions.
-    if (addon.type == "extension") {
-      if (manifest.bootstrap != "true") {
-        throw new Error("Non-restartless extensions no longer supported");
-      }
-
-      if (addon.optionsType &&
-          addon.optionsType != AddonManager.OPTIONS_TYPE_INLINE_BROWSER &&
-          addon.optionsType != AddonManager.OPTIONS_TYPE_TAB) {
-            throw new Error("Install manifest specifies unknown optionsType: " + addon.optionsType);
-      }
-    } else {
-      // Convert legacy dictionaries into a format the WebExtension
-      // dictionary loader can process.
-      if (addon.type === "dictionary") {
-        addon.loader = null;
-        let dictionaries = {};
-        await pkg.iterFiles(({path}) => {
-          let match = /^dictionaries\/([^\/]+)\.dic$/.exec(path);
-          if (match) {
-            let lang = match[1].replace(/_/g, "-");
-            dictionaries[lang] = match[0];
-          }
-        });
-        addon.startupData = {dictionaries};
-      }
-
-      // Only extensions are allowed to provide an optionsURL, optionsType,
-      // optionsBrowserStyle, or aboutURL. For all other types they are silently ignored
-      addon.aboutURL = null;
-      addon.optionsBrowserStyle = null;
-      addon.optionsType = null;
-      addon.optionsURL = null;
-    }
-
-    addon.defaultLocale = readLocale(manifest, true);
-
-    let seenLocales = [];
-    addon.locales = [];
-    for (let localeData of manifest.localized || []) {
-      let locale = readLocale(localeData, false, seenLocales);
-      if (locale)
-        addon.locales.push(locale);
-    }
-
-    let dependencies = new Set(manifest.dependencies);
-    addon.dependencies = Object.freeze(Array.from(dependencies));
-
-    let seenApplications = [];
-    addon.targetApplications = [];
-    for (let targetApp of manifest.targetApplications || []) {
-      if (!targetApp.id || !targetApp.minVersion ||
-          !targetApp.maxVersion) {
-            logger.warn("Ignoring invalid targetApplication entry in install manifest");
-            continue;
-      }
-      if (seenApplications.includes(targetApp.id)) {
-        logger.warn("Ignoring duplicate targetApplication entry for " + targetApp.id +
-                    " in install manifest");
-        continue;
-      }
-      seenApplications.push(targetApp.id);
-      addon.targetApplications.push(targetApp);
-    }
-
-    // Note that we don't need to check for duplicate targetPlatform entries since
-    // the RDF service coalesces them for us.
-    addon.targetPlatforms = [];
-    for (let targetPlatform of manifest.targetPlatforms || []) {
-      let platform = {
-        os: null,
-        abi: null,
-      };
-
-      let pos = targetPlatform.indexOf("_");
-      if (pos != -1) {
-        platform.os = targetPlatform.substring(0, pos);
-        platform.abi = targetPlatform.substring(pos + 1);
-      } else {
-        platform.os = targetPlatform;
-      }
-
-      addon.targetPlatforms.push(platform);
-    }
-
-    addon.userDisabled = false;
-    addon.softDisabled = addon.blocklistState == Blocklist.STATE_SOFTBLOCKED;
-    addon.applyBackgroundUpdates = AddonManager.AUTOUPDATE_DEFAULT;
-
-    addon.userPermissions = null;
-
-    addon.icons = {};
-    if (await pkg.hasResource("icon.png")) {
-      addon.icons[32] = "icon.png";
-      addon.icons[48] = "icon.png";
-    }
-
-    if (await pkg.hasResource("icon64.png")) {
-      addon.icons[64] = "icon64.png";
-    }
-
-    return addon;
-  },
-
-  loadScope(addon, file) {
-    let uri = getURIForResourceInFile(file, "bootstrap.js").spec;
-    let principal = Services.scriptSecurityManager.getSystemPrincipal();
-
-    let sandbox = new Cu.Sandbox(principal, {
-      sandboxName: uri,
-      addonId: addon.id,
-      wantGlobalProperties: ["ChromeUtils"],
-      metadata: { addonID: addon.id, URI: uri },
-    });
-
-    try {
-      Object.assign(sandbox, BOOTSTRAP_REASONS);
-
-      XPCOMUtils.defineLazyGetter(sandbox, "console", () =>
-        new ConsoleAPI({ consoleID: `addon/${addon.id}` }));
-
-      Services.scriptloader.loadSubScript(uri, sandbox);
-    } catch (e) {
-      logger.warn(`Error loading bootstrap.js for ${addon.id}`, e);
-    }
-
-    function findMethod(name) {
-      if (sandbox.name) {
-        return sandbox.name;
-      }
-
-      try {
-        let method = Cu.evalInSandbox(name, sandbox);
-        return method;
-      } catch (err) { }
-
-      return () => {
-        logger.warn(`Add-on ${addon.id} is missing bootstrap method ${name}`);
-      };
-    }
-
-    let install = findMethod("install");
-    let uninstall = findMethod("uninstall");
-    let startup = findMethod("startup");
-    let shutdown = findMethod("shutdown");
-
-    return {
-      install: (...args) => install(...args),
-      uninstall: (...args) => uninstall(...args),
-
-      startup(...args) {
-        if (addon.type == "extension") {
-          logger.debug(`Registering manifest for ${file.path}\n`);
-          Components.manager.addBootstrappedManifestLocation(file);
-        }
-        return startup(...args);
-      },
-
-      shutdown(data, reason) {
-        try {
-          return shutdown(data, reason);
-        } catch (err) {
-          throw err;
-        } finally {
-          if (reason != BOOTSTRAP_REASONS.APP_SHUTDOWN) {
-            logger.debug(`Removing manifest for ${file.path}\n`);
-            Components.manager.removeBootstrappedManifestLocation(file);
-          }
-        }
-      },
-    };
-  },
-};
-
diff --git a/toolkit/mozapps/extensions/internal/RDFManifestConverter.jsm b/toolkit/mozapps/extensions/internal/RDFManifestConverter.jsm
index c792a8b60c09..0b63d7c6e7a9 100644
--- a/toolkit/mozapps/extensions/internal/RDFManifestConverter.jsm
+++ b/toolkit/mozapps/extensions/internal/RDFManifestConverter.jsm
@@ -3,7 +3,7 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 "use strict";
 
-var EXPORTED_SYMBOLS = ["InstallRDF", "UpdateRDFConverter"];
+var EXPORTED_SYMBOLS = ["UpdateRDFConverter"];
 
 ChromeUtils.defineModuleGetter(this, "RDFDataSource",
                                "resource://gre/modules/addons/RDFDataSource.jsm");
@@ -16,8 +16,6 @@ const PREFIX_THEME          = "urn:mozilla:theme:";
 
 const TOOLKIT_ID            = "toolkit@mozilla.org";
 
-const RDFURI_INSTALL_MANIFEST_ROOT = "urn:mozilla:install-manifest";
-
 function EM_R(aProperty) {
   return `http://www.mozilla.org/2004/em-rdf#${aProperty}`;
 }
@@ -52,75 +50,6 @@ class Manifest {
   }
 }
 
-class InstallRDF extends Manifest {
-  _readProps(source, obj, props) {
-    for (let prop of props) {
-      let val = getProperty(source, prop);
-      if (val != null) {
-        obj[prop] = val;
-      }
-    }
-  }
-
-  _readArrayProp(source, obj, prop, target, decode = getValue) {
-    let result = Array.from(source.getObjects(EM_R(prop)),
-                            target => decode(target));
-    if (result.length) {
-      obj[target] = result;
-    }
-  }
-
-  _readArrayProps(source, obj, props, decode = getValue) {
-    for (let [prop, target] of Object.entries(props)) {
-      this._readArrayProp(source, obj, prop, target, decode);
-    }
-  }
-
-  _readLocaleStrings(source, obj) {
-    this._readProps(source, obj, ["name", "description", "creator", "homepageURL"]);
-    this._readArrayProps(source, obj, {
-      locale: "locales",
-      developer: "developers",
-      translator: "translators",
-      contributor: "contributors",
-    });
-  }
-
-  decode() {
-    let root = this.ds.getResource(RDFURI_INSTALL_MANIFEST_ROOT);
-    let result = {};
-
-    let props = ["id", "version", "type", "updateURL", "optionsURL",
-                 "optionsType", "aboutURL", "iconURL",
-                 "bootstrap", "unpack", "strictCompatibility"];
-    this._readProps(root, result, props);
-
-    let decodeTargetApplication = source => {
-      let app = {};
-      this._readProps(source, app, ["id", "minVersion", "maxVersion"]);
-      return app;
-    };
-
-    let decodeLocale = source => {
-      let localized = {};
-      this._readLocaleStrings(source, localized);
-      return localized;
-    };
-
-    this._readLocaleStrings(root, result);
-
-    this._readArrayProps(root, result, {"targetPlatform": "targetPlatforms"});
-    this._readArrayProps(root, result, {"targetApplication": "targetApplications"},
-                         decodeTargetApplication);
-    this._readArrayProps(root, result, {"localized": "localized"},
-                         decodeLocale);
-    this._readArrayProps(root, result, {"dependency": "dependencies"},
-                         source => getProperty(source, "id"));
-
-    return result;
-  }
-}
-
 class UpdateRDF extends Manifest {
   decode() {
     let addons = {};
diff --git a/toolkit/mozapps/extensions/internal/moz.build b/toolkit/mozapps/extensions/internal/moz.build
index 9c2f0483babe..f0a27b669980 100644
--- a/toolkit/mozapps/extensions/internal/moz.build
+++ b/toolkit/mozapps/extensions/internal/moz.build
@@ -8,7 +8,6 @@ EXTRA_JS_MODULES.addons += [
     'AddonRepository.jsm',
     'AddonSettings.jsm',
     'AddonUpdateChecker.jsm',
-    'BootstrapLoader.jsm',
     'Content.js',
     'GMPProvider.jsm',
     'LightweightThemeImageOptimizer.jsm',
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js
deleted file mode 100644
index 83d9bbcd029b..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap.js
+++ /dev/null
@@ -1,1183 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-const APP_STARTUP                     = 1;
-const APP_SHUTDOWN                    = 2;
-const ADDON_ENABLE                    = 3;
-const ADDON_DISABLE                   = 4;
-const ADDON_INSTALL                   = 5;
-const ADDON_UNINSTALL                 = 6;
-const ADDON_UPGRADE                   = 7;
-const ADDON_DOWNGRADE                 = 8;
-
-const ID1 = "bootstrap1@tests.mozilla.org";
-const ID2 = "bootstrap2@tests.mozilla.org";
-
-// This verifies that bootstrappable add-ons can be used without restarts.
-ChromeUtils.import("resource://gre/modules/Services.jsm");
-
-// Enable loading extensions from the user scopes
-Services.prefs.setIntPref("extensions.enabledScopes",
-                          AddonManager.SCOPE_PROFILE + AddonManager.SCOPE_USER);
-
-BootstrapMonitor.init();
-
-createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
-
-const profileDir = gProfD.clone();
-profileDir.append("extensions");
-const userExtDir = gProfD.clone();
-userExtDir.append("extensions2");
-userExtDir.append(gAppInfo.ID);
-registerDirectory("XREUSysExt", userExtDir.parent);
-
-
-const ADDONS = {
-  test_bootstrap1_1: {
-    "install.rdf": {
-      id: "bootstrap1@tests.mozilla.org",
-
-      name: "Test Bootstrap 1",
-
-      iconURL: "chrome://foo/skin/icon.png",
-      aboutURL: "chrome://foo/content/about.xul",
-      optionsURL: "chrome://foo/content/options.xul",
-    },
-    "bootstrap.js": BOOTSTRAP_MONITOR_BOOTSTRAP_JS,
-  },
-  test_bootstrap1_2: {
-    "install.rdf": {
-      id: "bootstrap1@tests.mozilla.org",
-      version: "2.0",
-
-      name: "Test Bootstrap 1",
-    },
-    "bootstrap.js": BOOTSTRAP_MONITOR_BOOTSTRAP_JS,
-  },
-  test_bootstrap1_3: {
-    "install.rdf": {
-      id: "bootstrap1@tests.mozilla.org",
-      version: "3.0",
-
-      name: "Test Bootstrap 1",
-
-      targetApplications: [{
-        id: "undefined",
-        minVersion: "1",
-        maxVersion: "1"}],
-    },
-    "bootstrap.js": BOOTSTRAP_MONITOR_BOOTSTRAP_JS,
-  },
-  test_bootstrap2_1: {
-    "install.rdf": {
-      id: "bootstrap2@tests.mozilla.org",
-    },
-    "bootstrap.js": BOOTSTRAP_MONITOR_BOOTSTRAP_JS,
-  },
-};
-
-var testserver = AddonTestUtils.createHttpServer({hosts: ["example.com"]});
-
-const XPIS = {};
-for (let [name, addon] of Object.entries(ADDONS)) {
-  XPIS[name] = AddonTestUtils.createTempXPIFile(addon);
-  testserver.registerFile(`/addons/${name}.xpi`, XPIS[name]);
-}
-
-
-function getStartupReason() {
-  let info = BootstrapMonitor.started.get(ID1);
-  return info ? info.reason : undefined;
-}
-
-function getShutdownReason() {
-  let info = BootstrapMonitor.stopped.get(ID1);
-  return info ? info.reason : undefined;
-}
-
-function getInstallReason() {
-  let info = BootstrapMonitor.installed.get(ID1);
-  return info ? info.reason : undefined;
-}
-
-function getUninstallReason() {
-  let info = BootstrapMonitor.uninstalled.get(ID1);
-  return info ? info.reason : undefined;
-}
-
-function getStartupOldVersion() {
-  let info = BootstrapMonitor.started.get(ID1);
-  return info ? info.data.oldVersion : undefined;
-}
-
-function getShutdownNewVersion() {
-  let info = BootstrapMonitor.stopped.get(ID1);
-  return info ? info.data.newVersion : undefined;
-}
-
-function getInstallOldVersion() {
-  let info = BootstrapMonitor.installed.get(ID1);
-  return info ? info.data.oldVersion : undefined;
-}
-
-function getUninstallNewVersion() {
-  let info = BootstrapMonitor.uninstalled.get(ID1);
-  return info ? info.data.newVersion : undefined;
-}
-
-async function checkBootstrappedPref() {
-  let XPIScope = ChromeUtils.import("resource://gre/modules/addons/XPIProvider.jsm", {});
-
-  let data = new Map();
-  for (let entry of XPIScope.XPIStates.enabledAddons()) {
-    data.set(entry.id, entry);
-  }
-
-  let addons = await AddonManager.getAddonsByTypes(["extension"]);
-  for (let addon of addons) {
-    if (!addon.id.endsWith("@tests.mozilla.org"))
-      continue;
-    if (!addon.isActive)
-      continue;
-    if (addon.operationsRequiringRestart != AddonManager.OP_NEEDS_RESTART_NONE)
-      continue;
-
-    ok(data.has(addon.id));
-    let addonData = data.get(addon.id);
-    data.delete(addon.id);
-
-    equal(addonData.version, addon.version);
-    equal(addonData.type, addon.type);
-    let file = addon.getResourceURI().QueryInterface(Ci.nsIFileURL).file;
-    equal(addonData.path, file.path);
-  }
-  equal(data.size, 0);
-}
-
-
-add_task(async function run_test() {
-  promiseStartupManager();
-
-  ok(!gExtensionsJSON.exists());
-  ok(!gAddonStartup.exists());
-});
-
-// Tests that installing doesn't require a restart
-add_task(async function test_1() {
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_1);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  equal(install.type, "extension");
-  equal(install.version, "1.0");
-  equal(install.name, "Test Bootstrap 1");
-  equal(install.state, AddonManager.STATE_DOWNLOADED);
-  notEqual(install.addon.syncGUID, null);
-  equal(install.addon.operationsRequiringRestart &
-               AddonManager.OP_NEEDS_RESTART_INSTALL, 0);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  let addon = install.addon;
-
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    new Promise(resolve => {
-      prepare_test({
-        [ID1]: [
-          ["onInstalling", false],
-          "onInstalled",
-        ],
-      }, [
-        "onInstallStarted",
-        "onInstallEnded",
-      ], function() {
-        // startup should not have been called yet.
-        BootstrapMonitor.checkAddonNotStarted(ID1);
-        resolve();
-      });
-      install.install();
-    }),
-  ]);
-
-  await checkBootstrappedPref();
-  let installSyncGUID = addon.syncGUID;
-
-  let installs = await AddonManager.getAllInstalls();
-  // There should be no active installs now since the install completed and
-  // doesn't require a restart.
-  equal(installs.length, 0);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  notEqual(b1.syncGUID, null);
-  equal(b1.syncGUID, installSyncGUID);
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_INSTALL);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  let dir = do_get_addon_root_uri(profileDir, ID1);
-  equal(b1.getResourceURI("bootstrap.js").spec, dir + "bootstrap.js");
-});
-
-// Tests that disabling doesn't require a restart
-add_task(async function test_2() {
-  let b1 = await AddonManager.getAddonByID(ID1);
-  prepare_test({
-    [ID1]: [
-      ["onDisabling", false],
-      "onDisabled",
-    ],
-  });
-
-  equal(b1.operationsRequiringRestart &
-        AddonManager.OP_NEEDS_RESTART_DISABLE, 0);
-  await b1.disable();
-  ensure_test_completed();
-
-  await new Promise(executeSoon);
-
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(b1.userDisabled);
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), ADDON_DISABLE);
-  equal(getShutdownNewVersion(), undefined);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  let newb1 = await AddonManager.getAddonByID(ID1);
-  notEqual(newb1, null);
-  equal(newb1.version, "1.0");
-  ok(!newb1.appDisabled);
-  ok(newb1.userDisabled);
-  ok(!newb1.isActive);
-
-  await checkBootstrappedPref();
-});
-
-// Test that restarting doesn't accidentally re-enable
-add_task(async function test_3() {
-  await promiseShutdownManager();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), ADDON_DISABLE);
-  equal(getShutdownNewVersion(), undefined);
-
-  await promiseStartupManager();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), ADDON_DISABLE);
-  equal(getShutdownNewVersion(), undefined);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  ok(gAddonStartup.exists());
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(b1.userDisabled);
-  ok(!b1.isActive);
-
-  await checkBootstrappedPref();
-});
-
-// Tests that enabling doesn't require a restart
-add_task(async function test_4() {
-  let b1 = await AddonManager.getAddonByID(ID1);
-  prepare_test({
-    [ID1]: [
-      ["onEnabling", false],
-      "onEnabled",
-    ],
-  });
-
-  equal(b1.operationsRequiringRestart &
-               AddonManager.OP_NEEDS_RESTART_ENABLE, 0);
-  await b1.enable();
-  ensure_test_completed();
-
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_ENABLE);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  let newb1 = await AddonManager.getAddonByID(ID1);
-  notEqual(newb1, null);
-  equal(newb1.version, "1.0");
-  ok(!newb1.appDisabled);
-  ok(!newb1.userDisabled);
-  ok(newb1.isActive);
-
-  await checkBootstrappedPref();
-});
-
-// Tests that a restart shuts down and restarts the add-on
-add_task(async function test_5() {
-  await promiseShutdownManager();
-  // By the time we've shut down, the database must have been written
-  ok(gExtensionsJSON.exists());
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), APP_SHUTDOWN);
-  equal(getShutdownNewVersion(), undefined);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-  await promiseStartupManager();
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), APP_STARTUP);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  await checkBootstrappedPref();
-});
-
-// Tests that installing an upgrade doesn't require a restart
-add_task(async function test_6() {
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_2);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  equal(install.type, "extension");
-  equal(install.version, "2.0");
-  equal(install.name, "Test Bootstrap 1");
-  equal(install.state, AddonManager.STATE_DOWNLOADED);
-
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    new Promise(resolve => {
-      prepare_test({
-        [ID1]: [
-          ["onInstalling", false],
-          "onInstalled",
-        ],
-      }, [
-        "onInstallStarted",
-        "onInstallEnded",
-      ], resolve);
-      install.install();
-    }),
-  ]);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "2.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "2.0");
-  equal(getStartupReason(), ADDON_UPGRADE);
-  equal(getInstallOldVersion(), 1);
-  equal(getStartupOldVersion(), 1);
-  equal(getShutdownReason(), ADDON_UPGRADE);
-  equal(getShutdownNewVersion(), 2);
-  equal(getUninstallNewVersion(), 2);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-  do_check_in_crash_annotation(ID1, "2.0");
-
-  await checkBootstrappedPref();
-});
-
-// Tests that uninstalling doesn't require a restart
-add_task(async function test_7() {
-  let b1 = await AddonManager.getAddonByID(ID1);
-  prepare_test({
-    [ID1]: [
-      ["onUninstalling", false],
-      "onUninstalled",
-    ],
-  });
-
-  equal(b1.operationsRequiringRestart &
-        AddonManager.OP_NEEDS_RESTART_UNINSTALL, 0);
-  await b1.uninstall();
-
-  await checkBootstrappedPref();
-
-  ensure_test_completed();
-  BootstrapMonitor.checkAddonNotInstalled(ID1);
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), ADDON_UNINSTALL);
-  equal(getShutdownNewVersion(), undefined);
-  do_check_not_in_crash_annotation(ID1, "2.0");
-
-  b1 = await AddonManager.getAddonByID(ID1);
-  equal(b1, null);
-
-  await promiseRestartManager();
-
-  let newb1 = await AddonManager.getAddonByID(ID1);
-  equal(newb1, null);
-
-  await checkBootstrappedPref();
-});
-
-// Test that a bootstrapped extension dropped into the profile loads properly
-// on startup and doesn't cause an EM restart
-add_task(async function test_8() {
-  await promiseShutdownManager();
-
-  await manuallyInstall(XPIS.test_bootstrap1_1, profileDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_INSTALL);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  await checkBootstrappedPref();
-});
-
-// Test that items detected as removed during startup get removed properly
-add_task(async function test_9() {
-  await promiseShutdownManager();
-
-  manuallyUninstall(profileDir, ID1);
-  BootstrapMonitor.clear(ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  equal(b1, null);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  await checkBootstrappedPref();
-});
-
-
-// Tests that installing a downgrade sends the right reason
-add_task(async function test_10() {
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_2);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  equal(install.type, "extension");
-  equal(install.version, "2.0");
-  equal(install.name, "Test Bootstrap 1");
-  equal(install.state, AddonManager.STATE_DOWNLOADED);
-  do_check_not_in_crash_annotation(ID1, "2.0");
-
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    new Promise(resolve => {
-      prepare_test({
-        [ID1]: [
-          ["onInstalling", false],
-          "onInstalled",
-        ],
-      }, [
-        "onInstallStarted",
-        "onInstallEnded",
-      ], resolve);
-      install.install();
-    }),
-  ]);
-
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "2.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "2.0");
-  equal(getStartupReason(), ADDON_INSTALL);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "2.0");
-
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_1);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  equal(install.type, "extension");
-  equal(install.version, "1.0");
-  equal(install.name, "Test Bootstrap 1");
-  equal(install.state, AddonManager.STATE_DOWNLOADED);
-
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    new Promise(resolve => {
-      prepare_test({
-        [ID1]: [
-          ["onInstalling", false],
-          "onInstalled",
-        ],
-      }, [
-        "onInstallStarted",
-        "onInstallEnded",
-      ], resolve);
-      install.install();
-    }),
-  ]);
-
-  b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_DOWNGRADE);
-  equal(getInstallOldVersion(), 2);
-  equal(getStartupOldVersion(), 2);
-  equal(getShutdownReason(), ADDON_DOWNGRADE);
-  equal(getShutdownNewVersion(), 1);
-  equal(getUninstallNewVersion(), 1);
-  do_check_in_crash_annotation(ID1, "1.0");
-  do_check_not_in_crash_annotation(ID1, "2.0");
-
-  await checkBootstrappedPref();
-});
-
-// Tests that uninstalling a disabled add-on still calls the uninstall method
-add_task(async function test_11() {
-  let b1 = await AddonManager.getAddonByID(ID1);
-  prepare_test({
-    [ID1]: [
-      ["onDisabling", false],
-      "onDisabled",
-      ["onUninstalling", false],
-      "onUninstalled",
-    ],
-  });
-
-  await b1.disable();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  equal(getShutdownReason(), ADDON_DISABLE);
-  equal(getShutdownNewVersion(), undefined);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  await b1.uninstall();
-
-  ensure_test_completed();
-  BootstrapMonitor.checkAddonNotInstalled(ID1);
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  do_check_not_in_crash_annotation(ID1, "1.0");
-
-  await checkBootstrappedPref();
-});
-
-// Tests that bootstrapped extensions are correctly loaded even if the app is
-// upgraded at the same time
-add_task(async function test_12() {
-  await promiseShutdownManager();
-
-  await manuallyInstall(XPIS.test_bootstrap1_1, profileDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_INSTALL);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  await b1.uninstall();
-
-  await promiseRestartManager();
-  await checkBootstrappedPref();
-});
-
-
-// Tests that installing a bootstrapped extension with an invalid application
-// entry doesn't call it's startup method
-add_task(async function test_13() {
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_3);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  equal(install.type, "extension");
-  equal(install.version, "3.0");
-  equal(install.name, "Test Bootstrap 1");
-  equal(install.state, AddonManager.STATE_DOWNLOADED);
-  do_check_not_in_crash_annotation(ID1, "3.0");
-
-  await new Promise(resolve => {
-    prepare_test({
-      [ID1]: [
-        ["onInstalling", false],
-        "onInstalled",
-      ],
-    }, [
-      "onInstallStarted",
-      "onInstallEnded",
-    ], resolve);
-    install.install();
-  });
-
-  let installs = await AddonManager.getAllInstalls();
-
-  // There should be no active installs now since the install completed and
-  // doesn't require a restart.
-  equal(installs.length, 0);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "3.0");
-  ok(b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "3.0"); // We call install even for disabled add-ons
-  BootstrapMonitor.checkAddonNotStarted(ID1); // Should not have called startup though
-  do_check_not_in_crash_annotation(ID1, "3.0");
-
-  await promiseRestartManager();
-
-  b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "3.0");
-  ok(b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "3.0"); // We call install even for disabled add-ons
-  BootstrapMonitor.checkAddonNotStarted(ID1); // Should not have called startup though
-  do_check_not_in_crash_annotation(ID1, "3.0");
-
-  await checkBootstrappedPref();
-  await b1.uninstall();
-});
-
-// Tests that a bootstrapped extension with an invalid target application entry
-// does not get loaded when detected during startup
-add_task(async function test_14() {
-  await promiseRestartManager();
-
-  await promiseShutdownManager();
-
-  await manuallyInstall(XPIS.test_bootstrap1_3, profileDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "3.0");
-  ok(b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "3.0"); // We call install even for disabled add-ons
-  BootstrapMonitor.checkAddonNotStarted(ID1); // Should not have called startup though
-  do_check_not_in_crash_annotation(ID1, "3.0");
-
-  await checkBootstrappedPref();
-  await b1.uninstall();
-});
-
-// Tests that upgrading a disabled bootstrapped extension still calls uninstall
-// and install but doesn't startup the new version
-add_task(async function test_15() {
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    promiseInstallFile(XPIS.test_bootstrap1_1),
-  ]);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-
-  await b1.disable();
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let install = await AddonManager.getInstallForFile(XPIS.test_bootstrap1_2);
-  ensure_test_completed();
-
-  notEqual(install, null);
-  ok(install.addon.userDisabled);
-
-  await new Promise(resolve => {
-    prepare_test({
-      [ID1]: [
-        ["onInstalling", false],
-        "onInstalled",
-      ],
-    }, [
-      "onInstallStarted",
-      "onInstallEnded",
-    ], resolve);
-    install.install();
-  });
-
-  b1 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1, null);
-  equal(b1.version, "2.0");
-  ok(!b1.appDisabled);
-  ok(b1.userDisabled);
-  ok(!b1.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-
-  await checkBootstrappedPref();
-  await promiseRestartManager();
-
-  let b1_2 = await AddonManager.getAddonByID(ID1);
-  notEqual(b1_2, null);
-  equal(b1_2.version, "2.0");
-  ok(!b1_2.appDisabled);
-  ok(b1_2.userDisabled);
-  ok(!b1_2.isActive);
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-
-  await b1_2.uninstall();
-});
-
-// Tests that bootstrapped extensions don't get loaded when in safe mode
-add_task(async function test_16() {
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    promiseInstallFile(XPIS.test_bootstrap1_1),
-  ]);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  equal(b1.iconURL, "chrome://foo/skin/icon.png");
-  equal(b1.aboutURL, "chrome://foo/content/about.xul");
-  equal(b1.optionsURL, "chrome://foo/content/options.xul");
-
-  await promiseShutdownManager();
-
-  // Should have stopped
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-
-  gAppInfo.inSafeMode = true;
-  await promiseStartupManager();
-
-  let b1_2 = await AddonManager.getAddonByID(ID1);
-  // Should still be stopped
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  ok(!b1_2.isActive);
-  equal(b1_2.iconURL, null);
-  equal(b1_2.aboutURL, null);
-  equal(b1_2.optionsURL, null);
-
-  await promiseShutdownManager();
-  gAppInfo.inSafeMode = false;
-  await promiseStartupManager();
-
-  // Should have started
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-
-  let b1_3 = await AddonManager.getAddonByID(ID1);
-  await b1_3.uninstall();
-});
-
-// Check that a bootstrapped extension in a non-profile location is loaded
-add_task(async function test_17() {
-  await promiseShutdownManager();
-
-  await manuallyInstall(XPIS.test_bootstrap1_1, userExtDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  await checkBootstrappedPref();
-});
-
-// Check that installing a new bootstrapped extension in the profile replaces
-// the existing one
-add_task(async function test_18() {
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID1),
-    promiseInstallFile(XPIS.test_bootstrap1_2),
-  ]);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "2.0");
-  notEqual(b1, null);
-  equal(b1.version, "2.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  equal(getShutdownReason(), ADDON_UPGRADE);
-  equal(getUninstallReason(), ADDON_UPGRADE);
-  equal(getInstallReason(), ADDON_UPGRADE);
-  equal(getStartupReason(), ADDON_UPGRADE);
-
-  equal(getShutdownNewVersion(), 2);
-  equal(getUninstallNewVersion(), 2);
-  equal(getInstallOldVersion(), 1);
-  equal(getStartupOldVersion(), 1);
-
-  await checkBootstrappedPref();
-});
-
-// Check that uninstalling the profile version reveals the non-profile one
-add_task(async function test_19() {
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // The revealed add-on gets activated asynchronously
-  await new Promise(resolve => {
-    prepare_test({
-      [ID1]: [
-        ["onUninstalling", false],
-        "onUninstalled",
-        ["onInstalling", false],
-        "onInstalled",
-      ],
-    }, [], resolve);
-
-    b1.uninstall();
-  });
-
-  b1 = await AddonManager.getAddonByID(ID1);
-  // Should have reverted to the older version
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  equal(getShutdownReason(), ADDON_DOWNGRADE);
-  equal(getUninstallReason(), ADDON_DOWNGRADE);
-  equal(getInstallReason(), ADDON_DOWNGRADE);
-  equal(getStartupReason(), ADDON_DOWNGRADE);
-
-  equal(getShutdownNewVersion(), "1.0");
-  equal(getUninstallNewVersion(), "1.0");
-  equal(getInstallOldVersion(), "2.0");
-  equal(getStartupOldVersion(), "2.0");
-
-  await checkBootstrappedPref();
-});
-
-// Check that a new profile extension detected at startup replaces the non-profile
-// one
-add_task(async function test_20() {
-  await promiseShutdownManager();
-
-  await manuallyInstall(XPIS.test_bootstrap1_2, profileDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "2.0");
-  notEqual(b1, null);
-  equal(b1.version, "2.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  equal(getShutdownReason(), APP_SHUTDOWN);
-  equal(getUninstallReason(), ADDON_UPGRADE);
-  equal(getInstallReason(), ADDON_UPGRADE);
-  equal(getStartupReason(), APP_STARTUP);
-
-  equal(getShutdownNewVersion(), undefined);
-  equal(getUninstallNewVersion(), 2);
-  equal(getInstallOldVersion(), 1);
-  equal(getStartupOldVersion(), undefined);
-});
-
-// Check that a detected removal reveals the non-profile one
-add_task(async function test_21() {
-  await promiseShutdownManager();
-
-  equal(getShutdownReason(), APP_SHUTDOWN);
-  equal(getShutdownNewVersion(), undefined);
-
-  manuallyUninstall(profileDir, ID1);
-  BootstrapMonitor.clear(ID1);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  // This won't be set as the bootstrap script was gone so we couldn't
-  // uninstall it properly
-  equal(getUninstallReason(), undefined);
-  equal(getUninstallNewVersion(), undefined);
-
-  equal(getInstallReason(), ADDON_DOWNGRADE);
-  equal(getInstallOldVersion(), 2);
-
-  equal(getStartupReason(), APP_STARTUP);
-  equal(getStartupOldVersion(), undefined);
-
-  await checkBootstrappedPref();
-  await promiseShutdownManager();
-
-  manuallyUninstall(userExtDir, ID1);
-  BootstrapMonitor.clear(ID1);
-
-  await promiseStartupManager();
-});
-
-// Check that an upgrade from the filesystem is detected and applied correctly
-add_task(async function test_22() {
-  await promiseShutdownManager();
-
-  let file = await manuallyInstall(XPIS.test_bootstrap1_1, profileDir, ID1);
-  if (file.isDirectory())
-    file.append("install.rdf");
-
-  // Make it look old so changes are detected
-  setExtensionModifiedTime(file, file.lastModifiedTime - 5000);
-
-  await promiseStartupManager();
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-
-  await promiseShutdownManager();
-
-  equal(getShutdownReason(), APP_SHUTDOWN);
-  equal(getShutdownNewVersion(), undefined);
-
-  manuallyUninstall(profileDir, ID1);
-  BootstrapMonitor.clear(ID1);
-  await manuallyInstall(XPIS.test_bootstrap1_2, profileDir, ID1);
-
-  await promiseStartupManager();
-
-  let b1_2 = await AddonManager.getAddonByID(ID1);
-  // Should have installed and started
-  BootstrapMonitor.checkAddonInstalled(ID1, "2.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "2.0");
-  notEqual(b1_2, null);
-  equal(b1_2.version, "2.0");
-  ok(b1_2.isActive);
-  ok(!b1_2.isSystem);
-
-  // This won't be set as the bootstrap script was gone so we couldn't
-  // uninstall it properly
-  equal(getUninstallReason(), undefined);
-  equal(getUninstallNewVersion(), undefined);
-
-  equal(getInstallReason(), ADDON_UPGRADE);
-  equal(getInstallOldVersion(), 1);
-  equal(getStartupReason(), APP_STARTUP);
-  equal(getStartupOldVersion(), undefined);
-
-  await checkBootstrappedPref();
-  await b1_2.uninstall();
-});
-
-
-// Tests that installing from a URL doesn't require a restart
-add_task(async function test_23() {
-  prepare_test({}, [
-    "onNewInstall",
-  ]);
-
-  let url = "http://example.com/addons/test_bootstrap1_1.xpi";
-  let install = await AddonManager.getInstallForURL(url, "application/x-xpinstall");
-
-  ensure_test_completed();
-
-  notEqual(install, null);
-
-  await new Promise(resolve => {
-    prepare_test({}, [
-      "onDownloadStarted",
-      "onDownloadEnded",
-    ], function() {
-      equal(install.type, "extension");
-      equal(install.version, "1.0");
-      equal(install.name, "Test Bootstrap 1");
-      equal(install.state, AddonManager.STATE_DOWNLOADED);
-      equal(install.addon.operationsRequiringRestart &
-                   AddonManager.OP_NEEDS_RESTART_INSTALL, 0);
-      do_check_not_in_crash_annotation(ID1, "1.0");
-
-      prepare_test({
-        [ID1]: [
-          ["onInstalling", false],
-          "onInstalled",
-        ],
-      }, [
-        "onInstallStarted",
-        "onInstallEnded",
-      ], resolve);
-    });
-    install.install();
-  });
-
-  await checkBootstrappedPref();
-
-  let installs = await AddonManager.getAllInstalls();
-
-  // There should be no active installs now since the install completed and
-  // doesn't require a restart.
-  equal(installs.length, 0);
-
-  let b1 = await AddonManager.getAddonByID(ID1);
-
-  notEqual(b1, null);
-  equal(b1.version, "1.0");
-  ok(!b1.appDisabled);
-  ok(!b1.userDisabled);
-  ok(b1.isActive);
-  ok(!b1.isSystem);
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  equal(getStartupReason(), ADDON_INSTALL);
-  equal(getStartupOldVersion(), undefined);
-  do_check_in_crash_annotation(ID1, "1.0");
-
-  let dir = do_get_addon_root_uri(profileDir, ID1);
-  equal(b1.getResourceURI("bootstrap.js").spec, dir + "bootstrap.js");
-
-  await promiseRestartManager();
-
-  let b1_2 = await AddonManager.getAddonByID(ID1);
-  await b1_2.uninstall();
-});
-
-// Tests that we recover from a broken preference
-add_task(async function test_24() {
-  info("starting 24");
-
-  await Promise.all([
-    BootstrapMonitor.promiseAddonStartup(ID2),
-    promiseInstallAllFiles([XPIS.test_bootstrap1_1, XPIS.test_bootstrap2_1]),
-  ]);
-
-  info("test 24 got prefs");
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  BootstrapMonitor.checkAddonInstalled(ID2, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID2, "1.0");
-
-  await promiseRestartManager();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  BootstrapMonitor.checkAddonInstalled(ID2, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID2, "1.0");
-
-  await promiseShutdownManager();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID1);
-  BootstrapMonitor.checkAddonInstalled(ID2, "1.0");
-  BootstrapMonitor.checkAddonNotStarted(ID2);
-
-  // Break the JSON.
-  let data = aomStartup.readStartupData();
-  data["app-profile"].addons[ID1].path += "foo";
-
-  await OS.File.writeAtomic(gAddonStartup.path,
-                            new TextEncoder().encode(JSON.stringify(data)),
-                            {compression: "lz4"});
-
-  await promiseStartupManager();
-
-  BootstrapMonitor.checkAddonInstalled(ID1, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID1, "1.0");
-  BootstrapMonitor.checkAddonInstalled(ID2, "1.0");
-  BootstrapMonitor.checkAddonStarted(ID2, "1.0");
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_const.js b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_const.js
deleted file mode 100644
index 50d35baae97b..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_const.js
+++ /dev/null
@@ -1,27 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
-
-const ADDONS = {
-  test_bootstrap_const: {
-    "install.rdf": {
-      "id": "bootstrap@tests.mozilla.org",
-    },
-    "bootstrap.js": "ChromeUtils.import(\"resource://gre/modules/Services.jsm\");\n\nconst install = function() {\n  Services.obs.notifyObservers(null, \"addon-install\");\n};\n",
-  },
-};
-
-add_task(async function() {
-  await promiseStartupManager();
-
-  let sawInstall = false;
-  Services.obs.addObserver(function() {
-    sawInstall = true;
-  }, "addon-install");
-
-  await AddonTestUtils.promiseInstallXPI(ADDONS.test_bootstrap_const);
-
-  ok(sawInstall);
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_globals.js b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_globals.js
deleted file mode 100644
index d1658b4782db..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrap_globals.js
+++ /dev/null
@@ -1,66 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-// This verifies that bootstrap.js has the expected globals defined
-ChromeUtils.import("resource://gre/modules/Services.jsm");
-
-createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
-
-const ADDONS = {
-  bootstrap_globals: {
-    "install.rdf": {
-      "id": "bootstrap_globals@tests.mozilla.org",
-    },
-    "bootstrap.js": String.raw`ChromeUtils.import("resource://gre/modules/Services.jsm");
-
-var seenGlobals = new Set();
-var scope = this;
-function checkGlobal(name, type) {
-  if (scope[name] && typeof(scope[name]) == type)
-    seenGlobals.add(name);
-}
-
-var wrapped = {};
-Services.obs.notifyObservers({ wrappedJSObject: wrapped }, "bootstrap-request-globals");
-for (let [name, type] of wrapped.expectedGlobals) {
-  checkGlobal(name, type);
-}
-
-function startup(data, reason) {
-  Services.obs.notifyObservers({ wrappedJSObject: seenGlobals }, "bootstrap-seen-globals");
-}
-
-function install(data, reason) {}
-function shutdown(data, reason) {}
-function uninstall(data, reason) {}
-`,
-  },
-};
-
-
-const EXPECTED_GLOBALS = [
-  ["console", "object"],
-];
-
-async function run_test() {
-  do_test_pending();
-  await promiseStartupManager();
-  let sawGlobals = false;
-
-  Services.obs.addObserver(function(subject) {
-    subject.wrappedJSObject.expectedGlobals = EXPECTED_GLOBALS;
-  }, "bootstrap-request-globals");
-
-  Services.obs.addObserver(function({ wrappedJSObject: seenGlobals }) {
-    for (let [name ] of EXPECTED_GLOBALS)
-      Assert.ok(seenGlobals.has(name));
-
-    sawGlobals = true;
-  }, "bootstrap-seen-globals");
-
-  await AddonTestUtils.promiseInstallXPI(ADDONS.bootstrap_globals);
-  Assert.ok(sawGlobals);
-  await promiseShutdownManager();
-  do_test_finished();
-}
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrapped_chrome_manifest.js b/toolkit/mozapps/extensions/test/xpcshell/test_bootstrapped_chrome_manifest.js
deleted file mode 100644
index b0d46faad82a..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_bootstrapped_chrome_manifest.js
+++ /dev/null
@@ -1,50 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-const ADDON = {
-  "install.rdf": {
-    "id": "bug675371@tests.mozilla.org",
-  },
-  "chrome.manifest": `content bug675371 .`,
-  "test.js": `var active = true;`,
-};
-
-add_task(async function run_test() {
-  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
-  await promiseStartupManager();
-});
-
-function checkActive(expected) {
-  let target = { active: false };
-  let load = () => {
-    Services.scriptloader.loadSubScript("chrome://bug675371/content/test.js", target);
-  };
-
-  if (expected) {
-    load();
-  } else {
-    Assert.throws(load, /Error opening input stream/);
-  }
-  equal(target.active, expected, "Manifest is active?");
-}
-
-add_task(async function test() {
-  let {addon} = await AddonTestUtils.promiseInstallXPI(ADDON);
-
-  Assert.ok(addon.isActive);
-
-  // Tests that chrome.manifest is registered when the addon is installed.
-  checkActive(true);
-
-  await addon.disable();
-  checkActive(false);
-
-  await addon.enable();
-  checkActive(true);
-
-  await promiseShutdownManager();
-
-  // Tests that chrome.manifest remains registered at app shutdown.
-  checkActive(true);
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_invalid_install_rdf.js b/toolkit/mozapps/extensions/test/xpcshell/test_invalid_install_rdf.js
deleted file mode 100644
index 879ad5d0c7b3..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_invalid_install_rdf.js
+++ /dev/null
@@ -1,113 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-// Test that side-loaded extensions with invalid install.rdf files are
-// not initialized at startup.
-
-const APP_ID = "xpcshell@tests.mozilla.org";
-
-Services.prefs.setIntPref("extensions.enabledScopes", AddonManager.SCOPE_USER);
-
-createAppInfo(APP_ID, "XPCShell", "1", "1.9.2");
-
-const userAppDir = AddonTestUtils.profileDir.clone();
-userAppDir.append("app-extensions");
-userAppDir.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
-AddonTestUtils.registerDirectory("XREUSysExt", userAppDir);
-
-const userExtensions = userAppDir.clone();
-userExtensions.append(APP_ID);
-userExtensions.create(Ci.nsIFile.DIRECTORY_TYPE, FileUtils.PERMS_DIRECTORY);
-
-XPCOMUtils.defineLazyServiceGetters(this, {
-  ChromeRegistry: ["@mozilla.org/chrome/chrome-registry;1", "nsIChromeRegistry"],
-});
-
-function hasChromeEntry(package) {
-  try {
-    void ChromeRegistry.convertChromeURL(Services.io.newURI(`chrome://${package}/content/`));
-    return true;
-  } catch (e) {
-    return false;
-  }
-}
-
-add_task(async function() {
-  await promiseWriteInstallRDFToXPI({
-    id: "langpack-foo@addons.mozilla.org",
-    version: "1.0",
-    type: 8,
-    targetApplications: [{
-      id: "xpcshell@tests.mozilla.org",
-      minVersion: "1",
-      maxVersion: "1",
-    }],
-    name: "Invalid install.rdf extension",
-  }, userExtensions, undefined, {
-    "chrome.manifest": `
-      content foo-langpack ./
-    `,
-  });
-
-  await promiseWriteInstallRDFToXPI({
-    id: "foo@addons.mozilla.org",
-    version: "1.0",
-    bootstrap: true,
-    targetApplications: [{
-      id: "xpcshell@tests.mozilla.org",
-      minVersion: "1",
-      maxVersion: "1",
-    }],
-    name: "Invalid install.rdf extension",
-  }, userExtensions, undefined, {
-    "chrome.manifest": `
-      content foo ./
-    `,
-  });
-
-  await promiseWriteInstallRDFToXPI({
-    id: "foo-legacy-legacy@addons.mozilla.org",
-    version: "1.0",
-    bootstrap: false,
-    targetApplications: [{
-      id: "xpcshell@tests.mozilla.org",
-      minVersion: "1",
-      maxVersion: "1",
-    }],
-    name: "Invalid install.rdf extension",
-  }, userExtensions, undefined, {
-    "chrome.manifest": `
-      content foo-legacy-legacy ./
-    `,
-  });
-
-  equal(hasChromeEntry("foo-langpack"), false,
-        "Should not have registered foo-langpack resource before AOM startup");
-  equal(hasChromeEntry("foo-legacy-legacy"), false,
-        "Should not have registered foo-legacy-legacy resource before AOM startup");
-  equal(hasChromeEntry("foo"), false,
-        "Should not have registered foo resource before AOM startup");
-
-  await promiseStartupManager();
-
-  equal(hasChromeEntry("foo-langpack"), false,
-        "Should not have registered chrome manifest for invalid extension");
-  equal(hasChromeEntry("foo-legacy-legacy"), false,
-        "Should not have registered chrome manifest for non-restartless extension");
-  equal(hasChromeEntry("foo"), true,
-        "Should have registered chrome manifest for valid extension");
-
-  await promiseRestartManager();
-
-  equal(hasChromeEntry("foo-langpack"), false,
-        "Should still not have registered chrome manifest for invalid extension after restart");
-  equal(hasChromeEntry("foo-legacy-legacy"), false,
-        "Should still not have registered chrome manifest for non-restartless extension");
-  equal(hasChromeEntry("foo"), true,
-        "Should still have registered chrome manifest for valid extension after restart");
-
-  await promiseShutdownManager();
-
-  userAppDir.remove(true);
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_legacy.js b/toolkit/mozapps/extensions/test/xpcshell/test_legacy.js
deleted file mode 100644
index 270e18a1bd5d..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_legacy.js
+++ /dev/null
@@ -1,118 +0,0 @@
-
-const LEGACY_PREF = "extensions.legacy.enabled";
-
-createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1");
-
-add_task(async function test_disable() {
-  await promiseStartupManager();
-
-  let legacy = [
-    {
-      id: "bootstrap@tests.mozilla.org",
-      name: "Bootstrap add-on",
-      version: "1.0",
-      bootstrap: true,
-    },
-  ];
-
-  let nonLegacy = [
-    {
-      id: "webextension@tests.mozilla.org",
-      manifest: {
-        applications: {gecko: {id: "webextension@tests.mozilla.org"}},
-      },
-    },
-    {
-      id: "privileged@tests.mozilla.org",
-      name: "Privileged Bootstrap add-on",
-      version: "1.0",
-      bootstrap: true,
-    },
-    {
-      id: "dictionary@tests.mozilla.org",
-      name: "Test Dictionary",
-      version: "1.0",
-      type: "64",
-    },
-  ];
-
-  function makeXPI(info) {
-    if (info.manifest) {
-      return createTempWebExtensionFile(info);
-    }
-
-    return createTempXPIFile(Object.assign({}, info, {
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    }));
-  }
-
-  AddonTestUtils.usePrivilegedSignatures = id => id.startsWith("privileged");
-
-  // Start out with legacy extensions disabled, installing non-legacy
-  // extensions should succeed.
-  Services.prefs.setBoolPref(LEGACY_PREF, false);
-  let installs = await Promise.all(nonLegacy.map(info => {
-    let xpi = makeXPI(info);
-    return AddonManager.getInstallForFile(xpi);
-  }));
-  await promiseCompleteAllInstalls(installs);
-  for (let install of installs) {
-    Assert.equal(install.state, AddonManager.STATE_INSTALLED);
-    Assert.equal(install.error, 0);
-  }
-  let addons = await AddonManager.getAddonsByIDs(nonLegacy.map(a => a.id));
-  for (let addon of addons) {
-    Assert.equal(addon.appDisabled, false);
-  }
-
-  // And installing legacy extensions should fail
-  let legacyXPIs = legacy.map(makeXPI);
-  installs = await Promise.all(legacyXPIs.map(xpi => AddonManager.getInstallForFile(xpi)));
-
-  // Yuck, the AddonInstall API is atrocious.  Installs of incompatible
-  // extensions are detected when the install reaches the DOWNLOADED state
-  // and the install is abandoned at that point.  Since this is a local file
-  // install we just start out in the DOWNLOADED state.
-  for (let install of installs) {
-    Assert.equal(install.state, AddonManager.STATE_DOWNLOADED);
-    Assert.equal(install.addon.appDisabled, true);
-  }
-
-  // Now enable legacy extensions, and we should be able to install
-  // the legacy extensions.
-  Services.prefs.setBoolPref(LEGACY_PREF, true);
-  installs = await Promise.all(legacyXPIs.map(xpi => AddonManager.getInstallForFile(xpi)));
-  for (let install of installs) {
-    Assert.equal(install.state, AddonManager.STATE_DOWNLOADED);
-    Assert.equal(install.addon.appDisabled, false);
-  }
-  await promiseCompleteAllInstalls(installs);
-  for (let install of installs) {
-    Assert.equal(install.state, AddonManager.STATE_INSTALLED);
-    Assert.equal(install.error, 0);
-  }
-  addons = await AddonManager.getAddonsByIDs(legacy.map(a => a.id));
-  for (let addon of addons) {
-    Assert.equal(addon.appDisabled, false);
-  }
-
-  // Flip the preference back, the legacy extensions should become disabled
-  // but non-legacy extensions should remain enabled.
-  Services.prefs.setBoolPref(LEGACY_PREF, false);
-  addons = await AddonManager.getAddonsByIDs(nonLegacy.map(a => a.id));
-  for (let addon of addons) {
-    Assert.equal(addon.appDisabled, false);
-    await addon.uninstall();
-  }
-  addons = await AddonManager.getAddonsByIDs(legacy.map(a => a.id));
-  for (let addon of addons) {
-    Assert.equal(addon.appDisabled, true);
-    await addon.uninstall();
-  }
-
-  Services.prefs.clearUserPref(LEGACY_PREF);
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_manifest.js b/toolkit/mozapps/extensions/test/xpcshell/test_manifest.js
deleted file mode 100644
index 48aa26790d09..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_manifest.js
+++ /dev/null
@@ -1,752 +0,0 @@
-/* Any copyright is dedicated to the Public Domain.
- * http://creativecommons.org/publicdomain/zero/1.0/
- */
-
-// This tests that all properties are read from the install manifests and that
-// items are correctly enabled/disabled based on them (blocklist tests are
-// elsewhere)
-
-const ADDONS = [
-  {
-    "install.rdf": {
-      id: "addon1@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      aboutURL: "chrome://test/content/about.xul",
-      iconURL: "chrome://test/skin/icon.png",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 1",
-      description: "Test Description",
-      creator: "Test Creator",
-      homepageURL: "http://www.example.com",
-      developer: [
-        "Test Developer 1",
-        "Test Developer 2",
-      ],
-      translator: [
-        "Test Translator 1",
-        "Test Translator 2",
-      ],
-      contributor: [
-        "Test Contributor 1",
-        "Test Contributor 2",
-      ],
-    },
-
-    expected: {
-      id: "addon1@tests.mozilla.org",
-      type: "extension",
-      version: "1.0",
-      optionsType: null,
-      aboutURL: "chrome://test/content/about.xul",
-      iconURL: "chrome://test/skin/icon.png",
-      icons: {32: "chrome://test/skin/icon.png", 48: "chrome://test/skin/icon.png"},
-      name: "Test Addon 1",
-      description: "Test Description",
-      creator: "Test Creator",
-      homepageURL: "http://www.example.com",
-      developers: ["Test Developer 1", "Test Developer 2"],
-      translators: ["Test Translator 1", "Test Translator 2"],
-      contributors: ["Test Contributor 1", "Test Contributor 2"],
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      providesUpdatesSecurely: true,
-      blocklistState: Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon2@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      updateURL: "https://www.foo.com",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 2",
-    },
-
-    expected: {
-      id: "addon2@tests.mozilla.org",
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      providesUpdatesSecurely: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon3@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      updateURL: "http://www.foo.com",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 3",
-    },
-
-    expected: {
-      id: "addon3@tests.mozilla.org",
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      providesUpdatesSecurely: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon4@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      updateURL: "http://www.foo.com",
-      updateKey: "foo",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 4",
-    },
-
-    expected: {
-      id: "addon4@tests.mozilla.org",
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      providesUpdatesSecurely: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon5@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "*",
-      }],
-      name: "Test Addon 5",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon6@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "0",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 6",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon7@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "0",
-        maxVersion: "0",
-      }],
-      name: "Test Addon 7",
-    },
-
-    expected: {
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      isCompatible: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon8@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1.1",
-        maxVersion: "*",
-      }],
-      name: "Test Addon 8",
-    },
-
-    expected: {
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      isCompatible: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon9@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9.2",
-        maxVersion: "1.9.*",
-      }],
-      name: "Test Addon 9",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon10@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9.2.1",
-        maxVersion: "1.9.*",
-      }],
-      name: "Test Addon 10",
-    },
-
-    expected: {
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      isCompatible: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon11@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9",
-        maxVersion: "1.9.2",
-      }],
-      name: "Test Addon 11",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon12@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9",
-        maxVersion: "1.9.1.*",
-      }],
-      name: "Test Addon 12",
-    },
-
-    expected: {
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      isCompatible: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon13@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9",
-        maxVersion: "1.9.*",
-      }, {
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "0",
-        maxVersion: "0.5",
-      }],
-      name: "Test Addon 13",
-    },
-
-    expected: {
-      isActive: false,
-      userDisabled: false,
-      appDisabled: true,
-      isCompatible: false,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon14@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "toolkit@mozilla.org",
-        minVersion: "1.9",
-        maxVersion: "1.9.1",
-      }, {
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 14",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon15@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      updateKey: "foo",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 15",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      providesUpdatesSecurely: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon16@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      updateKey: "foo",
-      updateURL: "https://www.foo.com",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 16",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      providesUpdatesSecurely: true,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon17@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsURL: "chrome://test/content/options.xul",
-      optionsType: "2",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 17",
-    },
-
-    // An obsolete optionsType means the add-on isn't registered.
-    expected: null,
-  },
-
-  {
-    "install.rdf": {
-      id: "addon18@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 18",
-    },
-    extraFiles: {"options.xul": ""},
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      optionsURL: null,
-      optionsType: null,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon19@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "99",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 19",
-    },
-
-    expected: null,
-  },
-
-  {
-    "install.rdf": {
-      id: "addon20@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsURL: "chrome://test/content/options.xul",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 20",
-    },
-
-    // Even with a defined optionsURL optionsType is null by default.
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      optionsURL: "chrome://test/content/options.xul",
-      optionsType: null,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon21@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "3",
-      optionsURL: "chrome://test/content/options.xul",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 21",
-    },
-
-    expected: {
-      isActive: true,
-      userDisabled: false,
-      appDisabled: false,
-      isCompatible: true,
-      optionsURL: "chrome://test/content/options.xul",
-      optionsType: AddonManager.OPTIONS_TYPE_TAB,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon22@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "2",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 22",
-    },
-
-    // An obsolete optionsType means the add-on isn't registered.
-    expected: null,
-  },
-
-  {
-    "install.rdf": {
-      id: "addon23@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "2",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 23",
-    },
-    extraFiles: {"options.xul": ""},
-
-    // An obsolete optionsType means the add-on isn't registered.
-    expected: null,
-  },
-
-  {
-    "install.rdf": {
-      id: "addon24@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 24",
-    },
-    extraFiles: {"options.xul": ""},
-
-    expected: {
-      optionsType: null,
-      optionsURL: null,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon25@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "3",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 25",
-    },
-
-    expected: {
-      optionsType: null,
-      optionsURL: null,
-    },
-  },
-
-  {
-    "install.rdf": {
-      id: "addon26@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      optionsType: "4",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-      name: "Test Addon 26",
-    },
-    extraFiles: {"options.xul": ""},
-    expected: null,
-  },
-
-  // Tests compatibility based on target platforms.
-
-  // No targetPlatforms so should be compatible
-  {
-    "install.rdf": {
-      id: "tp-addon1@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      name: "Test 1",
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    },
-
-    expected: {
-      appDisabled: false,
-      isPlatformCompatible: true,
-      isActive: true,
-    },
-  },
-
-  // Matches the OS
-  {
-    "install.rdf": {
-      id: "tp-addon2@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      name: "Test 2",
-      targetPlatforms: [
-        "XPCShell",
-        "WINNT_x86",
-        "XPCShell",
-      ],
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    },
-
-    expected: {
-      appDisabled: false,
-      isPlatformCompatible: true,
-      isActive: true,
-    },
-  },
-
-  // Matches the OS and ABI
-  {
-    "install.rdf": {
-      id: "tp-addon3@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      name: "Test 3",
-      targetPlatforms: [
-        "WINNT",
-        "XPCShell_noarch-spidermonkey",
-      ],
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    },
-
-    expected: {
-      appDisabled: false,
-      isPlatformCompatible: true,
-      isActive: true,
-    },
-  },
-
-  // Doesn't match
-  {
-    "install.rdf": {
-      id: "tp-addon4@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      name: "Test 4",
-      targetPlatforms: [
-        "WINNT_noarch-spidermonkey",
-        "Darwin",
-        "WINNT_noarch-spidermonkey",
-      ],
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    },
-
-    expected: {
-      appDisabled: true,
-      isPlatformCompatible: false,
-      isActive: false,
-    },
-  },
-
-  // Matches the OS but since a different entry specifies ABI this doesn't match.
-  {
-    "install.rdf": {
-      id: "tp-addon5@tests.mozilla.org",
-      version: "1.0",
-      bootstrap: true,
-      name: "Test 5",
-      targetPlatforms: [
-        "XPCShell",
-        "XPCShell_foo",
-      ],
-      targetApplications: [{
-        id: "xpcshell@tests.mozilla.org",
-        minVersion: "1",
-        maxVersion: "1",
-      }],
-    },
-
-    expected: {
-      appDisabled: true,
-      isPlatformCompatible: false,
-      isActive: false,
-    },
-  },
-];
-
-const IDS = ADDONS.map(a => a["install.rdf"].id);
-
-add_task(async function setup() {
-  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
-  const profileDir = gProfD.clone();
-  profileDir.append("extensions");
-
-  for (let addon of ADDONS) {
-    await promiseWriteInstallRDFForExtension(addon["install.rdf"], profileDir, undefined, addon.extraFiles);
-  }
-});
-
-add_task(async function test_values() {
-  await promiseStartupManager();
-
-  let addons = await getAddons(IDS);
-
-  for (let addon of ADDONS) {
-    let {id} = addon["install.rdf"];
-    checkAddon(id, addons.get(id), addon.expected);
-  }
-
-  await promiseShutdownManager();
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/test_manifest_locales.js b/toolkit/mozapps/extensions/test/xpcshell/test_manifest_locales.js
deleted file mode 100644
index 8c366eda17a0..000000000000
--- a/toolkit/mozapps/extensions/test/xpcshell/test_manifest_locales.js
+++ /dev/null
@@ -1,134 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- */
-
-const ID = "bug397778@tests.mozilla.org";
-
-const ADDON = {
-  id: "bug397778@tests.mozilla.org",
-  version: "1.0",
-  name: "Fallback Name",
-  description: "Fallback Description",
-  bootstrap: true,
-
-  targetApplications: [{
-    id: "xpcshell@tests.mozilla.org",
-    minVersion: "1",
-    maxVersion: "1"}],
-
-  localized: [
-    {
-      locale: ["fr"],
-      name: "fr Name",
-      description: "fr Description",
-    },
-    {
-      locale: ["de-DE"],
-      name: "Deutsches W\u00f6rterbuch",
-    },
-    {
-      locale: ["es-ES"],
-      name: "es-ES Name",
-      description: "es-ES Description",
-    },
-    {
-      locale: ["zh-TW"],
-      name: "zh-TW Name",
-      description: "zh-TW Description",
-    },
-    {
-      locale: ["zh-CN"],
-      name: "zh-CN Name",
-      description: "zh-CN Description",
-    },
-    {
-      locale: ["en-GB"],
-      name: "en-GB Name",
-      description: "en-GB Description",
-    },
-    {
-      locale: ["en"],
-      name: "en Name",
-      description: "en Description",
-    },
-    {
-      locale: ["en-CA"],
-      name: "en-CA Name",
-      description: "en-CA Description",
-    },
-  ],
-};
-
-add_task(async function setup() {
-  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1");
-  Services.locale.requestedLocales = ["fr-FR"];
-
-  await promiseStartupManager();
-  await promiseInstallXPI(ADDON);
-});
-
-add_task(async function test_1() {
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  Assert.equal(addon.name, "fr Name");
-  Assert.equal(addon.description, "fr Description");
-
-  await addon.disable();
-  await promiseRestartManager();
-
-  let newAddon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(newAddon, null);
-  Assert.equal(newAddon.name, "fr Name");
-});
-
-add_task(async function test_2() {
-  // Change locale. The more specific de-DE is the best match
-  await restartWithLocales(["de"]);
-
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  Assert.equal(addon.name, "Deutsches W\u00f6rterbuch");
-  Assert.equal(addon.description, null);
-});
-
-add_task(async function test_3() {
-  // Change locale. Locale case should have no effect
-  await restartWithLocales(["DE-de"]);
-
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  Assert.equal(addon.name, "Deutsches W\u00f6rterbuch");
-  Assert.equal(addon.description, null);
-});
-
-add_task(async function test_4() {
-  // Change locale. es-ES should closely match
-  await restartWithLocales(["es-AR"]);
-
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  Assert.equal(addon.name, "es-ES Name");
-  Assert.equal(addon.description, "es-ES Description");
-});
-
-add_task(async function test_5() {
-  // Change locale. Either zh-CN or zh-TW could match
-  await restartWithLocales(["zh"]);
-
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  ok(addon.name == "zh-TW Name" || addon.name == "zh-CN Name",
-     `Add-on name mismatch: ${addon.name}`);
-});
-
-add_task(async function test_6() {
-  // Unknown locale should try to match against en-US as well. Of en,en-GB
-  // en should match as being less specific
-  await restartWithLocales(["nl-NL"]);
-
-  let addon = await AddonManager.getAddonByID(ID);
-  Assert.notEqual(addon, null);
-  Assert.equal(addon.name, "en Name");
-  Assert.equal(addon.description, "en Description");
-});
diff --git a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
index aa2c7b31a977..39606f4924b4 100644
--- a/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
+++ b/toolkit/mozapps/extensions/test/xpcshell/xpcshell.ini
@@ -63,10 +63,6 @@ tags = blocklist
 # Times out during parallel runs on desktop
 requesttimeoutfactor = 2
 tags = blocklist
-[test_bootstrap.js]
-[test_bootstrap_const.js]
-[test_bootstrap_globals.js]
-[test_bootstrapped_chrome_manifest.js]
 [test_cache_certdb.js]
 [test_cacheflush.js]
 [test_childprocess.js]
@@ -134,17 +130,10 @@ skip-if = appname != "firefox"
 [test_install_icons.js]
 # Bug 676992: test consistently hangs on Android
 skip-if = os == "android"
-[test_invalid_install_rdf.js]
 [test_isDebuggable.js]
 [test_isReady.js]
 [test_json_updatecheck.js]
-[test_legacy.js]
-skip-if = !allow_legacy_extensions || appname == "thunderbird"
 [test_locale.js]
-[test_manifest.js]
-[test_manifest_locales.js]
-# Bug 676992: test consistently hangs on Android
-skip-if = os == "android"
 [test_moved_extension_metadata.js]
 skip-if = true
 [test_no_addons.js]

From fb55ca75b7a3786fc69e688e7f0c60abd058b59b Mon Sep 17 00:00:00 2001
From: Diego Pino Garcia <dpino@igalia.com>
Date: Thu, 29 Nov 2018 17:07:30 -0500
Subject: [PATCH 74/78] Bug 1249606 - Automatically generate operator== for
 WebIDL dictionaries. r=bzbarsky

---
 dom/base/ChromeUtils.cpp           |  5 +----
 dom/bindings/BindingDeclarations.h |  5 +++++
 dom/bindings/Codegen.py            | 27 +++++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 4 deletions(-)

diff --git a/dom/base/ChromeUtils.cpp b/dom/base/ChromeUtils.cpp
index 381339ab4cea..d69abf666169 100644
--- a/dom/base/ChromeUtils.cpp
+++ b/dom/base/ChromeUtils.cpp
@@ -619,10 +619,7 @@ ChromeUtils::IsOriginAttributesEqual(dom::GlobalObject& aGlobal,
 ChromeUtils::IsOriginAttributesEqual(const dom::OriginAttributesDictionary& aA,
                                      const dom::OriginAttributesDictionary& aB)
 {
-  return aA.mAppId == aB.mAppId &&
-         aA.mInIsolatedMozBrowser == aB.mInIsolatedMozBrowser &&
-         aA.mUserContextId == aB.mUserContextId &&
-         aA.mPrivateBrowsingId == aB.mPrivateBrowsingId;
+  return aA == aB;
 }
 
 #ifdef NIGHTLY_BUILD
diff --git a/dom/bindings/BindingDeclarations.h b/dom/bindings/BindingDeclarations.h
index cd1226adbfe1..710fb1f9b2bc 100644
--- a/dom/bindings/BindingDeclarations.h
+++ b/dom/bindings/BindingDeclarations.h
@@ -163,6 +163,11 @@ public:
     return mImpl == aOther.mImpl;
   }
 
+  bool operator!=(const Optional_base<T, InternalType>& aOther) const
+  {
+    return mImpl != aOther.mImpl;
+  }
+
   template<typename T1, typename T2>
   explicit Optional_base(const T1& aValue1, const T2& aValue2)
   {
diff --git a/dom/bindings/Codegen.py b/dom/bindings/Codegen.py
index d5ddd3134bf8..b4b6d45dc485 100644
--- a/dom/bindings/Codegen.py
+++ b/dom/bindings/Codegen.py
@@ -12996,6 +12996,30 @@ class CGDictionary(CGThing):
                       "aOther")],
             body=body.define())
 
+    def canHaveEqualsOperator(self):
+        return all(m.type.isString() or m.type.isPrimitive() for (m,_) in
+                   self.memberInfo)
+
+    def equalsOperator(self):
+        body = CGList([])
+
+        for m, _ in self.memberInfo:
+            memberName = self.makeMemberName(m.identifier.name)
+            memberTest = CGGeneric(fill(
+                """
+                if (${memberName} != aOther.${memberName}) {
+                    return false;
+                }
+                """,
+                memberName=memberName))
+            body.append(memberTest)
+        body.append(CGGeneric("return true;\n"))
+        return ClassMethod(
+            "operator==", "bool",
+            [Argument("const %s&" % self.makeClassName(self.dictionary),
+                      "aOther")
+            ], const=True, body=body.define())
+
     def getStructs(self):
         d = self.dictionary
         selfName = self.makeClassName(d)
@@ -13076,6 +13100,9 @@ class CGDictionary(CGThing):
         else:
             disallowCopyConstruction = True
 
+        if self.canHaveEqualsOperator():
+            methods.append(self.equalsOperator())
+
         struct = CGClass(selfName,
                          bases=[ClassBase(self.base())],
                          members=members,

From 6fba94b2f046d7a9d1d8b9c2cd486393fa61519b Mon Sep 17 00:00:00 2001
From: Adam Holm <asorholm@email.arizona.edu>
Date: Wed, 28 Nov 2018 12:08:52 -0800
Subject: [PATCH 75/78] Bug 1451591 - Flush CacheIR Spewer on regular
 intervals. r=mgaudet

---
 js/src/jit/CacheIRSpewer.cpp | 14 ++++++++++++--
 js/src/jit/CacheIRSpewer.h   | 13 +++++++++++++
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/js/src/jit/CacheIRSpewer.cpp b/js/src/jit/CacheIRSpewer.cpp
index f4fd6aa784bf..3c5f82777ede 100644
--- a/js/src/jit/CacheIRSpewer.cpp
+++ b/js/src/jit/CacheIRSpewer.cpp
@@ -31,8 +31,18 @@ using namespace js::jit;
 CacheIRSpewer CacheIRSpewer::cacheIRspewer;
 
 CacheIRSpewer::CacheIRSpewer()
-  : outputLock(mutexid::CacheIRSpewer)
-{ }
+  : outputLock(mutexid::CacheIRSpewer),
+    guardCount_(0)
+{
+
+    spewInterval_ = getenv("CACHEIR_LOG_FLUSH") ?
+                        atoi(getenv("CACHEIR_LOG_FLUSH")) :
+                        10000;
+
+    if (spewInterval_ < 1) {
+        spewInterval_ = 1;
+    }
+}
 
 CacheIRSpewer::~CacheIRSpewer()
 {
diff --git a/js/src/jit/CacheIRSpewer.h b/js/src/jit/CacheIRSpewer.h
index edf4254ab324..f44bc2141284 100644
--- a/js/src/jit/CacheIRSpewer.h
+++ b/js/src/jit/CacheIRSpewer.h
@@ -27,6 +27,16 @@ class CacheIRSpewer
     mozilla::Maybe<JSONPrinter> json;
     static CacheIRSpewer cacheIRspewer;
 
+    // Counter to record how many times Guard class is called. This is used to
+    // determine when to flush outputs based on the given interval value.
+    // For example, if |spewInterval_ = 2|, outputs will be flushed on
+    // guardCount_ values 0,2,4,6,...
+    uint32_t guardCount_;
+
+    // Interval at which to flush output files. This value can be set with the
+    // environment variable |CACHEIR_LOG_FLUSH|.
+    uint32_t spewInterval_;
+
     CacheIRSpewer();
     ~CacheIRSpewer();
 
@@ -68,6 +78,9 @@ class CacheIRSpewer
               sp_.attached(name_);
             }
             sp_.endCache();
+            if (sp_.guardCount_++ % sp_.spewInterval_ == 0) {
+              sp_.output.flush();
+            }
             sp_.lock().unlock();
           }
         }

From a5f6c680809796f2df198ce095365be71ec7ae33 Mon Sep 17 00:00:00 2001
From: Lee Salzman <lsalzman@mozilla.com>
Date: Thu, 29 Nov 2018 18:09:46 -0500
Subject: [PATCH 76/78] Bug 1490537 - init and shutdown Skia in the GPU
 process. r=mattwoodrow

---
 gfx/2d/2D.h           |  2 +-
 gfx/2d/Factory.cpp    | 40 ++++++++--------------------------------
 gfx/ipc/GPUParent.cpp | 18 ++++++++++++++++++
 gfx/ipc/moz.build     |  2 ++
 4 files changed, 29 insertions(+), 33 deletions(-)

diff --git a/gfx/2d/2D.h b/gfx/2d/2D.h
index 11704fb6b4ed..6c7d873dd5e8 100644
--- a/gfx/2d/2D.h
+++ b/gfx/2d/2D.h
@@ -1865,7 +1865,7 @@ public:
 
 private:
   static FT_Library mFTLibrary;
-  static Mutex* mFTLock;
+  static StaticMutex mFTLock;
 public:
 #endif
 
diff --git a/gfx/2d/Factory.cpp b/gfx/2d/Factory.cpp
index 8696fbed008d..125e9fc4a8b7 100644
--- a/gfx/2d/Factory.cpp
+++ b/gfx/2d/Factory.cpp
@@ -46,7 +46,6 @@
 #include <d3d10_1.h>
 #include "HelpersD2D.h"
 #include "HelpersWinFonts.h"
-#include "mozilla/Mutex.h"
 #endif
 
 #include "DrawTargetCapture.h"
@@ -67,8 +66,6 @@
 #ifdef MOZ_ENABLE_FREETYPE
 #include "ft2build.h"
 #include FT_FREETYPE_H
-
-#include "mozilla/Mutex.h"
 #endif
 #include "MainThreadUtils.h"
 
@@ -219,7 +216,7 @@ int32_t LoggingPrefs::sGfxLogLevel = LOG_DEFAULT;
 
 #ifdef MOZ_ENABLE_FREETYPE
 FT_Library Factory::mFTLibrary = nullptr;
-Mutex* Factory::mFTLock = nullptr;
+StaticMutex Factory::mFTLock;
 #endif
 
 #ifdef WIN32
@@ -243,10 +240,6 @@ Factory::Init(const Config& aConfig)
 {
   MOZ_ASSERT(!sConfig);
   sConfig = new Config(aConfig);
-
-#ifdef MOZ_ENABLE_FREETYPE
-  mFTLock = new Mutex("Factory::mFTLock");
-#endif
 }
 
 void
@@ -260,10 +253,6 @@ Factory::ShutDown()
 
 #ifdef MOZ_ENABLE_FREETYPE
   mFTLibrary = nullptr;
-  if (mFTLock) {
-    delete mFTLock;
-    mFTLock = nullptr;
-  }
 #endif
 }
 
@@ -764,22 +753,19 @@ Factory::ReleaseFTLibrary(FT_Library aFTLibrary)
 void
 Factory::LockFTLibrary(FT_Library aFTLibrary)
 {
-  MOZ_ASSERT(mFTLock);
-  mFTLock->Lock();
+  mFTLock.Lock();
 }
 
 void
 Factory::UnlockFTLibrary(FT_Library aFTLibrary)
 {
-  MOZ_ASSERT(mFTLock);
-  mFTLock->Unlock();
+  mFTLock.Unlock();
 }
 
 FT_Face
 Factory::NewFTFace(FT_Library aFTLibrary, const char* aFileName, int aFaceIndex)
 {
-  MOZ_ASSERT(mFTLock);
-  MutexAutoLock lock(*mFTLock);
+  StaticMutexAutoLock lock(mFTLock);
   if (!aFTLibrary) {
     aFTLibrary = mFTLibrary;
   }
@@ -793,8 +779,7 @@ Factory::NewFTFace(FT_Library aFTLibrary, const char* aFileName, int aFaceIndex)
 FT_Face
 Factory::NewFTFaceFromData(FT_Library aFTLibrary, const uint8_t* aData, size_t aDataSize, int aFaceIndex)
 {
-  MOZ_ASSERT(mFTLock);
-  MutexAutoLock lock(*mFTLock);
+  StaticMutexAutoLock lock(mFTLock);
   if (!aFTLibrary) {
     aFTLibrary = mFTLibrary;
   }
@@ -808,23 +793,14 @@ Factory::NewFTFaceFromData(FT_Library aFTLibrary, const uint8_t* aData, size_t a
 void
 Factory::ReleaseFTFace(FT_Face aFace)
 {
-  // May be called during shutdown when the lock is already destroyed.
-  // However, there are no other threads using the face by this point,
-  // so it is safe to skip locking if the lock is not around.
-  if (mFTLock) {
-    mFTLock->Lock();
-  }
+  StaticMutexAutoLock lock(mFTLock);
   FT_Done_Face(aFace);
-  if (mFTLock) {
-    mFTLock->Unlock();
-  }
 }
 
 FT_Error
 Factory::LoadFTGlyph(FT_Face aFace, uint32_t aGlyphIndex, int32_t aFlags)
 {
-  MOZ_ASSERT(mFTLock);
-  MutexAutoLock lock(*mFTLock);
+  StaticMutexAutoLock lock(mFTLock);
   return FT_Load_Glyph(aFace, aGlyphIndex, aFlags);
 }
 #endif
@@ -1028,7 +1004,7 @@ Factory::CreateScaledFontForDWriteFont(IDWriteFontFace* aFontFace,
                                          aStyle);
 }
 
-#endif // XP_WIN
+#endif // WIN32
 
 #ifdef USE_SKIA_GPU
 already_AddRefed<DrawTarget>
diff --git a/gfx/ipc/GPUParent.cpp b/gfx/ipc/GPUParent.cpp
index 4452fea83fe2..6e6be98a54ac 100644
--- a/gfx/ipc/GPUParent.cpp
+++ b/gfx/ipc/GPUParent.cpp
@@ -38,6 +38,7 @@
 #include "mozilla/webrender/RenderThread.h"
 #include "mozilla/webrender/WebRenderAPI.h"
 #include "mozilla/HangDetails.h"
+#include "nscore.h"
 #include "nsDebugImpl.h"
 #include "nsIGfxInfo.h"
 #include "nsThreadManager.h"
@@ -47,6 +48,8 @@
 #include "VRManager.h"
 #include "VRManagerParent.h"
 #include "VsyncBridgeParent.h"
+#include "cairo.h"
+#include "skia/include/core/SkGraphics.h"
 #if defined(XP_WIN)
 # include "mozilla/gfx/DeviceManagerDx.h"
 # include <process.h>
@@ -54,6 +57,7 @@
 #endif
 #ifdef MOZ_WIDGET_GTK
 # include <gtk/gtk.h>
+# include "skia/include/ports/SkTypeface_cairo.h"
 #endif
 #ifdef MOZ_GECKO_PROFILER
 #include "ChildProfilerController.h"
@@ -220,6 +224,10 @@ GPUParent::RecvInit(nsTArray<GfxPrefSetting>&& prefs,
     LayerTreeOwnerTracker::Get()->Map(map.layersId(), map.ownerId());
   }
 
+  // We bypass gfxPlatform::Init, so we must initialize any relevant libraries
+  // here that would normally be initialized there.
+  SkGraphics::Init();
+
 #if defined(XP_WIN)
   if (gfxConfig::IsEnabled(Feature::D3D11_COMPOSITING)) {
     DeviceManagerDx::Get()->CreateCompositorDevices();
@@ -260,6 +268,8 @@ GPUParent::RecvInit(nsTArray<GfxPrefSetting>&& prefs,
     FT_Library library = Factory::NewFTLibrary();
     MOZ_ASSERT(library);
     Factory::SetFTLibrary(library);
+
+    SkInitCairoFT(true);
   }
 #endif
 
@@ -552,6 +562,14 @@ GPUParent::ActorDestroy(ActorDestroyReason aWhy)
 #endif
 
   Factory::ShutDown();
+
+  // We bypass gfxPlatform shutdown, so we must shutdown any libraries here
+  // that would normally be handled by it.
+#ifdef NS_FREE_PERMANENT_DATA
+  SkGraphics::PurgeFontCache();
+  cairo_debug_reset_static_data();
+#endif
+
 #if defined(XP_WIN)
   DeviceManagerDx::Shutdown();
 #endif
diff --git a/gfx/ipc/moz.build b/gfx/ipc/moz.build
index 2179010c6390..8bf2b2c97578 100644
--- a/gfx/ipc/moz.build
+++ b/gfx/ipc/moz.build
@@ -86,3 +86,5 @@ FINAL_LIBRARY = 'xul'
 
 CXXFLAGS += CONFIG['MOZ_CAIRO_CFLAGS']
 CXXFLAGS += CONFIG['TK_CFLAGS']
+
+LOCAL_INCLUDES += CONFIG['SKIA_INCLUDES']

From 48f9912c8b6307ac7b162c403977b32209b5654c Mon Sep 17 00:00:00 2001
From: Andrew Swan <aswan@mozilla.com>
Date: Thu, 29 Nov 2018 15:42:08 -0800
Subject: [PATCH 77/78] Bug 857456 Follow-up: temporarily disable addon test in
 testDistribution.java rs=nalexander

--HG--
extra : rebase_source : 42558b53fd64c91f130c091ed415dbf25e61f06b
extra : amend_source : 0ebf6a2af83981b7350d3ac18f32e1776ed10e4f
---
 .../robocop/src/org/mozilla/gecko/tests/testDistribution.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/mobile/android/tests/browser/robocop/src/org/mozilla/gecko/tests/testDistribution.java b/mobile/android/tests/browser/robocop/src/org/mozilla/gecko/tests/testDistribution.java
index 54d7aa6c7d31..0af012acc8f3 100644
--- a/mobile/android/tests/browser/robocop/src/org/mozilla/gecko/tests/testDistribution.java
+++ b/mobile/android/tests/browser/robocop/src/org/mozilla/gecko/tests/testDistribution.java
@@ -385,6 +385,7 @@ public class testDistribution extends ContentProviderTest {
     }
 
     private void checkAddon() {
+        /* Bug 1511211
         try {
             final String[] prefNames = { "distribution.test.addonEnabled" };
             final JSONArray preferences = getPrefs(prefNames);
@@ -393,6 +394,7 @@ public class testDistribution extends ContentProviderTest {
         } catch (JSONException e) {
             mAsserter.ok(false, "exception getting preferences", e.toString());
         }
+        */
     }
 
     private JSONArray getPrefs(String[] prefNames) throws JSONException {

From eb222629612f2fb4b9c7a2bf8526336b2e277db8 Mon Sep 17 00:00:00 2001
From: Gabriel Luong <gabriel.luong@gmail.com>
Date: Thu, 29 Nov 2018 18:45:43 -0500
Subject: [PATCH 78/78] Bug 1509907 - Add telemetry to track flexbox
 highlighter usage. r=miker

This also adds a telemetry count for the grid highlighter being turned on by the
markup view, which was not in place when we added the telemetry for the grid
highlighter.
---
 devtools/client/inspector/flexbox/flexbox.js  |  2 +-
 .../client/inspector/flexbox/test/browser.ini |  1 +
 ...er_flexbox_highlighter_opened_telemetry.js | 32 +++++++++++
 .../client/inspector/flexbox/test/head.js     |  1 +
 .../client/inspector/grids/test/browser.ini   |  1 +
 ...wser_grids_highlighter-toggle-telemetry.js | 46 +++++++++++++++
 .../client/inspector/markup/test/browser.ini  |  2 +
 ...ser_markup_flex_display_badge_telemetry.js | 48 ++++++++++++++++
 ...ser_markup_grid_display_badge_telemetry.js | 43 ++++++++++++++
 devtools/client/inspector/markup/test/head.js |  1 +
 .../inspector/markup/views/element-editor.js  |  2 +-
 .../client/inspector/rules/test/browser.ini   |  2 +
 .../browser_rules_flexbox-toggle-telemetry.js | 46 +++++++++++++++
 .../browser_rules_grid-toggle-telemetry.js    | 42 ++++++++++++++
 devtools/client/inspector/rules/test/head.js  |  1 +
 .../inspector/shared/highlighters-overlay.js  | 54 ++++++++++++------
 devtools/client/shared/telemetry.js           |  8 +--
 toolkit/components/telemetry/Histograms.json  | 11 ++++
 toolkit/components/telemetry/Scalars.yaml     | 56 +++++++++++++++++++
 19 files changed, 376 insertions(+), 23 deletions(-)
 create mode 100644 devtools/client/inspector/flexbox/test/browser_flexbox_highlighter_opened_telemetry.js
 create mode 100644 devtools/client/inspector/grids/test/browser_grids_highlighter-toggle-telemetry.js
 create mode 100644 devtools/client/inspector/markup/test/browser_markup_flex_display_badge_telemetry.js
 create mode 100644 devtools/client/inspector/markup/test/browser_markup_grid_display_badge_telemetry.js
 create mode 100644 devtools/client/inspector/rules/test/browser_rules_flexbox-toggle-telemetry.js
 create mode 100644 devtools/client/inspector/rules/test/browser_rules_grid-toggle-telemetry.js

diff --git a/devtools/client/inspector/flexbox/flexbox.js b/devtools/client/inspector/flexbox/flexbox.js
index 3bda46821f60..e0d234768747 100644
--- a/devtools/client/inspector/flexbox/flexbox.js
+++ b/devtools/client/inspector/flexbox/flexbox.js
@@ -401,7 +401,7 @@ class FlexboxInspector {
    *         highlighter is toggled on/off for.
    */
   onToggleFlexboxHighlighter(node) {
-    this.highlighters.toggleFlexboxHighlighter(node);
+    this.highlighters.toggleFlexboxHighlighter(node, "layout");
     this.store.dispatch(updateFlexboxHighlighted(node !==
       this.highlighters.flexboxHighlighterShow));
   }
diff --git a/devtools/client/inspector/flexbox/test/browser.ini b/devtools/client/inspector/flexbox/test/browser.ini
index b41325aa7b2c..f259c80b553e 100644
--- a/devtools/client/inspector/flexbox/test/browser.ini
+++ b/devtools/client/inspector/flexbox/test/browser.ini
@@ -26,6 +26,7 @@ support-files =
 [browser_flexbox_empty_state.js]
 [browser_flexbox_highlighter_color_picker_on_ESC.js]
 [browser_flexbox_highlighter_color_picker_on_RETURN.js]
+[browser_flexbox_highlighter_opened_telemetry.js]
 [browser_flexbox_item_list_01.js]
 [browser_flexbox_item_list_02.js]
 [browser_flexbox_item_list_updates_on_change.js]
diff --git a/devtools/client/inspector/flexbox/test/browser_flexbox_highlighter_opened_telemetry.js b/devtools/client/inspector/flexbox/test/browser_flexbox_highlighter_opened_telemetry.js
new file mode 100644
index 000000000000..552b8ec185ae
--- /dev/null
+++ b/devtools/client/inspector/flexbox/test/browser_flexbox_highlighter_opened_telemetry.js
@@ -0,0 +1,32 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry is correct when the flexbox highlighter is activated from
+// the layout view.
+
+const TEST_URI = URL_ROOT + "doc_flexbox_specific_cases.html";
+
+add_task(async function() {
+  await addTab(TEST_URI);
+  startTelemetry();
+  const { inspector, flexboxInspector } = await openLayoutView();
+  const { document: doc } = flexboxInspector;
+  const { highlighters, store } = inspector;
+
+  const onFlexHighlighterToggleRendered = waitForDOM(doc, "#flexbox-checkbox-toggle");
+  await selectNode("#container", inspector);
+  const [flexHighlighterToggle] = await onFlexHighlighterToggleRendered;
+
+  await toggleHighlighterON(flexHighlighterToggle, highlighters, store);
+  await toggleHighlighterOFF(flexHighlighterToggle, highlighters, store);
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.layout.flexboxhighlighter.opened", "", 1, "scalar");
+  checkTelemetry("DEVTOOLS_FLEXBOX_HIGHLIGHTER_TIME_ACTIVE_SECONDS", "", null,
+    "hasentries");
+}
diff --git a/devtools/client/inspector/flexbox/test/head.js b/devtools/client/inspector/flexbox/test/head.js
index abaed3161a93..5cb52d5b6274 100644
--- a/devtools/client/inspector/flexbox/test/head.js
+++ b/devtools/client/inspector/flexbox/test/head.js
@@ -2,6 +2,7 @@
 /* Any copyright is dedicated to the Public Domain.
  http://creativecommons.org/publicdomain/zero/1.0/ */
 /* eslint no-unused-vars: [2, {"vars": "local"}] */
+/* import-globals-from ../../../shared/test/telemetry-test-helpers.js */
 /* import-globals-from ../../test/head.js */
 "use strict";
 
diff --git a/devtools/client/inspector/grids/test/browser.ini b/devtools/client/inspector/grids/test/browser.ini
index 7f4b8a745cf9..c0015939362d 100644
--- a/devtools/client/inspector/grids/test/browser.ini
+++ b/devtools/client/inspector/grids/test/browser.ini
@@ -39,6 +39,7 @@ skip-if = (verify && (os == 'win'))
 [browser_grids_grid-outline-writing-mode.js]
 skip-if = (verify && (os == 'win'))
 [browser_grids_highlighter-setting-rules-grid-toggle.js]
+[browser_grids_highlighter-toggle-telemetry.js]
 [browser_grids_number-of-css-grids-telemetry.js]
 [browser_grids_persist-color-palette.js]
 [browser_grids_restored-after-reload.js]
diff --git a/devtools/client/inspector/grids/test/browser_grids_highlighter-toggle-telemetry.js b/devtools/client/inspector/grids/test/browser_grids_highlighter-toggle-telemetry.js
new file mode 100644
index 000000000000..38ab6a4511ff
--- /dev/null
+++ b/devtools/client/inspector/grids/test/browser_grids_highlighter-toggle-telemetry.js
@@ -0,0 +1,46 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry count is correct when the grid highlighter is activated from
+// the layout view.
+
+const TEST_URI = `
+  <style type='text/css'>
+    #grid {
+      display: grid;
+    }
+  </style>
+  <div id="grid">
+    <div id="cell1">cell1</div>
+    <div id="cell2">cell2</div>
+  </div>
+`;
+
+add_task(async function() {
+  await addTab("data:text/html;charset=utf-8," + encodeURIComponent(TEST_URI));
+  startTelemetry();
+  const { gridInspector, inspector } = await openLayoutView();
+  const { document: doc } = gridInspector;
+  const { highlighters, store } = inspector;
+
+  await selectNode("#grid", inspector);
+  const gridList = doc.getElementById("grid-list");
+  const checkbox = gridList.children[0].querySelector("input");
+
+  info("Toggling ON the CSS grid highlighter from the layout panel.");
+  const onHighlighterShown = highlighters.once("grid-highlighter-shown");
+  const onCheckboxChange = waitUntilState(store, state =>
+    state.grids.length == 1 &&
+    state.grids[0].highlighted);
+  checkbox.click();
+  await onHighlighterShown;
+  await onCheckboxChange;
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.grid.gridinspector.opened", "", 1, "scalar");
+}
diff --git a/devtools/client/inspector/markup/test/browser.ini b/devtools/client/inspector/markup/test/browser.ini
index e7b40095b4b1..d50d24c52a3f 100644
--- a/devtools/client/inspector/markup/test/browser.ini
+++ b/devtools/client/inspector/markup/test/browser.ini
@@ -135,8 +135,10 @@ skip-if = true # Bug 1177550
 [browser_markup_events_source_map.js]
 [browser_markup_events-windowed-host.js]
 [browser_markup_flex_display_badge.js]
+[browser_markup_flex_display_badge_telemetry.js]
 [browser_markup_grid_display_badge_01.js]
 [browser_markup_grid_display_badge_02.js]
+[browser_markup_grid_display_badge_telemetry.js]
 [browser_markup_links_01.js]
 [browser_markup_links_02.js]
 [browser_markup_links_03.js]
diff --git a/devtools/client/inspector/markup/test/browser_markup_flex_display_badge_telemetry.js b/devtools/client/inspector/markup/test/browser_markup_flex_display_badge_telemetry.js
new file mode 100644
index 000000000000..cdd9e9888a84
--- /dev/null
+++ b/devtools/client/inspector/markup/test/browser_markup_flex_display_badge_telemetry.js
@@ -0,0 +1,48 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry is correct when the flexbox highlighter is activated from
+// the markup view.
+
+const TEST_URI = `
+  <style type="text/css">
+    #flex {
+      display: flex;
+    }
+  </style>
+  <div id="flex"></div>
+`;
+
+add_task(async function() {
+  await pushPref("devtools.inspector.flexboxHighlighter.enabled", true);
+  await pushPref("devtools.flexboxinspector.enabled", true);
+  await addTab("data:text/html;charset=utf-8," + encodeURIComponent(TEST_URI));
+  startTelemetry();
+  const { inspector } = await openLayoutView();
+  const { highlighters } = inspector;
+
+  await selectNode("#flex", inspector);
+  const flexContainer = await getContainerForSelector("#flex", inspector);
+  const flexDisplayBadge = flexContainer.elt.querySelector(
+    ".inspector-badge.interactive[data-display]");
+
+  info("Toggling ON the flexbox highlighter from the flex display badge.");
+  const onHighlighterShown = highlighters.once("flexbox-highlighter-shown");
+  flexDisplayBadge.click();
+  await onHighlighterShown;
+
+  info("Toggling OFF the flexbox highlighter from the flex display badge.");
+  const onHighlighterHidden = highlighters.once("flexbox-highlighter-hidden");
+  flexDisplayBadge.click();
+  await onHighlighterHidden;
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.markup.flexboxhighlighter.opened", "", 1, "scalar");
+  checkTelemetry("DEVTOOLS_FLEXBOX_HIGHLIGHTER_TIME_ACTIVE_SECONDS", "", null,
+    "hasentries");
+}
diff --git a/devtools/client/inspector/markup/test/browser_markup_grid_display_badge_telemetry.js b/devtools/client/inspector/markup/test/browser_markup_grid_display_badge_telemetry.js
new file mode 100644
index 000000000000..1b26b7980350
--- /dev/null
+++ b/devtools/client/inspector/markup/test/browser_markup_grid_display_badge_telemetry.js
@@ -0,0 +1,43 @@
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry count is correct when the grid highlighter is activated from
+// the markup view.
+
+const TEST_URI = `
+  <style type="text/css">
+    #grid {
+      display: grid;
+    }
+  </style>
+  <div id="grid"></div>
+`;
+
+add_task(async function() {
+  await addTab("data:text/html;charset=utf-8," + encodeURIComponent(TEST_URI));
+  startTelemetry();
+  const { inspector } = await openLayoutView();
+  const { highlighters, store } = inspector;
+
+  await selectNode("#grid", inspector);
+  const gridContainer = await getContainerForSelector("#grid", inspector);
+  const gridDisplayBadge = gridContainer.elt.querySelector(
+    ".inspector-badge.interactive[data-display]");
+
+  info("Toggling ON the CSS grid highlighter from the grid display badge.");
+  const onHighlighterShown = highlighters.once("grid-highlighter-shown");
+  const onCheckboxChange = waitUntilState(store, state =>
+    state.grids.length === 1 &&
+    state.grids[0].highlighted);
+  gridDisplayBadge.click();
+  await onHighlighterShown;
+  await onCheckboxChange;
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.markup.gridinspector.opened", "", 1, "scalar");
+}
diff --git a/devtools/client/inspector/markup/test/head.js b/devtools/client/inspector/markup/test/head.js
index 9be070fda20b..6ad666668e98 100644
--- a/devtools/client/inspector/markup/test/head.js
+++ b/devtools/client/inspector/markup/test/head.js
@@ -2,6 +2,7 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 /* eslint no-unused-vars: [2, {"vars": "local"}] */
+/* import-globals-from ../../../shared/test/telemetry-test-helpers.js */
 /* import-globals-from ../../test/head.js */
 "use strict";
 
diff --git a/devtools/client/inspector/markup/views/element-editor.js b/devtools/client/inspector/markup/views/element-editor.js
index cc5c5bc92c82..5336a3f66e0a 100644
--- a/devtools/client/inspector/markup/views/element-editor.js
+++ b/devtools/client/inspector/markup/views/element-editor.js
@@ -749,7 +749,7 @@ ElementEditor.prototype = {
       this.stopTrackingFlexboxHighlighterEvents();
 
       this._displayBadge.classList.toggle("active");
-      await this.highlighters.toggleFlexboxHighlighter(this.node);
+      await this.highlighters.toggleFlexboxHighlighter(this.node, "markup");
 
       this.startTrackingFlexboxHighlighterEvents();
     }
diff --git a/devtools/client/inspector/rules/test/browser.ini b/devtools/client/inspector/rules/test/browser.ini
index a203ffa2b2c1..02650f979f14 100644
--- a/devtools/client/inspector/rules/test/browser.ini
+++ b/devtools/client/inspector/rules/test/browser.ini
@@ -166,6 +166,7 @@ skip-if = (os == "win" && debug) # bug 963492: win.
 [browser_rules_flexbox-highlighter-on-navigate.js]
 [browser_rules_flexbox-highlighter-on-reload.js]
 [browser_rules_flexbox-highlighter-restored-after-reload.js]
+[browser_rules_flexbox-toggle-telemetry.js]
 [browser_rules_flexbox-toggle_01.js]
 [browser_rules_flexbox-toggle_01b.js]
 [browser_rules_flexbox-toggle_02.js]
@@ -176,6 +177,7 @@ skip-if = (os == "win" && debug) # bug 963492: win.
 [browser_rules_grid-highlighter-on-navigate.js]
 [browser_rules_grid-highlighter-on-reload.js]
 [browser_rules_grid-highlighter-restored-after-reload.js]
+[browser_rules_grid-toggle-telemetry.js]
 [browser_rules_grid-toggle_01.js]
 [browser_rules_grid-toggle_01b.js]
 [browser_rules_grid-toggle_02.js]
diff --git a/devtools/client/inspector/rules/test/browser_rules_flexbox-toggle-telemetry.js b/devtools/client/inspector/rules/test/browser_rules_flexbox-toggle-telemetry.js
new file mode 100644
index 000000000000..90a130b29d13
--- /dev/null
+++ b/devtools/client/inspector/rules/test/browser_rules_flexbox-toggle-telemetry.js
@@ -0,0 +1,46 @@
+/* vim: set ft=javascript ts=2 et sw=2 tw=80: */
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry is correct when the flexbox highlighter is activated from
+// the rules view.
+
+const TEST_URI = `
+  <style type='text/css'>
+    #flex {
+      display: flex;
+    }
+  </style>
+  <div id="flex"></div>
+`;
+
+add_task(async function() {
+  await addTab("data:text/html;charset=utf-8," + encodeURIComponent(TEST_URI));
+  startTelemetry();
+  const {inspector, view} = await openRuleView();
+  const {highlighters} = view;
+
+  await selectNode("#flex", inspector);
+  const container = getRuleViewProperty(view, "#flex", "display").valueSpan;
+  const flexboxToggle = container.querySelector(".ruleview-flex");
+
+  info("Toggling ON the flexbox highlighter from the rule-view.");
+  const onHighlighterShown = highlighters.once("flexbox-highlighter-shown");
+  flexboxToggle.click();
+  await onHighlighterShown;
+
+  info("Toggling OFF the flexbox highlighter from the rule-view.");
+  const onHighlighterHidden = highlighters.once("flexbox-highlighter-hidden");
+  flexboxToggle.click();
+  await onHighlighterHidden;
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.rules.flexboxhighlighter.opened", "", 1, "scalar");
+  checkTelemetry("DEVTOOLS_FLEXBOX_HIGHLIGHTER_TIME_ACTIVE_SECONDS", "", null,
+    "hasentries");
+}
diff --git a/devtools/client/inspector/rules/test/browser_rules_grid-toggle-telemetry.js b/devtools/client/inspector/rules/test/browser_rules_grid-toggle-telemetry.js
new file mode 100644
index 000000000000..0ab5e071a224
--- /dev/null
+++ b/devtools/client/inspector/rules/test/browser_rules_grid-toggle-telemetry.js
@@ -0,0 +1,42 @@
+/* vim: set ft=javascript ts=2 et sw=2 tw=80: */
+/* Any copyright is dedicated to the Public Domain.
+ http://creativecommons.org/publicdomain/zero/1.0/ */
+
+"use strict";
+
+// Test that the telemetry count is correct when the grid highlighter is activated from
+// the rules view.
+
+const TEST_URI = `
+  <style type='text/css'>
+    #grid {
+      display: grid;
+    }
+  </style>
+  <div id="grid">
+    <div id="cell1">cell1</div>
+    <div id="cell2">cell2</div>
+  </div>
+`;
+
+add_task(async function() {
+  await addTab("data:text/html;charset=utf-8," + encodeURIComponent(TEST_URI));
+  startTelemetry();
+  const {inspector, view} = await openRuleView();
+  const highlighters = view.highlighters;
+
+  await selectNode("#grid", inspector);
+  const container = getRuleViewProperty(view, "#grid", "display").valueSpan;
+  const gridToggle = container.querySelector(".ruleview-grid");
+
+  info("Toggling ON the CSS grid highlighter from the rule-view.");
+  const onHighlighterShown = highlighters.once("grid-highlighter-shown");
+  gridToggle.click();
+  await onHighlighterShown;
+
+  checkResults();
+});
+
+function checkResults() {
+  checkTelemetry("devtools.rules.gridinspector.opened", "", 1, "scalar");
+}
diff --git a/devtools/client/inspector/rules/test/head.js b/devtools/client/inspector/rules/test/head.js
index d86003cc5d01..6ad81ab7f6a3 100644
--- a/devtools/client/inspector/rules/test/head.js
+++ b/devtools/client/inspector/rules/test/head.js
@@ -2,6 +2,7 @@
 /* Any copyright is dedicated to the Public Domain.
  http://creativecommons.org/publicdomain/zero/1.0/ */
 /* eslint no-unused-vars: [2, {"vars": "local"}] */
+/* import-globals-from ../../../shared/test/telemetry-test-helpers.js */
 /* import-globals-from ../../test/head.js */
 "use strict";
 
diff --git a/devtools/client/inspector/shared/highlighters-overlay.js b/devtools/client/inspector/shared/highlighters-overlay.js
index d4f2730cb022..7b63c5da3f9d 100644
--- a/devtools/client/inspector/shared/highlighters-overlay.js
+++ b/devtools/client/inspector/shared/highlighters-overlay.js
@@ -258,16 +258,18 @@ class HighlightersOverlay {
    *
    * @param  {NodeFront} node
    *         The NodeFront of the flexbox container element to highlight.
-   * @param  {Object} options
-   *         Object used for passing options to the flexbox highlighter.
+   * @param. {String} trigger
+   *         String name matching "layout", "markup" or "rule" to indicate where the
+   *         flexbox highlighter was toggled on from. "layout" represents the layout view.
+   *         "markup" represents the markup view. "rule" represents the rule view.
    */
-  async toggleFlexboxHighlighter(node, options = {}) {
+  async toggleFlexboxHighlighter(node, trigger) {
     if (node == this.flexboxHighlighterShown) {
       await this.hideFlexboxHighlighter(node);
       return;
     }
 
-    await this.showFlexboxHighlighter(node, options);
+    await this.showFlexboxHighlighter(node, {}, trigger);
   }
 
   /**
@@ -277,8 +279,12 @@ class HighlightersOverlay {
    *         The NodeFront of the flexbox container element to highlight.
    * @param  {Object} options
    *         Object used for passing options to the flexbox highlighter.
+   * @param. {String} trigger
+   *         String name matching "layout", "markup" or "rule" to indicate where the
+   *         flexbox highlighter was toggled on from. "layout" represents the layout view.
+   *         "markup" represents the markup view. "rule" represents the rule view.
    */
-  async showFlexboxHighlighter(node, options) {
+  async showFlexboxHighlighter(node, options, trigger) {
     const highlighter = await this._getHighlighter("FlexboxHighlighter");
     if (!highlighter) {
       return;
@@ -293,6 +299,17 @@ class HighlightersOverlay {
 
     this._toggleRuleViewIcon(node, true, ".ruleview-flex");
 
+    this.telemetry.toolOpened("flexbox_highlighter", this.inspector.toolbox.sessionId,
+      this);
+
+    if (trigger === "layout") {
+      this.telemetry.scalarAdd("devtools.layout.flexboxhighlighter.opened", 1);
+    } else if (trigger === "markup") {
+      this.telemetry.scalarAdd("devtools.markup.flexboxhighlighter.opened", 1);
+    } else if (trigger === "rule") {
+      this.telemetry.scalarAdd("devtools.rules.flexboxhighlighter.opened", 1);
+    }
+
     try {
       // Save flexbox highlighter state.
       const { url } = this.inspector.target;
@@ -319,6 +336,9 @@ class HighlightersOverlay {
       return;
     }
 
+    this.telemetry.toolClosed("flexbox_highlighter", this.inspector.toolbox.sessionId,
+      this);
+
     this._toggleRuleViewIcon(node, false, ".ruleview-flex");
 
     await this.highlighters.FlexboxHighlighter.hide();
@@ -407,10 +427,10 @@ class HighlightersOverlay {
    *
    * @param  {NodeFront} node
    *         The NodeFront of the grid container element to highlight.
-   * @param. {String|null} trigger
-   *         String name matching "grid" or "rule" to indicate where the
-   *         grid highlighter was toggled on from. "grid" represents the grid view
-   *         "rule" represents the rule view.
+   * @param. {String} trigger
+   *         String name matching "grid", "markup" or "rule" to indicate where the
+   *         grid highlighter was toggled on from. "grid" represents the grid view.
+   *         "markup" represents the markup view. "rule" represents the rule view.
    */
   async toggleGridHighlighter(node, trigger) {
     if (this.gridHighlighters.has(node)) {
@@ -428,10 +448,10 @@ class HighlightersOverlay {
    *         The NodeFront of the grid container element to highlight.
    * @param  {Object} options
    *         Object used for passing options to the grid highlighter.
-   * @param. {String|null} trigger
-   *         String name matching "grid" or "rule" to indicate where the
-   *         grid highlighter was toggled on from. "grid" represents the grid view
-   *         "rule" represents the rule view.
+   * @param. {String} trigger
+   *         String name matching "grid", "markup" or "rule" to indicate where the
+   *         grid highlighter was toggled on from. "grid" represents the grid view.
+   *         "markup" represents the markup view. "rule" represents the rule view.
    */
   async showGridHighlighter(node, options, trigger) {
     // When the grid highlighter has the given node, it is probably called with new
@@ -464,9 +484,11 @@ class HighlightersOverlay {
 
     this._toggleRuleViewIcon(node, true, ".ruleview-grid");
 
-    if (trigger == "grid") {
+    if (trigger === "grid") {
       this.telemetry.scalarAdd("devtools.grid.gridinspector.opened", 1);
-    } else if (trigger == "rule") {
+    } else if (trigger === "markup") {
+      this.telemetry.scalarAdd("devtools.markup.gridinspector.opened", 1);
+    } else if (trigger === "rule") {
       this.telemetry.scalarAdd("devtools.rules.gridinspector.opened", 1);
     }
 
@@ -968,7 +990,7 @@ class HighlightersOverlay {
 
     if (this._isRuleViewDisplayFlex(event.target)) {
       event.stopPropagation();
-      this.toggleFlexboxHighlighter(this.inspector.selection.nodeFront);
+      this.toggleFlexboxHighlighter(this.inspector.selection.nodeFront, "rule");
     }
 
     if (this._isRuleViewShapeSwatch(event.target)) {
diff --git a/devtools/client/shared/telemetry.js b/devtools/client/shared/telemetry.js
index 0e0fa863f1b9..8f3f330a6366 100644
--- a/devtools/client/shared/telemetry.js
+++ b/devtools/client/shared/telemetry.js
@@ -732,16 +732,14 @@ function getChartsFromToolId(id) {
       timerHist = `DEVTOOLS_${id}_TIME_ACTIVE_SECONDS`;
       countHist = `DEVTOOLS_${id}_OPENED_COUNT`;
       break;
+    case "FLEXBOX_HIGHLIGHTER":
+      timerHist = `DEVTOOLS_${id}_TIME_ACTIVE_SECONDS`;
+      break;
     default:
       timerHist = `DEVTOOLS_CUSTOM_TIME_ACTIVE_SECONDS`;
       countHist = `DEVTOOLS_CUSTOM_OPENED_COUNT`;
   }
 
-  if (!timerHist || (!countHist && !countScalar)) {
-    throw new Error(`getChartsFromToolId cannot be called without a timer ` +
-                    `histogram and either a count histogram or count scalar.`);
-  }
-
   return {
     useTimedEvent: useTimedEvent,
     timerHist: timerHist,
diff --git a/toolkit/components/telemetry/Histograms.json b/toolkit/components/telemetry/Histograms.json
index 0d3b6e3817fc..7f033a739677 100644
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -9866,6 +9866,17 @@
     "releaseChannelCollection": "opt-out",
     "description": "How long has the application panel been active (seconds)."
   },
+  "DEVTOOLS_FLEXBOX_HIGHLIGHTER_TIME_ACTIVE_SECONDS": {
+    "record_in_processes": ["main", "content"],
+    "expires_in_version": "never",
+    "kind": "exponential",
+    "high": 86400,
+    "n_buckets": 100,
+    "bug_numbers": [1509907],
+    "alert_emails": ["dev-developer-tools@lists.mozilla.org", "mbalfanz@mozilla.com"],
+    "releaseChannelCollection": "opt-out",
+    "description": "How long has the flexbox highlighter been active (seconds)."
+  },
   "DEVTOOLS_TOOLBOX_TIME_ACTIVE_SECONDS": {
     "record_in_processes": ["main", "content"],
     "expires_in_version": "never",
diff --git a/toolkit/components/telemetry/Scalars.yaml b/toolkit/components/telemetry/Scalars.yaml
index 9999df3f554c..2d041639caf6 100644
--- a/toolkit/components/telemetry/Scalars.yaml
+++ b/toolkit/components/telemetry/Scalars.yaml
@@ -1038,6 +1038,62 @@ devtools.copy.xpath:
     record_in_processes:
       - 'main'
 
+devtools.layout.flexboxhighlighter:
+  opened:
+    bug_numbers:
+      - 1509907
+    description: >
+      Number of times the DevTools flexbox highlighter was activated from the layout view.
+    expires: never
+    kind: uint
+    notification_emails:
+      - dev-developer-tools@lists.mozilla.org
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+devtools.markup.flexboxhighlighter:
+  opened:
+    bug_numbers:
+      - 1509907
+    description: >
+      Number of times the DevTools flexbox highlighter was activated from the markup view.
+    expires: never
+    kind: uint
+    notification_emails:
+      - dev-developer-tools@lists.mozilla.org
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+devtools.rules.flexboxhighlighter:
+  opened:
+    bug_numbers:
+      - 1509907
+    description: >
+      Number of times the DevTools flexbox highlighter was activated from the rules view.
+    expires: never
+    kind: uint
+    notification_emails:
+      - dev-developer-tools@lists.mozilla.org
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
+devtools.markup.gridinspector:
+  opened:
+    bug_numbers:
+      - 1509907
+    description: >
+      Number of times the DevTools grid inspector was opened from the markup view.
+    expires: never
+    kind: uint
+    notification_emails:
+      - dev-developer-tools@lists.mozilla.org
+    release_channel_collection: opt-out
+    record_in_processes:
+      - 'main'
+
 devtools.rules.gridinspector:
   opened:
     bug_numbers: