Bug 1503072 - Navigation fault interception test support. r=dom-worker-reviewers,edenchuang

Depends on D111844

Differential Revision: https://phabricator.services.mozilla.com/D111845

dom/interfaces/base/nsIServiceWorkerManager.idl

@@ -67,6 +67,18 @@ interface nsIServiceWorkerInfo : nsISupports
   readonly attribute PRTime activatedTime;
   readonly attribute PRTime redundantTime;
 
+
+  // Total number of navigation faults experienced by this ServiceWorker since
+  // it was loaded from disk at startup or was installed.
+  readonly attribute unsigned long navigationFaultCount;
+
+  // Testing mechanism to induce synthetic failure of fetch events.  If set to
+  // something other than NS_OK, all fetch events dispatched will be propagated
+  // to the content process, but when it comes time to dispatch the fetch event,
+  // the cancellation control flow path will be triggered.
+  attribute nsresult testingInjectCancellation;
+
+
   void attachDebugger();
 
   void detachDebugger();
@@ -87,6 +99,7 @@ interface nsIServiceWorkerRegistrationInfo : nsISupports
   const unsigned short UPDATE_VIA_CACHE_NONE = 2;
 
   readonly attribute nsIPrincipal principal;
+  readonly attribute boolean unregistered;
 
   readonly attribute AString scope;
   readonly attribute AString scriptSpec;
@@ -99,6 +112,22 @@ interface nsIServiceWorkerRegistrationInfo : nsISupports
   readonly attribute nsIServiceWorkerInfo waitingWorker;
   readonly attribute nsIServiceWorkerInfo activeWorker;
 
+  // Exposes the number of times we have ever checked the usage of this origin
+  // for the purposes of mitigating ServiceWorker navigation faults that we
+  // suspect to be due to quota limit problems.  This should start out 0 and
+  // max out at 1 for the time being.
+  //
+  // Note that the underlying value is tracked on our per-Principal data, but
+  // we don't currently expose that data directly via XPCOM so we're exposing
+  // this here as the next best thing and because most non-test consumers would
+  // work in terms of the registration anyways.
+  //
+  // This will return -1 if there is no longer any per-origin data because the
+  // last registration for the origin (principal) has been unregistered.
+  // (Retaining a reference to this interface does not impact anything the
+  // underlying scope-to-registration map that is implemented per spec.)
+  readonly attribute long quotaUsageCheckCount;
+
   // Allows to get the related nsIServiceWorkerInfo for a given
   // nsIWorkerDebugger. Over time we shouldn't need this anymore,
   // and instead always control then nsIWorkerDebugger from

dom/serviceworkers/FetchEventOpChild.cpp

@@ -464,6 +464,14 @@ void FetchEventOpChild::CancelInterception(nsresult aStatus) {
   MOZ_ASSERT(!mInterceptedChannelHandled);
   MOZ_ASSERT(NS_FAILED(aStatus));
 
+  // Report a navigation fault if this is a navigation (and we have an active
+  // worker, which should be the case in non-shutdown/content-process-crash
+  // situations).
+  RefPtr<ServiceWorkerInfo> mActive = mRegistration->GetActive();
+  if (mActive && mArgs.isNonSubresourceRequest()) {
+    mActive->ReportNavigationFault();
+  }
+
   mInterceptedChannel->CancelInterception(aStatus);
   mInterceptedChannelHandled = true;
 

dom/serviceworkers/ServiceWorkerInfo.cpp

@@ -125,6 +125,31 @@ ServiceWorkerInfo::GetRedundantTime(PRTime* _retval) {
   return NS_OK;
 }
 
+NS_IMETHODIMP
+ServiceWorkerInfo::GetNavigationFaultCount(uint32_t* aNavigationFaultCount) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aNavigationFaultCount);
+  *aNavigationFaultCount = mNavigationFaultCount;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+ServiceWorkerInfo::GetTestingInjectCancellation(
+    nsresult* aTestingInjectCancellation) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aTestingInjectCancellation);
+  *aTestingInjectCancellation = mTestingInjectCancellation;
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+ServiceWorkerInfo::SetTestingInjectCancellation(
+    nsresult aTestingInjectCancellation) {
+  MOZ_ASSERT(NS_IsMainThread());
+  mTestingInjectCancellation = aTestingInjectCancellation;
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 ServiceWorkerInfo::AttachDebugger() {
   return mServiceWorkerPrivate->AttachDebugger();
@@ -188,7 +213,9 @@ ServiceWorkerInfo::ServiceWorkerInfo(nsIPrincipal* aPrincipal,
       mRedundantTime(0),
       mServiceWorkerPrivate(new ServiceWorkerPrivate(this)),
       mSkipWaitingFlag(false),
-      mHandlesFetch(Unknown) {
+      mHandlesFetch(Unknown),
+      mNavigationFaultCount(0),
+      mTestingInjectCancellation(NS_OK) {
   MOZ_ASSERT(XRE_GetProcessType() == GeckoProcessType_Default);
   MOZ_ASSERT(mPrincipal);
   // cache origin attributes so we can use them off main thread

dom/serviceworkers/ServiceWorkerInfo.h

@@ -62,6 +62,12 @@ class ServiceWorkerInfo final : public nsIServiceWorkerInfo {
 
   enum { Unknown, Enabled, Disabled } mHandlesFetch;
 
+  uint32_t mNavigationFaultCount;
+
+  // Testing helper to trigger fetch event cancellation when not NS_OK.
+  // See `nsIServiceWorkerInfo::testingInjectCancellation`.
+  nsresult mTestingInjectCancellation;
+
   ~ServiceWorkerInfo();
 
   // Generates a unique id for the service worker, with zero being treated as
@@ -97,6 +103,11 @@ class ServiceWorkerInfo final : public nsIServiceWorkerInfo {
     mSkipWaitingFlag = true;
   }
 
+  void ReportNavigationFault() {
+    MOZ_ASSERT(NS_IsMainThread());
+    mNavigationFaultCount++;
+  }
+
   ServiceWorkerInfo(nsIPrincipal* aPrincipal, const nsACString& aScope,
                     uint64_t aRegistrationId, uint64_t aRegistrationVersion,
                     const nsACString& aScriptSpec, const nsAString& aCacheName,
@@ -116,6 +127,8 @@ class ServiceWorkerInfo final : public nsIServiceWorkerInfo {
 
   const ServiceWorkerDescriptor& Descriptor() const { return mDescriptor; }
 
+  nsresult TestingInjectCancellation() { return mTestingInjectCancellation; }
+
   void UpdateState(ServiceWorkerState aState);
 
   // Only used to set initial state when loading from disk!

dom/serviceworkers/ServiceWorkerManager.cpp

@@ -396,6 +396,11 @@ struct ServiceWorkerManager::RegistrationDataPerPrincipal final {
 
   // Map scopes to scheduled update timers.
   nsInterfaceHashtable<nsCStringHashKey, nsITimer> mUpdateTimers;
+
+  // The number of times we have done a quota usage check for this origin for
+  // mitigation purposes.  See the docs on nsIServiceWorkerRegistrationInfo,
+  // where this value is exposed.
+  int32_t mQuotaUsageCheckCount = 0;
 };
 
 //////////////////////////
@@ -2230,6 +2235,22 @@ nsresult ServiceWorkerManager::GetClientRegistration(
   return NS_OK;
 }
 
+int32_t ServiceWorkerManager::GetPrincipalQuotaUsageCheckCount(
+    nsIPrincipal* aPrincipal) {
+  nsAutoCString scopeKey;
+  nsresult rv = PrincipalToScopeKey(aPrincipal, scopeKey);
+  if (NS_WARN_IF(NS_FAILED(rv))) {
+    return -1;
+  }
+
+  RegistrationDataPerPrincipal* data;
+  if (!mRegistrationInfos.Get(scopeKey, &data)) {
+    return -1;
+  }
+
+  return data->mQuotaUsageCheckCount;
+}
+
 void ServiceWorkerManager::SoftUpdate(const OriginAttributes& aOriginAttributes,
                                       const nsACString& aScope) {
   MOZ_ASSERT(NS_IsMainThread());

dom/serviceworkers/ServiceWorkerManager.h

@@ -252,6 +252,8 @@ class ServiceWorkerManager final : public nsIServiceWorkerManager,
       const ClientInfo& aClientInfo,
       ServiceWorkerRegistrationInfo** aRegistrationInfo);
 
+  int32_t GetPrincipalQuotaUsageCheckCount(nsIPrincipal* aPrincipal);
+
   // Returns the shutdown state ID (may be an invalid ID if an
   // nsIAsyncShutdownBlocker is not used).
   uint32_t MaybeInitServiceWorkerShutdownProgress() const;

dom/serviceworkers/ServiceWorkerOp.cpp

@@ -1533,6 +1533,31 @@ nsresult FetchEventOp::DispatchFetchEvent(JSContext* aCx,
   ServiceWorkerFetchEventOpArgs& args =
       mArgs.get_ServiceWorkerFetchEventOpArgs();
 
+  /**
+   * Testing: Failure injection.
+   *
+   * There are a number of different ways that this fetch event could have
+   * failed that would result in cancellation.  This injection point helps
+   * simulate them without worrying about shifting implementation details with
+   * full fidelity reproductions of current scenarios.
+   *
+   * Broadly speaking, we expect fetch event scenarios to fail because of:
+   * - Script load failure, which results in the CompileScriptRunnable closing
+   *   the worker and thereby cancelling all pending operations, including this
+   *   fetch.  The `ServiceWorkerOp::Cancel` impl just calls
+   *   RejectAll(NS_ERROR_DOM_ABORT_ERR) which we are able to approximate by
+   *   returning the same nsresult here, as our caller also calls RejectAll.
+   *   (And timing-wise, this rejection will happen in the correct sequence.)
+   * - An exception gets thrown in the processing of the promise that was passed
+   *   to respondWith and it ends up rejecting.  The rejection will be converted
+   *   by `FetchEventOp::RejectedCallback` into a cancellation with
+   *   NS_ERROR_INTERCEPTION_FAILED, and by returning that here we approximate
+   *   that failure mode.
+   */
+  if (NS_FAILED(args.testingInjectCancellation())) {
+    return args.testingInjectCancellation();
+  }
+
   /**
    * Step 1: get the InternalRequest. The InternalRequest can't be constructed
    * here from mArgs because the IPCStream has to be deserialized on the

dom/serviceworkers/ServiceWorkerOpArgs.ipdlh

@@ -70,6 +70,11 @@ struct ServiceWorkerFetchEventOpArgs {
   nsString clientId;
   nsString resultingClientId;
   bool isNonSubresourceRequest;
+  // Failure injection helper; non-NS_OK values indicate that the event, instead
+  // of dispatching should instead return a `CancelInterceptionArgs` response
+  // with this nsresult.  This value originates from
+  // `nsIServiceWorkerInfo::testingInjectCancellation`.
+  nsresult testingInjectCancellation;
 };
 
 union ServiceWorkerOpArgs {

dom/serviceworkers/ServiceWorkerPrivateImpl.cpp

@@ -827,7 +827,8 @@ nsresult ServiceWorkerPrivateImpl::SendFetchEvent(
   ServiceWorkerFetchEventOpArgs args(
       mOuter->mInfo->ScriptSpec(), std::move(request), nsString(aClientId),
       nsString(aResultingClientId),
-      nsContentUtils::IsNonSubresourceRequest(channel));
+      nsContentUtils::IsNonSubresourceRequest(channel),
+      mOuter->mInfo->TestingInjectCancellation());
 
   if (mOuter->mInfo->State() == ServiceWorkerState::Activating) {
     UniquePtr<PendingFunctionalEvent> pendingEvent =

dom/serviceworkers/ServiceWorkerRegistrationInfo.cpp

@@ -152,6 +152,7 @@ void ServiceWorkerRegistrationInfo::SetUnregistered() {
 #endif
 
   mUnregistered = true;
+  NotifyChromeRegistrationListeners();
 }
 
 NS_IMPL_ISUPPORTS(ServiceWorkerRegistrationInfo,
@@ -164,6 +165,14 @@ ServiceWorkerRegistrationInfo::GetPrincipal(nsIPrincipal** aPrincipal) {
   return NS_OK;
 }
 
+NS_IMETHODIMP ServiceWorkerRegistrationInfo::GetUnregistered(
+    bool* aUnregistered) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aUnregistered);
+  *aUnregistered = mUnregistered;
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 ServiceWorkerRegistrationInfo::GetScope(nsAString& aScope) {
   MOZ_ASSERT(NS_IsMainThread());
@@ -230,6 +239,23 @@ ServiceWorkerRegistrationInfo::GetActiveWorker(nsIServiceWorkerInfo** aResult) {
   return NS_OK;
 }
 
+NS_IMETHODIMP
+ServiceWorkerRegistrationInfo::GetQuotaUsageCheckCount(
+    int32_t* aQuotaUsageCheckCount) {
+  MOZ_ASSERT(NS_IsMainThread());
+  MOZ_ASSERT(aQuotaUsageCheckCount);
+
+  // This value is actually stored on SWM's internal-only
+  // RegistrationDataPerPrincipal structure, but we expose it here for
+  // simplicity for our consumers, so we have to ask SWM to look it up for us.
+  RefPtr<ServiceWorkerManager> swm = ServiceWorkerManager::GetInstance();
+  MOZ_ASSERT(swm);
+
+  *aQuotaUsageCheckCount = swm->GetPrincipalQuotaUsageCheckCount(mPrincipal);
+
+  return NS_OK;
+}
+
 NS_IMETHODIMP
 ServiceWorkerRegistrationInfo::GetWorkerByID(uint64_t aID,
                                              nsIServiceWorkerInfo** aResult) {

dom/serviceworkers/test/browser.ini

@@ -2,6 +2,7 @@
 support-files =
   browser_base_force_refresh.html
   browser_cached_force_refresh.html
+  browser_head.js
   download/window.html
   download/worker.js
   download_canceled/page_download_canceled.html
@@ -16,9 +17,11 @@ support-files =
   empty_with_utils.html
   empty.js
   intercepted_channel_process_swap_worker.js
+  network_with_utils.html
   page_post_controlled.html
   redirect.sjs
   storage_recovery_worker.sjs
+  sw_respondwith_serviceworker.js
   utils.js
 
 [browser_antitracking.js]
@@ -32,6 +35,7 @@ skip-if = verify # Bug 1603340
 skip-if = verify
 [browser_intercepted_channel_process_swap.js]
 skip-if = !fission
+[browser_navigation_fetch_fault_handling.js]
 [browser_remote_type_process_swap.js]
 skip-if = !e10s
 [browser_storage_permission.js]

dom/serviceworkers/test/browser_head.js

@@ -0,0 +1,218 @@
+/**
+ * This file contains common functionality for ServiceWorker browser tests.
+ *
+ * Note that the normal auto-import mechanics for browser mochitests only
+ * handles "head.js", but we currently store all of our different varieties of
+ * mochitest in a single directory, which potentially results in a collision
+ * for similar heuristics for xpcshell.
+ *
+ * Many of the storage-related helpers in this file come from:
+ * https://searchfox.org/mozilla-central/source/dom/localstorage/test/unit/head.js
+ **/
+
+// To use this file, explicitly import it via (including the eslint comment):
+//
+// /* import-globals-from browser_head.js */
+// Services.scriptloader.loadSubScript("chrome://mochitests/content/browser/dom/serviceworkers/test/browser_head.js", this);
+
+// Find the current parent directory of the test context we're being loaded into
+// such that one can do `${originNoTrailingSlash}/${DIR_PATH}/file_in_dir.foo`.
+const DIR_PATH = getRootDirectory(gTestPath)
+  .replace("chrome://mochitests/content/", "")
+  .slice(0, -1);
+
+const SWM = Cc["@mozilla.org/serviceworkers/manager;1"].getService(
+  Ci.nsIServiceWorkerManager
+);
+
+function getPrincipal(url, attrs) {
+  const uri = Services.io.newURI(url);
+  if (!attrs) {
+    attrs = {};
+  }
+  return Services.scriptSecurityManager.createContentPrincipal(uri, attrs);
+}
+
+async function _qm_requestFinished(request) {
+  await new Promise(function(resolve) {
+    request.callback = function() {
+      resolve();
+    };
+  });
+
+  if (request.resultCode !== Cr.NS_OK) {
+    throw new RequestError(request.resultCode, request.resultName);
+  }
+
+  return request.result;
+}
+
+async function get_qm_origin_usage(origin) {
+  return new Promise(resolve => {
+    const principal = Services.scriptSecurityManager.createContentPrincipalFromOrigin(
+      origin
+    );
+    Services.qms.getUsageForPrincipal(principal, request =>
+      resolve(request.result.usage)
+    );
+  });
+}
+
+/**
+ * Clear the group associated with the given origin via nsIClearDataService.  We
+ * are using nsIClearDataService here because nsIQuotaManagerService doesn't
+ * (directly) provide a means of clearing a group.
+ */
+async function clear_qm_origin_group_via_clearData(origin) {
+  const uri = Services.io.newURI(origin);
+  const baseDomain = Services.eTLD.getBaseDomain(uri);
+  info(`Clearing storage on domain ${baseDomain} (from origin ${origin})`);
+
+  // Initiate group clearing and wait for it.
+  await new Promise((resolve, reject) => {
+    Services.clearData.deleteDataFromHost(
+      baseDomain,
+      false,
+      Services.clearData.CLEAR_DOM_QUOTA,
+      failedFlags => {
+        if (failedFlags) {
+          reject(failedFlags);
+        } else {
+          resolve();
+        }
+      }
+    );
+  });
+}
+
+/**
+ * Look up the nsIServiceWorkerRegistrationInfo for a given SW descriptor.
+ */
+function swm_lookup_reg(swDesc) {
+  // Scopes always include the full origin.
+  const fullScope = `${swDesc.origin}/${DIR_PATH}/${swDesc.scope}`;
+  const principal = getPrincipal(fullScope);
+
+  const reg = SWM.getRegistrationByPrincipal(principal, fullScope);
+
+  return reg;
+}
+
+/**
+ * Install a ServiceWorker according to the provided descriptor by opening a
+ * fresh tab that will be closed when we are done.  Returns the
+ * `nsIServiceWorkerRegistrationInfo` corresponding to the registration.
+ *
+ * The descriptor may have the following properties:
+ * - scope: Optional.
+ * - script: The script, which usually just wants to be a relative path.
+ * - origin: Requred, the origin (which should not include a trailing slash).
+ */
+async function install_sw(swDesc) {
+  info(
+    `Installing ServiceWorker ${swDesc.script} at ${swDesc.scope} on origin ${swDesc.origin}`
+  );
+  const pageUrlStr = `${swDesc.origin}/${DIR_PATH}/empty_with_utils.html`;
+
+  await BrowserTestUtils.withNewTab(
+    {
+      gBrowser,
+      url: pageUrlStr,
+    },
+    async browser => {
+      await SpecialPowers.spawn(
+        browser,
+        [{ swScript: swDesc.script, swScope: swDesc.scope }],
+        async function({ swScript, swScope }) {
+          await content.wrappedJSObject.registerAndWaitForActive(
+            swScript,
+            swScope
+          );
+        }
+      );
+    }
+  );
+  info(`ServiceWorker installed`);
+
+  return swm_lookup_reg(swDesc);
+}
+
+/**
+ * Consume storage in the given origin by storing randomly generated Blobs into
+ * Cache API storage and IndexedDB storage.  We use both APIs in order to
+ * ensure that data clearing wipes both QM clients.
+ *
+ * Randomly generated Blobs means Blobs with literally random content.  This is
+ * done to compensate for the Cache API using snappy for compression.
+ */
+async function consume_storage(origin, storageDesc) {
+  info(`Consuming storage on origin ${origin}`);
+  const pageUrlStr = `${origin}/${DIR_PATH}/empty_with_utils.html`;
+
+  await BrowserTestUtils.withNewTab(
+    {
+      gBrowser,
+      url: pageUrlStr,
+    },
+    async browser => {
+      await SpecialPowers.spawn(browser, [storageDesc], async function({
+        cacheBytes,
+        idbBytes,
+      }) {
+        await content.wrappedJSObject.fillStorage(cacheBytes, idbBytes);
+      });
+    }
+  );
+}
+
+/**
+ * Perform a navigation, waiting until the navigation stops, then returning
+ * the `textContent` of the body node.  The expectation is this will be used
+ * with ServiceWorkers that return a body that indicates the ServiceWorker
+ * provided the result (possibly derived from the request) versus if
+ * interception didn't happen.
+ */
+async function navigate_and_get_body(swDesc, debugTag) {
+  let pageUrlStr = `${swDesc.origin}/${DIR_PATH}/${swDesc.scope}`;
+  if (debugTag) {
+    pageUrlStr += "?" + debugTag;
+  }
+  info(`Navigating to ${pageUrlStr}`);
+
+  const tabResult = await BrowserTestUtils.withNewTab(
+    {
+      gBrowser,
+      url: pageUrlStr,
+      // In the event of an aborted navigation, the load event will never
+      // happen...
+      waitForLoad: false,
+      // ...but the stop will.
+      waitForStateStop: true,
+    },
+    async browser => {
+      info(` Tab opened, querying body content.`);
+      const spawnResult = await SpecialPowers.spawn(browser, [], function() {
+        const controlled = !!content.navigator.serviceWorker.controller;
+        // Special-case about: URL's.
+        let loc = content.document.documentURI;
+        if (loc.startsWith("about:")) {
+          // about:neterror is parameterized by query string, so truncate that
+          // off because our tests just care if we're seeing the neterror page.
+          const idxQuestion = loc.indexOf("?");
+          if (idxQuestion !== -1) {
+            loc = loc.substring(0, idxQuestion);
+          }
+          return { controlled, body: loc };
+        }
+        return {
+          controlled,
+          body: content.document?.body?.textContent?.trim(),
+        };
+      });
+
+      return spawnResult;
+    }
+  );
+
+  return tabResult;
+}

dom/serviceworkers/test/browser_navigation_fetch_fault_handling.js

@@ -0,0 +1,259 @@
+/**
+ * This test file tests our automatic recovery and any related mitigating
+ * heuristics that occur during intercepted navigation fetch request.
+ * Specifically, we should be resetting interception so that we go to the
+ * network in these cases and then potentially taking actions like unregistering
+ * the ServiceWorker and/or clearing QuotaManager-managed storage for the
+ * origin.
+ *
+ * See specific test permutations for specific details inline in the test.
+ *
+ * NOTE THAT CURRENTLY THIS TEST IS DISCUSSING MITIGATIONS THAT ARE NOT YET
+ * IMPLEMENTED, JUST PLANNED.  These will be iterated on and added to the rest
+ * of the stack of patches on Bug 1503072.
+ *
+ * ## Test Mechanics
+ *
+ * ### Fetch Fault Injection
+ *
+ * We expose:
+ * - On nsIServiceWorkerInfo, the per-ServiceWorker XPCOM interface:
+ *   - A mechanism for creating synthetic faults by setting the
+ *     `nsIServiceWorkerInfo::testingInjectCancellation` attribute to a failing
+ *     nsresult.  The fault is applied at the beginning of the steps to dispatch
+ *     the fetch event on the global.
+ *   - A count of the number of times we experienced these navigation faults
+ *     that had to be reset as `nsIServiceWorkerInfo::navigationFaultCount`.
+ *     (This would also include real faults, but we only expect to see synthetic
+ *     faults in this test.)
+ * - On nsIServiceWorkerRegistrationInfo, the per-registration XPCOM interface:
+ *   - A readonly attribute that indicates how many times an origin storage
+ *     usage check has been initiated.
+ *
+ * We also use:
+ * - `nsIServiceWorkerManager::addListener(nsIServiceWorkerManagerListener)`
+ *   allows our test to listen for the unregistration of registrations.  This
+ *   allows us to be notified when unregistering or origin-clearing actions have
+ *   been taken as a mitigation.
+ *
+ * ### General Test Approach
+ *
+ * For each test we:
+ * - Ensure/confirm the testing origin has no QuotaManager storage in use.
+ * - Install the ServiceWorker.
+ * - If we are testing the situation where we want to simulate the origin being
+ *   near its quota limit, we also generate Cache API and IDB storage usage
+ *   sufficient to put our origin over the threshold.
+ *   - We run a quota check on the origin after doing this in order to make sure
+ *     that we did this correctly and that we properly constrained the limit for
+ *     the origin.  We fail the test for test implementation reasons if we
+ *     didn't accomplish this.
+ * - Verify a fetch navigation to the SW works without any fault injection,
+ *   producing a result produced by the ServiceWorker.
+ * - Begin fault permutations in a loop, where for each pass of the loop:
+ *   - We trigger a navigation which will result in an intercepted fetch
+ *     which will fault.  We wait until the navigation completes.
+ *   - We verify that we got the request from the network.
+ *   - We verify that the ServiceWorker's navigationFaultCount increased.
+ *   - If this the count at which we expect a mitigation to take place, we wait
+ *     for the registration to become unregistered AND:
+ *     - We check whether the storage for the origin was cleared or not, which
+ *       indicates which mitigation of the following happened:
+ *       - Unregister the registration directly.
+ *       - Clear the origin's data which will also unregister the registration
+ *         as a side effect.
+ *     - We check whether the registration indicates an origin quota check
+ *       happened or not.
+ *
+ * ### Disk Usage Limits
+ *
+ * In order to avoid gratuitous disk I/O and related overheads, we limit QM
+ * ("temporary") storage to 10 MiB which ends up limiting group usage to 10 MiB.
+ * This lets us set a threshold situation where we claim that a SW needs at
+ * least 4 MiB of storage for installation/operation, meaning that any usage
+ * beyond 6 MiB in the group will constitute a need to clear the group or
+ * origin.  We fill with the storage with 8 MiB of artificial usage to this end,
+ * storing 4 MiB in Cache API and 4 MiB in IDB.
+ **/
+
+// Because of the amount of I/O involved in this test, pernosco reproductions
+// may experience timeouts without a timeout multiplier.
+requestLongerTimeout(2);
+
+/* import-globals-from browser_head.js */
+Services.scriptloader.loadSubScript(
+  "chrome://mochitests/content/browser/dom/serviceworkers/test/browser_head.js",
+  this
+);
+
+// The origin we run the tests on.
+const TEST_ORIGIN = "https://test1.example.org";
+// An origin in the same group that impacts the usage of the TEST_ORIGIN.  Used
+// to verify heuristics related to group-clearing (where clearing the
+// TEST_ORIGIN itself would not be sufficient for us to mitigate quota limits
+// being reached.)
+const SAME_GROUP_ORIGIN = "https://test2.example.org";
+
+const TEST_SW_SETUP = {
+  origin: TEST_ORIGIN,
+  // Page with a body textContent of "NETWORK" and has utils.js loaded.
+  scope: "network_with_utils.html",
+  // SW that serves a body with a textContent of "SERVICEWORKER" and
+  // has utils.js loaded.
+  script: "sw_respondwith_serviceworker.js",
+};
+
+const TEST_STORAGE_SETUP = {
+  cacheBytes: 4 * 1024 * 1024,
+  idbBytes: 4 * 1024 * 1024,
+};
+
+const FAULTS_BEFORE_MITIGATION = 3;
+
+/**
+ * Core test iteration logic.
+ *
+ * Parameters:
+ * - name: Human readable name of the fault we're injecting.
+ * - useError: The nsresult failure code to inject into fetch.
+ * - errorPage: The "about" page that we expect errors to leave us on.
+ * - consumeQuotaOrigin: If truthy, the origin to place the storage usage in.
+ *   If falsey, we won't fill storage.
+ */
+async function do_fault_injection_test({
+  name,
+  useError,
+  errorPage,
+  consumeQuotaOrigin,
+}) {
+  info(
+    `### testing: error: ${name} (${useError}) consumeQuotaOrigin: ${consumeQuotaOrigin}`
+  );
+
+  // ## Ensure/confirm the testing origins have no QuotaManager storage in use.
+  await clear_qm_origin_group_via_clearData(TEST_ORIGIN);
+
+  // ## Install the ServiceWorker
+  const reg = await install_sw(TEST_SW_SETUP);
+  const sw = reg.activeWorker;
+
+  // ## Generate quota usage if appropriate
+  if (consumeQuotaOrigin) {
+    await consume_storage(consumeQuotaOrigin, TEST_SW_SETUP);
+  }
+
+  // ## Verify normal navigation is served by the SW.
+  info(`## Checking normal operation.`);
+  {
+    const debugTag = `err=${name}&fault=0`;
+    const docInfo = await navigate_and_get_body(TEST_SW_SETUP, debugTag);
+    is(
+      docInfo.body,
+      "SERVICEWORKER",
+      "navigation without injected fault originates from ServiceWorker"
+    );
+
+    is(
+      docInfo.controlled,
+      true,
+      "successfully intercepted navigation should be controlled"
+    );
+  }
+
+  // ## Inject faults in a loop until expected mitigation.
+  sw.testingInjectCancellation = useError;
+  for (let iFault = 0; iFault < FAULTS_BEFORE_MITIGATION; iFault++) {
+    info(`## Testing with injected fault number ${iFault + 1}`);
+    // We should never have triggered an origin quota usage check before the
+    // final fault injection.
+    is(reg.quotaUsageCheckCount, 0, "No quota usage check yet.");
+
+    // Make sure our loads encode the specific
+    const debugTag = `err=${name}&fault=${iFault + 1}`;
+
+    const docInfo = await navigate_and_get_body(TEST_SW_SETUP, debugTag);
+    // We should always be receiving network fallback.
+    is(
+      docInfo.body,
+      errorPage, // "NETWORK", once we have recovery.
+      "navigation with injected fault originates from network"
+    );
+
+    is(docInfo.controlled, false, "error pages shouldn't be controlled");
+
+    // The fault count should have increased
+    is(
+      sw.navigationFaultCount,
+      iFault + 1,
+      "navigation fault increased (to expected value)"
+    );
+  }
+
+  // The mitigations should have happened now, if they are going to happen.
+
+  // INITIALLY THERE ARE NO MITIGATIONS, SO WE ARE EXPECTING FAILURES (which we
+  // explicitly encode into the tests).  THE NEXT PATCHES WILL CHANGE THE
+  // EXPECTATIONS.
+
+  // We won't actually have been unregistered, unfortunately.
+  // (When fixed, we will actually instead want to wait for this value to
+  // change, as the unregister process is async.)
+  is(reg.unregistered, false, "registration should still exist.");
+
+  if (consumeQuotaOrigin) {
+    // Check that there is no longer any storage usaged by the origin in this
+    // case.
+    const originUsage = await get_qm_origin_usage(TEST_ORIGIN);
+    ok(originUsage > 0, "origin should still have usage until mitigated");
+
+    if (consumeQuotaOrigin === SAME_GROUP_ORIGIN) {
+      const sameGroupUsage = await get_qm_origin_usage(SAME_GROUP_ORIGIN);
+      ok(sameGroupUsage > 0, "same group should still have usage for now");
+    }
+  }
+}
+
+add_task(async function test_navigation_fetch_fault_handling() {
+  await SpecialPowers.pushPrefEnv({
+    set: [
+      ["dom.serviceWorkers.enabled", true],
+      ["dom.serviceWorkers.exemptFromPerDomainMax", true],
+      ["dom.serviceWorkers.testing.enabled", true],
+      // We want the temporary global limit to be 10 MiB (the pref is in KiB).
+      // This will result in the group limit also being 10 MiB because on small
+      // disks we provide a group limit value of min(10 MiB, global limit).
+      ["dom.quotaManager.temporaryStorage.fixedLimit", 10 * 1024],
+    ],
+  });
+
+  const quotaOriginVariations = [
+    // Don't put us near the storage limit.
+    undefined,
+    // Put us near the storage limit in the SW origin itself.
+    TEST_ORIGIN,
+    // Put us near the storage limit in the SW origin's group but not the origin
+    // itself.
+    SAME_GROUP_ORIGIN,
+  ];
+
+  for (const consumeQuotaOrigin of quotaOriginVariations) {
+    await do_fault_injection_test({
+      name: "NS_ERROR_DOM_ABORT_ERR",
+      useError: 0x80530014, // Not in `Cr`.
+      // Abort errors manifest as about:blank pages.
+      errorPage: "about:blank",
+      consumeQuotaOrigin,
+    });
+
+    await do_fault_injection_test({
+      name: "NS_ERROR_INTERCEPTION_FAILED",
+      useError: 0x804b0064, // Not in `Cr`.
+      // Interception failures manifest as corrupt content pages.
+      errorPage: "about:neterror",
+      consumeQuotaOrigin,
+    });
+  }
+
+  // Cleanup: wipe the origin and group so all the ServiceWorkers go away.
+  await clear_qm_origin_group_via_clearData(TEST_ORIGIN);
+});

dom/serviceworkers/test/network_with_utils.html

@@ -0,0 +1,14 @@
+<!--
+  Any copyright is dedicated to the Public Domain.
+  http://creativecommons.org/publicdomain/zero/1.0/
+-->
+<!DOCTYPE HTML>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="utils.js" type="text/javascript"></script>
+</head>
+<body>
+NETWORK
+</body>
+</html>

dom/serviceworkers/test/sw_respondwith_serviceworker.js

@@ -0,0 +1,24 @@
+const SERVICEWORKER_DOC = `<!DOCTYPE HTML>
+<html>
+<head>
+  <meta charset="utf-8">
+  <script src="utils.js" type="text/javascript"></script>
+</head>
+<body>
+SERVICEWORKER
+</body>
+</html>
+`;
+
+const SERVICEWORKER_RESPONSE = new Response(SERVICEWORKER_DOC, {
+  headers: { "content-type": "text/html" },
+});
+
+addEventListener("fetch", event => {
+  // Allow utils.js which we explicitly include to be loaded by resetting
+  // interception.
+  if (event.request.url.endsWith("/utils.js")) {
+    return;
+  }
+  event.respondWith(SERVICEWORKER_RESPONSE.clone());
+});

dom/serviceworkers/test/test_devtools_track_serviceworker_time.html

@@ -57,7 +57,13 @@ let expectedResults = [
     activatedTimeRecorded: true, redundantTimeRecorded: false
   },
 
-  // On unregister
+  // When first being marked as unregistered (but the worker can remain
+  // actively controlling pages)
+  {
+    state: State.ACTIVATED, installedTimeRecorded: true,
+    activatedTimeRecorded: true, redundantTimeRecorded: false
+  },
+  // When cleared (when idle)
   {
     state: State.REDUNDANT, installedTimeRecorded: true,
     activatedTimeRecorded: true, redundantTimeRecorded: true

dom/serviceworkers/test/utils.js

@@ -22,9 +22,13 @@ function waitForState(worker, state, context) {
  * From the ContentTask.spawn, use via
  * `content.wrappedJSObject.registerAndWaitForActive`.
  */
-async function registerAndWaitForActive(...args) {
+async function registerAndWaitForActive(script, maybeScope) {
   console.log("...calling register");
-  const reg = await navigator.serviceWorker.register(...args);
+  let opts = undefined;
+  if (maybeScope) {
+    opts = { scope: maybeScope };
+  }
+  const reg = await navigator.serviceWorker.register(script, opts);
   // Unless registration resurrection happens, the SW should be in the
   // installing slot.
   console.log("...waiting for activation");
@@ -83,3 +87,39 @@ async function unregisterAll() {
     await reg.unregister();
   }
 }
+
+/**
+ * Make a blob that contains random data and therefore shouldn't compress all
+ * that well.
+ */
+function makeRandomBlob(size) {
+  const arr = new Uint8Array(size);
+  window.crypto.getRandomValues(arr);
+  return new Blob([arr], { type: "application/octet-stream" });
+}
+
+async function fillStorage(cacheBytes, idbBytes) {
+  // ## Fill Cache API Storage
+  const cache = await caches.open("filler");
+  await cache.put("fill", new Response(makeRandomBlob(cacheBytes)));
+
+  // ## Fill IDB
+  const storeName = "filler";
+  let db = await new Promise((resolve, reject) => {
+    let openReq = indexedDB.open("filler", 1);
+    openReq.onerror = event => {
+      reject(event.target.error);
+    };
+    openReq.onsuccess = event => {
+      resolve(event.target.result);
+    };
+    openReq.onupgradeneeded = event => {
+      const useDB = event.target.result;
+      useDB.onerror = error => {
+        reject(error);
+      };
+      const store = useDB.createObjectStore(storeName);
+      store.put({ blob: makeRandomBlob(idbBytes) }, "filler-blob");
+    };
+  });
+}