Bug 1589407 - Part 2: Ensure that document.referrer sees the same URL as what we send in the Referer HTTP header per the ETP restrictions; r=tnguyen,baku

Differential Revision: https://phabricator.services.mozilla.com/D49773

--HG--
extra : moz-landing-system : lando

netwerk/protocol/http/HttpBaseChannel.cpp

@@ -1568,17 +1568,18 @@ HttpBaseChannel::GetReferrerInfo(nsIReferrerInfo** aReferrerInfo) {
   return NS_OK;
 }
 
-nsresult HttpBaseChannel::SetReferrerInfo(nsIReferrerInfo* aReferrerInfo,
-                                          bool aClone, bool aCompute,
-                                          bool aSetOriginal) {
-  LOG(("HttpBaseChannel::SetReferrerInfo [this=%p aClone(%d) aCompute(%d)]\n",
+nsresult HttpBaseChannel::SetReferrerInfoInternal(
+    nsIReferrerInfo* aReferrerInfo, bool aClone, bool aCompute,
+    bool aRespectBeforeConnect) {
+  LOG(
+      ("HttpBaseChannel::SetReferrerInfoInternal [this=%p aClone(%d) "
+       "aCompute(%d)]\n",
        this, aClone, aCompute));
-  ENSURE_CALLED_BEFORE_CONNECT();
+  if (aRespectBeforeConnect) {
+    ENSURE_CALLED_BEFORE_CONNECT();
+  }
 
   mReferrerInfo = aReferrerInfo;
-  if (aSetOriginal) {
-    mOriginalReferrerInfo = aReferrerInfo;
-  }
 
   // clear existing referrer, if any
   nsresult rv = ClearReferrerHeader();
@@ -1592,9 +1593,6 @@ nsresult HttpBaseChannel::SetReferrerInfo(nsIReferrerInfo* aReferrerInfo,
 
   if (aClone) {
     mReferrerInfo = static_cast<dom::ReferrerInfo*>(aReferrerInfo)->Clone();
-    if (aSetOriginal) {
-      mOriginalReferrerInfo = mReferrerInfo;
-    }
   }
 
   dom::ReferrerInfo* referrerInfo =
@@ -1624,17 +1622,17 @@ nsresult HttpBaseChannel::SetReferrerInfo(nsIReferrerInfo* aReferrerInfo,
     return rv;
   }
 
-  return SetReferrerHeader(spec);
+  return SetReferrerHeader(spec, aRespectBeforeConnect);
 }
 
 NS_IMETHODIMP
 HttpBaseChannel::SetReferrerInfo(nsIReferrerInfo* aReferrerInfo) {
-  return SetReferrerInfo(aReferrerInfo, true, true);
+  return SetReferrerInfoInternal(aReferrerInfo, true, true, true);
 }
 
 NS_IMETHODIMP
 HttpBaseChannel::SetReferrerInfoWithoutClone(nsIReferrerInfo* aReferrerInfo) {
-  return SetReferrerInfo(aReferrerInfo, false, true);
+  return SetReferrerInfoInternal(aReferrerInfo, false, true, true);
 }
 
 // Return the channel's proxy URI, or if it doesn't exist, the
@@ -3141,7 +3139,7 @@ HttpBaseChannel::CloneReplacementChannelConfig(bool aPreserveMethod,
     config.privateBrowsing = Some(mPrivateBrowsing);
   }
 
-  if (mOriginalReferrerInfo) {
+  if (mReferrerInfo) {
     dom::ReferrerPolicy referrerPolicy = dom::ReferrerPolicy::_empty;
     nsAutoCString tRPHeaderCValue;
     Unused << GetResponseHeader(NS_LITERAL_CSTRING("referrer-policy"),
@@ -3158,11 +3156,11 @@ HttpBaseChannel::CloneReplacementChannelConfig(bool aPreserveMethod,
       // changes, we must not use the old computed value, and have to compute
       // again.
       nsCOMPtr<nsIReferrerInfo> referrerInfo =
-          dom::ReferrerInfo::CreateFromOtherAndPolicyOverride(
-              mOriginalReferrerInfo, referrerPolicy);
+          dom::ReferrerInfo::CreateFromOtherAndPolicyOverride(mReferrerInfo,
+                                                              referrerPolicy);
       config.referrerInfo = referrerInfo;
     } else {
-      config.referrerInfo = mOriginalReferrerInfo;
+      config.referrerInfo = mReferrerInfo;
     }
   }
 

netwerk/protocol/http/HttpBaseChannel.h

@@ -447,8 +447,11 @@ class HttpBaseChannel : public nsHashPropertyBag,
     mUploadStreamHasHeaders = hasHeaders;
   }
 
-  virtual nsresult SetReferrerHeader(const nsACString& aReferrer) {
-    ENSURE_CALLED_BEFORE_CONNECT();
+  virtual nsresult SetReferrerHeader(const nsACString& aReferrer,
+                                     bool aRespectBeforeConnect = true) {
+    if (aRespectBeforeConnect) {
+      ENSURE_CALLED_BEFORE_CONNECT();
+    }
     return mRequestHead.SetHeader(nsHttp::Referer, aReferrer);
   }
 
@@ -470,8 +473,8 @@ class HttpBaseChannel : public nsHashPropertyBag,
   // Pass true for aSetOriginal if this is a new referrer and should
   // overwrite the 'original' value, false if this is a mutation (like
   // stripping the path).
-  nsresult SetReferrerInfo(nsIReferrerInfo* aReferrerInfo, bool aClone,
-                           bool aCompute, bool aSetOriginal = true);
+  nsresult SetReferrerInfoInternal(nsIReferrerInfo* aReferrerInfo, bool aClone,
+                                   bool aCompute, bool aRespectBeforeConnect);
 
   struct ReplacementChannelConfig {
     ReplacementChannelConfig() = default;
@@ -604,10 +607,6 @@ class HttpBaseChannel : public nsHashPropertyBag,
   nsCOMPtr<nsIInterfaceRequestor> mCallbacks;
   nsCOMPtr<nsIProgressEventSink> mProgressSink;
   nsCOMPtr<nsIReferrerInfo> mReferrerInfo;
-  // We cache the original value of mReferrerInfo, since
-  // we trim the referrer to not expose the full path to remote
-  // usage.
-  nsCOMPtr<nsIReferrerInfo> mOriginalReferrerInfo;
   nsCOMPtr<nsIApplicationCache> mApplicationCache;
   nsCOMPtr<nsIURI> mAPIRedirectToURI;
   nsCOMPtr<nsIURI> mProxyURI;

netwerk/protocol/http/HttpChannelChild.cpp

@@ -3337,6 +3337,19 @@ mozilla::ipc::IPCResult HttpChannelChild::RecvAltDataCacheInputStreamAvailable(
   return IPC_OK();
 }
 
+mozilla::ipc::IPCResult
+HttpChannelChild::RecvOverrideReferrerInfoDuringBeginConnect(
+    nsIReferrerInfo* aReferrerInfo) {
+  // The arguments passed to SetReferrerInfoInternal here should mirror the
+  // arguments passed in
+  // nsHttpChannel::ReEvaluateReferrerAfterTrackingStatusIsKnown(), except for
+  // aRespectBeforeConnect which we pass false here since we're intentionally
+  // overriding the referrer after BeginConnect().
+  Unused << SetReferrerInfoInternal(aReferrerInfo, false, true, false);
+
+  return IPC_OK();
+}
+
 //-----------------------------------------------------------------------------
 // HttpChannelChild::nsIResumableChannel
 //-----------------------------------------------------------------------------
@@ -3765,11 +3778,14 @@ nsresult HttpChannelChild::AsyncCallImpl(
   return rv;
 }
 
-nsresult HttpChannelChild::SetReferrerHeader(const nsACString& aReferrer) {
+nsresult HttpChannelChild::SetReferrerHeader(const nsACString& aReferrer,
+                                             bool aRespectBeforeConnect) {
   // Normally this would be ENSURE_CALLED_BEFORE_CONNECT, but since the
   // "connect" is done in the main process, and mRequestObserversCalled is never
   // set in the ChannelChild, before connect basically means before asyncOpen.
-  ENSURE_CALLED_BEFORE_ASYNC_OPEN();
+  if (aRespectBeforeConnect) {
+    ENSURE_CALLED_BEFORE_ASYNC_OPEN();
+  }
 
   // remove old referrer if any, loop backwards
   for (int i = mClientSetRequestHeaders.Length() - 1; i >= 0; --i) {
@@ -3779,7 +3795,7 @@ nsresult HttpChannelChild::SetReferrerHeader(const nsACString& aReferrer) {
     }
   }
 
-  return HttpBaseChannel::SetReferrerHeader(aReferrer);
+  return HttpBaseChannel::SetReferrerHeader(aReferrer, aRespectBeforeConnect);
 }
 
 class CancelEvent final : public NeckoTargetChannelEvent<HttpChannelChild> {

netwerk/protocol/http/HttpChannelChild.h

@@ -112,7 +112,8 @@ class HttpChannelChild final : public PHttpChannelChild,
   // nsIResumableChannel
   NS_IMETHOD ResumeAt(uint64_t startPos, const nsACString& entityID) override;
 
-  nsresult SetReferrerHeader(const nsACString& aReferrer) override;
+  nsresult SetReferrerHeader(const nsACString& aReferrer,
+                             bool aRespectBeforeConnect) override;
 
   MOZ_MUST_USE bool IsSuspended();
 
@@ -178,6 +179,9 @@ class HttpChannelChild final : public PHttpChannelChild,
   mozilla::ipc::IPCResult RecvAltDataCacheInputStreamAvailable(
       const Maybe<IPCStream>& aStream) override;
 
+  mozilla::ipc::IPCResult RecvOverrideReferrerInfoDuringBeginConnect(
+      nsIReferrerInfo* aReferrerInfo) override;
+
   virtual void ActorDestroy(ActorDestroyReason aWhy) override;
 
   virtual void DoNotifyListenerCleanup() override;

netwerk/protocol/http/HttpChannelParent.cpp

@@ -485,7 +485,8 @@ bool HttpChannelParent::DoAsyncOpen(
   if (docUri) httpChannel->SetDocumentURI(docUri);
   if (aReferrerInfo) {
     // Referrer header is computed in child no need to recompute here
-    rv = httpChannel->SetReferrerInfo(aReferrerInfo, false, false);
+    rv =
+        httpChannel->SetReferrerInfoInternal(aReferrerInfo, false, false, true);
     MOZ_ASSERT(NS_SUCCEEDED(rv));
   }
 
@@ -917,7 +918,8 @@ mozilla::ipc::IPCResult HttpChannelParent::RecvRedirect2Verify(
         MOZ_ASSERT(baseChannel);
         if (baseChannel) {
           // Referrer header is computed in child no need to recompute here
-          rv = baseChannel->SetReferrerInfo(aReferrerInfo, false, false);
+          rv = baseChannel->SetReferrerInfoInternal(aReferrerInfo, false, false,
+                                                    true);
           MOZ_ASSERT(NS_SUCCEEDED(rv));
         }
       }
@@ -2767,5 +2769,10 @@ nsresult HttpChannelParent::TriggerCrossProcessSwitch(nsIHttpChannel* aChannel,
   return NS_OK;
 }
 
+void HttpChannelParent::OverrideReferrerInfoDuringBeginConnect(
+    nsIReferrerInfo* aReferrerInfo) {
+  Unused << SendOverrideReferrerInfoDuringBeginConnect(aReferrerInfo);
+}
+
 }  // namespace net
 }  // namespace mozilla

netwerk/protocol/http/HttpChannelParent.h

@@ -136,6 +136,10 @@ class HttpChannelParent final : public nsIInterfaceRequestor,
   nsresult TriggerCrossProcessSwitch(nsIHttpChannel* aChannel,
                                      uint64_t aIdentifier);
 
+  // Inform the child actor that our referrer info was modified late during
+  // BeginConnect.
+  void OverrideReferrerInfoDuringBeginConnect(nsIReferrerInfo* aReferrerInfo);
+
  protected:
   // used to connect redirected-to channel in parent with just created
   // ChildChannel.  Used during redirects.

netwerk/protocol/http/PHttpChannel.ipdl

@@ -188,6 +188,8 @@ child:
 
   async AltDataCacheInputStreamAvailable(IPCStream? stream);
 
+  async OverrideReferrerInfoDuringBeginConnect(nsIReferrerInfo referrerInfo);
+
 both:
   // After receiving this message, the parent also calls
   // SendFinishInterceptedRedirect, and makes sure not to send any more messages

netwerk/protocol/http/nsHttpChannel.cpp

@@ -10434,9 +10434,17 @@ void nsHttpChannel::ReEvaluateReferrerAfterTrackingStatusIsKnown() {
                                                      isPrivate)) {
         nsCOMPtr<nsIReferrerInfo> newReferrerInfo =
             referrerInfo->CloneWithNewPolicy(ReferrerPolicy::_empty);
-        // Pass false for the 3rd bool to not overwrite the original
-        // referrer for these referrer policy mutations.
-        SetReferrerInfo(newReferrerInfo, false, true, false);
+        // The arguments passed to SetReferrerInfoInternal here should mirror
+        // the arguments passed in
+        // HttpChannelChild::RecvOverrideReferrerInfoDuringBeginConnect().
+        SetReferrerInfoInternal(newReferrerInfo, false, true, true);
+
+        nsCOMPtr<nsIParentChannel> parentChannel;
+        NS_QueryNotificationCallbacks(this, parentChannel);
+        RefPtr<HttpChannelParent> httpParent = do_QueryObject(parentChannel);
+        if (httpParent) {
+          httpParent->OverrideReferrerInfoDuringBeginConnect(newReferrerInfo);
+        }
       }
     }
   }

netwerk/test/unit/test_redirect_history.js

@@ -65,7 +65,7 @@ function run_test() {
 
   var chan = make_channel(URL + redirects[0]);
   var uri = NetUtil.newURI("http://test.com");
-  httpChan = chan.QueryInterface(Ci.nsIHttpChannel);
+  var httpChan = chan.QueryInterface(Ci.nsIHttpChannel);
   httpChan.referrerInfo = new ReferrerInfo(Ci.nsIReferrerInfo.EMPTY, true, uri);
   chan.asyncOpen(new ChannelListener(finish_test, null));
   do_test_pending();

toolkit/components/antitracking/test/browser/browser_referrerDefaultPolicy.js

@@ -23,7 +23,7 @@ async function testOnWindowBody(win, expectedReferrer, rp) {
   await promiseTabLoadEvent(tab, TEST_TOP_PAGE);
 
   info("Loading tracking scripts and tracking images");
-  await ContentTask.spawn(b, { rp }, async function({ rp }) {
+  let referrer = await ContentTask.spawn(b, { rp }, async function({ rp }) {
     {
       let src = content.document.createElement("script");
       let p = new content.Promise(resolve => {
@@ -51,8 +51,32 @@ async function testOnWindowBody(win, expectedReferrer, rp) {
         "https://tracking.example.org/browser/toolkit/components/antitracking/test/browser/referrer.sjs?what=image";
       await p;
     }
+
+    {
+      let iframe = content.document.createElement("iframe");
+      let p = new content.Promise(resolve => {
+        iframe.onload = resolve;
+      });
+      content.document.body.appendChild(iframe);
+      if (rp) {
+        iframe.referrerPolicy = rp;
+      }
+      iframe.src =
+        "https://tracking.example.org/browser/toolkit/components/antitracking/test/browser/referrer.sjs?what=iframe";
+      await p;
+
+      p = new content.Promise(resolve => {
+        content.onmessage = event => {
+          resolve(event.data);
+        };
+      });
+      iframe.contentWindow.postMessage("ping", "*");
+      return p;
+    }
   });
 
+  is(referrer, expectedReferrer, "The correct referrer must be read from DOM");
+
   await fetch(
     "https://tracking.example.org/browser/toolkit/components/antitracking/test/browser/referrer.sjs?result&what=script"
   )
@@ -68,6 +92,14 @@ async function testOnWindowBody(win, expectedReferrer, rp) {
     .then(text => {
       is(text, expectedReferrer, "We sent the correct Referer header");
     });
+
+  await fetch(
+    "https://tracking.example.org/browser/toolkit/components/antitracking/test/browser/referrer.sjs?result&what=iframe"
+  )
+    .then(r => r.text())
+    .then(text => {
+      is(text, expectedReferrer, "We sent the correct Referer header");
+    });
 }
 
 async function closeAWindow(win) {

toolkit/components/antitracking/test/browser/browser_urlDecorationStripping.js

@@ -144,7 +144,35 @@ AntiTracking._createTask({
       ok(false, "No query parameters should be found");
     }
   },
-  extraPrefs: null,
+  extraPrefs: [["network.http.referer.defaultPolicy.trackers", 3]],
+  expectedBlockingNotifications: 0,
+  runInPrivateWindow: false,
+  iframeSandbox: null,
+  accessRemoval: null,
+  callbackAfterRemoval: null,
+  topPage: TOP_PAGE_WITHOUT_TRACKING_IDENTIFIER,
+});
+
+AntiTracking._createTask({
+  name:
+    "Test that we do not downgrade document.referrer when it does not contain a tracking identifier even though it gets downgraded to origin only due to the default referrer policy",
+  cookieBehavior: BEHAVIOR_REJECT_TRACKER,
+  blockingByContentBlockingRTUI: true,
+  allowList: false,
+  callback: async _ => {
+    let ref = new URL(document.referrer);
+    is(
+      ref.hostname,
+      "sub1.xn--hxajbheg2az3al.xn--jxalpdlp",
+      "Hostname shouldn't be stripped"
+    );
+    is(ref.pathname.length, 1, "Path must be trimmed");
+    // eslint-disable-next-line no-unused-vars
+    for (let entry of ref.searchParams.entries()) {
+      ok(false, "No query parameters should be found");
+    }
+  },
+  extraPrefs: [["network.http.referer.defaultPolicy.trackers", 2]],
   expectedBlockingNotifications: 0,
   runInPrivateWindow: false,
   iframeSandbox: null,
@@ -172,7 +200,35 @@ AntiTracking._createTask({
       ok(false, "No query parameters should be found");
     }
   },
-  extraPrefs: null,
+  extraPrefs: [["network.http.referer.defaultPolicy.trackers", 3]],
+  expectedBlockingNotifications: 0,
+  runInPrivateWindow: false,
+  iframeSandbox: null,
+  accessRemoval: null,
+  callbackAfterRemoval: null,
+  topPage: TOP_PAGE_WITH_TRACKING_IDENTIFIER,
+});
+
+AntiTracking._createTask({
+  name:
+    "Test that we don't downgrade document.referrer when it contains a tracking identifier if it gets downgraded to origin only due to the default referrer policy because the tracking identifier wouldn't be present in the referrer any more",
+  cookieBehavior: BEHAVIOR_REJECT_TRACKER,
+  blockingByContentBlockingRTUI: true,
+  allowList: false,
+  callback: async _ => {
+    let ref = new URL(document.referrer);
+    is(
+      ref.hostname,
+      "sub1.xn--hxajbheg2az3al.xn--jxalpdlp",
+      "Hostname shouldn't be stripped"
+    );
+    is(ref.pathname.length, 1, "Path must be trimmed");
+    // eslint-disable-next-line no-unused-vars
+    for (let entry of ref.searchParams.entries()) {
+      ok(false, "No query parameters should be found");
+    }
+  },
+  extraPrefs: [["network.http.referer.defaultPolicy.trackers", 2]],
   expectedBlockingNotifications: 0,
   runInPrivateWindow: false,
   iframeSandbox: null,

toolkit/components/antitracking/test/browser/referrer.sjs

@@ -3,10 +3,18 @@
 const IMAGE = atob("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABAQMAAAAl21bKAAAAA1BMVEUAA" +
                    "ACnej3aAAAAAXRSTlMAQObYZgAAAApJREFUCNdjYAAAAAIAAeIhvDMAAAAASUVORK5CYII=");
 
+const IFRAME = "<!DOCTYPE html>\n" +
+               "<script>\n" +
+                 "onmessage = event => {\n" +
+                   "parent.postMessage(document.referrer, '*');\n" +
+                 "};\n" +
+               "</script>";
+
 function handleRequest(aRequest, aResponse) {
   aResponse.setStatusLine(aRequest.httpVersion, 200);
 
-  let key = aRequest.queryString.includes("what=script") ? "script" : "image";
+  let key = aRequest.queryString.includes("what=script") ? "script" :
+            (aRequest.queryString.includes("what=image") ? "image" : "iframe");
 
   if (aRequest.queryString.includes("result")) {
     aResponse.write(getState(key));
@@ -22,8 +30,11 @@ function handleRequest(aRequest, aResponse) {
   if (key == "script") {
     aResponse.setHeader("Content-Type", "text/javascript", false);
     aResponse.write("42;");
-  } else {
+  } else if (key == "image") {
     aResponse.setHeader("Content-Type", "image/png", false);
     aResponse.write(IMAGE);
+  } else {
+    aResponse.setHeader("Content-Type", "text/html", false);
+    aResponse.write(IFRAME);
   }
 }