Bug 1720221 proxy failover to direct for system requests r=kershaw,necko-reviewers,robwu

Differential Revision: https://phabricator.services.mozilla.com/D119695

modules/libpref/init/StaticPrefList.yaml

@@ -9420,6 +9420,14 @@
   value: 4
   mirror: always
 
+# Whether to force failover to direct for system requests.
+#ifdef MOZ_PROXY_DIRECT_FAILOVER
+- name: network.proxy.failover_direct
+  type: bool
+  value: true
+  mirror: always
+#endif
+
 # Some requests during a page load are marked as "tail", mainly trackers, but not only.
 # This pref controls whether such requests are put to the tail, behind other requests
 # emerging during page loading process.

netwerk/base/nsProtocolProxyService.cpp

@@ -1673,20 +1673,26 @@ NS_IMETHODIMP
 nsProtocolProxyService::GetFailoverForProxy(nsIProxyInfo* aProxy, nsIURI* aURI,
                                             nsresult aStatus,
                                             nsIProxyInfo** aResult) {
-  // We only support failover when a PAC file is configured, either
-  // directly or via system settings
-  if (mProxyConfig != PROXYCONFIG_PAC && mProxyConfig != PROXYCONFIG_WPAD &&
-      mProxyConfig != PROXYCONFIG_SYSTEM) {
-    return NS_ERROR_NOT_AVAILABLE;
-  }
+  // Failover is supported through a variety of methods including:
+  // * PAC scripts (PROXYCONFIG_PAC and PROXYCONFIG_WPAD)
+  // * System proxy
+  // * Extensions
+  // With extensions the mProxyConfig can be any type and the extension
+  // is still involved in the proxy filtering.  It may have also supplied
+  // any number of failover proxies.  We cannot determine what the mix is
+  // here, so we will attempt to get a failover regardless of the config
+  // type.  MANUAL configuration will not disable a proxy.
 
   // Verify that |aProxy| is one of our nsProxyInfo objects.
   nsCOMPtr<nsProxyInfo> pi = do_QueryInterface(aProxy);
   NS_ENSURE_ARG(pi);
   // OK, the QI checked out.  We can proceed.
 
-  // Remember that this proxy is down.
-  DisableProxy(pi);
+  // Remember that this proxy is down.  If the user has manually configured some
+  // proxies we do not want to disable them.
+  if (mProxyConfig != PROXYCONFIG_MANUAL) {
+    DisableProxy(pi);
+  }
 
   // NOTE: At this point, we might want to prompt the user if we have
   //       not already tried going DIRECT.  This is something that the

netwerk/protocol/http/nsHttpChannel.cpp

@@ -2647,7 +2647,25 @@ nsresult nsHttpChannel::ProxyFailover() {
   nsCOMPtr<nsIProxyInfo> pi;
   rv = pps->GetFailoverForProxy(mConnectionInfo->ProxyInfo(), mURI, mStatus,
                                 getter_AddRefs(pi));
-  if (NS_FAILED(rv)) return rv;
+#ifdef MOZ_PROXY_DIRECT_FAILOVER
+  if (NS_FAILED(rv)) {
+    if (!StaticPrefs::network_proxy_failover_direct()) {
+      return rv;
+    }
+    // If this request used a failed proxy and there is no failover available,
+    // fallback to DIRECT connections for system principal requests.
+    if (mLoadInfo->GetLoadingPrincipal() &&
+        mLoadInfo->GetLoadingPrincipal()->IsSystemPrincipal()) {
+      rv = pps->NewProxyInfo("direct"_ns, ""_ns, 0, ""_ns, ""_ns, 0, UINT32_MAX,
+                             nullptr, getter_AddRefs(pi));
+    }
+#endif
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+#ifdef MOZ_PROXY_DIRECT_FAILOVER
+  }
+#endif
 
   // XXXbz so where does this codepath remove us from the loadgroup,
   // exactly?

netwerk/test/unit/test_dns_proxy_bypass.js

@@ -30,12 +30,7 @@ var dnsRequestObserver = {
 
   observe(subject, topic, data) {
     if (topic == "dns-resolution-request") {
-      info(data);
-      if (data.indexOf("dnsleak.example.com") > -1) {
-        try {
-          Assert.ok(false);
-        } catch (e) {}
-      }
+      Assert.ok(!data.includes("dnsleak.example.com"), `no dnsleak: ${data}`);
     }
   },
 };
@@ -68,15 +63,15 @@ function run_test() {
     Ci.nsIWebSocketChannel
   );
 
+  var uri = ioService.newURI(url);
   chan.initLoadInfo(
     null, // aLoadingNode
-    Services.scriptSecurityManager.getSystemPrincipal(),
+    Services.scriptSecurityManager.createContentPrincipal(uri, {}),
     null, // aTriggeringPrincipal
     Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_SEC_CONTEXT_IS_NULL,
     Ci.nsIContentPolicy.TYPE_WEBSOCKET
   );
 
-  var uri = ioService.newURI(url);
   chan.asyncOpen(uri, url, 0, listener, null);
   do_test_pending();
 }

netwerk/test/unit/test_file_protocol.js

@@ -39,10 +39,14 @@ function new_file_input_stream(file, buffered) {
 }
 
 function new_file_channel(file) {
+  var ssm = Services.scriptSecurityManager;
   var ios = Cc["@mozilla.org/network/io-service;1"].getService(Ci.nsIIOService);
+  let uri = ios.newFileURI(file);
   return NetUtil.newChannel({
-    uri: ios.newFileURI(file),
-    loadUsingSystemPrincipal: true,
+    uri,
+    loadingPrincipal: ssm.createContentPrincipal(uri, {}),
+    securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_DOCUMENT,
   });
 }
 

netwerk/test/unit/test_http2-proxy.js

@@ -108,10 +108,20 @@ class AuthRequestor {
   }
 }
 
+function createPrincipal(url) {
+  var ssm = Services.scriptSecurityManager;
+  try {
+    return ssm.createContentPrincipal(Services.io.newURI(url), {});
+  } catch (e) {
+    return null;
+  }
+}
+
 function make_channel(url) {
   return NetUtil.newChannel({
     uri: url,
-    loadUsingSystemPrincipal: true,
+    loadingPrincipal: createPrincipal(url),
+    securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT,
     // Using TYPE_DOCUMENT for the authentication dialog test, it'd be blocked for other types
     contentPolicyType: Ci.nsIContentPolicy.TYPE_DOCUMENT,
   });

netwerk/test/unit/test_http3_alt_svc.js

@@ -52,10 +52,21 @@ function run_test() {
   run_next_test();
 }
 
+function createPrincipal(url) {
+  var ssm = Services.scriptSecurityManager;
+  try {
+    return ssm.createContentPrincipal(Services.io.newURI(url), {});
+  } catch (e) {
+    return null;
+  }
+}
+
 function makeChan(uri) {
   let chan = NetUtil.newChannel({
     uri,
-    loadUsingSystemPrincipal: true,
+    loadingPrincipal: createPrincipal(uri),
+    securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_DOCUMENT,
   }).QueryInterface(Ci.nsIHttpChannel);
   chan.loadFlags = Ci.nsIChannel.LOAD_INITIAL_DOCUMENT_URI;
   return chan;

netwerk/test/unit/test_protocolproxyservice.js

@@ -938,10 +938,15 @@ function failed_script_callback(pi) {
   obs = obs.QueryInterface(Ci.nsIObserverService);
   obs.addObserver(directFilterListener, "http-on-modify-request");
 
+  var ssm = Services.scriptSecurityManager;
+  let uri = "http://127.0.0.1:7247";
   var chan = NetUtil.newChannel({
-    uri: "http://127.0.0.1:7247",
-    loadUsingSystemPrincipal: true,
+    uri,
+    loadingPrincipal: ssm.createContentPrincipal(Services.io.newURI(uri), {}),
+    securityFlags: Ci.nsILoadInfo.SEC_ALLOW_CROSS_ORIGIN_INHERITS_SEC_CONTEXT,
+    contentPolicyType: Ci.nsIContentPolicy.TYPE_DOCUMENT,
   });
+
   chan.asyncOpen(directFilterListener);
 }
 

toolkit/components/extensions/test/xpcshell/test_proxy_failover.js

@@ -0,0 +1,245 @@
+"use strict";
+
+AddonTestUtils.init(this);
+AddonTestUtils.overrideCertDB();
+AddonTestUtils.createAppInfo(
+  "xpcshell@tests.mozilla.org",
+  "XPCShell",
+  "1",
+  "43"
+);
+
+// Necessary for the pac script to proxy localhost requests
+Services.prefs.setBoolPref("network.proxy.allow_hijacking_localhost", true);
+
+// Pref is not builtin if direct failover is disabled in compile config.
+XPCOMUtils.defineLazyGetter(this, "directFailoverDisabled", () => {
+  return (
+    Services.prefs.getPrefType("network.proxy.failover_direct") ==
+    Ci.nsIPrefBranch.PREF_INVALID
+  );
+});
+
+// Prevent the request from reaching out to the network.
+const { HttpServer } = ChromeUtils.import("resource://testing-common/httpd.js");
+
+// No hosts defined to avoid the default proxy filter setup.
+const nonProxiedServer = createHttpServer();
+nonProxiedServer.registerPathHandler("/", (request, response) => {
+  response.setStatusLine(request.httpVersion, 200, "OK");
+  response.write("ok!");
+});
+const { primaryHost, primaryPort } = nonProxiedServer.identity;
+
+// Get a free port with no listener to use in the proxyinfo.
+function getBadProxyPort() {
+  let server = new HttpServer();
+  server.start(-1);
+  const badPort = server.identity.primaryPort;
+  server.stop();
+  return badPort;
+}
+
+function xhr(url) {
+  return new Promise((resolve, reject) => {
+    let req = new XMLHttpRequest({ mozSystem: true });
+    req.open("GET", `${url}?t=${Math.random()}`);
+    req.onload = () => {
+      resolve(req.responseText);
+    };
+    req.onerror = () => {
+      reject(req.status);
+    };
+    req.send();
+  });
+}
+
+add_task(async function setup() {
+  await AddonTestUtils.promiseStartupManager();
+});
+
+async function getProxyExtension(proxyDetails) {
+  async function background(proxyDetails) {
+    browser.proxy.onRequest.addListener(
+      details => {
+        return proxyDetails;
+      },
+      { urls: ["<all_urls>"] }
+    );
+
+    browser.test.sendMessage("proxied");
+  }
+  let extensionData = {
+    manifest: {
+      permissions: ["proxy", "<all_urls>"],
+    },
+    background: `(${background})(${JSON.stringify(proxyDetails)})`,
+    incognitoOverride: "spanning",
+    useAddonManager: "temporary",
+  };
+
+  let extension = ExtensionTestUtils.loadExtension(extensionData);
+  await extension.startup();
+  await extension.awaitMessage("proxied");
+  return extension;
+}
+
+add_task(async function test_failover_content_direct() {
+  // load a content page for fetch and non-system principal, expect
+  // failover to direct will work.
+  const proxyDetails = [
+    { type: "http", host: "127.0.0.1", port: getBadProxyPort() },
+    { type: "direct" },
+  ];
+
+  // We need to load the content page before loading the proxy extension
+  // to ensure that we have a valid content page to run fetch from.
+  let contentUrl = `http://${primaryHost}:${primaryPort}/`;
+  let page = await ExtensionTestUtils.loadContentPage(contentUrl);
+
+  let extension = await getProxyExtension(proxyDetails);
+
+  await ExtensionTestUtils.fetch(contentUrl, `${contentUrl}?t=${Math.random()}`)
+    .then(text => {
+      equal(text, "ok!", "fetch completed");
+    })
+    .catch(() => {
+      ok(false, "fetch failed");
+    });
+
+  await extension.unload();
+  await page.close();
+});
+
+add_task(
+  { skip_if: () => directFailoverDisabled },
+  async function test_failover_content() {
+    // load a content page for fetch and non-system principal, expect
+    // no failover
+    const proxyDetails = [
+      { type: "http", host: "127.0.0.1", port: getBadProxyPort() },
+    ];
+
+    // We need to load the content page before loading the proxy extension
+    // to ensure that we have a valid content page to run fetch from.
+    let contentUrl = `http://${primaryHost}:${primaryPort}/`;
+    let page = await ExtensionTestUtils.loadContentPage(contentUrl);
+
+    let extension = await getProxyExtension(proxyDetails);
+
+    await ExtensionTestUtils.fetch(
+      contentUrl,
+      `${contentUrl}?t=${Math.random()}`
+    )
+      .then(text => {
+        ok(false, "xhr unexpectedly completed");
+      })
+      .catch(e => {
+        equal(
+          e.message,
+          "NetworkError when attempting to fetch resource.",
+          "fetch failed"
+        );
+      });
+
+    await extension.unload();
+    await page.close();
+  }
+);
+
+add_task(
+  { skip_if: () => directFailoverDisabled },
+  async function test_failover_system() {
+    const proxyDetails = [
+      { type: "http", host: "127.0.0.1", port: getBadProxyPort() },
+      { type: "http", host: "127.0.0.1", port: getBadProxyPort() },
+    ];
+
+    let extension = await getProxyExtension(proxyDetails);
+
+    await xhr(`http://${primaryHost}:${primaryPort}/`)
+      .then(text => {
+        equal(text, "ok!", "xhr completed");
+      })
+      .catch(() => {
+        ok(false, "xhr failed");
+      });
+
+    await extension.unload();
+  }
+);
+
+add_task(
+  {
+    skip_if: () =>
+      AppConstants.platform === "android" || directFailoverDisabled,
+  },
+  async function test_failover_pac() {
+    const badPort = getBadProxyPort();
+
+    async function background(badPort) {
+      let pac = `function FindProxyForURL(url, host) { return "PROXY 127.0.0.1:${badPort}"; }`;
+      let proxySettings = {
+        proxyType: "autoConfig",
+        autoConfigUrl: `data:application/x-ns-proxy-autoconfig;charset=utf-8,${encodeURIComponent(
+          pac
+        )}`,
+      };
+
+      await browser.proxy.settings.set({ value: proxySettings });
+      browser.test.sendMessage("proxied");
+    }
+    let extensionData = {
+      manifest: {
+        permissions: ["proxy", "<all_urls>"],
+      },
+      background: `(${background})(${badPort})`,
+      incognitoOverride: "spanning",
+      useAddonManager: "temporary",
+    };
+
+    let extension = ExtensionTestUtils.loadExtension(extensionData);
+    await extension.startup();
+    await extension.awaitMessage("proxied");
+    equal(
+      Services.prefs.getIntPref("network.proxy.type"),
+      2,
+      "autoconfig type set"
+    );
+    ok(
+      Services.prefs.getStringPref("network.proxy.autoconfig_url"),
+      "autoconfig url set"
+    );
+
+    await xhr(`http://${primaryHost}:${primaryPort}/`)
+      .then(text => {
+        equal(text, "ok!", "xhr completed");
+      })
+      .catch(() => {
+        ok(false, "xhr failed");
+      });
+
+    await extension.unload();
+  }
+);
+
+add_task(async function test_failover_system_off() {
+  // Test test failover failures, uncomment and set pref to false
+  Services.prefs.setBoolPref("network.proxy.failover_direct", false);
+
+  const proxyDetails = [
+    { type: "http", host: "127.0.0.1", port: getBadProxyPort() },
+  ];
+
+  let extension = await getProxyExtension(proxyDetails);
+
+  await xhr(`http://${primaryHost}:${primaryPort}/`)
+    .then(text => {
+      ok(false, "xhr completed");
+    })
+    .catch(status => {
+      equal(status, 0, "xhr failed");
+    });
+
+  await extension.unload();
+});

toolkit/components/extensions/test/xpcshell/xpcshell-common.ini

@@ -267,6 +267,7 @@ skip-if = appname == "thunderbird" || os == "android" # Bug 1350559
 skip-if = appname == "thunderbird" || os == "android" # Bug 1350559
 [test_ext_permissions_uninstall.js]
 skip-if = appname == "thunderbird" || os == "android" # Bug 1350559
+[test_proxy_failover.js]
 [test_proxy_listener.js]
 skip-if = appname == "thunderbird"
 [test_proxy_incognito.js]

toolkit/moz.configure

@@ -1290,6 +1290,24 @@ def proxy_bypass_protection(_):
 set_config("MOZ_PROXY_BYPASS_PROTECTION", proxy_bypass_protection)
 set_define("MOZ_PROXY_BYPASS_PROTECTION", proxy_bypass_protection)
 
+# Proxy direct failover
+# ==============================================================
+
+option(
+    "--disable-proxy-direct-failover",
+    help="Disable direct failover for system requests",
+)
+
+
+@depends_if("--disable-proxy-direct-failover")
+def proxy_direct_failover(value):
+    if value:
+        return True
+
+
+set_config("MOZ_PROXY_DIRECT_FAILOVER", proxy_direct_failover)
+set_define("MOZ_PROXY_DIRECT_FAILOVER", proxy_direct_failover)
+
 # MIDL
 # ==============================================================