Bug 1878506 - Add pref to make native HTTPS resolution exit early in automation r=necko-reviewers,kershaw

The purpose of this pref is to avoid breaking any tests that don't expect a
HTTPS record to be present. For example, if you're loading http://domain.com
in a test, and don't expect a HTTPS upgrade to happen, if that domain suddenly
adds a HTTPS record we might end up upgrading to HTTPS even in automation.

This pref does an early return with NS_ERROR_UNKNOWN_HOST just before doing
a call to the system API. This means we're still waiting in the DNS queue to
resolve the domain (simulating the same waiting times we might see
on users' computers), but we never actually use native DNS to resolve a HTTPS
record in automation.

Differential Revision: https://phabricator.services.mozilla.com/D200785

modules/libpref/init/StaticPrefList.yaml

@@ -11904,6 +11904,17 @@
   value: false
   mirror: always
 
+# When true, the HTTPS query will actually call the native
+# platform API. When false it will return before the call
+# to the platform API
+# This pref is necessary because having a HTTPS record
+# could cause channels to connect to a different port,
+# which is not desirable in automation.
+- name: network.dns.native_https_query_in_automation
+  type: RelaxedAtomicBool
+  value: false
+  mirror: always
+
 # When resolving a native HTTPS query with native APIs
 # the Android implementation has a max timeout
 - name: network.dns.native_https_timeout_android

netwerk/dns/PlatformDNSAndroid.cpp

@@ -43,6 +43,11 @@ nsresult ResolveHTTPSRecordImpl(const nsACString& aHost, uint16_t aFlags,
   nsAutoCString cname;
   nsresult rv;
 
+  if (xpc::IsInAutomation() &&
+      !StaticPrefs::network_dns_native_https_query_in_automation()) {
+    return NS_ERROR_UNKNOWN_HOST;
+  }
+
   if (!sLibLoading.exchange(true)) {
     // We're the first call here, load the library and symbols.
     void* handle = dlopen("libandroid.so", RTLD_LAZY | RTLD_LOCAL);

netwerk/dns/PlatformDNSUnix.cpp

@@ -8,6 +8,7 @@
 #include "mozilla/net/DNSPacket.h"
 #include "nsIDNSService.h"
 #include "mozilla/Maybe.h"
+#include "mozilla/StaticPrefs_network.h"
 
 #include <stdio.h>
 #include <stdlib.h>
@@ -27,6 +28,11 @@ nsresult ResolveHTTPSRecordImpl(const nsACString& aHost, uint16_t aFlags,
   nsAutoCString cname;
   nsresult rv;
 
+  if (xpc::IsInAutomation() &&
+      !StaticPrefs::network_dns_native_https_query_in_automation()) {
+    return NS_ERROR_UNKNOWN_HOST;
+  }
+
   LOG("resolving %s\n", host.get());
   // Perform the query
   rv = packet.FillBuffer(

netwerk/dns/PlatformDNSWin.cpp

@@ -9,6 +9,7 @@
 #include "nsIDNSService.h"
 #include "mozilla/Maybe.h"
 #include "mozilla/ScopeExit.h"
+#include "mozilla/StaticPrefs_network.h"
 
 #ifdef DNSQUERY_AVAILABLE
 // There is a bug in windns.h where the type of parameter ppQueryResultsSet for
@@ -33,6 +34,11 @@ nsresult ResolveHTTPSRecordImpl(const nsACString& aHost, uint16_t aFlags,
   nsAutoCString cname;
   aTTL = UINT32_MAX;
 
+  if (xpc::IsInAutomation() &&
+      !StaticPrefs::network_dns_native_https_query_in_automation()) {
+    return NS_ERROR_UNKNOWN_HOST;
+  }
+
   DNS_STATUS status =
       DnsQuery_A(host.get(), nsIDNSService::RESOLVE_TYPE_HTTPSSVC,
                  DNS_QUERY_STANDARD, nullptr, &result, nullptr);