Bug 1613790 - Implement unit tests for TRRPerformance module. r=johannh,dragana,valentin

Depends on D62845

Differential Revision: https://phabricator.services.mozilla.com/D62981

--HG--
extra : moz-landing-system : lando

browser/components/doh/moz.build

@@ -11,4 +11,4 @@ EXTRA_JS_MODULES += [
   'TRRPerformance.jsm',
 ]
 
-
+XPCSHELL_TESTS_MANIFESTS += ['test/unit/xpcshell.ini']

browser/components/doh/test/unit/head.js

@@ -0,0 +1,105 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+"use strict";
+
+const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
+const { NetUtil } = ChromeUtils.import("resource://gre/modules/NetUtil.jsm");
+const { PromiseUtils } = ChromeUtils.import(
+  "resource://gre/modules/PromiseUtils.jsm"
+);
+
+const { TestUtils } = ChromeUtils.import(
+  "resource://testing-common/TestUtils.jsm"
+);
+
+let h2Port, trrServer1, trrServer2;
+const { DNSLookup, LookupAggregator, TRRRacer } = ChromeUtils.import(
+  "resource:///modules/TRRPerformance.jsm"
+);
+
+function readFile(file) {
+  let fstream = Cc["@mozilla.org/network/file-input-stream;1"].createInstance(
+    Ci.nsIFileInputStream
+  );
+  fstream.init(file, -1, 0, 0);
+  let data = NetUtil.readInputStreamToString(fstream, fstream.available());
+  fstream.close();
+  return data;
+}
+
+function addCertFromFile(certdb, filename, trustString) {
+  let certFile = do_get_file(filename, false);
+  let pem = readFile(certFile)
+    .replace(/-----BEGIN CERTIFICATE-----/, "")
+    .replace(/-----END CERTIFICATE-----/, "")
+    .replace(/[\r\n]/g, "");
+  certdb.addCertFromBase64(pem, trustString);
+}
+
+function ensureNoTelemetry() {
+  let events =
+    Services.telemetry.snapshotEvents(
+      Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+      true
+    ).parent || [];
+  events = events.filter(e => e[1] == "security.doh.trrPerformance");
+  Assert.ok(!events.length);
+}
+
+function setup() {
+  let env = Cc["@mozilla.org/process/environment;1"].getService(
+    Ci.nsIEnvironment
+  );
+  h2Port = env.get("MOZHTTP2_PORT");
+  Assert.notEqual(h2Port, null);
+  Assert.notEqual(h2Port, "");
+
+  // Set to allow the cert presented by our H2 server
+  do_get_profile();
+
+  Services.prefs.setBoolPref("network.http.spdy.enabled", true);
+  Services.prefs.setBoolPref("network.http.spdy.enabled.http2", true);
+
+  // use the h2 server as DOH provider
+  trrServer1 = `https://foo.example.com:${h2Port}/doh?responseIP=1.1.1.1`;
+  trrServer2 = `https://foo.example.com:${h2Port}/doh?responseIP=2.2.2.2`;
+  // make all native resolve calls "secretly" resolve localhost instead
+  Services.prefs.setBoolPref("network.dns.native-is-localhost", true);
+
+  // The moz-http2 cert is for foo.example.com and is signed by http2-ca.pem
+  // so add that cert to the trust list as a signing cert.  // the foo.example.com domain name.
+  let certdb = Cc["@mozilla.org/security/x509certdb;1"].getService(
+    Ci.nsIX509CertDB
+  );
+  addCertFromFile(certdb, "http2-ca.pem", "CTu,u,u");
+
+  Services.prefs.setCharPref(
+    "doh-rollout.trrRace.trrList",
+    `${trrServer1}, ${trrServer2}`
+  );
+
+  Services.prefs.setIntPref("doh-rollout.trrRace.randomSubdomainCount", 2);
+
+  Services.prefs.setCharPref(
+    "doh-rollout.trrRace.popularDomains",
+    "foo.example.com, bar.example.com"
+  );
+
+  Services.prefs.setCharPref(
+    "doh-rollout.trrRace.canonicalDomain",
+    "firefox-dns-perf-test.net"
+  );
+
+  let oldCanRecord = Services.telemetry.canRecordExtended;
+  Services.telemetry.canRecordExtended = true;
+
+  registerCleanupFunction(() => {
+    Services.prefs.clearUserPref("network.http.spdy.enabled");
+    Services.prefs.clearUserPref("network.http.spdy.enabled.http2");
+    Services.prefs.clearUserPref("network.dns.native-is-localhost");
+
+    Services.telemetry.canRecordExtended = oldCanRecord;
+  });
+}

browser/components/doh/test/unit/test_DNSLookup.js

@@ -0,0 +1,62 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+"use strict";
+
+add_task(setup);
+
+add_task(async function test_SuccessfulRandomDNSLookup() {
+  let deferred = PromiseUtils.defer();
+  let lookup = new DNSLookup(
+    null,
+    trrServer1,
+    (request, record, status, usedDomain, retryCount) => {
+      deferred.resolve({ request, record, status, usedDomain, retryCount });
+    }
+  );
+  lookup.doLookup();
+  let result = await deferred.promise;
+  Assert.ok(result.usedDomain.endsWith(".firefox-dns-perf-test.net"));
+  Assert.equal(result.status, Cr.NS_OK);
+  Assert.ok(result.record);
+  Assert.ok(result.record.IsTRR());
+  Assert.greater(result.record.trrFetchDuration, 0);
+  Assert.equal(result.retryCount, 1);
+});
+
+add_task(async function test_SuccessfulSpecifiedDNSLookup() {
+  let deferred = PromiseUtils.defer();
+  let lookup = new DNSLookup(
+    "foo.example.com",
+    trrServer1,
+    (request, record, status, usedDomain, retryCount) => {
+      deferred.resolve({ request, record, status, usedDomain, retryCount });
+    }
+  );
+  lookup.doLookup();
+  let result = await deferred.promise;
+  Assert.equal(result.usedDomain, "foo.example.com");
+  Assert.equal(result.status, Cr.NS_OK);
+  Assert.ok(result.record);
+  Assert.ok(result.record.IsTRR());
+  Assert.greater(result.record.trrFetchDuration, 0);
+  Assert.equal(result.retryCount, 1);
+});
+
+add_task(async function test_FailedDNSLookup() {
+  let deferred = PromiseUtils.defer();
+  let lookup = new DNSLookup(
+    null,
+    `https://foo.example.com:${h2Port}/doh?responseIP=none`,
+    (request, record, status, usedDomain, retryCount) => {
+      deferred.resolve({ request, record, status, usedDomain, retryCount });
+    }
+  );
+  lookup.doLookup();
+  let result = await deferred.promise;
+  Assert.ok(result.usedDomain.endsWith(".firefox-dns-perf-test.net"));
+  Assert.notEqual(result.status, Cr.NS_OK);
+  Assert.equal(result.record, null);
+  Assert.equal(result.retryCount, 3);
+});

browser/components/doh/test/unit/test_LookupAggregator.js

@@ -0,0 +1,160 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+"use strict";
+
+const { setTimeout } = ChromeUtils.import("resource://gre/modules/Timer.jsm");
+
+add_task(setup);
+
+async function helper_SuccessfulLookupAggregator(
+  networkUnstable = false,
+  captivePortal = false
+) {
+  let deferred = PromiseUtils.defer();
+  let aggregator = new LookupAggregator(() => deferred.resolve());
+  // The aggregator's domain list should correctly reflect our set
+  // prefs for number of random subdomains (2) and the list of
+  // popular domains.
+  Assert.equal(aggregator.domains[0], null);
+  Assert.equal(aggregator.domains[1], null);
+  Assert.equal(aggregator.domains[2], "foo.example.com");
+  Assert.equal(aggregator.domains[3], "bar.example.com");
+  Assert.equal(aggregator.totalLookups, 8); // 2 TRRs * 4 domains.
+
+  if (networkUnstable) {
+    aggregator.markUnstableNetwork();
+  }
+  if (captivePortal) {
+    aggregator.markCaptivePortal();
+  }
+  aggregator.run();
+  await deferred.promise;
+  Assert.ok(!aggregator.aborted);
+  Assert.equal(aggregator.networkUnstable, networkUnstable);
+  Assert.equal(aggregator.captivePortal, captivePortal);
+  Assert.equal(aggregator.results.length, aggregator.totalLookups);
+
+  let events = Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    true
+  ).parent;
+  Assert.ok(events);
+  events = events.filter(e => e[1] == "security.doh.trrPerformance");
+  Assert.equal(events.length, aggregator.totalLookups);
+
+  for (let event of events) {
+    info(JSON.stringify(event));
+    Assert.equal(event[1], "security.doh.trrPerformance");
+    Assert.equal(event[2], "resolved");
+    Assert.equal(event[3], "record");
+    Assert.equal(event[4], "success");
+  }
+
+  // We only need to check the payload of each event from here on.
+  events = events.map(e => e[5]);
+
+  for (let trr of [trrServer1, trrServer2]) {
+    // There should be two results for random subdomains.
+    let results = aggregator.results.filter(r => {
+      return r.trr == trr && r.domain.endsWith(".firefox-dns-perf-test.net");
+    });
+    Assert.equal(results.length, 2);
+
+    for (let result of results) {
+      Assert.ok(result.domain.endsWith(".firefox-dns-perf-test.net"));
+      Assert.equal(result.trr, trr);
+      Assert.ok(Components.isSuccessCode(result.status));
+      Assert.greater(result.time, 0);
+      Assert.equal(result.retryCount, 1);
+
+      let matchingEvents = events.filter(
+        e => e.domain == result.domain && e.trr == result.trr
+      );
+      Assert.equal(matchingEvents.length, 1);
+      let e = matchingEvents.pop();
+      for (let key of Object.keys(result)) {
+        Assert.equal(e[key], result[key].toString());
+      }
+      Assert.equal(e.networkUnstable, networkUnstable.toString());
+      Assert.equal(e.captivePortal, captivePortal.toString());
+    }
+
+    // There should be two results for the popular domains.
+    results = aggregator.results.filter(r => {
+      return r.trr == trr && !r.domain.endsWith(".firefox-dns-perf-test.net");
+    });
+    Assert.equal(results.length, 2);
+
+    Assert.ok(
+      [results[0].domain, results[1].domain].includes("foo.example.com")
+    );
+    Assert.ok(
+      [results[0].domain, results[1].domain].includes("bar.example.com")
+    );
+    for (let result of results) {
+      Assert.equal(result.trr, trr);
+      Assert.equal(result.status, Cr.NS_OK);
+      Assert.greater(result.time, 0);
+      Assert.equal(result.retryCount, 1);
+
+      let matchingEvents = events.filter(
+        e => e.domain == result.domain && e.trr == result.trr
+      );
+      Assert.equal(matchingEvents.length, 1);
+      let e = matchingEvents.pop();
+      for (let key of Object.keys(result)) {
+        Assert.equal(e[key], result[key].toString());
+      }
+      Assert.equal(e.networkUnstable, networkUnstable.toString());
+      Assert.equal(e.captivePortal, captivePortal.toString());
+    }
+  }
+
+  Services.telemetry.clearEvents();
+}
+
+add_task(async function test_SuccessfulLookupAggregator() {
+  await helper_SuccessfulLookupAggregator(false, false);
+  await helper_SuccessfulLookupAggregator(false, true);
+  await helper_SuccessfulLookupAggregator(true, false);
+  await helper_SuccessfulLookupAggregator(true, true);
+});
+
+add_task(async function test_AbortedLookupAggregator() {
+  let deferred = PromiseUtils.defer();
+  let aggregator = new LookupAggregator(() => deferred.resolve());
+  // The aggregator's domain list should correctly reflect our set
+  // prefs for number of random subdomains (2) and the list of
+  // popular domains.
+  Assert.equal(aggregator.domains[0], null);
+  Assert.equal(aggregator.domains[1], null);
+  Assert.equal(aggregator.domains[2], "foo.example.com");
+  Assert.equal(aggregator.domains[3], "bar.example.com");
+  Assert.equal(aggregator.totalLookups, 8); // 2 TRRs * 4 domains.
+
+  // The aggregator should never call the onComplete callback. To test
+  // this, race the deferred promise with a 3 second timeout. The timeout
+  // should win, since the deferred promise should never resolve.
+  let timeoutPromise = new Promise(resolve => {
+    // eslint-disable-next-line mozilla/no-arbitrary-setTimeout
+    setTimeout(() => resolve("timeout"), 3000);
+  });
+  aggregator.run();
+  aggregator.abort();
+  let winner = await Promise.race([deferred.promise, timeoutPromise]);
+  Assert.equal(winner, "timeout");
+  Assert.ok(aggregator.aborted);
+  Assert.ok(!aggregator.networkUnstable);
+  Assert.ok(!aggregator.captivePortal);
+
+  // Ensure we send no telemetry for an aborted run!
+  let events = Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    true
+  ).parent;
+  Assert.ok(
+    !events || !events.filter(e => e[1] == "security.doh.trrPerformance").length
+  );
+});

browser/components/doh/test/unit/test_TRRRacer.js

@@ -0,0 +1,166 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+"use strict";
+
+add_task(setup);
+
+add_task(async function test_TRRRacer_cleanRun() {
+  let racer = new TRRRacer();
+  racer.run();
+
+  await TestUtils.waitForCondition(() => {
+    return Services.prefs.getBoolPref("doh-rollout.trrRace.complete", false);
+  });
+  Assert.equal(racer._retryCount, 1);
+
+  let events = Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    true
+  ).parent;
+  Assert.ok(events);
+  events = events.filter(e => e[1] == "security.doh.trrPerformance");
+  Assert.equal(events.length, racer._aggregator.totalLookups);
+
+  Services.telemetry.clearEvents();
+
+  // Simulate network changes and ensure no re-runs since it's already complete.
+  async function testNetworkChange(captivePortal = false) {
+    if (captivePortal) {
+      Services.obs.notifyObservers(null, "captive-portal-login");
+    } else {
+      Services.obs.notifyObservers(null, "network:link-status-changed", "down");
+    }
+
+    Assert.ok(!racer._aggregator.aborted);
+
+    if (captivePortal) {
+      Services.obs.notifyObservers(null, "captive-portal-login-success");
+    } else {
+      Services.obs.notifyObservers(null, "network:link-status-changed", "up");
+    }
+
+    Assert.equal(racer._retryCount, 1);
+    ensureNoTelemetry();
+
+    if (captivePortal) {
+      Services.obs.notifyObservers(null, "captive-portal-login-abort");
+    }
+  }
+
+  testNetworkChange(false);
+  testNetworkChange(true);
+
+  Services.prefs.clearUserPref("doh-rollout.trrRace.complete");
+});
+
+async function test_TRRRacer_networkFlux_helper(captivePortal = false) {
+  let racer = new TRRRacer();
+  racer.run();
+
+  if (captivePortal) {
+    Services.obs.notifyObservers(null, "captive-portal-login");
+  } else {
+    Services.obs.notifyObservers(null, "network:link-status-changed", "down");
+  }
+
+  Assert.ok(racer._aggregator.aborted);
+  ensureNoTelemetry();
+  Assert.equal(racer._retryCount, 1);
+  Assert.ok(!Services.prefs.getBoolPref("doh-rollout.trrRace.complete", false));
+
+  if (captivePortal) {
+    Services.obs.notifyObservers(null, "captive-portal-login-success");
+  } else {
+    Services.obs.notifyObservers(null, "network:link-status-changed", "up");
+  }
+
+  Assert.ok(!racer._aggregator.aborted);
+  await TestUtils.waitForCondition(() => {
+    return Services.prefs.getBoolPref("doh-rollout.trrRace.complete", false);
+  });
+
+  Assert.equal(racer._retryCount, 2);
+
+  let events = Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    true
+  ).parent;
+  Assert.ok(events);
+  events = events.filter(e => e[1] == "security.doh.trrPerformance");
+  Assert.equal(events.length, racer._aggregator.totalLookups);
+
+  Services.telemetry.clearEvents();
+  Services.prefs.clearUserPref("doh-rollout.trrRace.complete");
+  if (captivePortal) {
+    Services.obs.notifyObservers(null, "captive-portal-login-abort");
+  }
+}
+
+add_task(async function test_TRRRacer_networkFlux() {
+  await test_TRRRacer_networkFlux_helper(false);
+  await test_TRRRacer_networkFlux_helper(true);
+});
+
+async function test_TRRRacer_maxRetries_helper(captivePortal = false) {
+  let racer = new TRRRacer();
+  racer.run();
+  info("ran new racer");
+  // Start at i = 1 since we're already at retry #1.
+  for (let i = 1; i < 5; ++i) {
+    if (captivePortal) {
+      Services.obs.notifyObservers(null, "captive-portal-login");
+    } else {
+      Services.obs.notifyObservers(null, "network:link-status-changed", "down");
+    }
+
+    info("notified observers");
+
+    Assert.ok(racer._aggregator.aborted);
+    ensureNoTelemetry();
+    Assert.equal(racer._retryCount, i);
+
+    Assert.ok(
+      !Services.prefs.getBoolPref("doh-rollout.trrRace.complete", false)
+    );
+
+    if (captivePortal) {
+      Services.obs.notifyObservers(null, "captive-portal-login-success");
+    } else {
+      Services.obs.notifyObservers(null, "network:link-status-changed", "up");
+    }
+  }
+
+  // Simulate a "down" network event and ensure we still send telemetry
+  // since we've maxed out our retry count.
+  if (captivePortal) {
+    Services.obs.notifyObservers(null, "captive-portal-login");
+  } else {
+    Services.obs.notifyObservers(null, "network:link-status-changed", "down");
+  }
+  Assert.ok(!racer._aggregator.aborted);
+  await TestUtils.waitForCondition(() => {
+    return Services.prefs.getBoolPref("doh-rollout.trrRace.complete", false);
+  });
+  Assert.equal(racer._retryCount, 5);
+
+  let events = Services.telemetry.snapshotEvents(
+    Ci.nsITelemetry.DATASET_PRERELEASE_CHANNELS,
+    true
+  ).parent;
+  Assert.ok(events);
+  events = events.filter(e => e[1] == "security.doh.trrPerformance");
+  Assert.equal(events.length, racer._aggregator.totalLookups);
+
+  Services.telemetry.clearEvents();
+  Services.prefs.clearUserPref("doh-rollout.trrRace.complete");
+  if (captivePortal) {
+    Services.obs.notifyObservers(null, "captive-portal-login-abort");
+  }
+}
+
+add_task(async function test_TRRRacer_maxRetries() {
+  await test_TRRRacer_maxRetries_helper(false);
+  await test_TRRRacer_maxRetries_helper(true);
+});

browser/components/doh/test/unit/xpcshell.ini

@@ -0,0 +1,10 @@
+[DEFAULT]
+head = head.js
+firefox-appdir = browser
+support-files =
+  ../../../../../netwerk/test/unit/http2-ca.pem
+
+[test_DNSLookup.js]
+skip-if = debug # Bug 1617845
+[test_LookupAggregator.js]
+[test_TRRRacer.js]