Bug 1556789 minimize the chaos of search config testing r=daleharvey

700K extension reloads!  This minimizes tests to using the
necessary extensions as well as avoiding reloads of extensions while
the search config tests are run.  Local machine can now run a single
search config test file in under 2 minutes.  This was impossible before.

Differential Revision: https://phabricator.services.mozilla.com/D35104

--HG--
extra : moz-landing-system : lando

toolkit/components/search/SearchUtils.jsm

@@ -196,8 +196,12 @@ var SearchUtils = {
  */
 const SearchExtensionLoader = {
   _promises: new Map(),
-  // strict is used in tests.
+  // strict is used in tests, causes load errors to reject.
   _strict: false,
+  // chaos mode is used in search config tests.  It bypasses
+  // reloading extensions, otherwise over the course of these
+  // tests we do over 700K reloads of extensions.
+  _chaosMode: false,
 
   /**
    * Creates a deferred promise for an extension install.
@@ -281,6 +285,15 @@ const SearchExtensionLoader = {
       SearchUtils.log(
         `SearchExtensionLoader.installAddons: installing ${id} at ${path}`
       );
+      if (this._chaosMode) {
+        // If the extension is already loaded, we do not reload the extension.  Instead
+        // we call back to search service directly.
+        let policy = WebExtensionPolicy.getByID(id);
+        if (policy) {
+          Services.search.addEnginesFromExtension(policy.extension);
+          continue;
+        }
+      }
       // The AddonManager will install the engine asynchronously
       AddonManager.installBuiltinAddon(path).catch(error => {
         // Catch any install errors and propogate.

toolkit/components/search/tests/xpcshell/searchconfigs/head_searchconfig.js

@@ -3,6 +3,8 @@
 
 "use strict";
 
+Cu.importGlobalProperties(["fetch"]);
+
 const { XPCOMUtils } = ChromeUtils.import(
   "resource://gre/modules/XPCOMUtils.jsm"
 );
@@ -10,7 +12,9 @@ XPCOMUtils.defineLazyModuleGetters(this, {
   AddonTestUtils: "resource://testing-common/AddonTestUtils.jsm",
   ObjectUtils: "resource://gre/modules/ObjectUtils.jsm",
   OS: "resource://gre/modules/osfile.jsm",
+  SearchExtensionLoader: "resource://gre/modules/SearchUtils.jsm",
   SearchTestUtils: "resource://testing-common/SearchTestUtils.jsm",
+  SearchUtils: "resource://gre/modules/SearchUtils.jsm",
   Services: "resource://gre/modules/Services.jsm",
 });
 
@@ -26,6 +30,21 @@ const SUBMISSION_PURPOSES = [
   "newtab",
 ];
 
+const ORIG_LIST_JSON_URL = SearchUtils.LIST_JSON_URL;
+SearchExtensionLoader._chaosMode = true;
+
+function traverse(obj, fun) {
+  for (var i in obj) {
+    let r = fun.apply(this, [i, obj[i]]);
+    if (r !== undefined) {
+      obj[i] = r;
+    }
+    if (obj[i] !== null && typeof obj[i] == "object") {
+      traverse(obj[i], fun);
+    }
+  }
+}
+
 /**
  * This class implements the test harness for search configuration tests.
  * These tests are designed to ensure that the correct search engines are
@@ -81,10 +100,45 @@ class SearchConfigTest {
     this._testDebug = false;
   }
 
+  // Search init loads a bunch of extensions which can take a lot of time.
+  // This reduces that down to any default engines, and the engine that is
+  // specifically being tested.
+  async setupListJSON() {
+    let origListJSON = await fetch(ORIG_LIST_JSON_URL).then(req => req.json());
+    let target = this._config.identifier;
+    let listJSON = Object.assign({}, origListJSON);
+    // XXX these are all possible "searchDefault" engines in list.json.  Since the searchDefault
+    // value is a localized name, we have no good way to map these.  We need the engines that
+    // could be default, along with the engine being tested.
+    let defaults = ["google", "yandex", "baidu"];
+    // info(`testing with default engines ${defaults.values()}`)
+    traverse(listJSON, (key, val) => {
+      if (key === "searchOrder") {
+        return val.filter(
+          v =>
+            v.startsWith(target) ||
+            defaults.filter(d => v.startsWith(d)).length > 0
+        );
+      }
+      if (key === "visibleDefaultEngines") {
+        return val.filter(
+          v =>
+            v.startsWith(target) ||
+            defaults.filter(d => v.startsWith(d)).length > 0
+        );
+      }
+      return val;
+    });
+    SearchUtils.LIST_JSON_URL =
+      "data:application/json," + JSON.stringify(listJSON);
+  }
+
   /**
    * Sets up the test.
    */
   async setup() {
+    await this.setupListJSON();
+
     AddonTestUtils.init(GLOBAL_SCOPE);
     AddonTestUtils.createAppInfo(
       "xpcshell@tests.mozilla.org",
@@ -96,6 +150,9 @@ class SearchConfigTest {
     // Disable region checks.
     Services.prefs.setBoolPref("browser.search.geoSpecificDefaults", false);
     Services.prefs.setCharPref("browser.search.geoip.url", "");
+    Services.prefs.setIntPref("browser.search.addonLoadTimeout", 0);
+    // Shut off a bunch of logging from AOM.
+    Services.prefs.setBoolPref("extensions.logging.enabled", false);
 
     await AddonTestUtils.promiseStartupManager();
     await Services.search.init();