Bug 1755519 - Add JSM for langpack matching logic; r=platform-i18n-reviewers,nordzilla

The TestUtils file is broken out as it will be shared with the
about:welcome browser chrome tests.

Differential Revision: https://phabricator.services.mozilla.com/D138830

intl/locale/LangPackMatcher.jsm

@@ -0,0 +1,311 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+const { XPCOMUtils } = ChromeUtils.import(
+  "resource://gre/modules/XPCOMUtils.jsm"
+);
+
+XPCOMUtils.defineLazyModuleGetters(this, {
+  AddonRepository: "resource://gre/modules/addons/AddonRepository.jsm",
+  AddonManager: "resource://gre/modules/AddonManager.jsm",
+  Services: "resource://gre/modules/Services.jsm",
+});
+
+if (Services.appinfo.processType !== Services.appinfo.PROCESS_TYPE_DEFAULT) {
+  // This check ensures that the `mockable` API calls can be consisently mocked in tests.
+  // If this requirement needs to be eased, please ensure the test logic remains valid.
+  throw new Error("This code is assumed to run in the parent process.");
+}
+
+/**
+ * Attempts to find an appropriate langpack for a given language. The async function
+ * is infallible, but may not return a langpack.
+ *
+ * @returns {LangPack | null}
+ */
+async function negotiateLangPackForLanguageMismatch() {
+  const localeInfo = getAppAndSystemLocaleInfo();
+  if (!localeInfo.systemLocale) {
+    // The system locale info was not valid.
+    return null;
+  }
+
+  /**
+   * Fetch the available langpacks from AMO.
+   *
+   * @type {Array<LangPack>}
+   */
+  const availableLangpacks = await mockable.getAvailableLangpacks();
+  if (!availableLangpacks) {
+    return null;
+  }
+
+  /**
+   * Figure out a langpack to recommend.
+   * @type {LangPack | null}
+   */
+  return (
+    // First look for a langpack that matches the baseName.
+    // e.g. system "fr-FR" matches langpack "fr-FR"
+    //      system "en-GB" matches langpack "en-GB".
+    availableLangpacks.find(
+      ({ target_locale }) => target_locale === localeInfo.systemLocale.baseName
+    ) ||
+    // Next look for langpacks that just match the language.
+    // e.g. system "fr-FR" matches langpack "fr".
+    //      system "en-AU" matches langpack "en".
+    availableLangpacks.find(
+      ({ target_locale }) => target_locale === localeInfo.systemLocale.language
+    ) ||
+    // Next look for a langpack that matches the language, but not the region.
+    // e.g. "es-CL" (Chilean Spanish) as a system language matching
+    //      "es-ES" (European Spanish)
+    availableLangpacks.find(({ target_locale }) =>
+      target_locale.startsWith(`${localeInfo.systemLocale.language}-`)
+    ) ||
+    null
+  );
+}
+
+// If a langpack is being installed, allow blocking on that.
+let installingLangpack = new Map();
+
+/**
+ * @typedef {LangPack}
+ * @type {object}
+ * @property {string} target_locale
+ * @property {string} url
+ * @property {string} hash
+ */
+
+/**
+ * Ensure that a given lanpack is installed.
+ *
+ * @param {LangPack} langPack
+ * @returns {Promise<boolean>} Success or failure.
+ */
+function ensureLangPackInstalled(langPack) {
+  if (!langPack) {
+    throw new Error("Expected a LangPack to install.");
+  }
+  // Make sure any outstanding calls get resolved before attempting another call.
+  // This guards against any quick page refreshes attempting to install the langpack
+  // twice.
+  const inProgress = installingLangpack.get(langPack.hash);
+  if (inProgress) {
+    return inProgress;
+  }
+  const promise = _ensureLangPackInstalledImpl(langPack);
+  installingLangpack.set(langPack.hash, promise);
+  promise.finally(() => {
+    installingLangpack.delete(langPack.hash);
+  });
+  return promise;
+}
+
+/**
+ * @param {LangPack} langPack
+ * @returns {boolean} Success or failure.
+ */
+async function _ensureLangPackInstalledImpl(langPack) {
+  if (mockable.getAvailableLocales().includes(langPack.target_locale)) {
+    // The langpack is already installed.
+    return true;
+  }
+
+  return mockable.installLangPack(langPack);
+}
+
+/**
+ * These are all functions with side effects or configuration options that should be
+ * mockable for tests.
+ */
+const mockable = {
+  /**
+   * @returns {LangPack[] | null}
+   */
+  async getAvailableLangpacks() {
+    try {
+      return AddonRepository.getAvailableLangpacks();
+    } catch (error) {
+      Cu.reportError(
+        `Failed to get the list of available language packs: ${error?.message}`
+      );
+      return null;
+    }
+  },
+
+  /**
+   * Use the AddonManager to install an addon from the URL.
+   * @param {LangPack} langPack
+   */
+  async installLangPack(langPack) {
+    let install;
+    try {
+      install = await AddonManager.getInstallForURL(langPack.url, {
+        hash: langPack.hash,
+        telemetryInfo: {
+          source: "about:welcome",
+        },
+      });
+    } catch (error) {
+      Cu.reportError(error);
+      return false;
+    }
+
+    try {
+      await install.install();
+    } catch (error) {
+      Cu.reportError(error);
+      return false;
+    }
+    return true;
+  },
+
+  /**
+   * @returns {string[]}
+   */
+  getAvailableLocales() {
+    return Services.locale.availableLocales;
+  },
+
+  /**
+   * @returns {string}
+   */
+  getAppLocaleAsBCP47() {
+    return Services.locale.appLocaleAsBCP47;
+  },
+
+  /**
+   * @returns {string}
+   */
+  getSystemLocale() {
+    // Allow the system locale to be overridden for manual testing.
+    const systemLocaleOverride = Services.prefs.getCharPref(
+      "intl.multilingual.aboutWelcome.systemLocaleOverride",
+      null
+    );
+    if (systemLocaleOverride) {
+      try {
+        // If the locale can't be parsed, ignore the pref.
+        new Services.intl.Locale(systemLocaleOverride);
+        return systemLocaleOverride;
+      } catch (_error) {}
+    }
+
+    const osPrefs = Cc["@mozilla.org/intl/ospreferences;1"].getService(
+      Ci.mozIOSPreferences
+    );
+    return osPrefs.systemLocale;
+  },
+
+  /**
+   * @param {string[]} locales The BCP 47 locale identifiers.
+   */
+  setRequestedAppLocales(locales) {
+    Services.locale.requestedLocales = locales;
+  },
+};
+
+/**
+ * This function is really only setting `Services.locale.requestedLocales`, but it's
+ * using the `mockable` object to allow this behavior to be mocked in tests.
+ *
+ * @param {string[]} locales The BCP 47 locale identifiers.
+ */
+function setRequestedAppLocales(locales) {
+  mockable.setRequestedAppLocales(locales);
+}
+
+/**
+ * A serializable Intl.Locale.
+ *
+ * @typedef StructuredLocale
+ * @type {object}
+ * @property {string} baseName
+ * @property {string} language
+ * @property {string} region
+ */
+
+/**
+ * In telemetry data, some of the system locales show up as blank. Guard against this
+ * and any other malformed locale information provided by the system by wrapping the call
+ * into a catch/try.
+ *
+ * @param {string} locale
+ * @returns {StructuredLocale | null}
+ */
+function getStructuredLocaleOrNull(localeString) {
+  try {
+    const locale = new Services.intl.Locale(localeString);
+    return {
+      baseName: locale.baseName,
+      language: locale.language,
+      region: locale.region,
+    };
+  } catch (_err) {
+    return null;
+  }
+}
+
+/**
+ * Determine the system and app locales, and how much the locales match.
+ *
+ * @returns {{
+ *  systemLocale: StructuredLocale,
+ *  appLocale: StructuredLocale,
+ *  matchType: "unknown" | "language-mismatch" | "region-mismatch" | "match",
+ * }}
+ */
+function getAppAndSystemLocaleInfo() {
+  // Convert locale strings into structured locale objects.
+  const systemLocaleRaw = mockable.getSystemLocale();
+  const appLocaleRaw = mockable.getAppLocaleAsBCP47();
+
+  const systemLocale = getStructuredLocaleOrNull(systemLocaleRaw);
+  const appLocale = getStructuredLocaleOrNull(appLocaleRaw);
+
+  let matchType = "unknown";
+  if (systemLocale && appLocale) {
+    if (systemLocale.language !== appLocale.language) {
+      matchType = "language-mismatch";
+    } else if (systemLocale.region !== appLocale.region) {
+      matchType = "region-mismatch";
+    } else {
+      matchType = "match";
+    }
+  }
+
+  const displayNames = new Services.intl.DisplayNames(appLocaleRaw, {
+    type: "language",
+  });
+
+  return {
+    // Return the Intl.Locale in a serializable form.
+    systemLocaleRaw,
+    systemLocale,
+    appLocaleRaw,
+    appLocale,
+    matchType,
+
+    // These can be used as Fluent message args.
+    displayNames: {
+      systemLanguage: systemLocale
+        ? displayNames.of(systemLocale.baseName)
+        : null,
+      appLanguage: appLocale ? displayNames.of(appLocale.baseName) : null,
+    },
+  };
+}
+
+var LangPackMatcher = {
+  negotiateLangPackForLanguageMismatch,
+  ensureLangPackInstalled,
+  getAppAndSystemLocaleInfo,
+  setRequestedAppLocales,
+  mockable,
+};
+
+var EXPORTED_SYMBOLS = ["LangPackMatcher"];

intl/locale/moz.build

@@ -6,6 +6,10 @@
 
 XPCSHELL_TESTS_MANIFESTS += ["tests/unit/xpcshell.ini"]
 
+TESTING_JS_MODULES += [
+    "tests/LangPackMatcherTestUtils.jsm",
+]
+
 toolkit = CONFIG["MOZ_WIDGET_TOOLKIT"]
 
 if toolkit == "windows":
@@ -47,6 +51,7 @@ UNIFIED_SOURCES += [
 ]
 
 EXTRA_JS_MODULES += [
+    "LangPackMatcher.jsm",
     "PluralForm.jsm",
 ]
 

intl/locale/tests/LangPackMatcherTestUtils.jsm

@@ -0,0 +1,119 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+var EXPORTED_SYMBOLS = ["getAddonAndLocalAPIsMocker"];
+
+const { LangPackMatcher } = ChromeUtils.import(
+  "resource://gre/modules/LangPackMatcher.jsm"
+);
+
+/**
+ * LangPackMatcher.jsm calls out to to the addons store, which involves network requests.
+ * Other tests create a fake addons server, and install mock XPIs. At the time of this
+ * writing that infrastructure is not available for mochitests.
+ *
+ * Instead, this test mocks out APIs that have a side-effect, so the addons of the browser
+ * are never modified.
+ *
+ * The calls to get the app's locale and system's locale are also mocked so that the
+ * different language mismatch scenarios can be run through.
+ *
+ * The locales are BCP 47 identifiers:
+ *
+ * @param {{
+ *   sandbox: SinonSandbox,
+ *   systemLocale: string,
+ *   appLocale, string,
+ * }}
+ */
+function getAddonAndLocalAPIsMocker(testScope, sandbox) {
+  const { info } = testScope;
+  return function mockAddonAndLocaleAPIs({ systemLocale, appLocale }) {
+    info("Mocking LangPackMatcher.jsm APIs");
+
+    let resolveLangPacks;
+    const langPackPromise = new Promise(resolve => {
+      resolveLangPacks = availableLangpacks => {
+        info(
+          `Resolving which langpacks are available for download: ${JSON.stringify(
+            availableLangpacks
+          )}`
+        );
+        resolve(
+          availableLangpacks.map(locale => ({
+            guid: `langpack-${locale}@firefox.mozilla.org`,
+            type: "language",
+            target_locale: locale,
+            current_compatible_version: {
+              files: [
+                {
+                  platform: "all",
+                  url: `http://example.com/${locale}.langpack.xpi`,
+                },
+              ],
+            },
+          }))
+        );
+      };
+    });
+
+    let resolveInstaller;
+    const installerPromise = new Promise(resolve => {
+      resolveInstaller = () => {
+        info("LangPack install finished.");
+        resolve();
+      };
+    });
+
+    const { mockable } = LangPackMatcher;
+    if (appLocale) {
+      sandbox.stub(mockable, "getAvailableLocales").returns([appLocale]);
+      sandbox.stub(mockable, "getAppLocaleAsBCP47").returns(appLocale);
+    }
+    if (systemLocale) {
+      sandbox.stub(mockable, "getSystemLocale").returns(systemLocale);
+    }
+
+    sandbox.stub(mockable, "getAvailableLangpacks").callsFake(() => {
+      info("Requesting which langpacks are available for download");
+      return langPackPromise;
+    });
+
+    sandbox.stub(mockable, "installLangPack").callsFake(langPack => {
+      info(`LangPack install started, but pending: ${langPack.target_locale}`);
+      return installerPromise;
+    });
+
+    sandbox.stub(mockable, "setRequestedAppLocales").callsFake(locales => {
+      info(
+        `Changing the browser's requested locales to: ${JSON.stringify(
+          locales
+        )}`
+      );
+    });
+
+    return {
+      /**
+       * Resolves the addons API call with available langpacks. Call with a list
+       * of BCP 47 identifiers.
+       *
+       * @type {(availableLangpacks: string[]) => {}}
+       */
+      resolveLangPacks,
+
+      /**
+       * Resolves the pending call to install a langpack.
+       *
+       * @type {() => {}}
+       */
+      resolveInstaller,
+
+      /**
+       * The mocked APIs.
+       */
+      mockable,
+    };
+  };
+}

intl/locale/tests/unit/test_langPackMatcher.js

@@ -0,0 +1,201 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const { getAddonAndLocalAPIsMocker } = ChromeUtils.import(
+  "resource://testing-common/LangPackMatcherTestUtils.jsm"
+);
+const { LangPackMatcher } = ChromeUtils.import(
+  "resource://gre/modules/LangPackMatcher.jsm"
+);
+const { sinon } = ChromeUtils.import("resource://testing-common/Sinon.jsm");
+
+const sandbox = sinon.createSandbox();
+const mockAddonAndLocaleAPIs = getAddonAndLocalAPIsMocker(this, sandbox);
+
+add_task(function initSandbox() {
+  registerCleanupFunction(() => {
+    sandbox.restore();
+  });
+});
+
+add_task(function test_appLocaleLanguageMismatch() {
+  sandbox.restore();
+  mockAddonAndLocaleAPIs({
+    systemLocale: "es-ES",
+    appLocale: "en-US",
+  });
+
+  deepEqual(LangPackMatcher.getAppAndSystemLocaleInfo(), {
+    systemLocaleRaw: "es-ES",
+    systemLocale: { baseName: "es-ES", language: "es", region: "ES" },
+    appLocaleRaw: "en-US",
+    appLocale: { baseName: "en-US", language: "en", region: "US" },
+    matchType: "language-mismatch",
+    displayNames: {
+      systemLanguage: "European Spanish",
+      appLanguage: "American English",
+    },
+  });
+});
+
+add_task(function test_appLocaleRegionMismatch() {
+  sandbox.restore();
+  mockAddonAndLocaleAPIs({
+    sandbox,
+    systemLocale: "en-CA",
+    appLocale: "en-US",
+  });
+
+  deepEqual(LangPackMatcher.getAppAndSystemLocaleInfo(), {
+    systemLocaleRaw: "en-CA",
+    systemLocale: { baseName: "en-CA", language: "en", region: "CA" },
+    appLocaleRaw: "en-US",
+    appLocale: { baseName: "en-US", language: "en", region: "US" },
+    matchType: "region-mismatch",
+    displayNames: {
+      systemLanguage: "Canadian English",
+      appLanguage: "American English",
+    },
+  });
+});
+
+add_task(function test_appLocaleScriptMismatch() {
+  sandbox.restore();
+  // Script mismatch:
+  mockAddonAndLocaleAPIs({
+    sandbox,
+    systemLocale: "zh-Hans-CN",
+    appLocale: "zh-CN",
+  });
+
+  deepEqual(LangPackMatcher.getAppAndSystemLocaleInfo(), {
+    systemLocaleRaw: "zh-Hans-CN",
+    systemLocale: { baseName: "zh-Hans-CN", language: "zh", region: "CN" },
+    appLocaleRaw: "zh-CN",
+    appLocale: { baseName: "zh-CN", language: "zh", region: "CN" },
+    matchType: "match",
+    displayNames: {
+      systemLanguage: "简体中文（中国）",
+      appLanguage: "中文（中国）",
+    },
+  });
+});
+
+add_task(function test_appLocaleInvalidSystem() {
+  sandbox.restore();
+  // Script mismatch:
+  mockAddonAndLocaleAPIs({
+    sandbox,
+    systemLocale: "Not valid",
+    appLocale: "en-US",
+  });
+
+  deepEqual(LangPackMatcher.getAppAndSystemLocaleInfo(), {
+    systemLocaleRaw: "Not valid",
+    systemLocale: null,
+    appLocaleRaw: "en-US",
+    appLocale: { baseName: "en-US", language: "en", region: "US" },
+    matchType: "unknown",
+    displayNames: { systemLanguage: null, appLanguage: "American English" },
+  });
+});
+
+function shuffle(array) {
+  return array
+    .map(value => ({ value, sort: Math.random() }))
+    .sort((a, b) => a.sort - b.sort)
+    .map(({ value }) => value);
+}
+
+add_task(async function test_negotiateLangPacks() {
+  const negotiations = [
+    {
+      // Exact match found.
+      systemLocale: "en-US",
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: "en-US",
+    },
+    {
+      // Region-less match.
+      systemLocale: "en-CA",
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: "en",
+    },
+    {
+      // Fallback to a different region.
+      systemLocale: "en-CA",
+      availableLangPacks: ["en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: "en-US",
+    },
+    {
+      // Match with a script. zh-Hans-CN is the locale used with simplified
+      // Chinese scripts, while zh-CN uses the Latin script.
+      systemLocale: "zh-Hans-CN",
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: "zh-Hans-CN",
+    },
+    {
+      // No reasonable match could be found.
+      systemLocale: "tlh", // Klingon
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: null,
+    },
+    {
+      // Weird, but valid locale identifiers.
+      systemLocale: "en-US-u-hc-h23-ca-islamic-civil-ss-true",
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: "en-US",
+    },
+    {
+      // Invalid system locale
+      systemLocale: "Not valid",
+      availableLangPacks: ["en", "en-US", "zh", "zh-CN", "zh-Hans-CN"],
+      expected: null,
+    },
+  ];
+
+  for (const { systemLocale, availableLangPacks, expected } of negotiations) {
+    sandbox.restore();
+    const { resolveLangPacks } = mockAddonAndLocaleAPIs({
+      sandbox,
+      systemLocale,
+    });
+
+    const promise = LangPackMatcher.negotiateLangPackForLanguageMismatch();
+    // Shuffle the order to ensure that this test doesn't require on ordering of the
+    // langpack responses.
+    resolveLangPacks(shuffle(availableLangPacks));
+    const actual = (await promise)?.target_locale;
+    equal(
+      actual,
+      expected,
+      `Resolve the systemLocale "${systemLocale}" with available langpacks: ${JSON.stringify(
+        availableLangPacks
+      )}`
+    );
+  }
+});
+
+add_task(async function test_ensureLangPackInstalled() {
+  sandbox.restore();
+  const { resolveLangPacks, resolveInstaller } = mockAddonAndLocaleAPIs({
+    sandbox,
+    systemLocale: "es-ES",
+    appLocale: "en-US",
+  });
+
+  const negotiatePromise = LangPackMatcher.negotiateLangPackForLanguageMismatch();
+  resolveLangPacks(["es-ES"]);
+  const langPack = await negotiatePromise;
+
+  const installPromise1 = LangPackMatcher.ensureLangPackInstalled(langPack);
+  const installPromise2 = LangPackMatcher.ensureLangPackInstalled(langPack);
+
+  resolveInstaller(["fake langpack"]);
+
+  info("Ensure both installers resolve when called twice in a row.");
+  await installPromise1;
+  await installPromise2;
+  ok(true, "Both were called.");
+});

intl/locale/tests/unit/xpcshell.ini

@@ -10,7 +10,7 @@ skip-if = toolkit != "windows" && toolkit != "cocoa"
 [test_bug1086527.js]
 [test_intl_on_workers.js]
 skip-if = toolkit == "android" # bug 1309447
-
+[test_langPackMatcher.js]
 [test_pluralForm.js]
 [test_pluralForm_english.js]
 [test_pluralForm_makeGetter.js]