Bug 1414390 - Add intl.locale.requested locale list to replace general.useragent.locale. r=jfkthame,mossop

This patch moves us from using an old pref `general.useragent.locale`combined
with `intl.locale.matchOS` for retrieving user requested locale, to use a new
preference `intl.locale.requested` which stores a list of well-formed BCP47
language tags. If set to empty, the OS locales are used. If not set at all,
default locale is used.

We are also adding a piece of code to migrate from old to new system.


MozReview-Commit-ID: 854yQ1kC6Ee

--HG--
extra : rebase_source : c4a7171bc026f857f7878ee83d973ec01b536a84

browser/app/profile/firefox.js

@@ -224,7 +224,6 @@ pref("browser.uitour.surveyDuration", 7200);
 pref("keyword.enabled", true);
 pref("browser.fixup.domainwhitelist.localhost", true);
 
-pref("general.useragent.locale", "@AB_CD@");
 pref("general.skins.selectedSkin", "classic/1.0");
 
 pref("general.smoothScroll", true);
@@ -521,6 +520,8 @@ pref("javascript.options.showInConsole",          true);
 pref("general.warnOnAboutConfig",                 false);
 #endif
 
+pref("intl.locale.requested", "@AB_CD@");
+
 // This is the pref to control the location bar, change this to true to
 // force this - this makes the origin of popup windows more obvious to avoid
 // spoofing. We would rather not do it by default because it affects UE for web

browser/components/newtab/tests/browser/browser_packaged_as_locales.js

@@ -1,12 +1,11 @@
-const LOCALE_PREF = "general.useragent.locale";
-
+Components.utils.import("resource://gre/modules/Services.jsm");
 XPCOMUtils.defineLazyServiceGetter(this, "aboutNewTabService",
                                    "@mozilla.org/browser/aboutnewtab-service;1",
                                    "nsIAboutNewTabService");
 
 const DEFAULT_URL = "resource://activity-stream/prerendered/en-US/activity-stream-prerendered.html";
 async function getUrlForLocale(locale) {
-  await SpecialPowers.pushPrefEnv({set: [[LOCALE_PREF, locale]]});
+  Services.locale.setRequestedLocales([locale]);
   return aboutNewTabService.defaultURL;
 }
 
@@ -14,7 +13,7 @@ async function getUrlForLocale(locale) {
  * Test that an unknown locale defaults to en-US
  */
 add_task(async function test_unknown_locale() {
-  const url = await getUrlForLocale("foo-BAR");
+  const url = await getUrlForLocale("und");
   Assert.equal(url, DEFAULT_URL);
 });
 

browser/components/nsBrowserGlue.js

@@ -2208,6 +2208,27 @@ BrowserGlue.prototype = {
           Services.prefs.setCharPref("browser.search.reset.status", "silent");
         }
       });
+
+      // Migrate the old requested locales prefs to use the new model
+      const SELECTED_LOCALE_PREF = "general.useragent.locale";
+      const MATCHOS_LOCALE_PREF = "intl.locale.matchOS";
+
+      if (Services.prefs.prefHasUserValue(MATCHOS_LOCALE_PREF) ||
+          Services.prefs.prefHasUserValue(SELECTED_LOCALE_PREF)) {
+        if (Services.prefs.getBoolPref(MATCHOS_LOCALE_PREF, false)) {
+          Services.locale.setRequestedLocales([]);
+        } else {
+          let locale = Services.prefs.getComplexValue(SELECTED_LOCALE_PREF,
+            Ci.nsIPrefLocalizedString);
+          if (locale) {
+            try {
+              Services.locale.setRequestedLocales([locale.data]);
+            } catch (e) { /* Don't panic if the value is not a valid locale code. */ }
+          }
+        }
+        Services.prefs.clearUserPref(SELECTED_LOCALE_PREF);
+        Services.prefs.clearUserPref(MATCHOS_LOCALE_PREF);
+      }
     }
 
     // Update the migration version.

browser/locales/en-US/firefox-l10n.js

@@ -7,5 +7,3 @@
 # LOCALIZATION NOTE: this preference is set to true for en-US specifically,
 # locales without this line have the setting set to false by default.
 pref("browser.search.geoSpecificDefaults", true);
-
-pref("general.useragent.locale", "@AB_CD@");

intl/locale/LocaleService.cpp

@@ -21,12 +21,10 @@
 
 #define INTL_SYSTEM_LOCALES_CHANGED "intl:system-locales-changed"
 
-#define MATCH_OS_LOCALE_PREF "intl.locale.matchOS"
-#define SELECTED_LOCALE_PREF "general.useragent.locale"
+#define REQUESTED_LOCALES_PREF "intl.locale.requested"
 
 static const char* kObservedPrefs[] = {
-  MATCH_OS_LOCALE_PREF,
-  SELECTED_LOCALE_PREF,
+  REQUESTED_LOCALES_PREF,
   nullptr
 };
 
@@ -47,8 +45,8 @@ mozilla::StaticRefPtr<LocaleService> LocaleService::sInstance;
  * The BCP47 form should be used for all calls to ICU/Intl APIs.
  * The canonical form is used for all internal operations.
  */
-static void
-SanitizeForBCP47(nsACString& aLocale)
+static bool
+SanitizeForBCP47(nsACString& aLocale, bool strict)
 {
   // Currently, the only locale code we use that's not BCP47-conformant is
   // "ja-JP-mac" on OS X, but let's try to be more general than just
@@ -56,50 +54,65 @@ SanitizeForBCP47(nsACString& aLocale)
   const int32_t LANG_TAG_CAPACITY = 128;
   char langTag[LANG_TAG_CAPACITY];
   nsAutoCString locale(aLocale);
+  locale.Trim(" ");
   UErrorCode err = U_ZERO_ERROR;
   // This is a fail-safe method that will set langTag to "und" if it cannot
   // match any part of the input locale code.
   int32_t len = uloc_toLanguageTag(locale.get(), langTag, LANG_TAG_CAPACITY,
-                                   false, &err);
+                                   strict, &err);
   if (U_SUCCESS(err) && len > 0) {
     aLocale.Assign(langTag, len);
   }
+  return U_SUCCESS(err);
 }
 
 static bool
 ReadRequestedLocales(nsTArray<nsCString>& aRetVal)
 {
-  nsAutoCString locale;
+  nsAutoCString str;
+  nsresult rv = Preferences::GetCString(REQUESTED_LOCALES_PREF, str);
 
-  // First, we'll try to check if the user has `matchOS` pref selected
-  bool matchOSLocale = Preferences::GetBool(MATCH_OS_LOCALE_PREF);
-
-  if (matchOSLocale) {
-    // If he has, we'll pick the locale from the system
-    if (OSPreferences::GetInstance()->GetSystemLocales(aRetVal)) {
-      // If we succeeded, return.
-      return true;
+  // We handle three scenarios here:
+  //
+  // 1) The pref is not set - use default locale
+  // 2) The pref is set to "" - use OS locales
+  // 3) The pref is set to a value - parse the locale list and use it
+  if (NS_SUCCEEDED(rv)) {
+    if (str.Length() > 0) {
+      for (const nsACString& part : str.Split(',')) {
+        nsAutoCString locale(part);
+        if (SanitizeForBCP47(locale, true)) {
+          // This is a hack required for us to handle the special Mozilla `ja-JP-mac`
+          // locales. We sanitize the input to normalize the case etc. but in result
+          // the `ja-JP-mac` will get turned into a BCP47 tag. Here we're turning it
+          // back into the form expected by Gecko resources.
+          if (locale.EqualsLiteral("ja-JP-x-lvariant-mac")) {
+            locale.Assign("ja-JP-mac");
+          }
+          if (!aRetVal.Contains(locale)) {
+            aRetVal.AppendElement(locale);
+          }
+        }
+      }
+    } else {
+      // If the pref string is empty, we'll take requested locales
+      // from the OS.
+      OSPreferences::GetInstance()->GetSystemLocales(aRetVal);
     }
+  } else {
+    nsAutoCString defaultLocale;
+    LocaleService::GetInstance()->GetDefaultLocale(defaultLocale);
+    aRetVal.AppendElement(defaultLocale);
   }
 
-  // Otherwise, we'll try to get the requested locale from the prefs.
-  if (!NS_SUCCEEDED(Preferences::GetCString(SELECTED_LOCALE_PREF, locale))) {
-    return false;
-  }
-
-  // At the moment we just take a single locale, but in the future
-  // we'll want to allow user to specify a list of requested locales.
-  aRetVal.AppendElement(locale);
-
   // Last fallback locale is a locale for the requested locale chain.
   // In the future we'll want to make the fallback chain differ per-locale.
-  // For now, it'll always fallback on en-US.
   //
   // Notice: This is not the same as DefaultLocale,
   // which follows the default locale the build is in.
-  LocaleService::GetInstance()->GetLastFallbackLocale(locale);
-  if (!aRetVal.Contains(locale)) {
-    aRetVal.AppendElement(locale);
+  LocaleService::GetInstance()->GetLastFallbackLocale(str);
+  if (!aRetVal.Contains(str)) {
+    aRetVal.AppendElement(str);
   }
   return true;
 }
@@ -223,7 +236,7 @@ LocaleService::GetAppLocalesAsBCP47(nsTArray<nsCString>& aRetVal)
   }
   for (uint32_t i = 0; i < mAppLocales.Length(); i++) {
     nsAutoCString locale(mAppLocales[i]);
-    SanitizeForBCP47(locale);
+    SanitizeForBCP47(locale, false);
     aRetVal.AppendElement(locale);
   }
 }
@@ -298,7 +311,6 @@ LocaleService::GetRequestedLocales(nsTArray<nsCString>& aRetVal)
 {
   if (mRequestedLocales.IsEmpty()) {
     ReadRequestedLocales(mRequestedLocales);
-
   }
 
   aRetVal = mRequestedLocales;
@@ -572,8 +584,7 @@ LocaleService::Observe(nsISupports *aSubject, const char *aTopic,
     NS_ConvertUTF16toUTF8 pref(aData);
     // At the moment the only thing we're observing are settings indicating
     // user requested locales.
-    if (pref.EqualsLiteral(MATCH_OS_LOCALE_PREF) ||
-        pref.EqualsLiteral(SELECTED_LOCALE_PREF)) {
+    if (pref.EqualsLiteral(REQUESTED_LOCALES_PREF)) {
       RequestedLocalesChanged();
     }
   }
@@ -693,7 +704,7 @@ LocaleService::GetAppLocaleAsBCP47(nsACString& aRetVal)
   }
   aRetVal = mAppLocales[0];
 
-  SanitizeForBCP47(aRetVal);
+  SanitizeForBCP47(aRetVal, false);
   return NS_OK;
 }
 
@@ -977,19 +988,26 @@ NS_IMETHODIMP
 LocaleService::SetRequestedLocales(const char** aRequested,
                                    uint32_t aRequestedCount)
 {
-  nsAutoCString lastFallbackLocale;
-  GetLastFallbackLocale(lastFallbackLocale);
-  MOZ_ASSERT(aRequestedCount < 2 ||
-             (aRequestedCount == 2 && lastFallbackLocale.Equals(aRequested[1])),
-      "We can only handle one requested locale (optionally with last fallback)");
+  nsAutoCString str;
 
-  if (aRequestedCount == 0) {
-    Preferences::ClearUser(SELECTED_LOCALE_PREF);
-  } else {
-    Preferences::SetCString(SELECTED_LOCALE_PREF, aRequested[0]);
+  for (uint32_t i = 0; i < aRequestedCount; i++) {
+    nsAutoCString locale(aRequested[i]);
+    if (!SanitizeForBCP47(locale, true)) {
+      NS_ERROR("Invalid language tag provided to SetRequestedLocales!");
+      return NS_ERROR_INVALID_ARG;
+    }
+    if (locale.EqualsLiteral("ja-JP-x-lvariant-mac")) {
+      // This is a hack for our code to handle `ja-JP-mac` special case.
+      locale.Assign("ja-JP-mac");
+    }
+
+    if (i > 0) {
+      str.AppendLiteral(",");
+    }
+    str.Append(locale);
   }
+  Preferences::SetCString(REQUESTED_LOCALES_PREF, str);
 
-  Preferences::SetBool(MATCH_OS_LOCALE_PREF, aRequestedCount == 0);
   return NS_OK;
 }
 

intl/locale/tests/unit/test_localeService.js

@@ -49,9 +49,7 @@ add_test(function test_getAppLocalesAsLangTags() {
   run_next_test();
 });
 
-const PREF_MATCH_OS_LOCALE = "intl.locale.matchOS";
-const PREF_SELECTED_LOCALE = "general.useragent.locale";
-const PREF_OS_LOCALE       = "intl.locale.os";
+const PREF_REQUESTED_LOCALES = "intl.locale.requested";
 const REQ_LOC_CHANGE_EVENT = "intl:requested-locales-changed";
 
 add_test(function test_getRequestedLocales() {
@@ -72,9 +70,7 @@ add_test(function test_getRequestedLocales() {
 add_test(function test_getRequestedLocales_matchOS() {
   do_test_pending();
 
-  Services.prefs.setBoolPref(PREF_MATCH_OS_LOCALE, false);
-  Services.prefs.setCharPref(PREF_SELECTED_LOCALE, "ar-IR");
-  Services.prefs.setCharPref(PREF_OS_LOCALE, "en-US");
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "ar-IR");
 
   const observer = {
     observe: function (aSubject, aTopic, aData) {
@@ -89,7 +85,7 @@ add_test(function test_getRequestedLocales_matchOS() {
   };
 
   Services.obs.addObserver(observer, REQ_LOC_CHANGE_EVENT);
-  Services.prefs.setBoolPref(PREF_MATCH_OS_LOCALE, true);
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "");
 
   run_next_test();
 });
@@ -99,11 +95,10 @@ add_test(function test_getRequestedLocales_matchOS() {
  * event for requested locales change, it will be fired when the
  * pref for browser UI locale changes.
  */
-add_test(function test_getRequestedLocales_matchOS() {
+add_test(function test_getRequestedLocales_onChange() {
   do_test_pending();
 
-  Services.prefs.setBoolPref(PREF_MATCH_OS_LOCALE, false);
-  Services.prefs.setCharPref(PREF_SELECTED_LOCALE, "ar-IR");
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "ar-IR");
 
   const observer = {
     observe: function (aSubject, aTopic, aData) {
@@ -118,25 +113,18 @@ add_test(function test_getRequestedLocales_matchOS() {
   };
 
   Services.obs.addObserver(observer, REQ_LOC_CHANGE_EVENT);
-  Services.prefs.setCharPref(PREF_SELECTED_LOCALE, "sr-RU");
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "sr-RU");
 
   run_next_test();
 });
 
 add_test(function test_getRequestedLocale() {
-  Services.prefs.setBoolPref(PREF_MATCH_OS_LOCALE, false);
-  Services.prefs.setCharPref(PREF_SELECTED_LOCALE, "tlh");
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "tlh");
 
   let requestedLocale = localeService.getRequestedLocale();
   do_check_true(requestedLocale === "tlh", "requestedLocale returns the right value");
 
-  Services.prefs.setCharPref(PREF_SELECTED_LOCALE, "");
-
-  requestedLocale = localeService.getRequestedLocale();
-  do_check_true(requestedLocale === "", "requestedLocale returns empty value value");
-
-  Services.prefs.clearUserPref(PREF_MATCH_OS_LOCALE);
-  Services.prefs.clearUserPref(PREF_SELECTED_LOCALE);
+  Services.prefs.clearUserPref(PREF_REQUESTED_LOCALES);
 
   run_next_test();
 });
@@ -144,15 +132,12 @@ add_test(function test_getRequestedLocale() {
 add_test(function test_setRequestedLocales() {
   localeService.setRequestedLocales([]);
 
-  let matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE);
-  do_check_true(matchOS === true);
+  localeService.setRequestedLocales(['de-AT', 'de-DE', 'de-CH']);
 
-  localeService.setRequestedLocales(['de-AT']);
-
-  matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE);
-  do_check_true(matchOS === false);
   let locales = localeService.getRequestedLocales();;
   do_check_true(locales[0] === 'de-AT');
+  do_check_true(locales[1] === 'de-DE');
+  do_check_true(locales[2] === 'de-CH');
 
   run_next_test();
 });
@@ -163,8 +148,25 @@ add_test(function test_isAppLocaleRTL() {
   run_next_test();
 });
 
-do_register_cleanup(() => {
-  Services.prefs.clearUserPref(PREF_SELECTED_LOCALE);
-  Services.prefs.clearUserPref(PREF_MATCH_OS_LOCALE);
-  Services.prefs.clearUserPref(PREF_OS_LOCALE, "en-US");
+/**
+ * This test verifies that all values coming from the pref are sanitized.
+ */
+add_test(function test_getRequestedLocales_sanitize() {
+  Services.prefs.setCharPref(PREF_REQUESTED_LOCALES, "de,2,#$@#,pl,!a2,DE-at,,;");
+
+  let locales = localeService.getRequestedLocales();
+  do_check_eq(locales[0], "de");
+  do_check_eq(locales[1], "pl");
+  do_check_eq(locales[2], "de-AT");
+  do_check_eq(locales[3], "und");
+  do_check_eq(locales[4], localeService.lastFallbackLocale);
+  do_check_eq(locales.length, 5);
+
+  Services.prefs.clearUserPref(PREF_REQUESTED_LOCALES);
+
+  run_next_test();
+});
+
+do_register_cleanup(() => {
+  Services.prefs.clearUserPref(PREF_REQUESTED_LOCALES);
 });

mobile/android/installer/mobile-l10n.js

@@ -3,4 +3,4 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 // Inherit locale from the OS, used for multi-locale builds
-pref("intl.locale.matchOS", true);
+pref("intl.locale.requested", "");

mobile/android/locales/en-US/mobile-l10n.js

@@ -3,5 +3,3 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 #filter substitution
-
-pref("general.useragent.locale", "@AB_CD@");

modules/libpref/init/all.js

@@ -22,7 +22,6 @@
 pref("preferences.allow.omt-write", true);
 
 pref("keyword.enabled", false);
-pref("general.useragent.locale", "chrome://global/locale/intl.properties");
 pref("general.useragent.compatMode.firefox", false);
 
 // This pref exists only for testing purposes. In order to disable all
@@ -2295,7 +2294,6 @@ pref("intl.charset.fallback.override",      "");
 pref("intl.charset.fallback.tld",           true);
 pref("intl.charset.fallback.utf8_for_file", false);
 pref("intl.ellipsis",                       "chrome://global-platform/locale/intl.properties");
-pref("intl.locale.matchOS",                 false);
 // this pref allows user to request that all internationalization formatters
 // like date/time formatting, unit formatting, calendars etc. should use
 // OS locale set instead of the app locale set.

toolkit/locales/en-US/chrome/global/intl.properties

@@ -2,20 +2,10 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-# LOCALIZATION NOTE (general.useragent.locale):
-# This is the valid BCP 47 language tag representing your locale.
-#
-# In most cases, this will simply be your locale code. However, in rare cases
-# (such as 'jp-JP-mac'), you may need to modify your locale code in order to
-# make it a valid BCP 47 language tag. (If your locale code does not include a
-# region subtag, do not include one in the language tag representing your
-# locale.)
-general.useragent.locale=en-US
-
 # LOCALIZATION NOTE (intl.accept_languages):
 # This is a comma-separated list of valid BCP 47 language tags.
 #
-# Begin with the value of 'general.useragent.locale'. Next, include language
+# Begin with the language tag of your locale. Next, include language
 # tags for other languages that you expect most users of your locale to be
 # able to speak, so that their browsing experience degrades gracefully if
 # content is not available in their primary language.