gecko-dev/intl/l10n/L10nRegistry.jsm
2019-12-27 16:05:26 +00:00

792 lines
25 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

const { AppConstants } = ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
// eslint-disable-next-line mozilla/use-services
const appinfo = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULRuntime);
const { FluentBundle, FluentResource } = ChromeUtils.import("resource://gre/modules/Fluent.jsm");
const {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
ChromeUtils.defineModuleGetter(
this,
"NetUtil",
"resource://gre/modules/NetUtil.jsm"
);
const isParentProcess = appinfo.processType === appinfo.PROCESS_TYPE_DEFAULT;
/**
* L10nRegistry is a localization resource management system for Gecko.
*
* It manages the list of resource sources provided with the app and allows
* for additional sources to be added and updated.
*
* It's primary purpose is to allow for building an iterator over FluentBundle objects
* that will be utilized by a localization API.
*
* The generator creates all possible permutations of locales and sources to allow for
* complete fallbacking.
*
* Example:
*
* FileSource1:
* name: 'app'
* locales: ['en-US', 'de']
* resources: [
* '/browser/menu.ftl',
* '/platform/toolkit.ftl',
* ]
* FileSource2:
* name: 'platform'
* locales: ['en-US', 'de']
* resources: [
* '/platform/toolkit.ftl',
* ]
*
* If the user will request:
* L10nRegistry.generateBundles(['de', 'en-US'], [
* '/browser/menu.ftl',
* '/platform/toolkit.ftl'
* ]);
*
* the generator will return an async iterator over the following contexts:
*
* {
* locale: 'de',
* resources: [
* ['app', '/browser/menu.ftl'],
* ['app', '/platform/toolkit.ftl'],
* ]
* },
* {
* locale: 'de',
* resources: [
* ['app', '/browser/menu.ftl'],
* ['platform', '/platform/toolkit.ftl'],
* ]
* },
* {
* locale: 'en-US',
* resources: [
* ['app', '/browser/menu.ftl'],
* ['app', '/platform/toolkit.ftl'],
* ]
* },
* {
* locale: 'en-US',
* resources: [
* ['app', '/browser/menu.ftl'],
* ['platform', '/platform/toolkit.ftl'],
* ]
* }
*
* This allows the localization API to consume the FluentBundle and lazily fallback
* on the next in case of a missing string or error.
*
* If during the life-cycle of the app a new source is added, the generator can be called again
* and will produce a new set of permutations placing the language pack provided resources
* at the top.
*
* Notice: L10nRegistry is primarily an asynchronous API, but
* it does provide a synchronous version of it's main method
* for use by the `Localization` class when in `sync` state.
* This API should be only used in very specialized cases and
* the uses should be reviewed by the toolkit owner/peer.
*/
class L10nRegistryService {
constructor() {
this.sources = new Map();
if (isParentProcess) {
const locales = Services.locale.packagedLocales;
// Categories are sorted alphabetically, so we name our sources:
// - 0-toolkit
// - 5-browser
// - langpack-{locale}
//
// This should ensure that they're returned in the correct order.
for (let {entry, value} of Services.catMan.enumerateCategory("l10n-registry")) {
if (!this.hasSource(entry)) {
const source = new FileSource(entry, locales, value);
this.registerSource(source);
}
}
} else {
this._setSourcesFromSharedData();
Services.cpmm.sharedData.addEventListener("change", this);
}
}
handleEvent(event) {
if (event.type === "change") {
if (event.changedKeys.includes("L10nRegistry:Sources")) {
this._setSourcesFromSharedData();
}
}
}
/**
* Based on the list of requested languages and resource Ids,
* this function returns an lazy iterator over message context permutations.
*
* Notice: Any changes to this method should be copied
* to the `generateBundlesSync` equivalent below.
*
* @param {Array} requestedLangs
* @param {Array} resourceIds
* @returns {AsyncIterator<FluentBundle>}
*/
async* generateBundles(requestedLangs, resourceIds) {
const sourcesOrder = Array.from(this.sources.keys()).reverse();
const pseudoNameFromPref = Services.prefs.getStringPref("intl.l10n.pseudo", "");
for (const locale of requestedLangs) {
for await (const dataSets of generateResourceSetsForLocale(locale, sourcesOrder, resourceIds)) {
const bundle = new FluentBundle(locale, {
...MSG_CONTEXT_OPTIONS,
transform: PSEUDO_STRATEGIES[pseudoNameFromPref],
});
for (const data of dataSets) {
if (data === null) {
return;
}
bundle.addResource(data);
}
yield bundle;
}
}
}
/**
* This is a synchronous version of the `generateBundles`
* method and should stay completely in sync with it at all
* times except of the async/await changes.
*
* Notice: This method should be avoided at all costs
* You can think of it similarly to a synchronous XMLHttpRequest.
*
* @param {Array} requestedLangs
* @param {Array} resourceIds
* @returns {Iterator<FluentBundle>}
*/
* generateBundlesSync(requestedLangs, resourceIds) {
const sourcesOrder = Array.from(this.sources.keys()).reverse();
const pseudoNameFromPref = Services.prefs.getStringPref("intl.l10n.pseudo", "");
for (const locale of requestedLangs) {
for (const dataSets of generateResourceSetsForLocaleSync(locale, sourcesOrder, resourceIds)) {
const bundle = new FluentBundle(locale, {
...MSG_CONTEXT_OPTIONS,
transform: PSEUDO_STRATEGIES[pseudoNameFromPref],
});
for (const data of dataSets) {
if (data === null) {
return;
}
bundle.addResource(data);
}
yield bundle;
}
}
}
/**
* Check whether a source with the given known is already registered.
*
* @param {String} sourceName
* @returns {boolean} whether or not a source by that name is known.
*/
hasSource(sourceName) {
return this.sources.has(sourceName);
}
/**
* Adds a new resource source to the L10nRegistry.
*
* @param {FileSource} source
*/
registerSource(source) {
if (this.hasSource(source.name)) {
throw new Error(`Source with name "${source.name}" already registered.`);
}
this.sources.set(source.name, source);
if (isParentProcess) {
this._synchronizeSharedData();
Services.locale.availableLocales = this.getAvailableLocales();
}
}
/**
* Updates an existing source in the L10nRegistry
*
* That will usually happen when a new version of a source becomes
* available (for example, an updated version of a language pack).
*
* @param {FileSource} source
*/
updateSource(source) {
if (!this.hasSource(source.name)) {
throw new Error(`Source with name "${source.name}" is not registered.`);
}
this.sources.set(source.name, source);
if (isParentProcess) {
this._synchronizeSharedData();
Services.locale.availableLocales = this.getAvailableLocales();
}
}
/**
* Removes a source from the L10nRegistry.
*
* @param {String} sourceId
*/
removeSource(sourceName) {
this.sources.delete(sourceName);
if (isParentProcess) {
this._synchronizeSharedData();
Services.locale.availableLocales = this.getAvailableLocales();
}
}
_synchronizeSharedData() {
const sources = new Map();
for (const [name, source] of this.sources.entries()) {
if (source.indexed) {
continue;
}
sources.set(name, {
locales: source.locales,
prePath: source.prePath,
});
}
Services.ppmm.sharedData.set("L10nRegistry:Sources", sources);
}
_setSourcesFromSharedData() {
let sources = Services.cpmm.sharedData.get("L10nRegistry:Sources");
if (!sources) {
console.warn(`[l10nregistry] Failed to fetch sources from shared data.`);
return;
}
for (let [name, data] of sources.entries()) {
if (!this.hasSource(name)) {
const source = new FileSource(name, data.locales, data.prePath);
this.registerSource(source);
}
}
for (let name of this.sources.keys()) {
if (!sources.has(name)) {
this.removeSource(name);
}
}
}
/**
* Returns a list of locales for which at least one source
* has resources.
*
* @returns {Array<String>}
*/
getAvailableLocales() {
const locales = new Set();
for (const source of this.sources.values()) {
for (const locale of source.locales) {
locales.add(locale);
}
}
return Array.from(locales);
}
}
/**
* This function generates an iterator over FluentBundles for a single locale
* for a given list of resourceIds for all possible combinations of sources.
*
* This function is called recursively to generate all possible permutations
* and uses the last, optional parameter, to pass the already resolved
* sources order.
*
* Notice: Any changes to this method should be copied
* to the `generateResourceSetsForLocaleSync` equivalent below.
*
* @param {String} locale
* @param {Array} sourcesOrder
* @param {Array} resourceIds
* @param {Array} [resolvedOrder]
* @returns {AsyncIterator<FluentBundle>}
*/
async function* generateResourceSetsForLocale(locale, sourcesOrder, resourceIds, resolvedOrder = []) {
const resolvedLength = resolvedOrder.length;
const resourcesLength = resourceIds.length;
// Inside that loop we have a list of resources and the sources for them, like this:
// ['test.ftl', 'menu.ftl', 'foo.ftl']
// ['app', 'platform', 'app']
for (const sourceName of sourcesOrder) {
const order = resolvedOrder.concat(sourceName);
// We want to bail out early if we know that any of
// the (res)x(source) combinations in the permutation
// are unavailable.
// The combination may have been `undefined` when we
// stepped into this branch, and now is resolved to
// `false`.
//
// If the combination resolved to `false` is the last
// in the resolvedOrder, we want to continue in this
// loop, but if it's somewhere in the middle, we can
// safely bail from the whole branch.
for (let [idx, sourceName] of order.entries()) {
const source = L10nRegistry.sources.get(sourceName);
if (!source || source.hasFile(locale, resourceIds[idx]) === false) {
if (idx === order.length - 1) {
continue;
} else {
return;
}
}
}
// If the number of resolved sources equals the number of resources,
// create the right context and return it if it loads.
if (resolvedLength + 1 === resourcesLength) {
let dataSet = await generateResourceSet(locale, order, resourceIds);
// Here we check again to see if the newly resolved
// resources returned `false` on any position.
if (!dataSet.includes(false)) {
yield dataSet;
}
} else if (resolvedLength < resourcesLength) {
// otherwise recursively load another generator that walks over the
// partially resolved list of sources.
yield * generateResourceSetsForLocale(locale, sourcesOrder, resourceIds, order);
}
}
}
/**
* This is a synchronous version of the `generateResourceSetsForLocale`
* method and should stay completely in sync with it at all
* times except of the async/await changes.
*
* @param {String} locale
* @param {Array} sourcesOrder
* @param {Array} resourceIds
* @param {Array} [resolvedOrder]
* @returns {Iterator<FluentBundle>}
*/
function* generateResourceSetsForLocaleSync(locale, sourcesOrder, resourceIds, resolvedOrder = []) {
const resolvedLength = resolvedOrder.length;
const resourcesLength = resourceIds.length;
// Inside that loop we have a list of resources and the sources for them, like this:
// ['test.ftl', 'menu.ftl', 'foo.ftl']
// ['app', 'platform', 'app']
for (const sourceName of sourcesOrder) {
const order = resolvedOrder.concat(sourceName);
// We want to bail out early if we know that any of
// the (res)x(source) combinations in the permutation
// are unavailable.
// The combination may have been `undefined` when we
// stepped into this branch, and now is resolved to
// `false`.
//
// If the combination resolved to `false` is the last
// in the resolvedOrder, we want to continue in this
// loop, but if it's somewhere in the middle, we can
// safely bail from the whole branch.
for (let [idx, sourceName] of order.entries()) {
const source = L10nRegistry.sources.get(sourceName);
if (!source || source.hasFile(locale, resourceIds[idx]) === false) {
if (idx === order.length - 1) {
continue;
} else {
return;
}
}
}
// If the number of resolved sources equals the number of resources,
// create the right context and return it if it loads.
if (resolvedLength + 1 === resourcesLength) {
let dataSet = generateResourceSetSync(locale, order, resourceIds);
// Here we check again to see if the newly resolved
// resources returned `false` on any position.
if (!dataSet.includes(false)) {
yield dataSet;
}
} else if (resolvedLength < resourcesLength) {
// otherwise recursively load another generator that walks over the
// partially resolved list of sources.
yield * generateResourceSetsForLocaleSync(locale, sourcesOrder, resourceIds, order);
}
}
}
const MSG_CONTEXT_OPTIONS = {
// Temporarily disable bidi isolation due to Microsoft not supporting FSI/PDI.
// See bug 1439018 for details.
useIsolating: Services.prefs.getBoolPref("intl.l10n.enable-bidi-marks", false),
functions: {
/**
* PLATFORM is a built-in allowing localizers to differentiate message
* variants depending on the target platform.
*/
PLATFORM: () => {
switch (AppConstants.platform) {
case "linux":
case "android":
return AppConstants.platform;
case "win":
return "windows";
case "macosx":
return "macos";
default:
return "other";
}
},
},
};
/**
* Pseudolocalizations
*
* PSEUDO_STRATEGIES is a dict of strategies to be used to modify a
* context in order to create pseudolocalizations. These can be used by
* developers to test the localizability of their code without having to
* actually speak a foreign language.
*
* Currently, the following pseudolocales are supported:
*
* accented - Ȧȧƈƈḗḗƞŧḗḗḓ Ḗḗƞɠŀīīşħ
*
* In Accented English all Latin letters are replaced by accented
* Unicode counterparts which don't impair the readability of the content.
* This allows developers to quickly test if any given string is being
* correctly displayed in its 'translated' form. Additionally, simple
* heuristics are used to make certain words longer to better simulate the
* experience of international users.
*
* bidi - ɥsıʅƃuƎ ıpıԐ
*
* Bidi English is a fake RTL locale. All words are surrounded by
* Unicode formatting marks forcing the RTL directionality of characters.
* In addition, to make the reversed text easier to read, individual
* letters are flipped.
*
* Note: The name above is hardcoded to be RTL in case code editors have
* trouble with the RLO and PDF Unicode marks. In reality, it should be
* surrounded by those marks as well.
*
* See https://bugzil.la/1450781 for more information.
*
* In this implementation we use code points instead of inline unicode characters
* because the encoding of JSM files mangles them otherwise.
*/
const ACCENTED_MAP = {
// ȦƁƇḒḖƑƓĦĪĴĶĿḾȠǾƤɊŘŞŦŬṼẆẊẎẐ
"caps": [550, 385, 391, 7698, 7702, 401, 403, 294, 298, 308, 310, 319, 7742, 544, 510, 420, 586, 344, 350, 358, 364, 7804, 7814, 7818, 7822, 7824],
// ȧƀƈḓḗƒɠħīĵķŀḿƞǿƥɋřşŧŭṽẇẋẏẑ
"small": [551, 384, 392, 7699, 7703, 402, 608, 295, 299, 309, 311, 320, 7743, 414, 511, 421, 587, 345, 351, 359, 365, 7805, 7815, 7819, 7823, 7825],
};
const FLIPPED_MAP = {
// ∀ԐↃᗡƎℲ⅁HIſӼ⅂WNOԀÒᴚS⊥∩ɅMX⅄Z
"caps": [8704, 1296, 8579, 5601, 398, 8498, 8513, 72, 73, 383, 1276, 8514, 87, 78, 79, 1280, 210, 7450, 83, 8869, 8745, 581, 77, 88, 8516, 90],
// ɐqɔpǝɟƃɥıɾʞʅɯuodbɹsʇnʌʍxʎz
"small": [592, 113, 596, 112, 477, 607, 387, 613, 305, 638, 670, 645, 623, 117, 111, 100, 98, 633, 115, 647, 110, 652, 653, 120, 654, 122],
};
function transformString(map, elongate = false, prefix = "", postfix = "", msg) {
// Exclude access-keys and other single-char messages
if (msg.length === 1) {
return msg;
}
// XML entities (&#x202a;) and XML tags.
const reExcluded = /(&[#\w]+;|<\s*.+?\s*>)/;
const parts = msg.split(reExcluded);
const modified = parts.map((part) => {
if (reExcluded.test(part)) {
return part;
}
return prefix + part.replace(/[a-z]/ig, (ch) => {
let cc = ch.charCodeAt(0);
if (cc >= 97 && cc <= 122) {
const newChar = String.fromCodePoint(map.small[cc - 97]);
// duplicate "a", "e", "o" and "u" to emulate ~30% longer text
if (elongate && (cc === 97 || cc === 101 || cc === 111 || cc === 117)) {
return newChar + newChar;
}
return newChar;
}
if (cc >= 65 && cc <= 90) {
return String.fromCodePoint(map.caps[cc - 65]);
}
return ch;
}) + postfix;
});
return modified.join("");
}
const PSEUDO_STRATEGIES = {
"accented": transformString.bind(null, ACCENTED_MAP, true, "", ""),
"bidi": transformString.bind(null, FLIPPED_MAP, false, "\u202e", "\u202c"),
};
/**
* Generates a single FluentBundle by loading all resources
* from the listed sources for a given locale.
*
* The function casts all error cases into a Promise that resolves with
* value `null`.
* This allows the caller to be an async generator without using
* try/catch clauses.
*
* Notice: Any changes to this method should be copied
* to the `generateResourceSetSync` equivalent below.
*
* @param {String} locale
* @param {Array} sourcesOrder
* @param {Array} resourceIds
* @returns {Promise<FluentBundle>}
*/
function generateResourceSet(locale, sourcesOrder, resourceIds) {
return Promise.all(resourceIds.map((resourceId, i) => {
const source = L10nRegistry.sources.get(sourcesOrder[i]);
if (!source) {
return false;
}
return source.fetchFile(locale, resourceId);
}));
}
/**
* This is a synchronous version of the `generateResourceSet`
* method and should stay completely in sync with it at all
* times except of the async/await changes.
*
* @param {String} locale
* @param {Array} sourcesOrder
* @param {Array} resourceIds
* @returns {FluentBundle}
*/
function generateResourceSetSync(locale, sourcesOrder, resourceIds) {
return resourceIds.map((resourceId, i) => {
const source = L10nRegistry.sources.get(sourcesOrder[i]);
if (!source) {
return false;
}
return source.fetchFile(locale, resourceId, {sync: true});
});
}
/**
* This is a basic Source for L10nRegistry.
* It registers its own locales and a pre-path, and when asked for a file
* it attempts to download and cache it.
*
* The Source caches the downloaded files so any consecutive loads will
* come from the cache.
**/
class FileSource {
/**
* @param {string} name
* @param {Array<string>} locales
* @param {string} prePath
*
* @returns {FileSource}
*/
constructor(name, locales, prePath) {
this.name = name;
this.locales = locales;
this.prePath = prePath;
this.indexed = false;
// The cache object stores information about the resources available
// in the Source.
//
// It can take one of three states:
// * true - the resource is available but not fetched yet
// * false - the resource is not available
// * Promise - the resource has been fetched
//
// If the cache has no entry for a given path, that means that there
// is no information available about whether the resource is available.
//
// If the `indexed` property is set to `true` it will be treated as the
// resource not being available. Otherwise, the resource may be
// available and we do not have any information about it yet.
this.cache = {};
}
getPath(locale, path) {
// This is a special case for the only not BCP47-conformant locale
// code we have resources for.
if (locale === "ja-JP-macos") {
locale = "ja-JP-mac";
}
return (this.prePath + path).replace(/\{locale\}/g, locale);
}
hasFile(locale, path) {
if (!this.locales.includes(locale)) {
return false;
}
const fullPath = this.getPath(locale, path);
if (!this.cache.hasOwnProperty(fullPath)) {
return this.indexed ? false : undefined;
}
if (this.cache[fullPath] === false) {
return false;
}
if (this.cache[fullPath].then) {
return undefined;
}
return true;
}
fetchFile(locale, path, options = {sync: false}) {
if (!this.locales.includes(locale)) {
return false;
}
const fullPath = this.getPath(locale, path);
if (this.cache.hasOwnProperty(fullPath)) {
if (this.cache[fullPath] === false) {
return false;
}
// `true` means that the file is indexed, but hasn't
// been fetched yet.
if (this.cache[fullPath] !== true) {
if (this.cache[fullPath] instanceof Promise && options.sync) {
console.warn(`[l10nregistry] Attempting to synchronously load file
${fullPath} while it's being loaded asynchronously.`);
} else {
return this.cache[fullPath];
}
}
} else if (this.indexed) {
return false;
}
if (options.sync) {
let data = L10nRegistry.loadSync(fullPath);
if (data === false) {
this.cache[fullPath] = false;
} else {
this.cache[fullPath] = new FluentResource(data);
}
return this.cache[fullPath];
}
// async
return this.cache[fullPath] = L10nRegistry.load(fullPath).then(
data => {
return this.cache[fullPath] = new FluentResource(data);
},
err => {
this.cache[fullPath] = false;
return false;
}
);
}
}
/**
* This is an extension of the FileSource which should be used
* for sources that can provide the list of files available in the source.
*
* This allows for a faster lookup in cases where the source does not
* contain most of the files that the app will request for (e.g. an addon).
**/
class IndexedFileSource extends FileSource {
/**
* @param {string} name
* @param {Array<string>} locales
* @param {string} prePath
* @param {Array<string>} paths
*
* @returns {IndexedFileSource}
*/
constructor(name, locales, prePath, paths) {
super(name, locales, prePath);
this.indexed = true;
for (const path of paths) {
this.cache[path] = true;
}
}
}
this.L10nRegistry = new L10nRegistryService();
/**
* The low level wrapper around Fetch API. It unifies the error scenarios to
* always produce a promise rejection.
*
* We keep it as a method to make it easier to override for testing purposes.
*
* @param {string} url
*
* @returns {Promise<string>}
*/
this.L10nRegistry.load = function(url) {
return fetch(url).then(response => {
if (!response.ok) {
return Promise.reject(response.statusText);
}
return response.text();
});
};
/**
* This is a synchronous version of the `load`
* function and should stay completely in sync with it at all
* times except of the async/await changes.
*
* Notice: Any changes to this method should be copied
* to the `generateResourceSetSync` equivalent below.
*
* @param {string} url
*
* @returns {string}
*/
this.L10nRegistry.loadSync = function(uri) {
try {
let url = Services.io.newURI(uri);
let data = Cu.readUTF8URI(url);
return data;
} catch (e) {
if (
e.result == Cr.NS_ERROR_INVALID_ARG ||
e.result == Cr.NS_ERROR_NOT_INITIALIZED
) {
try {
// The preloader doesn't support this url or isn't initialized
// (xpcshell test). Try a synchronous channel load.
let stream = NetUtil.newChannel({
uri,
loadUsingSystemPrincipal: true,
}).open();
return NetUtil.readInputStreamToString(stream, stream.available(), {
charset: "UTF-8",
});
} catch (e) {
Cu.reportError(e);
}
} else if (e.result != Cr.NS_ERROR_FILE_NOT_FOUND) {
Cu.reportError(e);
}
}
return false;
};
this.FileSource = FileSource;
this.IndexedFileSource = IndexedFileSource;
var EXPORTED_SYMBOLS = ["L10nRegistry", "FileSource", "IndexedFileSource"];