Bug 1539192 - Update to Fluent.jsm 0.12.0, FluentSyntax 0.12.0. r=zbraniecki

Update the vendored Fluent libraries to their latest versions, both supporting Fluent Syntax 0.9.

Differential Revision: https://phabricator.services.mozilla.com/D25043

--HG--
extra : moz-landing-system : lando

intl/l10n/Fluent.jsm

@@ -1,6 +1,6 @@
 /* vim: set ts=2 et sw=2 tw=80 filetype=javascript: */
 
-/* Copyright 2017 Mozilla Foundation and others
+/* Copyright 2019 Mozilla Foundation and others
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -16,7 +16,7 @@
  */
 
 
-/* fluent@0.10.0 */
+/* fluent@0.12.0 */
 
 /* global Intl */
 
@@ -138,53 +138,7 @@ function values(opts) {
   return unwrapped;
 }
 
-/**
- * @overview
- *
- * The role of the Fluent resolver is to format a translation object to an
- * instance of `FluentType` or an array of instances.
- *
- * Translations can contain references to other messages or variables,
- * conditional logic in form of select expressions, traits which describe their
- * grammatical features, and can use Fluent builtins which make use of the
- * `Intl` formatters to format numbers, dates, lists and more into the
- * bundle's language. See the documentation of the Fluent syntax for more
- * information.
- *
- * In case of errors the resolver will try to salvage as much of the
- * translation as possible.  In rare situations where the resolver didn't know
- * how to recover from an error it will return an instance of `FluentNone`.
- *
- * `MessageReference`, `VariantExpression`, `AttributeExpression` and
- * `SelectExpression` resolve to raw Runtime Entries objects and the result of
- * the resolution needs to be passed into `Type` to get their real value.
- * This is useful for composing expressions.  Consider:
- *
- *     brand-name[nominative]
- *
- * which is a `VariantExpression` with properties `id: MessageReference` and
- * `key: Keyword`.  If `MessageReference` was resolved eagerly, it would
- * instantly resolve to the value of the `brand-name` message.  Instead, we
- * want to get the message object and look for its `nominative` variant.
- *
- * All other expressions (except for `FunctionReference` which is only used in
- * `CallExpression`) resolve to an instance of `FluentType`.  The caller should
- * use the `toString` method to convert the instance to a native value.
- *
- *
- * All functions in this file pass around a special object called `env`.
- * This object stores a set of elements used by all resolve functions:
- *
- *  * {FluentBundle} bundle
- *      bundle for which the given resolution is happening
- *  * {Object} args
- *      list of developer provided arguments that can be used
- *  * {Array} errors
- *      list of errors collected while resolving
- *  * {WeakSet} dirty
- *      Set of patterns already encountered during this resolution.
- *      This is used to prevent cyclic resolutions.
- */
+/* global Intl */
 
 // Prevent expansion of too long placeables.
 const MAX_PLACEABLE_LENGTH = 2500;
@@ -201,6 +155,7 @@ function match(bundle, selector, key) {
     return true;
   }
 
+  // XXX Consider comparing options too, e.g. minimumFractionDigits.
   if (key instanceof FluentNumber
     && selector instanceof FluentNumber
     && key.value === selector.value) {
@@ -220,28 +175,25 @@ function match(bundle, selector, key) {
 }
 
 // Helper: resolve the default variant from a list of variants.
-function getDefault(env, variants, star) {
+function getDefault(scope, variants, star) {
   if (variants[star]) {
-    return Type(env, variants[star]);
+    return Type(scope, variants[star]);
   }
 
-  const { errors } = env;
-  errors.push(new RangeError("No default"));
+  scope.errors.push(new RangeError("No default"));
   return new FluentNone();
 }
 
 // Helper: resolve arguments to a call expression.
-function getArguments(env, args) {
+function getArguments(scope, args) {
   const positional = [];
   const named = {};
 
-  if (args) {
-    for (const arg of args) {
-      if (arg.type === "narg") {
-        named[arg.name] = Type(env, arg.value);
-      } else {
-        positional.push(Type(env, arg));
-      }
+  for (const arg of args) {
+    if (arg.type === "narg") {
+      named[arg.name] = Type(scope, arg.value);
+    } else {
+      positional.push(Type(scope, arg));
     }
   }
 
@@ -249,12 +201,12 @@ function getArguments(env, args) {
 }
 
 // Resolve an expression to a Fluent type.
-function Type(env, expr) {
+function Type(scope, expr) {
   // A fast-path for strings which are the most common case. Since they
   // natively have the `toString` method they can be used as if they were
   // a FluentType instance without incurring the cost of creating one.
   if (typeof expr === "string") {
-    return env.bundle._transform(expr);
+    return scope.bundle._transform(expr);
   }
 
   // A fast-path for `FluentNone` which doesn't require any additional logic.
@@ -265,32 +217,33 @@ function Type(env, expr) {
   // The Runtime AST (Entries) encodes patterns (complex strings with
   // placeables) as Arrays.
   if (Array.isArray(expr)) {
-    return Pattern(env, expr);
+    return Pattern(scope, expr);
   }
 
   switch (expr.type) {
     case "str":
       return expr.value;
     case "num":
-      return new FluentNumber(expr.value);
+      return new FluentNumber(expr.value, {
+        minimumFractionDigits: expr.precision,
+      });
     case "var":
-      return VariableReference(env, expr);
+      return VariableReference(scope, expr);
+    case "mesg":
+      return MessageReference(scope, expr);
     case "term":
-      return TermReference({...env, args: {}}, expr);
-    case "ref":
-      return expr.args
-        ? FunctionReference(env, expr)
-        : MessageReference(env, expr);
+      return TermReference(scope, expr);
+    case "func":
+      return FunctionReference(scope, expr);
     case "select":
-      return SelectExpression(env, expr);
+      return SelectExpression(scope, expr);
     case undefined: {
       // If it's a node with a value, resolve the value.
       if (expr.value !== null && expr.value !== undefined) {
-        return Type(env, expr.value);
+        return Type(scope, expr.value);
       }
 
-      const { errors } = env;
-      errors.push(new RangeError("No value"));
+      scope.errors.push(new RangeError("No value"));
       return new FluentNone();
     }
     default:
@@ -299,15 +252,15 @@ function Type(env, expr) {
 }
 
 // Resolve a reference to a variable.
-function VariableReference(env, {name}) {
-  const { args, errors } = env;
-
-  if (!args || !args.hasOwnProperty(name)) {
-    errors.push(new ReferenceError(`Unknown variable: ${name}`));
+function VariableReference(scope, {name}) {
+  if (!scope.args || !scope.args.hasOwnProperty(name)) {
+    if (scope.insideTermReference === false) {
+      scope.errors.push(new ReferenceError(`Unknown variable: ${name}`));
+    }
     return new FluentNone(`$${name}`);
   }
 
-  const arg = args[name];
+  const arg = scope.args[name];
 
   // Return early if the argument already is an instance of FluentType.
   if (arg instanceof FluentType) {
@@ -325,7 +278,7 @@ function VariableReference(env, {name}) {
         return new FluentDateTime(arg);
       }
     default:
-      errors.push(
+      scope.errors.push(
         new TypeError(`Unsupported variable type: ${name}, ${typeof arg}`)
       );
       return new FluentNone(`$${name}`);
@@ -333,89 +286,69 @@ function VariableReference(env, {name}) {
 }
 
 // Resolve a reference to another message.
-function MessageReference(env, {name, attr}) {
-  const {bundle, errors} = env;
-  const message = bundle._messages.get(name);
+function MessageReference(scope, {name, attr}) {
+  const message = scope.bundle._messages.get(name);
   if (!message) {
     const err = new ReferenceError(`Unknown message: ${name}`);
-    errors.push(err);
+    scope.errors.push(err);
     return new FluentNone(name);
   }
 
   if (attr) {
     const attribute = message.attrs && message.attrs[attr];
     if (attribute) {
-      return Type(env, attribute);
+      return Type(scope, attribute);
     }
-    errors.push(new ReferenceError(`Unknown attribute: ${attr}`));
-    return Type(env, message);
+    scope.errors.push(new ReferenceError(`Unknown attribute: ${attr}`));
+    return Type(scope, message);
   }
 
-  return Type(env, message);
+  return Type(scope, message);
 }
 
 // Resolve a call to a Term with key-value arguments.
-function TermReference(env, {name, attr, selector, args}) {
-  const {bundle, errors} = env;
-
+function TermReference(scope, {name, attr, args}) {
   const id = `-${name}`;
-  const term = bundle._terms.get(id);
+  const term = scope.bundle._terms.get(id);
   if (!term) {
     const err = new ReferenceError(`Unknown term: ${id}`);
-    errors.push(err);
+    scope.errors.push(err);
     return new FluentNone(id);
   }
 
   // Every TermReference has its own args.
-  const [, keyargs] = getArguments(env, args);
-  const local = {...env, args: keyargs};
+  const [, keyargs] = getArguments(scope, args);
+  const local = {...scope, args: keyargs, insideTermReference: true};
 
   if (attr) {
     const attribute = term.attrs && term.attrs[attr];
     if (attribute) {
       return Type(local, attribute);
     }
-    errors.push(new ReferenceError(`Unknown attribute: ${attr}`));
+    scope.errors.push(new ReferenceError(`Unknown attribute: ${attr}`));
     return Type(local, term);
   }
 
-  const variantList = getVariantList(term);
-  if (selector && variantList) {
-    return SelectExpression(local, {...variantList, selector});
-  }
-
   return Type(local, term);
 }
 
-// Helper: convert a value into a variant list, if possible.
-function getVariantList(term) {
-  const value = term.value || term;
-  return Array.isArray(value)
-    && value[0].type === "select"
-    && value[0].selector === null
-    ? value[0]
-    : null;
-}
-
 // Resolve a call to a Function with positional and key-value arguments.
-function FunctionReference(env, {name, args}) {
+function FunctionReference(scope, {name, args}) {
   // Some functions are built-in. Others may be provided by the runtime via
   // the `FluentBundle` constructor.
-  const {bundle: {_functions}, errors} = env;
-  const func = _functions[name] || builtins[name];
-
+  const func = scope.bundle._functions[name] || builtins[name];
   if (!func) {
-    errors.push(new ReferenceError(`Unknown function: ${name}()`));
+    scope.errors.push(new ReferenceError(`Unknown function: ${name}()`));
     return new FluentNone(`${name}()`);
   }
 
   if (typeof func !== "function") {
-    errors.push(new TypeError(`Function ${name}() is not callable`));
+    scope.errors.push(new TypeError(`Function ${name}() is not callable`));
     return new FluentNone(`${name}()`);
   }
 
   try {
-    return func(...getArguments(env, args));
+    return func(...getArguments(scope, args));
   } catch (e) {
     // XXX Report errors.
     return new FluentNone();
@@ -423,60 +356,54 @@ function FunctionReference(env, {name, args}) {
 }
 
 // Resolve a select expression to the member object.
-function SelectExpression(env, {selector, variants, star}) {
-  if (selector === null) {
-    return getDefault(env, variants, star);
-  }
-
-  let sel = Type(env, selector);
+function SelectExpression(scope, {selector, variants, star}) {
+  let sel = Type(scope, selector);
   if (sel instanceof FluentNone) {
-    const variant = getDefault(env, variants, star);
-    return Type(env, variant);
+    const variant = getDefault(scope, variants, star);
+    return Type(scope, variant);
   }
 
   // Match the selector against keys of each variant, in order.
   for (const variant of variants) {
-    const key = Type(env, variant.key);
-    if (match(env.bundle, sel, key)) {
-      return Type(env, variant);
+    const key = Type(scope, variant.key);
+    if (match(scope.bundle, sel, key)) {
+      return Type(scope, variant);
     }
   }
 
-  const variant = getDefault(env, variants, star);
-  return Type(env, variant);
+  const variant = getDefault(scope, variants, star);
+  return Type(scope, variant);
 }
 
 // Resolve a pattern (a complex string with placeables).
-function Pattern(env, ptn) {
-  const { bundle, dirty, errors } = env;
-
-  if (dirty.has(ptn)) {
-    errors.push(new RangeError("Cyclic reference"));
+function Pattern(scope, ptn) {
+  if (scope.dirty.has(ptn)) {
+    scope.errors.push(new RangeError("Cyclic reference"));
     return new FluentNone();
   }
 
   // Tag the pattern as dirty for the purpose of the current resolution.
-  dirty.add(ptn);
+  scope.dirty.add(ptn);
   const result = [];
 
   // Wrap interpolations with Directional Isolate Formatting characters
   // only when the pattern has more than one element.
-  const useIsolating = bundle._useIsolating && ptn.length > 1;
+  const useIsolating = scope.bundle._useIsolating && ptn.length > 1;
 
   for (const elem of ptn) {
     if (typeof elem === "string") {
-      result.push(bundle._transform(elem));
+      result.push(scope.bundle._transform(elem));
       continue;
     }
 
-    const part = Type(env, elem).toString(bundle);
+    const part = Type(scope, elem).toString(scope.bundle);
 
     if (useIsolating) {
       result.push(FSI);
     }
 
     if (part.length > MAX_PLACEABLE_LENGTH) {
-      errors.push(
+      scope.errors.push(
         new RangeError(
           "Too many characters in placeable " +
           `(${part.length}, max allowed is ${MAX_PLACEABLE_LENGTH})`
@@ -492,7 +419,7 @@ function Pattern(env, ptn) {
     }
   }
 
-  dirty.delete(ptn);
+  scope.dirty.delete(ptn);
   return result.join("");
 }
 
@@ -512,10 +439,12 @@ function Pattern(env, ptn) {
  * @returns {FluentType}
  */
 function resolve(bundle, args, message, errors = []) {
-  const env = {
+  const scope = {
     bundle, args, errors, dirty: new WeakSet(),
+    // TermReferences are resolved in a new scope.
+    insideTermReference: false,
   };
-  return Type(env, message).toString(bundle);
+  return Type(scope, message).toString(bundle);
 }
 
 class FluentError extends Error {}
@@ -529,9 +458,10 @@ const RE_MESSAGE_START = /^(-?[a-zA-Z][\w-]*) *= */mg;
 const RE_ATTRIBUTE_START = /\.([a-zA-Z][\w-]*) *= */y;
 const RE_VARIANT_START = /\*?\[/y;
 
-const RE_NUMBER_LITERAL = /(-?[0-9]+(\.[0-9]+)?)/y;
+const RE_NUMBER_LITERAL = /(-?[0-9]+(?:\.([0-9]+))?)/y;
 const RE_IDENTIFIER = /([a-zA-Z][\w-]*)/y;
 const RE_REFERENCE = /([$-])?([a-zA-Z][\w-]*)(?:\.([a-zA-Z][\w-]*))?/y;
+const RE_FUNCTION_NAME = /^[A-Z][A-Z0-9_-]*$/;
 
 // A "run" is a sequence of text or string literal characters which don't
 // require any special handling. For TextElements such special characters are: {
@@ -791,13 +721,6 @@ class FluentResource extends Map {
     function parsePlaceable() {
       consumeToken(TOKEN_BRACE_OPEN, FluentError);
 
-      // VariantLists are parsed as selector-less SelectExpressions.
-      let onlyVariants = parseVariants();
-      if (onlyVariants) {
-        consumeToken(TOKEN_BRACE_CLOSE, FluentError);
-        return {type: "select", selector: null, ...onlyVariants};
-      }
-
       let selector = parseInlineExpression();
       if (consumeToken(TOKEN_BRACE_CLOSE)) {
         return selector;
@@ -820,18 +743,32 @@ class FluentResource extends Map {
 
       if (test(RE_REFERENCE)) {
         let [, sigil, name, attr = null] = match(RE_REFERENCE);
-        let type = {"$": "var", "-": "term"}[sigil] || "ref";
 
-        if (source[cursor] === "[") {
-          // DEPRECATED VariantExpressions will be removed before 1.0.
-          return {type, name, selector: parseVariantKey()};
+        if (sigil === "$") {
+          return {type: "var", name};
         }
 
         if (consumeToken(TOKEN_PAREN_OPEN)) {
-          return {type, name, attr, args: parseArguments()};
+          let args = parseArguments();
+
+          if (sigil === "-") {
+            // A parameterized term: -term(...).
+            return {type: "term", name, attr, args};
+          }
+
+          if (RE_FUNCTION_NAME.test(name)) {
+            return {type: "func", name, args};
+          }
+
+          throw new FluentError("Function names must be all upper-case");
         }
 
-        return {type, name, attr, args: null};
+        if (sigil === "-") {
+          // A non-parameterized term: -term.
+          return {type: "term", name, attr, args: []};
+        }
+
+        return {type: "mesg", name, attr};
       }
 
       return parseLiteral();
@@ -855,18 +792,18 @@ class FluentResource extends Map {
     }
 
     function parseArgument() {
-      let ref = parseInlineExpression();
-      if (ref.type !== "ref") {
-        return ref;
+      let expr = parseInlineExpression();
+      if (expr.type !== "mesg") {
+        return expr;
       }
 
       if (consumeToken(TOKEN_COLON)) {
         // The reference is the beginning of a named argument.
-        return {type: "narg", name: ref.name, value: parseLiteral()};
+        return {type: "narg", name: expr.name, value: parseLiteral()};
       }
 
       // It's a regular message reference.
-      return ref;
+      return expr;
     }
 
     function parseVariants() {
@@ -920,7 +857,9 @@ class FluentResource extends Map {
     }
 
     function parseNumberLiteral() {
-      return {type: "num", value: match1(RE_NUMBER_LITERAL)};
+      let [, value, fraction = ""] = match(RE_NUMBER_LITERAL);
+      let precision = fraction.length;
+      return {type: "num", value: parseFloat(value), precision};
     }
 
     function parseStringLiteral() {
@@ -1052,6 +991,7 @@ class FluentBundle {
    *
    *   - `useIsolating` - boolean specifying whether to use Unicode isolation
    *                    marks (FSI, PDI) for bidi interpolations.
+   *                    Default: true
    *
    *   - `transform` - a function used to transform string parts of patterns.
    *
@@ -1118,15 +1058,28 @@ class FluentBundle {
    *
    *     // Returns a raw representation of the 'foo' message.
    *
+   *     bundle.addMessages('bar = Bar');
+   *     bundle.addMessages('bar = Newbar', { allowOverrides: true });
+   *     bundle.getMessage('bar');
+   *
+   *     // Returns a raw representation of the 'bar' message: Newbar.
+   *
    * Parsed entities should be formatted with the `format` method in case they
    * contain logic (references, select expressions etc.).
    *
+   * Available options:
+   *
+   *   - `allowOverrides` - boolean specifying whether it's allowed to override
+   *                      an existing message or term with a new value.
+   *                      Default: false
+   *
    * @param   {string} source - Text resource with translations.
+   * @param   {Object} [options]
    * @returns {Array<Error>}
    */
-  addMessages(source) {
+  addMessages(source, options) {
     const res = FluentResource.fromString(source);
-    return this.addResource(res);
+    return this.addResource(res, options);
   }
 
   /**
@@ -1141,26 +1094,43 @@ class FluentBundle {
    *
    *     // Returns a raw representation of the 'foo' message.
    *
+   *     let res = FluentResource.fromString("bar = Bar");
+   *     bundle.addResource(res);
+   *     res = FluentResource.fromString("bar = Newbar");
+   *     bundle.addResource(res, { allowOverrides: true });
+   *     bundle.getMessage('bar');
+   *
+   *     // Returns a raw representation of the 'bar' message: Newbar.
+   *
    * Parsed entities should be formatted with the `format` method in case they
    * contain logic (references, select expressions etc.).
    *
+   * Available options:
+   *
+   *   - `allowOverrides` - boolean specifying whether it's allowed to override
+   *                      an existing message or term with a new value.
+   *                      Default: false
+   *
    * @param   {FluentResource} res - FluentResource object.
+   * @param   {Object} [options]
    * @returns {Array<Error>}
    */
-  addResource(res) {
+  addResource(res, {
+    allowOverrides = false,
+  } = {}) {
     const errors = [];
 
     for (const [id, value] of res) {
       if (id.startsWith("-")) {
         // Identifiers starting with a dash (-) define terms. Terms are private
         // and cannot be retrieved from FluentBundle.
-        if (this._terms.has(id)) {
+        if (allowOverrides === false && this._terms.has(id)) {
           errors.push(`Attempt to override an existing term: "${id}"`);
           continue;
         }
         this._terms.set(id, value);
       } else {
-        if (this._messages.has(id)) {
+        if (allowOverrides === false && this._messages.has(id)) {
           errors.push(`Attempt to override an existing message: "${id}"`);
           continue;
         }
@@ -1233,6 +1203,22 @@ class FluentBundle {
   }
 }
 
-this.FluentBundle = FluentBundle;
-this.FluentResource = FluentResource;
-var EXPORTED_SYMBOLS = ["FluentBundle", "FluentResource"];
+/*
+ * @module fluent
+ * @overview
+ *
+ * `fluent` is a JavaScript implementation of Project Fluent, a localization
+ * framework designed to unleash the expressive power of the natural language.
+ *
+ */
+
+this.EXPORTED_SYMBOLS = [
+  ...Object.keys({
+    FluentBundle,
+    FluentResource,
+    FluentError,
+    FluentType,
+    FluentNumber,
+    FluentDateTime,
+  }),
+];

intl/l10n/FluentSyntax.jsm

@@ -16,7 +16,7 @@
  */
 
 
-/* fluent-syntax@0.10.0 */
+/* fluent-syntax@0.12.0 */
 
 /*
  * Base class for all Fluent AST nodes.
@@ -27,6 +27,67 @@
  */
 class BaseNode {
   constructor() {}
+
+  equals(other, ignoredFields = ["span"]) {
+    const thisKeys = new Set(Object.keys(this));
+    const otherKeys = new Set(Object.keys(other));
+    if (ignoredFields) {
+      for (const fieldName of ignoredFields) {
+        thisKeys.delete(fieldName);
+        otherKeys.delete(fieldName);
+      }
+    }
+    if (thisKeys.size !== otherKeys.size) {
+      return false;
+    }
+    for (const fieldName of thisKeys) {
+      if (!otherKeys.has(fieldName)) {
+        return false;
+      }
+      const thisVal = this[fieldName];
+      const otherVal = other[fieldName];
+      if (typeof thisVal !== typeof otherVal) {
+        return false;
+      }
+      if (thisVal instanceof Array) {
+        if (thisVal.length !== otherVal.length) {
+          return false;
+        }
+        for (let i = 0; i < thisVal.length; ++i) {
+          if (!scalarsEqual(thisVal[i], otherVal[i], ignoredFields)) {
+            return false;
+          }
+        }
+      } else if (!scalarsEqual(thisVal, otherVal, ignoredFields)) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  clone() {
+    function visit(value) {
+      if (value instanceof BaseNode) {
+        return value.clone();
+      }
+      if (Array.isArray(value)) {
+        return value.map(visit);
+      }
+      return value;
+    }
+    const clone = Object.create(this.constructor.prototype);
+    for (const prop of Object.keys(this)) {
+      clone[prop] = visit(this[prop]);
+    }
+    return clone;
+  }
+}
+
+function scalarsEqual(thisVal, otherVal, ignoredFields) {
+  if (thisVal instanceof BaseNode) {
+    return thisVal.equals(otherVal, ignoredFields);
+  }
+  return thisVal === otherVal;
 }
 
 /*
@@ -73,14 +134,6 @@ class Term extends Entry {
   }
 }
 
-class VariantList extends SyntaxNode {
-  constructor(variants) {
-    super();
-    this.type = "VariantList";
-    this.variants = variants;
-  }
-}
-
 class Pattern extends SyntaxNode {
   constructor(elements) {
     super();
@@ -115,36 +168,87 @@ class Placeable extends PatternElement {
  */
 class Expression extends SyntaxNode {}
 
-class StringLiteral extends Expression {
-  constructor(raw, value) {
+// An abstract base class for Literals.
+class Literal extends Expression {
+  constructor(value) {
     super();
-    this.type = "StringLiteral";
-    this.raw = raw;
+    // The "value" field contains the exact contents of the literal,
+    // character-for-character.
     this.value = value;
   }
+
+  parse() {
+    return {value: this.value};
+  }
 }
 
-class NumberLiteral extends Expression {
+class StringLiteral extends Literal {
   constructor(value) {
-    super();
+    super(value);
+    this.type = "StringLiteral";
+  }
+
+  parse() {
+    // Backslash backslash, backslash double quote, uHHHH, UHHHHHH.
+    const KNOWN_ESCAPES =
+      /(?:\\\\|\\"|\\u([0-9a-fA-F]{4})|\\U([0-9a-fA-F]{6}))/g;
+
+    function from_escape_sequence(match, codepoint4, codepoint6) {
+      switch (match) {
+        case "\\\\":
+          return "\\";
+        case "\\\"":
+          return "\"";
+        default:
+          let codepoint = parseInt(codepoint4 || codepoint6, 16);
+          if (codepoint <= 0xD7FF || 0xE000 <= codepoint) {
+            // It's a Unicode scalar value.
+            return String.fromCodePoint(codepoint);
+          }
+          // Escape sequences reresenting surrogate code points are
+          // well-formed but invalid in Fluent. Replace them with U+FFFD
+          // REPLACEMENT CHARACTER.
+          return "<22>";
+      }
+    }
+
+    let value = this.value.replace(KNOWN_ESCAPES, from_escape_sequence);
+    return {value};
+  }
+}
+
+class NumberLiteral extends Literal {
+  constructor(value) {
+    super(value);
     this.type = "NumberLiteral";
-    this.value = value;
+  }
+
+  parse() {
+    let value = parseFloat(this.value);
+    let decimal_position = this.value.indexOf(".");
+    let precision = decimal_position > 0
+      ? this.value.length - decimal_position - 1
+      : 0;
+    return {value, precision};
   }
 }
 
 class MessageReference extends Expression {
-  constructor(id) {
+  constructor(id, attribute = null) {
     super();
     this.type = "MessageReference";
     this.id = id;
+    this.attribute = attribute;
   }
 }
 
 class TermReference extends Expression {
-  constructor(id) {
+  constructor(id, attribute = null, args = null) {
     super();
     this.type = "TermReference";
     this.id = id;
+    this.attribute = attribute;
+    this.arguments = args;
   }
 }
 
@@ -157,10 +261,11 @@ class VariableReference extends Expression {
 }
 
 class FunctionReference extends Expression {
-  constructor(id) {
+  constructor(id, args) {
     super();
     this.type = "FunctionReference";
     this.id = id;
+    this.arguments = args;
   }
 }
 
@@ -173,29 +278,10 @@ class SelectExpression extends Expression {
   }
 }
 
-class AttributeExpression extends Expression {
-  constructor(ref, name) {
+class CallArguments extends SyntaxNode {
+  constructor(positional = [], named = []) {
     super();
-    this.type = "AttributeExpression";
-    this.ref = ref;
-    this.name = name;
-  }
-}
-
-class VariantExpression extends Expression {
-  constructor(ref, key) {
-    super();
-    this.type = "VariantExpression";
-    this.ref = ref;
-    this.key = key;
-  }
-}
-
-class CallExpression extends Expression {
-  constructor(callee, positional = [], named = []) {
-    super();
-    this.type = "CallExpression";
-    this.callee = callee;
+    this.type = "CallArguments";
     this.positional = positional;
     this.named = named;
   }
@@ -292,22 +378,23 @@ class Annotation extends SyntaxNode {
     super();
     this.type = "Annotation";
     this.code = code;
-    this.args = args;
+    this.arguments = args;
     this.message = message;
   }
 }
 
 const ast = ({
+  BaseNode: BaseNode,
   Resource: Resource,
   Entry: Entry,
   Message: Message,
   Term: Term,
-  VariantList: VariantList,
   Pattern: Pattern,
   PatternElement: PatternElement,
   TextElement: TextElement,
   Placeable: Placeable,
   Expression: Expression,
+  Literal: Literal,
   StringLiteral: StringLiteral,
   NumberLiteral: NumberLiteral,
   MessageReference: MessageReference,
@@ -315,9 +402,7 @@ const ast = ({
   VariableReference: VariableReference,
   FunctionReference: FunctionReference,
   SelectExpression: SelectExpression,
-  AttributeExpression: AttributeExpression,
-  VariantExpression: VariantExpression,
-  CallExpression: CallExpression,
+  CallArguments: CallArguments,
   Attribute: Attribute,
   Variant: Variant,
   NamedArgument: NamedArgument,
@@ -368,7 +453,7 @@ function getErrorMessage(code, args) {
     case "E0008":
       return "The callee has to be an upper-case identifier or a term";
     case "E0009":
-      return "The key has to be a simple identifier";
+      return "The argument name has to be a simple identifier";
     case "E0010":
       return "Expected one of the variants to be marked as default (*)";
     case "E0011":
@@ -788,10 +873,9 @@ class FluentParser {
     // Poor man's decorators.
     const methodNames = [
       "getComment", "getMessage", "getTerm", "getAttribute", "getIdentifier",
-      "getVariant", "getNumber", "getPattern", "getVariantList",
-      "getTextElement", "getPlaceable", "getExpression",
-      "getInlineExpression", "getCallArgument", "getString",
-      "getSimpleExpression", "getLiteral",
+      "getVariant", "getNumber", "getPattern", "getTextElement",
+      "getPlaceable", "getExpression", "getInlineExpression",
+      "getCallArgument", "getCallArguments", "getString", "getLiteral",
     ];
     for (const name of methodNames) {
       this[name] = withSpan(this[name]);
@@ -994,9 +1078,7 @@ class FluentParser {
     ps.skipBlankInline();
     ps.expectChar("=");
 
-    // Syntax 0.8 compat: VariantLists are supported but deprecated. They can
-    // only be found as values of Terms. Nested VariantLists are not allowed.
-    const value = this.maybeGetVariantList(ps) || this.maybeGetPattern(ps);
+    const value = this.maybeGetPattern(ps);
     if (value === null) {
       throw new ParseError("E0006", id.name);
     }
@@ -1132,22 +1214,21 @@ class FluentParser {
   }
 
   getNumber(ps) {
-    let num = "";
+    let value = "";
 
     if (ps.currentChar === "-") {
-      num += "-";
       ps.next();
+      value += `-${this.getDigits(ps)}`;
+    } else {
+      value += this.getDigits(ps);
     }
 
-    num = `${num}${this.getDigits(ps)}`;
-
     if (ps.currentChar === ".") {
-      num += ".";
       ps.next();
-      num = `${num}${this.getDigits(ps)}`;
+      value += `.${this.getDigits(ps)}`;
     }
 
-    return new NumberLiteral(num);
+    return new NumberLiteral(value);
   }
 
   // maybeGetPattern distinguishes between patterns which start on the same line
@@ -1172,36 +1253,6 @@ class FluentParser {
     return null;
   }
 
-  // Deprecated in Syntax 0.8. VariantLists are only allowed as values of Terms.
-  // Values of Messages, Attributes and Variants must be Patterns. This method
-  // is only used in getTerm.
-  maybeGetVariantList(ps) {
-    ps.peekBlank();
-    if (ps.currentPeek === "{") {
-      const start = ps.peekOffset;
-      ps.peek();
-      ps.peekBlankInline();
-      if (ps.currentPeek === EOL) {
-        ps.peekBlank();
-        if (ps.isVariantStart()) {
-          ps.resetPeek(start);
-          ps.skipToPeek();
-          return this.getVariantList(ps);
-        }
-      }
-    }
-
-    ps.resetPeek();
-    return null;
-  }
-
-  getVariantList(ps) {
-    ps.expectChar("{");
-    var variants = this.getVariants(ps);
-    ps.expectChar("}");
-    return new VariantList(variants);
-  }
-
   getPattern(ps, {isBlock}) {
     const elements = [];
     if (isBlock) {
@@ -1212,7 +1263,7 @@ class FluentParser {
       elements.push(this.getIndent(ps, firstIndent, blankStart));
       var commonIndentLength = firstIndent.length;
     } else {
-      var commonIndentLength = Infinity;
+      commonIndentLength = Infinity;
     }
 
     let ch;
@@ -1343,7 +1394,7 @@ class FluentParser {
       case "\\":
       case "\"":
         ps.next();
-        return [`\\${next}`, next];
+        return `\\${next}`;
       case "u":
         return this.getUnicodeEscapeSequence(ps, next, 4);
       case "U":
@@ -1368,15 +1419,7 @@ class FluentParser {
       sequence += ch;
     }
 
-    const codepoint = parseInt(sequence, 16);
-    const unescaped = codepoint <= 0xD7FF || 0xE000 <= codepoint
-      // It's a Unicode scalar value.
-      ? String.fromCodePoint(codepoint)
-      // Escape sequences reresenting surrogate code points are well-formed
-      // but invalid in Fluent. Replace them with U+FFFD REPLACEMENT
-      // CHARACTER.
-      : "<22>";
-    return [`\\${u}${sequence}`, unescaped];
+    return `\\${u}${sequence}`;
   }
 
   getPlaceable(ps) {
@@ -1398,21 +1441,14 @@ class FluentParser {
       }
 
       if (selector.type === "MessageReference") {
-        throw new ParseError("E0016");
+        if (selector.attribute === null) {
+          throw new ParseError("E0016");
+        } else {
+          throw new ParseError("E0018");
+        }
       }
 
-      if (selector.type === "AttributeExpression"
-          && selector.ref.type === "MessageReference") {
-        throw new ParseError("E0018");
-      }
-
-      if (selector.type === "TermReference"
-          || selector.type === "VariantExpression") {
-        throw new ParseError("E0017");
-      }
-
-      if (selector.type === "CallExpression"
-          && selector.callee.type === "TermReference") {
+      if (selector.type === "TermReference" && selector.attribute === null) {
         throw new ParseError("E0017");
       }
 
@@ -1426,13 +1462,7 @@ class FluentParser {
       return new SelectExpression(selector, variants);
     }
 
-    if (selector.type === "AttributeExpression"
-        && selector.ref.type === "TermReference") {
-      throw new ParseError("E0019");
-    }
-
-    if (selector.type === "CallExpression"
-        && selector.callee.type === "AttributeExpression") {
+    if (selector.type === "TermReference" && selector.attribute !== null) {
       throw new ParseError("E0019");
     }
 
@@ -1444,60 +1474,6 @@ class FluentParser {
       return this.getPlaceable(ps);
     }
 
-    let expr = this.getSimpleExpression(ps);
-    switch (expr.type) {
-      case "NumberLiteral":
-      case "StringLiteral":
-      case "VariableReference":
-        return expr;
-      case "MessageReference": {
-        if (ps.currentChar === ".") {
-          ps.next();
-          const attr = this.getIdentifier(ps);
-          return new AttributeExpression(expr, attr);
-        }
-
-        if (ps.currentChar === "(") {
-          // It's a Function. Ensure it's all upper-case.
-          if (!/^[A-Z][A-Z_?-]*$/.test(expr.id.name)) {
-            throw new ParseError("E0008");
-          }
-
-          const func = new FunctionReference(expr.id);
-          if (this.withSpans) {
-            func.addSpan(expr.span.start, expr.span.end);
-          }
-          return new CallExpression(func, ...this.getCallArguments(ps));
-        }
-
-        return expr;
-      }
-      case "TermReference": {
-        if (ps.currentChar === "[") {
-          ps.next();
-          const key = this.getVariantKey(ps);
-          ps.expectChar("]");
-          return new VariantExpression(expr, key);
-        }
-
-        if (ps.currentChar === ".") {
-          ps.next();
-          const attr = this.getIdentifier(ps);
-          expr = new AttributeExpression(expr, attr);
-        }
-
-        if (ps.currentChar === "(") {
-          return new CallExpression(expr, ...this.getCallArguments(ps));
-        }
-
-        return expr;
-      }
-      default:
-        throw new ParseError("E0028");
-    }
-  }
-
-  getSimpleExpression(ps) {
     if (ps.isNumberStart()) {
       return this.getNumber(ps);
     }
@@ -1515,14 +1491,44 @@ class FluentParser {
     if (ps.currentChar === "-") {
       ps.next();
       const id = this.getIdentifier(ps);
-      return new TermReference(id);
+
+      let attr;
+      if (ps.currentChar === ".") {
+        ps.next();
+        attr = this.getIdentifier(ps);
+      }
+
+      let args;
+      if (ps.currentChar === "(") {
+        args = this.getCallArguments(ps);
+      }
+
+      return new TermReference(id, attr, args);
     }
 
     if (ps.isIdentifierStart()) {
       const id = this.getIdentifier(ps);
-      return new MessageReference(id);
+
+      if (ps.currentChar === "(") {
+        // It's a Function. Ensure it's all upper-case.
+        if (!/^[A-Z][A-Z0-9_-]*$/.test(id.name)) {
+          throw new ParseError("E0008");
+        }
+
+        let args = this.getCallArguments(ps);
+        return new FunctionReference(id, args);
+      }
+
+      let attr;
+      if (ps.currentChar === ".") {
+        ps.next();
+        attr = this.getIdentifier(ps);
+      }
+
+      return new MessageReference(id, attr);
     }
 
+
     throw new ParseError("E0028");
   }
 
@@ -1535,15 +1541,15 @@ class FluentParser {
       return exp;
     }
 
-    if (exp.type !== "MessageReference") {
-      throw new ParseError("E0009");
+    if (exp.type === "MessageReference" && exp.attribute === null) {
+      ps.next();
+      ps.skipBlank();
+
+      const value = this.getLiteral(ps);
+      return new NamedArgument(exp.id, value);
     }
 
-    ps.next();
-    ps.skipBlank();
-
-    const value = this.getLiteral(ps);
-    return new NamedArgument(exp.id, value);
+    throw new ParseError("E0009");
   }
 
   getCallArguments(ps) {
@@ -1584,23 +1590,18 @@ class FluentParser {
     }
 
     ps.expectChar(")");
-    return [positional, named];
+    return new CallArguments(positional, named);
   }
 
   getString(ps) {
-    let raw = "";
-    let value = "";
-
     ps.expectChar("\"");
+    let value = "";
 
     let ch;
     while ((ch = ps.takeChar(x => x !== '"' && x !== EOL))) {
       if (ch === "\\") {
-        const [sequence, unescaped] = this.getEscapeSequence(ps);
-        raw += sequence;
-        value += unescaped;
+        value += this.getEscapeSequence(ps);
       } else {
-        raw += ch;
         value += ch;
       }
     }
@@ -1611,7 +1612,7 @@ class FluentParser {
 
     ps.expectChar("\"");
 
-    return new StringLiteral(raw, value);
+    return new StringLiteral(value);
   }
 
   getLiteral(ps) {
@@ -1640,7 +1641,6 @@ function isSelectExpr(elem) {
     && elem.expression.type === "SelectExpression";
 }
 
-// Bit masks representing the state of the serializer.
 const HAS_ENTRIES = 1;
 
 class FluentSerializer {
@@ -1695,10 +1695,6 @@ class FluentSerializer {
         throw new Error(`Unknown entry type: ${entry.type}`);
     }
   }
-
-  serializeExpression(expr) {
-    return serializeExpression(expr);
-  }
 }
 
 
@@ -1726,7 +1722,7 @@ function serializeMessage(message) {
   parts.push(`${message.id.name} =`);
 
   if (message.value) {
-    parts.push(serializeValue(message.value));
+    parts.push(serializePattern(message.value));
   }
 
   for (const attribute of message.attributes) {
@@ -1746,7 +1742,7 @@ function serializeTerm(term) {
   }
 
   parts.push(`-${term.id.name} =`);
-  parts.push(serializeValue(term.value));
+  parts.push(serializePattern(term.value));
 
   for (const attribute of term.attributes) {
     parts.push(serializeAttribute(attribute));
@@ -1758,23 +1754,11 @@ function serializeTerm(term) {
 
 
 function serializeAttribute(attribute) {
-  const value = indent(serializeValue(attribute.value));
+  const value = indent(serializePattern(attribute.value));
   return `\n    .${attribute.id.name} =${value}`;
 }
 
 
-function serializeValue(value) {
-  switch (value.type) {
-    case "Pattern":
-      return serializePattern(value);
-    case "VariantList":
-      return serializeVariantList(value);
-    default:
-      throw new Error(`Unknown value type: ${value.type}`);
-  }
-}
-
-
 function serializePattern(pattern) {
   const content = pattern.elements.map(serializeElement).join("");
   const startOnNewLine =
@@ -1789,24 +1773,6 @@ function serializePattern(pattern) {
 }
 
 
-function serializeVariantList(varlist) {
-  const content = varlist.variants.map(serializeVariant).join("");
-  return `\n    {${indent(content)}\n    }`;
-}
-
-
-function serializeVariant(variant) {
-  const key = serializeVariantKey(variant.key);
-  const value = indent(serializeValue(variant.value));
-
-  if (variant.default) {
-    return `\n   *[${key}]${value}`;
-  }
-
-  return `\n    [${key}]${value}`;
-}
-
-
 function serializeElement(element) {
   switch (element.type) {
     case "TextElement":
@@ -1821,14 +1787,13 @@ function serializeElement(element) {
 
 function serializePlaceable(placeable) {
   const expr = placeable.expression;
-
   switch (expr.type) {
     case "Placeable":
       return `{${serializePlaceable(expr)}}`;
     case "SelectExpression":
       // Special-case select expression to control the whitespace around the
       // opening and the closing brace.
-      return `{ ${serializeSelectExpression(expr)}}`;
+      return `{ ${serializeExpression(expr)}}`;
     default:
       return `{ ${serializeExpression(expr)} }`;
   }
@@ -1838,24 +1803,37 @@ function serializePlaceable(placeable) {
 function serializeExpression(expr) {
   switch (expr.type) {
     case "StringLiteral":
-      return `"${expr.raw}"`;
+      return `"${expr.value}"`;
     case "NumberLiteral":
       return expr.value;
-    case "MessageReference":
-    case "FunctionReference":
-      return expr.id.name;
-    case "TermReference":
-      return `-${expr.id.name}`;
     case "VariableReference":
       return `$${expr.id.name}`;
-    case "AttributeExpression":
-      return serializeAttributeExpression(expr);
-    case "VariantExpression":
-      return serializeVariantExpression(expr);
-    case "CallExpression":
-      return serializeCallExpression(expr);
-    case "SelectExpression":
-      return serializeSelectExpression(expr);
+    case "TermReference": {
+      let out = `-${expr.id.name}`;
+      if (expr.attribute) {
+        out += `.${expr.attribute.name}`;
+      }
+      if (expr.arguments) {
+        out += serializeCallArguments(expr.arguments);
+      }
+      return out;
+    }
+    case "MessageReference": {
+      let out = expr.id.name;
+      if (expr.attribute) {
+        out += `.${expr.attribute.name}`;
+      }
+      return out;
+    }
+    case "FunctionReference":
+      return `${expr.id.name}${serializeCallArguments(expr.arguments)}`;
+    case "SelectExpression": {
+      let out = `${serializeExpression(expr.selector)} ->`;
+      for (let variant of expr.variants) {
+        out += serializeVariant(variant);
+      }
+      return `${out}\n`;
+    }
     case "Placeable":
       return serializePlaceable(expr);
     default:
@@ -1864,41 +1842,25 @@ function serializeExpression(expr) {
 }
 
 
-function serializeSelectExpression(expr) {
-  const parts = [];
-  const selector = `${serializeExpression(expr.selector)} ->`;
-  parts.push(selector);
+function serializeVariant(variant) {
+  const key = serializeVariantKey(variant.key);
+  const value = indent(serializePattern(variant.value));
 
-  for (const variant of expr.variants) {
-    parts.push(serializeVariant(variant));
+  if (variant.default) {
+    return `\n   *[${key}]${value}`;
   }
 
-  parts.push("\n");
-  return parts.join("");
+  return `\n    [${key}]${value}`;
 }
 
 
-function serializeAttributeExpression(expr) {
-  const ref = serializeExpression(expr.ref);
-  return `${ref}.${expr.name.name}`;
-}
-
-
-function serializeVariantExpression(expr) {
-  const ref = serializeExpression(expr.ref);
-  const key = serializeVariantKey(expr.key);
-  return `${ref}[${key}]`;
-}
-
-
-function serializeCallExpression(expr) {
-  const callee = serializeExpression(expr.callee);
+function serializeCallArguments(expr) {
   const positional = expr.positional.map(serializeExpression).join(", ");
   const named = expr.named.map(serializeNamedArgument).join(", ");
   if (expr.positional.length > 0 && expr.named.length > 0) {
-    return `${callee}(${positional}, ${named})`;
+    return `(${positional}, ${named})`;
   }
-  return `${callee}(${positional || named})`;
+  return `(${positional || named})`;
 }
 
 
@@ -1912,19 +1874,84 @@ function serializeVariantKey(key) {
   switch (key.type) {
     case "Identifier":
       return key.name;
+    case "NumberLiteral":
+      return key.value;
     default:
-      return serializeExpression(key);
+      throw new Error(`Unknown variant key type: ${key.type}`);
   }
 }
 
+/*
+ * Abstract Visitor pattern
+ */
+class Visitor {
+  visit(node) {
+    if (Array.isArray(node)) {
+      node.forEach(child => this.visit(child));
+      return;
+    }
+    if (!(node instanceof BaseNode)) {
+      return;
+    }
+    const visit = this[`visit${node.type}`] || this.genericVisit;
+    visit.call(this, node);
+  }
+
+  genericVisit(node) {
+    for (const propname of Object.keys(node)) {
+      this.visit(node[propname]);
+    }
+  }
+}
+
+/*
+ * Abstract Transformer pattern
+ */
+class Transformer extends Visitor {
+  visit(node) {
+    if (!(node instanceof BaseNode)) {
+      return node;
+    }
+    const visit = this[`visit${node.type}`] || this.genericVisit;
+    return visit.call(this, node);
+  }
+
+  genericVisit(node) {
+    for (const propname of Object.keys(node)) {
+      const propvalue = node[propname];
+      if (Array.isArray(propvalue)) {
+        const newvals = propvalue
+          .map(child => this.visit(child))
+          .filter(newchild => newchild !== undefined);
+        node[propname] = newvals;
+      }
+      if (propvalue instanceof BaseNode) {
+        const new_val = this.visit(propvalue);
+        if (new_val === undefined) {
+          delete node[propname];
+        } else {
+          node[propname] = new_val;
+        }
+      }
+    }
+    return node;
+  }
+}
+
+const visitor = ({
+  Visitor: Visitor,
+  Transformer: Transformer
+});
+
 /* eslint object-shorthand: "off",
-          no-unused-vars: "off",
-          no-redeclare: "off",
           comma-dangle: "off",
           no-labels: "off" */
 
 this.EXPORTED_SYMBOLS = [
-  "FluentParser",
-  "FluentSerializer",
+  ...Object.keys({
+    FluentParser,
+    FluentSerializer,
+  }),
   ...Object.keys(ast),
+  ...Object.keys(visitor),
 ];