Bug 1353680, update compare-locales to 7.2.1, r=flod

Differential Revision: https://phabricator.services.mozilla.com/D29000

--HG--
extra : moz-landing-system : lando

third_party/python/compare-locales/compare_locales/__init__.py

@@ -1 +1 @@
-version = "7.0.0"
+version = "7.2.1"

third_party/python/compare-locales/compare_locales/commands.py

@@ -129,16 +129,20 @@ Be careful to specify the right merge directory when using this option.""")
                     config = TOMLParser().parse(config_path, env=config_env)
                 except ConfigNotFound as e:
                     self.parser.exit('config file %s not found' % e.filename)
-                if locales:
-                    config.set_locales(locales, deep=locales_deep)
+                if locales_deep:
+                    if not locales:
+                        # no explicit locales given, force all locales
+                        config.set_locales(config.all_locales, deep=True)
+                    else:
+                        config.set_locales(locales, deep=True)
                 configs.append(config)
             else:
-                app = EnumerateApp(
-                    config_path, l10n_base_dir, locales)
+                app = EnumerateApp(config_path, l10n_base_dir)
                 configs.append(app.asConfig())
         try:
             observers = compareProjects(
                 configs,
+                locales,
                 l10n_base_dir,
                 quiet=quiet,
                 merge_stage=merge, clobber_merge=clobber)

third_party/python/compare-locales/compare_locales/compare/__init__.py

@@ -26,18 +26,19 @@ __all__ = [
 
 def compareProjects(
             project_configs,
+            locales,
             l10n_base_dir,
             stat_observer=None,
             merge_stage=None,
             clobber_merge=False,
             quiet=0,
         ):
-    locales = set()
+    all_locales = set(locales)
     comparer = ContentComparer(quiet)
     observers = comparer.observers
     for project in project_configs:
         # disable filter if we're in validation mode
-        if None in project.locales:
+        if None in locales:
             filter = None
         else:
             filter = project.filter
@@ -46,8 +47,9 @@ def compareProjects(
                 quiet=quiet,
                 filter=filter,
             ))
-        locales.update(project.locales)
-    for locale in sorted(locales):
+        if not locales:
+            all_locales.update(project.all_locales)
+    for locale in sorted(all_locales):
         files = paths.ProjectFiles(locale, project_configs,
                                    mergebase=merge_stage)
         if merge_stage is not None:

third_party/python/compare-locales/compare_locales/compare/content.py

@@ -195,11 +195,9 @@ class ContentComparer:
                 if isinstance(l10n_entities[entity_id],
                               parser.Junk):
                     junk = l10n_entities[entity_id]
-                    params = (junk.val,) + junk.position() + junk.position(-1)
                     self.observers.notify(
                         'error', l10n,
-                        'Unparsed content "%s" from line %d column %d'
-                        ' to line %d column %d' % params
+                        junk.error_message()
                     )
                     if merge_file is not None:
                         skips.append(junk)

third_party/python/compare-locales/compare_locales/compare/observer.py

@@ -158,11 +158,13 @@ class ObserverList(Observer):
         }
         for observer in self.observers:
             for loc, lst in summaries.items():
-                lst.append(observer.summary.get(loc))
+                # Not all locales are on all projects,
+                # default to empty summary
+                lst.append(observer.summary.get(loc, {}))
         if len(self.observers) > 1:
             # add ourselves if there's more than one project
             for loc, lst in summaries.items():
-                lst.append(self.summary.get(loc))
+                lst.append(self.summary.get(loc, {}))
         # normalize missing and missingInFiles -> missing
         for summarylist in summaries.values():
             for summary in summarylist:

third_party/python/compare-locales/compare_locales/lint/cli.py

@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import argparse
+import os
+
+from compare_locales.lint.linter import L10nLinter
+from compare_locales.lint.util import (
+    default_reference_and_tests,
+    mirror_reference_and_tests,
+    l10n_base_reference_and_tests,
+)
+from compare_locales import mozpath
+from compare_locales import paths
+from compare_locales import parser
+from compare_locales import version
+
+
+epilog = '''\
+moz-l10n-lint checks for common mistakes in localizable files. It tests for
+duplicate entries, parsing errors, and the like. Optionally, it can compare
+the strings to an external reference with strings and warn if a string might
+need to get a new ID.
+'''
+
+
+def main():
+    p = argparse.ArgumentParser(
+        description='Validate localizable strings',
+        epilog=epilog,
+    )
+    p.add_argument('l10n_toml')
+    p.add_argument(
+        '--version', action='version', version='%(prog)s ' + version
+    )
+    p.add_argument('-W', action='store_true', help='error on warnings')
+    p.add_argument(
+        '--l10n-reference',
+        dest='l10n_reference',
+        metavar='PATH',
+        help='check for conflicts against an l10n-only reference repository '
+        'like gecko-strings',
+    )
+    p.add_argument(
+        '--reference-project',
+        dest='ref_project',
+        metavar='PATH',
+        help='check for conflicts against a reference project like '
+        'android-l10n',
+    )
+    args = p.parse_args()
+    if args.l10n_reference:
+        l10n_base, locale = \
+            os.path.split(os.path.abspath(args.l10n_reference))
+        if not locale or not os.path.isdir(args.l10n_reference):
+            p.error('Pass an existing l10n reference')
+    else:
+        l10n_base = '.'
+        locale = None
+    pc = paths.TOMLParser().parse(args.l10n_toml, env={'l10n_base': l10n_base})
+    if locale:
+        pc.set_locales([locale], deep=True)
+    files = paths.ProjectFiles(locale, [pc])
+    get_reference_and_tests = default_reference_and_tests
+    if args.l10n_reference:
+        get_reference_and_tests = l10n_base_reference_and_tests(files)
+    elif args.ref_project:
+        get_reference_and_tests = mirror_reference_and_tests(
+            files, args.ref_project
+        )
+    linter = L10nLinter()
+    results = linter.lint(
+        (f for f, _, _, _ in files.iter_reference() if parser.hasParser(f)),
+        get_reference_and_tests
+    )
+    rv = 0
+    if results:
+        rv = 1
+        if all(r['level'] == 'warning' for r in results) and not args.W:
+            rv = 0
+    for result in results:
+        print('{} ({}:{}): {}'.format(
+            mozpath.relpath(result['path'], '.'),
+            result.get('lineno', 0),
+            result.get('column', 0),
+            result['message']
+        ))
+    return rv
+
+
+if __name__ == '__main__':
+    main()

third_party/python/compare-locales/compare_locales/lint/linter.py

@@ -0,0 +1,120 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from collections import Counter
+import os
+
+from compare_locales import parser, checks
+from compare_locales.paths import File, REFERENCE_LOCALE
+
+
+class L10nLinter(object):
+
+    def lint(self, files, get_reference_and_tests):
+        results = []
+        for path in files:
+            if not parser.hasParser(path):
+                continue
+            ref, extra_tests = get_reference_and_tests(path)
+            results.extend(self.lint_file(path, ref, extra_tests))
+        return results
+
+    def lint_file(self, path, ref, extra_tests):
+        file_parser = parser.getParser(path)
+        if os.path.isfile(ref):
+            file_parser.readFile(ref)
+            reference = file_parser.parse()
+        else:
+            reference = {}
+        file_parser.readFile(path)
+        current = file_parser.parse()
+        checker = checks.getChecker(
+            File(path, path, locale=REFERENCE_LOCALE),
+            extra_tests=extra_tests
+        )
+        if checker and checker.needs_reference:
+            checker.set_reference(current)
+        linter = EntityLinter(current, checker, reference)
+        for current_entity in current:
+            for result in linter.lint_entity(current_entity):
+                result['path'] = path
+                yield result
+
+
+class EntityLinter(object):
+    '''Factored out helper to run linters on a single entity.'''
+    def __init__(self, current, checker, reference):
+        self.key_count = Counter(entity.key for entity in current)
+        self.checker = checker
+        self.reference = reference
+
+    def lint_entity(self, current_entity):
+        res = self.handle_junk(current_entity)
+        if res:
+            yield res
+            return
+        for res in self.lint_full_entity(current_entity):
+            yield res
+        for res in self.lint_value(current_entity):
+            yield res
+
+    def lint_full_entity(self, current_entity):
+        '''Checks that go good or bad for a full entity,
+        without a particular spot inside the entity.
+        '''
+        lineno = col = None
+        if self.key_count[current_entity.key] > 1:
+            lineno, col = current_entity.position()
+            yield {
+                'lineno': lineno,
+                'column': col,
+                'level': 'error',
+                'message': 'Duplicate string with ID: {}'.format(
+                    current_entity.key
+                )
+            }
+
+        if current_entity.key in self.reference:
+            reference_entity = self.reference[current_entity.key]
+            if not current_entity.equals(reference_entity):
+                if lineno is None:
+                    lineno, col = current_entity.position()
+                msg = 'Changes to string require a new ID: {}'.format(
+                    current_entity.key
+                )
+                yield {
+                    'lineno': lineno,
+                    'column': col,
+                    'level': 'warning',
+                    'message': msg,
+                }
+
+    def lint_value(self, current_entity):
+        '''Checks that error on particular locations in the entity value.
+        '''
+        if self.checker:
+            for tp, pos, msg, cat in self.checker.check(
+                current_entity, current_entity
+            ):
+                lineno, col = current_entity.value_position(pos)
+                yield {
+                    'lineno': lineno,
+                    'column': col,
+                    'level': tp,
+                    'message': msg,
+                }
+
+    def handle_junk(self, current_entity):
+        if not isinstance(current_entity, parser.Junk):
+            return None
+
+        lineno, col = current_entity.position()
+        return {
+            'lineno': lineno,
+            'column': col,
+            'level': 'error',
+            'message': current_entity.error_message()
+        }

third_party/python/compare-locales/compare_locales/lint/util.py

@@ -0,0 +1,40 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+from compare_locales import paths
+
+
+def default_reference_and_tests(path):
+    return None, None
+
+
+def mirror_reference_and_tests(files, basedir):
+    '''Get reference files to check for conflicts in android-l10n and friends.
+    '''
+    def get_reference_and_tests(path):
+        for matchers in files.matchers:
+            if 'reference' not in matchers:
+                continue
+            matcher = matchers['reference']
+            if matcher.match(path) is None:
+                continue
+            ref_matcher = paths.Matcher(matcher, root=basedir)
+            ref_path = matcher.sub(ref_matcher, path)
+            return ref_path, matchers.get('test')
+        return None, None
+    return get_reference_and_tests
+
+
+def l10n_base_reference_and_tests(files):
+    '''Get reference files to check for conflicts in gecko-strings and friends.
+    '''
+    def get_reference_and_tests(path):
+        match = files.match(path)
+        if match is None:
+            return None, None
+        ref, _, _, extra_tests = match
+        return ref, extra_tests
+    return get_reference_and_tests

third_party/python/compare-locales/compare_locales/parser/__init__.py

@@ -57,6 +57,13 @@ def getParser(path):
     raise UserWarning("Cannot find Parser")
 
 
+def hasParser(path):
+    try:
+        return bool(getParser(path))
+    except UserWarning:
+        return False
+
+
 __constructors = [
     ('strings.*\\.xml$', AndroidParser()),
     ('\\.dtd$', DTDParser()),

third_party/python/compare-locales/compare_locales/parser/android.py

@@ -61,6 +61,9 @@ class AndroidEntity(Entity):
     def raw_val(self):
         return self._raw_val_literal
 
+    def position(self, offset=0):
+        return (0, offset)
+
     def value_position(self, offset=0):
         return (0, offset)
 
@@ -178,22 +181,22 @@ class AndroidParser(Parser):
         except Exception:
             yield XMLJunk(contents)
             return
-        if doc.documentElement.nodeName != 'resources':
+        docElement = doc.documentElement
+        if docElement.nodeName != 'resources':
             yield XMLJunk(doc.toxml())
             return
-        root_children = doc.documentElement.childNodes
+        root_children = docElement.childNodes
         if not only_localizable:
-            attributes = ''.join(
-                ' {}="{}"'.format(attr_name, attr_value)
-                for attr_name, attr_value in
-                doc.documentElement.attributes.items()
-            )
             yield DocumentWrapper(
                 '<?xml?><resources>',
-                '<?xml version="1.0" encoding="utf-8"?>\n<resources{}>'.format(
-                    attributes
-                )
+                '<?xml version="1.0" encoding="utf-8"?>\n<resources'
             )
+            for attr_name, attr_value in docElement.attributes.items():
+                yield DocumentWrapper(
+                    attr_name,
+                    ' {}="{}"'.format(attr_name, attr_value)
+                )
+            yield DocumentWrapper('>', '>')
         child_num = 0
         while child_num < len(root_children):
             node = root_children[child_num]

third_party/python/compare-locales/compare_locales/parser/base.py

@@ -266,6 +266,13 @@ class Junk(object):
     def val(self):
         return self.all
 
+    def error_message(self):
+        params = (self.val,) + self.position() + self.position(-1)
+        return (
+            'Unparsed content "%s" from line %d column %d'
+            ' to line %d column %d' % params
+        )
+
     def __repr__(self):
         return self.key
 

third_party/python/compare-locales/compare_locales/parser/fluent.py

@@ -109,7 +109,13 @@ class FluentEntity(Entity):
 
     # In Fluent we treat entries as a whole.  FluentChecker reports errors at
     # offsets calculated from the beginning of the entry.
-    def value_position(self, offset=0):
+    def value_position(self, offset=None):
+        if offset is None:
+            # no offset given, use our value start or id end
+            if self.val_span:
+                offset = self.val_span[0] - self.span[0]
+            else:
+                offset = self.key_span[1] - self.span[0]
         return self.position(offset)
 
     @property

third_party/python/compare-locales/compare_locales/paths/files.py

@@ -23,9 +23,13 @@ class ProjectFiles(object):
         self.mergebase = mergebase
         configs = []
         for project in projects:
+            # Only add this project if we're not in validation mode,
+            # and the given locale is enabled for the project.
+            if locale is not None and locale not in project.all_locales:
+                continue
             configs.extend(project.configs)
         for pc in configs:
-            if locale and locale not in pc.locales:
+            if locale and pc.locales is not None and locale not in pc.locales:
                 continue
             for paths in pc.paths:
                 if (

third_party/python/compare-locales/compare_locales/paths/ini.py

@@ -167,14 +167,12 @@ class SourceTreeConfigParser(L10nConfigParser):
 class EnumerateApp(object):
     reference = 'en-US'
 
-    def __init__(self, inipath, l10nbase, locales=None):
+    def __init__(self, inipath, l10nbase):
         self.setupConfigParser(inipath)
         self.modules = defaultdict(dict)
         self.l10nbase = mozpath.abspath(l10nbase)
         self.filters = []
         self.addFilters(*self.config.getFilters())
-        self.locales = locales or self.config.allLocales()
-        self.locales.sort()
 
     def setupConfigParser(self, inipath):
         self.config = L10nConfigParser(inipath)
@@ -193,7 +191,7 @@ class EnumerateApp(object):
         filters = self.config.getFilters()
         if filters:
             config.set_filter_py(filters[0])
-        config.locales += self.locales
+        config.set_locales(self.config.allLocales(), deep=True)
         return config
 
     def _config_for_ini(self, projectconfig, aConfig):

third_party/python/compare-locales/compare_locales/paths/project.py

@@ -25,7 +25,9 @@ class ProjectConfig(object):
         self.root = None
         self.paths = []
         self.rules = []
-        self.locales = []
+        self.locales = None
+        # cache for all_locales, as that's not in `filter`
+        self._all_locales = None
         self.environ = {}
         self.children = []
         self._cache = None
@@ -63,7 +65,7 @@ class ProjectConfig(object):
         An optional key `test` is allowed to enable additional tests for this
         path pattern.
         '''
-
+        self._all_locales = None  # clear cache
         for d in paths:
             rv = {
                 'l10n': Matcher(d['l10n'], env=self.environ, root=self.root),
@@ -109,16 +111,16 @@ class ProjectConfig(object):
             self.rules.extend(self._compile_rule(rule))
 
     def add_child(self, child):
+        self._all_locales = None  # clear cache
         self.children.append(child)
 
     def set_locales(self, locales, deep=False):
+        self._all_locales = None  # clear cache
         self.locales = locales
+        if not deep:
+            return
         for child in self.children:
-            if not child.locales or deep:
-                child.set_locales(locales, deep=deep)
-            else:
-                locs = [loc for loc in locales if loc in child.locales]
-                child.set_locales(locs)
+            child.set_locales(locales, deep=deep)
 
     @property
     def configs(self):
@@ -128,9 +130,25 @@ class ProjectConfig(object):
             for config in child.configs:
                 yield config
 
+    @property
+    def all_locales(self):
+        'Recursively get all locales in this project and its paths'
+        if self._all_locales is None:
+            all_locales = set()
+            for config in self.configs:
+                if config.locales is not None:
+                    all_locales.update(config.locales)
+                for paths in config.paths:
+                    if 'locales' in paths:
+                        all_locales.update(paths['locales'])
+            self._all_locales = sorted(all_locales)
+        return self._all_locales
+
     def filter(self, l10n_file, entity=None):
         '''Filter a localization file or entities within, according to
         this configuration file.'''
+        if l10n_file.locale not in self.all_locales:
+            return 'ignore'
         if self.filter_py is not None:
             return self.filter_py(l10n_file.module, l10n_file.file,
                                   entity=entity)

third_party/python/compare-locales/compare_locales/plurals.py

@@ -70,6 +70,7 @@ CATEGORIES_BY_LOCALE = {
     'az': CATEGORIES_BY_INDEX[1],
     'be': CATEGORIES_BY_INDEX[7],
     'bg': CATEGORIES_BY_INDEX[1],
+    'bn': CATEGORIES_BY_INDEX[2],
     'bn-BD': CATEGORIES_BY_INDEX[2],
     'bn-IN': CATEGORIES_BY_INDEX[2],
     'br': CATEGORIES_BY_INDEX[16],

third_party/python/compare-locales/compare_locales/tests/android/test_merge.py

@@ -55,4 +55,28 @@ class TestMerge(unittest.TestCase):
   <!-- Foo -->
   <string name="foo">value</string>
 </resources>
+''')
+
+    def test_namespaces(self):
+        channels = (
+            b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns1="urn:ns1">
+    <string ns1:one="test">string</string>
+</resources>
+''',
+            b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns2="urn:ns2">
+    <string ns2:two="test">string</string>
+</resources>
+'''
+        )
+        self.assertEqual(
+            merge_channels(self.name, channels), b'''\
+<?xml version="1.0" encoding="utf-8"?>
+<resources xmlns:ns2="urn:ns2" xmlns:ns1="urn:ns1">
+    <string ns2:two="test">string</string>
+    <string ns1:one="test">string</string>
+</resources>
 ''')

third_party/python/compare-locales/compare_locales/tests/android/test_parser.py

@@ -39,6 +39,7 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('foo', 'value', 'bar'),
                 (Whitespace, '\n'),
@@ -79,6 +80,7 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('first', 'value'),
                 (Whitespace, '\n  '),
@@ -115,6 +117,7 @@ class TestAndroidParser(ParserTestMixin, unittest.TestCase):
             source,
             (
                 (DocumentWrapper, '<?xml'),
+                (DocumentWrapper, '>'),
                 (Whitespace, '\n  '),
                 ('one', ''),
                 (Whitespace, '\n  '),

third_party/python/compare-locales/compare_locales/tests/fluent/test_parser.py

@@ -113,6 +113,8 @@ abc =
 
     def test_message_with_attribute(self):
         self.parser.readContents(b'''\
+
+
 abc = ABC
     .attr = Attr
 ''')
@@ -121,6 +123,10 @@ abc = ABC
         self.assertEqual(abc.key, 'abc')
         self.assertEqual(abc.raw_val, 'ABC')
         self.assertEqual(abc.all, 'abc = ABC\n    .attr = Attr')
+        self.assertEqual(abc.position(), (3, 1))
+        self.assertEqual(abc.value_position(), (3, 7))
+        attr = list(abc.attributes)[0]
+        self.assertEqual(attr.value_position(), (4, 13))
 
     def test_message_with_attribute_and_no_value(self):
         self.parser.readContents(b'''\
@@ -137,6 +143,8 @@ abc =
         attr = attributes[0]
         self.assertEqual(attr.key, 'attr')
         self.assertEqual(attr.raw_val, 'Attr')
+        self.assertEqual(abc.value_position(), (1, 4))
+        self.assertEqual(attr.value_position(), (2, 13))
 
     def test_non_localizable(self):
         self.parser.readContents(b'''\

third_party/python/compare-locales/compare_locales/tests/lint/test_linter.py

@@ -0,0 +1,97 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+import unittest
+
+from compare_locales.lint import linter
+from compare_locales.parser import base as parser
+
+
+class MockChecker(object):
+    def __init__(self, mocked):
+        self.results = mocked
+
+    def check(self, ent, ref):
+        for r in self.results:
+            yield r
+
+
+class EntityTest(unittest.TestCase):
+    def test_junk(self):
+        el = linter.EntityLinter([], None, {})
+        ctx = parser.Parser.Context('foo\nbar\n')
+        ent = parser.Junk(ctx, (4, 7))
+        res = el.handle_junk(ent)
+        self.assertIsNotNone(res)
+        self.assertEqual(res['lineno'], 2)
+        self.assertEqual(res['column'], 1)
+        ent = parser.LiteralEntity('one', 'two', 'one = two')
+        self.assertIsNone(el.handle_junk(ent))
+
+    def test_full_entity(self):
+        ctx = parser.Parser.Context('''\
+one = two
+two = three
+one = four
+''')
+        entities = [
+            parser.Entity(ctx, None, None, (0, 10), (0, 3), (6, 9)),
+            parser.Entity(ctx, None, None, (10, 22), (10, 13), (16, 21)),
+            parser.Entity(ctx, None, None, (22, 33), (22, 25), (28, 32)),
+        ]
+        self.assertEqual(
+            (entities[0].all, entities[0].key, entities[0].val),
+            ('one = two\n', 'one', 'two')
+        )
+        self.assertEqual(
+            (entities[1].all, entities[1].key, entities[1].val),
+            ('two = three\n', 'two', 'three')
+        )
+        self.assertEqual(
+            (entities[2].all, entities[2].key, entities[2].val),
+            ('one = four\n', 'one', 'four')
+        )
+        el = linter.EntityLinter(entities, None, {})
+        results = list(el.lint_full_entity(entities[1]))
+        self.assertListEqual(results, [])
+        results = list(el.lint_full_entity(entities[2]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'error')
+        self.assertEqual(result['lineno'], 3)
+        self.assertEqual(result['column'], 1)
+        # finally check for conflict
+        el.reference = {
+            'two': parser.LiteralEntity('two = other', 'two', 'other')
+        }
+        results = list(el.lint_full_entity(entities[1]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'warning')
+        self.assertEqual(result['lineno'], 2)
+        self.assertEqual(result['column'], 1)
+
+    def test_in_value(self):
+        ctx = parser.Parser.Context('''\
+one = two
+''')
+        entities = [
+            parser.Entity(ctx, None, None, (0, 10), (0, 3), (6, 9)),
+        ]
+        self.assertEqual(
+            (entities[0].all, entities[0].key, entities[0].val),
+            ('one = two\n', 'one', 'two')
+        )
+        checker = MockChecker([
+            ('error', 2, 'Incompatible resource types', 'android'),
+        ])
+        el = linter.EntityLinter(entities, checker, {})
+        results = list(el.lint_value(entities[0]))
+        self.assertEqual(len(results), 1)
+        result = results[0]
+        self.assertEqual(result['level'], 'error')
+        self.assertEqual(result['lineno'], 1)
+        self.assertEqual(result['column'], 9)

third_party/python/compare-locales/compare_locales/tests/lint/test_util.py

@@ -0,0 +1,91 @@
+# -*- coding: utf-8 -*-
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import
+
+import unittest
+
+from compare_locales.lint import util
+from compare_locales.paths.project import ProjectConfig
+from compare_locales.paths.files import ProjectFiles
+from compare_locales import mozpath
+
+
+class MirrorReferenceTest(unittest.TestCase):
+    def test_empty(self):
+        files = ProjectFiles(None, [])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertIsNone(ref)
+        self.assertIsNone(tests)
+
+    def test_no_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': 'some/{locale}/file.ftl',
+        })
+        files = ProjectFiles(None, [pc])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
+        self.assertEqual(tests, set())
+
+    def test_with_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': 'some/{locale}/file.ftl',
+            'test': ['more_stuff'],
+        })
+        files = ProjectFiles(None, [pc])
+        get_reference_and_tests = util.mirror_reference_and_tests(files, 'tld')
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(mozpath.relpath(ref, 'tld'), 'some/path/file.ftl')
+        self.assertEqual(tests, {'more_stuff'})
+
+
+class L10nBaseReferenceTest(unittest.TestCase):
+    def test_empty(self):
+        files = ProjectFiles(None, [])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertIsNone(ref)
+        self.assertIsNone(tests)
+
+    def test_no_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_environment(l10n_base='l10n_orig')
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': '{l10n_base}/{locale}/some/file.ftl',
+        })
+        pc.set_locales(['gecko'], deep=True)
+        files = ProjectFiles('gecko', [pc])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(
+            mozpath.relpath(ref, 'l10n_orig/gecko'),
+            'some/file.ftl'
+        )
+        self.assertEqual(tests, set())
+
+    def test_with_tests(self):
+        pc = ProjectConfig(None)
+        pc.add_environment(l10n_base='l10n_orig')
+        pc.add_paths({
+            'reference': 'some/path/file.ftl',
+            'l10n': '{l10n_base}/{locale}/some/file.ftl',
+            'test': ['more_stuff'],
+        })
+        pc.set_locales(['gecko'], deep=True)
+        files = ProjectFiles('gecko', [pc])
+        get_reference_and_tests = util.l10n_base_reference_and_tests(files)
+        ref, tests = get_reference_and_tests('some/path/file.ftl')
+        self.assertEqual(
+            mozpath.relpath(ref, 'l10n_orig/gecko'),
+            'some/file.ftl'
+        )
+        self.assertEqual(tests, {'more_stuff'})

third_party/python/compare-locales/compare_locales/tests/paths/__init__.py

@@ -33,6 +33,7 @@ class SetupMixin(object):
             '/tmp/somedir/de/toolkit/two/one/file.ftl',
             'file.ftl',
             module='toolkit', locale='de')
+        self.cfg.set_locales(['de'])
 
 
 class MockNode(object):

third_party/python/compare-locales/compare_locales/tests/paths/test_files.py

@@ -20,7 +20,7 @@ class TestProjectPaths(Rooted, unittest.TestCase):
     def test_l10n_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.root)
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*'
         })
@@ -65,7 +65,7 @@ class TestProjectPaths(Rooted, unittest.TestCase):
     def test_single_reference_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/good.ftl',
             'reference': self.path('/reference/good.ftl')
@@ -101,7 +101,7 @@ class TestProjectPaths(Rooted, unittest.TestCase):
     def test_reference_path(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*',
             'reference': self.path('/reference/*')
@@ -175,7 +175,7 @@ class TestProjectPaths(Rooted, unittest.TestCase):
 
     def test_partial_l10n(self):
         cfg = ProjectConfig(None)
-        cfg.locales.extend(['de', 'fr'])
+        cfg.set_locales(['de', 'fr'])
         cfg.add_paths({
             'l10n': self.path('/{locale}/major/*')
         }, {
@@ -216,7 +216,7 @@ class TestProjectPaths(Rooted, unittest.TestCase):
     def test_validation_mode(self):
         cfg = ProjectConfig(None)
         cfg.add_environment(l10n_base=self.path('/l10n'))
-        cfg.locales.append('de')
+        cfg.set_locales(['de'])
         cfg.add_paths({
             'l10n': '{l10n_base}/{locale}/*',
             'reference': self.path('/reference/*')

third_party/python/compare-locales/compare_locales/tests/paths/test_project.py

@@ -171,6 +171,32 @@ class TestProjectConfig(unittest.TestCase):
         pc.add_child(child)
         self.assertListEqual([pc, child], list(pc.configs))
 
+    def test_locales_in_children(self):
+        pc = ProjectConfig(None)
+        child = ProjectConfig(None)
+        child.add_paths({
+            'l10n': '/tmp/somedir/{locale}/toolkit/**',
+        })
+        child.set_locales([])
+        pc.add_child(child)
+        self.assertListEqual(pc.all_locales, [])
+        pc.set_locales(['de', 'fr'])
+        self.assertListEqual(child.locales, [])
+        self.assertListEqual(pc.all_locales, ['de', 'fr'])
+
+    def test_locales_in_paths(self):
+        pc = ProjectConfig(None)
+        child = ProjectConfig(None)
+        child.add_paths({
+            'l10n': '/tmp/somedir/{locale}/toolkit/**',
+            'locales': ['it']
+        })
+        child.set_locales([])
+        pc.add_child(child)
+        self.assertListEqual(pc.all_locales, ['it'])
+        pc.set_locales(['de', 'fr'])
+        self.assertListEqual(pc.all_locales, ['de', 'fr', 'it'])
+
 
 class TestSameConfig(unittest.TestCase):
 

third_party/python/compare-locales/compare_locales/tests/test_apps.py

@@ -81,7 +81,7 @@ class TestApp(unittest.TestCase):
         app = EnumerateApp(
             mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
             mozpath.join(self.stage, 'l10n-central'))
-        self.assertListEqual(app.locales, ['af', 'de', 'fr'])
+        self.assertListEqual(app.config.allLocales(), ['af', 'de', 'fr'])
         self.assertEqual(len(app.config.children), 1)
         projectconfig = app.asConfig()
         self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])