Bug 1579479, update compare-locales to 7.2.5, r=flod

This uplifts https://hg.mozilla.org/l10n/compare-locales/log?rev=reverse%28%3A%3ARELEASE_7_2_5+-+%3A%3ARELEASE_7_2_1%29.
Most of this has been reviewed by flod already, so putting this into his
queue once more.
Notably 5c6a60f129
fixes bug 1579479.

Differential Revision: https://phabricator.services.mozilla.com/D45203

--HG--
extra : moz-landing-system : lando

third_party/python/compare-locales/compare_locales/__init__.py

@@ -1 +1 @@
-version = "7.2.1"
+version = "7.2.5"

third_party/python/compare-locales/compare_locales/checks/__init__.py

@@ -5,12 +5,19 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+from .base import Checker, EntityPos
 from .android import AndroidChecker
 from .dtd import DTDChecker
 from .fluent import FluentChecker
 from .properties import PropertiesChecker
 
 
+__all__ = [
+    'Checker', 'EntityPos',
+    'AndroidChecker', 'DTDChecker', 'FluentChecker', 'PropertiesChecker',
+]
+
+
 def getChecker(file, extra_tests=None):
     if PropertiesChecker.use(file):
         return PropertiesChecker(extra_tests, locale=file.locale)
@@ -20,4 +27,4 @@ def getChecker(file, extra_tests=None):
         return FluentChecker(extra_tests, locale=file.locale)
     if AndroidChecker.use(file):
         return AndroidChecker(extra_tests, locale=file.locale)
-    return None
+    return Checker(extra_tests, locale=file.locale)

third_party/python/compare-locales/compare_locales/checks/android.py

@@ -23,6 +23,10 @@ class AndroidChecker(Checker):
         - tuple of line, column info for the error within the string
         - description string to be shown in the report
         '''
+        for encoding_trouble in super(
+            AndroidChecker, self
+        ).check(refEnt, l10nEnt):
+            yield encoding_trouble
         refNode = refEnt.node
         l10nNode = l10nEnt.node
         # Apples and oranges, error out.

third_party/python/compare-locales/compare_locales/checks/base.py

@@ -5,6 +5,15 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import re
+
+
+class EntityPos(int):
+    pass
+
+
+mochibake = re.compile('\ufffd')
+
 
 class Checker(object):
     '''Abstract class to implement checks per file type.
@@ -29,10 +38,16 @@ class Checker(object):
         - "warning" or "error", depending on what should be reported,
         - tuple of line, column info for the error within the string
         - description string to be shown in the report
+
+        By default, check for possible encoding errors.
         '''
-        if True:
-            raise NotImplementedError("Need to subclass")
-        yield ("error", (0, 0), "This is an example error", "example")
+        for m in mochibake.finditer(l10nEnt.all):
+            yield (
+                "warning",
+                EntityPos(m.start()),
+                "\ufffd in: {}".format(l10nEnt.key),
+                "encodings"
+            )
 
     def set_reference(self, reference):
         '''Set the reference entities.

third_party/python/compare-locales/compare_locales/checks/dtd.py

@@ -80,6 +80,10 @@ class DTDChecker(Checker):
 
         Return a checker that offers just those entities.
         """
+        for encoding_trouble in super(
+            DTDChecker, self
+        ).check(refEnt, l10nEnt):
+            yield encoding_trouble
         refValue, l10nValue = refEnt.raw_val, l10nEnt.raw_val
         # find entities the refValue references,
         # reusing markup from DTDParser.

third_party/python/compare-locales/compare_locales/checks/fluent.py

@@ -296,6 +296,10 @@ class FluentChecker(Checker):
         return l10n_data.messages
 
     def check(self, refEnt, l10nEnt):
+        for encoding_trouble in super(
+            FluentChecker, self
+        ).check(refEnt, l10nEnt):
+            yield encoding_trouble
         l10n_entry = l10nEnt.entry
         if isinstance(l10n_entry, ftl.Message):
             ref_entry = refEnt.entry

third_party/python/compare-locales/compare_locales/checks/properties.py

@@ -33,6 +33,10 @@ class PropertiesChecker(Checker):
     def check(self, refEnt, l10nEnt):
         '''Test for the different variable formats.
         '''
+        for encoding_trouble in super(
+            PropertiesChecker, self
+        ).check(refEnt, l10nEnt):
+            yield encoding_trouble
         refValue, l10nValue = refEnt.val, l10nEnt.val
         refSpecs = None
         # check for PluralForm.jsm stuff, should have the docs in the

third_party/python/compare-locales/compare_locales/compare/content.py

@@ -13,7 +13,7 @@ import re
 
 from compare_locales import parser
 from compare_locales import mozpath
-from compare_locales.checks import getChecker
+from compare_locales.checks import getChecker, EntityPos
 from compare_locales.keyedtuple import KeyedTuple
 
 from .observer import ObserverList
@@ -224,7 +224,10 @@ class ContentComparer:
                         # run checks:
                 if checker:
                     for tp, pos, msg, cat in checker.check(refent, l10nent):
-                        line, col = l10nent.value_position(pos)
+                        if isinstance(pos, EntityPos):
+                            line, col = l10nent.position(pos)
+                        else:
+                            line, col = l10nent.value_position(pos)
                         # skip error entities when merging
                         if tp == 'error' and merge_file is not None:
                             skips.append(l10nent)

third_party/python/compare-locales/compare_locales/lint/linter.py

@@ -24,7 +24,7 @@ class L10nLinter(object):
 
     def lint_file(self, path, ref, extra_tests):
         file_parser = parser.getParser(path)
-        if os.path.isfile(ref):
+        if ref is not None and os.path.isfile(ref):
             file_parser.readFile(ref)
             reference = file_parser.parse()
         else:
@@ -99,7 +99,10 @@ class EntityLinter(object):
             for tp, pos, msg, cat in self.checker.check(
                 current_entity, current_entity
             ):
-                lineno, col = current_entity.value_position(pos)
+                if isinstance(pos, checks.EntityPos):
+                    lineno, col = current_entity.position(pos)
+                else:
+                    lineno, col = current_entity.value_position(pos)
                 yield {
                     'lineno': lineno,
                     'column': col,

third_party/python/compare-locales/compare_locales/parser/android.py

@@ -102,6 +102,12 @@ class NodeMixin(object):
     def raw_val(self):
         return self._val_literal
 
+    def position(self, offset=0):
+        return (0, offset)
+
+    def value_position(self, offset=0):
+        return (0, offset)
+
 
 class XMLWhitespace(NodeMixin, Whitespace):
     pass
@@ -139,6 +145,12 @@ class XMLJunk(Junk):
     def all(self):
         return self._all_literal
 
+    def position(self, offset=0):
+        return (0, offset)
+
+    def value_position(self, offset=0):
+        return (0, offset)
+
 
 def textContent(node):
     if node.childNodes.length == 0:

third_party/python/compare-locales/compare_locales/parser/base.py

@@ -8,7 +8,6 @@ import re
 import bisect
 import codecs
 from collections import Counter
-import logging
 from compare_locales.keyedtuple import KeyedTuple
 from compare_locales.paths import File
 
@@ -335,25 +334,21 @@ class Parser(object):
         # python 3 doesn't. Let's split code paths
         if six.PY2:
             with open(file, 'rbU') as f:
-                try:
-                    self.readContents(f.read())
-                except UnicodeDecodeError as e:
-                    (logging.getLogger('locales')
-                            .error("Can't read file: " + file + '; ' + str(e)))
+                self.readContents(f.read())
         else:
-            with open(file, 'r', encoding=self.encoding, newline=None) as f:
-                try:
-                    self.readUnicode(f.read())
-                except UnicodeDecodeError as e:
-                    (logging.getLogger('locales')
-                            .error("Can't read file: " + file + '; ' + str(e)))
+            with open(
+                file, 'r',
+                encoding=self.encoding, errors='replace',
+                newline=None
+            ) as f:
+                self.readUnicode(f.read())
 
     def readContents(self, contents):
         '''Read contents and create parsing context.
 
         contents are in native encoding, but with normalized line endings.
         '''
-        (contents, _) = codecs.getdecoder(self.encoding)(contents)
+        (contents, _) = codecs.getdecoder(self.encoding)(contents, 'replace')
         self.readUnicode(contents)
 
     def readUnicode(self, contents):

third_party/python/compare-locales/compare_locales/paths/ini.py

@@ -216,11 +216,10 @@ class EnumerateSourceTreeApp(EnumerateApp):
     be checked out for building.
     '''
 
-    def __init__(self, inipath, basepath, l10nbase, redirects,
-                 locales=None):
+    def __init__(self, inipath, basepath, l10nbase, redirects):
         self.basepath = basepath
         self.redirects = redirects
-        EnumerateApp.__init__(self, inipath, l10nbase, locales)
+        EnumerateApp.__init__(self, inipath, l10nbase)
 
     def setupConfigParser(self, inipath):
         self.config = SourceTreeConfigParser(inipath, self.basepath,

third_party/python/compare-locales/compare_locales/tests/android/test_checks.py

@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
@@ -70,6 +71,19 @@ class SimpleStringsTest(BaseHelper):
             )
         )
 
+    def test_bad_encoding(self):
+        self._test(
+            ANDROID_WRAPPER % 'touché'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    24,
+                    "\ufffd in: foo",
+                    "encodings"
+                ),
+            )
+        )
+
 
 class QuotesTest(BaseHelper):
     file = File('values/strings.xml', 'values/strings.xml')

third_party/python/compare-locales/compare_locales/tests/dtd/test_checks.py

@@ -95,6 +95,19 @@ stuff">
         self._test(b'''<!ENTITY style "width:12ch;height:200px;">''', tuple())
         self._test(b'''<!ENTITY ftd "0">''', tuple())
 
+    def test_bad_encoding(self):
+        self._test(
+            '<!ENTITY foo "touché">'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    19,
+                    "\ufffd in: foo",
+                    "encodings"
+                ),
+            )
+        )
+
 
 class TestEntitiesInDTDs(BaseHelper):
     file = File('foo.dtd', 'foo.dtd')

third_party/python/compare-locales/compare_locales/tests/fluent/test_checks.py

@@ -112,6 +112,19 @@ class TestMessage(BaseHelper):
             )
         )
 
+    def test_bad_encoding(self):
+        self._test(
+            'simple = touché'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    14,
+                    "\ufffd in: simple",
+                    "encodings"
+                ),
+            )
+        )
+
 
 class TestTerm(BaseHelper):
     file = File('foo.ftl', 'foo.ftl')

third_party/python/compare-locales/compare_locales/tests/properties/test_checks.py

@@ -24,6 +24,19 @@ class TestProperties(BaseHelper):
                    (('warning', 20, r'unknown escape sequence, \e',
                      'escape'),))
 
+    def test_bad_encoding(self):
+        self._test(
+            'some = touché"'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    12,
+                    "\ufffd in: some",
+                    "encodings"
+                ),
+            )
+        )
+
 
 class TestPlurals(BaseHelper):
     file = File('foo.properties', 'foo.properties')

third_party/python/compare-locales/compare_locales/tests/test_apps.py

@@ -5,7 +5,11 @@ import tempfile
 import shutil
 
 from compare_locales import mozpath
-from compare_locales.paths import EnumerateApp, ProjectFiles
+from compare_locales.paths import (
+    EnumerateApp,
+    EnumerateSourceTreeApp,
+    ProjectFiles,
+)
 
 MAIL_INI = '''\
 [general]
@@ -115,3 +119,50 @@ class TestApp(unittest.TestCase):
             ['comm', 'mozilla', 'toolkit', 'locales', 'en-US', 'platform.ftl'])
         self.assertIsNone(mergefile)
         self.assertSetEqual(test, set())
+
+    def test_src_app(self):
+        'Test parsing a App in source setup'
+        # move toolkit to toplevel
+        shutil.move(mozpath.join(self.stage, 'comm', 'mozilla'), self.stage)
+        app = EnumerateSourceTreeApp(
+            mozpath.join(self.stage, 'comm', 'mail', 'locales', 'l10n.ini'),
+            self.stage,
+            mozpath.join(self.stage, 'l10n-central'),
+            {
+                'mozilla-central': mozpath.join(self.stage, 'mozilla')
+            }
+        )
+        self.assertListEqual(app.config.allLocales(), ['af', 'de', 'fr'])
+        self.assertEqual(len(app.config.children), 1)
+        projectconfig = app.asConfig()
+        self.assertListEqual(projectconfig.locales, ['af', 'de', 'fr'])
+        files = ProjectFiles('de', [projectconfig])
+        files = list(files)
+        self.assertEqual(len(files), 3)
+
+        l10nfile, reffile, mergefile, test = files[0]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'mail', 'mail.ftl'])
+        self.assertListEqual(mozpath.split(reffile)[-4:],
+                             ['mail', 'locales', 'en-US', 'mail.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())
+
+        l10nfile, reffile, mergefile, test = files[1]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'toolkit', 'localized.ftl'])
+        self.assertListEqual(
+            mozpath.split(reffile)[-5:],
+            ['mozilla', 'toolkit',
+             'locales', 'en-US', 'localized.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())
+
+        l10nfile, reffile, mergefile, test = files[2]
+        self.assertListEqual(mozpath.split(l10nfile)[-3:],
+                             ['de', 'toolkit', 'platform.ftl'])
+        self.assertListEqual(
+            mozpath.split(reffile)[-5:],
+            ['mozilla', 'toolkit', 'locales', 'en-US', 'platform.ftl'])
+        self.assertIsNone(mergefile)
+        self.assertSetEqual(test, set())

third_party/python/compare-locales/compare_locales/tests/test_defines.py

@@ -7,7 +7,8 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 import unittest
 
-from compare_locales.tests import ParserTestMixin
+from compare_locales.tests import ParserTestMixin, BaseHelper
+from compare_locales.paths import File
 from compare_locales.parser import (
     Comment,
     DefinesInstruction,
@@ -224,5 +225,27 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
         )
 
 
-if __name__ == '__main__':
-    unittest.main()
+class TestChecks(BaseHelper):
+    file = File('defines.inc', 'defines.inc')
+    refContent = b'''\
+#define foo bar
+'''
+
+    def test_ok(self):
+        self._test(
+            b'#define foo other',
+            tuple()
+        )
+
+    def test_bad_encoding(self):
+        self._test(
+            '#define foo touché'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    17,
+                    "\ufffd in: foo",
+                    "encodings"
+                ),
+            )
+        )

third_party/python/compare-locales/compare_locales/tests/test_ini.py

@@ -7,7 +7,8 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 import unittest
 
-from compare_locales.tests import ParserTestMixin
+from compare_locales.tests import ParserTestMixin, BaseHelper
+from compare_locales.paths import File
 from compare_locales.parser import (
     Comment,
     IniSection,
@@ -195,5 +196,28 @@ Good=other string
         self._test(' \n\n', ((Whitespace, ' \n\n'),))
 
 
-if __name__ == '__main__':
-    unittest.main()
+class TestChecks(BaseHelper):
+    file = File('foo.ini', 'foo.ini')
+    refContent = b'''\
+[Strings]
+foo=good
+'''
+
+    def test_ok(self):
+        self._test(
+            b'[Strings]\nfoo=other',
+            tuple()
+        )
+
+    def test_bad_encoding(self):
+        self._test(
+            'foo=touché'.encode('latin-1'),
+            (
+                (
+                    "warning",
+                    9,
+                    "\ufffd in: foo",
+                    "encodings"
+                ),
+            )
+        )