Magic-Mirror/gecko-dev
1
0
Fork 0
mirror of https://github.com/mozilla/gecko-dev.git synced 2024-10-09 11:25:00 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

bug 1432229, land compare-locales 2.7 and python-fluent 0.6.2, r=gandalf

Browse Source 
MozReview-Commit-ID: 2Oep88PhVXF

--HG--
extra : rebase_source : 68af20d1f29ff2914547598079ad22fe7346b5fd
This commit is contained in:
Axel Hecht 2018-02-07 20:22:37 +01:00
parent 4c9045f5b1
commit afe9fce49c
37 changed files with 1618 additions and 869 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
third_party/python/compare-locales/compare_locales/__init__.py vendored
View File

@ -1 +1 @@
version = "2.5.1"
version = "2.7.0"

132
third_party/python/compare-locales/compare_locales/checks.py vendored
View File

@ -11,7 +11,10 @@ try:
except ImportError:
from StringIO import StringIO
from compare_locales.parser import DTDParser, PropertiesEntity
from fluent.syntax import ast as ftl
from compare_locales.parser import DTDParser, PropertiesEntity, FluentMessage
from compare_locales import plurals
class Checker(object):
@ -25,8 +28,9 @@ class Checker(object):
def use(cls, file):
return cls.pattern.match(file.file)
def __init__(self, extra_tests):
def __init__(self, extra_tests, locale=None):
self.extra_tests = extra_tests
self.locale = locale
self.reference = None
def check(self, refEnt, l10nEnt):
@ -71,23 +75,14 @@ class PropertiesChecker(Checker):
refSpecs = None
# check for PluralForm.jsm stuff, should have the docs in the
# comment
# That also includes intl.properties' pluralRule, so exclude
# entities with that key and values with just numbers
if (refEnt.pre_comment
and 'Localization_and_Plurals' in refEnt.pre_comment.all):
# For plurals, common variable pattern is #1. Try that.
pats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
refValue))
if len(pats) == 0:
return
lpats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
l10nValue))
if pats - lpats:
yield ('warning', 0, 'not all variables used in l10n',
'plural')
return
if lpats - pats:
yield ('error', 0, 'unreplaced variables in l10n',
'plural')
return
and 'Localization_and_Plurals' in refEnt.pre_comment.all
and refEnt.key != 'pluralRule'
and not re.match(r'\d+$', refValue)):
for msg_tuple in self.check_plural(refValue, l10nValue):
yield msg_tuple
return
# check for lost escapes
raw_val = l10nEnt.raw_val
@ -106,6 +101,35 @@ class PropertiesChecker(Checker):
yield t
return
def check_plural(self, refValue, l10nValue):
'''Check for the stringbundle plurals logic.
The common variable pattern is #1.
'''
if self.locale in plurals.CATEGORIES_BY_LOCALE:
expected_forms = len(plurals.CATEGORIES_BY_LOCALE[self.locale])
found_forms = l10nValue.count(';') + 1
msg = 'expecting {} plurals, found {}'.format(
expected_forms,
found_forms
)
if expected_forms > found_forms:
yield ('warning', 0, msg, 'plural')
if expected_forms < found_forms:
yield ('warning', 0, msg, 'plural')
pats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
refValue))
if len(pats) == 0:
return
lpats = set(int(m.group(1)) for m in re.finditer('#([0-9]+)',
l10nValue))
if pats - lpats:
yield ('warning', 0, 'not all variables used in l10n',
'plural')
return
if lpats - pats:
yield ('error', 0, 'unreplaced variables in l10n',
'plural')
def checkPrintf(self, refSpecs, l10nValue):
try:
l10nSpecs = self.getPrintfSpecs(l10nValue)
@ -202,8 +226,8 @@ class DTDChecker(Checker):
'''
xmllist = set(('amp', 'lt', 'gt', 'apos', 'quot'))
def __init__(self, extra_tests):
super(DTDChecker, self).__init__(extra_tests)
def __init__(self, extra_tests, locale=None):
super(DTDChecker, self).__init__(extra_tests, locale=locale)
self.processContent = False
if self.extra_tests is not None and 'android-dtd' in self.extra_tests:
self.processContent = True
@ -435,17 +459,52 @@ class FluentChecker(Checker):
'''
pattern = re.compile('.*\.ftl')
def check(self, refEnt, l10nEnt):
ref_entry = refEnt.entry
l10n_entry = l10nEnt.entry
# verify that values match, either both have a value or none
def find_message_references(self, entry):
refs = {}
def collect_message_references(node):
if isinstance(node, ftl.MessageReference):
# The key is the name of the referenced message and it will
# be used in set algebra to find missing and obsolete
# references. The value is the node itself and its span
# will be used to pinpoint the error.
refs[node.id.name] = node
# BaseNode.traverse expects this function to return the node.
return node
entry.traverse(collect_message_references)
return refs
def check_values(self, ref_entry, l10n_entry):
'''Verify that values match, either both have a value or none.'''
if ref_entry.value is not None and l10n_entry.value is None:
yield ('error', 0, 'Missing value', 'fluent')
if ref_entry.value is None and l10n_entry.value is not None:
offset = l10n_entry.value.span.start - l10n_entry.span.start
yield ('error', offset, 'Obsolete value', 'fluent')
# verify that we're having the same set of attributes
def check_message_references(self, ref_entry, l10n_entry):
'''Verify that message references are the same.'''
ref_msg_refs = self.find_message_references(ref_entry)
l10n_msg_refs = self.find_message_references(l10n_entry)
# create unique sets of message names referenced in both entries
ref_msg_refs_names = set(ref_msg_refs.keys())
l10n_msg_refs_names = set(l10n_msg_refs.keys())
missing_msg_ref_names = ref_msg_refs_names - l10n_msg_refs_names
for msg_name in missing_msg_ref_names:
yield ('warning', 0, 'Missing message reference: ' + msg_name,
'fluent')
obsolete_msg_ref_names = l10n_msg_refs_names - ref_msg_refs_names
for msg_name in obsolete_msg_ref_names:
pos = l10n_msg_refs[msg_name].span.start - l10n_entry.span.start
yield ('warning', pos, 'Obsolete message reference: ' + msg_name,
'fluent')
def check_attributes(self, ref_entry, l10n_entry):
'''Verify that ref_entry and l10n_entry have the same attributes.'''
ref_attr_names = set((attr.id.name for attr in ref_entry.attributes))
ref_pos = dict((attr.id.name, i)
for i, attr in enumerate(ref_entry.attributes))
@ -484,12 +543,29 @@ class FluentChecker(Checker):
yield ('error', attr.span.start - l10n_entry.span.start,
'Obsolete attribute: ' + attr.id.name, 'fluent')
def check(self, refEnt, l10nEnt):
ref_entry = refEnt.entry
l10n_entry = l10nEnt.entry
# PY3 Replace with `yield from` in Python 3.3+
for check in self.check_values(ref_entry, l10n_entry):
yield check
for check in self.check_message_references(ref_entry, l10n_entry):
yield check
# Only compare attributes of Fluent Messages. Attributes defined on
# Fluent Terms are private.
if isinstance(refEnt, FluentMessage):
for check in self.check_attributes(ref_entry, l10n_entry):
yield check
def getChecker(file, extra_tests=None):
if PropertiesChecker.use(file):
return PropertiesChecker(extra_tests)
return PropertiesChecker(extra_tests, locale=file.locale)
if DTDChecker.use(file):
return DTDChecker(extra_tests)
return DTDChecker(extra_tests, locale=file.locale)
if FluentChecker.use(file):
return FluentChecker(extra_tests)
return FluentChecker(extra_tests, locale=file.locale)
return None

21
third_party/python/compare-locales/compare_locales/commands.py vendored
View File

@ -30,10 +30,13 @@ or the all-locales file referenced by the application\'s l10n.ini."""
parser = ArgumentParser(description=self.__doc__)
parser.add_argument('--version', action='version',
version='%(prog)s ' + version)
parser.add_argument('-v', '--verbose', action='count', dest='v',
parser.add_argument('-v', '--verbose', action='count',
default=0, help='Make more noise')
parser.add_argument('-q', '--quiet', action='count', dest='q',
default=0, help='Make less noise')
parser.add_argument('-q', '--quiet', action='count',
default=0, help='''Show less data.
Specified once, doesn't record entities. Specified twice, also drops
missing and obsolete files. Specify thrice to hide errors and warnings and
just show stats''')
parser.add_argument('-m', '--merge',
help='''Use this directory to stage merged files,
use {ab_CD} to specify a different directory for each locale''')
@ -82,17 +85,16 @@ data in a json useful for Exhibit
self.parser = self.get_parser()
args = self.parser.parse_args()
# log as verbose or quiet as we want, warn by default
logging_level = logging.WARNING - (args.verbose - args.quiet) * 10
logging.basicConfig()
logging.getLogger().setLevel(logging.WARNING -
(args.v - args.q) * 10)
logging.getLogger().setLevel(logging_level)
kwargs = vars(args)
# strip handeld arguments
kwargs.pop('q')
kwargs.pop('v')
kwargs.pop('verbose')
return self.handle(**kwargs)
def handle(self, config_paths, l10n_base_dir, locales,
merge=None, defines=None, unified=False, full=False,
merge=None, defines=None, unified=False, full=False, quiet=0,
clobber=False, data='text'):
# using nargs multiple times in argparser totally screws things
# up, repair that.
@ -139,9 +141,10 @@ data in a json useful for Exhibit
try:
unified_observer = None
if unified:
unified_observer = Observer()
unified_observer = Observer(quiet=quiet)
observers = compareProjects(
configs,
quiet=quiet,
stat_observer=unified_observer,
merge_stage=merge, clobber_merge=clobber)
except (OSError, IOError), exc:

32
third_party/python/compare-locales/compare_locales/compare.py vendored
View File

@ -147,9 +147,15 @@ class AddRemove(object):
class Observer(object):
def __init__(self, filter=None, file_stats=False):
def __init__(self, quiet=0, filter=None, file_stats=False):
'''Create Observer
For quiet=1, skip per-entity missing and obsolete strings,
for quiet=2, skip missing and obsolete files. For quiet=3,
skip warnings and errors.
'''
self.summary = defaultdict(lambda: defaultdict(int))
self.details = Tree(list)
self.quiet = quiet
self.filter = filter
self.file_stats = None
if file_stats:
@ -217,7 +223,7 @@ class Observer(object):
if category in ['missingFile', 'obsoleteFile']:
if self.filter is not None:
rv = self.filter(file)
if rv != "ignore":
if rv != "ignore" and self.quiet < 2:
self.details[file].append({category: rv})
return rv
if category in ['missingEntity', 'obsoleteEntity']:
@ -225,9 +231,10 @@ class Observer(object):
rv = self.filter(file, data)
if rv == "ignore":
return rv
self.details[file].append({category: data})
if self.quiet < 1:
self.details[file].append({category: data})
return rv
if category in ('error', 'warning'):
if category in ('error', 'warning') and self.quiet < 3:
self.details[file].append({category: data})
self.summary[file.locale][category + 's'] += 1
return rv
@ -586,14 +593,23 @@ class ContentComparer:
pass
def compareProjects(project_configs, stat_observer=None,
file_stats=False,
merge_stage=None, clobber_merge=False):
def compareProjects(
project_configs,
stat_observer=None,
file_stats=False,
merge_stage=None,
clobber_merge=False,
quiet=0,
):
locales = set()
observers = []
for project in project_configs:
observers.append(
Observer(filter=project.filter, file_stats=file_stats))
Observer(
quiet=quiet,
filter=project.filter,
file_stats=file_stats,
))
locales.update(project.locales)
if stat_observer is not None:
stat_observers = [stat_observer]

5
third_party/python/compare-locales/compare_locales/integration_tests/__init__.py vendored Normal file
View File

@ -0,0 +1,5 @@
'''Tests that are not run by default.
They might just take long, or depend on external services, or both.
They might also fail for external changes.
'''

82
third_party/python/compare-locales/compare_locales/integration_tests/test_plurals.py vendored Normal file
View File

@ -0,0 +1,82 @@
# -*- coding: utf-8 -*-
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import ast
import json
import os
import unittest
import urllib2
TRANSVISION_URL = (
'https://transvision.mozfr.org/'
'api/v1/entity/gecko_strings/'
'?id=toolkit/chrome/global/intl.properties:pluralRule'
)
class TestPlural(unittest.TestCase):
'''Integration test for plural forms and l10n-central.
Having more plural forms than in l10n-central is OK, missing or
mismatching ones isn't.
Depends on transvision.
'''
maxDiff = None
def test_valid_forms(self):
reference_form_map = self._load_transvision()
compare_locales_map = self._parse_plurals_py()
# Notify locales in compare-locales but not in transvision
# Might be incubator locales
extra_locales = set()
extra_locales.update(compare_locales_map)
extra_locales.difference_update(reference_form_map)
for locale in sorted(extra_locales):
print("{} not in transvision, OK".format(locale))
compare_locales_map.pop(locale)
# Strip matches from dicts, to make diff for test small
locales = set()
locales.update(compare_locales_map)
locales.intersection_update(reference_form_map)
for locale in locales:
if compare_locales_map[locale] == reference_form_map[locale]:
compare_locales_map.pop(locale)
reference_form_map.pop(locale)
self.assertDictEqual(reference_form_map, compare_locales_map)
def _load_transvision(self):
'''Use the Transvision API to load all values of pluralRule
in intl.properties.
Skip test on load failure.
'''
try:
data = urllib2.urlopen(TRANSVISION_URL).read()
except urllib2.URLError:
raise unittest.SkipTest("Couldn't load transvision API.")
return json.loads(data)
def _parse_plurals_py(self):
'''Load compare_locales.plurals, parse the AST, and inspect
the dictionary assigned to CATEGORIES_BY_LOCALE to find
the actual plural number.
Convert both number and locale code to unicode for comparing
to json.
'''
path = os.path.join(os.path.dirname(__file__), '..', 'plurals.py')
with open(path) as source_file:
plurals_ast = ast.parse(source_file.read())
assign_cats_statement = [
s for s in plurals_ast.body
if isinstance(s, ast.Assign)
and any(t.id == 'CATEGORIES_BY_LOCALE' for t in s.targets)
][0]
return dict(
(unicode(k.s), unicode(v.slice.value.n))
for k, v in zip(
assign_cats_statement.value.keys,
assign_cats_statement.value.values
)
)

4
third_party/python/compare-locales/compare_locales/merge.py vendored
View File

@ -35,10 +35,10 @@ def merge_channels(name, *resources):
def get_key_value(entity, counter):
if isinstance(entity, cl.Comment):
counter[entity.all] += 1
counter[entity.val] += 1
# Use the (value, index) tuple as the key. AddRemove will
# de-deplicate identical comments at the same index.
return ((entity.all, counter[entity.all]), entity)
return ((entity.val, counter[entity.val]), entity)
if isinstance(entity, cl.Whitespace):
# Use the Whitespace instance as the key so that it's always

112
third_party/python/compare-locales/compare_locales/parser.py vendored
View File

@ -126,15 +126,38 @@ class Comment(EntityBase):
self.ctx = ctx
self.span = span
self.val_span = None
self._val_cache = None
@property
def key(self):
return None
@property
def val(self):
if self._val_cache is None:
self._val_cache = self.all
return self._val_cache
def __repr__(self):
return self.all
class OffsetComment(Comment):
'''Helper for file formats that have a constant number of leading
chars to strip from comments.
Offset defaults to 1
'''
comment_offset = 1
@property
def val(self):
if self._val_cache is None:
self._val_cache = ''.join((
l[self.comment_offset:] for l in self.all.splitlines(True)
))
return self._val_cache
class Junk(object):
'''
An almost-Entity, representing junk data that we didn't parse.
@ -189,6 +212,7 @@ class Whitespace(EntityBase):
class Parser(object):
capabilities = CAN_SKIP | CAN_MERGE
reWhitespace = re.compile('\s+', re.M)
Comment = Comment
class Context(object):
"Fixture for content and line numbers"
@ -268,7 +292,7 @@ class Parser(object):
return self.createEntity(ctx, m)
m = self.reComment.match(ctx.contents, offset)
if m:
self.last_comment = Comment(ctx, m.span())
self.last_comment = self.Comment(ctx, m.span())
return self.last_comment
return self.getJunk(ctx, offset, self.reKey, self.reComment)
@ -361,6 +385,14 @@ class DTDParser(Parser):
u'%' + Name + ';'
u'(?:[ \t]*(?:' + XmlComment + u'\s*)*\n?)?')
class Comment(Comment):
@property
def val(self):
if self._val_cache is None:
# Strip "<!--" and "-->" to comment contents
self._val_cache = self.all[4:-3]
return self._val_cache
def getNext(self, ctx, offset):
'''
Overload Parser.getNext to special-case ParsedEntities.
@ -408,6 +440,9 @@ class PropertiesEntity(Entity):
class PropertiesParser(Parser):
Comment = OffsetComment
def __init__(self):
self.reKey = re.compile(
'(?P<key>[^#!\s\n][^=:\n]*?)\s*[:=][ \t]*', re.M)
@ -426,7 +461,7 @@ class PropertiesParser(Parser):
m = self.reComment.match(contents, offset)
if m:
self.last_comment = Comment(ctx, m.span())
self.last_comment = self.Comment(ctx, m.span())
return self.last_comment
m = self.reKey.match(contents, offset)
@ -484,6 +519,9 @@ class DefinesParser(Parser):
EMPTY_LINES = 1 << 0
PAST_FIRST_LINE = 1 << 1
class Comment(OffsetComment):
comment_offset = 2
def __init__(self):
self.reComment = re.compile('(?:^# .*?\n)*(?:^# [^\n]*)', re.M)
# corresponds to
@ -510,7 +548,7 @@ class DefinesParser(Parser):
m = self.reComment.match(contents, offset)
if m:
self.last_comment = Comment(ctx, m.span())
self.last_comment = self.Comment(ctx, m.span())
return self.last_comment
m = self.reKey.match(contents, offset)
if m:
@ -549,6 +587,9 @@ class IniParser(Parser):
string=value
...
'''
Comment = OffsetComment
def __init__(self):
self.reComment = re.compile('(?:^[;#][^\n]*\n)*(?:^[;#][^\n]*)', re.M)
self.reSection = re.compile('\[(?P<val>.*?)\]', re.M)
@ -562,7 +603,7 @@ class IniParser(Parser):
return Whitespace(ctx, m.span())
m = self.reComment.match(contents, offset)
if m:
self.last_comment = Comment(ctx, m.span())
self.last_comment = self.Comment(ctx, m.span())
return self.last_comment
m = self.reSection.match(contents, offset)
if m:
@ -592,7 +633,7 @@ class FluentAttribute(EntityBase):
class FluentEntity(Entity):
# Fields ignored when comparing two entities.
ignored_fields = ['comment', 'span', 'tags']
ignored_fields = ['comment', 'span']
def __init__(self, ctx, entry):
start = entry.span.start
@ -616,6 +657,14 @@ class FluentEntity(Entity):
# are not separate Comment instances.
self.pre_comment = None
@property
def root_node(self):
'''AST node at which to start traversal for count_words.
By default we count words in the value and in all attributes.
'''
return self.entry
_word_count = None
def count_words(self):
@ -627,7 +676,7 @@ class FluentEntity(Entity):
self._word_count += len(node.value.split())
return node
self.entry.traverse(count_words)
self.root_node.traverse(count_words)
return self._word_count
@ -646,14 +695,28 @@ class FluentEntity(Entity):
yield FluentAttribute(self, attr_node)
class FluentSection(EntityBase):
def __init__(self, ctx, entry):
self.entry = entry
self.ctx = ctx
class FluentMessage(FluentEntity):
pass
self.span = (entry.span.start, entry.span.end)
self.key_span = self.val_span = (
entry.name.span.start, entry.name.span.end)
class FluentTerm(FluentEntity):
# Fields ignored when comparing two terms.
ignored_fields = ['attributes', 'comment', 'span']
@property
def root_node(self):
'''AST node at which to start traversal for count_words.
In Fluent Terms we only count words in the value. Attributes are
private and do not count towards the word total.
'''
return self.entry.value
class FluentComment(Comment):
def __init__(self, ctx, span, entry):
super(FluentComment, self).__init__(ctx, span)
self._val_cache = entry.content
class FluentParser(Parser):
@ -670,18 +733,7 @@ class FluentParser(Parser):
resource = self.ftl_parser.parse(self.ctx.contents)
if resource.comment:
last_span_end = resource.comment.span.end
if not only_localizable:
if 0 < resource.comment.span.start:
yield Whitespace(
self.ctx, (0, resource.comment.span.start))
yield Comment(
self.ctx,
(resource.comment.span.start, resource.comment.span.end))
else:
last_span_end = 0
last_span_end = 0
for entry in resource.body:
if not only_localizable:
@ -690,7 +742,9 @@ class FluentParser(Parser):
self.ctx, (last_span_end, entry.span.start))
if isinstance(entry, ftl.Message):
yield FluentEntity(self.ctx, entry)
yield FluentMessage(self.ctx, entry)
elif isinstance(entry, ftl.Term):
yield FluentTerm(self.ctx, entry)
elif isinstance(entry, ftl.Junk):
start = entry.span.start
end = entry.span.end
@ -700,11 +754,9 @@ class FluentParser(Parser):
ws, we = re.search('\s*$', entry.content).span()
end -= we - ws
yield Junk(self.ctx, (start, end))
elif isinstance(entry, ftl.Comment) and not only_localizable:
elif isinstance(entry, ftl.BaseComment) and not only_localizable:
span = (entry.span.start, entry.span.end)
yield Comment(self.ctx, span)
elif isinstance(entry, ftl.Section) and not only_localizable:
yield FluentSection(self.ctx, entry)
yield FluentComment(self.ctx, span, entry)
last_span_end = entry.span.end

42
third_party/python/compare-locales/compare_locales/plurals.py vendored
View File

@ -38,16 +38,18 @@ CATEGORIES_BY_INDEX = (
('one', 'two', 'few', 'many', 'other', 'zero'),
# 13 (Maltese)
('one', 'few', 'many', 'other'),
# 14 (Macedonian)
# 14 (Unused)
# CLDR: one, other
('one', 'two', 'other'),
# 15 (Icelandic)
# 15 (Icelandic, Macedonian)
('one', 'other'),
# 16 (Breton)
('one', 'two', 'few', 'many', 'other'),
# 17 (Shuar)
# CLDR: (missing)
('zero', 'other')
('zero', 'other'),
# 18 (Welsh),
('zero', 'one', 'two', 'few', 'many', 'other'),
)
CATEGORIES_BY_LOCALE = {
@ -57,17 +59,17 @@ CATEGORIES_BY_LOCALE = {
'ar': CATEGORIES_BY_INDEX[12],
'as': CATEGORIES_BY_INDEX[1],
'ast': CATEGORIES_BY_INDEX[1],
'az': CATEGORIES_BY_INDEX[0],
'az': CATEGORIES_BY_INDEX[1],
'be': CATEGORIES_BY_INDEX[7],
'bg': CATEGORIES_BY_INDEX[1],
'bn-BD': CATEGORIES_BY_INDEX[1],
'bn-IN': CATEGORIES_BY_INDEX[1],
'bn-BD': CATEGORIES_BY_INDEX[2],
'bn-IN': CATEGORIES_BY_INDEX[2],
'br': CATEGORIES_BY_INDEX[1],
'bs': CATEGORIES_BY_INDEX[1],
'bs': CATEGORIES_BY_INDEX[7],
'ca': CATEGORIES_BY_INDEX[1],
'cak': CATEGORIES_BY_INDEX[1],
'cs': CATEGORIES_BY_INDEX[8],
'cy': CATEGORIES_BY_INDEX[1],
'cy': CATEGORIES_BY_INDEX[18],
'da': CATEGORIES_BY_INDEX[1],
'de': CATEGORIES_BY_INDEX[1],
'dsb': CATEGORIES_BY_INDEX[10],
@ -82,7 +84,7 @@ CATEGORIES_BY_LOCALE = {
'es-MX': CATEGORIES_BY_INDEX[1],
'et': CATEGORIES_BY_INDEX[1],
'eu': CATEGORIES_BY_INDEX[1],
'fa': CATEGORIES_BY_INDEX[0],
'fa': CATEGORIES_BY_INDEX[2],
'ff': CATEGORIES_BY_INDEX[1],
'fi': CATEGORIES_BY_INDEX[1],
'fr': CATEGORIES_BY_INDEX[2],
@ -93,7 +95,7 @@ CATEGORIES_BY_LOCALE = {
'gn': CATEGORIES_BY_INDEX[1],
'gu-IN': CATEGORIES_BY_INDEX[2],
'he': CATEGORIES_BY_INDEX[1],
'hi-IN': CATEGORIES_BY_INDEX[1],
'hi-IN': CATEGORIES_BY_INDEX[2],
'hr': CATEGORIES_BY_INDEX[7],
'hsb': CATEGORIES_BY_INDEX[10],
'hu': CATEGORIES_BY_INDEX[1],
@ -105,10 +107,10 @@ CATEGORIES_BY_LOCALE = {
'ja': CATEGORIES_BY_INDEX[0],
'ja-JP-mac': CATEGORIES_BY_INDEX[0],
'jiv': CATEGORIES_BY_INDEX[17],
'ka': CATEGORIES_BY_INDEX[0],
'ka': CATEGORIES_BY_INDEX[1],
'kab': CATEGORIES_BY_INDEX[1],
'kk': CATEGORIES_BY_INDEX[1],
'km': CATEGORIES_BY_INDEX[1],
'km': CATEGORIES_BY_INDEX[0],
'kn': CATEGORIES_BY_INDEX[1],
'ko': CATEGORIES_BY_INDEX[0],
'lij': CATEGORIES_BY_INDEX[1],
@ -120,20 +122,20 @@ CATEGORIES_BY_LOCALE = {
'mk': CATEGORIES_BY_INDEX[15],
'ml': CATEGORIES_BY_INDEX[1],
'mr': CATEGORIES_BY_INDEX[1],
'ms': CATEGORIES_BY_INDEX[1],
'my': CATEGORIES_BY_INDEX[1],
'ms': CATEGORIES_BY_INDEX[0],
'my': CATEGORIES_BY_INDEX[0],
'nb-NO': CATEGORIES_BY_INDEX[1],
'ne-NP': CATEGORIES_BY_INDEX[1],
'nl': CATEGORIES_BY_INDEX[1],
'nn-NO': CATEGORIES_BY_INDEX[1],
'oc': CATEGORIES_BY_INDEX[1],
'oc': CATEGORIES_BY_INDEX[2],
'or': CATEGORIES_BY_INDEX[1],
'pa-IN': CATEGORIES_BY_INDEX[1],
'pa-IN': CATEGORIES_BY_INDEX[2],
'pl': CATEGORIES_BY_INDEX[9],
'pt-BR': CATEGORIES_BY_INDEX[1],
'pt-PT': CATEGORIES_BY_INDEX[1],
'rm': CATEGORIES_BY_INDEX[1],
'ro': CATEGORIES_BY_INDEX[1],
'ro': CATEGORIES_BY_INDEX[5],
'ru': CATEGORIES_BY_INDEX[7],
'si': CATEGORIES_BY_INDEX[1],
'sk': CATEGORIES_BY_INDEX[8],
@ -146,12 +148,12 @@ CATEGORIES_BY_LOCALE = {
'te': CATEGORIES_BY_INDEX[1],
'th': CATEGORIES_BY_INDEX[0],
'tl': CATEGORIES_BY_INDEX[1],
'tr': CATEGORIES_BY_INDEX[0],
'tr': CATEGORIES_BY_INDEX[1],
'trs': CATEGORIES_BY_INDEX[1],
'uk': CATEGORIES_BY_INDEX[7],
'ur': CATEGORIES_BY_INDEX[1],
'uz': CATEGORIES_BY_INDEX[0],
'vi': CATEGORIES_BY_INDEX[1],
'uz': CATEGORIES_BY_INDEX[1],
'vi': CATEGORIES_BY_INDEX[0],
'wo': CATEGORIES_BY_INDEX[0],
'xh': CATEGORIES_BY_INDEX[1],
'zam': CATEGORIES_BY_INDEX[1],

2
third_party/python/compare-locales/compare_locales/tests/__init__.py vendored
View File

@ -46,5 +46,5 @@ class ParserTestMixin():
self.assertEqual(entity.key, ref[0])
self.assertEqual(entity.val, ref[1])
else:
self.assertEqual(type(entity).__name__, ref[0])
self.assertIsInstance(entity, ref[0])
self.assertIn(ref[1], entity.all)

54
third_party/python/compare-locales/compare_locales/tests/test_checks.py vendored
View File

@ -90,6 +90,35 @@ downloadsTitleFiles=#1 file - Downloads;#1 files - #2;#1 #3
(('error', 0, 'unreplaced variables in l10n', 'plural'),))
class TestPluralForms(BaseHelper):
file = File('foo.properties', 'foo.properties', locale='en-GB')
refContent = '''\
# LOCALIZATION NOTE (downloadsTitleFiles): Semi-colon list of plural forms.
# See: http://developer.mozilla.org/en/docs/Localization_and_Plurals
# #1 number of files
# example: 111 files - Downloads
downloadsTitleFiles=#1 file;#1 files
'''
def test_matching_forms(self):
self._test('''\
downloadsTitleFiles=#1 fiiilee;#1 fiiilees
''',
tuple())
def test_lacking_forms(self):
self._test('''\
downloadsTitleFiles=#1 fiiilee
''',
(('warning', 0, 'expecting 2 plurals, found 1', 'plural'),))
def test_excess_forms(self):
self._test('''\
downloadsTitleFiles=#1 fiiilee;#1 fiiilees;#1 fiiilees
''',
(('warning', 0, 'expecting 2 plurals, found 3', 'plural'),))
class TestDTDs(BaseHelper):
file = File('foo.dtd', 'foo.dtd')
refContent = '''<!ENTITY foo "This is &apos;good&apos;">
@ -228,19 +257,16 @@ class TestAndroid(unittest.TestCase):
quot_msg = u"Quotes in Android DTDs need escaping with \\\" or " + \
u"\\u0022, or put string in apostrophes."
def getEntity(self, v):
def getNext(self, v):
ctx = Parser.Context(v)
return DTDEntity(
ctx, '', (0, len(v)), (), (), (), (0, len(v)), ())
ctx, '', (0, len(v)), (), (0, len(v)))
def getDTDEntity(self, v):
v = v.replace('"', '&quot;')
ctx = Parser.Context('<!ENTITY foo "%s">' % v)
return DTDEntity(
ctx,
'',
(0, len(v) + 16), (), (), (9, 12),
(14, len(v) + 14), ())
ctx, '', (0, len(v) + 16), (9, 12), (14, len(v) + 14))
def test_android_dtd(self):
"""Testing the actual android checks. The logic is involved,
@ -326,23 +352,23 @@ class TestAndroid(unittest.TestCase):
"embedding/android")
checker = getChecker(f, extra_tests=['android-dtd'])
# good plain string
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string")
ref = self.getNext("plain string")
l10n = self.getNext("plain localized string")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# no dtd warning
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string &ref;")
ref = self.getNext("plain string")
l10n = self.getNext("plain localized string &ref;")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# no report on stray ampersand
ref = self.getEntity("plain string")
l10n = self.getEntity("plain localized string with apos: '")
ref = self.getNext("plain string")
l10n = self.getNext("plain localized string with apos: '")
self.assertEqual(tuple(checker.check(ref, l10n)),
())
# report on bad printf
ref = self.getEntity("string with %s")
l10n = self.getEntity("string with %S")
ref = self.getNext("string with %s")
l10n = self.getNext("string with %S")
self.assertEqual(tuple(checker.check(ref, l10n)),
(('error', 0, 'argument 1 `S` should be `s`',
'printf'),))

112
third_party/python/compare-locales/compare_locales/tests/test_defines.py vendored
View File

@ -6,6 +6,12 @@
import unittest
from compare_locales.tests import ParserTestMixin
from compare_locales.parser import (
Comment,
DefinesInstruction,
Junk,
Whitespace,
)
mpl2 = '''\
@ -31,16 +37,16 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#unfilter emptyLines
''', (
('Comment', mpl2),
('Whitespace', '\n'),
('DefinesInstruction', 'filter emptyLines'),
('Whitespace', '\n\n'),
(Comment, mpl2),
(Whitespace, '\n'),
(DefinesInstruction, 'filter emptyLines'),
(Whitespace, '\n\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n\n'),
('Comment', '#define'),
('Whitespace', '\n\n'),
('DefinesInstruction', 'unfilter emptyLines'),
('Junk', '\n\n')))
(Whitespace, '\n\n'),
(Comment, '#define'),
(Whitespace, '\n\n'),
(DefinesInstruction, 'unfilter emptyLines'),
(Junk, '\n\n')))
def testBrowserWithContributors(self):
self._test(mpl2 + '''
@ -55,19 +61,19 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#unfilter emptyLines
''', (
('Comment', mpl2),
('Whitespace', '\n'),
('DefinesInstruction', 'filter emptyLines'),
('Whitespace', '\n\n'),
(Comment, mpl2),
(Whitespace, '\n'),
(DefinesInstruction, 'filter emptyLines'),
(Whitespace, '\n\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n\n'),
('Comment', 'non-English'),
('Whitespace', '\n'),
(Whitespace, '\n\n'),
(Comment, 'non-English'),
(Whitespace, '\n'),
('MOZ_LANGPACK_CONTRIBUTORS',
'<em:contributor>Joe Solon</em:contributor>'),
('Whitespace', '\n\n'),
('DefinesInstruction', 'unfilter emptyLines'),
('Junk', '\n\n')))
(Whitespace, '\n\n'),
(DefinesInstruction, 'unfilter emptyLines'),
(Junk, '\n\n')))
def testCommentWithNonAsciiCharacters(self):
self._test(mpl2 + '''
@ -79,25 +85,25 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#unfilter emptyLines
''', (
('Comment', mpl2),
('Whitespace', '\n'),
('DefinesInstruction', 'filter emptyLines'),
('Whitespace', '\n\n'),
('Comment', u'češtině'),
('Whitespace', '\n'),
(Comment, mpl2),
(Whitespace, '\n'),
(DefinesInstruction, 'filter emptyLines'),
(Whitespace, '\n\n'),
(Comment, u'češtině'),
(Whitespace, '\n'),
('seamonkey_l10n_long', ''),
('Whitespace', '\n\n'),
('DefinesInstruction', 'unfilter emptyLines'),
('Junk', '\n\n')))
(Whitespace, '\n\n'),
(DefinesInstruction, 'unfilter emptyLines'),
(Junk, '\n\n')))
def test_no_empty_lines(self):
self._test('''#define MOZ_LANGPACK_CREATOR mozilla.org
#define MOZ_LANGPACK_CREATOR mozilla.org
''', (
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def test_empty_line_between(self):
self._test('''#define MOZ_LANGPACK_CREATOR mozilla.org
@ -105,20 +111,20 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#define MOZ_LANGPACK_CREATOR mozilla.org
''', (
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Junk', '\n'),
(Junk, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def test_empty_line_at_the_beginning(self):
self._test('''
#define MOZ_LANGPACK_CREATOR mozilla.org
#define MOZ_LANGPACK_CREATOR mozilla.org
''', (
('Junk', '\n'),
(Junk, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def test_filter_empty_lines(self):
self._test('''#filter emptyLines
@ -126,13 +132,13 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#define MOZ_LANGPACK_CREATOR mozilla.org
#define MOZ_LANGPACK_CREATOR mozilla.org
#unfilter emptyLines''', (
('DefinesInstruction', 'filter emptyLines'),
('Whitespace', '\n\n'),
(DefinesInstruction, 'filter emptyLines'),
(Whitespace, '\n\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
('DefinesInstruction', 'unfilter emptyLines')))
(Whitespace, '\n'),
(DefinesInstruction, 'unfilter emptyLines')))
def test_unfilter_empty_lines_with_trailing(self):
self._test('''#filter emptyLines
@ -141,20 +147,20 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#define MOZ_LANGPACK_CREATOR mozilla.org
#unfilter emptyLines
''', (
('DefinesInstruction', 'filter emptyLines'),
('Whitespace', '\n\n'),
(DefinesInstruction, 'filter emptyLines'),
(Whitespace, '\n\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('MOZ_LANGPACK_CREATOR', 'mozilla.org'),
('Whitespace', '\n'),
('DefinesInstruction', 'unfilter emptyLines'),
('Whitespace', '\n')))
(Whitespace, '\n'),
(DefinesInstruction, 'unfilter emptyLines'),
(Whitespace, '\n')))
def testToolkit(self):
self._test('''#define MOZ_LANG_TITLE English (US)
''', (
('MOZ_LANG_TITLE', 'English (US)'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def testToolkitEmpty(self):
self._test('', tuple())
@ -165,9 +171,9 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
defines.inc are interesting that way, as their
content is added to the generated file.
'''
self._test('\n', (('Junk', '\n'),))
self._test('\n\n', (('Junk', '\n\n'),))
self._test(' \n\n', (('Junk', ' \n\n'),))
self._test('\n', ((Junk, '\n'),))
self._test('\n\n', ((Junk, '\n\n'),))
self._test(' \n\n', ((Junk, ' \n\n'),))
def test_whitespace_value(self):
'''Test that there's only one whitespace between key and value
@ -180,11 +186,11 @@ class TestDefinesParser(ParserTestMixin, unittest.TestCase):
#define tre \n\
''', (
('one', ''),
('Whitespace', '\n'),
(Whitespace, '\n'),
('two', ' '),
('Whitespace', '\n'),
(Whitespace, '\n'),
('tre', ' '),
('Whitespace', '\n'),))
(Whitespace, '\n'),))
if __name__ == '__main__':

66
third_party/python/compare-locales/compare_locales/tests/test_dtd.py vendored
View File

@ -9,6 +9,11 @@ import unittest
import re
from compare_locales import parser
from compare_locales.parser import (
Comment,
Junk,
Whitespace,
)
from compare_locales.tests import ParserTestMixin
@ -30,17 +35,17 @@ class TestDTD(ParserTestMixin, unittest.TestCase):
'''
quoteRef = (
('good.one', 'one'),
('Whitespace', '\n'),
('Junk', '<!ENTITY bad.one "bad " quote">\n'),
(Whitespace, '\n'),
(Junk, '<!ENTITY bad.one "bad " quote">\n'),
('good.two', 'two'),
('Whitespace', '\n'),
('Junk', '<!ENTITY bad.two "bad "quoted" word">\n'),
(Whitespace, '\n'),
(Junk, '<!ENTITY bad.two "bad "quoted" word">\n'),
('good.three', 'three'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('good.four', 'good \' quote'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('good.five', 'good \'quoted\' word'),
('Whitespace', '\n'),)
(Whitespace, '\n'),)
def test_quotes(self):
self._test(self.quoteContent, self.quoteRef)
@ -69,17 +74,17 @@ class TestDTD(ParserTestMixin, unittest.TestCase):
''',
(
('first', 'string'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('second', 'string'),
('Whitespace', '\n'),
('Comment', 'out'),
('Whitespace', '\n')))
(Whitespace, '\n'),
(Comment, 'out'),
(Whitespace, '\n')))
def test_license_header(self):
p = parser.getParser('foo.dtd')
p.readContents(self.resource('triple-license.dtd'))
entities = list(p.walk())
self.assert_(isinstance(entities[0], parser.Comment))
self.assertIsInstance(entities[0], parser.Comment)
self.assertIn('MPL', entities[0].all)
e = entities[2]
self.assert_(isinstance(e, parser.Entity))
@ -107,17 +112,17 @@ class TestDTD(ParserTestMixin, unittest.TestCase):
def test_trailing_whitespace(self):
self._test('<!ENTITY foo.label "stuff">\n \n',
(('foo.label', 'stuff'), ('Whitespace', '\n \n')))
(('foo.label', 'stuff'), (Whitespace, '\n \n')))
def test_unicode_comment(self):
self._test('<!-- \xe5\x8f\x96 -->',
(('Comment', u'\u53d6'),))
((Comment, u'\u53d6'),))
def test_empty_file(self):
self._test('', tuple())
self._test('\n', (('Whitespace', '\n'),))
self._test('\n\n', (('Whitespace', '\n\n'),))
self._test(' \n\n', (('Whitespace', ' \n\n'),))
self._test('\n', ((Whitespace, '\n'),))
self._test('\n\n', ((Whitespace, '\n\n'),))
self._test(' \n\n', ((Whitespace, ' \n\n'),))
def test_positions(self):
self.parser.readContents('''\
@ -177,6 +182,33 @@ escaped value">
self.assertEqual(entity.raw_val, '&unknownEntity;')
self.assertEqual(entity.val, '&unknownEntity;')
def test_comment_val(self):
self.parser.readContents('''\
<!-- comment
spanning lines --> <!--
-->
<!-- last line -->
''')
entities = self.parser.walk()
entity = next(entities)
self.assertIsInstance(entity, parser.Comment)
self.assertEqual(entity.val, ' comment\nspanning lines ')
entity = next(entities)
self.assertIsInstance(entity, parser.Whitespace)
entity = next(entities)
self.assertIsInstance(entity, parser.Comment)
self.assertEqual(entity.val, '\n')
entity = next(entities)
self.assertIsInstance(entity, parser.Whitespace)
entity = next(entities)
self.assertIsInstance(entity, parser.Comment)
self.assertEqual(entity.val, ' last line ')
entity = next(entities)
self.assertIsInstance(entity, parser.Whitespace)
if __name__ == '__main__':
unittest.main()

149
third_party/python/compare-locales/compare_locales/tests/test_ftl.py vendored
View File

@ -46,9 +46,9 @@ d =
*[x] Two three
[y] Four
} five.
e
e =
.attr = One
f
f =
.attr1 = One
.attr2 = Two
g = One two
@ -103,7 +103,7 @@ abc =
[abc] = list(self.parser)
self.assertEqual(abc.key, 'abc')
self.assertEqual(abc.val, ' A\n B\n C')
self.assertEqual(abc.val, 'A\n B\n C')
self.assertEqual(abc.all, 'abc =\n A\n B\n C')
def test_message_with_attribute(self):
@ -135,6 +135,82 @@ abc
def test_non_localizable(self):
self.parser.readContents('''\
### Resource Comment
foo = Foo
## Group Comment
-bar = Bar
##
# Standalone Comment
# Baz Comment
baz = Baz
''')
entities = self.parser.walk()
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '### Resource Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentMessage))
self.assertEqual(entity.val, 'Foo')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '## Group Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentTerm))
self.assertEqual(entity.val, 'Bar')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '##')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '# Standalone Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentMessage))
self.assertEqual(entity.val, 'Baz')
self.assertEqual(entity.entry.comment.content, 'Baz Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n')
def test_non_localizable_syntax_zero_four(self):
self.parser.readContents('''\
// Resource Comment
foo = Foo
@ -144,6 +220,8 @@ foo = Foo
bar = Bar
[[ Another Section ]]
// Standalone Comment
// Baz Comment
@ -152,7 +230,7 @@ baz = Baz
entities = self.parser.walk()
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Comment))
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '// Resource Comment')
entity = next(entities)
@ -168,12 +246,11 @@ baz = Baz
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentSection))
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(
entity.all, '// Section Comment\n[[ Section Header ]]')
self.assertEqual(entity.val, 'Section Header ')
self.assertEqual(
entity.entry.comment.content, 'Section Comment')
entity.all,
'// Section Comment\n[[ Section Header ]]'
)
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
@ -188,7 +265,15 @@ baz = Baz
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Comment))
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '[[ Another Section ]]')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n\n')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.FluentComment))
self.assertEqual(entity.all, '// Standalone Comment')
entity = next(entities)
@ -203,3 +288,47 @@ baz = Baz
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
self.assertEqual(entity.all, '\n')
def test_comments_val(self):
self.parser.readContents('''\
// Legacy Comment
### Resource Comment
## Section Comment
# Standalone Comment
''')
entities = self.parser.walk()
entity = next(entities)
# ensure that fluent comments are FluentComments and Comments
self.assertTrue(isinstance(entity, parser.FluentComment))
# now test the actual .val values
self.assertTrue(isinstance(entity, parser.Comment))
self.assertEqual(entity.val, 'Legacy Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Comment))
self.assertEqual(entity.val, 'Resource Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Comment))
self.assertEqual(entity.val, 'Section Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Comment))
self.assertEqual(entity.val, 'Standalone Comment')
entity = next(entities)
self.assertTrue(isinstance(entity, parser.Whitespace))

125
third_party/python/compare-locales/compare_locales/tests/test_ini.py vendored
View File

@ -6,13 +6,18 @@
import unittest
from compare_locales.tests import ParserTestMixin
from compare_locales.parser import (
Comment,
IniSection,
Junk,
Whitespace,
)
mpl2 = '''\
; This Source Code Form is subject to the terms of the Mozilla Public
; License, v. 2.0. If a copy of the MPL was not distributed with this file,
; You can obtain one at http://mozilla.org/MPL/2.0/.
'''
; You can obtain one at http://mozilla.org/MPL/2.0/.'''
class TestIniParser(ParserTestMixin, unittest.TestCase):
@ -24,67 +29,92 @@ class TestIniParser(ParserTestMixin, unittest.TestCase):
[Strings]
TitleText=Some Title
''', (
('Comment', 'UTF-8 encoding'),
('IniSection', 'Strings'),
('TitleText', 'Some Title'),))
(Comment, 'UTF-8 encoding'),
(Whitespace, '\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
(Whitespace, '\n')))
def testMPL2_Space_UTF(self):
self._test(mpl2 + '''
; This file is in the UTF-8 encoding
[Strings]
TitleText=Some Title
''', (
('Comment', mpl2),
('Comment', 'UTF-8'),
('IniSection', 'Strings'),
('TitleText', 'Some Title'),))
(Comment, mpl2),
(Whitespace, '\n\n'),
(Comment, 'UTF-8'),
(Whitespace, '\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
(Whitespace, '\n')))
def testMPL2_Space(self):
self._test(mpl2 + '''
[Strings]
TitleText=Some Title
''', (
('Comment', mpl2),
('IniSection', 'Strings'),
('TitleText', 'Some Title'),))
(Comment, mpl2),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
(Whitespace, '\n')))
def testMPL2_MultiSpace(self):
self._test(mpl2 + '''\
self._test(mpl2 + '''
; more comments
[Strings]
TitleText=Some Title
''', (
('Comment', mpl2),
('Comment', 'more comments'),
('IniSection', 'Strings'),
('TitleText', 'Some Title'),))
(Comment, mpl2),
(Whitespace, '\n\n'),
(Comment, 'more comments'),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
(Whitespace, '\n')))
def testMPL2_JunkBeforeCategory(self):
self._test(mpl2 + '''\
self._test(mpl2 + '''
Junk
[Strings]
TitleText=Some Title
''', (
('Comment', mpl2),
('Junk', 'Junk'),
('IniSection', 'Strings'),
('TitleText', 'Some Title')))
(Comment, mpl2),
(Whitespace, '\n'),
(Junk, 'Junk\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
(Whitespace, '\n')))
def test_TrailingComment(self):
self._test(mpl2 + '''
[Strings]
TitleText=Some Title
;Stray trailing comment
''', (
('Comment', mpl2),
('IniSection', 'Strings'),
(Comment, mpl2),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
('Comment', 'Stray trailing')))
(Whitespace, '\n'),
(Comment, 'Stray trailing'),
(Whitespace, '\n')))
def test_SpacedTrailingComments(self):
self._test(mpl2 + '''
[Strings]
TitleText=Some Title
@ -92,13 +122,18 @@ TitleText=Some Title
;Second stray comment
''', (
('Comment', mpl2),
('IniSection', 'Strings'),
(Comment, mpl2),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
('Comment', 'Second stray comment')))
(Whitespace, '\n\n'),
(Comment, 'Second stray comment'),
(Whitespace, '\n\n')))
def test_TrailingCommentsAndJunk(self):
self._test(mpl2 + '''
[Strings]
TitleText=Some Title
@ -107,15 +142,21 @@ Junk
;Second stray comment
''', (
('Comment', mpl2),
('IniSection', 'Strings'),
(Comment, mpl2),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
('Comment', 'Stray trailing'),
('Junk', 'Junk'),
('Comment', 'Second stray comment')))
(Whitespace, '\n\n'),
(Comment, 'Stray trailing'),
(Whitespace, '\n'),
(Junk, 'Junk\n'),
(Comment, 'Second stray comment'),
(Whitespace, '\n\n')))
def test_JunkInbetweenEntries(self):
self._test(mpl2 + '''
[Strings]
TitleText=Some Title
@ -123,17 +164,21 @@ Junk
Good=other string
''', (
('Comment', mpl2),
('IniSection', 'Strings'),
(Comment, mpl2),
(Whitespace, '\n\n'),
(IniSection, 'Strings'),
(Whitespace, '\n'),
('TitleText', 'Some Title'),
('Junk', 'Junk'),
('Good', 'other string')))
(Whitespace, '\n\n'),
(Junk, 'Junk\n\n'),
('Good', 'other string'),
(Whitespace, '\n')))
def test_empty_file(self):
self._test('', tuple())
self._test('\n', (('Whitespace', '\n'),))
self._test('\n\n', (('Whitespace', '\n\n'),))
self._test(' \n\n', (('Whitespace', ' \n\n'),))
self._test('\n', ((Whitespace, '\n'),))
self._test('\n\n', ((Whitespace, '\n\n'),))
self._test(' \n\n', ((Whitespace, ' \n\n'),))
if __name__ == '__main__':

201
third_party/python/compare-locales/compare_locales/tests/test_merge.py vendored
View File

@ -389,12 +389,12 @@ class TestFluent(unittest.TestCase):
self.reference("""\
foo = fooVal
bar = barVal
eff = effVal
-eff = effVal
""")
self.localized("""\
foo = lFoo
bar = lBar
eff = lEff
-eff = lEff
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
@ -420,6 +420,7 @@ eff = lEff
self.reference("""\
foo = fooVal
bar = barVal
-baz = bazVal
eff = effVal
""")
self.localized("""\
@ -436,15 +437,16 @@ eff = lEff
{
'details': {
'l10n.ftl': [
{'missingEntity': u'bar'}
{'missingEntity': u'bar'},
{'missingEntity': u'-baz'},
],
},
'summary': {
None: {
'changed': 2,
'changed_w': 2,
'missing': 1,
'missing_w': 1
'missing': 2,
'missing_w': 2,
}
}
}
@ -517,6 +519,92 @@ eff = lEff {
l10n_foo = l10n_entities[l10n_map['foo']]
self.assertTrue(merged_foo.equals(l10n_foo))
def testMatchingReferences(self):
self.reference("""\
foo = Reference { bar }
""")
self.localized("""\
foo = Localized { bar }
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{
'details': {},
'summary': {
None: {
'changed': 1,
'changed_w': 1
}
}
}
)
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(not os.path.exists(mergepath))
def testMismatchingReferences(self):
self.reference("""\
foo = Reference { bar }
bar = Reference { baz }
baz = Reference
""")
self.localized("""\
foo = Localized { qux }
bar = Localized
baz = Localized { qux }
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{
'details': {
'l10n.ftl': [
{
'warning':
u'Missing message reference: bar '
u'at line 1, column 1 for foo'
},
{
'warning':
u'Obsolete message reference: qux '
u'at line 1, column 19 for foo'
},
{
'warning':
u'Missing message reference: baz '
u'at line 2, column 1 for bar'
},
{
'warning':
u'Obsolete message reference: qux '
u'at line 3, column 19 for baz'
},
],
},
'summary': {
None: {
'changed': 3,
'changed_w': 3,
'warnings': 4
}
}
}
)
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(not os.path.exists(mergepath))
def testMismatchingAttributes(self):
self.reference("""
foo = Foo
@ -574,6 +662,53 @@ eff = lEff
l10n_eff = l10n_entities[l10n_map['eff']]
self.assertTrue(merged_eff.equals(l10n_eff))
def test_term_attributes(self):
self.reference("""
-foo = Foo
-bar = Bar
-baz = Baz
.attr = Baz Attribute
-qux = Qux
.attr = Qux Attribute
-missing = Missing
.attr = An Attribute
""")
self.localized("""\
-foo = Localized Foo
-bar = Localized Bar
.attr = Locale-specific Bar Attribute
-baz = Localized Baz
-qux = Localized Qux
.other = Locale-specific Qux Attribute
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{
'details': {
'l10n.ftl': [
{'missingEntity': u'-missing'},
],
},
'summary': {
None: {
'changed': 4,
'changed_w': 4,
'missing': 1,
'missing_w': 1,
}
}
}
)
# validate merge results
mergepath = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assert_(not os.path.exists(mergepath))
def testMismatchingValues(self):
self.reference("""
foo = Foo
@ -622,11 +757,11 @@ bar = lBar
merged_entities, _ = p.parse()
self.assertEqual([e.key for e in merged_entities], [])
def testMissingSection(self):
def testMissingGroupComment(self):
self.reference("""\
foo = fooVal
[[ Section ]]
## Group Comment
bar = barVal
""")
self.localized("""\
@ -659,7 +794,7 @@ bar = lBar
self.reference("""\
foo = fooVal
// Attached Comment
# Attached Comment
bar = barVal
""")
self.localized("""\
@ -698,7 +833,7 @@ bar = barVal
self.localized("""\
foo = lFoo
// Standalone Comment
# Standalone Comment
bar = lBar
""")
@ -789,54 +924,6 @@ bar = duplicated bar
mergefile = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assertFalse(os.path.isfile(mergefile))
def test_unmatched_tags(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
#yes
""")
self.localized("""foo = fooVal
#no
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{'summary':
{None: {
'unchanged': 1,
'unchanged_w': 1
}},
'details': {}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assertFalse(os.path.isfile(mergefile))
def test_matching_tags(self):
self.assertTrue(os.path.isdir(self.tmp))
self.reference("""foo = fooVal
#yes
""")
self.localized("""foo = fooVal
#yes
""")
cc = ContentComparer([Observer()])
cc.compare(File(self.ref, "en-reference.ftl", ""),
File(self.l10n, "l10n.ftl", ""),
mozpath.join(self.tmp, "merge", "l10n.ftl"))
self.assertDictEqual(
cc.observers[0].toJSON(),
{'summary':
{None: {
'unchanged': 1,
'unchanged_w': 1
}},
'details': {}
})
mergefile = mozpath.join(self.tmp, "merge", "l10n.ftl")
self.assertFalse(os.path.isfile(mergefile))
if __name__ == '__main__':
unittest.main()

179
third_party/python/compare-locales/compare_locales/tests/test_merge_ftl.py vendored
View File

@ -59,96 +59,74 @@ foo = Foo 1
.attr = Attr 1
""")
def test_tag_in_first(self):
def test_group_comment_in_first(self):
channels = (b"""
## Group Comment 1
foo = Foo 1
#tag
""", b"""
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
foo = Foo 1
#tag
""")
def test_tag_in_last(self):
channels = (b"""
foo = Foo 1
""", b"""
foo = Foo 2
#tag
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
## Group Comment 1
foo = Foo 1
""")
def test_tag_changed(self):
def test_group_comment_in_last(self):
channels = (b"""
foo = Foo 1
#tag1
""", b"""
foo = Foo 2
#tag2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
foo = Foo 1
#tag1
""")
def test_section_in_first(self):
channels = (b"""
[[ Section 1 ]]
foo = Foo 1
""", b"""
## Group Comment 2
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
[[ Section 1 ]]
## Group Comment 2
foo = Foo 1
""")
def test_section_in_last(self):
def test_group_comment_changed(self):
channels = (b"""
## Group Comment 1
foo = Foo 1
""", b"""
[[ Section 2 ]]
## Group Comment 2
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
[[ Section 2 ]]
## Group Comment 2
## Group Comment 1
foo = Foo 1
""")
def test_section_changed(self):
def test_group_comment_and_section(self):
channels = (b"""
[[ Section 1 ]]
## Group Comment
foo = Foo 1
""", b"""
[[ Section 2 ]]
// Section Comment
[[ Section ]]
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
[[ Section 2 ]]
[[ Section 1 ]]
// Section Comment
[[ Section ]]
## Group Comment
foo = Foo 1
""")
def test_message_comment_in_first(self):
channels = (b"""
// Comment 1
# Comment 1
foo = Foo 1
""", b"""
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Comment 1
# Comment 1
foo = Foo 1
""")
@ -156,7 +134,7 @@ foo = Foo 1
channels = (b"""
foo = Foo 1
""", b"""
// Comment 2
# Comment 2
foo = Foo 2
""")
self.assertEqual(
@ -166,62 +144,23 @@ foo = Foo 1
def test_message_comment_changed(self):
channels = (b"""
// Comment 1
# Comment 1
foo = Foo 1
""", b"""
// Comment 2
# Comment 2
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Comment 1
# Comment 1
foo = Foo 1
""")
def test_section_comment_in_first(self):
channels = (b"""
// Comment 1
[[ Section ]]
""", b"""
[[ Section ]]
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Comment 1
[[ Section ]]
""")
def test_section_comment_in_last(self):
channels = (b"""
[[ Section ]]
""", b"""
// Comment 2
[[ Section ]]
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
[[ Section ]]
""")
def test_section_comment_changed(self):
channels = (b"""
// Comment 1
[[ Section ]]
""", b"""
// Comment 2
[[ Section ]]
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Comment 1
[[ Section ]]
""")
def test_standalone_comment_in_first(self):
channels = (b"""
foo = Foo 1
// Comment 1
# Comment 1
""", b"""
foo = Foo 2
""")
@ -229,7 +168,7 @@ foo = Foo 2
merge_channels(self.name, *channels), b"""
foo = Foo 1
// Comment 1
# Comment 1
""")
def test_standalone_comment_in_last(self):
@ -238,37 +177,37 @@ foo = Foo 1
""", b"""
foo = Foo 2
// Comment 2
# Comment 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
foo = Foo 1
// Comment 2
# Comment 2
""")
def test_standalone_comment_changed(self):
channels = (b"""
foo = Foo 1
// Comment 1
# Comment 1
""", b"""
foo = Foo 2
// Comment 2
# Comment 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
foo = Foo 1
// Comment 2
# Comment 2
// Comment 1
# Comment 1
""")
def test_resource_comment_in_first(self):
channels = (b"""
// Resource Comment 1
### Resource Comment 1
foo = Foo 1
""", b"""
@ -276,7 +215,7 @@ foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Resource Comment 1
### Resource Comment 1
foo = Foo 1
""")
@ -285,32 +224,70 @@ foo = Foo 1
channels = (b"""
foo = Foo 1
""", b"""
// Resource Comment 1
### Resource Comment 1
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Resource Comment 1
### Resource Comment 1
foo = Foo 1
""")
def test_resource_comment_changed(self):
channels = (b"""
// Resource Comment 1
### Resource Comment 1
foo = Foo 1
""", b"""
// Resource Comment 2
### Resource Comment 2
foo = Foo 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
// Resource Comment 2
### Resource Comment 2
// Resource Comment 1
### Resource Comment 1
foo = Foo 1
""")
def test_cross_grammar(self):
channels = (b"""
# Comment 1
foo =
.attr = Attribute 1
""", b"""
// Comment 2
foo
.attr = Attribute 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
# Comment 1
foo =
.attr = Attribute 1
""")
def test_cross_grammar_standalone_comment(self):
'''This is in particular going to be triggered for license headers.'''
channels = (b"""
# Same comment
foo =
.attr = Attribute 1
""", b"""
// Same comment
foo
.attr = Attribute 2
""")
self.assertEqual(
merge_channels(self.name, *channels), b"""
# Same comment
foo =
.attr = Attribute 1
""")

20
third_party/python/compare-locales/compare_locales/tests/test_parser.py vendored
View File

@ -2,6 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import textwrap
import unittest
from compare_locales import parser
@ -46,3 +47,22 @@ third line
_map,
{}
)
class TestOffsetComment(unittest.TestCase):
def test_offset(self):
ctx = parser.Parser.Context(textwrap.dedent('''\
#foo
#bar
# baz
'''
)) # noqa
offset_comment = parser.OffsetComment(ctx, (0, len(ctx.contents)))
self.assertEqual(
offset_comment.val,
textwrap.dedent('''\
foo
bar
baz
''')
)

55
third_party/python/compare-locales/compare_locales/tests/test_properties.py vendored
View File

@ -6,6 +6,11 @@
import unittest
from compare_locales.tests import ParserTestMixin
from compare_locales.parser import (
Comment,
Junk,
Whitespace,
)
class TestPropertiesParser(ParserTestMixin, unittest.TestCase):
@ -22,15 +27,15 @@ two_lines_triple = This line is one of two and ends in \\\
and still has another line coming
''', (
('one_line', 'This is one line'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('two_line', u'This is the first of two lines'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('one_line_trailing', u'This line ends in \\'),
('Whitespace', '\n'),
('Junk', 'and has junk\n'),
(Whitespace, '\n'),
(Junk, 'and has junk\n'),
('two_lines_triple', 'This line is one of two and ends in \\'
'and still has another line coming'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def testProperties(self):
# port of netwerk/test/PropertiesTest.cpp
@ -68,10 +73,10 @@ and an end''', (('bar', 'one line with a # part that looks like a comment '
foo=value
''', (
('Comment', 'MPL'),
('Whitespace', '\n\n'),
(Comment, 'MPL'),
(Whitespace, '\n\n'),
('foo', 'value'),
('Whitespace', '\n')))
(Whitespace, '\n')))
def test_escapes(self):
self.parser.readContents(r'''
@ -97,18 +102,18 @@ second = string
#commented out
''', (
('first', 'string'),
('Whitespace', '\n'),
(Whitespace, '\n'),
('second', 'string'),
('Whitespace', '\n\n'),
('Comment', 'commented out'),
('Whitespace', '\n')))
(Whitespace, '\n\n'),
(Comment, 'commented out'),
(Whitespace, '\n')))
def test_trailing_newlines(self):
self._test('''\
foo = bar
\x20\x20
''', (('foo', 'bar'), ('Whitespace', '\n\n\x20\x20\n ')))
''', (('foo', 'bar'), (Whitespace, '\n\n\x20\x20\n ')))
def test_just_comments(self):
self._test('''\
@ -119,10 +124,10 @@ foo = bar
# LOCALIZATION NOTE These strings are used inside the Promise debugger
# which is available as a panel in the Debugger.
''', (
('Comment', 'MPL'),
('Whitespace', '\n\n'),
('Comment', 'LOCALIZATION NOTE'),
('Whitespace', '\n')))
(Comment, 'MPL'),
(Whitespace, '\n\n'),
(Comment, 'LOCALIZATION NOTE'),
(Whitespace, '\n')))
def test_just_comments_without_trailing_newline(self):
self._test('''\
@ -132,9 +137,9 @@ foo = bar
# LOCALIZATION NOTE These strings are used inside the Promise debugger
# which is available as a panel in the Debugger.''', (
('Comment', 'MPL'),
('Whitespace', '\n\n'),
('Comment', 'LOCALIZATION NOTE')))
(Comment, 'MPL'),
(Whitespace, '\n\n'),
(Comment, 'LOCALIZATION NOTE')))
def test_trailing_comment_and_newlines(self):
self._test('''\
@ -144,14 +149,14 @@ foo = bar
''', (
('Comment', 'LOCALIZATION NOTE'),
('Whitespace', '\n\n\n')))
(Comment, 'LOCALIZATION NOTE'),
(Whitespace, '\n\n\n')))
def test_empty_file(self):
self._test('', tuple())
self._test('\n', (('Whitespace', '\n'),))
self._test('\n\n', (('Whitespace', '\n\n'),))
self._test(' \n\n', (('Whitespace', '\n\n'),))
self._test('\n', ((Whitespace, '\n'),))
self._test('\n\n', ((Whitespace, '\n\n'),))
self._test(' \n\n', ((Whitespace, '\n\n'),))
def test_positions(self):
self.parser.readContents('''\

10
third_party/python/fluent/fluent/migrate/__init__.py vendored
View File

@ -1,13 +1,5 @@
# coding=utf8
from .context import MergeContext # noqa: F401
from .errors import ( # noqa: F401
MigrationError, NotSupportedError, UnreadableReferenceError
)
from .transforms import ( # noqa: F401
Source, COPY, REPLACE_IN_TEXT, REPLACE, PLURALS, CONCAT
CONCAT, COPY, PLURALS, REPLACE, REPLACE_IN_TEXT
)
from .helpers import ( # noqa: F401
EXTERNAL_ARGUMENT, MESSAGE_REFERENCE
)
from .changesets import convert_blame_to_changesets # noqa: F401

2
third_party/python/fluent/fluent/migrate/cldr.py vendored
View File

@ -48,7 +48,7 @@ def get_plural_categories(lang):
fallback_lang, _, _ = lang.rpartition('-')
if fallback_lang == '':
raise RuntimeError('Unknown language: {}'.format(lang))
raise RuntimeError('Missing plural categories for {}'.format(lang))
return get_plural_categories(fallback_lang)

153
third_party/python/fluent/fluent/migrate/context.py vendored
View File

@ -14,16 +14,14 @@ import fluent.syntax.ast as FTL
from fluent.syntax.parser import FluentParser
from fluent.syntax.serializer import FluentSerializer
from fluent.util import fold
try:
from compare_locales.parser import getParser
except ImportError:
def getParser(path):
raise RuntimeError('compare-locales required')
from compare_locales.parser import getParser
from .cldr import get_plural_categories
from .transforms import Source
from .merge import merge_resource
from .errors import NotSupportedError, UnreadableReferenceError
from .util import get_message
from .errors import (
EmptyLocalizationError, NotSupportedError, UnreadableReferenceError)
class MergeContext(object):
@ -59,8 +57,8 @@ class MergeContext(object):
try:
self.plural_categories = get_plural_categories(lang)
except RuntimeError as e:
print(e.message)
self.plural_categories = 'en'
logging.getLogger('migrate').warn(e)
self.plural_categories = get_plural_categories('en')
# Paths to directories with input data, relative to CWD.
self.reference_dir = reference_dir
@ -114,6 +112,47 @@ class MergeContext(object):
# Transform the parsed result which is an iterator into a dict.
return {entity.key: entity.val for entity in parser}
def read_reference_ftl(self, path):
"""Read and parse a reference FTL file.
A missing resource file is a fatal error and will raise an
UnreadableReferenceError.
"""
fullpath = os.path.join(self.reference_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError as err:
error_message = 'Missing reference file: {}'.format(fullpath)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
except UnicodeDecodeError as err:
error_message = 'Error reading file {}: {}'.format(fullpath, err)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
def read_localization_ftl(self, path):
"""Read and parse an existing localization FTL file.
Create a new FTL.Resource if the file doesn't exist or can't be
decoded.
"""
fullpath = os.path.join(self.localization_dir, path)
try:
return self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.info(
'Localization file {} does not exist and '
'it will be created'.format(path))
return FTL.Resource()
except UnicodeDecodeError:
logger = logging.getLogger('migrate')
logger.warn(
'Localization file {} has broken encoding. '
'It will be re-created and some translations '
'may be lost'.format(path))
return FTL.Resource()
def maybe_add_localization(self, path):
"""Add a localization resource to migrate translations from.
@ -159,52 +198,52 @@ class MergeContext(object):
acc.add((cur.path, cur.key))
return acc
refpath = os.path.join(self.reference_dir, reference)
try:
ast = self.read_ftl_resource(refpath)
except IOError as err:
error_message = 'Missing reference file: {}'.format(refpath)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
except UnicodeDecodeError as err:
error_message = 'Error reading file {}: {}'.format(refpath, err)
logging.getLogger('migrate').error(error_message)
raise UnreadableReferenceError(error_message)
else:
# The reference file will be used by the merge function as
# a template for serializing the merge results.
self.reference_resources[target] = ast
reference_ast = self.read_reference_ftl(reference)
self.reference_resources[target] = reference_ast
for node in transforms:
ident = node.id.name
# Scan `node` for `Source` nodes and collect the information they
# store into a set of dependencies.
dependencies = fold(get_sources, node, set())
# Set these sources as dependencies for the current transform.
self.dependencies[(target, node.id.name)] = dependencies
self.dependencies[(target, ident)] = dependencies
# Read all legacy translation files defined in Source transforms.
# The target Fluent message should exist in the reference file. If
# it doesn't, it's probably a typo.
if get_message(reference_ast.body, ident) is None:
logger = logging.getLogger('migrate')
logger.warn(
'{} "{}" was not found in {}'.format(
type(node).__name__, ident, reference))
# Keep track of localization resource paths which were defined as
# sources in the transforms.
expected_paths = set()
# Read all legacy translation files defined in Source transforms. This
# may fail but a single missing legacy resource doesn't mean that the
# migration can't succeed.
for dependencies in self.dependencies.values():
for path in set(path for path, _ in dependencies):
expected_paths.add(path)
self.maybe_add_localization(path)
# However, if all legacy resources are missing, bail out early. There
# are no translations to migrate. We'd also get errors in hg annotate.
if len(expected_paths) > 0 and len(self.localization_resources) == 0:
error_message = 'No localization files were found'
logging.getLogger('migrate').error(error_message)
raise EmptyLocalizationError(error_message)
# Add the current transforms to any other transforms added earlier for
# this path.
path_transforms = self.transforms.setdefault(target, [])
path_transforms += transforms
if target not in self.localization_resources:
fullpath = os.path.join(self.localization_dir, target)
try:
ast = self.read_ftl_resource(fullpath)
except IOError:
logger = logging.getLogger('migrate')
logger.info(
'Localization file {} does not exist and '
'it will be created'.format(target))
except UnicodeDecodeError:
logger = logging.getLogger('migrate')
logger.warn(
'Localization file {} will be re-created and some '
'translations might be lost'.format(target))
else:
self.localization_resources[target] = ast
target_ast = self.read_localization_ftl(target)
self.localization_resources[target] = target_ast
def get_source(self, path, key):
"""Get an entity value from a localized legacy source.
@ -242,7 +281,15 @@ class MergeContext(object):
The input data must be configured earlier using the `add_*` methods.
if given, `changeset` must be a set of (path, key) tuples describing
which legacy translations are to be merged.
which legacy translations are to be merged. If `changeset` is None,
all legacy translations will be allowed to be migrated in a single
changeset.
The inner `in_changeset` function is used to determine if a message
should be migrated for the given changeset. It compares the legacy
dependencies of the transform defined for the message with legacy
translations available in the changeset. If all dependencies are
present, the message will be migrated.
Given `changeset`, return a dict whose keys are resource paths and
values are `FTL.Resource` instances. The values will also be used to
@ -259,14 +306,22 @@ class MergeContext(object):
}
for path, reference in self.reference_resources.iteritems():
current = self.localization_resources.get(path, FTL.Resource())
current = self.localization_resources[path]
transforms = self.transforms.get(path, [])
def in_changeset(ident):
"""Check if entity should be merged.
"""Check if a message should be migrated.
If at least one dependency of the entity is in the current
set of changeset, merge it.
A message will be migrated only if all of its dependencies
are present in the currently processed changeset.
If a transform defined for this message points to a missing
legacy translation, this message will not be merged. The
missing legacy dependency won't be present in the changeset.
This also means that partially translated messages (e.g.
constructed from two legacy strings out of which only one is
avaiable) will never be migrated.
"""
message_deps = self.dependencies.get((path, ident), None)
@ -281,9 +336,11 @@ class MergeContext(object):
if len(message_deps) == 0:
return True
# If the intersection of the dependencies and the current
# changeset is non-empty, merge this message.
return message_deps & changeset
# Make sure all the dependencies are present in the current
# changeset. Partial migrations are not currently supported.
# See https://bugzilla.mozilla.org/show_bug.cgi?id=1321271
available_deps = message_deps & changeset
return message_deps == available_deps
# Merge legacy translations with the existing ones using the
# reference as a template.

4
third_party/python/fluent/fluent/migrate/errors.py vendored
View File

@ -2,6 +2,10 @@ class MigrationError(ValueError):
pass
class EmptyLocalizationError(MigrationError):
pass
class NotSupportedError(MigrationError):
pass

8
third_party/python/fluent/fluent/migrate/merge.py vendored
View File

@ -26,11 +26,7 @@ def merge_resource(ctx, reference, current, transforms, in_changeset):
def merge_entry(entry):
# All standalone comments will be merged.
if isinstance(entry, FTL.Comment):
return entry
# All section headers will be merged.
if isinstance(entry, FTL.Section):
if isinstance(entry, FTL.BaseComment):
return entry
# Ignore Junk
@ -56,4 +52,4 @@ def merge_resource(ctx, reference, current, transforms, in_changeset):
return evaluate(ctx, transform)
body = merge_body(reference.body)
return FTL.Resource(body, reference.comment)
return FTL.Resource(body)

162
third_party/python/fluent/fluent/migrate/transforms.py vendored
View File

@ -64,6 +64,7 @@ them.
"""
from __future__ import unicode_literals
import itertools
import fluent.syntax.ast as FTL
from .errors import NotSupportedError
@ -89,6 +90,38 @@ class Transform(FTL.BaseNode):
def __call__(self, ctx):
raise NotImplementedError
@staticmethod
def flatten_elements(elements):
'''Flatten a list of FTL nodes into valid Pattern's elements'''
flattened = []
for element in elements:
if isinstance(element, FTL.Pattern):
flattened.extend(element.elements)
elif isinstance(element, FTL.PatternElement):
flattened.append(element)
elif isinstance(element, FTL.Expression):
flattened.append(FTL.Placeable(element))
else:
raise RuntimeError(
'Expected Pattern, PatternElement or Expression')
return flattened
@staticmethod
def prune_text_elements(elements):
'''Join adjacent TextElements and remove empty ones'''
pruned = []
# Group elements in contiguous sequences of the same type.
for elem_type, elems in itertools.groupby(elements, key=type):
if elem_type is FTL.TextElement:
# Join adjacent TextElements.
text = FTL.TextElement(''.join(elem.value for elem in elems))
# And remove empty ones.
if len(text.value) > 0:
pruned.append(text)
else:
pruned.extend(elems)
return pruned
class Source(Transform):
"""Declare the source translation to be migrated with other transforms.
@ -128,11 +161,11 @@ class COPY(Source):
class REPLACE_IN_TEXT(Transform):
"""Replace various placeables in the translation with FTL placeables.
"""Replace various placeables in the translation with FTL.
The original placeables are defined as keys on the `replacements` dict.
For each key the value is defined as a list of FTL Expressions to be
interpolated.
For each key the value is defined as a FTL Pattern, Placeable,
TextElement or Expressions to be interpolated.
"""
def __init__(self, value, replacements):
@ -141,7 +174,7 @@ class REPLACE_IN_TEXT(Transform):
def __call__(self, ctx):
# Only replace placeable which are present in the translation.
# Only replace placeables which are present in the translation.
replacements = {
key: evaluate(ctx, repl)
for key, repl in self.replacements.iteritems()
@ -154,41 +187,25 @@ class REPLACE_IN_TEXT(Transform):
lambda x, y: self.value.find(x) - self.value.find(y)
)
# Used to reduce the `keys_in_order` list.
def replace(acc, cur):
"""Convert original placeables and text into FTL Nodes.
# A list of PatternElements built from the legacy translation and the
# FTL replacements. It may contain empty or adjacent TextElements.
elements = []
tail = self.value
For each original placeable the translation will be partitioned
around it and the text before it will be converted into an
`FTL.TextElement` and the placeable will be replaced with its
replacement. The text following the placebale will be fed again to
the `replace` function.
"""
# Convert original placeables and text into FTL Nodes. For each
# original placeable the translation will be partitioned around it and
# the text before it will be converted into an `FTL.TextElement` and
# the placeable will be replaced with its replacement.
for key in keys_in_order:
before, key, tail = tail.partition(key)
elements.append(FTL.TextElement(before))
elements.append(replacements[key])
parts, rest = acc
before, key, after = rest.value.partition(cur)
placeable = FTL.Placeable(replacements[key])
# Return the elements found and converted so far, and the remaining
# text which hasn't been scanned for placeables yet.
return (
parts + [FTL.TextElement(before), placeable],
FTL.TextElement(after)
)
def is_non_empty(elem):
"""Used for filtering empty `FTL.TextElement` nodes out."""
return not isinstance(elem, FTL.TextElement) or len(elem.value)
# Start with an empty list of elements and the original translation.
init = ([], FTL.TextElement(self.value))
parts, tail = reduce(replace, keys_in_order, init)
# Explicitly concat the trailing part to get the full list of elements
# and filter out the empty ones.
elements = filter(is_non_empty, parts + [tail])
# Dont' forget about the tail after the loop ends.
elements.append(FTL.TextElement(tail))
elements = self.flatten_elements(elements)
elements = self.prune_text_elements(elements)
return FTL.Pattern(elements)
@ -245,7 +262,7 @@ class PLURALS(Source):
# variant. Then evaluate it to a migrated FTL node.
value = evaluate(ctx, self.foreach(variant))
return FTL.Variant(
key=FTL.Symbol(key),
key=FTL.VariantName(key),
value=value,
default=index == last_index
)
@ -260,69 +277,16 @@ class PLURALS(Source):
class CONCAT(Transform):
"""Concatenate elements of many patterns."""
"""Create a new Pattern from Patterns, PatternElements and Expressions."""
def __init__(self, *patterns):
self.patterns = list(patterns)
def __init__(self, *elements, **kwargs):
# We want to support both passing elements as *elements in the
# migration specs and as elements=[]. The latter is used by
# FTL.BaseNode.traverse when it recreates the traversed node using its
# attributes as kwargs.
self.elements = list(kwargs.get('elements', elements))
def __call__(self, ctx):
# Flatten the list of patterns of which each has a list of elements.
def concat_elements(acc, cur):
if isinstance(cur, FTL.Pattern):
acc.extend(cur.elements)
return acc
elif (isinstance(cur, FTL.TextElement) or
isinstance(cur, FTL.Placeable)):
acc.append(cur)
return acc
raise RuntimeError(
'CONCAT accepts FTL Patterns, TextElements and Placeables.'
)
# Merge adjecent `FTL.TextElement` nodes.
def merge_adjecent_text(acc, cur):
if type(cur) == FTL.TextElement and len(acc):
last = acc[-1]
if type(last) == FTL.TextElement:
last.value += cur.value
else:
acc.append(cur)
else:
acc.append(cur)
return acc
elements = reduce(concat_elements, self.patterns, [])
elements = reduce(merge_adjecent_text, elements, [])
elements = self.flatten_elements(self.elements)
elements = self.prune_text_elements(elements)
return FTL.Pattern(elements)
def traverse(self, fun):
def visit(value):
if isinstance(value, FTL.BaseNode):
return value.traverse(fun)
if isinstance(value, list):
return fun(map(visit, value))
else:
return fun(value)
node = self.__class__(
*[
visit(value) for value in self.patterns
]
)
return fun(node)
def to_json(self):
def to_json(value):
if isinstance(value, FTL.BaseNode):
return value.to_json()
else:
return value
return {
'type': self.__class__.__name__,
'patterns': [
to_json(value) for value in self.patterns
]
}

4
third_party/python/fluent/fluent/migrate/util.py vendored
View File

@ -42,10 +42,12 @@ def to_json(merged_iter):
}
LOCALIZABLE_ENTRIES = (FTL.Message, FTL.Term)
def get_message(body, ident):
"""Get message called `ident` from the `body` iterable."""
for entity in body:
if isinstance(entity, FTL.Message) and entity.id.name == ident:
if isinstance(entity, LOCALIZABLE_ENTRIES) and entity.id.name == ident:
return entity

75
third_party/python/fluent/fluent/syntax/ast.py vendored
View File

@ -8,6 +8,8 @@ def to_json(value):
return value.to_json()
if isinstance(value, list):
return list(map(to_json, value))
if isinstance(value, tuple):
return list(map(to_json, value))
else:
return value
@ -64,12 +66,10 @@ class BaseNode(object):
else:
return fun(value)
# Use all attributes found on the node as kwargs to the constructor.
kwargs = vars(self).items()
node = self.__class__(
**{
name: visit(value)
for name, value in vars(self).items()
}
)
**{name: visit(value) for name, value in kwargs})
return fun(node)
@ -77,7 +77,7 @@ class BaseNode(object):
"""Compare two nodes.
Nodes are deeply compared on a field by field basis. If possible, False
is returned early. When comparing attributes, tags and variants in
is returned early. When comparing attributes and variants in
SelectExpressions, the order doesn't matter. By default, spans are not
taken into account.
"""
@ -98,7 +98,7 @@ class BaseNode(object):
field2 = getattr(other, key)
# List-typed nodes are compared item-by-item. When comparing
# attributes, tags and variants, the order of items doesn't matter.
# attributes and variants, the order of items doesn't matter.
if isinstance(field1, list) and isinstance(field2, list):
if len(field1) != len(field2):
return False
@ -108,7 +108,6 @@ class BaseNode(object):
# can't be keyed on any of their fields reliably.
field_sorting = {
'attributes': lambda elem: elem.id.name,
'tags': lambda elem: elem.name.name,
'variants': lambda elem: elem.key.name,
}
@ -152,10 +151,9 @@ class SyntaxNode(BaseNode):
class Resource(SyntaxNode):
def __init__(self, body=None, comment=None, **kwargs):
def __init__(self, body=None, **kwargs):
super(Resource, self).__init__(**kwargs)
self.body = body or []
self.comment = comment
class Entry(SyntaxNode):
@ -168,13 +166,21 @@ class Entry(SyntaxNode):
class Message(Entry):
def __init__(self, id, value=None, attributes=None, tags=None,
def __init__(self, id, value=None, attributes=None,
comment=None, **kwargs):
super(Message, self).__init__(**kwargs)
self.id = id
self.value = value
self.attributes = attributes or []
self.tags = tags or []
self.comment = comment
class Term(Entry):
def __init__(self, id, value, attributes=None,
comment=None, **kwargs):
super(Term, self).__init__(**kwargs)
self.id = id
self.value = value
self.attributes = attributes or []
self.comment = comment
class Pattern(SyntaxNode):
@ -182,12 +188,15 @@ class Pattern(SyntaxNode):
super(Pattern, self).__init__(**kwargs)
self.elements = elements
class TextElement(SyntaxNode):
class PatternElement(SyntaxNode):
pass
class TextElement(PatternElement):
def __init__(self, value, **kwargs):
super(TextElement, self).__init__(**kwargs)
self.value = value
class Placeable(SyntaxNode):
class Placeable(PatternElement):
def __init__(self, expression, **kwargs):
super(Placeable, self).__init__(**kwargs)
self.expression = expression
@ -235,10 +244,10 @@ class VariantExpression(Expression):
self.key = key
class CallExpression(Expression):
def __init__(self, callee, args, **kwargs):
def __init__(self, callee, args=None, **kwargs):
super(CallExpression, self).__init__(**kwargs)
self.callee = callee
self.args = args
self.args = args or []
class Attribute(SyntaxNode):
def __init__(self, id, value, **kwargs):
@ -246,11 +255,6 @@ class Attribute(SyntaxNode):
self.id = id
self.value = value
class Tag(SyntaxNode):
def __init__(self, name, **kwargs):
super(Tag, self).__init__(**kwargs)
self.name = name
class Variant(SyntaxNode):
def __init__(self, key, value, default=False, **kwargs):
super(Variant, self).__init__(**kwargs)
@ -269,20 +273,31 @@ class Identifier(SyntaxNode):
super(Identifier, self).__init__(**kwargs)
self.name = name
class Symbol(Identifier):
class VariantName(Identifier):
def __init__(self, name, **kwargs):
super(Symbol, self).__init__(name, **kwargs)
super(VariantName, self).__init__(name, **kwargs)
class Comment(Entry):
class BaseComment(Entry):
def __init__(self, content=None, **kwargs):
super(Comment, self).__init__(**kwargs)
super(BaseComment, self).__init__(**kwargs)
self.content = content
class Section(Entry):
def __init__(self, name, comment=None, **kwargs):
super(Section, self).__init__(**kwargs)
self.name = name
self.comment = comment
class Comment(BaseComment):
def __init__(self, content=None, **kwargs):
super(Comment, self).__init__(content, **kwargs)
class GroupComment(BaseComment):
def __init__(self, content=None, **kwargs):
super(GroupComment, self).__init__(content, **kwargs)
class ResourceComment(BaseComment):
def __init__(self, content=None, **kwargs):
super(ResourceComment, self).__init__(content, **kwargs)
class Function(Identifier):
def __init__(self, name, **kwargs):

21
third_party/python/fluent/fluent/syntax/errors.py vendored
View File

@ -18,26 +18,37 @@ def get_error_message(code, args):
if code == 'E0004':
return 'Expected a character from range: "{}"'.format(args[0])
if code == 'E0005':
msg = 'Expected entry "{}" to have a value or attributes'
msg = 'Expected message "{}" to have a value or attributes'
return msg.format(args[0])
if code == 'E0006':
return 'Expected field: "{}"'.format(args[0])
msg = 'Expected term "{}" to have a value'
return msg.format(args[0])
if code == 'E0007':
return 'Keyword cannot end with a whitespace'
if code == 'E0008':
return 'Callee has to be a simple identifier'
return 'The callee has to be a simple, upper-case identifier'
if code == 'E0009':
return 'Key has to be a simple identifier'
return 'The key has to be a simple identifier'
if code == 'E0010':
return 'Expected one of the variants to be marked as default (*)'
if code == 'E0011':
return 'Expected at least one variant after "->"'
if code == 'E0012':
return 'Tags cannot be added to messages with attributes'
return 'Expected value'
if code == 'E0013':
return 'Expected variant key'
if code == 'E0014':
return 'Expected literal'
if code == 'E0015':
return 'Only one variant can be marked as default (*)'
if code == 'E0016':
return 'Message references cannot be used as selectors'
if code == 'E0017':
return 'Variants cannot be used as selectors'
if code == 'E0018':
return 'Attributes of messages cannot be used as selectors'
if code == 'E0019':
return 'Attributes of terms cannot be used as placeables'
if code == 'E0020':
return 'Unterminated string expression'
return code

171
third_party/python/fluent/fluent/syntax/ftlstream.py vendored
View File

@ -4,9 +4,18 @@ from .errors import ParseError
INLINE_WS = (' ', '\t')
SPECIAL_LINE_START_CHARS = ('}', '.', '[', '*')
class FTLParserStream(ParserStream):
last_comment_zero_four_syntax = False
def skip_inline_ws(self):
while self.ch:
if self.ch not in INLINE_WS:
break
self.next()
def peek_inline_ws(self):
ch = self.current_peek()
while ch:
@ -25,11 +34,21 @@ class FTLParserStream(ParserStream):
self.reset_peek()
break
def skip_inline_ws(self):
while self.ch:
if self.ch not in INLINE_WS:
def peek_blank_lines(self):
while True:
line_start = self.get_peek_index()
self.peek_inline_ws()
if self.current_peek_is('\n'):
self.peek()
else:
self.reset_peek(line_start)
break
self.next()
def skip_indent(self):
self.skip_blank_lines()
self.skip_inline_ws()
def expect_char(self, ch):
if self.ch == ch:
@ -42,6 +61,12 @@ class FTLParserStream(ParserStream):
raise ParseError('E0003', ch)
def expect_indent(self):
self.expect_char('\n')
self.skip_blank_lines()
self.expect_char(' ')
self.skip_inline_ws()
def take_char_if(self, ch):
if self.ch == ch:
self.next()
@ -55,20 +80,87 @@ class FTLParserStream(ParserStream):
return ch
return None
def is_id_start(self):
if self.ch is None:
def is_char_id_start(self, ch=None):
if ch is None:
return False
cc = ord(self.ch)
cc = ord(ch)
return (cc >= 97 and cc <= 122) or \
(cc >= 65 and cc <= 90) or \
cc == 95
(cc >= 65 and cc <= 90)
def is_entry_id_start(self):
if self.current_is('-'):
self.peek()
ch = self.current_peek()
is_id = self.is_char_id_start(ch)
self.reset_peek()
return is_id
def is_number_start(self):
cc = ord(self.ch)
if self.current_is('-'):
self.peek()
return (cc >= 48 and cc <= 57) or cc == 45
cc = ord(self.current_peek())
is_digit = cc >= 48 and cc <= 57
self.reset_peek()
return is_digit
def is_char_pattern_continuation(self, ch):
return ch not in SPECIAL_LINE_START_CHARS
def is_peek_pattern_start(self):
self.peek_inline_ws()
ch = self.current_peek()
# Inline Patterns may start with any char.
if ch is not None and ch != '\n':
return True
return self.is_peek_next_line_pattern_start()
def is_peek_next_line_zero_four_style_comment(self):
if not self.current_peek_is('\n'):
return False
self.peek()
if self.current_peek_is('/'):
self.peek()
if self.current_peek_is('/'):
self.reset_peek()
return True
self.reset_peek()
return False
# -1 - any
# 0 - comment
# 1 - group comment
# 2 - resource comment
def is_peek_next_line_comment(self, level=-1):
if not self.current_peek_is('\n'):
return False
i = 0
while (i <= level or (level == -1 and i < 3)):
self.peek()
if not self.current_peek_is('#'):
if i != level and level != -1:
self.reset_peek()
return False
break
i += 1
self.peek()
if self.current_peek() in [' ', '\n']:
self.reset_peek()
return True
self.reset_peek()
return False
def is_peek_next_line_variant_start(self):
if not self.current_peek_is('\n'):
@ -76,6 +168,8 @@ class FTLParserStream(ParserStream):
self.peek()
self.peek_blank_lines()
ptr = self.get_peek_index()
self.peek_inline_ws()
@ -100,6 +194,8 @@ class FTLParserStream(ParserStream):
self.peek()
self.peek_blank_lines()
ptr = self.get_peek_index()
self.peek_inline_ws()
@ -115,12 +211,14 @@ class FTLParserStream(ParserStream):
self.reset_peek()
return False
def is_peek_next_line_pattern(self):
def is_peek_next_line_pattern_start(self):
if not self.current_peek_is('\n'):
return False
self.peek()
self.peek_blank_lines()
ptr = self.get_peek_index()
self.peek_inline_ws()
@ -129,57 +227,38 @@ class FTLParserStream(ParserStream):
self.reset_peek()
return False
if (self.current_peek_is('}') or
self.current_peek_is('.') or
self.current_peek_is('#') or
self.current_peek_is('[') or
self.current_peek_is('*')):
if not self.is_char_pattern_continuation(self.current_peek()):
self.reset_peek()
return False
self.reset_peek()
return True
def is_peek_next_line_tag_start(self):
if not self.current_peek_is('\n'):
return False
self.peek()
ptr = self.get_peek_index()
self.peek_inline_ws()
if (self.get_peek_index() - ptr == 0):
self.reset_peek()
return False
if self.current_peek_is('#'):
self.reset_peek()
return True
self.reset_peek()
return False
def skip_to_next_entry_start(self):
while self.ch:
if self.current_is('\n') and not self.peek_char_is('\n'):
self.next()
if self.ch is None or self.is_id_start() or \
if self.ch is None or \
self.is_entry_id_start() or \
self.current_is('#') or \
(self.current_is('/') and self.peek_char_is('/')) or \
(self.current_is('[') and self.peek_char_is('[')):
break
self.next()
def take_id_start(self):
if self.is_id_start():
def take_id_start(self, allow_term):
if allow_term and self.current_is('-'):
self.next()
return '-'
if self.is_char_id_start(self.ch):
ret = self.ch
self.next()
return ret
raise ParseError('E0004', 'a-zA-Z_')
allowed_range = 'a-zA-Z-' if allow_term else 'a-zA-Z'
raise ParseError('E0004', allowed_range)
def take_id_char(self):
def closure(ch):
@ -190,7 +269,7 @@ class FTLParserStream(ParserStream):
cc == 95 or cc == 45)
return self.take_char(closure)
def take_symb_char(self):
def take_variant_name_char(self):
def closure(ch):
if ch is None:
return False
@ -198,7 +277,7 @@ class FTLParserStream(ParserStream):
return (cc >= 97 and cc <= 122) or \
(cc >= 65 and cc <= 90) or \
(cc >= 48 and cc <= 57) or \
cc == 95 or cc == 45 or cc == 32
cc == 95 or cc == 45 or cc == 32
return self.take_char(closure)
def take_digit(self):

273
third_party/python/fluent/fluent/syntax/parser.py vendored
View File

@ -18,8 +18,8 @@ def with_span(fn):
if node.span is not None:
return node
# Spans of Messages and Sections should include the attached Comment.
if isinstance(node, ast.Message) or isinstance(node, ast.Section):
# Spans of Messages should include the attached Comment.
if isinstance(node, ast.Message):
if node.comment is not None:
start = node.comment.span.start
@ -35,8 +35,6 @@ class FluentParser(object):
self.with_spans = with_spans
def parse(self, source):
comment = None
ps = FTLParserStream(source)
ps.skip_blank_lines()
@ -45,14 +43,22 @@ class FluentParser(object):
while ps.current():
entry = self.get_entry_or_junk(ps)
if isinstance(entry, ast.Comment) and len(entries) == 0:
comment = entry
if entry is None:
continue
if isinstance(entry, ast.Comment) \
and ps.last_comment_zero_four_syntax \
and len(entries) == 0:
comment = ast.ResourceComment(entry.content)
comment.span = entry.span
entries.append(comment)
else:
entries.append(entry)
ps.last_comment_zero_four_syntax = False
ps.skip_blank_lines()
res = ast.Resource(entries, comment)
res = ast.Resource(entries)
if self.with_spans:
res.add_span(0, ps.get_index())
@ -87,13 +93,23 @@ class FluentParser(object):
def get_entry(self, ps):
comment = None
if ps.current_is('/'):
if ps.current_is('/') or ps.current_is('#'):
comment = self.get_comment(ps)
if ps.current_is('['):
return self.get_section(ps, comment)
# The Comment content doesn't include the trailing newline. Consume
# this newline to be ready for the next entry. None stands for EOF.
ps.expect_char('\n' if ps.current() else None)
if ps.is_id_start():
if ps.current_is('['):
group_comment = self.get_group_comment_from_section(ps, comment)
if comment and self.with_spans:
# The Group Comment should start where the section comment
# starts.
group_comment.span.start = comment.span.start
return group_comment
if ps.is_entry_id_start() \
and (comment is None or isinstance(comment, ast.Comment)):
return self.get_message(ps, comment)
if comment:
@ -102,82 +118,123 @@ class FluentParser(object):
raise ParseError('E0002')
@with_span
def get_comment(self, ps):
def get_zero_four_style_comment(self, ps):
ps.expect_char('/')
ps.expect_char('/')
ps.take_char_if(' ')
content = ''
def until_eol(x):
return x != '\n'
while True:
ch = ps.take_char(until_eol)
ch = ps.take_char(lambda x: x != '\n')
while ch:
content += ch
ch = ps.take_char(until_eol)
ch = ps.take_char(lambda x: x != '\n')
ps.next()
if ps.current_is('/'):
if ps.is_peek_next_line_zero_four_style_comment():
content += '\n'
ps.next()
ps.expect_char('/')
ps.expect_char('/')
ps.take_char_if(' ')
else:
break
return ast.Comment(content)
comment = ast.Comment(content)
ps.last_comment_zero_four_syntax = True
return comment
@with_span
def get_section(self, ps, comment):
def get_comment(self, ps):
if ps.current_is('/'):
return self.get_zero_four_style_comment(ps)
# 0 - comment
# 1 - group comment
# 2 - resource comment
level = -1
content = ''
while True:
i = -1
while ps.current_is('#') and (i < (2 if level == -1 else level)):
ps.next()
i += 1
if level == -1:
level = i
if not ps.current_is('\n'):
ps.expect_char(' ')
ch = ps.take_char(lambda x: x != '\n')
while ch:
content += ch
ch = ps.take_char(lambda x: x != '\n')
if ps.is_peek_next_line_comment(level):
content += '\n'
ps.next()
else:
break
if level == 0:
return ast.Comment(content)
elif level == 1:
return ast.GroupComment(content)
elif level == 2:
return ast.ResourceComment(content)
@with_span
def get_group_comment_from_section(self, ps, comment):
ps.expect_char('[')
ps.expect_char('[')
ps.skip_inline_ws()
symb = self.get_symbol(ps)
self.get_variant_name(ps)
ps.skip_inline_ws()
ps.expect_char(']')
ps.expect_char(']')
ps.skip_inline_ws()
if comment:
return ast.GroupComment(comment.content)
ps.expect_char('\n')
return ast.Section(symb, comment)
# A Section without a comment is like an empty Group Comment.
# Semantically it ends the previous group and starts a new one.
return ast.GroupComment('')
@with_span
def get_message(self, ps, comment):
id = self.get_identifier(ps)
id = self.get_entry_identifier(ps)
ps.skip_inline_ws()
pattern = None
attrs = None
tags = None
# XXX Syntax 0.4 compat
if ps.current_is('='):
ps.next()
ps.skip_inline_ws()
pattern = self.get_pattern(ps)
if ps.is_peek_pattern_start():
ps.skip_indent()
pattern = self.get_pattern(ps)
if id.name.startswith('-') and pattern is None:
raise ParseError('E0006', id.name)
if ps.is_peek_next_line_attribute_start():
attrs = self.get_attributes(ps)
if ps.is_peek_next_line_tag_start():
if attrs is not None:
raise ParseError('E0012')
tags = self.get_tags(ps)
if id.name.startswith('-'):
return ast.Term(id, pattern, attrs, comment)
if pattern is None and attrs is None:
raise ParseError('E0005', id.name)
return ast.Message(id, pattern, attrs, tags, comment)
return ast.Message(id, pattern, attrs, comment)
@with_span
def get_attribute(self, ps):
@ -187,22 +244,19 @@ class FluentParser(object):
ps.skip_inline_ws()
ps.expect_char('=')
ps.skip_inline_ws()
value = self.get_pattern(ps)
if ps.is_peek_pattern_start():
ps.skip_indent()
value = self.get_pattern(ps)
return ast.Attribute(key, value)
if value is None:
raise ParseError('E0006', 'value')
return ast.Attribute(key, value)
raise ParseError('E0012')
def get_attributes(self, ps):
attrs = []
while True:
ps.expect_char('\n')
ps.skip_inline_ws()
ps.expect_indent()
attr = self.get_attribute(ps)
attrs.append(attr)
@ -210,31 +264,14 @@ class FluentParser(object):
break
return attrs
@with_span
def get_tag(self, ps):
ps.expect_char('#')
symb = self.get_symbol(ps)
return ast.Tag(symb)
def get_tags(self, ps):
tags = []
while True:
ps.expect_char('\n')
ps.skip_inline_ws()
tag = self.get_tag(ps)
tags.append(tag)
if not ps.is_peek_next_line_tag_start():
break
return tags
def get_entry_identifier(self, ps):
return self.get_identifier(ps, True)
@with_span
def get_identifier(self, ps):
def get_identifier(self, ps, allow_term=False):
name = ''
name += ps.take_id_start()
name += ps.take_id_start(allow_term)
ch = ps.take_id_char()
while ch:
@ -249,10 +286,11 @@ class FluentParser(object):
if ch is None:
raise ParseError('E0013')
if ps.is_number_start():
cc = ord(ch)
if ((cc >= 48 and cc <= 57) or cc == 45): # 0-9, -
return self.get_number(ps)
return self.get_symbol(ps)
return self.get_variant_name(ps)
@with_span
def get_variant(self, ps, has_default):
@ -270,23 +308,19 @@ class FluentParser(object):
ps.expect_char(']')
ps.skip_inline_ws()
if ps.is_peek_pattern_start():
ps.skip_indent()
value = self.get_pattern(ps)
return ast.Variant(key, value, default_index)
value = self.get_pattern(ps)
if value is None:
raise ParseError('E0006', 'value')
return ast.Variant(key, value, default_index)
raise ParseError('E0012')
def get_variants(self, ps):
variants = []
has_default = False
while True:
ps.expect_char('\n')
ps.skip_inline_ws()
ps.expect_indent()
variant = self.get_variant(ps, has_default)
if variant.default:
@ -303,19 +337,19 @@ class FluentParser(object):
return variants
@with_span
def get_symbol(self, ps):
def get_variant_name(self, ps):
name = ''
name += ps.take_id_start()
name += ps.take_id_start(False)
while True:
ch = ps.take_symb_char()
ch = ps.take_variant_name_char()
if ch:
name += ch
else:
break
return ast.Symbol(name.rstrip())
return ast.VariantName(name.rstrip())
def get_digits(self, ps):
num = ''
@ -352,17 +386,12 @@ class FluentParser(object):
elements = []
ps.skip_inline_ws()
# Special-case: trim leading whitespace and newlines.
if ps.is_peek_next_line_pattern():
ps.skip_blank_lines()
ps.skip_inline_ws()
while ps.current():
ch = ps.current()
# The end condition for get_pattern's while loop is a newline
# which is not followed by a valid pattern continuation.
if ch == '\n' and not ps.is_peek_next_line_pattern():
if ch == '\n' and not ps.is_peek_next_line_pattern_start():
break
if ch == '{':
@ -385,7 +414,7 @@ class FluentParser(object):
return ast.TextElement(buf)
if ch == '\n':
if not ps.is_peek_next_line_pattern():
if not ps.is_peek_next_line_pattern_start():
return ast.TextElement(buf)
ps.next()
@ -423,9 +452,7 @@ class FluentParser(object):
if ps.is_peek_next_line_variant_start():
variants = self.get_variants(ps)
ps.expect_char('\n')
ps.expect_char(' ')
ps.skip_inline_ws()
ps.expect_indent()
return ast.SelectExpression(None, variants)
@ -437,24 +464,37 @@ class FluentParser(object):
if ps.current_is('-'):
ps.peek()
if not ps.current_peek_is('>'):
ps.reset_peek()
else:
ps.next()
ps.next()
return selector
ps.skip_inline_ws()
if isinstance(selector, ast.MessageReference):
raise ParseError('E0016')
variants = self.get_variants(ps)
if isinstance(selector, ast.AttributeExpression) and \
not selector.id.name.startswith('-'):
raise ParseError('E0018')
if len(variants) == 0:
raise ParseError('E0011')
if isinstance(selector, ast.VariantExpression):
raise ParseError('E0017')
ps.expect_char('\n')
ps.expect_char(' ')
ps.skip_inline_ws()
ps.next()
ps.next()
return ast.SelectExpression(selector, variants)
ps.skip_inline_ws()
variants = self.get_variants(ps)
if len(variants) == 0:
raise ParseError('E0011')
ps.expect_indent()
return ast.SelectExpression(selector, variants)
elif isinstance(selector, ast.AttributeExpression) and \
selector.id.name.startswith('-'):
raise ParseError('E0019')
return selector
@ -485,7 +525,7 @@ class FluentParser(object):
ps.expect_char(')')
if not re.match('^[A-Z_-]+$', literal.id.name):
if not re.match('^[A-Z][A-Z_?-]*$', literal.id.name):
raise ParseError('E0008')
return ast.CallExpression(
@ -542,7 +582,7 @@ class FluentParser(object):
return self.get_number(ps)
elif ps.current_is('"'):
return self.get_string(ps)
raise ParseError('E0006', 'value')
raise ParseError('E0012')
@with_span
def get_string(self, ps):
@ -550,10 +590,13 @@ class FluentParser(object):
ps.expect_char('"')
ch = ps.take_char(lambda x: x != '"')
ch = ps.take_char(lambda x: x != '"' and x != '\n')
while ch:
val += ch
ch = ps.take_char(lambda x: x != '"')
ch = ps.take_char(lambda x: x != '"' and x != '\n')
if ps.current_is('\n'):
raise ParseError('E0020')
ps.next()
@ -566,14 +609,16 @@ class FluentParser(object):
if ch is None:
raise ParseError('E0014')
if ps.is_number_start():
return self.get_number(ps)
elif ch == '"':
return self.get_string(ps)
elif ch == '$':
if ch == '$':
ps.next()
name = self.get_identifier(ps)
return ast.ExternalArgument(name)
elif ps.is_entry_id_start():
name = self.get_entry_identifier(ps)
return ast.MessageReference(name)
elif ps.is_number_start():
return self.get_number(ps)
elif ch == '"':
return self.get_string(ps)
name = self.get_identifier(ps)
return ast.MessageReference(name)
raise ParseError('E0014')

114
third_party/python/fluent/fluent/syntax/serializer.py vendored
View File

@ -16,52 +16,70 @@ def contain_new_line(elems):
class FluentSerializer(object):
HAS_ENTRIES = 1
def __init__(self, with_junk=False):
self.with_junk = with_junk
def serialize(self, resource):
if not isinstance(resource, ast.Resource):
raise Exception('Unknown resource type: {}'.format(type(resource)))
state = 0
parts = []
if resource.comment:
parts.append(
"{}\n\n".format(
serialize_comment(resource.comment)
)
)
for entry in resource.body:
if not isinstance(entry, ast.Junk) or self.with_junk:
parts.append(self.serialize_entry(entry))
parts.append(self.serialize_entry(entry, state))
if not state & self.HAS_ENTRIES:
state |= self.HAS_ENTRIES
return "".join(parts)
def serialize_entry(self, entry):
def serialize_entry(self, entry, state=0):
if isinstance(entry, ast.Message):
return serialize_message(entry)
if isinstance(entry, ast.Section):
return serialize_section(entry)
if isinstance(entry, ast.Term):
return serialize_message(entry)
if isinstance(entry, ast.Comment):
return "\n{}\n\n".format(serialize_comment(entry))
if state & self.HAS_ENTRIES:
return "\n{}\n\n".format(serialize_comment(entry))
return "{}\n\n".format(serialize_comment(entry))
if isinstance(entry, ast.GroupComment):
if state & self.HAS_ENTRIES:
return "\n{}\n\n".format(serialize_group_comment(entry))
return "{}\n\n".format(serialize_group_comment(entry))
if isinstance(entry, ast.ResourceComment):
if state & self.HAS_ENTRIES:
return "\n{}\n\n".format(serialize_resource_comment(entry))
return "{}\n\n".format(serialize_resource_comment(entry))
if isinstance(entry, ast.Junk):
return serialize_junk(entry)
raise Exception('Unknown entry type: {}'.format(entry.type))
raise Exception('Unknown entry type: {}'.format(type(entry)))
def serialize_expression(self, expr):
return serialize_expression(expr)
def serialize_comment(comment):
return "".join([
"{}{}".format("// ", line)
for line in comment.content.splitlines(True)
return "\n".join([
"#" if len(line) == 0 else "# {}".format(line)
for line in comment.content.splitlines(False)
])
def serialize_section(section):
if section.comment:
return "\n\n{}\n[[ {} ]]\n\n".format(
serialize_comment(section.comment),
serialize_symbol(section.name)
)
else:
return "\n\n[[ {} ]]\n\n".format(
serialize_symbol(section.name)
)
def serialize_group_comment(comment):
return "\n".join([
"##" if len(line) == 0 else "## {}".format(line)
for line in comment.content.splitlines(False)
])
def serialize_resource_comment(comment):
return "\n".join([
"###" if len(line) == 0 else "### {}".format(line)
for line in comment.content.splitlines(False)
])
def serialize_junk(junk):
@ -76,15 +94,11 @@ def serialize_message(message):
parts.append("\n")
parts.append(serialize_identifier(message.id))
parts.append(" =")
if message.value:
parts.append(" =")
parts.append(serialize_value(message.value))
if message.tags:
for tag in message.tags:
parts.append(serialize_tag(tag))
if message.attributes:
for attribute in message.attributes:
parts.append(serialize_attribute(attribute))
@ -94,12 +108,6 @@ def serialize_message(message):
return ''.join(parts)
def serialize_tag(tag):
return "\n #{}".format(
serialize_symbol(tag.name),
)
def serialize_attribute(attribute):
return "\n .{} ={}".format(
serialize_identifier(attribute.id),
@ -127,7 +135,7 @@ def serialize_element(element):
return serialize_text_element(element)
if isinstance(element, ast.Placeable):
return serialize_placeable(element)
raise Exception('Unknown element type: {}'.format(element.type))
raise Exception('Unknown element type: {}'.format(type(element)))
def serialize_text_element(text):
@ -138,14 +146,18 @@ def serialize_placeable(placeable):
expr = placeable.expression
if isinstance(expr, ast.Placeable):
return "{{{}}}".format(
serialize_placeable(expr))
return "{{{}}}".format(serialize_placeable(expr))
if isinstance(expr, ast.SelectExpression):
return "{{{}}}".format(
serialize_select_expression(expr))
# Special-case select expressions to control the withespace around the
# opening and the closing brace.
if expr.expression is not None:
# A select expression with a selector.
return "{{ {}}}".format(serialize_select_expression(expr))
else:
# A variant list without a selector.
return "{{{}}}".format(serialize_select_expression(expr))
if isinstance(expr, ast.Expression):
return "{{ {} }}".format(
serialize_expression(expr))
return "{{ {} }}".format(serialize_expression(expr))
def serialize_expression(expression):
@ -163,7 +175,9 @@ def serialize_expression(expression):
return serialize_variant_expression(expression)
if isinstance(expression, ast.CallExpression):
return serialize_call_expression(expression)
raise Exception('Unknown expression type: {}'.format(expression.type))
if isinstance(expression, ast.SelectExpression):
return serialize_select_expression(expression)
raise Exception('Unknown expression type: {}'.format(type(expression)))
def serialize_string_expression(expr):
@ -186,7 +200,7 @@ def serialize_select_expression(expr):
parts = []
if expr.expression:
selector = " {} ->".format(
selector = "{} ->".format(
serialize_expression(expr.expression)
)
parts.append(selector)
@ -250,23 +264,23 @@ def serialize_argument_value(argval):
return serialize_string_expression(argval)
if isinstance(argval, ast.NumberExpression):
return serialize_number_expression(argval)
raise Exception('Unknown argument type: {}'.format(argval.type))
raise Exception('Unknown argument type: {}'.format(type(argval)))
def serialize_identifier(identifier):
return identifier.name
def serialize_symbol(symbol):
def serialize_variant_name(symbol):
return symbol.name
def serialize_variant_key(key):
if isinstance(key, ast.Symbol):
return serialize_symbol(key)
if isinstance(key, ast.VariantName):
return serialize_variant_name(key)
if isinstance(key, ast.NumberExpression):
return serialize_number_expression(key)
raise Exception('Unknown variant key type: {}'.format(key.type))
raise Exception('Unknown variant key type: {}'.format(type(key)))
def serialize_function(function):

13
third_party/python/fluent/fluent/syntax/stream.py vendored
View File

@ -43,7 +43,7 @@ class ParserStream():
self.index += 1
if self.ch == None:
if self.ch is None:
self.iter_end = True
self.peek_end = True
@ -104,9 +104,14 @@ class ParserStream():
return ret == ch
def reset_peek(self):
self.peek_index = self.index
self.peek_end = self.iter_end
def reset_peek(self, pos=False):
if pos:
if pos < self.peek_index:
self.peek_end = False
self.peek_index = pos
else:
self.peek_index = self.index
self.peek_end = self.iter_end
def skip_to_peek(self):
diff = self.peek_index - self.index

32
third_party/python/fluent/tools/migrate/blame.py vendored
View File

@ -1,19 +1,22 @@
import argparse
import json
import os
from compare_locales.parser import getParser, Junk
import hglib
from hglib.util import b, cmdbuilder
from compare_locales.parser import getParser, Junk
class Blame(object):
def __init__(self, repopath):
self.client = hglib.open(repopath)
def __init__(self, client):
self.client = client
self.users = []
self.blame = {}
def main(self):
def attribution(self, file_paths):
args = cmdbuilder(
b('annotate'), self.client.root(), d=True, u=True, T='json')
b('annotate'), *map(b, file_paths), template='json',
date=True, user=True, cwd=self.client.root())
blame_json = ''.join(self.client.rawcommand(args))
file_blames = json.loads(blame_json)
@ -24,16 +27,16 @@ class Blame(object):
'blame': self.blame}
def handleFile(self, file_blame):
abspath = file_blame['abspath']
path = file_blame['path']
try:
parser = getParser(abspath)
parser = getParser(path)
except UserWarning:
return
self.blame[abspath] = {}
self.blame[path] = {}
parser.readFile(file_blame['path'])
parser.readFile(os.path.join(self.client.root(), path))
entities, emap = parser.parse()
for e in entities:
if isinstance(e, Junk):
@ -49,12 +52,13 @@ class Blame(object):
if user not in self.users:
self.users.append(user)
userid = self.users.index(user)
self.blame[abspath][e.key] = [userid, timestamp]
self.blame[path][e.key] = [userid, timestamp]
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument("repopath")
parser.add_argument('repo_path')
parser.add_argument('file_path', nargs='+')
args = parser.parse_args()
blame = Blame(args.repopath)
blimey = blame.main()
print(json.dumps(blimey, indent=4, separators=(',', ': ')))
blame = Blame(hglib.open(args.repo_path))
attrib = blame.attribution(args.file_path)
print(json.dumps(attrib, indent=4, separators=(',', ': ')))

5
third_party/python/fluent/tools/migrate/examples/about_dialog.py vendored
View File

@ -1,9 +1,8 @@
# coding=utf8
import fluent.syntax.ast as FTL
from fluent.migrate import (
CONCAT, EXTERNAL_ARGUMENT, MESSAGE_REFERENCE, COPY, REPLACE
)
from fluent.migrate import CONCAT, COPY, REPLACE
from fluent.migrate.helpers import EXTERNAL_ARGUMENT, MESSAGE_REFERENCE
def migrate(ctx):

3
third_party/python/fluent/tools/migrate/examples/about_downloads.py vendored
View File

@ -1,7 +1,8 @@
# coding=utf8
import fluent.syntax.ast as FTL
from fluent.migrate import EXTERNAL_ARGUMENT, COPY, PLURALS, REPLACE_IN_TEXT
from fluent.migrate import COPY, PLURALS, REPLACE_IN_TEXT
from fluent.migrate.helpers import EXTERNAL_ARGUMENT
def migrate(ctx):

3
third_party/python/fluent/tools/migrate/examples/bug_1291693.py vendored
View File

@ -1,7 +1,8 @@
# coding=utf8
import fluent.syntax.ast as FTL
from fluent.migrate import MESSAGE_REFERENCE, COPY, REPLACE
from fluent.migrate import COPY, REPLACE
from fluent.migrate.helpers import MESSAGE_REFERENCE
def migrate(ctx):

39
third_party/python/fluent/tools/migrate/migrate-l10n.py vendored
View File

@ -11,20 +11,20 @@ import importlib
import hglib
from hglib.util import b
from fluent.migrate import (
MergeContext, MigrationError, convert_blame_to_changesets
)
from fluent.migrate.context import MergeContext
from fluent.migrate.errors import MigrationError
from fluent.migrate.changesets import convert_blame_to_changesets
from blame import Blame
def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
def main(lang, reference_dir, localization_dir, migrations, dry_run):
"""Run migrations and commit files with the result."""
changesets = convert_blame_to_changesets(blame)
client = hglib.open(localization_dir)
for migration in migrations:
print('Running migration {}'.format(migration.__name__))
print('\nRunning migration {} for {}'.format(
migration.__name__, lang))
# For each migration create a new context.
ctx = MergeContext(lang, reference_dir, localization_dir)
@ -32,12 +32,22 @@ def main(lang, reference_dir, localization_dir, blame, migrations, dry_run):
try:
# Add the migration spec.
migration.migrate(ctx)
except MigrationError as err:
sys.exit(err.message)
except MigrationError:
print(' Skipping migration {} for {}'.format(
migration.__name__, lang))
continue
# Keep track of how many changesets we're committing.
index = 0
# Annotate legacy localization files used as sources by this migration
# to preserve attribution of translations.
files = (
path for path in ctx.localization_resources.keys()
if not path.endswith('.ftl'))
blame = Blame(client).attribution(files)
changesets = convert_blame_to_changesets(blame)
for changeset in changesets:
# Run the migration for the changeset.
snapshot = ctx.serialize_changeset(changeset['changes'])
@ -91,10 +101,6 @@ if __name__ == '__main__':
'--localization-dir', type=str,
help='directory for localization files'
)
parser.add_argument(
'--blame', type=argparse.FileType(), default=None,
help='path to a JSON with blame information'
)
parser.add_argument(
'--dry-run', action='store_true',
help='do not write to disk nor commit any changes'
@ -106,19 +112,10 @@ if __name__ == '__main__':
args = parser.parse_args()
if args.blame:
# Load pre-computed blame from a JSON file.
blame = json.load(args.blame)
else:
# Compute blame right now.
print('Annotating {}'.format(args.localization_dir))
blame = Blame(args.localization_dir).main()
main(
lang=args.lang,
reference_dir=args.reference_dir,
localization_dir=args.localization_dir,
blame=blame,
migrations=map(importlib.import_module, args.migrations),
dry_run=args.dry_run
)