Merge pull request #923 from CleoQc/feature/long_integer_with_underscores

Feature/long integer with underscores

CHANGES.rst

@@ -23,6 +23,8 @@ unreleased
     :func:`meta.find_undeclared_variables`. #931
 -   Float literals can be written with scientific notation, like
     ``{{ 2.56e-3 }}``. #912, #922
+-   Int and float literals can be written with the '_' separator for
+    legibility, like ``{{ 12_345 }}``. #923
 
 .. _#557: https://github.com/pallets/jinja/issues/557
 .. _#765: https://github.com/pallets/jinja/issues/765

docs/templates.rst

@@ -1177,13 +1177,16 @@ for Python objects such as strings and numbers.  The following literals exist:
     arguments to function calls and filters, or just to extend or include a
     template).
 
-42:
-    Integers are whole numbers without a decimal part.
+42 / 123_456:
+    Integers are whole numbers without a decimal part. The '_' character
+    can be used to separate groups for legibility.
 
-42.23 / 42.1e2:
+42.23 / 42.1e2 / 123_456.789:
     Floating point numbers can be written using a '.' as a decimal mark.
     They can also be written in scientific notation with an upper or
-    lower case 'e' to indicate the exponent part.
+    lower case 'e' to indicate the exponent part. The '_' character can
+    be used to separate groups for legibility, but cannot be used in the
+    exponent part.
 
 ['list', 'of', 'objects']:
     Everything between two brackets is a list.  Lists are useful for storing

jinja2/lexer.py

@@ -23,15 +23,31 @@ from jinja2._compat import implements_iterator, intern, iteritems, text_type
 from jinja2.exceptions import TemplateSyntaxError
 from jinja2.utils import LRUCache
 
+from ast import literal_eval # to support scientific notation
+
 # cache for the lexers. Exists in order to be able to have multiple
 # environments with the same lexer
 _lexer_cache = LRUCache(50)
 
 # static regular expressions
 whitespace_re = re.compile(r'\s+', re.U)
+newline_re = re.compile(r'(\r\n|\r|\n)')
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
-integer_re = re.compile(r'\d+')
+integer_re = re.compile(r'(\d+_)*\d+')
+float_re = re.compile(
+    r"""
+    (?<!\.)  # doesn't start with a .
+    (\d+_)*\d+  # digits, possibly _ separated
+    (
+        (\.(\d+_)*\d+)?  # optional fractional part
+        e[+\-]?(\d+_)*\d+  # exponent part
+    |
+        \.(\d+_)*\d+  # required fractional part
+    )
+    """,
+    re.IGNORECASE | re.VERBOSE,
+)
 
 try:
     # check if this Python supports Unicode identifiers
@@ -53,9 +69,6 @@ else:
     del jinja2._identifier
     del _identifier
 
-float_re = re.compile(r'(?<!\.)\d+(?:\.\d+)?(?:e[+\-]?\d+)?', re.IGNORECASE)
-newline_re = re.compile(r'(\r\n|\r|\n)')
-
 # internal the tokens and keep references to them
 TOKEN_ADD = intern('add')
 TOKEN_ASSIGN = intern('assign')
@@ -600,9 +613,10 @@ class Lexer(object):
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
             elif token == 'integer':
-                value = int(value)
+                value = int(value.replace("_", ""))
             elif token == 'float':
-                value = literal_eval(value)
+                # remove all "_" first to support more Python versions
+                value = literal_eval(value.replace("_", ""))
             elif token == 'operator':
                 token = operators[value]
             yield Token(lineno, token, value)

tests/test_filters.py

@@ -133,14 +133,16 @@ class TestFilter(object):
         out = tmpl.render(foo=list(range(10)))
         assert out == '0'
 
-    def test_float(self, env):
-        tmpl = env.from_string('{{ "42"|float }}|'
-                               '{{ "ajsghasjgd"|float }}|'
-                               '{{ "10e1"|float }}|'
-                               '{{ "10.5e-10"|float }}|'
-                               '{{ "32.32"|float }}')
-        out = tmpl.render()
-        assert out == '42.0|0.0|100.0|1.05e-09|32.32'
+    @pytest.mark.parametrize(
+        ("value", "expect"), (("42", "42.0"), ("abc", "0.0"), ("32.32", "32.32"),)
+    )
+    def test_float(self, env, value, expect):
+        t = env.from_string("{{ '%s'|float }}" % value)
+        assert t.render() == expect
+
+    def test_float_default(self, env):
+        t = env.from_string("{{ value|float(default=1.0) }}")
+        assert t.render(value="abc") == "1.0"
 
     def test_format(self, env):
         tmpl = env.from_string('''{{ "%s|%s"|format("a", "b") }}''')
@@ -180,17 +182,42 @@ class TestFilter(object):
         with pytest.warns(DeprecationWarning):
             env.from_string('{{ "jinja"|indent(indentfirst=true) }}').render()
 
-    def test_int(self, env):
+    @pytest.mark.parametrize(
+        ("value", "expect"),
+        (
+            ("42", "42"),
+            ("abc", "0"),
+            ("32.32", "32"),
+            ("12345678901234567890", "12345678901234567890"),
+        )
+    )
+    def test_int(self, env, value, expect):
+        t = env.from_string("{{ '%s'|int }}" % value)
+        assert t.render() == expect
+
+    @pytest.mark.parametrize(
+        ("value", "base", "expect"),
+        (
+            ("0x4d32", 16, "19762"),
+            ("011", 8, "9"),
+            ("0x33Z", 16, "0"),
+        )
+    )
+    def test_int_base(self, env, value, base, expect):
+        t = env.from_string("{{ '%s'|int(base=%d) }}" % (value, base))
+        assert t.render() == expect
+
+    def test_int_default(self, env):
+        t = env.from_string("{{ value|int(default=1) }}")
+        assert t.render(value="abc") == "1"
+
+    def test_int_special_method(self, env):
         class IntIsh(object):
             def __int__(self):
                 return 42
 
-        tmpl = env.from_string('{{ "42"|int }}|{{ "ajsghasjgd"|int }}|'
-                               '{{ "32.32"|int }}|{{ "0x4d32"|int(0, 16) }}|'
-                               '{{ "011"|int(0, 8)}}|{{ "0x33FU"|int(0, 16) }}|'
-                               '{{ obj|int }}')
-        out = tmpl.render(obj=IntIsh())
-        assert out == '42|0|32|19762|9|0|42'
+        t = env.from_string("{{ value|int }}")
+        assert t.render(value=IntIsh()) == "42"
 
     def test_join(self, env):
         tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}')

tests/test_lexnparse.py

@@ -341,29 +341,24 @@ class TestSyntax(object):
         t = env.from_string("{{ %s }}" % value)
         assert t.render() == value
 
-    @pytest.mark.parametrize("value", ("1", "123"))
-    def test_int_literal(self, env, value):
-        t = env.from_string("{{ %s }}" % value)
-        assert t.render() == value
-
     @pytest.mark.parametrize(
-        "value",
+        ("value", "expect"),
         (
-            "1.2",
-            "34.56",
+            ("1", "1"),
+            ("123", "123"),
+            ("12_34_56", "123456"),
+            ("1.2", "1.2"),
+            ("34.56", "34.56"),
+            ("3_4.5_6", "34.56"),
             ("1e0", "1.0"),
             ("10e1", "100.0"),
             ("2.5e100", "2.5e+100"),
-            "2.5e+100",
+            ("2.5e+100", "2.5e+100"),
             ("25.6e-10", "2.56e-09"),
-        ),
+            ("1_2.3_4e5_6", "1.234e+57"),
+        )
     )
-    def test_float_literal(self, env, value):
-        if isinstance(value, tuple):
-            value, expect = value
-        else:
-            expect = value
-
+    def test_numeric_literal(self, env, value, expect):
         t = env.from_string("{{ %s }}" % value)
         assert t.render() == expect