clean up numeric underscore support

add changelog
clean up docs
parametrize tests
explain float regex

CHANGES.rst

@@ -23,6 +23,8 @@ unreleased
     :func:`meta.find_undeclared_variables`. #931
 -   Float literals can be written with scientific notation, like
     ``{{ 2.56e-3 }}``. #912, #922
+-   Int and float literals can be written with the '_' separator for
+    legibility, like ``{{ 12_345 }}``. #923
 
 .. _#557: https://github.com/pallets/jinja/issues/557
 .. _#765: https://github.com/pallets/jinja/issues/765

jinja2/lexer.py

@@ -31,9 +31,23 @@ _lexer_cache = LRUCache(50)
 
 # static regular expressions
 whitespace_re = re.compile(r'\s+', re.U)
+newline_re = re.compile(r'(\r\n|\r|\n)')
 string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
                        r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
-integer_re = re.compile(r'(\d+_)*(\d+)')
+integer_re = re.compile(r'(\d+_)*\d+')
+float_re = re.compile(
+    r"""
+    (?<!\.)  # doesn't start with a .
+    (\d+_)*\d+  # digits, possibly _ separated
+    (
+        (\.(\d+_)*\d+)?  # optional fractional part
+        e[+\-]?(\d+_)*\d+  # exponent part
+    |
+        \.(\d+_)*\d+  # required fractional part
+    )
+    """,
+    re.IGNORECASE | re.VERBOSE,
+)
 
 try:
     # check if this Python supports Unicode identifiers
@@ -55,9 +69,6 @@ else:
     del jinja2._identifier
     del _identifier
 
-float_re = re.compile(r'(?<!\.)((((\d+_)*)\d+)(((\.\d+)?e-?\d+)|(\.((\d+_)*\d+))+))', re.IGNORECASE)
-newline_re = re.compile(r'(\r\n|\r|\n)')
-
 # internal the tokens and keep references to them
 TOKEN_ADD = intern('add')
 TOKEN_ASSIGN = intern('assign')

tests/test_filters.py

@@ -10,7 +10,7 @@
 """
 import random
 import pytest
-from jinja2 import Markup, Environment, Template
+from jinja2 import Markup, Environment
 from jinja2._compat import text_type, implements_to_string
 
 
@@ -133,18 +133,16 @@ class TestFilter(object):
         out = tmpl.render(foo=list(range(10)))
         assert out == '0'
 
-    def test_float(self, env):
-        tmpl = env.from_string('{{ "42"|float }}|'
-                               '{{ "ajsghasjgd"|float }}|'
-                               '{{ "1e2"|float }}|'
-                               '{{ "10e1"|float }}|'
-                               '{{ "10.5e-10"|float }}|'
-                               '{{ "32.32"|float }}')
-        out = tmpl.render()
-        assert out == '42.0|0.0|100.0|100.0|1.05e-09|32.32'
+    @pytest.mark.parametrize(
+        ("value", "expect"), (("42", "42.0"), ("abc", "0.0"), ("32.32", "32.32"),)
+    )
+    def test_float(self, env, value, expect):
+        t = env.from_string("{{ '%s'|float }}" % value)
+        assert t.render() == expect
 
-        out = Template("{{12_3_4.5_6}}|{{12_34e0}}").render()
-        assert out == '1234.56|1234.0'
+    def test_float_default(self, env):
+        t = env.from_string("{{ value|float(default=1.0) }}")
+        assert t.render(value="abc") == "1.0"
 
     def test_format(self, env):
         tmpl = env.from_string('''{{ "%s|%s"|format("a", "b") }}''')
@@ -184,20 +182,42 @@ class TestFilter(object):
         with pytest.warns(DeprecationWarning):
             env.from_string('{{ "jinja"|indent(indentfirst=true) }}').render()
 
-    def test_int(self, env):
+    @pytest.mark.parametrize(
+        ("value", "expect"),
+        (
+            ("42", "42"),
+            ("abc", "0"),
+            ("32.32", "32"),
+            ("12345678901234567890", "12345678901234567890"),
+        )
+    )
+    def test_int(self, env, value, expect):
+        t = env.from_string("{{ '%s'|int }}" % value)
+        assert t.render() == expect
+
+    @pytest.mark.parametrize(
+        ("value", "base", "expect"),
+        (
+            ("0x4d32", 16, "19762"),
+            ("011", 8, "9"),
+            ("0x33Z", 16, "0"),
+        )
+    )
+    def test_int_base(self, env, value, base, expect):
+        t = env.from_string("{{ '%s'|int(base=%d) }}" % (value, base))
+        assert t.render() == expect
+
+    def test_int_default(self, env):
+        t = env.from_string("{{ value|int(default=1) }}")
+        assert t.render(value="abc") == "1"
+
+    def test_int_special_method(self, env):
         class IntIsh(object):
             def __int__(self):
                 return 42
 
-        tmpl = env.from_string('{{ "42"|int }}|{{ "ajsghasjgd"|int }}|'
-                               '{{ "32.32"|int }}|{{ "0x4d32"|int(0, 16) }}|'
-                               '{{ "1234567890"|int }}|'
-                               '{{ "011"|int(0, 8)}}|{{ "0x33FU"|int(0, 16) }}|'
-                               '{{ obj|int }}')
-        out = tmpl.render(obj=IntIsh())
-        assert out == '42|0|32|19762|1234567890|9|0|42'
-        out = Template('{{ 5_555 }}|{{ 123_456_789 }}|{{ 12_34_56_78 }}').render()
-        assert out == '5555|123456789|12345678'
+        t = env.from_string("{{ value|int }}")
+        assert t.render(value=IntIsh()) == "42"
 
     def test_join(self, env):
         tmpl = env.from_string('{{ [1, 2, 3]|join("|") }}')

tests/test_lexnparse.py

@@ -341,29 +341,24 @@ class TestSyntax(object):
         t = env.from_string("{{ %s }}" % value)
         assert t.render() == value
 
-    @pytest.mark.parametrize("value", ("1", "123"))
-    def test_int_literal(self, env, value):
-        t = env.from_string("{{ %s }}" % value)
-        assert t.render() == value
-
     @pytest.mark.parametrize(
-        "value",
+        ("value", "expect"),
         (
-            "1.2",
-            "34.56",
+            ("1", "1"),
+            ("123", "123"),
+            ("12_34_56", "123456"),
+            ("1.2", "1.2"),
+            ("34.56", "34.56"),
+            ("3_4.5_6", "34.56"),
             ("1e0", "1.0"),
             ("10e1", "100.0"),
             ("2.5e100", "2.5e+100"),
-            "2.5e+100",
+            ("2.5e+100", "2.5e+100"),
             ("25.6e-10", "2.56e-09"),
-        ),
+            ("1_2.3_4e5_6", "1.234e+57"),
+        )
     )
-    def test_float_literal(self, env, value):
-        if isinstance(value, tuple):
-            value, expect = value
-        else:
-            expect = value
-
+    def test_numeric_literal(self, env, value, expect):
         t = env.from_string("{{ %s }}" % value)
         assert t.render() == expect