Merge pull request #1366 from mvolfik/fix-unicode-newlines

Fix unicode newlines

CHANGES.rst

@@ -56,6 +56,8 @@ Unreleased
     instead of a ``TypeError``. :issue:`1198`
 -   ``Undefined`` is iterable in an async environment. :issue:`1294`
 -   ``NativeEnvironment`` supports async mode. :issue:`1362`
+-   Template rendering only treats ``\n``, ``\r\n`` and ``\r`` as line
+    breaks. Other characters are left unchanged. :issue:`769, 952, 1313`
 
 
 Version 2.11.3

src/jinja2/lexer.py

@@ -638,12 +638,17 @@ class Lexer:
 
     def tokeniter(self, source, name, filename=None, state=None):
         """This method tokenizes the text and returns the tokens in a
-        generator.  Use this method if you just want to tokenize a template.
+        generator. Use this method if you just want to tokenize a template.
+
+        .. versionchanged:: 3.0
+            Only ``\\n``, ``\\r\\n`` and ``\\r`` are treated as line
+            breaks.
         """
-        lines = source.splitlines()
-        if self.keep_trailing_newline and source:
-            if source.endswith(("\r\n", "\r", "\n")):
-                lines.append("")
+        lines = newline_re.split(source)[::2]
+
+        if not self.keep_trailing_newline and lines[-1] == "":
+            del lines[-1]
+
         source = "\n".join(lines)
         pos = 0
         lineno = 1

tests/test_regression.py

@@ -745,3 +745,10 @@ End"""
 
         tmpl = env.get_template("base")
         assert tmpl.render() == "42 y"
+
+
+@pytest.mark.parametrize("unicode_char", ["\N{FORM FEED}", "\x85"])
+def test_unicode_whitespace(env, unicode_char):
+    content = "Lorem ipsum\n" + unicode_char + "\nMore text"
+    tmpl = env.from_string(content)
+    assert tmpl.render() == content