From 028f058370265f86ce36fc1313520c81e63fb9f0 Mon Sep 17 00:00:00 2001
From: Armin Ronacher <armin.ronacher@active-4.com>
Date: Sat, 7 Jan 2017 14:57:44 +0100
Subject: [PATCH] Add a policy for the ascii literal behavior.  Fixes #392

---
 CHANGES                |  2 ++
 docs/api.rst           | 10 ++++++++++
 jinja2/compiler.py     |  2 +-
 jinja2/defaults.py     |  1 +
 jinja2/lexer.py        |  9 ---------
 jinja2/nodes.py        | 11 +++++++++--
 tests/test_features.py | 26 +++++++++++++++++++++++++-
 7 files changed, 48 insertions(+), 13 deletions(-)

diff --git a/CHANGES b/CHANGES
index 65bf969..da6774f 100644
--- a/CHANGES
+++ b/CHANGES
@@ -28,6 +28,8 @@ Version 2.9
 - Ported a modified version of the `tojson` filter from Flask to Jinja2
   and hooked it up with the new policy framework.
 - Block sets are now marked `safe` by default.
+- On Python 2 the asciification of ASCII strings can now be disabled with
+  the `compiler.ascii_str` policy.
 
 Version 2.8.2
 -------------
diff --git a/docs/api.rst b/docs/api.rst
index 8bf0fdf..3b5bccd 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -556,6 +556,16 @@ Example::
 
     env.policies['urlize.rel'] = 'nofollow noopener'
 
+``compiler.ascii_str``:
+    This boolean controls on Python 2 if Jinja2 should store ASCII only
+    literals as bytestring instead of unicode strings.  This used to be
+    always enabled for Jinja versions below 2.9 and now can be changed.
+    Traditionally it was done this way since some APIs in Python 2 failed
+    badly for unicode strings (for instance the datetime strftime API).
+    Now however sometimes the inverse is true (for instance str.format).
+    If this is set to False then all strings are stored as unicode
+    internally.
+
 ``urlize.rel``:
     A string that defines the items for the `rel` attribute of generated
     links with the `urlize` filter.  These items are always added.  The
diff --git a/jinja2/compiler.py b/jinja2/compiler.py
index 65bffe6..6595d63 100644
--- a/jinja2/compiler.py
+++ b/jinja2/compiler.py
@@ -1340,7 +1340,7 @@ class CodeGenerator(NodeVisitor):
         self.write(ref)
 
     def visit_Const(self, node, frame):
-        val = node.value
+        val = node.as_const(frame.eval_ctx)
         if isinstance(val, float):
             self.write(str(val))
         else:
diff --git a/jinja2/defaults.py b/jinja2/defaults.py
index 90ccb65..38c5376 100644
--- a/jinja2/defaults.py
+++ b/jinja2/defaults.py
@@ -41,6 +41,7 @@ DEFAULT_NAMESPACE = {
 
 # default policies
 DEFAULT_POLICIES = {
+    'compiler.ascii_str':   True,
     'urlize.rel':           'noopener',
     'urlize.target':        None,
     'json.dumps_function':  None,
diff --git a/jinja2/lexer.py b/jinja2/lexer.py
index c8dac21..d2ca32f 100644
--- a/jinja2/lexer.py
+++ b/jinja2/lexer.py
@@ -574,15 +574,6 @@ class Lexer(object):
                 except Exception as e:
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
-                # if we can express it as bytestring (ascii only)
-                # we do that for support of semi broken APIs
-                # as datetime.datetime.strftime.  On python 3 this
-                # call becomes a noop thanks to 2to3
-                if PY2:
-                    try:
-                        value = value.encode('ascii')
-                    except UnicodeError:
-                        pass
             elif token == 'integer':
                 value = int(value)
             elif token == 'float':
diff --git a/jinja2/nodes.py b/jinja2/nodes.py
index 4d62ccc..5e0726a 100644
--- a/jinja2/nodes.py
+++ b/jinja2/nodes.py
@@ -17,7 +17,7 @@ import operator
 
 from collections import deque
 from jinja2.utils import Markup
-from jinja2._compat import izip, with_metaclass, text_type
+from jinja2._compat import izip, with_metaclass, text_type, PY2
 
 
 #: the types we support for context functions
@@ -470,7 +470,14 @@ class Const(Literal):
     fields = ('value',)
 
     def as_const(self, eval_ctx=None):
-        return self.value
+        rv = self.value
+        if PY2 and type(rv) is text_type and \
+           self.environment.policies['compiler.ascii_str']:
+            try:
+                rv = rv.encode('ascii')
+            except UnicodeError:
+                pass
+        return rv
 
     @classmethod
     def from_untrusted(cls, value, lineno=None, environment=None):
diff --git a/tests/test_features.py b/tests/test_features.py
index 25d58e4..3187890 100644
--- a/tests/test_features.py
+++ b/tests/test_features.py
@@ -1,7 +1,7 @@
 import sys
 import pytest
 
-from jinja2 import Template
+from jinja2 import Template, Environment, contextfilter
 
 
 @pytest.mark.skipif(sys.version_info < (3, 5),
@@ -14,3 +14,27 @@ def test_generator_stop():
     t = Template('a{{ bad.bar() }}b')
     with pytest.raises(RuntimeError):
         t.render(bad=X())
+
+
+@pytest.mark.skipif(sys.version_info[0] > 2,
+                    reason='Feature only supported on 2.x')
+def test_ascii_str():
+    @contextfilter
+    def assert_func(context, value):
+        assert type(value) is context['expected_type']
+
+    env = Environment()
+    env.filters['assert'] = assert_func
+
+    env.policies['compiler.ascii_str'] = False
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=unicode)
+
+    env.policies['compiler.ascii_str'] = True
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=str)
+
+    for val in True, False:
+        env.policies['compiler.ascii_str'] = val
+        t = env.from_string(u'{{ "\N{SNOWMAN}"|assert }}')
+        t.render(expected_type=unicode)