Add a policy for the ascii literal behavior. Fixes #392

CHANGES

@@ -28,6 +28,8 @@ Version 2.9
 - Ported a modified version of the `tojson` filter from Flask to Jinja2
   and hooked it up with the new policy framework.
 - Block sets are now marked `safe` by default.
+- On Python 2 the asciification of ASCII strings can now be disabled with
+  the `compiler.ascii_str` policy.
 
 Version 2.8.2
 -------------

docs/api.rst

@@ -556,6 +556,16 @@ Example::
 
     env.policies['urlize.rel'] = 'nofollow noopener'
 
+``compiler.ascii_str``:
+    This boolean controls on Python 2 if Jinja2 should store ASCII only
+    literals as bytestring instead of unicode strings.  This used to be
+    always enabled for Jinja versions below 2.9 and now can be changed.
+    Traditionally it was done this way since some APIs in Python 2 failed
+    badly for unicode strings (for instance the datetime strftime API).
+    Now however sometimes the inverse is true (for instance str.format).
+    If this is set to False then all strings are stored as unicode
+    internally.
+
 ``urlize.rel``:
     A string that defines the items for the `rel` attribute of generated
     links with the `urlize` filter.  These items are always added.  The

jinja2/compiler.py

@@ -1340,7 +1340,7 @@ class CodeGenerator(NodeVisitor):
         self.write(ref)
 
     def visit_Const(self, node, frame):
-        val = node.value
+        val = node.as_const(frame.eval_ctx)
         if isinstance(val, float):
             self.write(str(val))
         else:

jinja2/defaults.py

@@ -41,6 +41,7 @@ DEFAULT_NAMESPACE = {
 
 # default policies
 DEFAULT_POLICIES = {
+    'compiler.ascii_str':   True,
     'urlize.rel':           'noopener',
     'urlize.target':        None,
     'json.dumps_function':  None,

jinja2/lexer.py

@@ -574,15 +574,6 @@ class Lexer(object):
                 except Exception as e:
                     msg = str(e).split(':')[-1].strip()
                     raise TemplateSyntaxError(msg, lineno, name, filename)
-                # if we can express it as bytestring (ascii only)
-                # we do that for support of semi broken APIs
-                # as datetime.datetime.strftime.  On python 3 this
-                # call becomes a noop thanks to 2to3
-                if PY2:
-                    try:
-                        value = value.encode('ascii')
-                    except UnicodeError:
-                        pass
             elif token == 'integer':
                 value = int(value)
             elif token == 'float':

jinja2/nodes.py

@@ -17,7 +17,7 @@ import operator
 
 from collections import deque
 from jinja2.utils import Markup
-from jinja2._compat import izip, with_metaclass, text_type
+from jinja2._compat import izip, with_metaclass, text_type, PY2
 
 
 #: the types we support for context functions
@@ -470,7 +470,14 @@ class Const(Literal):
     fields = ('value',)
 
     def as_const(self, eval_ctx=None):
-        return self.value
+        rv = self.value
+        if PY2 and type(rv) is text_type and \
+           self.environment.policies['compiler.ascii_str']:
+            try:
+                rv = rv.encode('ascii')
+            except UnicodeError:
+                pass
+        return rv
 
     @classmethod
     def from_untrusted(cls, value, lineno=None, environment=None):

tests/test_features.py

@@ -1,7 +1,7 @@
 import sys
 import pytest
 
-from jinja2 import Template
+from jinja2 import Template, Environment, contextfilter
 
 
 @pytest.mark.skipif(sys.version_info < (3, 5),
@@ -14,3 +14,27 @@ def test_generator_stop():
     t = Template('a{{ bad.bar() }}b')
     with pytest.raises(RuntimeError):
         t.render(bad=X())
+
+
+@pytest.mark.skipif(sys.version_info[0] > 2,
+                    reason='Feature only supported on 2.x')
+def test_ascii_str():
+    @contextfilter
+    def assert_func(context, value):
+        assert type(value) is context['expected_type']
+
+    env = Environment()
+    env.filters['assert'] = assert_func
+
+    env.policies['compiler.ascii_str'] = False
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=unicode)
+
+    env.policies['compiler.ascii_str'] = True
+    t = env.from_string('{{ "foo"|assert }}')
+    t.render(expected_type=str)
+
+    for val in True, False:
+        env.policies['compiler.ascii_str'] = val
+        t = env.from_string(u'{{ "\N{SNOWMAN}"|assert }}')
+        t.render(expected_type=unicode)