Make the runtime leak less of the version hacks

jinja2/_compat.py

@@ -21,55 +21,58 @@ if not PY2:
     text_type = str
     string_types = (str,)
 
-    _iterkeys = 'keys'
-    _itervalues = 'values'
-    _iteritems = 'items'
+    iterkeys = lambda d: iter(d.keys())
+    itervalues = lambda d: iter(d.values())
+    iteritems = lambda d: iter(d.items())
 
+    import pickle
     from io import BytesIO, StringIO
     NativeStringIO = StringIO
 
-    ifilter = filter
-    imap = map
-    izip = zip
-
     def reraise(tp, value, tb=None):
         if value.__traceback__ is not tb:
             raise value.with_traceback(tb)
         raise value
 
-    Iterator = object
+    ifilter = filter
+    imap = map
+    izip = zip
+    intern = sys.intern
 
-    class UnicodeMixin(object):
-        __slots__ = ()
-        def __str__(self):
-            return self.__unicode__()
+    implements_iterator = lambda x: x
+    implements_to_string = lambda x: x
+    get_next = lambda x: x.__next__
 else:
-    text_type = unicode
     unichr = unichr
+    text_type = unicode
+    range_type = xrange
     string_types = (str, unicode)
 
-    _iterkeys = 'iterkeys'
-    _itervalues = 'itervalues'
-    _iteritems = 'iteritems'
+    iterkeys = lambda d: d.iterkeys()
+    itervalues = lambda d: d.itervalues()
+    iteritems = lambda d: d.iteritems()
 
-    from itertools import imap, izip, ifilter
-    range_type = xrange
-
-    from cStringIO import StringIO as BytesIO
-    from StringIO import StringIO
+    import cPickle as pickle
+    from cStringIO import StringIO as BytesIO, StringIO
     NativeStringIO = BytesIO
 
     exec('def reraise(tp, value, tb=None):\n raise tp, value, tb')
 
-    class UnicodeMixin(object):
-        __slots__ = ()
-        def __str__(self):
-            return self.__unicode__().encode('utf-8')
+    from itertools import imap, izip, ifilter
+    intern = intern
+
+    def implements_iterator(cls):
+        cls.next = cls.__next__
+        del cls.__next__
+        return cls
+
+    def implements_to_string(cls):
+        cls.__unicode__ = cls.__str__
+        cls.__str__ = lambda x: x.__unicode__().encode('utf-8')
+        return cls
+
+    get_next = lambda x: x.next
 
-    class Iterator(object):
-        __slots__ = ()
-        def next(self):
-            return self.__next__()
 
 try:
     next = next
@@ -79,22 +82,15 @@ except NameError:
 
 
 def with_metaclass(meta, *bases):
-    """Create a base class with a metaclass."""
-    return meta('NewBase', bases, {})
+    class __metaclass__(meta):
+        __call__ = type.__call__
+        __init__ = type.__init__
+        def __new__(cls, name, this_bases, d):
+            if this_bases is None:
+                return type.__new__(cls, name, (), d)
+            return meta(name, bases, d)
+    return __metaclass__('<dummy_class>', None, {})
 
-def iterkeys(d, **kw):
-    return iter(getattr(d, _iterkeys)(**kw))
-
-def itervalues(d, **kw):
-    return iter(getattr(d, _itervalues)(**kw))
-
-def iteritems(d, **kw):
-    return iter(getattr(d, _iteritems)(**kw))
-
-try:
-    import cPickle as pickle
-except ImportError:
-    import pickle
 
 try:
     from collections import Mapping as mapping_types

jinja2/environment.py

@@ -28,7 +28,8 @@ from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \
 from jinja2.utils import import_string, LRUCache, Markup, missing, \
      concat, consume, internalcode, _encode_filename
 from jinja2._compat import imap, ifilter, string_types, iteritems, \
-     text_type, reraise, Iterator, next, UnicodeMixin
+     text_type, reraise, implements_iterator, implements_to_string, \
+     get_next
 from functools import reduce
 
 
@@ -1051,7 +1052,8 @@ class Template(object):
         return '<%s %s>' % (self.__class__.__name__, name)
 
 
-class TemplateModule(UnicodeMixin):
+@implements_to_string
+class TemplateModule(object):
     """Represents an imported template.  All the exported names of the
     template are available as attributes on this object.  Additionally
     converting it into an unicode- or bytestrings renders the contents.
@@ -1065,7 +1067,7 @@ class TemplateModule(UnicodeMixin):
     def __html__(self):
         return Markup(concat(self._body_stream))
 
-    def __unicode__(self):
+    def __str__(self):
         return concat(self._body_stream)
 
     def __repr__(self):
@@ -1095,7 +1097,8 @@ class TemplateExpression(object):
         return rv
 
 
-class TemplateStream(Iterator):
+@implements_iterator
+class TemplateStream(object):
     """A template stream works pretty much like an ordinary python generator
     but it can buffer multiple items to reduce the number of total iterations.
     Per default the output is unbuffered which means that for every unbuffered
@@ -1139,7 +1142,7 @@ class TemplateStream(Iterator):
 
     def disable_buffering(self):
         """Disable the output buffering."""
-        self._next = lambda: next(self._gen)
+        self._next = get_next(self._gen)
         self.buffered = False
 
     def enable_buffering(self, size=5):
@@ -1167,7 +1170,7 @@ class TemplateStream(Iterator):
                 c_size = 0
 
         self.buffered = True
-        self._next = lambda: next(generator(lambda: next(self._gen)))
+        self._next = get_next(generator(get_next(self._gen)))
 
     def __iter__(self):
         return self

jinja2/exceptions.py

@@ -8,7 +8,7 @@
     :copyright: (c) 2010 by the Jinja Team.
     :license: BSD, see LICENSE for more details.
 """
-from jinja2._compat import imap, text_type, PY2, UnicodeMixin
+from jinja2._compat import imap, text_type, PY2, implements_to_string
 
 
 class TemplateError(Exception):
@@ -36,7 +36,8 @@ class TemplateError(Exception):
                     return message
 
 
-class TemplateNotFound(IOError, LookupError, TemplateError, UnicodeMixin):
+@implements_to_string
+class TemplateNotFound(IOError, LookupError, TemplateError):
     """Raised if a template does not exist."""
 
     # looks weird, but removes the warning descriptor that just
@@ -51,7 +52,7 @@ class TemplateNotFound(IOError, LookupError, TemplateError, UnicodeMixin):
         self.name = name
         self.templates = [name]
 
-    def __unicode__(self):
+    def __str__(self):
         return self.message
 
 
@@ -71,7 +72,8 @@ class TemplatesNotFound(TemplateNotFound):
         self.templates = list(names)
 
 
-class TemplateSyntaxError(UnicodeMixin, TemplateError):
+@implements_to_string
+class TemplateSyntaxError(TemplateError):
     """Raised to tell the user that there is a problem with the template."""
 
     def __init__(self, message, lineno, name=None, filename=None):
@@ -85,7 +87,7 @@ class TemplateSyntaxError(UnicodeMixin, TemplateError):
         # function translated the syntax error into a new traceback
         self.translated = False
 
-    def __unicode__(self):
+    def __str__(self):
         # for translated errors we only return the message
         if self.translated:
             return self.message

jinja2/lexer.py

@@ -20,7 +20,8 @@ from operator import itemgetter
 from collections import deque
 from jinja2.exceptions import TemplateSyntaxError
 from jinja2.utils import LRUCache
-from jinja2._compat import next, iteritems, Iterator, text_type
+from jinja2._compat import next, iteritems, implements_iterator, text_type, \
+     intern
 
 
 # cache for the lexers. Exists in order to be able to have multiple
@@ -47,12 +48,6 @@ else:
 float_re = re.compile(r'(?<!\.)\d+\.\d+')
 newline_re = re.compile(r'(\r\n|\r|\n)')
 
-try:
-    intern = intern  # py2
-except NameError:
-    import sys
-    intern = sys.intern  # py3
-
 # internal the tokens and keep references to them
 TOKEN_ADD = intern('add')
 TOKEN_ASSIGN = intern('assign')
@@ -270,7 +265,8 @@ class Token(tuple):
         )
 
 
-class TokenStreamIterator(Iterator):
+@implements_iterator
+class TokenStreamIterator(object):
     """The iterator for tokenstreams.  Iterate over the stream
     until the eof token is reached.
     """
@@ -290,7 +286,8 @@ class TokenStreamIterator(Iterator):
         return token
 
 
-class TokenStream(Iterator):
+@implements_iterator
+class TokenStream(object):
     """A token stream is an iterable that yields :class:`Token`\s.  The
     parser however does not iterate over it but calls :meth:`next` to go
     one token ahead.  The current active token is stored as :attr:`current`.

jinja2/runtime.py

@@ -14,8 +14,8 @@ from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \
      internalcode, object_type_repr
 from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \
      TemplateNotFound
-from jinja2._compat import next, imap, text_type, iteritems, Iterator, \
-     string_types, PY2, UnicodeMixin
+from jinja2._compat import next, imap, text_type, iteritems, \
+     implements_iterator, implements_to_string, string_types
 
 
 # these variables are exported to the template runtime
@@ -25,12 +25,8 @@ __all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup',
            'TemplateNotFound']
 
 #: the name of the function that is used to convert something into
-#: a string.  2to3 will adopt that automatically and the generated
-#: code can take advantage of it.
-try:
-    to_string = unicode
-except NameError:
-    to_string = str
+#: a string.  We can just use the text type here.
+to_string = text_type
 
 #: the identity function.  Useful for certain things in the environment
 identity = lambda x: x
@@ -356,7 +352,8 @@ class LoopContext(object):
         )
 
 
-class LoopContextIterator(Iterator):
+@implements_iterator
+class LoopContextIterator(object):
     """The iterator for a loop context."""
     __slots__ = ('context',)
 
@@ -440,7 +437,8 @@ class Macro(object):
         )
 
 
-class Undefined(UnicodeMixin):
+@implements_to_string
+class Undefined(object):
     """The default undefined type.  This undefined type can be printed and
     iterated over, but every other access will raise an :exc:`UndefinedError`:
 
@@ -498,7 +496,7 @@ class Undefined(UnicodeMixin):
     __float__ = __complex__ = __pow__ = __rpow__ = \
         _fail_with_undefined_error
 
-    def __unicode__(self):
+    def __str__(self):
         return u''
 
     def __len__(self):
@@ -515,6 +513,7 @@ class Undefined(UnicodeMixin):
         return 'Undefined'
 
 
+@implements_to_string
 class DebugUndefined(Undefined):
     """An undefined that returns the debug info when printed.
 
@@ -530,7 +529,7 @@ class DebugUndefined(Undefined):
     """
     __slots__ = ()
 
-    def __unicode__(self):
+    def __str__(self):
         if self._undefined_hint is None:
             if self._undefined_obj is missing:
                 return u'{{ %s }}' % self._undefined_name
@@ -541,6 +540,7 @@ class DebugUndefined(Undefined):
         return u'{{ undefined value printed: %s }}' % self._undefined_hint
 
 
+@implements_to_string
 class StrictUndefined(Undefined):
     """An undefined that barks on print and iteration as well as boolean
     tests and all kinds of comparisons.  In other words: you can do nothing
@@ -561,7 +561,7 @@ class StrictUndefined(Undefined):
     UndefinedError: 'foo' is undefined
     """
     __slots__ = ()
-    __iter__ = __unicode__ = __len__ = __nonzero__ = __eq__ = \
+    __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \
         __ne__ = __bool__ = Undefined._fail_with_undefined_error
 
 

jinja2/sandbox.py

@@ -16,7 +16,7 @@ import operator
 from jinja2.environment import Environment
 from jinja2.exceptions import SecurityError
 from jinja2._compat import string_types, function_type, method_type, \
-     traceback_type, code_type, frame_type, generator_type
+     traceback_type, code_type, frame_type, generator_type, PY2
 
 
 #: maximum number of items a range may produce
@@ -29,6 +29,13 @@ UNSAFE_FUNCTION_ATTRIBUTES = set(['func_closure', 'func_code', 'func_dict',
 #: unsafe method attributes.  function attributes are unsafe for methods too
 UNSAFE_METHOD_ATTRIBUTES = set(['im_class', 'im_func', 'im_self'])
 
+#: unsafe generator attirbutes.
+UNSAFE_GENERATOR_ATTRIBUTES = set(['gi_frame', 'gi_code'])
+
+# On versions > python 2 the special attributes on functions are gone,
+# but they remain on methods and generators for whatever reason.
+if not PY2:
+    UNSAFE_FUNCTION_ATTRIBUTES = set()
 
 import warnings
 
@@ -137,7 +144,7 @@ def is_internal_attribute(obj, attr):
     elif isinstance(obj, (code_type, traceback_type, frame_type)):
         return True
     elif isinstance(obj, generator_type):
-        if attr == 'gi_frame':
+        if attr in UNSAFE_GENERATOR_ATTRIBUTES:
             return True
     return attr.startswith('__')
 

jinja2/tests.py

@@ -17,11 +17,7 @@ number_re = re.compile(r'^-?\d+(\.\d+)?$')
 regex_type = type(number_re)
 
 
-try:
-    test_callable = callable
-except NameError:
-    def test_callable(x):
-        return hasattr(x, '__call__')
+test_callable = callable
 
 
 def test_odd(value):

jinja2/testsuite/filters.py

@@ -12,7 +12,7 @@ import unittest
 from jinja2.testsuite import JinjaTestCase
 
 from jinja2 import Markup, Environment
-from jinja2._compat import text_type, UnicodeMixin
+from jinja2._compat import text_type, implements_to_string
 
 env = Environment()
 
@@ -294,10 +294,11 @@ class FilterTestCase(JinjaTestCase):
         assert tmpl.render() == "['Bar', 'blah', 'foo']"
 
     def test_sort4(self):
-        class Magic(UnicodeMixin):
+        @implements_to_string
+        class Magic(object):
             def __init__(self, value):
                 self.value = value
-            def __unicode__(self):
+            def __str__(self):
                 return text_type(self.value)
         tmpl = env.from_string('''{{ items|sort(attribute='value')|join }}''')
         assert tmpl.render(items=map(Magic, [3, 2, 4, 1])) == '1234'

jinja2/utils.py

@@ -23,7 +23,7 @@ except ImportError:
     except ImportError:
         from dummy_thread import allocate_lock
 from collections import deque
-from jinja2._compat import text_type, string_types, Iterator, PY2
+from jinja2._compat import text_type, string_types, implements_iterator, PY2
 
 
 _word_split_re = re.compile(r'(\s+)')
@@ -506,7 +506,8 @@ except ImportError:
     pass
 
 
-class Cycler(Iterator):
+@implements_iterator
+class Cycler(object):
     """A cycle helper for templates."""
 
     def __init__(self, *items):