!10 jinja版本更替

Merge pull request !10 from oumeng/mycheck

README.OpenSource

@@ -3,9 +3,9 @@
         "Name": "Jinja2",
         "License": "BSD 3-clause License",
         "License File": "LICENSE.rst",
-        "Version Number": "2.11.1",
-        "Owner": "yaoxiaoyu1@huawei.com",
-        "Upstream URL": "https://pypi.org/project/Jinja2/",
+        "Version Number": "2.11.3",
+        "Owner": "anguanglin@huawei.com",
+        "Upstream URL": "https://github.com/pallets/jinja",
         "Description": "Jinja2 is a template engine written in pure Python. It provides a Django inspired non-XML syntax but supports inline expressions and an optional sandboxed environment."
     }
 ]

__init__.py

@@ -41,4 +41,4 @@ from .utils import evalcontextfunction
 from .utils import is_undefined
 from .utils import select_autoescape
 
-__version__ = "2.11.1"
+__version__ = "2.11.3"

asyncfilters.py

@@ -26,17 +26,16 @@ async def async_select_or_reject(args, kwargs, modfunc, lookup_attr):
 
 def dualfilter(normal_filter, async_filter):
     wrap_evalctx = False
-    if getattr(normal_filter, "environmentfilter", False):
+    if getattr(normal_filter, "environmentfilter", False) is True:
 
         def is_async(args):
             return args[0].is_async
 
         wrap_evalctx = False
     else:
-        if not getattr(normal_filter, "evalcontextfilter", False) and not getattr(
-            normal_filter, "contextfilter", False
-        ):
-            wrap_evalctx = True
+        has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True
+        has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True
+        wrap_evalctx = not has_evalctxfilter and not has_ctxfilter
 
         def is_async(args):
             return args[0].environment.is_async

compiler.py

@@ -1307,13 +1307,13 @@ class CodeGenerator(NodeVisitor):
             def finalize(value):
                 return default(env_finalize(value))
 
-            if getattr(env_finalize, "contextfunction", False):
+            if getattr(env_finalize, "contextfunction", False) is True:
                 src += "context, "
                 finalize = None  # noqa: F811
-            elif getattr(env_finalize, "evalcontextfunction", False):
+            elif getattr(env_finalize, "evalcontextfunction", False) is True:
                 src += "context.eval_ctx, "
                 finalize = None
-            elif getattr(env_finalize, "environmentfunction", False):
+            elif getattr(env_finalize, "environmentfunction", False) is True:
                 src += "environment, "
 
                 def finalize(value):
@@ -1689,11 +1689,11 @@ class CodeGenerator(NodeVisitor):
         func = self.environment.filters.get(node.name)
         if func is None:
             self.fail("no filter named %r" % node.name, node.lineno)
-        if getattr(func, "contextfilter", False):
+        if getattr(func, "contextfilter", False) is True:
             self.write("context, ")
-        elif getattr(func, "evalcontextfilter", False):
+        elif getattr(func, "evalcontextfilter", False) is True:
             self.write("context.eval_ctx, ")
-        elif getattr(func, "environmentfilter", False):
+        elif getattr(func, "environmentfilter", False) is True:
             self.write("environment, ")
 
         # if the filter node is None we are inside a filter block

debug.py

@@ -245,10 +245,7 @@ else:
     class _CTraceback(ctypes.Structure):
         _fields_ = [
             # Extra PyObject slots when compiled with Py_TRACE_REFS.
-            (
-                "PyObject_HEAD",
-                ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16),
-            ),
+            ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()),
             # Only care about tb_next as an object, not a traceback.
             ("tb_next", ctypes.py_object),
         ]

environment.py

@@ -492,20 +492,20 @@ class Environment(object):
         if func is None:
             fail_for_missing_callable("no filter named %r", name)
         args = [value] + list(args or ())
-        if getattr(func, "contextfilter", False):
+        if getattr(func, "contextfilter", False) is True:
             if context is None:
                 raise TemplateRuntimeError(
                     "Attempted to invoke context filter without context"
                 )
             args.insert(0, context)
-        elif getattr(func, "evalcontextfilter", False):
+        elif getattr(func, "evalcontextfilter", False) is True:
             if eval_ctx is None:
                 if context is not None:
                     eval_ctx = context.eval_ctx
                 else:
                     eval_ctx = EvalContext(self)
             args.insert(0, eval_ctx)
-        elif getattr(func, "environmentfilter", False):
+        elif getattr(func, "environmentfilter", False) is True:
             args.insert(0, self)
         return func(*args, **(kwargs or {}))
 

filters.py

@@ -268,16 +268,16 @@ def do_dictsort(value, case_sensitive=False, by="key", reverse=False):
 
     .. sourcecode:: jinja
 
-        {% for item in mydict|dictsort %}
+        {% for key, value in mydict|dictsort %}
             sort the dict by key, case insensitive
 
-        {% for item in mydict|dictsort(reverse=true) %}
+        {% for key, value in mydict|dictsort(reverse=true) %}
             sort the dict by key, case insensitive, reverse order
 
-        {% for item in mydict|dictsort(true) %}
+        {% for key, value in mydict|dictsort(true) %}
             sort the dict by key, case sensitive
 
-        {% for item in mydict|dictsort(false, 'value') %}
+        {% for key, value in mydict|dictsort(false, 'value') %}
             sort the dict by value, case insensitive
     """
     if by == "key":
@@ -761,7 +761,7 @@ def do_wordwrap(
 
 def do_wordcount(s):
     """Count the words in that string."""
-    return len(_word_re.findall(s))
+    return len(_word_re.findall(soft_unicode(s)))
 
 
 def do_int(value, default=0, base=10):

lexer.py

@@ -681,6 +681,8 @@ class Lexer(object):
         source_length = len(source)
         balancing_stack = []
         lstrip_unless_re = self.lstrip_unless_re
+        newlines_stripped = 0
+        line_starting = True
 
         while 1:
             # tokenizer loop
@@ -717,7 +719,9 @@ class Lexer(object):
 
                         if strip_sign == "-":
                             # Strip all whitespace between the text and the tag.
-                            groups = (text.rstrip(),) + groups[1:]
+                            stripped = text.rstrip()
+                            newlines_stripped = text[len(stripped) :].count("\n")
+                            groups = (stripped,) + groups[1:]
                         elif (
                             # Not marked for preserving whitespace.
                             strip_sign != "+"
@@ -728,11 +732,11 @@ class Lexer(object):
                         ):
                             # The start of text between the last newline and the tag.
                             l_pos = text.rfind("\n") + 1
-
-                            # If there's only whitespace between the newline and the
-                            # tag, strip it.
-                            if not lstrip_unless_re.search(text, l_pos):
-                                groups = (text[:l_pos],) + groups[1:]
+                            if l_pos > 0 or line_starting:
+                                # If there's only whitespace between the newline and the
+                                # tag, strip it.
+                                if not lstrip_unless_re.search(text, l_pos):
+                                    groups = (text[:l_pos],) + groups[1:]
 
                     for idx, token in enumerate(tokens):
                         # failure group
@@ -758,7 +762,8 @@ class Lexer(object):
                             data = groups[idx]
                             if data or token not in ignore_if_empty:
                                 yield lineno, token, data
-                            lineno += data.count("\n")
+                            lineno += data.count("\n") + newlines_stripped
+                            newlines_stripped = 0
 
                 # strings as token just are yielded as it.
                 else:
@@ -790,6 +795,8 @@ class Lexer(object):
                         yield lineno, tokens, data
                     lineno += data.count("\n")
 
+                line_starting = m.group()[-1:] == "\n"
+
                 # fetch new position into new variable so that we can check
                 # if there is a internal parsing error which would result
                 # in an infinite loop

loaders.py

@@ -3,11 +3,9 @@
 sources.
 """
 import os
-import pkgutil
 import sys
 import weakref
 from hashlib import sha1
-from importlib import import_module
 from os import path
 from types import ModuleType
 
@@ -217,141 +215,75 @@ class FileSystemLoader(BaseLoader):
 
 
 class PackageLoader(BaseLoader):
-    """Load templates from a directory in a Python package.
+    """Load templates from python eggs or packages.  It is constructed with
+    the name of the python package and the path to the templates in that
+    package::
 
-    :param package_name: Import name of the package that contains the
-        template directory.
-    :param package_path: Directory within the imported package that
-        contains the templates.
-    :param encoding: Encoding of template files.
+        loader = PackageLoader('mypackage', 'views')
 
-    The following example looks up templates in the ``pages`` directory
-    within the ``project.ui`` package.
+    If the package path is not given, ``'templates'`` is assumed.
 
-    .. code-block:: python
-
-        loader = PackageLoader("project.ui", "pages")
-
-    Only packages installed as directories (standard pip behavior) or
-    zip/egg files (less common) are supported. The Python API for
-    introspecting data in packages is too limited to support other
-    installation methods the way this loader requires.
-
-    There is limited support for :pep:`420` namespace packages. The
-    template directory is assumed to only be in one namespace
-    contributor. Zip files contributing to a namespace are not
-    supported.
-
-    .. versionchanged:: 2.11.0
-        No longer uses ``setuptools`` as a dependency.
-
-    .. versionchanged:: 2.11.0
-        Limited PEP 420 namespace package support.
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.  Due to the nature
+    of eggs it's only possible to reload templates if the package was loaded
+    from the file system and not a zip file.
     """
 
     def __init__(self, package_name, package_path="templates", encoding="utf-8"):
-        if package_path == os.path.curdir:
-            package_path = ""
-        elif package_path[:2] == os.path.curdir + os.path.sep:
-            package_path = package_path[2:]
+        from pkg_resources import DefaultProvider
+        from pkg_resources import get_provider
+        from pkg_resources import ResourceManager
 
-        package_path = os.path.normpath(package_path).rstrip(os.path.sep)
-        self.package_path = package_path
-        self.package_name = package_name
+        provider = get_provider(package_name)
         self.encoding = encoding
-
-        # Make sure the package exists. This also makes namespace
-        # packages work, otherwise get_loader returns None.
-        import_module(package_name)
-        self._loader = loader = pkgutil.get_loader(package_name)
-
-        # Zip loader's archive attribute points at the zip.
-        self._archive = getattr(loader, "archive", None)
-        self._template_root = None
-
-        if hasattr(loader, "get_filename"):
-            # A standard directory package, or a zip package.
-            self._template_root = os.path.join(
-                os.path.dirname(loader.get_filename(package_name)), package_path
-            )
-        elif hasattr(loader, "_path"):
-            # A namespace package, limited support. Find the first
-            # contributor with the template directory.
-            for root in loader._path:
-                root = os.path.join(root, package_path)
-
-                if os.path.isdir(root):
-                    self._template_root = root
-                    break
-
-        if self._template_root is None:
-            raise ValueError(
-                "The %r package was not installed in a way that"
-                " PackageLoader understands." % package_name
-            )
+        self.manager = ResourceManager()
+        self.filesystem_bound = isinstance(provider, DefaultProvider)
+        self.provider = provider
+        self.package_path = package_path
 
     def get_source(self, environment, template):
-        p = os.path.join(self._template_root, *split_template_path(template))
+        pieces = split_template_path(template)
+        p = "/".join((self.package_path,) + tuple(pieces))
 
-        if self._archive is None:
-            # Package is a directory.
-            if not os.path.isfile(p):
-                raise TemplateNotFound(template)
+        if not self.provider.has_resource(p):
+            raise TemplateNotFound(template)
 
-            with open(p, "rb") as f:
-                source = f.read()
+        filename = uptodate = None
 
-            mtime = os.path.getmtime(p)
+        if self.filesystem_bound:
+            filename = self.provider.get_resource_filename(self.manager, p)
+            mtime = path.getmtime(filename)
 
-            def up_to_date():
-                return os.path.isfile(p) and os.path.getmtime(p) == mtime
+            def uptodate():
+                try:
+                    return path.getmtime(filename) == mtime
+                except OSError:
+                    return False
 
-        else:
-            # Package is a zip file.
-            try:
-                source = self._loader.get_data(p)
-            except OSError:
-                raise TemplateNotFound(template)
-
-            # Could use the zip's mtime for all template mtimes, but
-            # would need to safely reload the module if it's out of
-            # date, so just report it as always current.
-            up_to_date = None
-
-        return source.decode(self.encoding), p, up_to_date
+        source = self.provider.get_resource_string(self.manager, p)
+        return source.decode(self.encoding), filename, uptodate
 
     def list_templates(self):
+        path = self.package_path
+
+        if path[:2] == "./":
+            path = path[2:]
+        elif path == ".":
+            path = ""
+
+        offset = len(path)
         results = []
 
-        if self._archive is None:
-            # Package is a directory.
-            offset = len(self._template_root)
+        def _walk(path):
+            for filename in self.provider.resource_listdir(path):
+                fullname = path + "/" + filename
 
-            for dirpath, _, filenames in os.walk(self._template_root):
-                dirpath = dirpath[offset:].lstrip(os.path.sep)
-                results.extend(
-                    os.path.join(dirpath, name).replace(os.path.sep, "/")
-                    for name in filenames
-                )
-        else:
-            if not hasattr(self._loader, "_files"):
-                raise TypeError(
-                    "This zip import does not have the required"
-                    " metadata to list templates."
-                )
-
-            # Package is a zip file.
-            prefix = (
-                self._template_root[len(self._archive) :].lstrip(os.path.sep)
-                + os.path.sep
-            )
-            offset = len(prefix)
-
-            for name in self._loader._files.keys():
-                # Find names under the templates directory that aren't directories.
-                if name.startswith(prefix) and name[-1] != os.path.sep:
-                    results.append(name[offset:].replace(os.path.sep, "/"))
+                if self.provider.resource_isdir(fullname):
+                    _walk(fullname)
+                else:
+                    results.append(fullname[offset:].lstrip("/"))
 
+        _walk(path)
         results.sort()
         return results
 

nativetypes.py

@@ -1,4 +1,3 @@
-import types
 from ast import literal_eval
 from itertools import chain
 from itertools import islice
@@ -11,7 +10,7 @@ from .environment import Environment
 from .environment import Template
 
 
-def native_concat(nodes, preserve_quotes=True):
+def native_concat(nodes):
     """Return a native Python type from the list of compiled nodes. If
     the result is a single node, its value is returned. Otherwise, the
     nodes are concatenated as strings. If the result can be parsed with
@@ -19,9 +18,6 @@ def native_concat(nodes, preserve_quotes=True):
     the string is returned.
 
     :param nodes: Iterable of nodes to concatenate.
-    :param preserve_quotes: Whether to re-wrap literal strings with
-        quotes, to preserve quotes around expressions for later parsing.
-        Should be ``False`` in :meth:`NativeEnvironment.render`.
     """
     head = list(islice(nodes, 2))
 
@@ -31,29 +27,17 @@ def native_concat(nodes, preserve_quotes=True):
     if len(head) == 1:
         raw = head[0]
     else:
-        if isinstance(nodes, types.GeneratorType):
-            nodes = chain(head, nodes)
-        raw = u"".join([text_type(v) for v in nodes])
+        raw = u"".join([text_type(v) for v in chain(head, nodes)])
 
     try:
-        literal = literal_eval(raw)
+        return literal_eval(raw)
     except (ValueError, SyntaxError, MemoryError):
         return raw
 
-    # If literal_eval returned a string, re-wrap with the original
-    # quote character to avoid dropping quotes between expression nodes.
-    # Without this, "'{{ a }}', '{{ b }}'" results in "a, b", but should
-    # be ('a', 'b').
-    if preserve_quotes and isinstance(literal, str):
-        return "{quote}{}{quote}".format(literal, quote=raw[0])
-
-    return literal
-
 
 class NativeCodeGenerator(CodeGenerator):
     """A code generator which renders Python types by not adding
-    ``to_string()`` around output nodes, and using :func:`native_concat`
-    to convert complex strings back to Python types if possible.
+    ``to_string()`` around output nodes.
     """
 
     @staticmethod
@@ -61,7 +45,7 @@ class NativeCodeGenerator(CodeGenerator):
         return value
 
     def _output_const_repr(self, group):
-        return repr(native_concat(group))
+        return repr(u"".join([text_type(v) for v in group]))
 
     def _output_child_to_const(self, node, frame, finalize):
         const = node.as_const(frame.eval_ctx)
@@ -100,10 +84,9 @@ class NativeTemplate(Template):
         Otherwise, the string is returned.
         """
         vars = dict(*args, **kwargs)
+
         try:
-            return native_concat(
-                self.root_render_func(self.new_context(vars)), preserve_quotes=False
-            )
+            return native_concat(self.root_render_func(self.new_context(vars)))
         except Exception:
             return self.environment.handle_exception()
 

nodes.py

@@ -671,7 +671,7 @@ class Filter(Expr):
         # python 3.  because of that, do not rename filter_ to filter!
         filter_ = self.environment.filters.get(self.name)
 
-        if filter_ is None or getattr(filter_, "contextfilter", False):
+        if filter_ is None or getattr(filter_, "contextfilter", False) is True:
             raise Impossible()
 
         # We cannot constant handle async filters, so we need to make sure
@@ -684,9 +684,9 @@ class Filter(Expr):
         args, kwargs = args_as_const(self, eval_ctx)
         args.insert(0, self.node.as_const(eval_ctx))
 
-        if getattr(filter_, "evalcontextfilter", False):
+        if getattr(filter_, "evalcontextfilter", False) is True:
             args.insert(0, eval_ctx)
-        elif getattr(filter_, "environmentfilter", False):
+        elif getattr(filter_, "environmentfilter", False) is True:
             args.insert(0, self.environment)
 
         try:

runtime.py

@@ -280,11 +280,11 @@ class Context(with_metaclass(ContextMeta)):
                     break
 
         if callable(__obj):
-            if getattr(__obj, "contextfunction", 0):
+            if getattr(__obj, "contextfunction", False) is True:
                 args = (__self,) + args
-            elif getattr(__obj, "evalcontextfunction", 0):
+            elif getattr(__obj, "evalcontextfunction", False) is True:
                 args = (__self.eval_ctx,) + args
-            elif getattr(__obj, "environmentfunction", 0):
+            elif getattr(__obj, "environmentfunction", False) is True:
                 args = (__self.environment,) + args
         try:
             return __obj(*args, **kwargs)

utils.py

@@ -6,6 +6,8 @@ import warnings
 from collections import deque
 from random import choice
 from random import randrange
+from string import ascii_letters as _letters
+from string import digits as _digits
 from threading import Lock
 
 from markupsafe import escape
@@ -16,20 +18,6 @@ from ._compat import string_types
 from ._compat import text_type
 from ._compat import url_quote
 
-_word_split_re = re.compile(r"(\s+)")
-_punctuation_re = re.compile(
-    "^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$"
-    % (
-        "|".join(map(re.escape, ("(", "<", "&lt;"))),
-        "|".join(map(re.escape, (".", ",", ")", ">", "\n", "&gt;"))),
-    )
-)
-_simple_email_re = re.compile(r"^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$")
-_striptags_re = re.compile(r"(<!--.*?-->|<[^>]*>)")
-_entity_re = re.compile(r"&([^;]+);")
-_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
-_digits = "0123456789"
-
 # special singleton representing missing values for the runtime
 missing = type("MissingType", (), {"__repr__": lambda x: "missing"})()
 
@@ -165,11 +153,15 @@ def object_type_repr(obj):
         return "None"
     elif obj is Ellipsis:
         return "Ellipsis"
+
+    cls = type(obj)
+
     # __builtin__ in 2.x, builtins in 3.x
-    if obj.__class__.__module__ in ("__builtin__", "builtins"):
-        name = obj.__class__.__name__
+    if cls.__module__ in ("__builtin__", "builtins"):
+        name = cls.__name__
     else:
-        name = obj.__class__.__module__ + "." + obj.__class__.__name__
+        name = cls.__module__ + "." + cls.__name__
+
     return "%s object" % name
 
 
@@ -206,48 +198,65 @@ def urlize(text, trim_url_limit=None, rel=None, target=None):
         and (x[:limit] + (len(x) >= limit and "..." or ""))
         or x
     )
-    words = _word_split_re.split(text_type(escape(text)))
+    words = re.split(r"(\s+)", text_type(escape(text)))
     rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or ""
     target_attr = target and ' target="%s"' % escape(target) or ""
 
     for i, word in enumerate(words):
-        match = _punctuation_re.match(word)
+        head, middle, tail = "", word, ""
+        match = re.match(r"^([(<]|&lt;)+", middle)
+
         if match:
-            lead, middle, trail = match.groups()
-            if middle.startswith("www.") or (
-                "@" not in middle
-                and not middle.startswith("http://")
-                and not middle.startswith("https://")
-                and len(middle) > 0
-                and middle[0] in _letters + _digits
-                and (
-                    middle.endswith(".org")
-                    or middle.endswith(".net")
-                    or middle.endswith(".com")
-                )
-            ):
-                middle = '<a href="http://%s"%s%s>%s</a>' % (
-                    middle,
-                    rel_attr,
-                    target_attr,
-                    trim_url(middle),
-                )
-            if middle.startswith("http://") or middle.startswith("https://"):
-                middle = '<a href="%s"%s%s>%s</a>' % (
-                    middle,
-                    rel_attr,
-                    target_attr,
-                    trim_url(middle),
-                )
-            if (
-                "@" in middle
-                and not middle.startswith("www.")
-                and ":" not in middle
-                and _simple_email_re.match(middle)
-            ):
-                middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
-            if lead + middle + trail != word:
-                words[i] = lead + middle + trail
+            head = match.group()
+            middle = middle[match.end() :]
+
+        # Unlike lead, which is anchored to the start of the string,
+        # need to check that the string ends with any of the characters
+        # before trying to match all of them, to avoid backtracking.
+        if middle.endswith((")", ">", ".", ",", "\n", "&gt;")):
+            match = re.search(r"([)>.,\n]|&gt;)+$", middle)
+
+            if match:
+                tail = match.group()
+                middle = middle[: match.start()]
+
+        if middle.startswith("www.") or (
+            "@" not in middle
+            and not middle.startswith("http://")
+            and not middle.startswith("https://")
+            and len(middle) > 0
+            and middle[0] in _letters + _digits
+            and (
+                middle.endswith(".org")
+                or middle.endswith(".net")
+                or middle.endswith(".com")
+            )
+        ):
+            middle = '<a href="http://%s"%s%s>%s</a>' % (
+                middle,
+                rel_attr,
+                target_attr,
+                trim_url(middle),
+            )
+
+        if middle.startswith("http://") or middle.startswith("https://"):
+            middle = '<a href="%s"%s%s>%s</a>' % (
+                middle,
+                rel_attr,
+                target_attr,
+                trim_url(middle),
+            )
+
+        if (
+            "@" in middle
+            and not middle.startswith("www.")
+            and ":" not in middle
+            and re.match(r"^\S+@\w[\w.-]*\.\w+$", middle)
+        ):
+            middle = '<a href="mailto:%s">%s</a>' % (middle, middle)
+
+        words[i] = head + middle + tail
+
     return u"".join(words)
 
 
@@ -693,7 +702,8 @@ class Namespace(object):
         self.__attrs = dict(*args, **kwargs)
 
     def __getattribute__(self, name):
-        if name == "_Namespace__attrs":
+        # __class__ is needed for the awaitable check in async mode
+        if name in {"_Namespace__attrs", "__class__"}:
             return object.__getattribute__(self, name)
         try:
             return self.__attrs[name]