From be0e9cde33b0795bb1f4f9ab70aa1c540411507e Mon Sep 17 00:00:00 2001 From: oumeng Date: Wed, 30 Mar 2022 17:08:30 +0800 Subject: [PATCH] updated jinja2 version Signed-off-by: oumeng --- README.OpenSource | 6 +- __init__.py | 2 +- asyncfilters.py | 9 ++- compiler.py | 12 ++-- debug.py | 5 +- environment.py | 6 +- filters.py | 10 +-- lexer.py | 21 ++++-- loaders.py | 164 ++++++++++++++-------------------------------- nativetypes.py | 31 ++------- nodes.py | 6 +- runtime.py | 6 +- utils.py | 120 +++++++++++++++++---------------- 13 files changed, 163 insertions(+), 235 deletions(-) diff --git a/README.OpenSource b/README.OpenSource index d71ed08..b644212 100644 --- a/README.OpenSource +++ b/README.OpenSource @@ -3,9 +3,9 @@ "Name": "Jinja2", "License": "BSD 3-clause License", "License File": "LICENSE.rst", - "Version Number": "2.11.1", - "Owner": "yaoxiaoyu1@huawei.com", - "Upstream URL": "https://pypi.org/project/Jinja2/", + "Version Number": "2.11.3", + "Owner": "anguanglin@huawei.com", + "Upstream URL": "https://github.com/pallets/jinja", "Description": "Jinja2 is a template engine written in pure Python. It provides a Django inspired non-XML syntax but supports inline expressions and an optional sandboxed environment." } ] diff --git a/__init__.py b/__init__.py index 7f4a1c5..f17866f 100755 --- a/__init__.py +++ b/__init__.py @@ -41,4 +41,4 @@ from .utils import evalcontextfunction from .utils import is_undefined from .utils import select_autoescape -__version__ = "2.11.1" +__version__ = "2.11.3" diff --git a/asyncfilters.py b/asyncfilters.py index d29f6c6..3d98dbc 100755 --- a/asyncfilters.py +++ b/asyncfilters.py @@ -26,17 +26,16 @@ async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): def dualfilter(normal_filter, async_filter): wrap_evalctx = False - if getattr(normal_filter, "environmentfilter", False): + if getattr(normal_filter, "environmentfilter", False) is True: def is_async(args): return args[0].is_async wrap_evalctx = False else: - if not getattr(normal_filter, "evalcontextfilter", False) and not getattr( - normal_filter, "contextfilter", False - ): - wrap_evalctx = True + has_evalctxfilter = getattr(normal_filter, "evalcontextfilter", False) is True + has_ctxfilter = getattr(normal_filter, "contextfilter", False) is True + wrap_evalctx = not has_evalctxfilter and not has_ctxfilter def is_async(args): return args[0].environment.is_async diff --git a/compiler.py b/compiler.py index f450ec6..63297b4 100755 --- a/compiler.py +++ b/compiler.py @@ -1307,13 +1307,13 @@ class CodeGenerator(NodeVisitor): def finalize(value): return default(env_finalize(value)) - if getattr(env_finalize, "contextfunction", False): + if getattr(env_finalize, "contextfunction", False) is True: src += "context, " finalize = None # noqa: F811 - elif getattr(env_finalize, "evalcontextfunction", False): + elif getattr(env_finalize, "evalcontextfunction", False) is True: src += "context.eval_ctx, " finalize = None - elif getattr(env_finalize, "environmentfunction", False): + elif getattr(env_finalize, "environmentfunction", False) is True: src += "environment, " def finalize(value): @@ -1689,11 +1689,11 @@ class CodeGenerator(NodeVisitor): func = self.environment.filters.get(node.name) if func is None: self.fail("no filter named %r" % node.name, node.lineno) - if getattr(func, "contextfilter", False): + if getattr(func, "contextfilter", False) is True: self.write("context, ") - elif getattr(func, "evalcontextfilter", False): + elif getattr(func, "evalcontextfilter", False) is True: self.write("context.eval_ctx, ") - elif getattr(func, "environmentfilter", False): + elif getattr(func, "environmentfilter", False) is True: self.write("environment, ") # if the filter node is None we are inside a filter block diff --git a/debug.py b/debug.py index d2c5a06..5d8aec3 100755 --- a/debug.py +++ b/debug.py @@ -245,10 +245,7 @@ else: class _CTraceback(ctypes.Structure): _fields_ = [ # Extra PyObject slots when compiled with Py_TRACE_REFS. - ( - "PyObject_HEAD", - ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16), - ), + ("PyObject_HEAD", ctypes.c_byte * object().__sizeof__()), # Only care about tb_next as an object, not a traceback. ("tb_next", ctypes.py_object), ] diff --git a/environment.py b/environment.py index bf44b9d..8430390 100755 --- a/environment.py +++ b/environment.py @@ -492,20 +492,20 @@ class Environment(object): if func is None: fail_for_missing_callable("no filter named %r", name) args = [value] + list(args or ()) - if getattr(func, "contextfilter", False): + if getattr(func, "contextfilter", False) is True: if context is None: raise TemplateRuntimeError( "Attempted to invoke context filter without context" ) args.insert(0, context) - elif getattr(func, "evalcontextfilter", False): + elif getattr(func, "evalcontextfilter", False) is True: if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) args.insert(0, eval_ctx) - elif getattr(func, "environmentfilter", False): + elif getattr(func, "environmentfilter", False) is True: args.insert(0, self) return func(*args, **(kwargs or {})) diff --git a/filters.py b/filters.py index 1af7ac8..74b108d 100755 --- a/filters.py +++ b/filters.py @@ -268,16 +268,16 @@ def do_dictsort(value, case_sensitive=False, by="key", reverse=False): .. sourcecode:: jinja - {% for item in mydict|dictsort %} + {% for key, value in mydict|dictsort %} sort the dict by key, case insensitive - {% for item in mydict|dictsort(reverse=true) %} + {% for key, value in mydict|dictsort(reverse=true) %} sort the dict by key, case insensitive, reverse order - {% for item in mydict|dictsort(true) %} + {% for key, value in mydict|dictsort(true) %} sort the dict by key, case sensitive - {% for item in mydict|dictsort(false, 'value') %} + {% for key, value in mydict|dictsort(false, 'value') %} sort the dict by value, case insensitive """ if by == "key": @@ -761,7 +761,7 @@ def do_wordwrap( def do_wordcount(s): """Count the words in that string.""" - return len(_word_re.findall(s)) + return len(_word_re.findall(soft_unicode(s))) def do_int(value, default=0, base=10): diff --git a/lexer.py b/lexer.py index a2b44e9..552356a 100755 --- a/lexer.py +++ b/lexer.py @@ -681,6 +681,8 @@ class Lexer(object): source_length = len(source) balancing_stack = [] lstrip_unless_re = self.lstrip_unless_re + newlines_stripped = 0 + line_starting = True while 1: # tokenizer loop @@ -717,7 +719,9 @@ class Lexer(object): if strip_sign == "-": # Strip all whitespace between the text and the tag. - groups = (text.rstrip(),) + groups[1:] + stripped = text.rstrip() + newlines_stripped = text[len(stripped) :].count("\n") + groups = (stripped,) + groups[1:] elif ( # Not marked for preserving whitespace. strip_sign != "+" @@ -728,11 +732,11 @@ class Lexer(object): ): # The start of text between the last newline and the tag. l_pos = text.rfind("\n") + 1 - - # If there's only whitespace between the newline and the - # tag, strip it. - if not lstrip_unless_re.search(text, l_pos): - groups = (text[:l_pos],) + groups[1:] + if l_pos > 0 or line_starting: + # If there's only whitespace between the newline and the + # tag, strip it. + if not lstrip_unless_re.search(text, l_pos): + groups = (text[:l_pos],) + groups[1:] for idx, token in enumerate(tokens): # failure group @@ -758,7 +762,8 @@ class Lexer(object): data = groups[idx] if data or token not in ignore_if_empty: yield lineno, token, data - lineno += data.count("\n") + lineno += data.count("\n") + newlines_stripped + newlines_stripped = 0 # strings as token just are yielded as it. else: @@ -790,6 +795,8 @@ class Lexer(object): yield lineno, tokens, data lineno += data.count("\n") + line_starting = m.group()[-1:] == "\n" + # fetch new position into new variable so that we can check # if there is a internal parsing error which would result # in an infinite loop diff --git a/loaders.py b/loaders.py index ce5537a..457c4b5 100755 --- a/loaders.py +++ b/loaders.py @@ -3,11 +3,9 @@ sources. """ import os -import pkgutil import sys import weakref from hashlib import sha1 -from importlib import import_module from os import path from types import ModuleType @@ -217,141 +215,75 @@ class FileSystemLoader(BaseLoader): class PackageLoader(BaseLoader): - """Load templates from a directory in a Python package. + """Load templates from python eggs or packages. It is constructed with + the name of the python package and the path to the templates in that + package:: - :param package_name: Import name of the package that contains the - template directory. - :param package_path: Directory within the imported package that - contains the templates. - :param encoding: Encoding of template files. + loader = PackageLoader('mypackage', 'views') - The following example looks up templates in the ``pages`` directory - within the ``project.ui`` package. + If the package path is not given, ``'templates'`` is assumed. - .. code-block:: python - - loader = PackageLoader("project.ui", "pages") - - Only packages installed as directories (standard pip behavior) or - zip/egg files (less common) are supported. The Python API for - introspecting data in packages is too limited to support other - installation methods the way this loader requires. - - There is limited support for :pep:`420` namespace packages. The - template directory is assumed to only be in one namespace - contributor. Zip files contributing to a namespace are not - supported. - - .. versionchanged:: 2.11.0 - No longer uses ``setuptools`` as a dependency. - - .. versionchanged:: 2.11.0 - Limited PEP 420 namespace package support. + Per default the template encoding is ``'utf-8'`` which can be changed + by setting the `encoding` parameter to something else. Due to the nature + of eggs it's only possible to reload templates if the package was loaded + from the file system and not a zip file. """ def __init__(self, package_name, package_path="templates", encoding="utf-8"): - if package_path == os.path.curdir: - package_path = "" - elif package_path[:2] == os.path.curdir + os.path.sep: - package_path = package_path[2:] + from pkg_resources import DefaultProvider + from pkg_resources import get_provider + from pkg_resources import ResourceManager - package_path = os.path.normpath(package_path).rstrip(os.path.sep) - self.package_path = package_path - self.package_name = package_name + provider = get_provider(package_name) self.encoding = encoding - - # Make sure the package exists. This also makes namespace - # packages work, otherwise get_loader returns None. - import_module(package_name) - self._loader = loader = pkgutil.get_loader(package_name) - - # Zip loader's archive attribute points at the zip. - self._archive = getattr(loader, "archive", None) - self._template_root = None - - if hasattr(loader, "get_filename"): - # A standard directory package, or a zip package. - self._template_root = os.path.join( - os.path.dirname(loader.get_filename(package_name)), package_path - ) - elif hasattr(loader, "_path"): - # A namespace package, limited support. Find the first - # contributor with the template directory. - for root in loader._path: - root = os.path.join(root, package_path) - - if os.path.isdir(root): - self._template_root = root - break - - if self._template_root is None: - raise ValueError( - "The %r package was not installed in a way that" - " PackageLoader understands." % package_name - ) + self.manager = ResourceManager() + self.filesystem_bound = isinstance(provider, DefaultProvider) + self.provider = provider + self.package_path = package_path def get_source(self, environment, template): - p = os.path.join(self._template_root, *split_template_path(template)) + pieces = split_template_path(template) + p = "/".join((self.package_path,) + tuple(pieces)) - if self._archive is None: - # Package is a directory. - if not os.path.isfile(p): - raise TemplateNotFound(template) + if not self.provider.has_resource(p): + raise TemplateNotFound(template) - with open(p, "rb") as f: - source = f.read() + filename = uptodate = None - mtime = os.path.getmtime(p) + if self.filesystem_bound: + filename = self.provider.get_resource_filename(self.manager, p) + mtime = path.getmtime(filename) - def up_to_date(): - return os.path.isfile(p) and os.path.getmtime(p) == mtime + def uptodate(): + try: + return path.getmtime(filename) == mtime + except OSError: + return False - else: - # Package is a zip file. - try: - source = self._loader.get_data(p) - except OSError: - raise TemplateNotFound(template) - - # Could use the zip's mtime for all template mtimes, but - # would need to safely reload the module if it's out of - # date, so just report it as always current. - up_to_date = None - - return source.decode(self.encoding), p, up_to_date + source = self.provider.get_resource_string(self.manager, p) + return source.decode(self.encoding), filename, uptodate def list_templates(self): + path = self.package_path + + if path[:2] == "./": + path = path[2:] + elif path == ".": + path = "" + + offset = len(path) results = [] - if self._archive is None: - # Package is a directory. - offset = len(self._template_root) + def _walk(path): + for filename in self.provider.resource_listdir(path): + fullname = path + "/" + filename - for dirpath, _, filenames in os.walk(self._template_root): - dirpath = dirpath[offset:].lstrip(os.path.sep) - results.extend( - os.path.join(dirpath, name).replace(os.path.sep, "/") - for name in filenames - ) - else: - if not hasattr(self._loader, "_files"): - raise TypeError( - "This zip import does not have the required" - " metadata to list templates." - ) - - # Package is a zip file. - prefix = ( - self._template_root[len(self._archive) :].lstrip(os.path.sep) - + os.path.sep - ) - offset = len(prefix) - - for name in self._loader._files.keys(): - # Find names under the templates directory that aren't directories. - if name.startswith(prefix) and name[-1] != os.path.sep: - results.append(name[offset:].replace(os.path.sep, "/")) + if self.provider.resource_isdir(fullname): + _walk(fullname) + else: + results.append(fullname[offset:].lstrip("/")) + _walk(path) results.sort() return results diff --git a/nativetypes.py b/nativetypes.py index 9866c96..a9ead4e 100755 --- a/nativetypes.py +++ b/nativetypes.py @@ -1,4 +1,3 @@ -import types from ast import literal_eval from itertools import chain from itertools import islice @@ -11,7 +10,7 @@ from .environment import Environment from .environment import Template -def native_concat(nodes, preserve_quotes=True): +def native_concat(nodes): """Return a native Python type from the list of compiled nodes. If the result is a single node, its value is returned. Otherwise, the nodes are concatenated as strings. If the result can be parsed with @@ -19,9 +18,6 @@ def native_concat(nodes, preserve_quotes=True): the string is returned. :param nodes: Iterable of nodes to concatenate. - :param preserve_quotes: Whether to re-wrap literal strings with - quotes, to preserve quotes around expressions for later parsing. - Should be ``False`` in :meth:`NativeEnvironment.render`. """ head = list(islice(nodes, 2)) @@ -31,29 +27,17 @@ def native_concat(nodes, preserve_quotes=True): if len(head) == 1: raw = head[0] else: - if isinstance(nodes, types.GeneratorType): - nodes = chain(head, nodes) - raw = u"".join([text_type(v) for v in nodes]) + raw = u"".join([text_type(v) for v in chain(head, nodes)]) try: - literal = literal_eval(raw) + return literal_eval(raw) except (ValueError, SyntaxError, MemoryError): return raw - # If literal_eval returned a string, re-wrap with the original - # quote character to avoid dropping quotes between expression nodes. - # Without this, "'{{ a }}', '{{ b }}'" results in "a, b", but should - # be ('a', 'b'). - if preserve_quotes and isinstance(literal, str): - return "{quote}{}{quote}".format(literal, quote=raw[0]) - - return literal - class NativeCodeGenerator(CodeGenerator): """A code generator which renders Python types by not adding - ``to_string()`` around output nodes, and using :func:`native_concat` - to convert complex strings back to Python types if possible. + ``to_string()`` around output nodes. """ @staticmethod @@ -61,7 +45,7 @@ class NativeCodeGenerator(CodeGenerator): return value def _output_const_repr(self, group): - return repr(native_concat(group)) + return repr(u"".join([text_type(v) for v in group])) def _output_child_to_const(self, node, frame, finalize): const = node.as_const(frame.eval_ctx) @@ -100,10 +84,9 @@ class NativeTemplate(Template): Otherwise, the string is returned. """ vars = dict(*args, **kwargs) + try: - return native_concat( - self.root_render_func(self.new_context(vars)), preserve_quotes=False - ) + return native_concat(self.root_render_func(self.new_context(vars))) except Exception: return self.environment.handle_exception() diff --git a/nodes.py b/nodes.py index 9f3edc0..95bd614 100755 --- a/nodes.py +++ b/nodes.py @@ -671,7 +671,7 @@ class Filter(Expr): # python 3. because of that, do not rename filter_ to filter! filter_ = self.environment.filters.get(self.name) - if filter_ is None or getattr(filter_, "contextfilter", False): + if filter_ is None or getattr(filter_, "contextfilter", False) is True: raise Impossible() # We cannot constant handle async filters, so we need to make sure @@ -684,9 +684,9 @@ class Filter(Expr): args, kwargs = args_as_const(self, eval_ctx) args.insert(0, self.node.as_const(eval_ctx)) - if getattr(filter_, "evalcontextfilter", False): + if getattr(filter_, "evalcontextfilter", False) is True: args.insert(0, eval_ctx) - elif getattr(filter_, "environmentfilter", False): + elif getattr(filter_, "environmentfilter", False) is True: args.insert(0, self.environment) try: diff --git a/runtime.py b/runtime.py index 527d4b5..3ad7968 100755 --- a/runtime.py +++ b/runtime.py @@ -280,11 +280,11 @@ class Context(with_metaclass(ContextMeta)): break if callable(__obj): - if getattr(__obj, "contextfunction", 0): + if getattr(__obj, "contextfunction", False) is True: args = (__self,) + args - elif getattr(__obj, "evalcontextfunction", 0): + elif getattr(__obj, "evalcontextfunction", False) is True: args = (__self.eval_ctx,) + args - elif getattr(__obj, "environmentfunction", 0): + elif getattr(__obj, "environmentfunction", False) is True: args = (__self.environment,) + args try: return __obj(*args, **kwargs) diff --git a/utils.py b/utils.py index e3285e8..6afca81 100755 --- a/utils.py +++ b/utils.py @@ -6,6 +6,8 @@ import warnings from collections import deque from random import choice from random import randrange +from string import ascii_letters as _letters +from string import digits as _digits from threading import Lock from markupsafe import escape @@ -16,20 +18,6 @@ from ._compat import string_types from ._compat import text_type from ._compat import url_quote -_word_split_re = re.compile(r"(\s+)") -_punctuation_re = re.compile( - "^(?P(?:%s)*)(?P.*?)(?P(?:%s)*)$" - % ( - "|".join(map(re.escape, ("(", "<", "<"))), - "|".join(map(re.escape, (".", ",", ")", ">", "\n", ">"))), - ) -) -_simple_email_re = re.compile(r"^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$") -_striptags_re = re.compile(r"(|<[^>]*>)") -_entity_re = re.compile(r"&([^;]+);") -_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" -_digits = "0123456789" - # special singleton representing missing values for the runtime missing = type("MissingType", (), {"__repr__": lambda x: "missing"})() @@ -165,11 +153,15 @@ def object_type_repr(obj): return "None" elif obj is Ellipsis: return "Ellipsis" + + cls = type(obj) + # __builtin__ in 2.x, builtins in 3.x - if obj.__class__.__module__ in ("__builtin__", "builtins"): - name = obj.__class__.__name__ + if cls.__module__ in ("__builtin__", "builtins"): + name = cls.__name__ else: - name = obj.__class__.__module__ + "." + obj.__class__.__name__ + name = cls.__module__ + "." + cls.__name__ + return "%s object" % name @@ -206,48 +198,65 @@ def urlize(text, trim_url_limit=None, rel=None, target=None): and (x[:limit] + (len(x) >= limit and "..." or "")) or x ) - words = _word_split_re.split(text_type(escape(text))) + words = re.split(r"(\s+)", text_type(escape(text))) rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or "" target_attr = target and ' target="%s"' % escape(target) or "" for i, word in enumerate(words): - match = _punctuation_re.match(word) + head, middle, tail = "", word, "" + match = re.match(r"^([(<]|<)+", middle) + if match: - lead, middle, trail = match.groups() - if middle.startswith("www.") or ( - "@" not in middle - and not middle.startswith("http://") - and not middle.startswith("https://") - and len(middle) > 0 - and middle[0] in _letters + _digits - and ( - middle.endswith(".org") - or middle.endswith(".net") - or middle.endswith(".com") - ) - ): - middle = '%s' % ( - middle, - rel_attr, - target_attr, - trim_url(middle), - ) - if middle.startswith("http://") or middle.startswith("https://"): - middle = '%s' % ( - middle, - rel_attr, - target_attr, - trim_url(middle), - ) - if ( - "@" in middle - and not middle.startswith("www.") - and ":" not in middle - and _simple_email_re.match(middle) - ): - middle = '%s' % (middle, middle) - if lead + middle + trail != word: - words[i] = lead + middle + trail + head = match.group() + middle = middle[match.end() :] + + # Unlike lead, which is anchored to the start of the string, + # need to check that the string ends with any of the characters + # before trying to match all of them, to avoid backtracking. + if middle.endswith((")", ">", ".", ",", "\n", ">")): + match = re.search(r"([)>.,\n]|>)+$", middle) + + if match: + tail = match.group() + middle = middle[: match.start()] + + if middle.startswith("www.") or ( + "@" not in middle + and not middle.startswith("http://") + and not middle.startswith("https://") + and len(middle) > 0 + and middle[0] in _letters + _digits + and ( + middle.endswith(".org") + or middle.endswith(".net") + or middle.endswith(".com") + ) + ): + middle = '%s' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + + if middle.startswith("http://") or middle.startswith("https://"): + middle = '%s' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + + if ( + "@" in middle + and not middle.startswith("www.") + and ":" not in middle + and re.match(r"^\S+@\w[\w.-]*\.\w+$", middle) + ): + middle = '%s' % (middle, middle) + + words[i] = head + middle + tail + return u"".join(words) @@ -693,7 +702,8 @@ class Namespace(object): self.__attrs = dict(*args, **kwargs) def __getattribute__(self, name): - if name == "_Namespace__attrs": + # __class__ is needed for the awaitable check in async mode + if name in {"_Namespace__attrs", "__class__"}: return object.__getattribute__(self, name) try: return self.__attrs[name]