Merge branch '3.0.x'

CHANGES.rst

@@ -6,6 +6,23 @@ Version 3.1.0
 Unreleased
 
 
+Version 3.0.3
+-------------
+
+Released 2021-11-09
+
+-   Fix traceback rewriting internals for Python 3.10 and 3.11.
+    :issue:`1535`
+-   Fix how the native environment treats leading and trailing spaces
+    when parsing values on Python 3.10. :pr:`1537`
+-   Improve async performance by avoiding checks for common types.
+    :issue:`1514`
+-   Revert change to ``hash(Node)`` behavior. Nodes are hashed by id
+    again :issue:`1521`
+-   ``PackageLoader`` works when the package is a single module file.
+    :issue:`1512`
+
+
 Version 3.0.2
 -------------
 
@@ -414,7 +431,7 @@ Released 2017-01-08
     possible. For more information and a discussion see :issue:`641`
 -   Resolved an issue where ``block scoped`` would not take advantage of
     the new scoping rules. In some more exotic cases a variable
-    overriden in a local scope would not make it into a block.
+    overridden in a local scope would not make it into a block.
 -   Change the code generation of the ``with`` statement to be in line
     with the new scoping rules. This resolves some unlikely bugs in edge
     cases. This also introduces a new internal ``With`` node that can be

docs/templates.rst

@@ -587,17 +587,26 @@ When combined with ``scoped``, the ``required`` modifier must be placed
 Template Objects
 ~~~~~~~~~~~~~~~~
 
-.. versionchanged:: 2.4
+``extends``, ``include``, and ``import`` can take a template object
+instead of the name of a template to load. This could be useful in some
+advanced situations, since you can use Python code to load a template
+first and pass it in to ``render``.
 
-If a template object was passed in the template context, you can
-extend from that object as well.  Assuming the calling code passes
-a layout template as `layout_template` to the environment, this
-code works::
+.. code-block:: python
 
-    {% extends layout_template %}
+    if debug_mode:
+        layout = env.get_template("debug_layout.html")
+    else:
+        layout = env.get_template("layout.html")
 
-Previously, the `layout_template` variable had to be a string with
-the layout template's filename for this to work.
+    user_detail = env.get_template("user/detail.html", layout=layout)
+
+.. code-block:: jinja
+
+    {% extends layout %}
+
+Note how ``extends`` is passed the variable with the template object
+that was passed to ``render``, instead of a string.
 
 
 HTML Escaping
@@ -914,9 +923,6 @@ are available on a macro object:
 `arguments`
     A tuple of the names of arguments the macro accepts.
 
-`defaults`
-    A tuple of default values.
-
 `catch_kwargs`
     This is `true` if the macro accepts extra keyword arguments (i.e.: accesses
     the special `kwargs` variable).
@@ -1338,8 +1344,19 @@ but exists for completeness' sake.  The following operators are supported:
     ``{{ '=' * 80 }}`` would print a bar of 80 equal signs.
 
 ``**``
-    Raise the left operand to the power of the right operand.  ``{{ 2**3 }}``
-    would return ``8``.
+    Raise the left operand to the power of the right operand.
+    ``{{ 2**3 }}`` would return ``8``.
+
+    Unlike Python, chained pow is evaluated left to right.
+    ``{{ 3**3**3 }}`` is evaluated as ``(3**3)**3`` in Jinja, but would
+    be evaluated as ``3**(3**3)`` in Python. Use parentheses in Jinja
+    to be explicit about what order you want. It is usually preferable
+    to do extended math in Python and pass the results to ``render``
+    rather than doing it in the template.
+
+    This behavior may be changed in the future to match Python, if it's
+    possible to introduce an upgrade path.
+
 
 Comparisons
 ~~~~~~~~~~~

src/jinja2/async_utils.py

@@ -44,7 +44,14 @@ def async_variant(normal_func):  # type: ignore
     return decorator
 
 
+_common_primitives = {int, float, bool, str, list, dict, tuple, type(None)}
+
+
 async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V":
+    # Avoid a costly call to isawaitable
+    if type(value) in _common_primitives:
+        return t.cast("V", value)
+
     if inspect.isawaitable(value):
         return await t.cast("t.Awaitable[V]", value)
 

src/jinja2/debug.py

@@ -102,62 +102,42 @@ def fake_traceback(  # type: ignore
         "__jinja_exception__": exc_value,
     }
     # Raise an exception at the correct line number.
-    code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec")
+    code: CodeType = compile(
+        "\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec"
+    )
 
     # Build a new code object that points to the template file and
     # replaces the location with a block name.
-    try:
-        location = "template"
+    location = "template"
 
-        if tb is not None:
-            function = tb.tb_frame.f_code.co_name
+    if tb is not None:
+        function = tb.tb_frame.f_code.co_name
 
-            if function == "root":
-                location = "top-level template code"
-            elif function.startswith("block_"):
-                location = f"block {function[6:]!r}"
+        if function == "root":
+            location = "top-level template code"
+        elif function.startswith("block_"):
+            location = f"block {function[6:]!r}"
 
-        # Collect arguments for the new code object. CodeType only
-        # accepts positional arguments, and arguments were inserted in
-        # new Python versions.
-        code_args = []
-
-        for attr in (
-            "argcount",
-            "posonlyargcount",  # Python 3.8
-            "kwonlyargcount",
-            "nlocals",
-            "stacksize",
-            "flags",
-            "code",  # codestring
-            "consts",  # constants
-            "names",
-            "varnames",
-            ("filename", filename),
-            ("name", location),
-            "firstlineno",
-            "lnotab",
-            "freevars",
-            "cellvars",
-            "linetable",  # Python 3.10
-        ):
-            if isinstance(attr, tuple):
-                # Replace with given value.
-                code_args.append(attr[1])
-                continue
-
-            try:
-                # Copy original value if it exists.
-                code_args.append(getattr(code, "co_" + t.cast(str, attr)))
-            except AttributeError:
-                # Some arguments were added later.
-                continue
-
-        code = CodeType(*code_args)
-    except Exception:
-        # Some environments such as Google App Engine don't support
-        # modifying code objects.
-        pass
+    if sys.version_info >= (3, 8):
+        code = code.replace(co_name=location)
+    else:
+        code = CodeType(
+            code.co_argcount,
+            code.co_kwonlyargcount,
+            code.co_nlocals,
+            code.co_stacksize,
+            code.co_flags,
+            code.co_code,
+            code.co_consts,
+            code.co_names,
+            code.co_varnames,
+            code.co_filename,
+            location,
+            code.co_firstlineno,
+            code.co_lnotab,
+            code.co_freevars,
+            code.co_cellvars,
+        )
 
     # Execute the new code, which is guaranteed to raise, and return
     # the new traceback without this frame.

src/jinja2/environment.py

@@ -1114,33 +1114,20 @@ class Environment:
 
 
 class Template:
-    """The central template object.  This class represents a compiled template
-    and is used to evaluate it.
+    """A compiled template that can be rendered.
 
-    Normally the template object is generated from an :class:`Environment` but
-    it also has a constructor that makes it possible to create a template
-    instance directly using the constructor.  It takes the same arguments as
-    the environment constructor but it's not possible to specify a loader.
+    Use the methods on :class:`Environment` to create or load templates.
+    The environment is used to configure how templates are compiled and
+    behave.
 
-    Every template object has a few methods and members that are guaranteed
-    to exist.  However it's important that a template object should be
-    considered immutable.  Modifications on the object are not supported.
+    It is also possible to create a template object directly. This is
+    not usually recommended. The constructor takes most of the same
+    arguments as :class:`Environment`. All templates created with the
+    same environment arguments share the same ephemeral ``Environment``
+    instance behind the scenes.
 
-    Template objects created from the constructor rather than an environment
-    do have an `environment` attribute that points to a temporary environment
-    that is probably shared with other templates created with the constructor
-    and compatible settings.
-
-    >>> template = Template('Hello {{ name }}!')
-    >>> template.render(name='John Doe') == u'Hello John Doe!'
-    True
-    >>> stream = template.stream(name='John Doe')
-    >>> next(stream) == u'Hello John Doe!'
-    True
-    >>> next(stream)
-    Traceback (most recent call last):
-        ...
-    StopIteration
+    A template object should be considered immutable. Modifications on
+    the object are not supported.
     """
 
     #: Type of environment to create when creating a template directly

src/jinja2/loaders.py

@@ -297,10 +297,18 @@ class PackageLoader(BaseLoader):
             self._archive = loader.archive
             pkgdir = next(iter(spec.submodule_search_locations))  # type: ignore
             template_root = os.path.join(pkgdir, package_path)
-        elif spec.submodule_search_locations:
-            # This will be one element for regular packages and multiple
-            # for namespace packages.
-            for root in spec.submodule_search_locations:
+        else:
+            roots: t.List[str] = []
+
+            # One element for regular packages, multiple for namespace
+            # packages, or None for single module file.
+            if spec.submodule_search_locations:
+                roots.extend(spec.submodule_search_locations)
+            # A single module file, use the parent directory instead.
+            elif spec.origin is not None:
+                roots.append(os.path.dirname(spec.origin))
+
+            for root in roots:
                 root = os.path.join(root, package_path)
 
                 if os.path.isdir(root):

src/jinja2/nativetypes.py

@@ -1,5 +1,6 @@
 import typing as t
 from ast import literal_eval
+from ast import parse
 from itertools import chain
 from itertools import islice
 
@@ -33,7 +34,12 @@ def native_concat(values: t.Iterable[t.Any]) -> t.Optional[t.Any]:
         raw = "".join([str(v) for v in chain(head, values)])
 
     try:
-        return literal_eval(raw)
+        return literal_eval(
+            # In Python 3.10+ ast.literal_eval removes leading spaces/tabs
+            # from the given string. For backwards compatibility we need to
+            # parse the string ourselves without removing leading spaces/tabs.
+            parse(raw, mode="eval")
+        )
     except (ValueError, SyntaxError, MemoryError):
         return raw
 

src/jinja2/nodes.py

@@ -241,8 +241,7 @@ class Node(metaclass=NodeType):
 
         return tuple(self.iter_fields()) == tuple(other.iter_fields())
 
-    def __hash__(self) -> int:
-        return hash(tuple(self.iter_fields()))
+    __hash__ = object.__hash__
 
     def __repr__(self) -> str:
         args_str = ", ".join(f"{a}={getattr(self, a, None)!r}" for a in self.fields)
@@ -956,7 +955,7 @@ class Div(BinExpr):
 
 
 class FloorDiv(BinExpr):
-    """Divides the left by the right node and truncates conver the
+    """Divides the left by the right node and converts the
     result into an integer by truncating.
     """
 

tests/test_inheritance.py

@@ -37,7 +37,7 @@ WORKINGTEMPLATE = """\
 {% block block1 %}
   {% if false %}
     {% block block2 %}
-      this should workd
+      this should work
     {% endblock %}
   {% endif %}
 {% endblock %}
@@ -49,7 +49,7 @@ DOUBLEEXTENDS = """\
 {% block block1 %}
   {% if false %}
     {% block block2 %}
-      this should workd
+      this should work
     {% endblock %}
   {% endif %}
 {% endblock %}

tests/test_loader.py

@@ -313,6 +313,28 @@ def test_package_dir_list(package_dir_loader):
     assert "test.html" in templates
 
 
+@pytest.fixture()
+def package_file_loader(monkeypatch):
+    monkeypatch.syspath_prepend(Path(__file__).parent / "res")
+    return PackageLoader("__init__")
+
+
+@pytest.mark.parametrize(
+    ("template", "expect"), [("foo/test.html", "FOO"), ("test.html", "BAR")]
+)
+def test_package_file_source(package_file_loader, template, expect):
+    source, name, up_to_date = package_file_loader.get_source(None, template)
+    assert source.rstrip() == expect
+    assert name.endswith(os.path.join(*split_template_path(template)))
+    assert up_to_date()
+
+
+def test_package_file_list(package_file_loader):
+    templates = package_file_loader.list_templates()
+    assert "foo/test.html" in templates
+    assert "test.html" in templates
+
+
 @pytest.fixture()
 def package_zip_loader(monkeypatch):
     package_zip = (Path(__file__) / ".." / "res" / "package.zip").resolve()

tests/test_nativetypes.py

@@ -147,3 +147,9 @@ def test_no_intermediate_eval(env):
 def test_spontaneous_env():
     t = NativeTemplate("{{ true }}")
     assert isinstance(t.environment, NativeEnvironment)
+
+
+def test_leading_spaces(env):
+    t = env.from_string(" {{ True }}")
+    result = t.render()
+    assert result == " True"

tests/test_nodes.py

@@ -0,0 +1,3 @@
+def test_template_hash(env):
+    template = env.parse("hash test")
+    hash(template)