test on trio, fix all missing aclose related warnings (#1960)

Author: graingert

CHANGES.rst

@@ -7,6 +7,13 @@ Unreleased
 
 -   Calling sync ``render`` for an async template uses ``asyncio.run``.
     :pr:`1952`
+-   Avoid unclosed ``auto_aiter`` warnings. :pr:`1960`
+-   Return an ``aclose``-able ``AsyncGenerator`` from
+    ``Template.generate_async``. :pr:`1960`
+-   Avoid leaving ``root_render_func()`` unclosed in
+    ``Template.generate_async``. :pr:`1960`
+-   Avoid leaving async generators unclosed in blocks, includes and extends.
+    :pr:`1960`
 
 
 Version 3.1.4

requirements/docs.txt

@@ -15,7 +15,7 @@ charset-normalizer==3.1.0
     # via requests
 docutils==0.20.1
     # via sphinx
-idna==3.4
+idna==3.6
     # via requests
 imagesize==1.4.1
     # via sphinx

requirements/tests.in

@@ -1 +1,2 @@
 pytest
+trio<=0.22.2  # for Python3.7 support

requirements/tests.txt

@@ -1,19 +1,35 @@
-# SHA1:0eaa389e1fdb3a1917c0f987514bd561be5718ee
+# SHA1:b8d151f902b43c4435188a9d3494fb8d4af07168
 #
 # This file is autogenerated by pip-compile-multi
 # To update, run:
 #
 #    pip-compile-multi
 #
+attrs==23.2.0
+    # via
+    #   outcome
+    #   trio
 exceptiongroup==1.1.1
-    # via pytest
+    # via
+    #   pytest
+    #   trio
+idna==3.6
+    # via trio
 iniconfig==2.0.0
     # via pytest
+outcome==1.3.0.post0
+    # via trio
 packaging==23.1
     # via pytest
 pluggy==1.2.0
     # via pytest
 pytest==7.4.0
     # via -r requirements/tests.in
+sniffio==1.3.1
+    # via trio
+sortedcontainers==2.4.0
+    # via trio
 tomli==2.0.1
     # via pytest
+trio==0.22.2
+    # via -r requirements/tests.in

src/jinja2/async_utils.py

@@ -6,6 +6,9 @@
 from .utils import _PassArg
 from .utils import pass_eval_context
 
+if t.TYPE_CHECKING:
+    import typing_extensions as te
+
 V = t.TypeVar("V")
 
 
@@ -67,15 +70,27 @@ async def auto_await(value: t.Union[t.Awaitable["V"], "V"]) -> "V":
     return t.cast("V", value)
 
 
-async def auto_aiter(
+class _IteratorToAsyncIterator(t.Generic[V]):
+    def __init__(self, iterator: "t.Iterator[V]"):
+        self._iterator = iterator
+
+    def __aiter__(self) -> "te.Self":
+        return self
+
+    async def __anext__(self) -> V:
+        try:
+            return next(self._iterator)
+        except StopIteration as e:
+            raise StopAsyncIteration(e.value) from e
+
+
+def auto_aiter(
     iterable: "t.Union[t.AsyncIterable[V], t.Iterable[V]]",
 ) -> "t.AsyncIterator[V]":
     if hasattr(iterable, "__aiter__"):
-        async for item in t.cast("t.AsyncIterable[V]", iterable):
-            yield item
+        return iterable.__aiter__()
     else:
-        for item in iterable:
-            yield item
+        return _IteratorToAsyncIterator(iter(iterable))
 
 
 async def auto_to_list(

src/jinja2/compiler.py

@@ -902,12 +902,15 @@ def visit_Template(
             if not self.environment.is_async:
                 self.writeline("yield from parent_template.root_render_func(context)")
             else:
-                self.writeline(
-                    "async for event in parent_template.root_render_func(context):"
-                )
+                self.writeline("agen = parent_template.root_render_func(context)")
+                self.writeline("try:")
+                self.indent()
+                self.writeline("async for event in agen:")
                 self.indent()
                 self.writeline("yield event")
                 self.outdent()
+                self.outdent()
+                self.writeline("finally: await agen.aclose()")
             self.outdent(1 + (not self.has_known_extends))
 
         # at this point we now have the blocks collected and can visit them too.
@@ -977,14 +980,20 @@ def visit_Block(self, node: nodes.Block, frame: Frame) -> None:
                 f"yield from context.blocks[{node.name!r}][0]({context})", node
             )
         else:
+            self.writeline(f"gen = context.blocks[{node.name!r}][0]({context})")
+            self.writeline("try:")
+            self.indent()
             self.writeline(
-                f"{self.choose_async()}for event in"
-                f" context.blocks[{node.name!r}][0]({context}):",
+                f"{self.choose_async()}for event in gen:",
                 node,
             )
             self.indent()
             self.simple_write("event", frame)
             self.outdent()
+            self.outdent()
+            self.writeline(
+                f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}"
+            )
 
         self.outdent(level)
 
@@ -1057,26 +1066,33 @@ def visit_Include(self, node: nodes.Include, frame: Frame) -> None:
             self.writeline("else:")
             self.indent()
 
-        skip_event_yield = False
+        def loop_body() -> None:
+            self.indent()
+            self.simple_write("event", frame)
+            self.outdent()
+
         if node.with_context:
             self.writeline(
-                f"{self.choose_async()}for event in template.root_render_func("
+                f"gen = template.root_render_func("
                 "template.new_context(context.get_all(), True,"
-                f" {self.dump_local_context(frame)})):"
+                f" {self.dump_local_context(frame)}))"
+            )
+            self.writeline("try:")
+            self.indent()
+            self.writeline(f"{self.choose_async()}for event in gen:")
+            loop_body()
+            self.outdent()
+            self.writeline(
+                f"finally: {self.choose_async('await gen.aclose()', 'gen.close()')}"
             )
         elif self.environment.is_async:
             self.writeline(
                 "for event in (await template._get_default_module_async())"
                 "._body_stream:"
             )
+            loop_body()
         else:
             self.writeline("yield from template._get_default_module()._body_stream")
-            skip_event_yield = True
-
-        if not skip_event_yield:
-            self.indent()
-            self.simple_write("event", frame)
-            self.outdent()
 
         if node.ignore_missing:
             self.outdent()

src/jinja2/environment.py

@@ -1346,7 +1346,7 @@ async def to_list() -> t.List[str]:
 
     async def generate_async(
         self, *args: t.Any, **kwargs: t.Any
-    ) -> t.AsyncIterator[str]:
+    ) -> t.AsyncGenerator[str, object]:
         """An async version of :meth:`generate`.  Works very similarly but
         returns an async iterator instead.
         """
@@ -1358,8 +1358,14 @@ async def generate_async(
         ctx = self.new_context(dict(*args, **kwargs))
 
         try:
-            async for event in self.root_render_func(ctx):  # type: ignore
-                yield event
+            agen = self.root_render_func(ctx)
+            try:
+                async for event in agen:  # type: ignore
+                    yield event
+            finally:
+                # we can't use async with aclosing(...) because that's only
+                # in 3.10+
+                await agen.aclose()  # type: ignore
         except Exception:
             yield self.environment.handle_exception()