From bf795a12478761c804baf98aa4488ed165689bcc Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 13 Dec 2024 19:09:21 -0800 Subject: [PATCH 1/6] add _GeneratorContextManagerBase.__init__ I think that #6676 showed that Paramspec didn't work, but that wasn't actually the fault of _GeneratorContextManagerBase. --- stdlib/@tests/stubtest_allowlists/common.txt | 1 - stdlib/contextlib.pyi | 27 +++++--------------- 2 files changed, 7 insertions(+), 21 deletions(-) diff --git a/stdlib/@tests/stubtest_allowlists/common.txt b/stdlib/@tests/stubtest_allowlists/common.txt index 6c3e3086f9e4..83df7d8cfdd2 100644 --- a/stdlib/@tests/stubtest_allowlists/common.txt +++ b/stdlib/@tests/stubtest_allowlists/common.txt @@ -12,7 +12,6 @@ asyncio.BaseEventLoop.subprocess_exec # BaseEventLoop adds several parameters a asyncio.base_events.BaseEventLoop.subprocess_exec # BaseEventLoop adds several parameters and stubtest fails on the difference if we add them builtins.dict.get collections\.ChainMap\.fromkeys # https://github.com/python/mypy/issues/17023 -contextlib._GeneratorContextManagerBase.__init__ # skipped in the stubs in favor of its child classes ctypes.CDLL._FuncPtr # None at class level but initialized in __init__ to this value ctypes.memmove # CFunctionType ctypes.memset # CFunctionType diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index dc5d926775f3..d347903d2621 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -2,7 +2,7 @@ import abc import sys from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator +from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable from typing_extensions import ParamSpec, Self, TypeAlias @@ -64,16 +64,15 @@ class ContextDecorator: def _recreate_cm(self) -> Self: ... def __call__(self, func: _F) -> _F: ... -class _GeneratorContextManagerBase: ... - -class _GeneratorContextManager(_GeneratorContextManagerBase, AbstractContextManager[_T_co, bool | None], ContextDecorator): - # __init__ and all instance attributes are actually inherited from _GeneratorContextManagerBase - # adding them there is more trouble than it's worth to include in the stub; see #6676 +class _GeneratorContextManagerBase(Generic[_T_co]): + # Ideally this would use Paramspec, but that requires (*args, **kwargs), which this isn't. see #6676 def __init__(self, func: Callable[..., Iterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... gen: Generator[_T_co, Any, Any] func: Callable[..., Generator[_T_co, Any, Any]] args: tuple[Any, ...] kwds: dict[str, Any] + +class _GeneratorContextManager(_GeneratorContextManagerBase[_T_co], AbstractContextManager[_T_co, bool | None], ContextDecorator): if sys.version_info >= (3, 9): def __exit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None @@ -93,26 +92,14 @@ if sys.version_info >= (3, 10): def __call__(self, func: _AF) -> _AF: ... class _AsyncGeneratorContextManager( - _GeneratorContextManagerBase, AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator + _GeneratorContextManagerBase[_T_co], AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator ): - # __init__ and these attributes are actually defined in the base class _GeneratorContextManagerBase, - # adding them there is more trouble than it's worth to include in the stub (see #6676) - def __init__(self, func: Callable[..., AsyncIterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... - gen: AsyncGenerator[_T_co, Any] - func: Callable[..., AsyncGenerator[_T_co, Any]] - args: tuple[Any, ...] - kwds: dict[str, Any] async def __aexit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... else: - class _AsyncGeneratorContextManager(_GeneratorContextManagerBase, AbstractAsyncContextManager[_T_co, bool | None]): - def __init__(self, func: Callable[..., AsyncIterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... - gen: AsyncGenerator[_T_co, Any] - func: Callable[..., AsyncGenerator[_T_co, Any]] - args: tuple[Any, ...] - kwds: dict[str, Any] + class _AsyncGeneratorContextManager(_GeneratorContextManagerBase[_T_co], AbstractAsyncContextManager[_T_co, bool | None]): async def __aexit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... From 8180e7647a1907dc4fba30bbe5d914dc484ff54e Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 13 Dec 2024 19:32:03 -0800 Subject: [PATCH 2/6] generic over the generator --- stdlib/contextlib.pyi | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index d347903d2621..ce4caafba9d7 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -2,7 +2,7 @@ import abc import sys from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Generator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable from typing_extensions import ParamSpec, Self, TypeAlias @@ -33,6 +33,7 @@ _T_co = TypeVar("_T_co", covariant=True) _T_io = TypeVar("_T_io", bound=IO[str] | None) _ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None) _F = TypeVar("_F", bound=Callable[..., Any]) +_G = TypeVar("_G", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any]) _P = ParamSpec("_P") _ExitFunc: TypeAlias = Callable[[type[BaseException] | None, BaseException | None, TracebackType | None], bool | None] @@ -64,15 +65,17 @@ class ContextDecorator: def _recreate_cm(self) -> Self: ... def __call__(self, func: _F) -> _F: ... -class _GeneratorContextManagerBase(Generic[_T_co]): +class _GeneratorContextManagerBase(Generic[_G]): # Ideally this would use Paramspec, but that requires (*args, **kwargs), which this isn't. see #6676 - def __init__(self, func: Callable[..., Iterator[_T_co]], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... - gen: Generator[_T_co, Any, Any] - func: Callable[..., Generator[_T_co, Any, Any]] + def __init__(self, func: Callable[..., _G], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... + gen: _G + func: Callable[..., _G] args: tuple[Any, ...] kwds: dict[str, Any] -class _GeneratorContextManager(_GeneratorContextManagerBase[_T_co], AbstractContextManager[_T_co, bool | None], ContextDecorator): +class _GeneratorContextManager( + _GeneratorContextManagerBase[Generator[_T_co, Any, Any]], AbstractContextManager[_T_co, bool | None], ContextDecorator +): if sys.version_info >= (3, 9): def __exit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None @@ -92,14 +95,18 @@ if sys.version_info >= (3, 10): def __call__(self, func: _AF) -> _AF: ... class _AsyncGeneratorContextManager( - _GeneratorContextManagerBase[_T_co], AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator + _GeneratorContextManagerBase[AsyncGenerator[_T_co, Any]], + AbstractAsyncContextManager[_T_co, bool | None], + AsyncContextDecorator, ): async def __aexit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... else: - class _AsyncGeneratorContextManager(_GeneratorContextManagerBase[_T_co], AbstractAsyncContextManager[_T_co, bool | None]): + class _AsyncGeneratorContextManager( + _GeneratorContextManagerBase[AsyncGenerator[_T_co, Any]], AbstractAsyncContextManager[_T_co, bool | None] + ): async def __aexit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... From 4f834195c12731ba78f4ba564379ddd8e9be25fb Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 13 Dec 2024 20:47:30 -0800 Subject: [PATCH 3/6] variance --- stdlib/contextlib.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index ce4caafba9d7..ca0d59b8b500 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -33,7 +33,7 @@ _T_co = TypeVar("_T_co", covariant=True) _T_io = TypeVar("_T_io", bound=IO[str] | None) _ExitT_co = TypeVar("_ExitT_co", covariant=True, bound=bool | None, default=bool | None) _F = TypeVar("_F", bound=Callable[..., Any]) -_G = TypeVar("_G", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any]) +_G = TypeVar("_G", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any], covariant=True) _P = ParamSpec("_P") _ExitFunc: TypeAlias = Callable[[type[BaseException] | None, BaseException | None, TracebackType | None], bool | None] From 73661f6c07ee05462a22cd7ac49047adda5a1967 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 13 Dec 2024 21:33:32 -0800 Subject: [PATCH 4/6] add the extra type variables --- stdlib/contextlib.pyi | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index ca0d59b8b500..2d23536b815c 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -2,7 +2,7 @@ import abc import sys from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator from types import TracebackType from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable from typing_extensions import ParamSpec, Self, TypeAlias @@ -36,6 +36,9 @@ _F = TypeVar("_F", bound=Callable[..., Any]) _G = TypeVar("_G", bound=Generator[Any, Any, Any] | AsyncGenerator[Any, Any], covariant=True) _P = ParamSpec("_P") +_SendT_contra = TypeVar("_SendT_contra", contravariant=True, default=None) +_ReturnT_co = TypeVar("_ReturnT_co", covariant=True, default=None) + _ExitFunc: TypeAlias = Callable[[type[BaseException] | None, BaseException | None, TracebackType | None], bool | None] _CM_EF = TypeVar("_CM_EF", bound=AbstractContextManager[Any, Any] | _ExitFunc) @@ -74,7 +77,9 @@ class _GeneratorContextManagerBase(Generic[_G]): kwds: dict[str, Any] class _GeneratorContextManager( - _GeneratorContextManagerBase[Generator[_T_co, Any, Any]], AbstractContextManager[_T_co, bool | None], ContextDecorator + _GeneratorContextManagerBase[Generator[_T_co, _SendT_contra, _ReturnT_co]], + AbstractContextManager[_T_co, bool | None], + ContextDecorator, ): if sys.version_info >= (3, 9): def __exit__( @@ -85,7 +90,9 @@ class _GeneratorContextManager( self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... -def contextmanager(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, _GeneratorContextManager[_T_co]]: ... +def contextmanager( + func: Callable[_P, Generator[_T_co, _SendT_contra, _ReturnT_co]] +) -> Callable[_P, _GeneratorContextManager[_T_co, _SendT_contra, _ReturnT_co]]: ... if sys.version_info >= (3, 10): _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]]) @@ -95,7 +102,7 @@ if sys.version_info >= (3, 10): def __call__(self, func: _AF) -> _AF: ... class _AsyncGeneratorContextManager( - _GeneratorContextManagerBase[AsyncGenerator[_T_co, Any]], + _GeneratorContextManagerBase[AsyncGenerator[_T_co, _SendT_contra]], AbstractAsyncContextManager[_T_co, bool | None], AsyncContextDecorator, ): @@ -105,7 +112,7 @@ if sys.version_info >= (3, 10): else: class _AsyncGeneratorContextManager( - _GeneratorContextManagerBase[AsyncGenerator[_T_co, Any]], AbstractAsyncContextManager[_T_co, bool | None] + _GeneratorContextManagerBase[AsyncGenerator[_T_co, _SendT_contra]], AbstractAsyncContextManager[_T_co, bool | None] ): async def __aexit__( self, typ: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None From 01ee3d539c925474f8fa662eeff8aadbf27aa9a3 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Fri, 13 Dec 2024 23:15:51 -0800 Subject: [PATCH 5/6] don't mess with contextmanager --- stdlib/contextlib.pyi | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index 2d23536b815c..f5d7160166da 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -2,7 +2,7 @@ import abc import sys from _typeshed import FileDescriptorOrPath, Unused from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator +from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Iterator from types import TracebackType from typing import IO, Any, Generic, Protocol, TypeVar, overload, runtime_checkable from typing_extensions import ParamSpec, Self, TypeAlias @@ -90,9 +90,7 @@ class _GeneratorContextManager( self, type: type[BaseException] | None, value: BaseException | None, traceback: TracebackType | None ) -> bool | None: ... -def contextmanager( - func: Callable[_P, Generator[_T_co, _SendT_contra, _ReturnT_co]] -) -> Callable[_P, _GeneratorContextManager[_T_co, _SendT_contra, _ReturnT_co]]: ... +def contextmanager(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, _GeneratorContextManager[_T_co]]: ... if sys.version_info >= (3, 10): _AF = TypeVar("_AF", bound=Callable[..., Awaitable[Any]]) From 1602f86d81f08f3b1841dad67c8efc3a711d1173 Mon Sep 17 00:00:00 2001 From: Stephen Morton Date: Sat, 14 Dec 2024 13:40:02 -0800 Subject: [PATCH 6/6] Update stdlib/contextlib.pyi Co-authored-by: Jelle Zijlstra --- stdlib/contextlib.pyi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stdlib/contextlib.pyi b/stdlib/contextlib.pyi index f5d7160166da..f57e7fa67036 100644 --- a/stdlib/contextlib.pyi +++ b/stdlib/contextlib.pyi @@ -69,7 +69,7 @@ class ContextDecorator: def __call__(self, func: _F) -> _F: ... class _GeneratorContextManagerBase(Generic[_G]): - # Ideally this would use Paramspec, but that requires (*args, **kwargs), which this isn't. see #6676 + # Ideally this would use ParamSpec, but that requires (*args, **kwargs), which this isn't. see #6676 def __init__(self, func: Callable[..., _G], args: tuple[Any, ...], kwds: dict[str, Any]) -> None: ... gen: _G func: Callable[..., _G]