From 3841a10e2c3f92f742551b0612eaa3a39d68b417 Mon Sep 17 00:00:00 2001 From: Daverball Date: Sat, 9 Dec 2023 17:43:30 +0100 Subject: [PATCH 1/3] gevent: Use `TypeVarTuple` instead of old `ParamSpec` workaround. --- stubs/gevent/@tests/stubtest_allowlist.txt | 8 ------- stubs/gevent/gevent/_ffi/loop.pyi | 11 ++++------ stubs/gevent/gevent/_ffi/watcher.pyi | 22 ++++++++----------- stubs/gevent/gevent/_hub_primitives.pyi | 10 ++++----- stubs/gevent/gevent/_types.pyi | 25 ++++++++++------------ stubs/gevent/gevent/baseserver.pyi | 23 +++++++++----------- stubs/gevent/gevent/server.pyi | 4 ++-- 7 files changed, 40 insertions(+), 63 deletions(-) diff --git a/stubs/gevent/@tests/stubtest_allowlist.txt b/stubs/gevent/@tests/stubtest_allowlist.txt index 7ae0546ea3cb..534b1fb95cba 100644 --- a/stubs/gevent/@tests/stubtest_allowlist.txt +++ b/stubs/gevent/@tests/stubtest_allowlist.txt @@ -121,14 +121,6 @@ gevent._config._PositiveValueMixin.validate # internal API implementation detail we don't care about gevent._ffi.watcher.AbstractWatcherType.__new__ -# these are inconsistent due to the ParamSpec hack for positional only callables -gevent._ffi.loop.AbstractLoop.run_callback -gevent._ffi.loop.AbstractLoop.run_callback_threadsafe -gevent._ffi.watcher.watcher.start -gevent._hub_primitives.WaitOperationsGreenlet.cancel_waits_close_and_then -gevent.baseserver.BaseServer.do_close -gevent.baseserver.BaseServer.do_handle - # we don't care about write/writeall allowing a named parameter gevent._fileobjectcommon.FlushingBufferedWriter.write gevent._fileobjectcommon.WriteIsWriteallMixin.write diff --git a/stubs/gevent/gevent/_ffi/loop.pyi b/stubs/gevent/gevent/_ffi/loop.pyi index a73068044ee1..b06e90c2a46d 100644 --- a/stubs/gevent/gevent/_ffi/loop.pyi +++ b/stubs/gevent/gevent/_ffi/loop.pyi @@ -3,11 +3,11 @@ from _typeshed import FileDescriptor from collections.abc import Callable from types import TracebackType from typing import Protocol -from typing_extensions import ParamSpec, TypeAlias +from typing_extensions import TypeAlias, TypeVarTuple, Unpack from gevent._types import _AsyncWatcher, _Callback, _ChildWatcher, _IoWatcher, _StatWatcher, _TimerWatcher, _Watcher -_P = ParamSpec("_P") +_Ts = TypeVarTuple("_Ts") _ErrorHandlerFunc: TypeAlias = Callable[ [object | None, type[BaseException] | None, BaseException | None, TracebackType | None], object ] @@ -74,10 +74,7 @@ class AbstractLoop: def async_(self, ref: bool = True, priority: int | None = None) -> _AsyncWatcher: ... def stat(self, path: str, interval: float = 0.0, ref: bool = True, priority: bool | None = ...) -> _StatWatcher: ... - # These technically don't allow the functions arguments to be passed in as kwargs - # but there's no way to express that yet with ParamSpec, however, we would still like - # to verify that the arguments match - def run_callback(self, func: Callable[_P, object], *args: _P.args, **_: _P.kwargs) -> _Callback: ... - def run_callback_threadsafe(self, func: Callable[_P, object], *args: _P.args, **_: _P.kwargs) -> _Callback: ... + def run_callback(self, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> _Callback: ... + def run_callback_threadsafe(self, func: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> _Callback: ... def callback(self, priority: float | None = ...) -> _Callback: ... def fileno(self) -> FileDescriptor | None: ... diff --git a/stubs/gevent/gevent/_ffi/watcher.pyi b/stubs/gevent/gevent/_ffi/watcher.pyi index 70b976657371..9b1e36f762fa 100644 --- a/stubs/gevent/gevent/_ffi/watcher.pyi +++ b/stubs/gevent/gevent/_ffi/watcher.pyi @@ -2,11 +2,11 @@ from _typeshed import FileDescriptor, StrOrBytesPath from collections.abc import Callable from types import TracebackType from typing import Any, overload -from typing_extensions import Concatenate, Literal, ParamSpec, Self +from typing_extensions import Literal, Self, TypeVarTuple, Unpack from gevent._types import _Loop, _StatResult -_P = ParamSpec("_P") +_Ts = TypeVarTuple("_Ts") class AbstractWatcherType(type): def new_handle(cls, obj: object) -> int: ... @@ -22,7 +22,7 @@ class watcher(metaclass=AbstractWatcherType): def ref(self) -> bool: ... callback: Callable[..., Any] args: tuple[Any, ...] - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... def stop(self) -> None: ... @property def priority(self) -> int | None: ... @@ -36,27 +36,23 @@ class watcher(metaclass=AbstractWatcherType): class IoMixin: EVENT_MASK: int def __init__(self, loop: _Loop, fd: FileDescriptor, events: int, ref: bool = True, priority: int | None = None) -> None: ... - # pass_events means the first argument of the callback needs to be an integer, but we can't - # type check the other passed in args in this case @overload - def start(self, callback: Callable[Concatenate[int, _P], Any], *args: Any, pass_events: Literal[True]) -> None: ... + def start(self, callback: Callable[[int, Unpack[_Ts]], Any], *args: Unpack[_Ts], pass_events: Literal[True]) -> None: ... @overload - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... class TimerMixin: def __init__( self, loop: _Loop, after: float = 0.0, repeat: float = 0.0, ref: bool = True, priority: int | None = None ) -> None: ... - # this has one specific allowed keyword argument, if it is given we don't try to check - # the passed in arguments, but if it isn't passed in, then we do. @overload - def start(self, callback: Callable[..., Any], *args: Any, update: bool) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], update: bool) -> None: ... @overload - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> None: ... @overload - def again(self, callback: Callable[..., Any], *args: Any, update: bool) -> None: ... + def again(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts], update: bool) -> None: ... @overload - def again(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def again(self, callback: Callable[[Unpack[_Ts]], object], *args: Unpack[_Ts]) -> None: ... class SignalMixin: def __init__(self, loop: _Loop, signalnum: int, ref: bool = True, priority: int | None = None) -> None: ... diff --git a/stubs/gevent/gevent/_hub_primitives.pyi b/stubs/gevent/gevent/_hub_primitives.pyi index 94f11da3002f..f6efa4610481 100644 --- a/stubs/gevent/gevent/_hub_primitives.pyi +++ b/stubs/gevent/gevent/_hub_primitives.pyi @@ -2,7 +2,7 @@ from _typeshed import FileDescriptor from collections.abc import Callable, Collection, Iterable from types import TracebackType from typing import Any, Generic, Protocol, TypeVar, overload -from typing_extensions import ParamSpec, Self +from typing_extensions import Self, TypeVarTuple, Unpack from gevent._greenlet_primitives import SwitchOutGreenletWithLoop from gevent._types import _Loop, _Watcher @@ -12,8 +12,8 @@ from gevent.socket import socket __all__ = ["WaitOperationsGreenlet", "iwait_on_objects", "wait_on_objects", "wait_read", "wait_write", "wait_readwrite"] _T = TypeVar("_T") +_Ts = TypeVarTuple("_Ts") _WaitableT = TypeVar("_WaitableT", bound=_Waitable) -_P = ParamSpec("_P") class _Waitable(Protocol): def rawlink(self, __callback: Callable[[Any], object]) -> object: ... @@ -22,14 +22,12 @@ class _Waitable(Protocol): class WaitOperationsGreenlet(SwitchOutGreenletWithLoop): loop: _Loop def wait(self, watcher: _Watcher) -> None: ... - # These then doesn't allow keyword arguments, but ParamSpec doesn't allow for that def cancel_waits_close_and_then( self, watchers: Iterable[_Watcher], exc_kind: type[BaseException] | BaseException, - then: Callable[_P, object], - *then_args: _P.args, - **_: _P.kwargs, + then: Callable[[Unpack[_Ts]], object], + *then_args: Unpack[_Ts], ) -> None: ... def cancel_wait(self, watcher: _Watcher, error: type[BaseException] | BaseException, close_watcher: bool = False) -> None: ... diff --git a/stubs/gevent/gevent/_types.pyi b/stubs/gevent/gevent/_types.pyi index bc633d3f234e..e790e7e54b38 100644 --- a/stubs/gevent/gevent/_types.pyi +++ b/stubs/gevent/gevent/_types.pyi @@ -3,9 +3,9 @@ from _typeshed import FileDescriptor, StrOrBytesPath from collections.abc import Callable from types import TracebackType from typing import Any, Protocol, overload -from typing_extensions import Concatenate, Literal, ParamSpec, TypeAlias +from typing_extensions import Literal, TypeAlias, TypeVarTuple, Unpack -_P = ParamSpec("_P") +_Ts = TypeVarTuple("_Ts") # gevent uses zope.interface interanlly which does not work well with type checkers # partially due to the odd call signatures without self and partially due to them @@ -62,16 +62,13 @@ class _Loop(Protocol): # noqa: Y046 def async_(self, ref: bool = True, priority: int | None = None) -> _AsyncWatcher: ... def stat(self, path: str, interval: float = 0.0, ref: bool = True, priority: bool | None = ...) -> _StatWatcher: ... - # These technically don't allow the functions arguments to be passed in as kwargs - # but there's no way to express that yet with ParamSpec, however, we would still like - # to verify that the arguments match - def run_callback(self, func: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> _Callback: ... - def run_callback_threadsafe(self, func: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> _Callback: ... + def run_callback(self, func: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> _Callback: ... + def run_callback_threadsafe(self, func: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> _Callback: ... def fileno(self) -> FileDescriptor | None: ... class _Watcher(Protocol): # while IWatcher allows for kwargs the actual implementation does not... - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... def stop(self) -> None: ... def close(self) -> None: ... @@ -80,13 +77,13 @@ class _TimerWatcher(_Watcher, Protocol): # this has one specific allowed keyword argument, if it is given we don't try to check # the passed in arguments, but if it isn't passed in, then we do. @overload - def start(self, callback: Callable[..., Any], *args: Any, update: bool) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts], update: bool) -> None: ... @overload - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... @overload - def again(self, callback: Callable[..., Any], *args: Any, update: bool) -> None: ... + def again(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts], update: bool) -> None: ... @overload - def again(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def again(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... # this matches Intersection[_Watcher, IoMixin] class _IoWatcher(_Watcher, Protocol): @@ -94,9 +91,9 @@ class _IoWatcher(_Watcher, Protocol): # pass_events means the first argument of the callback needs to be an integer, but we can't # type check the other passed in args in this case @overload - def start(self, callback: Callable[Concatenate[int, _P], Any], *args: Any, pass_events: Literal[True]) -> None: ... + def start(self, callback: Callable[[int, Unpack[_Ts]], Any], *args: Unpack[_Ts], pass_events: Literal[True]) -> None: ... @overload - def start(self, callback: Callable[_P, Any], *args: _P.args, **_: _P.kwargs) -> None: ... + def start(self, callback: Callable[[Unpack[_Ts]], Any], *args: Unpack[_Ts]) -> None: ... # this matches Intersection[_Watcher, ChildMixin] class _ChildWatcher(_Watcher, Protocol): diff --git a/stubs/gevent/gevent/baseserver.pyi b/stubs/gevent/gevent/baseserver.pyi index 74ae39294f27..58247ac66f69 100644 --- a/stubs/gevent/gevent/baseserver.pyi +++ b/stubs/gevent/gevent/baseserver.pyi @@ -1,13 +1,14 @@ from collections.abc import Callable, Container from types import TracebackType -from typing import Any, Generic, Protocol -from typing_extensions import Literal, ParamSpec, Self, TypeAlias +from typing import Generic, Protocol +from typing_extensions import Literal, ParamSpec, Self, TypeAlias, TypeVarTuple, Unpack from gevent._types import _Loop from gevent.pool import Pool from gevent.socket import socket as _GeventSocket from greenlet import greenlet +_Ts = TypeVarTuple("_Ts") _P = ParamSpec("_P") class _SpawnFunc(Protocol): @@ -15,7 +16,7 @@ class _SpawnFunc(Protocol): _Spawner: TypeAlias = Pool | _SpawnFunc | int | Literal["default"] | None -class BaseServer(Generic[_P]): +class BaseServer(Generic[Unpack[_Ts]]): min_delay: float max_delay: float max_accept: int @@ -27,27 +28,23 @@ class BaseServer(Generic[_P]): family: int address: str | tuple[str, int] socket: _GeventSocket - handle: Callable[..., object] + handle: Callable[[Unpack[_Ts]], object] def __init__( self, listener: _GeventSocket | tuple[str, int] | str, - handle: Callable[_P, object] | None = None, + handle: Callable[[Unpack[_Ts]], object] | None = None, spawn: _Spawner = "default", ) -> None: ... def __enter__(self) -> Self: ... def __exit__(self, __typ: type[BaseException] | None, __value: BaseException | None, __tb: TracebackType | None) -> None: ... def set_listener(self, listener: _GeventSocket | tuple[str, int] | str) -> None: ... def set_spawn(self, spawn: _Spawner) -> None: ... - def set_handle(self, handle: Callable[_P, object]) -> None: ... + def set_handle(self, handle: Callable[[Unpack[_Ts]], object]) -> None: ... def start_accepting(self) -> None: ... def stop_accepting(self) -> None: ... - # neither of these accept keyword arguments, but if we omit them, then ParamSpec - # won't match the arguments correctly - def do_handle(self, *args: _P.args, **_: _P.kwargs) -> None: ... - def do_close(self, *args: _P.args, **_: _P.kwargs) -> None: ... - # we would like to return _P.args here, however pyright will complain - # mypy doesn't seem to mind - def do_read(self) -> tuple[Any, ...] | None: ... + def do_handle(self, *args: Unpack[_Ts]) -> None: ... + def do_close(self, *args: Unpack[_Ts]) -> None: ... + def do_read(self) -> tuple[Unpack[_Ts]] | None: ... def full(self) -> bool: ... @property def server_host(self) -> str | None: ... diff --git a/stubs/gevent/gevent/server.pyi b/stubs/gevent/gevent/server.pyi index cce84c07e4c5..53663f71abd8 100644 --- a/stubs/gevent/gevent/server.pyi +++ b/stubs/gevent/gevent/server.pyi @@ -23,7 +23,7 @@ class _SSLArguments(TypedDict, total=False): do_handshake_on_connect: bool ciphers: str -class StreamServer(BaseServer[[_GeventSocket, _Address]]): +class StreamServer(BaseServer[_GeventSocket, _Address]): backlog: int reuse_addr: ClassVar[int | None] wrap_socket = ssl_wrap_socket @@ -67,7 +67,7 @@ class StreamServer(BaseServer[[_GeventSocket, _Address]]): def do_close(self, sock: _GeventSocket, address: _Address) -> None: ... def wrap_socket_and_handle(self, client_socket: _GeventSocket, address: _StrictAddress) -> Any: ... -class DatagramServer(BaseServer[[_GeventSocket, _Address]]): +class DatagramServer(BaseServer[_GeventSocket, _Address]): reuse_addr: ClassVar[int | None] def __init__( self, From 14ce6bb615613bd30761c014de1e65b76b83ea23 Mon Sep 17 00:00:00 2001 From: Daverball Date: Sat, 9 Dec 2023 18:12:39 +0100 Subject: [PATCH 2/3] Reverts changes to `BaseServer` --- stubs/gevent/gevent/baseserver.pyi | 23 +++++++++++++---------- stubs/gevent/gevent/server.pyi | 4 ++-- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/stubs/gevent/gevent/baseserver.pyi b/stubs/gevent/gevent/baseserver.pyi index 58247ac66f69..74ae39294f27 100644 --- a/stubs/gevent/gevent/baseserver.pyi +++ b/stubs/gevent/gevent/baseserver.pyi @@ -1,14 +1,13 @@ from collections.abc import Callable, Container from types import TracebackType -from typing import Generic, Protocol -from typing_extensions import Literal, ParamSpec, Self, TypeAlias, TypeVarTuple, Unpack +from typing import Any, Generic, Protocol +from typing_extensions import Literal, ParamSpec, Self, TypeAlias from gevent._types import _Loop from gevent.pool import Pool from gevent.socket import socket as _GeventSocket from greenlet import greenlet -_Ts = TypeVarTuple("_Ts") _P = ParamSpec("_P") class _SpawnFunc(Protocol): @@ -16,7 +15,7 @@ class _SpawnFunc(Protocol): _Spawner: TypeAlias = Pool | _SpawnFunc | int | Literal["default"] | None -class BaseServer(Generic[Unpack[_Ts]]): +class BaseServer(Generic[_P]): min_delay: float max_delay: float max_accept: int @@ -28,23 +27,27 @@ class BaseServer(Generic[Unpack[_Ts]]): family: int address: str | tuple[str, int] socket: _GeventSocket - handle: Callable[[Unpack[_Ts]], object] + handle: Callable[..., object] def __init__( self, listener: _GeventSocket | tuple[str, int] | str, - handle: Callable[[Unpack[_Ts]], object] | None = None, + handle: Callable[_P, object] | None = None, spawn: _Spawner = "default", ) -> None: ... def __enter__(self) -> Self: ... def __exit__(self, __typ: type[BaseException] | None, __value: BaseException | None, __tb: TracebackType | None) -> None: ... def set_listener(self, listener: _GeventSocket | tuple[str, int] | str) -> None: ... def set_spawn(self, spawn: _Spawner) -> None: ... - def set_handle(self, handle: Callable[[Unpack[_Ts]], object]) -> None: ... + def set_handle(self, handle: Callable[_P, object]) -> None: ... def start_accepting(self) -> None: ... def stop_accepting(self) -> None: ... - def do_handle(self, *args: Unpack[_Ts]) -> None: ... - def do_close(self, *args: Unpack[_Ts]) -> None: ... - def do_read(self) -> tuple[Unpack[_Ts]] | None: ... + # neither of these accept keyword arguments, but if we omit them, then ParamSpec + # won't match the arguments correctly + def do_handle(self, *args: _P.args, **_: _P.kwargs) -> None: ... + def do_close(self, *args: _P.args, **_: _P.kwargs) -> None: ... + # we would like to return _P.args here, however pyright will complain + # mypy doesn't seem to mind + def do_read(self) -> tuple[Any, ...] | None: ... def full(self) -> bool: ... @property def server_host(self) -> str | None: ... diff --git a/stubs/gevent/gevent/server.pyi b/stubs/gevent/gevent/server.pyi index 53663f71abd8..cce84c07e4c5 100644 --- a/stubs/gevent/gevent/server.pyi +++ b/stubs/gevent/gevent/server.pyi @@ -23,7 +23,7 @@ class _SSLArguments(TypedDict, total=False): do_handshake_on_connect: bool ciphers: str -class StreamServer(BaseServer[_GeventSocket, _Address]): +class StreamServer(BaseServer[[_GeventSocket, _Address]]): backlog: int reuse_addr: ClassVar[int | None] wrap_socket = ssl_wrap_socket @@ -67,7 +67,7 @@ class StreamServer(BaseServer[_GeventSocket, _Address]): def do_close(self, sock: _GeventSocket, address: _Address) -> None: ... def wrap_socket_and_handle(self, client_socket: _GeventSocket, address: _StrictAddress) -> Any: ... -class DatagramServer(BaseServer[_GeventSocket, _Address]): +class DatagramServer(BaseServer[[_GeventSocket, _Address]]): reuse_addr: ClassVar[int | None] def __init__( self, From b1a90844180319200d20537f738cc4d52536df1b Mon Sep 17 00:00:00 2001 From: Daverball Date: Sat, 9 Dec 2023 18:14:39 +0100 Subject: [PATCH 3/3] Fixes stubtest_allowlist --- stubs/gevent/@tests/stubtest_allowlist.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/stubs/gevent/@tests/stubtest_allowlist.txt b/stubs/gevent/@tests/stubtest_allowlist.txt index 534b1fb95cba..c9008a6bfa11 100644 --- a/stubs/gevent/@tests/stubtest_allowlist.txt +++ b/stubs/gevent/@tests/stubtest_allowlist.txt @@ -121,6 +121,10 @@ gevent._config._PositiveValueMixin.validate # internal API implementation detail we don't care about gevent._ffi.watcher.AbstractWatcherType.__new__ +# these are inconsistent due to the ParamSpec hack for positional only callables +gevent.baseserver.BaseServer.do_close +gevent.baseserver.BaseServer.do_handle + # we don't care about write/writeall allowing a named parameter gevent._fileobjectcommon.FlushingBufferedWriter.write gevent._fileobjectcommon.WriteIsWriteallMixin.write