From 6a6d0050f475e0922c2494fa983f025ff64e6e69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Francis=20Clairicia-Rose-Claire-Jos=C3=A9phine?= Date: Sat, 29 Jun 2024 13:00:22 +0200 Subject: [PATCH] Low-level API: Lazy asyncio import (#313) --- .../api_async/backend/_asyncio/backend.py | 112 +++++++++++------- .../test_asyncio_backend/test_backend.py | 57 ++++----- 2 files changed, 99 insertions(+), 70 deletions(-) diff --git a/src/easynetwork/lowlevel/api_async/backend/_asyncio/backend.py b/src/easynetwork/lowlevel/api_async/backend/_asyncio/backend.py index 19d8b831..7c47cd69 100644 --- a/src/easynetwork/lowlevel/api_async/backend/_asyncio/backend.py +++ b/src/easynetwork/lowlevel/api_async/backend/_asyncio/backend.py @@ -19,8 +19,6 @@ __all__ = ["AsyncIOBackend"] -import asyncio -import asyncio.base_events import contextvars import functools import math @@ -32,21 +30,9 @@ from .... import _utils from ....constants import HAPPY_EYEBALLS_DELAY as _DEFAULT_HAPPY_EYEBALLS_DELAY +from ...transports.abc import AsyncDatagramListener, AsyncDatagramTransport, AsyncListener, AsyncStreamTransport from .. import _sniffio_helpers -from ..abc import AsyncBackend as AbstractAsyncBackend, ILock, TaskInfo -from ._asyncio_utils import ( - create_connection, - create_datagram_connection, - open_listener_sockets_from_getaddrinfo_result, - resolve_local_addresses, -) -from .datagram.endpoint import create_datagram_endpoint -from .datagram.listener import DatagramListenerSocketAdapter -from .datagram.socket import AsyncioTransportDatagramSocketAdapter -from .stream.listener import AcceptedSocketFactory, ListenerSocketAdapter -from .stream.socket import AsyncioTransportStreamSocketAdapter, StreamReaderBufferedProtocol -from .tasks import CancelScope, TaskGroup, TaskUtils -from .threads import ThreadsPortal +from ..abc import AsyncBackend as AbstractAsyncBackend, CancelScope, ICondition, IEvent, ILock, TaskGroup, TaskInfo, ThreadsPortal _P = ParamSpec("_P") _T = TypeVar("_T") @@ -55,7 +41,23 @@ class AsyncIOBackend(AbstractAsyncBackend): - __slots__ = () + __slots__ = ( + "__asyncio", + "__coro_yield", + "__cancel_shielded_coro_yield", + "__cancel_shielded_await", + ) + + def __init__(self) -> None: + import asyncio + + from .tasks import TaskUtils + + self.__asyncio = asyncio + + self.__coro_yield = TaskUtils.coro_yield + self.__cancel_shielded_coro_yield = TaskUtils.cancel_shielded_coro_yield + self.__cancel_shielded_await = TaskUtils.cancel_shielded_await def bootstrap( self, @@ -63,40 +65,46 @@ def bootstrap( *args: *_T_PosArgs, runner_options: Mapping[str, Any] | None = None, ) -> _T: - with asyncio.Runner(**(runner_options or {})) as runner: + with self.__asyncio.Runner(**(runner_options or {})) as runner: return runner.run(coro_func(*args)) async def coro_yield(self) -> None: - await TaskUtils.coro_yield() + await self.__coro_yield() async def cancel_shielded_coro_yield(self) -> None: - await TaskUtils.cancel_shielded_coro_yield() + await self.__cancel_shielded_coro_yield() def get_cancelled_exc_class(self) -> type[BaseException]: - return asyncio.CancelledError + return self.__asyncio.CancelledError async def ignore_cancellation(self, coroutine: Awaitable[_T_co]) -> _T_co: - return await TaskUtils.cancel_shielded_await(coroutine) + return await self.__cancel_shielded_await(coroutine) def open_cancel_scope(self, *, deadline: float = math.inf) -> CancelScope: + from .tasks import CancelScope + return CancelScope(deadline=deadline) def current_time(self) -> float: - loop = asyncio.get_running_loop() + loop = self.__asyncio.get_running_loop() return loop.time() async def sleep(self, delay: float) -> None: - await asyncio.sleep(delay) + await self.__asyncio.sleep(delay) async def sleep_forever(self) -> NoReturn: - loop = asyncio.get_running_loop() + loop = self.__asyncio.get_running_loop() await loop.create_future() raise AssertionError("await an unused future cannot end in any other way than by cancellation") def create_task_group(self) -> TaskGroup: + from .tasks import TaskGroup + return TaskGroup() def get_current_task(self) -> TaskInfo: + from .tasks import TaskUtils + current_task = TaskUtils.current_asyncio_task() return TaskUtils.create_task_info(current_task) @@ -107,10 +115,12 @@ async def create_tcp_connection( *, local_address: tuple[str, int] | None = None, happy_eyeballs_delay: float | None = None, - ) -> AsyncioTransportStreamSocketAdapter: + ) -> AsyncStreamTransport: if happy_eyeballs_delay is None: happy_eyeballs_delay = _DEFAULT_HAPPY_EYEBALLS_DELAY + from ._asyncio_utils import create_connection + socket = await create_connection( host, port, @@ -120,9 +130,11 @@ async def create_tcp_connection( return await self.wrap_stream_socket(socket) - async def wrap_stream_socket(self, socket: _socket.socket) -> AsyncioTransportStreamSocketAdapter: + async def wrap_stream_socket(self, socket: _socket.socket) -> AsyncStreamTransport: + from .stream.socket import AsyncioTransportStreamSocketAdapter, StreamReaderBufferedProtocol + socket.setblocking(False) - loop = asyncio.get_running_loop() + loop = self.__asyncio.get_running_loop() transport, protocol = await loop.create_connection( _utils.make_callback(StreamReaderBufferedProtocol, loop=loop), sock=socket, @@ -136,10 +148,13 @@ async def create_tcp_listeners( backlog: int, *, reuse_port: bool = False, - ) -> Sequence[ListenerSocketAdapter[AsyncioTransportStreamSocketAdapter]]: + ) -> Sequence[AsyncListener[AsyncStreamTransport]]: if not isinstance(backlog, int): raise TypeError("backlog: Expected an integer") + from ._asyncio_utils import open_listener_sockets_from_getaddrinfo_result, resolve_local_addresses + from .stream.listener import AcceptedSocketFactory, ListenerSocketAdapter + reuse_address: bool = os.name not in ("nt", "cygwin") and sys.platform != "cygwin" hosts: Sequence[str | None] if host == "" or host is None: @@ -165,7 +180,8 @@ async def create_tcp_listeners( ) factory = AcceptedSocketFactory() - return [ListenerSocketAdapter(self, sock, factory) for sock in sockets] + listeners = [ListenerSocketAdapter(self, sock, factory) for sock in sockets] + return listeners async def create_udp_endpoint( self, @@ -174,7 +190,9 @@ async def create_udp_endpoint( *, local_address: tuple[str, int] | None = None, family: int = _socket.AF_UNSPEC, - ) -> AsyncioTransportDatagramSocketAdapter: + ) -> AsyncDatagramTransport: + from ._asyncio_utils import create_datagram_connection + socket = await create_datagram_connection( remote_host, remote_port, @@ -183,7 +201,10 @@ async def create_udp_endpoint( ) return await self.wrap_connected_datagram_socket(socket) - async def wrap_connected_datagram_socket(self, socket: _socket.socket) -> AsyncioTransportDatagramSocketAdapter: + async def wrap_connected_datagram_socket(self, socket: _socket.socket) -> AsyncDatagramTransport: + from .datagram.endpoint import create_datagram_endpoint + from .datagram.socket import AsyncioTransportDatagramSocketAdapter + socket.setblocking(False) endpoint = await create_datagram_endpoint(sock=socket) return AsyncioTransportDatagramSocketAdapter(self, endpoint) @@ -194,10 +215,11 @@ async def create_udp_listeners( port: int, *, reuse_port: bool = False, - ) -> Sequence[DatagramListenerSocketAdapter]: - from .datagram.listener import DatagramListenerProtocol + ) -> Sequence[AsyncDatagramListener[tuple[Any, ...]]]: + from ._asyncio_utils import open_listener_sockets_from_getaddrinfo_result, resolve_local_addresses + from .datagram.listener import DatagramListenerProtocol, DatagramListenerSocketAdapter - loop = asyncio.get_running_loop() + loop = self.__asyncio.get_running_loop() hosts: Sequence[str | None] if host == "" or host is None: @@ -226,22 +248,24 @@ async def create_udp_listeners( listeners = [await loop.create_datagram_endpoint(protocol_factory, sock=sock) for sock in sockets] return [DatagramListenerSocketAdapter(self, transport, protocol) for transport, protocol in listeners] - def create_lock(self) -> asyncio.Lock: - return asyncio.Lock() + def create_lock(self) -> ILock: + return self.__asyncio.Lock() - def create_event(self) -> asyncio.Event: - return asyncio.Event() + def create_event(self) -> IEvent: + return self.__asyncio.Event() - def create_condition_var(self, lock: ILock | None = None) -> asyncio.Condition: + def create_condition_var(self, lock: ILock | None = None) -> ICondition: if lock is not None: - assert isinstance(lock, asyncio.Lock) # nosec assert_used + assert isinstance(lock, self.__asyncio.Lock) # nosec assert_used - return asyncio.Condition(lock) + return self.__asyncio.Condition(lock) async def run_in_thread(self, func: Callable[_P, _T], /, *args: _P.args, **kwargs: _P.kwargs) -> _T: - loop = asyncio.get_running_loop() + loop = self.__asyncio.get_running_loop() ctx = contextvars.copy_context() + from .tasks import TaskUtils + _sniffio_helpers.setup_sniffio_contextvar(ctx, None) future = loop.run_in_executor(None, functools.partial(ctx.run, func, *args, **kwargs)) @@ -251,4 +275,6 @@ async def run_in_thread(self, func: Callable[_P, _T], /, *args: _P.args, **kwarg del future def create_threads_portal(self) -> ThreadsPortal: + from .threads import ThreadsPortal + return ThreadsPortal() diff --git a/tests/unit_test/test_async/test_asyncio_backend/test_backend.py b/tests/unit_test/test_async/test_asyncio_backend/test_backend.py index 9a46b49b..b38a7688 100644 --- a/tests/unit_test/test_async/test_asyncio_backend/test_backend.py +++ b/tests/unit_test/test_async/test_asyncio_backend/test_backend.py @@ -4,7 +4,7 @@ import contextvars from collections.abc import Callable, Coroutine, Sequence from socket import AF_INET, AF_INET6, AF_UNSPEC, AI_ADDRCONFIG, AI_PASSIVE, IPPROTO_TCP, IPPROTO_UDP, SOCK_DGRAM, SOCK_STREAM -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Final from easynetwork.lowlevel.api_async.backend._asyncio.backend import AsyncIOBackend from easynetwork.lowlevel.api_async.backend._asyncio.datagram.listener import DatagramListenerProtocol @@ -22,6 +22,9 @@ from pytest_mock import MockerFixture +_ASYNCIO_BACKEND_MODULE: Final[str] = "easynetwork.lowlevel.api_async.backend._asyncio" + + class TestAsyncIOBackendSync: @pytest.fixture @staticmethod @@ -218,7 +221,7 @@ async def test____create_tcp_connection____use_loop_create_connection( mock_asyncio_transport = mocker.NonCallableMagicMock(spec=asyncio.Transport) mock_protocol = mocker.NonCallableMagicMock(spec=StreamReaderBufferedProtocol) mock_AsyncioTransportStreamSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.AsyncioTransportStreamSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.socket.AsyncioTransportStreamSocketAdapter", return_value=mocker.sentinel.socket, ) mock_event_loop_create_connection: AsyncMock = mocker.patch.object( @@ -228,7 +231,7 @@ async def test____create_tcp_connection____use_loop_create_connection( return_value=(mock_asyncio_transport, mock_protocol), ) mock_own_create_connection: AsyncMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.create_connection", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.create_connection", new_callable=mocker.AsyncMock, return_value=mock_tcp_socket, ) @@ -268,7 +271,7 @@ async def test____wrap_stream_socket____use_asyncio_open_connection( mock_asyncio_transport = mocker.NonCallableMagicMock(spec=asyncio.Transport) mock_protocol = mocker.NonCallableMagicMock(spec=StreamReaderBufferedProtocol) mock_AsyncioTransportStreamSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.AsyncioTransportStreamSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.socket.AsyncioTransportStreamSocketAdapter", return_value=mocker.sentinel.socket, ) mock_event_loop_create_connection: AsyncMock = mocker.patch.object( @@ -315,11 +318,11 @@ async def test____create_tcp_listeners____open_listener_sockets( return_value=addrinfo_list, ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_tcp_socket], ) mock_ListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.ListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.listener.ListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) expected_factory: AbstractAcceptedSocketFactory[Any] = AcceptedSocketFactory() @@ -384,11 +387,11 @@ async def test____create_tcp_listeners____bind_to_any_interfaces( return_value=addrinfo_list, ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_tcp_socket, mock_tcp_socket], ) mock_ListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.ListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.listener.ListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) expected_factory: AbstractAcceptedSocketFactory[Any] = AcceptedSocketFactory() @@ -454,11 +457,11 @@ async def test____create_tcp_listeners____bind_to_several_hosts( side_effect=[[info] for info in addrinfo_list], ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_tcp_socket, mock_tcp_socket], ) mock_ListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.ListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.listener.ListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) expected_factory: AbstractAcceptedSocketFactory[Any] = AcceptedSocketFactory() @@ -511,11 +514,11 @@ async def test____create_tcp_listeners____error_getaddrinfo_returns_empty_list( return_value=[], ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", side_effect=AssertionError, ) mock_ListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.ListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.listener.ListenerSocketAdapter", side_effect=AssertionError, ) @@ -556,11 +559,11 @@ async def test____create_tcp_listeners____invalid_backlog( return_value=[], ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", side_effect=AssertionError, ) mock_ListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.ListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.stream.listener.ListenerSocketAdapter", side_effect=AssertionError, ) @@ -592,16 +595,16 @@ async def test____create_udp_endpoint____use_loop_create_datagram_endpoint( # Arrange mock_endpoint = mock_datagram_endpoint_factory() mock_AsyncioTransportDatagramSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.AsyncioTransportDatagramSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.socket.AsyncioTransportDatagramSocketAdapter", return_value=mocker.sentinel.socket, ) mock_create_datagram_endpoint: AsyncMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.create_datagram_endpoint", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.endpoint.create_datagram_endpoint", new_callable=mocker.AsyncMock, return_value=mock_endpoint, ) mock_own_create_connection: AsyncMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.create_datagram_connection", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.create_datagram_connection", new_callable=mocker.AsyncMock, return_value=mock_udp_socket, ) @@ -633,11 +636,11 @@ async def test____wrap_connected_datagram_socket____use_loop_create_datagram_end # Arrange mock_endpoint = mock_datagram_endpoint_factory() mock_AsyncioTransportDatagramSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.AsyncioTransportDatagramSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.socket.AsyncioTransportDatagramSocketAdapter", return_value=mocker.sentinel.socket, ) mock_create_datagram_endpoint: AsyncMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.create_datagram_endpoint", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.endpoint.create_datagram_endpoint", new_callable=mocker.AsyncMock, return_value=mock_endpoint, ) @@ -679,7 +682,7 @@ async def test____create_udp_listeners____open_listener_sockets( return_value=addrinfo_list, ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_udp_socket], ) mock_create_datagram_endpoint: AsyncMock = mocker.patch.object( @@ -689,7 +692,7 @@ async def test____create_udp_listeners____open_listener_sockets( return_value=(mock_transport, mock_protocol), ) mock_DatagramListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.DatagramListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.listener.DatagramListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) @@ -758,7 +761,7 @@ async def test____create_udp_listeners____bind_to_local_interfaces( return_value=addrinfo_list, ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_udp_socket, mock_udp_socket], ) mock_create_datagram_endpoint: AsyncMock = mocker.patch.object( @@ -768,7 +771,7 @@ async def test____create_udp_listeners____bind_to_local_interfaces( return_value=(mock_transport, mock_protocol), ) mock_DatagramListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.DatagramListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.listener.DatagramListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) @@ -841,7 +844,7 @@ async def test____create_udp_listeners____bind_to_several_hosts( return_value=addrinfo_list, ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", return_value=[mock_udp_socket, mock_udp_socket], ) mock_create_datagram_endpoint: AsyncMock = mocker.patch.object( @@ -851,7 +854,7 @@ async def test____create_udp_listeners____bind_to_several_hosts( return_value=(mock_transport, mock_protocol), ) mock_DatagramListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.DatagramListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.listener.DatagramListenerSocketAdapter", return_value=mocker.sentinel.listener_socket, ) @@ -908,7 +911,7 @@ async def test____create_udp_listeners____error_getaddrinfo_returns_empty_list( return_value=[], ) mock_open_listeners = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.open_listener_sockets_from_getaddrinfo_result", + f"{_ASYNCIO_BACKEND_MODULE}._asyncio_utils.open_listener_sockets_from_getaddrinfo_result", side_effect=AssertionError, ) mock_create_datagram_endpoint: AsyncMock = mocker.patch.object( @@ -918,7 +921,7 @@ async def test____create_udp_listeners____error_getaddrinfo_returns_empty_list( side_effect=AssertionError, ) mock_DatagramListenerSocketAdapter: MagicMock = mocker.patch( - "easynetwork.lowlevel.api_async.backend._asyncio.backend.DatagramListenerSocketAdapter", + f"{_ASYNCIO_BACKEND_MODULE}.datagram.listener.DatagramListenerSocketAdapter", side_effect=AssertionError, )