Replace aioredis with redis 4.3 or higher (#701)

* Change aioredis extra dependency to use redis-py 4.3 or later

- redis-py now includes the aioredis as an official asyncio client
  since 4.2.0.

* Adapt existing codes to redis 4.3

- Drop use of distutils (deprecated in Python 3.10)

* Add news fragment

* Apply review comments

- Since this module encodes/decodes the get/set data explicitly,
  I set the generic type parameter of Redis to bytes.
- As you've made as a PR to typeshed, VERSION is not recognized by
  mypy currently.  Temporarily let mypy ignore the check error.

* Fix mistake

* Update test mocking

* Update types-redis to remove a 'type: ignore' directive

* Fix type errors following types-redis 4.3.1

* Upgrade the development requirements

* Reveal type of the encoded value

* I don't prefer the Hungarian notation, but let's allow to avoid
  reusing a single variable for multiple different types.

* Now redis-py explicitly declares VERSION as an int-tuple.

* ref: https://github.com/redis/redis-py/blob/bea7299/redis/__init__.py#L54

* Fix further type errors in test suite

* Apply `__future__.annotations` flag

* Fix another missing lint error

* Update tests/test_redis_storage.py

* Update aiohttp_session/redis_storage.py

* Update redis_storage.py

* Update test_redis_storage.py

Co-authored-by: Sam Bull <[email protected]>

Author: achimnol

.isort.cfg

@@ -4,3 +4,4 @@ include_trailing_comma=True
 multi_line_output=3
 force_grid_wrap=0
 combine_as_imports=True
+known_first_party=aiohttp_session

aiohttp_session/redis_storage.py

@@ -1,14 +1,14 @@
 import json
 import uuid
-from distutils.version import StrictVersion
 from typing import Any, Callable, Optional
 
 from aiohttp import web
 
 from . import AbstractStorage, Session
 
 try:
-    import aioredis
+    from redis import asyncio as aioredis
+    from redis import VERSION as REDIS_VERSION
 except ImportError:  # pragma: no cover
     aioredis = None  # type: ignore[assignment]
 
@@ -18,7 +18,7 @@ class RedisStorage(AbstractStorage):
 
     def __init__(
         self,
-        redis_pool: "aioredis.Redis",
+        redis_pool: "aioredis.Redis[bytes]",
         *,
         cookie_name: str = "AIOHTTP_SESSION",
         domain: Optional[str] = None,
@@ -43,13 +43,13 @@ def __init__(
             decoder=decoder,
         )
         if aioredis is None:
-            raise RuntimeError("Please install aioredis")
+            raise RuntimeError("Please install redis")
         # May have installed aioredis separately (without aiohttp-session[aioredis]).
-        if StrictVersion(aioredis.__version__).version < (2, 0):
-            raise RuntimeError("aioredis<2.0 is not supported")
+        if REDIS_VERSION < (4, 3):
+            raise RuntimeError("redis<4.3 is not supported")
         self._key_factory = key_factory
         if not isinstance(redis_pool, aioredis.Redis):
-            raise TypeError(f"Expected aioredis.Redis got {type(redis_pool)}")
+            raise TypeError(f"Expected redis.asyncio.Redis got {type(redis_pool)}")
         self._redis = redis_pool
 
     async def load_session(self, request: web.Request) -> Session:
@@ -58,12 +58,12 @@ async def load_session(self, request: web.Request) -> Session:
             return Session(None, data=None, new=True, max_age=self.max_age)
         else:
             key = str(cookie)
-            data = await self._redis.get(self.cookie_name + "_" + key)
-            if data is None:
+            data_bytes = await self._redis.get(self.cookie_name + "_" + key)
+            if data_bytes is None:
                 return Session(None, data=None, new=True, max_age=self.max_age)
-            data = data.decode("utf-8")
+            data_str = data_bytes.decode("utf-8")
             try:
-                data = self._decoder(data)
+                data = self._decoder(data_str)
             except ValueError:
                 data = None
             return Session(key, data=data, new=False, max_age=self.max_age)
@@ -82,9 +82,9 @@ async def save_session(
                 key = str(key)
                 self.save_cookie(response, key, max_age=session.max_age)
 
-        data = self._encoder(self._get_session_data(session))
+        data_str = self._encoder(self._get_session_data(session))
         await self._redis.set(
             self.cookie_name + "_" + key,
-            data,
+            data_str,
             ex=session.max_age,
         )

changes/701.fix

@@ -0,0 +1 @@
+Replace aioredis with redis 4.3 or higher in favor of the official asyncio support

demo/redis_storage.py

@@ -18,7 +18,7 @@ async def handler(request: web.Request) -> web.Response:
 
 async def redis_pool(app: web.Application) -> AsyncIterator[None]:
     redis_address = "redis://127.0.0.1:6379"
-    async with aioredis.from_url(  # type: ignore[no-untyped-call]
+    async with aioredis.from_url(
         redis_address,
         timeout=1,
     ) as redis:

requirements-dev.txt

@@ -1,7 +1,7 @@
 -e .
 aiohttp==3.8.3
 aiomcache==0.7.0
-aioredis==2.0.1
+redis==4.3.3
 attrs==22.1.0
 chardet==5.0.0
 cryptography==38.0.1
@@ -22,5 +22,6 @@ pytest-cov==4.0.0
 pytest-mock==3.10.0
 pytest-sugar==0.9.5
 sphinx==4.3.2
+types-redis==4.3.1
 typing-extensions==4.4.0
 yarl==1.8.1

setup.py

@@ -21,7 +21,7 @@ def read(f):
 
 install_requires = ["aiohttp>=3.8", 'typing_extensions>=3.7.4; python_version<"3.8"']
 extras_require = {
-    "aioredis": ["aioredis>=2.0.0"],
+    "aioredis": ["redis>=4.3.1"],
     "aiomcache": ["aiomcache>=0.5.2"],
     "pycrypto": ["cryptography"],
     "secure": ["cryptography"],

tests/conftest.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import gc
 import socket
@@ -7,20 +9,16 @@
 from typing import Iterator
 
 import aiomcache
-import aioredis
 import pytest
 from docker import DockerClient, from_env as docker_from_env, models as docker_models
+from redis import asyncio as aioredis
 
 if sys.version_info >= (3, 8):
     from typing import TypedDict
 else:
     from typing_extensions import TypedDict
 
 
-# TODO: Remove once fixed: https://github.com/aio-libs/aioredis-py/issues/1115
-aioredis.Redis.__del__ = lambda *args: None  # type: ignore
-
-
 class _ContainerInfo(TypedDict):
     host: str
     port: int
@@ -106,10 +104,10 @@ def redis_server(  # type: ignore[misc]  # No docker types.
     delay = 0.1
     for _i in range(20):
         try:
-            conn = aioredis.from_url(f"redis://{host}:{port}")  # type: ignore[no-untyped-call]  # noqa
+            conn = aioredis.from_url(f"redis://{host}:{port}")
             event_loop.run_until_complete(conn.set("foo", "bar"))
             break
-        except aioredis.exceptions.ConnectionError:
+        except aioredis.ConnectionError:
             time.sleep(delay)
             delay *= 2
         finally:
@@ -134,15 +132,15 @@ def redis_url(redis_server: _ContainerInfo) -> str:  # type: ignore[misc]
 def redis(
     event_loop: asyncio.AbstractEventLoop,
     redis_url: str,
-) -> Iterator[aioredis.Redis]:
-    async def start(pool: aioredis.ConnectionPool) -> aioredis.Redis:
+) -> Iterator[aioredis.Redis[bytes]]:
+    async def start(pool: aioredis.ConnectionPool) -> aioredis.Redis[bytes]:
         return aioredis.Redis(connection_pool=pool)
 
     asyncio.set_event_loop(event_loop)
     pool = aioredis.ConnectionPool.from_url(redis_url)
     redis = event_loop.run_until_complete(start(pool))
     yield redis
-    event_loop.run_until_complete(redis.close())  # type: ignore[no-untyped-call]
+    event_loop.run_until_complete(redis.close())
     event_loop.run_until_complete(pool.disconnect())
 
 

tests/test_redis_storage.py

@@ -1,14 +1,16 @@
+from __future__ import annotations
+
 import asyncio
 import json
 import time
 import uuid
 from typing import Any, Callable, Dict, MutableMapping, Optional, cast
 
-import aioredis
 import pytest
 from aiohttp import web
 from aiohttp.test_utils import TestClient
 from pytest_mock import MockFixture
+from redis import asyncio as aioredis
 
 from aiohttp_session import Handler, Session, get_session, session_middleware
 from aiohttp_session.redis_storage import RedisStorage
@@ -18,7 +20,7 @@
 
 def create_app(
     handler: Handler,
-    redis: aioredis.Redis,
+    redis: aioredis.Redis[bytes],
     max_age: Optional[int] = None,
     key_factory: Callable[[], str] = lambda: uuid.uuid4().hex,
 ) -> web.Application:
@@ -31,7 +33,7 @@ def create_app(
 
 
 async def make_cookie(
-    client: TestClient, redis: aioredis.Redis, data: Dict[Any, Any]
+    client: TestClient, redis: aioredis.Redis[bytes], data: Dict[Any, Any]
 ) -> None:
     session_data = {"session": data, "created": int(time.time())}
     value = json.dumps(session_data)
@@ -40,23 +42,21 @@ async def make_cookie(
     client.session.cookie_jar.update_cookies({"AIOHTTP_SESSION": key})
 
 
-async def make_cookie_with_bad_value(client: TestClient, redis: aioredis.Redis) -> None:
+async def make_cookie_with_bad_value(client: TestClient, redis: aioredis.Redis[bytes]) -> None:
     key = uuid.uuid4().hex
     await redis.set("AIOHTTP_SESSION_" + key, "")
     client.session.cookie_jar.update_cookies({"AIOHTTP_SESSION": key})
 
 
-async def load_cookie(client: TestClient, redis: aioredis.Redis) -> Any:
+async def load_cookie(client: TestClient, redis: aioredis.Redis[bytes]) -> Any:
     cookies = client.session.cookie_jar.filter_cookies(client.make_url("/"))
     key = cookies["AIOHTTP_SESSION"]
-    encoded = await redis.get("AIOHTTP_SESSION_" + key.value)
-    s = encoded.decode("utf-8")
-    value = json.loads(s)
-    return value
+    value_bytes = await redis.get("AIOHTTP_SESSION_" + key.value)
+    return None if value_bytes is None else json.loads(value_bytes)
 
 
 async def test_create_new_session(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -72,7 +72,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_load_existing_session(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -89,7 +89,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_load_bad_session(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -106,7 +106,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_change_session(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -133,7 +133,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_clear_cookie_on_session_invalidation(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -154,7 +154,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_create_cookie_in_handler(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -181,7 +181,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_set_ttl_on_session_saving(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -201,7 +201,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_set_ttl_manually_set(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -222,7 +222,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_create_new_session_if_key_doesnt_exists_in_redis(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -236,7 +236,7 @@ async def handler(request: web.Request) -> web.StreamResponse:
 
 
 async def test_create_storage_with_custom_key_factory(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -259,7 +259,7 @@ def key_factory() -> str:
 
 
 async def test_redis_session_fixation(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def login(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)
@@ -288,7 +288,7 @@ async def test_redis_from_create_pool(redis_url: str) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         pass
 
-    redis = aioredis.from_url(redis_url)  # type: ignore[no-untyped-call]
+    redis = aioredis.from_url(redis_url)
     create_app(handler=handler, redis=redis)
     await redis.close()
 
@@ -314,17 +314,13 @@ async def test_old_aioredis_version(mocker: MockFixture) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         pass
 
-    class Dummy:
-        def __init__(self, *args: object, **kwargs: object) -> None:
-            self.version = (0, 3)
-
-    mocker.patch("aiohttp_session.redis_storage.StrictVersion", Dummy)
+    mocker.patch("aiohttp_session.redis_storage.REDIS_VERSION", (0, 3, "dev0"))
     with pytest.raises(RuntimeError):
         create_app(handler=handler, redis=None)  # type: ignore[arg-type]
 
 
 async def test_load_session_dont_load_expired_session(
-    aiohttp_client: AiohttpClient, redis: aioredis.Redis
+    aiohttp_client: AiohttpClient, redis: aioredis.Redis[bytes]
 ) -> None:
     async def handler(request: web.Request) -> web.StreamResponse:
         session = await get_session(request)