Uses testcontainers to provide a redis instance for the unit tests

.github/workflows/ci.yml

@@ -5,96 +5,90 @@ on:
     branches:
       - main
     tags:
-      - '**'
+      - "**"
   pull_request: {}
 
 jobs:
   lint:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: set up python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
+      - name: set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
 
-    - run: pip install -r requirements/linting.txt -r requirements/pyproject.txt pre-commit
+      - run: pip install -r requirements/linting.txt -r requirements/pyproject.txt pre-commit
 
-    - run: pre-commit run -a --verbose
-      env:
-        SKIP: no-commit-to-branch
+      - run: pre-commit run -a --verbose
+        env:
+          SKIP: no-commit-to-branch
 
   docs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: set up python
-      uses: actions/setup-python@v5
-      with:
-        python-version: '3.11'
+      - name: set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
 
-    - run: pip install -r requirements/docs.txt -r requirements/pyproject.txt
-    - run: pip install .
+      - run: pip install -r requirements/docs.txt -r requirements/pyproject.txt
+      - run: pip install .
 
-    - run: make docs
+      - run: make docs
 
-    - name: Store docs site
-      uses: actions/upload-artifact@v4
-      with:
-        name: docs
-        path: docs/_build/
+      - name: Store docs site
+        uses: actions/upload-artifact@v4
+        with:
+          name: docs
+          path: docs/_build/
 
   test:
     name: test py${{ matrix.python }} with redis:${{ matrix.redis }} on ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: [ubuntu]
-        python: ['3.8', '3.9', '3.10', '3.11', '3.12']
-        redis: ['5']
+        python: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        redis: ["5"]
         include:
-          - python: '3.11'
-            redis: '6'
-            os: 'ubuntu'
-          - python: '3.11'
-            redis: '7'
-            os: 'ubuntu'
+          - python: "3.11"
+            redis: "6"
+            os: "ubuntu"
+          - python: "3.11"
+            redis: "7"
+            os: "ubuntu"
 
     env:
       PYTHON: ${{ matrix.python }}
       OS: ${{ matrix.os }}
+      ARQ_TEST_REDIS_VERSION: ${{ matrix.redis }}
 
     runs-on: ${{ matrix.os }}-latest
 
-    services:
-      redis:
-        image: redis:${{ matrix.redis }}
-        ports:
-        - 6379:6379
-        options: --entrypoint redis-server
-
     steps:
-    - uses: actions/checkout@v4
+      - uses: actions/checkout@v4
 
-    - name: set up python
-      uses: actions/setup-python@v5
-      with:
-        python-version: ${{ matrix.python }}
+      - name: set up python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python }}
 
-    - run: pip install -r requirements/testing.txt -r requirements/pyproject.txt
+      - run: pip install -r requirements/testing.txt -r requirements/pyproject.txt
 
-    - run: make test
+      - run: make test
 
-    - run: coverage xml
+      - run: coverage xml
 
-    - uses: codecov/codecov-action@v4
-      with:
-        file: ./coverage.xml
-        env_vars: PYTHON,OS
+      - uses: codecov/codecov-action@v4
+        with:
+          file: ./coverage.xml
+          env_vars: PYTHON,OS
 
   check:
     if: always()
@@ -130,7 +124,7 @@ jobs:
       - name: set up python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.11'
+          python-version: "3.11"
 
       - name: install
         run: pip install -U build
@@ -139,7 +133,7 @@ jobs:
         id: check-version
         uses: samuelcolvin/[email protected]
         with:
-          version_file_path: 'arq/version.py'
+          version_file_path: "arq/version.py"
 
       - name: build
         run: python -m build
@@ -148,7 +142,7 @@ jobs:
         uses: pypa/gh-action-pypi-publish@release/v1
 
       - name: publish docs
-        if: '!fromJSON(steps.check-version.outputs.IS_PRERELEASE)'
+        if: "!fromJSON(steps.check-version.outputs.IS_PRERELEASE)"
         run: make publish-docs
         env:
           NETLIFY: ${{ secrets.netlify_token }}

.gitignore

@@ -18,3 +18,4 @@ __pycache__/
 .venv/
 /.auto-format
 /scratch/
+.python-version

requirements/docs.txt

@@ -8,13 +8,13 @@ alabaster==0.7.16
     # via sphinx
 babel==2.14.0
     # via sphinx
-certifi==2024.2.2
+certifi==2024.7.4
     # via requests
 charset-normalizer==3.3.2
     # via requests
 docutils==0.19
     # via sphinx
-idna==3.6
+idna==3.7
     # via requests
 imagesize==1.4.1
     # via sphinx
@@ -26,7 +26,7 @@ packaging==24.0
     # via sphinx
 pygments==2.17.2
     # via sphinx
-requests==2.31.0
+requests==2.32.3
     # via sphinx
 snowballstemmer==2.2.0
     # via sphinx
@@ -44,5 +44,5 @@ sphinxcontrib-qthelp==1.0.7
     # via sphinx
 sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
-urllib3==2.2.1
+urllib3==2.2.2
     # via requests

requirements/pyproject.txt

@@ -10,7 +10,7 @@ click==8.1.7
     # via arq (pyproject.toml)
 hiredis==2.3.2
     # via redis
-idna==3.6
+idna==3.7
     # via anyio
 redis==4.6.0
     # via arq (pyproject.toml)

requirements/testing.in

@@ -8,3 +8,4 @@ pytest-mock
 pytest-pretty
 pytest-timeout
 pytz
+testcontainers

requirements/testing.txt

@@ -6,10 +6,18 @@
 #
 annotated-types==0.6.0
     # via pydantic
+certifi==2024.7.4
+    # via requests
+charset-normalizer==3.3.2
+    # via requests
 coverage==7.4.4
     # via -r requirements/testing.in
 dirty-equals==0.7.1.post0
     # via -r requirements/testing.in
+docker==7.1.0
+    # via testcontainers
+idna==3.7
+    # via requests
 iniconfig==2.0.0
     # via pytest
 markdown-it-py==3.0.0
@@ -47,9 +55,21 @@ pytz==2024.1
     # via
     #   -r requirements/testing.in
     #   dirty-equals
+requests==2.32.3
+    # via docker
 rich==13.7.1
     # via pytest-pretty
+testcontainers==4.7.2
+    # via -r requirements/testing.in
 typing-extensions==4.10.0
     # via
     #   pydantic
     #   pydantic-core
+    #   testcontainers
+urllib3==2.2.2
+    # via
+    #   docker
+    #   requests
+    #   testcontainers
+wrapt==1.16.0
+    # via testcontainers

tests/conftest.py

@@ -2,27 +2,55 @@
 import functools
 import os
 import sys
+from typing import Generator
 
 import msgpack
 import pytest
 import redis.exceptions
 from redis.asyncio.retry import Retry
 from redis.backoff import NoBackoff
+from testcontainers.redis import RedisContainer
 
-from arq.connections import ArqRedis, create_pool
+from arq.connections import ArqRedis, RedisSettings, create_pool
 from arq.worker import Worker
 
 
 @pytest.fixture(name='loop')
-def _fix_loop(event_loop):
+def _fix_loop(event_loop: asyncio.AbstractEventLoop) -> asyncio.AbstractEventLoop:
     return event_loop
 
 
+@pytest.fixture(scope='session')
+def redis_version() -> str:
+    return os.getenv('ARQ_TEST_REDIS_VERSION', 'latest')
+
+
+@pytest.fixture(scope='session')
+def redis_container(redis_version: str) -> Generator[RedisContainer, None, None]:
+    with RedisContainer(f'redis:{redis_version}') as redis:
+        yield redis
+
+
+@pytest.fixture(scope='session')
+def test_redis_host(redis_container: RedisContainer) -> str:
+    return redis_container.get_container_host_ip()
+
+
+@pytest.fixture(scope='session')
+def test_redis_port(redis_container: RedisContainer) -> int:
+    return redis_container.get_exposed_port(redis_container.port)
+
+
+@pytest.fixture(scope='session')
+def test_redis_settings(test_redis_host: str, test_redis_port: int) -> RedisSettings:
+    return RedisSettings(host=test_redis_host, port=test_redis_port)
+
+
 @pytest.fixture
-async def arq_redis(loop):
+async def arq_redis(test_redis_host: str, test_redis_port: int):
     redis_ = ArqRedis(
-        host='localhost',
-        port=6379,
+        host=test_redis_host,
+        port=test_redis_port,
         encoding='utf-8',
     )
 
@@ -34,10 +62,10 @@ async def arq_redis(loop):
 
 
 @pytest.fixture
-async def arq_redis_msgpack(loop):
+async def arq_redis_msgpack(test_redis_host: str, test_redis_port: int):
     redis_ = ArqRedis(
-        host='localhost',
-        port=6379,
+        host=test_redis_host,
+        port=test_redis_port,
         encoding='utf-8',
         job_serializer=msgpack.packb,
         job_deserializer=functools.partial(msgpack.unpackb, raw=False),
@@ -48,10 +76,10 @@ async def arq_redis_msgpack(loop):
 
 
 @pytest.fixture
-async def arq_redis_retry(loop):
+async def arq_redis_retry(test_redis_host: str, test_redis_port: int):
     redis_ = ArqRedis(
-        host='localhost',
-        port=6379,
+        host=test_redis_host,
+        port=test_redis_port,
         encoding='utf-8',
         retry=Retry(backoff=NoBackoff(), retries=3),
         retry_on_timeout=True,

tests/test_cli.py

@@ -3,6 +3,7 @@
 
 from arq import logs
 from arq.cli import cli
+from arq.connections import RedisSettings
 
 
 async def foobar(ctx):
@@ -14,6 +15,11 @@ class WorkerSettings:
     functions = [foobar]
 
 
+@pytest.fixture(scope='module', autouse=True)
+def setup_worker_connection(test_redis_host: str, test_redis_port: int):
+    WorkerSettings.redis_settings = RedisSettings(host=test_redis_host, port=test_redis_port)
+
+
 def test_help():
     runner = CliRunner()
     result = runner.invoke(cli, ['--help'])

tests/test_jobs.py

@@ -114,9 +114,9 @@ async def test_enqueue_job_alt_queue(arq_redis: ArqRedis, worker):
     await test_enqueue_job(arq_redis, worker, queue_name='custom_queue')
 
 
-async def test_enqueue_job_nondefault_queue(worker):
+async def test_enqueue_job_nondefault_queue(test_redis_settings: RedisSettings, worker):
     """Test initializing arq_redis with a queue name, and the worker using it."""
-    arq_redis = await create_pool(RedisSettings(), default_queue_name='test_queue')
+    arq_redis = await create_pool(test_redis_settings, default_queue_name='test_queue')
     await test_enqueue_job(
         arq_redis,
         lambda functions, **_: worker(functions=functions, arq_redis=arq_redis, queue_name=None),