Add Python 3.11 to workflows and apply ruff linting (#68)

* Add Python 3.11 to workflows

* Add ruff to dependencies

* Update makefile clean commands

* Apply ruff linting suggestions

.github/workflows/build.yml

@@ -8,14 +8,15 @@ jobs:
 
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
+        python-version: ["3.9", "3.10", "3.11"]
 
     steps:
 
     - name: Checkout the repo
       uses: actions/checkout@v4
 
     - name: Set up Python ${{ matrix.python-version }}
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
@@ -31,7 +32,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .venv
-        key: ${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+        key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
       if: steps.cached-dependencies.outputs.cache-hit != 'true'

.github/workflows/documentation.yml

@@ -15,6 +15,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Python 3.9
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: 3.9
@@ -30,7 +31,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .venv
-        key: ${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+        key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
       if: steps.cached-dependencies.outputs.cache-hit != 'true'

.github/workflows/lint.yml

@@ -12,6 +12,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Set up Python 3.9
+      id: setup-python
       uses: actions/setup-python@v5
       with:
         python-version: 3.9
@@ -27,7 +28,7 @@ jobs:
       uses: actions/cache@v4
       with:
         path: .venv
-        key: ${{ runner.os }}-${{ hashFiles('**/poetry.lock') }}
+        key: ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}
 
     - name: Install dependencies
       if: steps.cached-dependencies.outputs.cache-hit != 'true'

Makefile

@@ -1,21 +1,22 @@
 .PHONY: clean build docs
 
 clean: # clean all build, python, and testing files
-	rm -fr build/
-	rm -fr dist/
-	rm -fr .eggs/
-	find . -name '*.egg-info' -exec rm -fr {} +
-	find . -name '*.egg' -exec rm -f {} +
+	rm -rf build/
+	rm -rf dist/
+	rm -rf .eggs/
+	find . -name '*.egg-info' -exec rm -rf {} +
+	find . -name '*.egg' -exec rm -rf {} +
 	find . -name '*.pyc' -exec rm -f {} +
 	find . -name '*.pyo' -exec rm -f {} +
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
-	rm -fr .tox/
-	rm -fr .coverage
-	rm -fr coverage.xml
-	rm -fr htmlcov/
-	rm -fr .pytest_cache
-	rm -fr .mypy_cache
+	rm -rf .tox/
+	rm -rf .coverage
+	rm -rf coverage.xml
+	rm -rf htmlcov/
+	rm -rf .pytest_cache
+	rm -rf .mypy_cache
+	rm -rf .ruff_cache
 
 build: # run tox tests and lint
 	tox

pyproject.toml

@@ -31,6 +31,7 @@ furo = "^2023.5.20"
 myst-parser = "^2.0.0"
 sphinx-copybutton = "^0.5.2"
 tox = "^4.5.1"
+ruff = "^0.6.3"
 moto = {extras = ["ec2", "batch", "logs"], version = "^5.0.11"}
 
 [tool.isort]
@@ -68,11 +69,52 @@ module = [
 ]
 ignore_missing_imports = true
 
+[tool.ruff]
+line-length = 100
+target-version = "py39"
+
+[tool.ruff.lint]
+select = ["ALL"]
+ignore = [
+    "COM812", # missing-trailing-comma
+    "D100",   # undocumented-public-module
+    "D105",   # undocumented-magic-method
+    "D107",   # undocumented-public-init
+    "D202",   # no-blank-line-after-function
+    "D203",   # one-blank-line-before-class
+    "D212",   # multi-line-summary-first-line
+    "D413",   # blank-line-after-last-section
+    "D416",   # section-name-ends-in-colon
+]
+
+[tool.ruff.lint.pylint]
+max-args = 10
+
+[tool.ruff.lint.per-file-ignores]
+"tests/*.py" = [
+    "D",      # pydocstyle
+    "PT009",  # pytest-unittest-assertion
+    "PT027",  # pytest-unittest-raises-assertion
+    "INP001", # implicit-namespace-package
+    "ANN201", # missing-return-type-undocumented-public-function
+    "S311",   # suspicious-non-cryptographic-random-usage
+    "ANN001", # missing-type-function-argument
+    "ANN003", # missing-type-kwargs
+    "ANN202", # missing-type-args
+    "ANN206", # missing-return-type-class-method
+]
+
+[tool.coverage.report]
+exclude_lines = [
+    "pragma: no cover",
+    "if TYPE_CHECKING:",
+]
+
 [tool.tox]
 legacy_tox_ini = """
 [tox]
 isolated_build = True
-envlist = py{39,310}, format, lint, typecheck
+envlist = py{39,310,311}, format, lint, typecheck
 skipsdist=True
 
 [testenv]
@@ -88,6 +130,7 @@ commands =
 [testenv:lint]
 commands =
     poetry run pylint --ignore-patterns=test.*?py src/ tests/ --fail-under=9.0
+    poetry run ruff check src/ tests/
 
 [testenv:typecheck]
 commands =

src/container_collection/__init__.py

@@ -0,0 +1 @@
+"""Collection of tasks for running containerized models."""

src/container_collection/batch/check_batch_job.py

@@ -1,5 +1,6 @@
+from __future__ import annotations
+
 from time import sleep
-from typing import Union
 
 import boto3
 from prefect.context import TaskRunContext
@@ -9,7 +10,7 @@
 """Exit code used when task run retries exceed the maximum retries."""
 
 
-def check_batch_job(job_arn: str, max_retries: int) -> Union[int, State, bool]:
+def check_batch_job(job_arn: str, max_retries: int) -> int | State | bool:
     """
     Check for exit code of an AWS Batch job.
 
@@ -60,7 +61,7 @@ def check_batch_job(job_arn: str, max_retries: int) -> Union[int, State, bool]:
     if context is not None and status == "RUNNING":
         return Failed()
     if status == "RUNNING":
-        raise RuntimeError("Job is in RUNNING state and does not have exit code.")
+        message = "Job is in RUNNING state and does not have exit code."
+        raise RuntimeError(message)
 
-    exitcode = response[0]["attempts"][0]["container"]["exitCode"]
-    return exitcode
+    return response[0]["attempts"][0]["container"]["exitCode"]

src/container_collection/batch/make_batch_job.py

@@ -1,13 +1,13 @@
-from typing import Optional
+from __future__ import annotations
 
 
 def make_batch_job(
     name: str,
     image: str,
     vcpus: int,
     memory: int,
-    environment: Optional[list[dict[str, str]]] = None,
-    job_role_arn: Optional[str] = None,
+    environment: list[dict[str, str]] | None = None,
+    job_role_arn: str | None = None,
 ) -> dict:
     """
     Create batch job definition.

src/container_collection/batch/submit_batch_job.py

@@ -1,4 +1,4 @@
-from typing import Any
+from __future__ import annotations
 
 import boto3
 
@@ -9,7 +9,7 @@ def submit_batch_job(
     user: str,
     queue: str,
     size: int,
-    **kwargs: Any,
+    **kwargs: list | str | int | bool | dict,
 ) -> list[str]:
     """
     Submit AWS Batch job.
@@ -44,7 +44,7 @@ def submit_batch_job(
     }
 
     if size > 1:
-        default_job_submission["arrayProperties"] = {"size": size}  # type: ignore
+        default_job_submission["arrayProperties"] = {"size": size}  # type: ignore[assignment]
 
     client = boto3.client("batch")
     job_submission = default_job_submission | kwargs

src/container_collection/docker/check_docker_job.py

@@ -1,16 +1,18 @@
-from typing import Union
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
 
-from docker import APIClient
 from prefect.context import TaskRunContext
 from prefect.states import Failed, State
 
+if TYPE_CHECKING:
+    from docker import APIClient
+
 RETRIES_EXCEEDED_EXIT_CODE = 80
 """Exit code used when task run retries exceed the maximum retries."""
 
 
-def check_docker_job(
-    api_client: APIClient, container_id: str, max_retries: int
-) -> Union[int, State]:
+def check_docker_job(api_client: APIClient, container_id: str, max_retries: int) -> int | State:
     """
     Check for exit code of a Docker container.
 
@@ -49,7 +51,7 @@ def check_docker_job(
     if context is not None and status == "running":
         return Failed()
     if status == "running":
-        raise RuntimeError("Job is in RUNNING state and does not have exit code.")
+        message = "Job is in RUNNING state and does not have exit code."
+        raise RuntimeError(message)
 
-    exitcode = api_client.wait(container_id, timeout=1)["StatusCode"]
-    return exitcode
+    return api_client.wait(container_id, timeout=1)["StatusCode"]

src/container_collection/docker/make_docker_job.py

@@ -1,7 +1,7 @@
-from typing import Optional
+from __future__ import annotations
 
 
-def make_docker_job(name: str, image: str, environment: Optional[list[str]] = None) -> dict:
+def make_docker_job(name: str, image: str, environment: list[str] | None = None) -> dict:
     """
     Create docker job definition.
 

src/container_collection/docker/run_docker_command.py

@@ -1,22 +1,26 @@
-from typing import Optional
+from __future__ import annotations
 
-from docker import DockerClient
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from docker import DockerClient
 
 
 def run_docker_command(
     client: DockerClient,
     image: str,
     command: list[str],
-    volume_name: Optional[str] = None,
-    environment: Optional[list] = None,
-    detach: bool = False,
+    volume_name: str | None = None,
+    environment: list | None = None,
+    *,
+    detach: bool,
 ) -> None:
     """
     Run container from image with given command.
 
     Parameters
     ----------
-    api_client
+    client
         Docker API client.
     image
         Docker image.

src/container_collection/fargate/check_fargate_task.py

@@ -1,5 +1,6 @@
+from __future__ import annotations
+
 from time import sleep
-from typing import Union
 
 import boto3
 from prefect.context import TaskRunContext
@@ -9,7 +10,7 @@
 """Exit code used when task run retries exceed the maximum retries."""
 
 
-def check_fargate_task(cluster: str, task_arn: str, max_retries: int) -> Union[int, State]:
+def check_fargate_task(cluster: str, task_arn: str, max_retries: int) -> int | State:
     """
     Check for exit code of an AWS Fargate task.
 
@@ -62,7 +63,7 @@ def check_fargate_task(cluster: str, task_arn: str, max_retries: int) -> Union[i
     if context is not None and status == "RUNNING":
         return Failed()
     if status == "RUNNING":
-        raise RuntimeError("Task is in RUNNING state and does not have exit code.")
+        message = "Task is in RUNNING state and does not have exit code."
+        raise RuntimeError(message)
 
-    exitcode = response[0]["containers"][0]["exitCode"]
-    return exitcode
+    return response[0]["containers"][0]["exitCode"]

src/container_collection/fargate/submit_fargate_task.py

@@ -1,4 +1,4 @@
-from typing import Any
+from __future__ import annotations
 
 import boto3
 
@@ -11,7 +11,7 @@ def submit_fargate_task(
     security_groups: list[str],
     subnets: list[str],
     command: list[str],
-    **kwargs: Any,
+    **kwargs: list | str | int | bool | dict,
 ) -> list[str]:
     """
     Submit task to AWS Fargate.