Merge and solving conflicts

.git-blame-ignore-revs

@@ -0,0 +1 @@
+4bef04b904374e539a59bead67ab93aaed52b3e8  # apply new rules max-length 120 and format whole project with ruff

.github/workflows/tests.yaml

@@ -0,0 +1,15 @@
+name: Automated tests
+on: [pull_request]
+jobs:
+  tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: KengoTODA/actions-setup-docker-compose@v1
+        with:
+          version: '2.23.3'
+      - name: "Running tests"
+        run: |
+          cp .env.default .env
+          docker compose up -d
+          make dtest

.pre-commit-config.yaml

@@ -22,7 +22,7 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        exclude: "apps/girderformindlogger|apps/migrate"
+        exclude: "apps/girderformindlogger|apps/migrate|apps/jsonld_converter"
 
       - id: git-secrets
         name: git-secrets

compose/postgres/init-db/003-main.sql

@@ -1 +1,2 @@
-CREATE EXTENSION pg_cron;
+CREATE EXTENSION pg_cron;
+CREATE EXTENSION IF NOT EXISTS pgcrypto;

conftest.py

@@ -1,6 +1,7 @@
+import datetime
 import os
 import uuid
-from typing import Any, AsyncGenerator, Generator, cast
+from typing import Any, AsyncGenerator, Callable, Generator, cast
 
 import pytest
 import taskiq_fastapi
@@ -9,14 +10,12 @@
 from fastapi import FastAPI
 from pytest import Parser
 from pytest_asyncio import is_async_test
+from pytest_mock import MockerFixture
 from sqlalchemy import event
 from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine, AsyncSession
 from sqlalchemy.orm import Session, SessionTransaction
 
-from apps.answers.deps.preprocess_arbitrary import (
-    get_answer_session,
-    get_answer_session_by_subject,
-)
+from apps.answers.deps.preprocess_arbitrary import get_answer_session, get_answer_session_by_subject
 from apps.shared.test.client import TestClient
 from broker import broker
 from config import settings
@@ -30,10 +29,40 @@
     "apps.activities.tests.fixtures.items",
     "apps.activities.tests.fixtures.conditional_logic",
     "apps.activities.tests.fixtures.scores_reports",
+    "apps.activities.tests.fixtures.activities",
     "apps.users.tests.fixtures.users",
+    "apps.applets.tests.fixtures.applets",
+    "apps.users.tests.fixtures.user_devices",
 ]
 
 
+@pytest.fixture(scope="session")
+async def global_engine():
+    engine = build_engine(settings.database.url)
+    yield engine
+    await engine.dispose()
+
+
+@pytest.fixture(scope="session")
+async def global_session(global_engine: AsyncEngine):
+    """
+    Global session is used to create pre-defined objects in database for ALL pytest session.
+    Inside tests and for local/intermediate fixtures please use session fixture.
+    """
+    async with AsyncSession(bind=global_engine) as session:
+        yield session
+
+
+# TODO: Instead of custom faketime for tests add function wrapper `now`
+# to use it instead of builtin datetime.datetime.utcnow
+class FakeTime(datetime.datetime):
+    current_utc = datetime.datetime(2024, 1, 1, 0, 0, 0)
+
+    @classmethod
+    def utcnow(cls):
+        return cls.current_utc
+
+
 alembic_configs = [Config("alembic.ini"), Config("alembic_arbitrary.ini")]
 
 
@@ -46,11 +75,25 @@ def pytest_addoption(parser: Parser) -> None:
     )
 
 
-def pytest_sessionstart(session):
+def before():
+    os.environ["PYTEST_APP_TESTING"] = "1"
+    for alembic_cfg in alembic_configs:
+        command.upgrade(alembic_cfg, "head")
+
+
+def after():
+    for alembic_cfg in alembic_configs[::-1]:
+        command.downgrade(alembic_cfg, "base")
+    os.environ.pop("PYTEST_APP_TESTING", None)
+
+
+def pytest_sessionstart(session) -> None:
     before()
 
 
-def pytest_sessionfinish(session):
+@pytest.hookimpl(trylast=True)
+def pytest_sessionfinish(session, exitstatus) -> None:
+    # Don't run downgrade migrations
     keepdb = session.config.getvalue("keepdb")
     if not keepdb:
         after()
@@ -61,6 +104,12 @@ def app() -> FastAPI:
     return create_app()
 
 
+@pytest.fixture(scope="session")
+def arbitrary_db_url() -> str:
+    host = settings.database.host
+    return f"postgresql+asyncpg://postgres:postgres@{host}:5432/test_arbitrary"
+
+
 @pytest.fixture()
 async def engine() -> AsyncGenerator[AsyncEngine, Any]:
     engine = build_engine(settings.database.url)
@@ -69,11 +118,10 @@ async def engine() -> AsyncGenerator[AsyncEngine, Any]:
 
 
 @pytest.fixture()
-async def arbitrary_engine() -> AsyncGenerator[AsyncEngine, Any]:
-    host = settings.database.host
-    engine = build_engine(
-        f"postgresql+asyncpg://postgres:postgres@{host}:5432/test_arbitrary"
-    )
+async def arbitrary_engine(
+    arbitrary_db_url: str,
+) -> AsyncGenerator[AsyncEngine, Any]:
+    engine = build_engine(arbitrary_db_url)
     yield engine
     await engine.dispose()
 
@@ -85,12 +133,8 @@ async def session(engine: AsyncEngine) -> AsyncGenerator:
         await conn.begin_nested()
         async with AsyncSession(bind=conn) as async_session:
 
-            @event.listens_for(
-                async_session.sync_session, "after_transaction_end"
-            )
-            def end_savepoint(
-                session: Session, transaction: SessionTransaction
-            ) -> None:
+            @event.listens_for(async_session.sync_session, "after_transaction_end")
+            def end_savepoint(session: Session, transaction: SessionTransaction) -> None:
                 nonlocal conn
                 conn = cast(AsyncConnection, conn)
                 if conn.closed:
@@ -110,12 +154,8 @@ async def arbitrary_session(arbitrary_engine: AsyncEngine) -> AsyncGenerator:
         await conn.begin_nested()
         async with AsyncSession(bind=conn) as async_session:
 
-            @event.listens_for(
-                async_session.sync_session, "after_transaction_end"
-            )
-            def end_savepoint(
-                session: Session, transaction: SessionTransaction
-            ) -> None:
+            @event.listens_for(async_session.sync_session, "after_transaction_end")
+            def end_savepoint(session: Session, transaction: SessionTransaction) -> None:
                 if conn.closed:
                     return
                 if not conn.in_nested_transaction():
@@ -154,19 +194,7 @@ def arbitrary_client(
 
 
 
-def before():
-    os.environ["PYTEST_APP_TESTING"] = "1"
-    for alembic_cfg in alembic_configs:
-        command.upgrade(alembic_cfg, "head")
-
-
-def after():
-    for alembic_cfg in alembic_configs[::-1]:
-        command.downgrade(alembic_cfg, "base")
-    os.environ.pop("PYTEST_APP_TESTING")
-
-
-def pytest_collection_modifyitems(items):
+def pytest_collection_modifyitems(items) -> None:
     pytest_asyncio_tests = (item for item in items if is_async_test(item))
     session_scope_marker = pytest.mark.asyncio(scope="session")
     for async_test in pytest_asyncio_tests:
@@ -180,15 +208,17 @@ def remote_image() -> str:
 
 
 @pytest.fixture
-async def mock_kiq_report(mocker):
+async def mock_kiq_report(mocker) -> AsyncGenerator[Any, Any]:
     mock = mocker.patch("apps.answers.service.create_report.kiq")
     yield mock
-    mock.stop_all()
 
 
 @pytest.fixture
-async def mock_report_server_response(mocker):
-    def json_():
+async def mock_report_server_response(mocker) -> AsyncGenerator[Any, Any]:
+    Recipients = list[str]
+    FakeBody = dict[str, str | dict[str, str | Recipients]]
+
+    def json_() -> FakeBody:
         return dict(
             pdf="cGRmIGJvZHk=",
             email=dict(
@@ -203,16 +233,43 @@ def json_():
     mock.return_value.__aenter__.return_value.status = 200
     mock.return_value.__aenter__.return_value.json.side_effect = json_
     yield mock
-    mock.stop_all()
 
 
 @pytest.fixture
-def mock_reencrypt_kiq(mocker):
+async def mock_reencrypt_kiq(mocker) -> AsyncGenerator[Any, Any]:
     mock = mocker.patch("apps.users.api.password.reencrypt_answers.kiq")
     yield mock
-    mock.stop_all()
 
 
 @pytest.fixture(scope="session")
 def uuid_zero() -> uuid.UUID:
     return uuid.UUID("00000000-0000-0000-0000-000000000000")
+
+
+@pytest.fixture
+def faketime(mocker: MockerFixture) -> type[FakeTime]:
+    mock = mocker.patch("datetime.datetime", new=FakeTime)
+    return mock
+
+
+@pytest.fixture
+def mock_get_session(
+    session: AsyncSession,
+    arbitrary_session: AsyncSession,
+    mocker: MockerFixture,
+    arbitrary_db_url: str,
+) -> Callable[..., Callable[[], AsyncSession]]:
+    # Add stub for first argument, because orig get_session takes instanace as first argument after mock
+    def get_session(_, url: str = settings.database.url) -> Callable[[], AsyncSession]:
+        def f() -> AsyncSession:
+            if url == arbitrary_db_url:
+                return arbitrary_session
+            return session
+
+        return f
+
+    mock = mocker.patch(
+        "infrastructure.database.core.SessionManager.get_session",
+        new=get_session,
+    )
+    return mock

pyproject.toml

@@ -18,15 +18,16 @@ exclude = [
     "dist",
     "venv",
     "src/infrastructure/database/migrations/versions",
+    "src/infrastructure/database/migrations_arbitrary/versions",
     "src/apps/girderformindlogger",
     "src/apps/migrate",
 ]
-line-length = 79
+line-length = 120
 indent-width = 4
 target-version = "py310"
 
 [tool.ruff.lint]
-select = ["E4", "E7", "E9", "F", "C901"]
+select = ["E4", "E7", "E9", "F", "C901", "E5"]
 ignore = []
 
 [tool.ruff.format]
@@ -43,12 +44,12 @@ multi_line_output = 3
 include_trailing_comma = true
 force_grid_wrap = 0
 use_parentheses = true
-line_length = 79
+line_length = 120
 skip = '.venv,venv,env,migrate'
 src_paths = ["src"]
 
 [tool.pytest.ini_options]
-addopts = '-s -v --cache-clear'
+addopts = '-s -v'
 asyncio_mode = 'auto'
 cache_dir = '/tmp'
 python_files = 'tests.py test_*.py *_test.py'
@@ -62,6 +63,7 @@ env = [
     "DATABASE__USER=postgres",
     "DATABASE__DB=test",
     "ARBITRARY_DB=test_arbitrary",
+    "TASK_ANSWER_ENCRYPTION__BATCH_LIMIT=1",
 ]
 
 [tool.coverage.run]
@@ -86,7 +88,7 @@ warn_redundant_casts = true
 warn_unused_ignores = true
 show_error_codes = true
 namespace_packages = true
-exclude = ["apps/girderformindlogger", "apps/migrate"]
+exclude = ["apps/girderformindlogger", "apps/migrate", "apps/jsonld_converter"]
 
 # Silence "type import errors" as our 3rd-party libs does not have types
 # Check: https://mypy.readthedocs.io/en/latest/config_file.html#import-discovery