Update to Python 3.11, upgrade to SQLAlchemy 2.0 and Flask 2.4, refactor DB code

.github/workflows/format.yml

@@ -36,7 +36,7 @@ jobs:
     name: Check Python formatting
     runs-on: ubuntu-latest
     env:
-      PYTHON_VERSION: "3.10"
+      PYTHON_VERSION: "3.11"
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/lint.yml

@@ -46,7 +46,7 @@ jobs:
     name: Lint Python
     runs-on: ubuntu-latest
     env:
-      PYTHON_VERSION: "3.10"
+      PYTHON_VERSION: "3.11"
 
     needs: skip_check
     if: needs.skip_check.outputs.should_skip != 'true'

cli/commands/dev/build.py

@@ -27,18 +27,29 @@
 # 2. Specify the task and valid tags in the BUILD_TASKS dictionary.
 
 
-def build_tim(tag: Optional[str], no_cache: bool) -> Optional[List[str]]:
+def build_tim(
+    tag: Optional[str], no_cache: bool, build_args: List[str]
+) -> Optional[List[str]]:
+    assert tag is not None
     config = get_config()
     image_suffix = "-base" if tag == "base" else ""
     image_name = f"{config.images_repository}/tim{image_suffix}"
     image_name_specific = f"{image_name}:{tim_image_tag()}"
     image_name_latest = f"{image_name}:latest"
     cwd = Path.cwd()
     dockerfile = cwd / "timApp" / "Dockerfile"
+    build_args_cli = []
+    if build_args:
+        for arg in build_args:
+            build_args_cli.append("--build-arg")
+            build_args_cli.append(arg)
     run_docker(
         [
             "build",
             *(["--no-cache"] if no_cache else []),
+            *build_args_cli,
+            "--target",
+            tag,
             "--tag",
             image_name_specific,
             "--tag",
@@ -51,15 +62,23 @@ def build_tim(tag: Optional[str], no_cache: bool) -> Optional[List[str]]:
     return [image_name_specific, image_name_latest]
 
 
-def build_csplugin(tag: Optional[str], no_cache: bool) -> Optional[List[str]]:
+def build_csplugin(
+    tag: Optional[str], no_cache: bool, build_args: List[str]
+) -> Optional[List[str]]:
     assert tag is not None
     config = get_config()
     image_name = f"{config.images_repository}/cs3:{tag}-{csplugin_image_tag()}"
     context = Path.cwd() / "timApp" / "modules" / "cs"
+    build_args_cli = []
+    if build_args:
+        for arg in build_args:
+            build_args_cli.append("--build-arg")
+            build_args_cli.append(arg)
     run_docker(
         [
             "build",
             *(["--no-cache"] if no_cache else []),
+            *build_args_cli,
             "--tag",
             image_name,
             "--target",
@@ -76,12 +95,13 @@ def build_csplugin(tag: Optional[str], no_cache: bool) -> Optional[List[str]]:
 class Arguments:
     push: bool
     no_cache: bool
+    build_args: List[str]
     tasks: List[str]
 
 
 class BuildTask(NamedTuple):
     tags: Optional[List[str]]
-    build: Callable[[Optional[str], bool], Optional[List[str]]]
+    build: Callable[[Optional[str], bool, List[str]], Optional[List[str]]]
 
 
 BUILD_TASKS: Dict[str, BuildTask] = {
@@ -104,12 +124,12 @@ def run(args: Arguments) -> None:
         if build_tags:
             for tag in build_tags:
                 log_info(f"Building {task}:{tag}")
-                images = build_task.build(tag, args.no_cache)
+                images = build_task.build(tag, args.no_cache, args.build_args)
                 if images:
                     built_images.extend(images)
         else:
             log_info(f"Building {task}")
-            images = build_task.build(None, args.no_cache)
+            images = build_task.build(None, args.no_cache, args.build_args)
             if images:
                 built_images.extend(images)
 
@@ -137,6 +157,12 @@ def init(parser: ArgumentParser) -> None:
         action="store_true",
         dest="no_cache",
     )
+    parser.add_argument(
+        "--build-arg",
+        help="Build arguments to pass to the Docker build command.",
+        action="append",
+        dest="build_args",
+    )
     parser.add_argument(
         "tasks",
         nargs="*",

cli/commands/test.py

@@ -15,6 +15,7 @@ class Arguments:
     down: bool
     up: bool
     new_screenshots: bool
+    coverage: bool
 
 
 BROWSER_TEST_SCRIPT = """
@@ -72,7 +73,8 @@ def run(args: Arguments) -> None:
     else:
         test_parameters = ["discover", "-v", f"tests/{args.target}", "test_*.py", "."]
 
-    test_command = ["python3", "-m", "unittest", *test_parameters]
+    base_command = ["python3"] if not args.coverage else ["coverage", "run"]
+    test_command = [*base_command, "-m", "unittest", *test_parameters]
 
     # Browser tests can be flaky (in part of lackluster flask support for Selenium, in part of the tests).
     # It's better to retry it a few times
@@ -125,3 +127,9 @@ def init(parser: ArgumentParser) -> None:
         "Run a specific test module or test function. Format is <module>[.<function>]. "
         "Special value 'all' runs all tests.",
     )
+    parser.add_argument(
+        "--coverage",
+        help="Run tests with coverage",
+        action="store_true",
+        dest="coverage",
+    )

cli/templates/docker/docker-compose.tmpl.yml

@@ -71,9 +71,9 @@ services:
   profiles:
    - prod
    - dev
-  image: postgres:11
+  image: postgres:15
   volumes:
-   - data11:/var/lib/postgresql/data
+   - db_data:/var/lib/postgresql/data
   command: [
     "postgres",
     "-c", "log_statement=mod",
@@ -88,7 +88,9 @@ services:
    - /run/postgresql
    - /tmp
   environment:
+   POSTGRES_USER: postgres
    POSTGRES_PASSWORD: postgresql
+   POSTGRES_HOST_AUTH_METHOD: "md5"
   ports: ${ jsonify(["5432:5432"] if tim.is_dev else []) }
   logging:
    # According to docs, "local" is more compact and performant than the default json-file driver.
@@ -97,7 +99,7 @@ services:
     max-size: "50m"
     max-file: "100"
  postgresql-test:
-  image: postgres:11
+  image: postgres:15
   profiles:
    - test
    - dev
@@ -106,7 +108,9 @@ services:
    - /var/lib/postgresql/data
   ports: ${ jsonify(["5433:5432"] if tim.is_dev else []) }
   environment:
+   POSTGRES_USER: postgres
    POSTGRES_PASSWORD: postgresql
+   POSTGRES_HOST_AUTH_METHOD: "md5"
  csplugin:
   image: ${compose.images_repository}/cs3:${csplugin.target}-${csplugin.image_tag}
   build:
@@ -258,7 +262,7 @@ ${ partial("docker", "csplugin_mongodb.tmpl.yml") if csplugin.is_mongodb_enabled
    - redis_data:/data
  celery:
   image: ${compose.images_repository}/tim:${tim.image_tag}
-  command: celery worker -A timApp.tim_celery.celery --concurrency 4
+  command: celery -A timApp.tim_celery.celery worker --concurrency 4
   profiles:
    - prod
    - dev
@@ -291,8 +295,8 @@ ${ partial("docker", "csplugin_mongodb.tmpl.yml") if csplugin.is_mongodb_enabled
    - prod
    - dev
   command: >-
-   celery beat
-   -A timApp.tim_celery.celery
+   celery -A timApp.tim_celery.celery
+   beat
    -S timApp.celery_sqlalchemy_scheduler.schedulers:DatabaseScheduler
    --pidfile /var/run/celery/celerybeat.pid
   volumes:
@@ -396,6 +400,7 @@ ${ partial("docker", "csplugin_mongodb.tmpl.yml") if csplugin.is_mongodb_enabled
   volumes:
    - .:/service:rw
   environment:
+   TIM_TESTING: 1
    TIM_SETTINGS: testconfig.py
    PYTHONPATH: /service
    CI: ${CI:-false}
@@ -426,7 +431,7 @@ ${ partial("docker", "csplugin_mongodb.tmpl.yml") if csplugin.is_mongodb_enabled
   read_only: true
 ${ partial("docker", "mailman.tmpl.yml") if mailman.is_dev else "" }
 volumes:
- data11:
+ db_data:
  cache:
  caddy_data:
  caddy_config: