Merge pull request #572 from scidsg/use-postgres-everywhere

Use postgres everywhere

.github/workflows/tests.yml

@@ -10,6 +10,15 @@ on:
 jobs:
   run-lint-and-tests:
     runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:16.4-alpine3.20
+        env:
+          POSTGRES_USER: hushline
+          POSTGRES_PASSWORD: hushline
+          POSTGRES_DB: hushline
+        ports:
+          - 127.0.0.1:5432:5432
     steps:
       - uses: actions/checkout@v4
 

.prettierignore

@@ -2,4 +2,4 @@
 build
 coverage
 hushline/static/vendor/*
-.pytest_cache
+.pytest_cache

dev_env.sh

@@ -1,7 +1,7 @@
 export FLASK_APP=hushline
 export ENCRYPTION_KEY=bi5FDwhZGKfc4urLJ_ChGtIAaOPgxd3RDOhnvct10mw=
 export SECRET_KEY=cb3f4afde364bfb3956b97ca22ef4d2b593d9d980a4330686267cabcd2c0befd
-export SQLALCHEMY_DATABASE_URI=sqlite:///hushline.db
+export SQLALCHEMY_DATABASE_URI=postgresql://hushline:[email protected]:5432/hushline
 export REGISTRATION_CODES_REQUIRED=False
 export SESSION_COOKIE_NAME=session
 export [email protected]

docker-compose.yaml

@@ -1,27 +1,28 @@
+---
 services:
   app:
     build: .
     ports:
-      - "8080:8080"
+      - 127.0.0.1:8080:8080
     environment:
-      - SQLALCHEMY_DATABASE_URI=postgresql://hushline:hushline@postgres:5432/hushline
-      - FLASK_ENV=development
-      - SECRET_KEY="cb3f4afde364bfb3956b97ca22ef4d2b593d9d980a4330686267cabcd2c0befd"
-      - ENCRYPTION_KEY="bi5FDwhZGKfc4urLJ_ChGtIAaOPgxd3RDOhnvct10mw="
-      - REGISTRATION_CODES_REQUIRED=false
+      SQLALCHEMY_DATABASE_URI: postgresql://hushline:hushline@postgres:5432/hushline
+      FLASK_ENV: development
+      SECRET_KEY: "cb3f4afde364bfb3956b97ca22ef4d2b593d9d980a4330686267cabcd2c0befd"
+      ENCRYPTION_KEY: "bi5FDwhZGKfc4urLJ_ChGtIAaOPgxd3RDOhnvct10mw="
+      REGISTRATION_CODES_REQUIRED: false
     volumes:
       - ./volumes/app:/data
       - ./:/app
     depends_on:
       - postgres
 
   postgres:
-    image: "postgres:alpine"
+    image: postgres:16.4-alpine3.20
     environment:
-      - POSTGRES_USER=hushline
-      - POSTGRES_PASSWORD=hushline
-      - POSTGRES_DB=hushline
+      POSTGRES_USER: hushline
+      POSTGRES_PASSWORD: hushline
+      POSTGRES_DB: hushline
     ports:
-      - "5432:5432"
+      - 127.0.0.1:5432:5432
     volumes:
       - ./volumes/postgres:/var/lib/postgresql/data

docs/DEV.md

@@ -56,6 +56,9 @@ make run
 Run the tests:
 
 ```sh
+# in one terminal
+./scripts/local-postgres.sh
+# in another terminal
 make test
 ```
 

scripts/local-postgres.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+docker run --rm -t -p 127.0.0.1:5432:5432 \
+    -e POSTGRES_USER=hushline \
+    -e POSTGRES_PASSWORD=hushline \
+    -e POSTGRES_DB=hushline \
+    postgres:16.4-alpine3.20

tests/conftest.py

@@ -1,17 +1,105 @@
 import os
+import random
+import string
+import time
+from contextlib import contextmanager
 from typing import Generator
 
 import pytest
 from cryptography.fernet import Fernet
 from flask import Flask
 from flask.testing import FlaskClient
 from pytest_mock import MockFixture
+from sqlalchemy import create_engine, text
+from sqlalchemy.exc import OperationalError
+from sqlalchemy.orm import Session, sessionmaker
 
 from hushline import create_app, db
 
-# TODO once we refactor `fernet` to not be global, move this into the `config` fixture.
-# this needs to be imported before importing `hushline`
-os.environ["ENCRYPTION_KEY"] = Fernet.generate_key().decode()
+CONN_FMT_STR = "postgresql+psycopg://hushline:[email protected]:5432/{database}"
+TEMPLATE_DB_NAME = "app_db_template"
+
+
+def random_name(size: int) -> str:
+    return "".join([random.choice(string.ascii_lowercase) for _ in range(size)])  # noqa: S311
+
+
+@contextmanager
+def temp_session(conn_str: str) -> Generator[Session, None, None]:
+    engine = create_engine(conn_str)
+    engine = engine.execution_options(isolation_level="AUTOCOMMIT")
+    session = sessionmaker(bind=engine)()
+
+    yield session
+
+    # aggressively terminate all connections
+    session.close()
+    session.connection().connection.invalidate()
+    engine.dispose()
+
+
+@pytest.fixture(scope="session")
+def _db_template() -> Generator[None, None, None]:
+    """A template database that all other databases are created from.
+    Effectively allows caching of `db.create_all()`"""
+
+    conn_str = CONN_FMT_STR.format(database="hushline")
+
+    counter = 5
+    while True:
+        try:
+            with temp_session(conn_str) as session:
+                session.execute(text(f"CREATE DATABASE {TEMPLATE_DB_NAME} WITH TEMPLATE template1"))
+            break
+        except OperationalError:
+            if counter > 0:
+                counter -= 1
+                time.sleep(1)
+            else:
+                raise
+
+    db_uri = CONN_FMT_STR.format(database=TEMPLATE_DB_NAME)
+    init_db_via_create_all(db_uri)
+
+    yield
+
+    with temp_session(conn_str) as session:
+        session.execute(text(f"DROP DATABASE {TEMPLATE_DB_NAME}"))
+
+
+def init_db_via_create_all(db_uri: str) -> None:
+    # dumb hack to easily get the create_all() functionality
+    os.environ["SQLALCHEMY_DATABASE_URI"] = db_uri
+    app = create_app()
+
+    with app.app_context():
+        db.session.commit()
+        db.create_all()
+        db.session.close()
+        db.session.connection().connection.invalidate()  # type: ignore
+
+
+@pytest.fixture()
+def database(_db_template: None) -> str:
+    """A clean Postgres database from the template with DDLs applied"""
+    db_name = random_name(16)
+    conn_str = CONN_FMT_STR.format(database="hushline")
+    engine = create_engine(conn_str)
+    engine = engine.execution_options(isolation_level="AUTOCOMMIT")
+
+    session = sessionmaker(bind=engine)()
+
+    sql = text(f"CREATE DATABASE {db_name} WITH TEMPLATE {TEMPLATE_DB_NAME}")
+    session.execute(sql)
+
+    # aggressively terminate all connections
+    session.close()
+    session.connection().connection.invalidate()
+    engine.dispose()
+
+    print(f"Postgres DB: {db_name}, template: {TEMPLATE_DB_NAME}")  # to help with debugging tests
+
+    return db_name
 
 
 @pytest.fixture()
@@ -20,10 +108,11 @@ def _config(mocker: MockFixture) -> None:
 
 
 @pytest.fixture()
-def app(_config: None) -> Generator[Flask, None, None]:
-    os.environ["SQLALCHEMY_DATABASE_URI"] = "sqlite:///:memory:"
+def app(_config: None, database: str) -> Generator[Flask, None, None]:
     os.environ["SQLALCHEMY_TRACK_MODIFICATIONS"] = "False"
     os.environ["REGISTRATION_CODES_REQUIRED"] = "False"
+    os.environ["ENCRYPTION_KEY"] = Fernet.generate_key().decode()
+    os.environ["SQLALCHEMY_DATABASE_URI"] = CONN_FMT_STR.format(database=database)
 
     app = create_app()
     app.config["TESTING"] = True
@@ -32,9 +121,7 @@ def app(_config: None) -> Generator[Flask, None, None]:
     app.config["PREFERRED_URL_SCHEME"] = "http"
 
     with app.app_context():
-        db.create_all()
         yield app
-        db.drop_all()
 
 
 @pytest.fixture()