Record webhook payloads in a local SQLite database

Dockerfile

@@ -32,14 +32,14 @@ RUN true \
     && dnf update -y \
     && dnf install -y --allowerasing \
         ca-certificates \
-        curl \
         glibc-langpack-en \
         httpd \
         mod_auth_openidc \
         mod_ssl \
         procps \
         ${PY_PKG}-pip \
         ${PY_PKG}-mod_wsgi \
+        sqlite \
         supervisor \
     && dnf install -y \
         https://research.cs.wisc.edu/htcondor/repo/current/htcondor-release-current.el9.noarch.rpm \
@@ -61,13 +61,18 @@ RUN ${PY_EXE} -m pip install --ignore-installed --no-cache-dir -r /srv/requireme
 COPY registry /srv/registry/
 COPY set_version.py wsgi.py /srv/
 
-RUN (cd /srv/ && env FLASK_APP="registry" ${PY_EXE} -m flask assets build) \
+RUN true \
     #
-    && find /srv/ -name ".*" -exec rm -rf {} \; -prune \
+    && pushd /srv \
+    && env DATA_DIR=/tmp FLASK_APP=registry ${PY_EXE} -m flask assets build \
+    && popd \
     #
     && rm -rf /srv/instance/* \
+    && rm -rf /tmp/* \
     && chown apache:apache /srv/instance/ \
-    && ${PY_EXE} /srv/set_version.py
+    #
+    && ${PY_EXE} /srv/set_version.py \
+    && true
 
 # Configure container startup.
 

Makefile

@@ -35,6 +35,7 @@ clean:
 	rm -rf dist/$(PY_PACKAGE_NAME)-*.whl
 	-rmdir dist/
 
+	rm -rf instance/data
 	rm -rf instance/log
 	-docker image rm soteria-webapp:dev
 
@@ -46,6 +47,7 @@ local: \
 		secrets/oidc/id secrets/oidc/passphrase secrets/oidc/secret \
 		secrets/tls.crt secrets/tls.key
 
+	mkdir -p instance/data
 	mkdir -p instance/log
 
 	# Run `docker compose build --no-cache --pull` manually to ensure

docker-compose.yaml

@@ -33,6 +33,7 @@ services:
         target: /certs/tls.key
 
     volumes:
+      - ${PWD}/instance/data:/data
       - ${PWD}/instance/log:/srv/instance/log
 
 secrets:

registry/api/v1.py

@@ -194,9 +194,11 @@ def webhook_for_harbor():
         and auth.token == flask.current_app.config["WEBHOOKS_HARBOR_BEARER_TOKEN"]
     ):
         payload = flask.request.get_json()
+        payload_as_msg = json.dumps(payload, indent=2)
+        payload_as_text = json.dumps(payload, separators=(",", ":"))
 
-        flask.current_app.logger.info(f"Webhook called from Harbor: {json.dumps(payload)}")
-
+        flask.current_app.logger.info(f"Webhook called from Harbor: {payload_as_msg}")
+        registry.database.insert_new_payload(payload_as_text)
         return make_ok_response({"message": "webhook completed succesfully"})
 
     return make_error_response(401, "Missing authorization")

registry/app.py

@@ -7,12 +7,13 @@
 from typing import Any, Dict
 
 import flask
-import flask_assets  # type: ignore[import]
+import flask_assets  # type: ignore[import-untyped]
 
 import registry.api.debug
 import registry.api.harbor
 import registry.api.v1
 import registry.cli
+import registry.database
 import registry.public
 import registry.util
 import registry.website
@@ -32,15 +33,16 @@ def load_config(app: flask.Flask) -> None:
         app.config.from_pyfile(os.fspath(p))
 
     for key in [
+        "DATA_DIR",
         "FRESHDESK_API_KEY",
         "HARBOR_ADMIN_PASSWORD",
         "HARBOR_ADMIN_USERNAME",
-        "REGISTRY_API_USERNAME",
-        "REGISTRY_API_PASSWORD",
         "LDAP_BASE_DN",
         "LDAP_PASSWORD",
         "LDAP_URL",
         "LDAP_USERNAME",
+        "REGISTRY_API_PASSWORD",
+        "REGISTRY_API_USERNAME",
         "SECRET_KEY",
         "WEBHOOKS_HARBOR_BEARER_TOKEN",
     ]:
@@ -134,14 +136,9 @@ def create_app() -> flask.Flask:
     define_assets(app)
     add_context_processor(app)
 
+    registry.database.init(app)
     cache.init_app(app)
 
     app.logger.info("Created and configured app!")
 
     return app
-
-
-# Use to test locally
-if __name__ == "__main__":
-    app = create_app()
-    app.run(debug=True, use_reloader=True, port=9876)

registry/database.py

@@ -0,0 +1,66 @@
+"""
+Manage the web application's local database.
+
+The database serves as a permanent store for webhook payloads received from
+the associated Harbor instance, as well as a mechanism for tracking HTCondor
+jobs that process those payloads.
+"""
+
+import pathlib
+import sqlite3
+
+import flask
+
+__all__ = [
+    "get_db_conn",
+    "init",
+    "insert_new_payload",
+]
+
+DB_FILE = "database/soteria.sqlite"
+
+
+def get_db_conn() -> sqlite3.Connection:
+    """
+    Create a connection object to the database.
+    """
+    return sqlite3.connect(pathlib.Path(flask.current_app.config["DATA_DIR"]) / DB_FILE)
+
+
+def init(app: flask.Flask) -> None:
+    """
+    Ensure that the database exists and has the required tables and columns.
+    """
+    db_file = pathlib.Path(app.config["DATA_DIR"]) / DB_FILE
+    db_file.parent.mkdir(parents=True, exist_ok=True)
+
+    if not db_file.exists():
+        db_file.touch()
+    with sqlite3.connect(db_file) as conn:
+        conn.execute(
+            """
+            CREATE TABLE IF NOT EXISTS webhook_payloads
+            (
+              id INTEGER PRIMARY KEY
+            , payload TEXT
+            )
+            """,
+        )
+        conn.commit()
+
+
+def insert_new_payload(data: str) -> None:
+    """
+    Add a new webhook payload to the database.
+    """
+    with get_db_conn() as conn:
+        conn.execute(
+            """
+            INSERT INTO webhook_payloads
+            ( payload )
+            VALUES
+            ( :payload )
+            """,
+            {"payload": data},
+        )
+        conn.commit()

templates/config.py

@@ -10,6 +10,12 @@
 #
 CONTACT_EMAIL = "[email protected]"
 
+#
+# The directory to use for long-term storage of data.
+# An environment variable of the same name will override the value set here.
+#
+DATA_DIR = "/data"
+
 #
 # The base URL for SOTERIA's online documentation.
 #