From 061113bee1f030bde6fc04b40d823414077969c4 Mon Sep 17 00:00:00 2001 From: Brian Aydemir Date: Tue, 26 Mar 2024 13:14:15 -0500 Subject: [PATCH] Record webhook payloads in a local SQLite database --- Dockerfile | 13 ++++++--- Makefile | 2 ++ docker-compose.yaml | 1 + registry/api/v1.py | 6 ++-- registry/app.py | 15 ++++------ registry/database.py | 66 ++++++++++++++++++++++++++++++++++++++++++++ templates/config.py | 6 ++++ 7 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 registry/database.py diff --git a/Dockerfile b/Dockerfile index bd969a8..3703e7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,7 +32,6 @@ RUN true \ && dnf update -y \ && dnf install -y --allowerasing \ ca-certificates \ - curl \ glibc-langpack-en \ httpd \ mod_auth_openidc \ @@ -40,6 +39,7 @@ RUN true \ procps \ ${PY_PKG}-pip \ ${PY_PKG}-mod_wsgi \ + sqlite \ supervisor \ && dnf install -y \ https://research.cs.wisc.edu/htcondor/repo/current/htcondor-release-current.el9.noarch.rpm \ @@ -61,13 +61,18 @@ RUN ${PY_EXE} -m pip install --ignore-installed --no-cache-dir -r /srv/requireme COPY registry /srv/registry/ COPY set_version.py wsgi.py /srv/ -RUN (cd /srv/ && env FLASK_APP="registry" ${PY_EXE} -m flask assets build) \ +RUN true \ # - && find /srv/ -name ".*" -exec rm -rf {} \; -prune \ + && pushd /srv \ + && env DATA_DIR=/tmp FLASK_APP=registry ${PY_EXE} -m flask assets build \ + && popd \ # && rm -rf /srv/instance/* \ + && rm -rf /tmp/* \ && chown apache:apache /srv/instance/ \ - && ${PY_EXE} /srv/set_version.py + # + && ${PY_EXE} /srv/set_version.py \ + && true # Configure container startup. diff --git a/Makefile b/Makefile index 75213ff..152bb59 100644 --- a/Makefile +++ b/Makefile @@ -35,6 +35,7 @@ clean: rm -rf dist/$(PY_PACKAGE_NAME)-*.whl -rmdir dist/ + rm -rf instance/data rm -rf instance/log -docker image rm soteria-webapp:dev @@ -46,6 +47,7 @@ local: \ secrets/oidc/id secrets/oidc/passphrase secrets/oidc/secret \ secrets/tls.crt secrets/tls.key + mkdir -p instance/data mkdir -p instance/log # Run `docker compose build --no-cache --pull` manually to ensure diff --git a/docker-compose.yaml b/docker-compose.yaml index c44d1aa..71000ea 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -33,6 +33,7 @@ services: target: /certs/tls.key volumes: + - ${PWD}/instance/data:/data - ${PWD}/instance/log:/srv/instance/log secrets: diff --git a/registry/api/v1.py b/registry/api/v1.py index 348037c..a11667f 100644 --- a/registry/api/v1.py +++ b/registry/api/v1.py @@ -194,9 +194,11 @@ def webhook_for_harbor(): and auth.token == flask.current_app.config["WEBHOOKS_HARBOR_BEARER_TOKEN"] ): payload = flask.request.get_json() + payload_as_msg = json.dumps(payload, indent=2) + payload_as_text = json.dumps(payload, separators=(",", ":")) - flask.current_app.logger.info(f"Webhook called from Harbor: {json.dumps(payload)}") - + flask.current_app.logger.info(f"Webhook called from Harbor: {payload_as_msg}") + registry.database.insert_new_payload(payload_as_text) return make_ok_response({"message": "webhook completed succesfully"}) return make_error_response(401, "Missing authorization") diff --git a/registry/app.py b/registry/app.py index ed3ea71..cafe85f 100644 --- a/registry/app.py +++ b/registry/app.py @@ -7,12 +7,13 @@ from typing import Any, Dict import flask -import flask_assets # type: ignore[import] +import flask_assets # type: ignore[import-untyped] import registry.api.debug import registry.api.harbor import registry.api.v1 import registry.cli +import registry.database import registry.public import registry.util import registry.website @@ -32,15 +33,16 @@ def load_config(app: flask.Flask) -> None: app.config.from_pyfile(os.fspath(p)) for key in [ + "DATA_DIR", "FRESHDESK_API_KEY", "HARBOR_ADMIN_PASSWORD", "HARBOR_ADMIN_USERNAME", - "REGISTRY_API_USERNAME", - "REGISTRY_API_PASSWORD", "LDAP_BASE_DN", "LDAP_PASSWORD", "LDAP_URL", "LDAP_USERNAME", + "REGISTRY_API_PASSWORD", + "REGISTRY_API_USERNAME", "SECRET_KEY", "WEBHOOKS_HARBOR_BEARER_TOKEN", ]: @@ -134,14 +136,9 @@ def create_app() -> flask.Flask: define_assets(app) add_context_processor(app) + registry.database.init(app) cache.init_app(app) app.logger.info("Created and configured app!") return app - - -# Use to test locally -if __name__ == "__main__": - app = create_app() - app.run(debug=True, use_reloader=True, port=9876) diff --git a/registry/database.py b/registry/database.py new file mode 100644 index 0000000..7df275c --- /dev/null +++ b/registry/database.py @@ -0,0 +1,66 @@ +""" +Manage the web application's local database. + +The database serves as a permanent store for webhook payloads received from +the associated Harbor instance, as well as a mechanism for tracking HTCondor +jobs that process those payloads. +""" + +import pathlib +import sqlite3 + +import flask + +__all__ = [ + "get_db_conn", + "init", + "insert_new_payload", +] + +DB_FILE = "database/soteria.sqlite" + + +def get_db_conn() -> sqlite3.Connection: + """ + Create a connection object to the database. + """ + return sqlite3.connect(pathlib.Path(flask.current_app.config["DATA_DIR"]) / DB_FILE) + + +def init(app: flask.Flask) -> None: + """ + Ensure that the database exists and has the required tables and columns. + """ + db_file = pathlib.Path(app.config["DATA_DIR"]) / DB_FILE + db_file.parent.mkdir(parents=True, exist_ok=True) + + if not db_file.exists(): + db_file.touch() + with sqlite3.connect(db_file) as conn: + conn.execute( + """ + CREATE TABLE IF NOT EXISTS webhook_payloads + ( + id INTEGER PRIMARY KEY + , payload TEXT + ) + """, + ) + conn.commit() + + +def insert_new_payload(data: str) -> None: + """ + Add a new webhook payload to the database. + """ + with get_db_conn() as conn: + conn.execute( + """ + INSERT INTO webhook_payloads + ( payload ) + VALUES + ( :payload ) + """, + {"payload": data}, + ) + conn.commit() diff --git a/templates/config.py b/templates/config.py index 1e7b8ac..9d4fd28 100644 --- a/templates/config.py +++ b/templates/config.py @@ -10,6 +10,12 @@ # CONTACT_EMAIL = "someone@example.com" +# +# The directory to use for long-term storage of data. +# An environment variable of the same name will override the value set here. +# +DATA_DIR = "/data" + # # The base URL for SOTERIA's online documentation. #