using postgres

.dockerignore

@@ -6,3 +6,4 @@
 !pyproject.toml
 !conftest.py
 !pytest.ini
+!init.sql

.env

@@ -1,4 +1,3 @@
-BASE_API_URL="http://localhost:8000"
 IsDebug=True
 LOG_LEVEL=DEBUG
 MUSIC_BOX_BUILD_DIR=/music-box/build
@@ -11,3 +10,8 @@ RABBIT_MQ_PORT=5672
 RABBIT_MQ_USER=guest
 SWAGGER_BASE_PATH=''
 SECRET_KEY=
+DATABASE_NAME=musicbox
+DATABASE_USER=musicbox_user
+DATABASE_PASSWORD=musicbox_password
+DATABASE_HOST=postgres
+DATABASE_PORT=5432

.github/workflows/pytest.yml

@@ -25,14 +25,6 @@ jobs:
         with:
           compose-file: "./docker-compose.yml"
 
-      - name: Run migrations
-        run: |
-          docker exec api-server bash -c "
-          cd interactive &&
-          poetry run python manage.py makemigrations &&
-          poetry run python manage.py migrate"
-
-
       - name: Run pytest
         run: |
           docker compose exec -T api-server sh -c "poetry run pytest"

docker-compose-prod.yml

@@ -8,10 +8,8 @@ services:
       - 8000:8000
     env_file:
       - /home/shared/.env
-    entrypoint: ["poetry", "run", "python", "interactive/manage.py"]
-    command: ["runserver_plus", "--cert-file", "/certificates/acom.ucar.edu.crt", "--key-file", "/certificates/acom.ucar.edu.key", "0.0.0.0:8000"]
+    entrypoint: ["sh", "-c", "poetry run python3 interactive/manage.py makemigrations && poetry run python3 interactive/manage.py migrate && poetry run python3 interactive/manage.py collectstatic --noinput && poetry run python3 interactive/manage.py runserver_plus --cert-file /certificates/acom.ucar.edu.crt --key-file /certificates/acom.ucar.edu.key 0.0.0.0:8000"]
     volumes:
-      - db-data:/music-box-interactive/interactive
       - /etc/pki/:/certificates
       # - partmc-data:/music-box-interactive/interactive/partmc-volume
     network_mode: "host"
@@ -31,15 +29,22 @@ services:
     deploy:
       mode: replicated
       replicas: 6
-    volumes:
-      # - partmc-data:/partmc/partmc-volume
-      - db-data:/music-box-interactive/interactive
     network_mode: "host"
     logging:
       driver: "json-file"
       options:
         max-size: "10m"    # Maximum size of each log file, e.g., 10 megabytes
         max-file: "5"      # Maximum number of log files to retain
+  postgres:
+    container_name: 'postgres'
+    network_mode: "host"
+    build: 
+      context: .
+      dockerfile: ./docker/Dockerfile.postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
 volumes:
-  db-data:
   # partmc-data:
+  postgres-data:

docker-compose.yml

@@ -10,11 +10,33 @@ services:
       - 8000:8000
     env_file:
       - ./.env
-    entrypoint: ["poetry", "run", "python", "interactive/manage.py"]
-    command: ["runserver_plus", "0.0.0.0:8000"]
+    command: >
+      sh -c "poetry run python3 interactive/manage.py makemigrations &&
+             poetry run python3 interactive/manage.py migrate &&
+             poetry run python3 interactive/manage.py collectstatic --noinput &&
+             poetry run python3 interactive/manage.py runserver_plus 0.0.0.0:8000"
     volumes:
-      - db-data:/music-box-interactive/interactive
       - partmc-data:/music-box-interactive/interactive/partmc-volume
+    depends_on:
+      - postgres
+  postgres:
+    container_name: 'postgres'
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres_password
+    build: 
+      context: .
+      dockerfile: ./docker/Dockerfile.postgres
+    ports:
+      - "5432:5432"
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U postgres"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
   model-runner:
     build: 
       context: .
@@ -25,15 +47,14 @@ services:
       - ./.env
     depends_on:
       - rabbitmq
+      - postgres
     restart: on-failure
     deploy:
       mode: replicated
       replicas: 6
-    security_opt:
-      - seccomp:unconfined
+    command: ["poetry", "run", "python3", "/music-box-interactive/interactive/rabbit_mq_model_runner.py"]
     volumes:
       - partmc-data:/partmc/partmc-volume
-      - db-data:/music-box-interactive/interactive
   rabbitmq:
     container_name: 'rabbitmq'
     image: rabbitmq:3.11-management
@@ -45,9 +66,9 @@ services:
       - ~/.docker-conf/rabbitmq/log/:/var/log/rabbitmq
     healthcheck:
       test: rabbitmq-diagnostics -q ping
-      interval: 5s
-      timeout: 15s
-      retries: 1
+      interval: 10s
+      timeout: 30s
+      retries: 10
 volumes:
-  db-data:
-  partmc-data:
+  partmc-data:
+  postgres-data:

docker/Dockerfile.api

@@ -30,8 +30,4 @@ RUN poetry config virtualenvs.in-project true
 ENV CXXFLAGS="-Wno-error=redundant-move"
 RUN if [ "$INSTALL_PyPartMC" = "true" ]; then poetry install --all-extras; else poetry install; fi
 
-RUN poetry run python3 interactive/manage.py makemigrations
-RUN poetry run python3 interactive/manage.py migrate
-RUN poetry run python3 interactive/manage.py collectstatic --noinput
-
 EXPOSE 8000

docker/Dockerfile.postgres

@@ -0,0 +1,3 @@
+FROM postgres:latest
+
+COPY init.sql /docker-entrypoint-initdb.d/

init.sql

@@ -0,0 +1,14 @@
+-- Create the database
+CREATE DATABASE musicbox;
+
+-- Create the user with the specified password
+CREATE USER musicbox_user WITH PASSWORD 'musicbox_password';
+
+-- Grant all privileges on the database to the user
+GRANT ALL PRIVILEGES ON DATABASE musicbox TO musicbox_user;
+
+-- Optional: Set the default privileges for the user to own tables they create
+ALTER DATABASE musicbox OWNER TO musicbox_user;
+
+-- allow the user to create databases
+ALTER USER musicbox_user CREATEDB;

interactive/api/models.py

@@ -4,10 +4,7 @@
 
 class ModelRun(models.Model):
     # uid of user [CharField]
-    uid = models.CharField(max_length=50, primary_key=True)
-
-    # checksum of config files [CharField]
-    config_checksum = models.CharField(max_length=50)
+    uid = models.CharField(max_length=60, primary_key=True)
 
     # status of the model run [CharField]
     status = models.CharField(
@@ -16,6 +13,3 @@ class ModelRun(models.Model):
 
     # results name and binary data [JSONField]
     results = models.JSONField(default=dict)  # {name: binary data}
-
-    # should cache?
-    should_cache = models.BooleanField(default=True)

interactive/manage/settings.py

@@ -82,8 +82,12 @@
 
 DATABASES = {
     'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(BASE_DIR, 'db.sqlite3'),
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': os.environ.get('DB_NAME', 'musicbox'),
+        'USER': os.environ.get('DB_USER', 'musicbox_user'),
+        'PASSWORD': os.environ.get('DB_PASSWORD', 'musicbox_password'),
+        'HOST': os.environ.get('DB_HOST', 'postgres'),
+        'PORT': os.environ.get('DB_PORT', '5432'),
     }
 }
 
@@ -182,6 +186,10 @@
         },
     },
     'loggers': {
+        'django.db.backends': {
+            'handlers': ['console'],
+            'level': 'INFO',
+        },
         'django': {
             'handlers': ['console'],
             'level': 'INFO',

pyproject.toml

@@ -20,6 +20,7 @@ pandas = "^2.0"
 pika = "^1.2"
 pyOpenSSL = "^22.0"
 python = "^3.9"
+psycopg2-binary = "^2.9"
 scipy = "^1.7"
 setuptools = "^70.0"
 Werkzeug = "^2.0"