Feat minio hot storage

.devcontainer/devcontainer.json

@@ -7,7 +7,7 @@
       "../deployment/docker-compose.override.devcontainer.yml"
     ],
     "service": "dev",
-    "runServices": ["db", "redis", "s3", "celery_beat", "worker", "dev", "createbuckets"],
+    "runServices": ["db", "redis", "minio", "s3", "celery_beat", "worker", "dev", "createbuckets"],
     "workspaceFolder": "/home/web/project",
     "runArgs": [
       "--env-file",

.vscode/launch.json

@@ -13,7 +13,7 @@
                 "runserver", "8000"
             ],
             "django": true,
-            "justMyCode": true,
+            "justMyCode": false,
         }
     ]
 }

deployment/.template.env

@@ -8,5 +8,9 @@ REDIS_HOST=redis
 REDIS_PASSWORD=redis_password
 RABBITMQ_HOST=rabbitmq
 SENTRY_DSN=sentry_dsn
-AWS_ACCESS_KEY_ID=minios3admin
-AWS_SECRET_ACCESS_KEY=minios3admin
+S3_AWS_ACCESS_KEY_ID=minios3admin
+S3_AWS_SECRET_ACCESS_KEY=minios3admin
+AWS_S3_ENDPOINT=http://s3:9000
+MINIO_ACCESS_KEY_ID=miniocplus
+MINIO_SECRET_ACCESS_KEY=miniocplus
+MINIO_ENDPOINT=http://minio:9000

deployment/.template.test.env

@@ -8,5 +8,9 @@ REDIS_HOST=redis
 REDIS_PASSWORD=redis_password
 RABBITMQ_HOST=rabbitmq
 SENTRY_DSN=sentry_dsn
-AWS_ACCESS_KEY_ID=minios3admin
-AWS_SECRET_ACCESS_KEY=minios3admin
+S3_AWS_ACCESS_KEY_ID=minios3admin
+S3_AWS_SECRET_ACCESS_KEY=minios3admin
+AWS_S3_ENDPOINT=http://s3:9000
+MINIO_ACCESS_KEY_ID=miniocplus
+MINIO_SECRET_ACCESS_KEY=miniocplus
+MINIO_ENDPOINT=http://minio:9000

deployment/docker-compose.override.devcontainer.yml

@@ -21,8 +21,6 @@ services:
       - ../:/home/web/project
       - ../django_project:/home/web/django_project
       - ./volumes/media:/home/web/media
-    links:
-      - s3
 
   celery_beat:
     image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_worker_dev
@@ -45,9 +43,6 @@ services:
       - ../:/home/web/project
       - ./volumes/media:/home/web/media
       - ./volumes/static:/home/web/static
-    links:
-      - db
-      - worker
 
   s3:
     image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
@@ -56,22 +51,31 @@ services:
       - "9000:9000"
       - "9001:9001"
     environment:
-      MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID:-minios3admin}
-      MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_ROOT_USER: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+      MINIO_ROOT_PASSWORD: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
     volumes:
       - ./volumes/s3_data:/data
     restart: always
 
+  minio:
+    volumes:
+      - ./volumes/minio_data:/data
+
   createbuckets:
     image: minio/mc
     depends_on:
       - s3
+      - minio
     environment:
-      MINIO_S3_USER: ${AWS_ACCESS_KEY_ID:-minios3admin}
-      MINIO_S3_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_S3_USER: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+      MINIO_S3_PASSWORD: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_USER: ${MINIO_ACCESS_KEY_ID:-miniocplus}
+      MINIO_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-miniocplus}
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc alias set s3minio http://s3:9000 $${MINIO_S3_USER} $${MINIO_S3_PASSWORD};
       /usr/bin/mc mb --ignore-existing s3minio/cplus;
+      /usr/bin/mc alias set myminio http://minio:9000 $${MINIO_USER} $${MINIO_PASSWORD};
+      /usr/bin/mc mb --ignore-existing myminio/cplus;
       exit 0;
       "

deployment/docker-compose.override.template.yml

@@ -53,7 +53,6 @@ services:
       - ./volumes/static:/home/web/static
       - ./volumes/media:/home/web/media
     links:
-      - db
       - worker
 
   nginx:
@@ -62,8 +61,10 @@ services:
       - ./volumes/static:/home/web/static
     ports:
       - "${HTTP_PORT:-8888}:80"
-    links:
-      - django
+
+  minio:
+    volumes:
+      - ./volumes/minio_data:/data
 
   s3:
     image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
@@ -72,8 +73,8 @@ services:
       - "9000:9000"
       - "9001:9001"
     environment:
-      MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID:-minios3admin}
-      MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_ROOT_USER: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+      MINIO_ROOT_PASSWORD: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
     volumes:
       - ./volumes/s3_data:/data
     restart: always

deployment/docker-compose.override.test.yml

@@ -14,6 +14,7 @@ services:
       - ./volumes/media:/home/web/media
     links:
       - s3
+      - minio
 
   dev:
     image: kartoza/cplus-api:test
@@ -27,6 +28,10 @@ services:
       - worker
     depends_on:
       - createbuckets
+
+  minio:
+    volumes:
+      - ./volumes/minio_data:/data
 
   s3:
     image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
@@ -35,8 +40,8 @@ services:
       - "9000:9000"
       - "9001:9001"
     environment:
-      MINIO_ROOT_USER: ${AWS_ACCESS_KEY_ID:-minios3admin}
-      MINIO_ROOT_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_ROOT_USER: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+      MINIO_ROOT_PASSWORD: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
     volumes:
       - ./volumes/s3_data:/data
     restart: always
@@ -45,12 +50,17 @@ services:
     image: minio/mc
     depends_on:
       - s3
+      - minio
     environment:
-      MINIO_S3_USER: ${AWS_ACCESS_KEY_ID:-minios3admin}
-      MINIO_S3_PASSWORD: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_S3_USER: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+      MINIO_S3_PASSWORD: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
+      MINIO_USER: ${MINIO_ACCESS_KEY_ID:-miniocplus}
+      MINIO_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-miniocplus}
     entrypoint: >
       /bin/sh -c "
       /usr/bin/mc alias set s3minio http://s3:9000 $${MINIO_S3_USER} $${MINIO_S3_PASSWORD};
       /usr/bin/mc mb --ignore-existing s3minio/cplus;
+      /usr/bin/mc alias set myminio http://minio:9000 $${MINIO_USER} $${MINIO_PASSWORD};
+      /usr/bin/mc mb --ignore-existing myminio/cplus;
       exit 0;
       "

deployment/docker-compose.yml

@@ -8,6 +8,7 @@ volumes:
   nginx-cache:
   backups-data:
   data-volume:
+  minio-data:
 
 x-common-variables: &common-variables
   # editable in .env
@@ -30,8 +31,13 @@ x-common-variables: &common-variables
   # worker variables
   CPLUS_QUEUE_CONCURRENCY: ${CPLUS_QUEUE_CONCURRENCY:-1}
   # s3 variable
-  AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-minios3admin}
-  AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-minios3admin}
+  S3_AWS_ACCESS_KEY_ID: ${S3_AWS_ACCESS_KEY_ID:-minios3admin}
+  S3_AWS_SECRET_ACCESS_KEY: ${S3_AWS_SECRET_ACCESS_KEY:-minios3admin}
+  AWS_S3_ENDPOINT: ${AWS_S3_ENDPOINT}
+  # minio variable
+  MINIO_ACCESS_KEY_ID: ${MINIO_ACCESS_KEY_ID:-miniocplus}
+  MINIO_SECRET_ACCESS_KEY: ${MINIO_SECRET_ACCESS_KEY:-miniocplus}
+  MINIO_ENDPOINT: ${MINIO_ENDPOINT}
 
 
 x-common-django:
@@ -109,6 +115,10 @@ services:
       CPLUS_WORKER: 1
     volumes:
       - media-data:/home/web/media
+    links:
+      - db
+      - minio
+      - redis
 
   dev:
     image: kartoza/${COMPOSE_PROJECT_NAME:-django_project}_dev
@@ -133,3 +143,30 @@ services:
       - static-data:/home/web/static
     links:
       - django
+
+  minio:
+    image: quay.io/minio/minio:RELEASE.2024-03-30T09-41-56Z.fips
+    command: minio server /data --console-address ":9001"
+    ports:
+      - "9010:9000"
+      - "9011:9001"
+    environment:
+      MINIO_ROOT_USER: ${MINIO_ACCESS_KEY_ID:-miniocplus}
+      MINIO_ROOT_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-miniocplus}
+    volumes:
+      - minio-data:/data
+    restart: always
+
+  createbuckets:
+    image: minio/mc
+    depends_on:
+      - minio
+    environment:
+      MINIO_USER: ${MINIO_ACCESS_KEY_ID:-miniocplus}
+      MINIO_PASSWORD: ${MINIO_SECRET_ACCESS_KEY:-miniocplus}
+    entrypoint: >
+      /bin/sh -c "
+      /usr/bin/mc alias set myminio http://minio:9000 $${MINIO_USER} $${MINIO_PASSWORD};
+      /usr/bin/mc mb --ignore-existing myminio/cplus;
+      exit 0;
+      "

django_project/core/settings/dev.py

@@ -54,19 +54,3 @@
         'level': 'WARNING'
     }
 }
-
-# aws s3 dev configuration
-STORAGES = {
-    "default": {
-        "BACKEND": "storages.backends.s3.S3Storage",
-        "OPTIONS": {
-          "bucket_name": "cplus",
-          "endpoint_url": "http://s3:9000",
-          "file_overwrite": False,
-          "max_memory_size": 300 * 1024 * 1024,  # 300MB
-        },
-    },
-    "staticfiles": {
-        "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
-    },
-}

django_project/core/settings/project.py

@@ -49,12 +49,27 @@
     "default": {
         "BACKEND": "storages.backends.s3.S3Storage",
         "OPTIONS": {
+          "access_key": os.environ.get("S3_AWS_ACCESS_KEY_ID"),
+          "secret_key": os.environ.get("S3_AWS_SECRET_ACCESS_KEY"),
           "bucket_name": "cplus",
           "file_overwrite": False,
           "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "endpoint_url": os.environ.get("AWS_S3_ENDPOINT"),
+          "session_profile": None
         },
     },
     "staticfiles": {
         "BACKEND": "django.contrib.staticfiles.storage.ManifestStaticFilesStorage",
     },
+    "minio": {
+        "BACKEND": "storages.backends.s3.S3Storage",
+        "OPTIONS": {
+          "access_key": os.environ.get("MINIO_ACCESS_KEY_ID"),
+          "secret_key": os.environ.get("MINIO_SECRET_ACCESS_KEY"),
+          "bucket_name": "cplus",
+          "file_overwrite": False,
+          "max_memory_size": 300 * 1024 * 1024,  # 300MB
+          "endpoint_url": os.environ.get("MINIO_ENDPOINT"),
+        },
+    },
 }

django_project/cplus_api/migrations/0002_outputlayer_inputlayer.py

@@ -1,4 +1,4 @@
-# Generated by Django 4.2.7 on 2024-04-04 15:53
+# Generated by Django 4.2.7 on 2024-04-05 17:23
 
 import cplus_api.models.layer
 from django.conf import settings
@@ -44,10 +44,11 @@ class Migration(migrations.Migration):
                 ('created_on', models.DateTimeField()),
                 ('layer_type', models.IntegerField(choices=[(0, 'Raster'), (1, 'Vector'), (-1, 'Undefined')])),
                 ('size', models.IntegerField(blank=True, default=0, null=True)),
-                ('file', models.FileField(upload_to=cplus_api.models.layer.input_layer_dir_path)),
+                ('file', models.FileField(storage=cplus_api.models.layer.select_input_layer_storage, upload_to=cplus_api.models.layer.input_layer_dir_path)),
                 ('component_type', models.CharField(choices=[('ncs_pathway', 'ncs_pathway'), ('ncs_carbon', 'ncs_carbon'), ('priority_layer', 'priority_layer')], max_length=255)),
                 ('privacy_type', models.CharField(choices=[('private', 'private'), ('internal', 'internal'), ('common', 'common')], default='private', max_length=255)),
                 ('last_used_on', models.DateTimeField(blank=True, null=True)),
+                ('client_id', models.TextField(blank=True, null=True)),
                 ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
             ],
             options={

django_project/cplus_api/models/layer.py

@@ -4,9 +4,11 @@
 from django.utils.translation import gettext_lazy as _
 from django.conf import settings
 from django.utils import timezone
+from django.core.files.storage import storages
 
 
 def input_layer_dir_path(instance, filename):
+    """Return upload directory path for Input Layer."""
     file_path = f'{str(instance.owner.pk)}/'
     if instance.privacy_type == InputLayer.PrivacyTypes.COMMON:
         file_path = 'common_layers/'
@@ -17,13 +19,19 @@ def input_layer_dir_path(instance, filename):
 
 
 def output_layer_dir_path(instance, filename):
+    """Return upload directory path for Output Layer."""
     file_path = f'{str(instance.owner.pk)}/{str(instance.scenario.uuid)}/'
     if not instance.is_final_output:
         file_path = file_path + f'{instance.group}/'
     file_path = file_path + filename
     return file_path
 
 
+def select_input_layer_storage():
+    """Return minio storage for input layer."""
+    return storages['minio']
+
+
 class BaseLayer(models.Model):
     class LayerTypes(models.IntegerChoices):
         RASTER = 0, _('Raster')
@@ -70,7 +78,8 @@ class PrivacyTypes(models.TextChoices):
         COMMON = 'common', _('common')
 
     file = models.FileField(
-        upload_to=input_layer_dir_path
+        upload_to=input_layer_dir_path,
+        storage=select_input_layer_storage
     )
 
     component_type = models.CharField(
@@ -89,6 +98,11 @@ class PrivacyTypes(models.TextChoices):
         blank=True
     )
 
+    client_id = models.TextField(
+        null=True,
+        blank=True
+    )
+
     def download_to_working_directory(self, base_dir):
         if not self.file.storage.exists(self.file.name):
             return None