diff --git a/compose.yml b/compose.yml index 2ccbbb6a..130afe09 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ x-common: &common target: python_dev_deps platform: linux/amd64 environment: - - DEBUG=true + # - DEBUG=true - ADMIN_EMAIL=adm@hde.org - ADMIN_PASSWORD=123 - CACHE_URL=redis://redis:6379/1 @@ -18,6 +18,8 @@ x-common: &common - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - MEDIA_ROOT=/var/hope_dedupe_engine/media - PYTHONPATH=/code/src/:/code/__pypackages__/3.12/lib/ - SECRET_KEY=very-secret-key @@ -98,15 +100,15 @@ services: celery-worker: <<: *common - entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] - command: worker - # command: > - # sh -c ' - # mkdir -p /var/hope_dedupe_engine/default && - # chown -R user:app /var/hope_dedupe_engine && - # gosu user:app django-admin syncdnn && - # gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=INFO --concurrency=4 - # ' + # entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] + # command: worker + command: > + sh -c ' + mkdir -p /var/hope_dedupe_engine/default && + chown -R user:app /var/hope_dedupe_engine && + gosu user:app django-admin syncdnn && + gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --concurrency=4 + ' celery-beat: <<: *common diff --git a/src/hope_dedup_engine/apps/core/management/commands/demo.py b/src/hope_dedup_engine/apps/core/management/commands/demo.py index 73b9d1de..1a1f1b67 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/demo.py +++ b/src/hope_dedup_engine/apps/core/management/commands/demo.py @@ -1,6 +1,7 @@ import logging import sys from argparse import ArgumentParser +from dataclasses import dataclass, field from pathlib import Path from typing import Any, Final @@ -23,6 +24,7 @@ MESSAGES: Final[dict[str, str]] = { "upload": "Starting upload of files...", "not_exist": "Directory '%s' does not exist.", + "container_success": "Container for storage '%s' created successfully.", "storage_success": "Files uploaded to storage '%s' successfully.", "success": "Finished uploading files to storage.", "failed": "Failed to upload files to storage '%s': %s", @@ -31,6 +33,13 @@ } +@dataclass(frozen=True) +class Storage: + name: str + src: Path | None = field(default=None) + options: dict[str, str] = field(default_factory=dict) + + class Command(BaseCommand): help = "Create demo app" @@ -69,40 +78,43 @@ def handle(self, *args: Any, **options: dict[str, Any]) -> None: Exception: For any other unexpected errors that may arise during the execution of the command. """ - storages: dict[str, Path] = { - "hope": Path(options["demo_images"]), - "dnn": Path(options["dnn_files"]), - } + storages = ( + Storage(name="hope", src=Path(options["demo_images"])), + Storage(name="dnn", src=Path(options["dnn_files"])), + Storage(name="media"), + Storage(name="staticfiles", options={"public_access": "blob"}), + ) self.stdout.write(self.style.WARNING(MESSAGES["upload"])) logger.info(MESSAGES["upload"]) - try: - for storage_name, images_src_path in storages.items(): - am = AzuriteManager(storage_name) - if images_src_path is None: + for storage in storages: + try: + am = AzuriteManager(storage.name, storage.options) + self.stdout.write(MESSAGES["container_success"] % storage.name) + if storage.src is None: continue - if images_src_path.exists(): - am.upload_files(images_src_path) + if storage.src.exists(): + am.upload_files(storage.src) else: self.stdout.write( - self.style.ERROR(MESSAGES["not_exist"] % images_src_path) - ) - logger.error(MESSAGES["not_exist"] % images_src_path) - self.halt( - FileNotFoundError(MESSAGES["not_exist"] % images_src_path) + self.style.ERROR(MESSAGES["not_exist"] % storage.src) ) - self.stdout.write(MESSAGES["storage_success"] % storage_name) - logger.info(MESSAGES["storage_success"] % storage_name) - except (CommandError, SystemCheckError) as e: - self.stdout.write(self.style.ERROR(MESSAGES["failed"] % (storage_name, e))) - logger.error(MESSAGES["failed"] % (storage_name, e)) - self.halt(e) - except Exception as e: - self.stdout.write( - self.style.ERROR(MESSAGES["unexpected"] % (storage_name, e)) - ) - logger.exception(MESSAGES["unexpected"] % (storage_name, e)) - self.halt(e) + logger.error(MESSAGES["not_exist"] % storage.src) + self.halt(FileNotFoundError(MESSAGES["not_exist"] % storage.src)) + self.stdout.write(MESSAGES["storage_success"] % storage.name) + logger.info(MESSAGES["storage_success"] % storage.name) + except (CommandError, SystemCheckError) as e: + self.stdout.write( + self.style.ERROR(MESSAGES["failed"] % (storage.name, e)) + ) + logger.error(MESSAGES["failed"] % (storage.name, e)) + self.halt(e) + except Exception as e: + self.stdout.write( + self.style.ERROR(MESSAGES["unexpected"] % (storage.name, e)) + ) + logger.exception(MESSAGES["unexpected"] % (storage.name, e)) + self.halt(e) self.stdout.write(self.style.SUCCESS(MESSAGES["success"])) diff --git a/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py index e428c819..1b995c40 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py +++ b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py @@ -10,12 +10,17 @@ class AzuriteManager: # pragma: no cover - def __init__(self, storage_name: str) -> None: + def __init__( + self, storage_name: str, container_options: dict | None = None + ) -> None: """ Initializes the AzuriteManager with the specified storage configuration. Args: - storage_name (str): The name of the storage configuration as defined in settings.STORAGES. + storage_name (str): + The name of the storage configuration as defined in settings.STORAGES. + container_options (dict, optional): + Additional options to configure the Azure Blob Storage container. Defaults to an empty dictionary. """ storage = settings.STORAGES.get(storage_name).get("OPTIONS", {}) self.container_client: ContainerClient = ( @@ -23,18 +28,23 @@ def __init__(self, storage_name: str) -> None: storage.get("connection_string") ).get_container_client(storage.get("azure_container")) ) - self._create_container() + self._create_container(container_options) - def _create_container(self) -> None: + def _create_container(self, options: dict | None = None) -> None: """ - Creates container if it does not already exist. + Creates a container if it does not already exist. + + Args: + options (dict, optional): + Additional options to configure the container creation. Defaults to an empty dictionary. Raises: Exception: If the container creation fails for any reason. """ + options = options or {} try: if not self.container_client.exists(): - self.container_client.create_container() + self.container_client.create_container(**options) logger.info( "Container '%s' created successfully.", self.container_client.container_name, diff --git a/tests/extras/demoapp/compose.yml b/tests/extras/demoapp/compose.yml index 7136f46b..3a92052f 100644 --- a/tests/extras/demoapp/compose.yml +++ b/tests/extras/demoapp/compose.yml @@ -14,6 +14,8 @@ x-common: &common - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage?location=/var/hope_dedupe_engine/default - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - MEDIA_ROOT=/var/hope_dedupe_engine/media - PYTHONPATH=/code/__pypackages__/3.12/lib/ - SECRET_KEY=very-secret-key @@ -50,6 +52,7 @@ services: command: > /bin/sh -c " django-admin demo --skip-checks && + django-admin upgrade && docker-entrypoint.sh run " diff --git a/tests/extras/demoapp/dnn_files/deploy.prototxt b/tests/extras/demoapp/dnn_files/deploy.prototxt.txt similarity index 100% rename from tests/extras/demoapp/dnn_files/deploy.prototxt rename to tests/extras/demoapp/dnn_files/deploy.prototxt.txt