From 3d7a8df6bae321091d52989dd08c9c58201cc43e Mon Sep 17 00:00:00 2001 From: vitali-yanushchyk-valor <168179384+vitali-yanushchyk-valor@users.noreply.github.com> Date: Mon, 30 Sep 2024 12:33:11 -0300 Subject: [PATCH] chg ! demoapp (#92) --- compose.yml | 22 +++--- .../apps/core/management/commands/demo.py | 70 +++++++++++-------- .../commands/utils/azurite_manager.py | 22 ++++-- tests/extras/demoapp/compose.yml | 3 + .../{deploy.prototxt => deploy.prototxt.txt} | 0 tests/test_command_demo.py | 48 +++++++++++++ tests/test_commands.py | 34 --------- 7 files changed, 119 insertions(+), 80 deletions(-) rename tests/extras/demoapp/dnn_files/{deploy.prototxt => deploy.prototxt.txt} (100%) create mode 100644 tests/test_command_demo.py diff --git a/compose.yml b/compose.yml index 2ccbbb6a..130afe09 100644 --- a/compose.yml +++ b/compose.yml @@ -5,7 +5,7 @@ x-common: &common target: python_dev_deps platform: linux/amd64 environment: - - DEBUG=true + # - DEBUG=true - ADMIN_EMAIL=adm@hde.org - ADMIN_PASSWORD=123 - CACHE_URL=redis://redis:6379/1 @@ -18,6 +18,8 @@ x-common: &common - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - MEDIA_ROOT=/var/hope_dedupe_engine/media - PYTHONPATH=/code/src/:/code/__pypackages__/3.12/lib/ - SECRET_KEY=very-secret-key @@ -98,15 +100,15 @@ services: celery-worker: <<: *common - entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] - command: worker - # command: > - # sh -c ' - # mkdir -p /var/hope_dedupe_engine/default && - # chown -R user:app /var/hope_dedupe_engine && - # gosu user:app django-admin syncdnn && - # gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=INFO --concurrency=4 - # ' + # entrypoint: ["sh", "-c", "exec docker-entrypoint.sh \"$0\" \"$@\""] + # command: worker + command: > + sh -c ' + mkdir -p /var/hope_dedupe_engine/default && + chown -R user:app /var/hope_dedupe_engine && + gosu user:app django-admin syncdnn && + gosu user:app celery -A hope_dedup_engine.config.celery worker -E --loglevel=WARNING --concurrency=4 + ' celery-beat: <<: *common diff --git a/src/hope_dedup_engine/apps/core/management/commands/demo.py b/src/hope_dedup_engine/apps/core/management/commands/demo.py index 8fb260ca..1a1f1b67 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/demo.py +++ b/src/hope_dedup_engine/apps/core/management/commands/demo.py @@ -1,6 +1,7 @@ import logging import sys from argparse import ArgumentParser +from dataclasses import dataclass, field from pathlib import Path from typing import Any, Final @@ -23,7 +24,7 @@ MESSAGES: Final[dict[str, str]] = { "upload": "Starting upload of files...", "not_exist": "Directory '%s' does not exist.", - "container_success": "Container '%s' created successfully.", + "container_success": "Container for storage '%s' created successfully.", "storage_success": "Files uploaded to storage '%s' successfully.", "success": "Finished uploading files to storage.", "failed": "Failed to upload files to storage '%s': %s", @@ -32,7 +33,14 @@ } -class Command(BaseCommand): # pragma: no cover +@dataclass(frozen=True) +class Storage: + name: str + src: Path | None = field(default=None) + options: dict[str, str] = field(default_factory=dict) + + +class Command(BaseCommand): help = "Create demo app" def add_arguments(self, parser: ArgumentParser) -> None: @@ -70,41 +78,43 @@ def handle(self, *args: Any, **options: dict[str, Any]) -> None: Exception: For any other unexpected errors that may arise during the execution of the command. """ - storages: dict[str, Path] = { - "hope": Path(options["demo_images"]), - "dnn": Path(options["dnn_files"]), - } + storages = ( + Storage(name="hope", src=Path(options["demo_images"])), + Storage(name="dnn", src=Path(options["dnn_files"])), + Storage(name="media"), + Storage(name="staticfiles", options={"public_access": "blob"}), + ) self.stdout.write(self.style.WARNING(MESSAGES["upload"])) logger.info(MESSAGES["upload"]) - try: - for storage_name, images_src_path in storages.items(): - am = AzuriteManager(storage_name) - self.stdout.write(MESSAGES["container_success"] % storage_name) - if images_src_path is None: + for storage in storages: + try: + am = AzuriteManager(storage.name, storage.options) + self.stdout.write(MESSAGES["container_success"] % storage.name) + if storage.src is None: continue - if images_src_path.exists(): - am.upload_files(images_src_path) + if storage.src.exists(): + am.upload_files(storage.src) else: self.stdout.write( - self.style.ERROR(MESSAGES["not_exist"] % images_src_path) - ) - logger.error(MESSAGES["not_exist"] % images_src_path) - self.halt( - FileNotFoundError(MESSAGES["not_exist"] % images_src_path) + self.style.ERROR(MESSAGES["not_exist"] % storage.src) ) - self.stdout.write(MESSAGES["storage_success"] % storage_name) - logger.info(MESSAGES["storage_success"] % storage_name) - except (CommandError, SystemCheckError) as e: - self.stdout.write(self.style.ERROR(MESSAGES["failed"] % (storage_name, e))) - logger.error(MESSAGES["failed"] % (storage_name, e)) - self.halt(e) - except Exception as e: - self.stdout.write( - self.style.ERROR(MESSAGES["unexpected"] % (storage_name, e)) - ) - logger.exception(MESSAGES["unexpected"] % (storage_name, e)) - self.halt(e) + logger.error(MESSAGES["not_exist"] % storage.src) + self.halt(FileNotFoundError(MESSAGES["not_exist"] % storage.src)) + self.stdout.write(MESSAGES["storage_success"] % storage.name) + logger.info(MESSAGES["storage_success"] % storage.name) + except (CommandError, SystemCheckError) as e: + self.stdout.write( + self.style.ERROR(MESSAGES["failed"] % (storage.name, e)) + ) + logger.error(MESSAGES["failed"] % (storage.name, e)) + self.halt(e) + except Exception as e: + self.stdout.write( + self.style.ERROR(MESSAGES["unexpected"] % (storage.name, e)) + ) + logger.exception(MESSAGES["unexpected"] % (storage.name, e)) + self.halt(e) self.stdout.write(self.style.SUCCESS(MESSAGES["success"])) diff --git a/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py index e428c819..1b995c40 100644 --- a/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py +++ b/src/hope_dedup_engine/apps/core/management/commands/utils/azurite_manager.py @@ -10,12 +10,17 @@ class AzuriteManager: # pragma: no cover - def __init__(self, storage_name: str) -> None: + def __init__( + self, storage_name: str, container_options: dict | None = None + ) -> None: """ Initializes the AzuriteManager with the specified storage configuration. Args: - storage_name (str): The name of the storage configuration as defined in settings.STORAGES. + storage_name (str): + The name of the storage configuration as defined in settings.STORAGES. + container_options (dict, optional): + Additional options to configure the Azure Blob Storage container. Defaults to an empty dictionary. """ storage = settings.STORAGES.get(storage_name).get("OPTIONS", {}) self.container_client: ContainerClient = ( @@ -23,18 +28,23 @@ def __init__(self, storage_name: str) -> None: storage.get("connection_string") ).get_container_client(storage.get("azure_container")) ) - self._create_container() + self._create_container(container_options) - def _create_container(self) -> None: + def _create_container(self, options: dict | None = None) -> None: """ - Creates container if it does not already exist. + Creates a container if it does not already exist. + + Args: + options (dict, optional): + Additional options to configure the container creation. Defaults to an empty dictionary. Raises: Exception: If the container creation fails for any reason. """ + options = options or {} try: if not self.container_client.exists(): - self.container_client.create_container() + self.container_client.create_container(**options) logger.info( "Container '%s' created successfully.", self.container_client.container_name, diff --git a/tests/extras/demoapp/compose.yml b/tests/extras/demoapp/compose.yml index 7136f46b..3a92052f 100644 --- a/tests/extras/demoapp/compose.yml +++ b/tests/extras/demoapp/compose.yml @@ -14,6 +14,8 @@ x-common: &common - FILE_STORAGE_DEFAULT=django.core.files.storage.FileSystemStorage?location=/var/hope_dedupe_engine/default - FILE_STORAGE_DNN=storages.backends.azure_storage.AzureStorage?azure_container=dnn&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - FILE_STORAGE_HOPE=storages.backends.azure_storage.AzureStorage?azure_container=hope&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_MEDIA=storages.backends.azure_storage.AzureStorage?azure_container=media&overwrite_files=True&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; + - FILE_STORAGE_STATIC=storages.backends.azure_storage.AzureStorage?azure_container=static&overwrite_files=True&custom_domain=localhost:10000/&connection_string=DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azurite:10000/devstoreaccount1; - MEDIA_ROOT=/var/hope_dedupe_engine/media - PYTHONPATH=/code/__pypackages__/3.12/lib/ - SECRET_KEY=very-secret-key @@ -50,6 +52,7 @@ services: command: > /bin/sh -c " django-admin demo --skip-checks && + django-admin upgrade && docker-entrypoint.sh run " diff --git a/tests/extras/demoapp/dnn_files/deploy.prototxt b/tests/extras/demoapp/dnn_files/deploy.prototxt.txt similarity index 100% rename from tests/extras/demoapp/dnn_files/deploy.prototxt rename to tests/extras/demoapp/dnn_files/deploy.prototxt.txt diff --git a/tests/test_command_demo.py b/tests/test_command_demo.py new file mode 100644 index 00000000..68d0bf97 --- /dev/null +++ b/tests/test_command_demo.py @@ -0,0 +1,48 @@ +import os +from io import StringIO +from unittest import mock + +from django.core.management import call_command + +import pytest +from pytest_mock import MockerFixture + + +@pytest.fixture() +def environment(): + return { + "DEMO_IMAGES_PATH": "demo_images", + "DNN_FILES_PATH": "dnn_files", + } + + +@pytest.fixture +def mock_azurite_manager(mocker: MockerFixture): + with mock.patch( + "hope_dedup_engine.apps.core.management.commands.utils.azurite_manager.AzuriteManager" + ) as MockAzuriteManager: + yield MockAzuriteManager + + +def test_demo_handle_success(environment, mock_azurite_manager): + out = StringIO() + with ( + mock.patch.dict("os.environ", environment, clear=True), + mock.patch("pathlib.Path.exists", return_value=True), + ): + call_command( + "demo", + demo_images="/path/to/demo/images", + dnn_files="/path/to/dnn/files", + stdout=out, + ) + assert "error" not in str(out.getvalue()) + assert mock_azurite_manager.call_count == 4 + assert mock_azurite_manager.return_value.upload_files.call_count == 2 + + +def test_demo_handle_exception(environment, mock_azurite_manager): + mock_azurite_manager.side_effect = Exception() + with mock.patch.dict(os.environ, environment, clear=True): + with pytest.raises(Exception): + call_command("demo", ignore_errors=False) diff --git a/tests/test_commands.py b/tests/test_commands.py index 49f6c559..3c8b4a95 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -23,19 +23,9 @@ def environment(): "STATIC_ROOT": "/tmp/static", "SECURE_SSL_REDIRECT": "1", "SESSION_COOKIE_SECURE": "1", - "DEMO_IMAGES_PATH": "demo_images", - "DNN_FILES_PATH": "dnn_files", } -@pytest.fixture -def mock_azurite_manager(): - with mock.patch( - "hope_dedup_engine.apps.core.management.commands.utils.azurite_manager.AzuriteManager" - ) as MockAzuriteManager: - yield MockAzuriteManager - - @pytest.fixture def mock_settings(): with mock.patch("django.conf.settings") as mock_settings: @@ -161,27 +151,3 @@ def test_upgrade_exception(mocked_responses, environment): ): with pytest.raises(SystemExit): call_command("upgrade", stdout=out, check=True, admin_email="") - - -def test_demo_handle_success(environment, mock_azurite_manager, mock_settings): - out = StringIO() - with ( - mock.patch.dict("os.environ", environment, clear=True), - mock.patch("pathlib.Path.exists", return_value=True), - ): - call_command( - "demo", - demo_images="/path/to/demo/images", - dnn_files="/path/to/dnn/files", - stdout=out, - ) - assert "error" not in str(out.getvalue()) - assert mock_azurite_manager.call_count == 2 - assert mock_azurite_manager.return_value.upload_files.call_count == 2 - - -def test_demo_handle_exception(environment, mock_azurite_manager, mock_settings): - mock_azurite_manager.side_effect = Exception() - with mock.patch.dict(os.environ, environment, clear=True): - with pytest.raises(Exception): - call_command("demo", ignore_errors=False)