From 065c6055cf407f38eade2d9b2e59fc679c18d595 Mon Sep 17 00:00:00 2001 From: vitali-yanushchyk-valor <168179384+vitali-yanushchyk-valor@users.noreply.github.com> Date: Tue, 15 Oct 2024 06:25:57 -0300 Subject: [PATCH] chg ! use djago-smart-env (#97) --- compose.yml | 2 +- pdm.lock | 16 +- pyproject.toml | 1 + .../apps/core/management/commands/env.py | 89 ------ src/hope_dedup_engine/config/__init__.py | 260 ++++++++++-------- src/hope_dedup_engine/config/settings.py | 1 + tests/extras/demoapp/compose.yml | 1 + tests/test_commands.py | 37 +-- tests/test_smartenv.py | 71 ----- 9 files changed, 164 insertions(+), 314 deletions(-) delete mode 100644 src/hope_dedup_engine/apps/core/management/commands/env.py delete mode 100644 tests/test_smartenv.py diff --git a/compose.yml b/compose.yml index 130afe09..7b3c81a3 100644 --- a/compose.yml +++ b/compose.yml @@ -5,9 +5,9 @@ x-common: &common target: python_dev_deps platform: linux/amd64 environment: - # - DEBUG=true - ADMIN_EMAIL=adm@hde.org - ADMIN_PASSWORD=123 + - ALLOWED_HOSTS=localhost,127.0.0.1 - CACHE_URL=redis://redis:6379/1 - CELERY_BROKER_URL=redis://redis:6379/9 - CELERY_TASK_ALWAYS_EAGER=False diff --git a/pdm.lock b/pdm.lock index 1724e8e3..ef4cdf81 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev"] strategy = ["inherit_metadata"] lock_version = "4.5.0" -content_hash = "sha256:d53cf1c2ac1f0ac27bcea7ea3ee174e9b610f52bd455f466fe80fadd9acd2da9" +content_hash = "sha256:e929a0f5bde349def56aecf6e2ae3ca3ef6227aa8be9b84f37c520c5af489ed2" [[metadata.targets]] requires_python = ">=3.12" @@ -686,6 +686,20 @@ files = [ {file = "django-regex-0.5.0.tar.gz", hash = "sha256:6af1add11ae5232f133a42754c9291f9113996b1294b048305d9f1a427bca27c"}, ] +[[package]] +name = "django-smart-env" +version = "0.1.0" +requires_python = ">=3.12" +summary = "Add your description here" +groups = ["default"] +dependencies = [ + "django-environ>=0.11.2", +] +files = [ + {file = "django_smart_env-0.1.0-py3-none-any.whl", hash = "sha256:ffcbc03ab2b28808d1ac80b5165543549396dde4a24107e969a9635ba9321849"}, + {file = "django_smart_env-0.1.0.tar.gz", hash = "sha256:09ef06a2ae9223c68ba893dae2b6188938f41e464cb38e4714c341950fc1caf3"}, +] + [[package]] name = "django-storages" version = "1.14.4" diff --git a/pyproject.toml b/pyproject.toml index f531813e..71797b64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ dependencies = [ "numpy>=1.26.4,<2.0.0", "flower>=2.0.1", "setuptools>=74.1.2", + "django-smart-env>=0.1.0", ] [build-system] diff --git a/src/hope_dedup_engine/apps/core/management/commands/env.py b/src/hope_dedup_engine/apps/core/management/commands/env.py deleted file mode 100644 index a1bfff67..00000000 --- a/src/hope_dedup_engine/apps/core/management/commands/env.py +++ /dev/null @@ -1,89 +0,0 @@ -import shlex -from typing import TYPE_CHECKING - -from django.core.management import BaseCommand, CommandError, CommandParser - -if TYPE_CHECKING: - from typing import Any - -DEVELOP = { - "DEBUG": True, - "SECRET_KEY": "only-development-secret-key", -} - - -def clean(value): - if isinstance(value, (list, tuple)): - ret = ",".join(value) - else: - ret = str(value) - return shlex.quote(ret) - - -class Command(BaseCommand): - requires_migrations_checks = False - requires_system_checks = [] - - def add_arguments(self, parser: "CommandParser") -> None: - - parser.add_argument( - "--pattern", - action="store", - dest="pattern", - default="export {key}={value}", - help="Check env for variable availability (default: 'export {key}=\"{value}\"')", - ) - parser.add_argument( - "--develop", action="store_true", help="Display development values" - ) - parser.add_argument( - "--config", action="store_true", help="Only list changed values" - ) - parser.add_argument("--diff", action="store_true", help="Mark changed values") - parser.add_argument( - "--check", - action="store_true", - dest="check", - default=False, - help="Check env for variable availability", - ) - parser.add_argument( - "--ignore-errors", - action="store_true", - dest="ignore_errors", - default=False, - help="Do not fail", - ) - - def handle(self, *args: "Any", **options: "Any") -> None: - from hope_dedup_engine.config import CONFIG, EXPLICIT_SET, env - - check_failure = False - pattern = options["pattern"] - - for k, __ in sorted(CONFIG.items()): - help: str = env.get_help(k) - default = env.get_default(k) - if options["check"]: - if k in EXPLICIT_SET and k not in env.ENVIRON: - self.stderr.write(self.style.ERROR(f"- Missing env variable: {k}")) - check_failure = True - else: - if options["develop"]: - value: Any = env.for_develop(k) - else: - value: Any = env.get_value(k) - - line: str = pattern.format( - key=k, value=clean(value), help=help, default=default - ) - if options["diff"]: - if value != default: - line = self.style.SUCCESS(line) - elif options["config"]: - if value == default and k not in EXPLICIT_SET: - continue - self.stdout.write(line) - - if check_failure and not options["ignore_errors"]: - raise CommandError("Env check command failure!") diff --git a/src/hope_dedup_engine/config/__init__.py b/src/hope_dedup_engine/config/__init__.py index 68be3b30..5f915371 100644 --- a/src/hope_dedup_engine/config/__init__.py +++ b/src/hope_dedup_engine/config/__init__.py @@ -1,14 +1,13 @@ from enum import Enum from typing import TYPE_CHECKING, Any, Dict, Tuple, TypeAlias, Union -from environ import Env +from smart_env import SmartEnv if TYPE_CHECKING: ConfigItem: TypeAlias = Union[ Tuple[type, Any, str, Any], Tuple[type, Any, str], Tuple[type, Any] ] - DJANGO_HELP_BASE = "https://docs.djangoproject.com/en/5.1/ref/settings" @@ -16,78 +15,130 @@ def setting(anchor: str) -> str: return f"@see {DJANGO_HELP_BASE}#{anchor}" +def celery_doc(anchor: str) -> str: + return ( + f"@see https://docs.celeryq.dev/en/stable/" + f"userguide/configuration.html#{anchor}" + ) + + class Group(Enum): DJANGO = 1 -NOT_SET = "<- not set ->" -EXPLICIT_SET = [ - "DATABASE_URL", - "SECRET_KEY", - "CACHE_URL", - "CELERY_BROKER_URL", - "MEDIA_ROOT", - "STATIC_ROOT", - "DEFAULT_ROOT", -] - CONFIG: "Dict[str, ConfigItem]" = { - "ADMIN_EMAIL": (str, "", "Initial user created at first deploy"), - "ADMIN_PASSWORD": (str, "", "Password for initial user created at first deploy"), - "ALLOWED_HOSTS": (list, ["127.0.0.1", "localhost"], setting("allowed-hosts")), - "AUTHENTICATION_BACKENDS": (list, [], setting("authentication-backends")), - "CACHE_URL": (str, "redis://localhost:6379/0"), - "CATCH_ALL_EMAIL": (str, "If set all the emails will be sent to this address"), + "ADMIN_EMAIL": ( + str, + SmartEnv.NOTSET, + "admin", + True, + "Initial user created at first deploy", + ), + "ADMIN_PASSWORD": ( + str, + "", + "", + True, + "Password for initial user created at first deploy", + ), + "ALLOWED_HOSTS": ( + list, + [], + ["127.0.0.1", "localhost"], + True, + setting("allowed-hosts"), + ), + "AUTHENTICATION_BACKENDS": ( + list, + [], + [], + False, + setting("authentication-backends"), + ), + "AZURE_CLIENT_SECRET": (str, ""), + "AZURE_TENANT_ID": (str, ""), + "AZURE_CLIENT_KEY": (str, ""), + "CACHE_URL": ( + str, + SmartEnv.NOTSET, + "redis://localhost:6379/0", + True, + setting("cache-url"), + ), + "CATCH_ALL_EMAIL": ( + str, + "", + "", + False, + "If set all the emails will be sent to this address", + ), "CELERY_BROKER_URL": ( str, - NOT_SET, + "", + "", + True, "https://docs.celeryq.dev/en/stable/django/first-steps-with-django.html", ), "CELERY_TASK_ALWAYS_EAGER": ( bool, False, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#std-setting-task_always_eager", True, + False, + f"{celery_doc}#std-setting-task_always_eager", ), "CELERY_TASK_EAGER_PROPAGATES": ( bool, True, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#task-eager-propagates", + True, + False, + f"{celery_doc}#task-eager-propagates", ), "CELERY_VISIBILITY_TIMEOUT": ( int, 1800, - "https://docs.celeryq.dev/en/stable/userguide/configuration.html#broker-transport-options", + 1800, + False, + f"{celery_doc}#broker-transport-options", ), - "CSRF_COOKIE_SECURE": (bool, True, setting("csrf-cookie-secure")), + "CSRF_COOKIE_SECURE": (bool, True, False, setting("csrf-cookie-secure")), "DATABASE_URL": ( str, - "postgres://127.0.0.1:5432/dedupe", + SmartEnv.NOTSET, + SmartEnv.NOTSET, + True, "https://django-environ.readthedocs.io/en/latest/types.html#environ-env-db-url", - "postgres://127.0.0.1:5432/dedupe", ), - "DEBUG": (bool, False, setting("debug"), True), - "DEFAULT_ROOT": (str, "", "Default root for stored locally files"), - "EMAIL_BACKEND": ( + "DEBUG": (bool, False, True, False, setting("debug")), + "DEFAULT_ROOT": ( str, - "django.core.mail.backends.smtp.EmailBackend", - setting("email-backend"), + "/var/default/", + "/tmp/default", # nosec True, + "Default root for stored locally files", ), - "EMAIL_HOST": (str, "localhost", setting("email-host"), True), - "EMAIL_HOST_USER": (str, "", setting("email-host-user"), True), - "EMAIL_HOST_PASSWORD": (str, "", setting("email-host-password"), True), - "EMAIL_PORT": (int, "25", setting("email-port"), True), + "DEMO_IMAGES_PATH": (str, "demo_images"), + "DNN_FILES_PATH": (str, "dnn_files"), + # "EMAIL_BACKEND": ( + # str, + # "django.core.mail.backends.smtp.EmailBackend", + # setting("email-backend"), + # True, + # ), + "EMAIL_HOST": (str, "", "", False, setting("email-host")), + "EMAIL_HOST_USER": (str, "", "", False, setting("email-host-user")), + "EMAIL_HOST_PASSWORD": (str, "", "", False, setting("email-host-password")), + "EMAIL_PORT": (int, "25", "25", False, setting("email-port")), "EMAIL_SUBJECT_PREFIX": ( str, "[Hope-dedupe]", + "[Hope-dedupe-dev]", + False, setting("email-subject-prefix"), - True, ), - "EMAIL_USE_LOCALTIME": (bool, False, setting("email-use-localtime"), True), - "EMAIL_USE_TLS": (bool, False, setting("email-use-tls"), True), - "EMAIL_USE_SSL": (bool, False, setting("email-use-ssl"), True), - "EMAIL_TIMEOUT": (str, None, setting("email-timeout"), True), + "EMAIL_USE_LOCALTIME": (bool, False, False, False, setting("email-use-localtime")), + "EMAIL_USE_TLS": (bool, False, False, False, setting("email-use-tls")), + "EMAIL_USE_SSL": (bool, False, False, False, setting("email-use-ssl")), + "EMAIL_TIMEOUT": (str, None, None, False, setting("email-timeout")), "FILE_STORAGE_DEFAULT": ( str, "django.core.files.storage.FileSystemStorage", @@ -113,96 +164,71 @@ class Group(Enum): "storages.backends.azure_storage.AzureStorage", setting("storages"), ), - "LOG_LEVEL": (str, "CRITICAL", setting("logging")), - "LOGGING_LEVEL": (str, "CRITICAL", setting("logging-level")), - "MEDIA_ROOT": (str, None, setting("media-root")), - "MEDIA_URL": (str, "/media/", setting("media-url")), - "ROOT_TOKEN": (str, "", ""), - "SECRET_KEY": (str, NOT_SET, setting("secret-key")), - "SECURE_HSTS_PRELOAD": (bool, True, setting("secure-hsts-preload"), False), - "SECURE_HSTS_SECONDS": (int, 60, setting("secure-hsts-seconds")), - "SECURE_SSL_REDIRECT": (bool, True, setting("secure-ssl-redirect"), False), - "SENTRY_DSN": (str, "", "Sentry DSN"), - "SENTRY_ENVIRONMENT": (str, "production", "Sentry Environment"), - "SENTRY_URL": (str, "", "Sentry server url"), - "SESSION_COOKIE_DOMAIN": ( + "LOG_LEVEL": (str, "CRITICAL", "DEBUG", False, setting("logging-level")), + "MEDIA_ROOT": ( + str, + "/var/media/", + "/tmp/media", # nosec + True, + setting("media-root"), + ), + "MEDIA_URL": (str, "/media/", "/media", False, setting("media-root")), # nosec + "ROOT_TOKEN_HEADER": (str, "x-root-token", "x-root-token"), + "ROOT_TOKEN": (str, ""), + "SECRET_KEY": ( str, "", - setting("std-setting-SESSION_COOKIE_DOMAIN"), + "super_sensitive_key_just_for_testing", + True, + setting("secret-key"), + ), + "SECURE_HSTS_PRELOAD": (bool, True, False, False, setting("secure-hsts-preload")), + "SECURE_HSTS_SECONDS": (int, 60, 0, False, setting("secure-hsts-seconds")), + "SECURE_SSL_REDIRECT": (bool, True, False, False, setting("secure-ssl-redirect")), + "SENTRY_DSN": (str, "", "", False, "Sentry DSN"), + "SENTRY_ENVIRONMENT": (str, "production", "develop", False, "Sentry Environment"), + "SENTRY_URL": (str, "", "", False, "Sentry server url"), + "SESSION_COOKIE_DOMAIN": ( + str, + SmartEnv.NOTSET, "localhost", + False, + setting("std-setting-SESSION_COOKIE_DOMAIN"), + ), + "SESSION_COOKIE_HTTPONLY": ( + bool, + True, + False, + False, + setting("session-cookie-httponly"), ), - "SESSION_COOKIE_HTTPONLY": (bool, True, setting("session-cookie-httponly"), False), "SESSION_COOKIE_NAME": (str, "dedupe_session", setting("session-cookie-name")), "SESSION_COOKIE_PATH": (str, "/", setting("session-cookie-path")), - "SESSION_COOKIE_SECURE": (bool, True, setting("session-cookie-secure"), False), + "SESSION_COOKIE_SECURE": ( + bool, + True, + False, + False, + setting("session-cookie-secure"), + ), "SIGNING_BACKEND": ( str, "django.core.signing.TimestampSigner", setting("signing-backend"), ), - "SOCIAL_AUTH_LOGIN_URL": (str, "/login/", "", ""), - "SOCIAL_AUTH_RAISE_EXCEPTIONS": (bool, False, "", True), - "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True, "", False), - "STATIC_FILE_STORAGE": ( + "SOCIAL_AUTH_LOGIN_URL": (str, "/login/", "", False, ""), + "SOCIAL_AUTH_RAISE_EXCEPTIONS": (bool, False, True, False), + "SOCIAL_AUTH_REDIRECT_IS_HTTPS": (bool, True, False, False, ""), + "STATIC_ROOT": ( str, - "django.core.files.storage.FileSystemStorage", - setting("storages"), - ), - "STATIC_ROOT": (str, "", setting("static-root")), - "STATIC_URL": (str, "/static/", setting("static-url")), - "TIME_ZONE": (str, "UTC", setting("std-setting-TIME_ZONE")), - "DEMO_IMAGES_PATH": (str, "demo_images"), - "DNN_FILES_PATH": (str, "dnn_files"), + "/var/static", + "/tmp/static", + True, + setting("static-root"), + ), # nosec + "STATIC_URL": (str, "/static/", "/static/", False, setting("static-url")), # nosec + "TIME_ZONE": (str, "UTC", "UTC", False, setting("std-setting-TIME_ZONE")), } -class SmartEnv(Env): - def __init__(self, **scheme): # type: ignore[no-untyped-def] - self.raw = scheme - values = {k: v[:2] for k, v in scheme.items()} - super().__init__(**values) - - def get_help(self, key: str) -> str: - entry: "ConfigItem" = self.raw.get(key, "") - if len(entry) > 2: - return entry[2] - return "" - - def for_develop(self, key: str) -> Any: - entry: ConfigItem = self.raw.get(key, "") - if len(entry) > 3: - value = entry[3] - else: - value = self.get_value(key) - return value - - def storage(self, value: str) -> dict[str, str | dict[str, Any]] | None: - raw_value = self.get_value(value, str) - if not raw_value: - return None - options = {} - if "?" in raw_value: - value, args = raw_value.split("?", 1) - for entry in args.split("&"): - k, v = entry.split("=", 1) - options[k] = v - else: - value = raw_value - - return {"BACKEND": value, "OPTIONS": options} - - def get_default(self, var: str) -> Any: - var_name = f"{self.prefix}{var}" - value = "" - if var_name in self.scheme: - var_info = self.scheme[var_name] - value = var_info[1] - try: - cast = var_info[0] - return cast(value) - except TypeError as e: - raise TypeError(f"Can't cast {var} to {cast}") from e - return value - - env = SmartEnv(**CONFIG) # type: ignore[no-untyped-call] diff --git a/src/hope_dedup_engine/config/settings.py b/src/hope_dedup_engine/config/settings.py index c006505c..ce1adf73 100644 --- a/src/hope_dedup_engine/config/settings.py +++ b/src/hope_dedup_engine/config/settings.py @@ -43,6 +43,7 @@ "hope_dedup_engine.apps.api", "hope_dedup_engine.apps.faces", "storages", + "smart_env", ) MIDDLEWARE = ( diff --git a/tests/extras/demoapp/compose.yml b/tests/extras/demoapp/compose.yml index 3a92052f..9c3dd2e6 100644 --- a/tests/extras/demoapp/compose.yml +++ b/tests/extras/demoapp/compose.yml @@ -4,6 +4,7 @@ x-common: &common environment: - ADMIN_EMAIL=adm@hde.org - ADMIN_PASSWORD=123 + - ALLOWED_HOSTS=localhost,127.0.0.1 - CACHE_URL=redis://redis:6379/1 - CELERY_BROKER_URL=redis://redis:6379/9 - CELERY_TASK_ALWAYS_EAGER=False diff --git a/tests/test_commands.py b/tests/test_commands.py index f0d0225b..3cf7ee04 100644 --- a/tests/test_commands.py +++ b/tests/test_commands.py @@ -2,7 +2,7 @@ from io import StringIO from unittest import mock -from django.core.management import CommandError, call_command +from django.core.management import call_command import pytest from testutils.factories import SuperUserFactory @@ -23,6 +23,7 @@ def environment(): "STATIC_ROOT": "/tmp/static", "SECURE_SSL_REDIRECT": "1", "SESSION_COOKIE_SECURE": "1", + "DJANGO_SETTINGS_MODULE": "hope_dedup_engine.config.settings", } @@ -115,40 +116,6 @@ def test_upgrade_admin(db, mocked_responses, environment, admin): ) -@pytest.mark.parametrize("verbosity", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("develop", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("diff", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("config", [0, 1], ids=["0", "1"]) -@pytest.mark.parametrize("check", [0, 1], ids=["0", "1"]) -def test_env(mocked_responses, verbosity, develop, diff, config, check): - out = StringIO() - environ = { - "ADMIN_URL_PREFIX": "test", - "SECURE_SSL_REDIRECT": "1", - "SECRET_KEY": "a" * 120, - "SESSION_COOKIE_SECURE": "1", - } - with mock.patch.dict(os.environ, environ, clear=True): - call_command( - "env", - ignore_errors=True if check == 1 else False, - stdout=out, - verbosity=verbosity, - develop=develop, - diff=diff, - config=config, - check=check, - ) - assert "error" not in str(out.getvalue()) - - -def test_env_raise(mocked_responses): - environ = {"ADMIN_URL_PREFIX": "test"} - with mock.patch.dict(os.environ, environ, clear=True): - with pytest.raises(CommandError): - call_command("env", ignore_errors=False, check=True) - - def test_upgrade_exception(mocked_responses, environment): with mock.patch( "hope_dedup_engine.apps.core.management.commands.upgrade.call_command" diff --git a/tests/test_smartenv.py b/tests/test_smartenv.py deleted file mode 100644 index 637e2473..00000000 --- a/tests/test_smartenv.py +++ /dev/null @@ -1,71 +0,0 @@ -import os -from unittest import mock - -import pytest - -from hope_dedup_engine.config import SmartEnv - - -@pytest.fixture() -def env(): - return SmartEnv(STORAGE_DEFAULT=(str, "")) - - -@pytest.mark.parametrize( - "storage", - [ - "storage.SampleStorage?bucket=container&option=value&connection_string=Defaul", - "storage.SampleStorage?bucket=container&option=value&connection_string=DefaultEndpointsProtocol=http;Account" - "Name=devstoreaccount1;AccountKey=ke1==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", - ], -) -def test_storage_options(storage, env): - with mock.patch.dict(os.environ, {"STORAGE_DEFAULT": storage}, clear=True): - ret = env.storage("STORAGE_DEFAULT") - - assert ret["BACKEND"] == "storage.SampleStorage" - assert sorted(ret["OPTIONS"].keys()) == ["bucket", "connection_string", "option"] - - -@pytest.mark.parametrize("storage", ["storage.SampleStorage"]) -def test_storage(storage, env): - with mock.patch.dict(os.environ, {"STORAGE_DEFAULT": storage}, clear=True): - ret = env.storage("STORAGE_DEFAULT") - - assert ret["BACKEND"] == "storage.SampleStorage" - - -def test_storage_empty(env): - with mock.patch.dict(os.environ, {}, clear=True): - assert not env.storage("STORAGE_DEFAULT") - - -def test_env(): - e = SmartEnv( - **{ - "T1": (str, "a@b.com"), - "T2": (str, "a@b.com", "help"), - "T3": (str, "a@b.com", "help", "dev@b.com"), - "T4": (int, None), - } - ) - - assert e("T1") == "a@b.com" - assert e.get_help("T1") == "" - assert e.for_develop("T1") == "a@b.com" - assert e.get_default("T1") == "a@b.com" - - assert e("T2") == "a@b.com" - assert e.get_help("T2") == "help" - assert e.for_develop("T2") == "a@b.com" - assert e.get_default("T2") == "a@b.com" - - assert e("T3") == "a@b.com" - assert e.get_help("T3") == "help" - assert e.for_develop("T3") == "dev@b.com" - assert e.get_default("T3") == "a@b.com" - - assert e.get_default("cc") == "" - - with pytest.raises(TypeError): - assert e.get_default("T4")