From 7f22aa6314af8339a90c0467f0b3dcb64142d7da Mon Sep 17 00:00:00 2001 From: Karl Sutt Date: Mon, 10 Jul 2023 15:51:21 +0300 Subject: [PATCH] Add configuration entrypoint for ECS --- judoscale/core/config.py | 16 ++++++++ poetry.lock | 85 +++++++++++++++++++++++++++++++++++++++- pyproject.toml | 1 + tests/test_config.py | 37 +++++++++++++++++ 4 files changed, 138 insertions(+), 1 deletion(-) diff --git a/judoscale/core/config.py b/judoscale/core/config.py index 7215755..9712e7e 100644 --- a/judoscale/core/config.py +++ b/judoscale/core/config.py @@ -3,6 +3,8 @@ from collections import UserDict from typing import Mapping +import requests + from judoscale.core.logger import logger DEFAULTS = {"REPORT_INTERVAL_SECONDS": 10, "LOG_LEVEL": "WARN"} @@ -49,6 +51,8 @@ def initialize(cls, env: Mapping = os.environ): return cls.for_heroku(env) elif env.get("RENDER_INSTANCE_ID"): return cls.for_render(env) + elif env.get("ECS_CONTAINER_METADATA_URI_V4"): + return cls.for_ecs(env) else: return cls(None, "", env) @@ -71,6 +75,18 @@ def for_render(cls, env: Mapping): api_base_url = f"https://adapter.judoscale.com/api/{service_id}" return cls(runtime_container, api_base_url, env) + @classmethod + def for_ecs(cls, env: Mapping): + container_metadata = requests.get(env["ECS_CONTAINER_METADATA_URI_V4"]).json() + service_name = container_metadata["DockerName"] + instance = container_metadata["DockerId"] + + # All ECS containers have to be manually assigned as web instances, + # because ECS does not distinguish between web and worker instances. + runtime_container = RuntimeContainer(service_name, instance, "web") + api_base_url = env.get("JUDOSCALE_URL") + return cls(runtime_container, api_base_url, env) + @property def is_enabled(self) -> bool: return bool(self["API_BASE_URL"]) diff --git a/poetry.lock b/poetry.lock index d336476..c548eea 100644 --- a/poetry.lock +++ b/poetry.lock @@ -714,6 +714,56 @@ files = [ {file = "pytz-2022.7.1.tar.gz", hash = "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0"}, ] +[[package]] +name = "pyyaml" +version = "6.0" +description = "YAML parser and emitter for Python" +category = "dev" +optional = false +python-versions = ">=3.6" +files = [ + {file = "PyYAML-6.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53"}, + {file = "PyYAML-6.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b"}, + {file = "PyYAML-6.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"}, + {file = "PyYAML-6.0-cp310-cp310-win32.whl", hash = "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513"}, + {file = "PyYAML-6.0-cp310-cp310-win_amd64.whl", hash = "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358"}, + {file = "PyYAML-6.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f"}, + {file = "PyYAML-6.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782"}, + {file = "PyYAML-6.0-cp311-cp311-win32.whl", hash = "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7"}, + {file = "PyYAML-6.0-cp311-cp311-win_amd64.whl", hash = "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf"}, + {file = "PyYAML-6.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92"}, + {file = "PyYAML-6.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4"}, + {file = "PyYAML-6.0-cp36-cp36m-win32.whl", hash = "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293"}, + {file = "PyYAML-6.0-cp36-cp36m-win_amd64.whl", hash = "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57"}, + {file = "PyYAML-6.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4"}, + {file = "PyYAML-6.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9"}, + {file = "PyYAML-6.0-cp37-cp37m-win32.whl", hash = "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737"}, + {file = "PyYAML-6.0-cp37-cp37m-win_amd64.whl", hash = "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d"}, + {file = "PyYAML-6.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34"}, + {file = "PyYAML-6.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287"}, + {file = "PyYAML-6.0-cp38-cp38-win32.whl", hash = "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78"}, + {file = "PyYAML-6.0-cp38-cp38-win_amd64.whl", hash = "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b"}, + {file = "PyYAML-6.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3"}, + {file = "PyYAML-6.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0"}, + {file = "PyYAML-6.0-cp39-cp39-win32.whl", hash = "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb"}, + {file = "PyYAML-6.0-cp39-cp39-win_amd64.whl", hash = "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c"}, + {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, +] + [[package]] name = "redis" version = "4.4.2" @@ -755,6 +805,27 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "responses" +version = "0.23.1" +description = "A utility library for mocking out the `requests` Python library." +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "responses-0.23.1-py3-none-any.whl", hash = "sha256:8a3a5915713483bf353b6f4079ba8b2a29029d1d1090a503c70b0dc5d9d0c7bd"}, + {file = "responses-0.23.1.tar.gz", hash = "sha256:c4d9aa9fc888188f0c673eff79a8dadbe2e75b7fe879dc80a221a06e0a68138f"}, +] + +[package.dependencies] +pyyaml = "*" +requests = ">=2.22.0,<3.0" +types-PyYAML = "*" +urllib3 = ">=1.25.10" + +[package.extras] +tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "tomli", "tomli-w", "types-requests"] + [[package]] name = "rq" version = "1.13.0" @@ -807,6 +878,18 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "types-pyyaml" +version = "6.0.12.10" +description = "Typing stubs for PyYAML" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "types-PyYAML-6.0.12.10.tar.gz", hash = "sha256:ebab3d0700b946553724ae6ca636ea932c1b0868701d4af121630e78d695fc97"}, + {file = "types_PyYAML-6.0.12.10-py3-none-any.whl", hash = "sha256:662fa444963eff9b68120d70cda1af5a5f2aa57900003c2006d7626450eaae5f"}, +] + [[package]] name = "typing-extensions" version = "4.4.0" @@ -916,4 +999,4 @@ rq = ["rq"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "9b479c9ee07f8485e7fb436550aed6eed23a80ff2af0a15d4515e1cf0006d62c" +content-hash = "fadbc79520598406e33dd5beaaa8367d811b43ad05717fa184147c5276a867cc" diff --git a/pyproject.toml b/pyproject.toml index f59185e..cf715a3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ black = "^22.12.0" isort = "^5.11.2" flake8 = { version = "^6.0.0", python = ">=3.8.1,<4.0.0" } pytest = "^7.2.2" +responses = "^0.23.1" [tool.poetry.extras] django = ["django"] diff --git a/tests/test_config.py b/tests/test_config.py index 03882b1..bd68585 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,19 @@ +import pytest +import responses + from judoscale.core.config import Config, RuntimeContainer +ecs_response = { + "DockerId": "b25c2505a33f4b3ab8673c3200c2f245-2750272591", + "DockerName": "backend", +} + + +@pytest.fixture +def mocked_responses(): + with responses.RequestsMock() as rsps: + yield rsps + class TestConfig: def test_on_heroku(self): @@ -31,6 +45,29 @@ def test_on_render(self): assert config["LOG_LEVEL"] == "WARN" assert config["API_BASE_URL"] == "https://adapter.judoscale.com/api/srv-123" + def test_on_ecs(self, mocked_responses): + mocked_responses.get( + "https://127.0.0.1/api/123456", + status=200, + json=ecs_response, + ) + + fake_env = { + "ECS_CONTAINER_METADATA_URI_V4": "https://127.0.0.1/api/123456", + "JUDOSCALE_URL": "https://adapter.judoscale.com/api/srv-123", + "LOG_LEVEL": "WARN", + } + config = Config.for_ecs(fake_env) + + assert config["RUNTIME_CONTAINER"].service_name == "backend" + assert ( + config["RUNTIME_CONTAINER"].instance + == "b25c2505a33f4b3ab8673c3200c2f245-2750272591" + ) + assert config["RUNTIME_CONTAINER"].service_type == "web" + assert config["LOG_LEVEL"] == "WARN" + assert config["API_BASE_URL"] == "https://adapter.judoscale.com/api/srv-123" + def test_is_enabled(self): config = Config(None, "", {}) assert not config.is_enabled