✨Autoscaling: 1st draft on auto-scaling computational clusters (ITISF…

…oundation#4711)
ignapas · Oct 20, 2023 · 60dbc85 · 60dbc85
1 parent ff62322
commit 60dbc85
Show file tree

Hide file tree

Showing 32 changed files with 2,161 additions and 562 deletions.
diff --git a/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py b/packages/pytest-simcore/src/pytest_simcore/dask_scheduler.py
@@ -63,14 +63,34 @@ async def dask_spec_local_cluster(
     monkeypatch: pytest.MonkeyPatch,
     dask_workers_config: dict[str, Any],
     dask_scheduler_config: dict[str, Any],
-) -> AsyncIterable[distributed.SpecCluster]:
+) -> AsyncIterator[distributed.SpecCluster]:
     # in this mode we can precisely create a specific cluster
 
     async with distributed.SpecCluster(
         workers=dask_workers_config,
         scheduler=dask_scheduler_config,
         asynchronous=True,
-        name="pytest_cluster",
+        name="pytest_dask_spec_local_cluster",
+    ) as cluster:
+        scheduler_address = URL(cluster.scheduler_address)
+        monkeypatch.setenv(
+            "COMPUTATIONAL_BACKEND_DEFAULT_CLUSTER_URL",
+            f"{scheduler_address}" or "invalid",
+        )
+        yield cluster
+
+
+@pytest.fixture
+async def dask_local_cluster_without_workers(
+    monkeypatch: pytest.MonkeyPatch,
+    dask_scheduler_config: dict[str, Any],
+) -> AsyncIterable[distributed.SpecCluster]:
+    # in this mode we can precisely create a specific cluster
+
+    async with distributed.SpecCluster(
+        scheduler=dask_scheduler_config,
+        asynchronous=True,
+        name="pytest_dask_local_cluster_without_workers",
     ) as cluster:
         scheduler_address = URL(cluster.scheduler_address)
         monkeypatch.setenv(

diff --git a/services/autoscaling/requirements/_base.in b/services/autoscaling/requirements/_base.in
@@ -4,6 +4,7 @@
 # NOTE: ALL version constraints MUST be commented
 --constraint ../../../requirements/constraints.txt
 --constraint ./constraints.txt
+--constraint ../../../services/dask-sidecar/requirements/_dask-distributed.txt
 
 # intra-repo required dependencies
 --requirement ../../../packages/models-library/requirements/_base.in
@@ -15,6 +16,7 @@
 
 aiodocker
 aioboto3
+dask[distributed]
 fastapi
 packaging
 types-aiobotocore[ec2]
diff --git a/services/autoscaling/requirements/_base.txt b/services/autoscaling/requirements/_base.txt
@@ -11,7 +11,9 @@ aio-pika==9.1.2
 aioboto3==10.4.0
     # via -r requirements/_base.in
 aiobotocore==2.4.2
-    # via aioboto3
+    # via
+    #   aioboto3
+    #   aiobotocore
 aiodebug==2.3.0
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
@@ -87,10 +89,27 @@ certifi==2023.7.22
     #   httpx
 charset-normalizer==3.0.1
     # via aiohttp
-click==8.1.3
+click==8.1.7
     # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
+    #   distributed
     #   typer
     #   uvicorn
+cloudpickle==2.2.1
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
+    #   distributed
+dask==2023.3.2
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   -r requirements/_base.in
+    #   distributed
+distributed==2023.3.2
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
 dnspython==2.3.0
     # via email-validator
 email-validator==1.3.1
@@ -112,6 +131,10 @@ frozenlist==1.3.3
     # via
     #   aiohttp
     #   aiosignal
+fsspec==2023.6.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
 h11==0.14.0
     # via
     #   httpcore
@@ -136,6 +159,23 @@ idna==3.4
     #   email-validator
     #   httpx
     #   yarl
+importlib-metadata==6.8.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
+jinja2==3.1.2
+    # via
+    #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
+    #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 jmespath==1.0.1
     # via
     #   boto3
@@ -145,10 +185,23 @@ jsonschema==3.2.0
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
+locket==1.0.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
+    #   partd
 markdown-it-py==3.0.0
     # via rich
+markupsafe==2.1.3
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   jinja2
 mdurl==0.1.2
     # via markdown-it-py
+msgpack==1.0.5
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 multidict==6.0.4
     # via
     #   aiohttp
@@ -158,10 +211,22 @@ orjson==3.9.7
     #   -r requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
     #   -r requirements/../../../packages/service-library/requirements/./../../../packages/models-library/requirements/_base.in
-packaging==23.0
-    # via -r requirements/_base.in
+packaging==23.1
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   -r requirements/_base.in
+    #   dask
+    #   distributed
 pamqp==3.2.1
     # via aiormq
+partd==1.4.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   dask
+psutil==5.9.5
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 pydantic==1.10.2
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -206,7 +271,10 @@ pyyaml==6.0.1
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   dask
+    #   distributed
 redis==4.5.4
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -236,6 +304,10 @@ sniffio==1.3.0
     #   anyio
     #   httpcore
     #   httpx
+sortedcontainers==2.4.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 starlette==0.27.0
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -248,14 +320,26 @@ starlette==0.27.0
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
     #   fastapi
+tblib==2.0.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 tenacity==8.2.1
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
     #   -r requirements/../../../packages/service-library/requirements/_base.in
 toolz==0.12.0
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
     #   -r requirements/../../../packages/service-library/requirements/_base.in
+    #   dask
+    #   distributed
+    #   partd
+tornado==6.3.3
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
 tqdm==4.64.1
     # via
     #   -c requirements/../../../packages/service-library/requirements/./_base.in
@@ -278,7 +362,7 @@ typing-extensions==4.5.0
     #   pydantic
     #   types-aiobotocore
     #   types-aiobotocore-ec2
-urllib3==1.26.14
+urllib3==1.26.16
     # via
     #   -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
@@ -289,7 +373,9 @@ urllib3==1.26.14
     #   -c requirements/../../../packages/service-library/requirements/./../../../requirements/constraints.txt
     #   -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
     #   botocore
+    #   distributed
 uvicorn==0.20.0
     # via -r requirements/../../../packages/service-library/requirements/_fastapi.in
 wrapt==1.14.1
@@ -299,6 +385,14 @@ yarl==1.9.2
     #   aio-pika
     #   aiohttp
     #   aiormq
+zict==3.0.0
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   distributed
+zipp==3.16.2
+    # via
+    #   -c requirements/../../../services/dask-sidecar/requirements/_dask-distributed.txt
+    #   importlib-metadata
 
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools
diff --git a/services/autoscaling/requirements/_test.txt b/services/autoscaling/requirements/_test.txt
@@ -53,7 +53,7 @@ charset-normalizer==3.0.1
     # via
     #   -c requirements/_base.txt
     #   requests
-click==8.1.3
+click==8.1.7
     # via
     #   -c requirements/_base.txt
     #   flask
@@ -121,6 +121,7 @@ itsdangerous==2.1.2
 jinja2==3.1.2
     # via
     #   -c requirements/../../../requirements/constraints.txt
+    #   -c requirements/_base.txt
     #   flask
     #   moto
 jmespath==1.0.1
@@ -151,6 +152,7 @@ lupa==2.0
     # via fakeredis
 markupsafe==2.1.3
     # via
+    #   -c requirements/_base.txt
     #   jinja2
     #   werkzeug
 moto==4.2.2
@@ -165,7 +167,7 @@ openapi-spec-validator==0.4.0
     # via moto
 ordered-set==4.1.0
     # via deepdiff
-packaging==23.0
+packaging==23.1
     # via
     #   -c requirements/_base.txt
     #   docker
@@ -179,7 +181,9 @@ pluggy==1.3.0
 pprintpp==0.4.0
     # via pytest-icdiff
 psutil==5.9.5
-    # via -r requirements/_test.in
+    # via
+    #   -c requirements/_base.txt
+    #   -r requirements/_test.in
 py-partiql-parser==0.3.6
     # via moto
 pyasn1==0.5.0
@@ -277,7 +281,9 @@ sniffio==1.3.0
     #   httpcore
     #   httpx
 sortedcontainers==2.4.0
-    # via fakeredis
+    # via
+    #   -c requirements/_base.txt
+    #   fakeredis
 sshpubkeys==3.3.1
     # via moto
 sympy==1.12
@@ -293,7 +299,7 @@ typing-extensions==4.5.0
     #   -c requirements/_base.txt
     #   aws-sam-translator
     #   pydantic
-urllib3==1.26.14
+urllib3==1.26.16
     # via
     #   -c requirements/../../../requirements/constraints.txt
     #   -c requirements/_base.txt

diff --git a/services/autoscaling/requirements/_tools.txt b/services/autoscaling/requirements/_tools.txt
@@ -14,7 +14,7 @@ bump2version==1.0.1
     # via -r requirements/../../../requirements/devenv.txt
 cfgv==3.4.0
     # via pre-commit
-click==8.1.3
+click==8.1.7
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt
@@ -40,7 +40,7 @@ mypy-extensions==1.0.0
     # via black
 nodeenv==1.8.0
     # via pre-commit
-packaging==23.0
+packaging==23.1
     # via
     #   -c requirements/_base.txt
     #   -c requirements/_test.txt