From 7c718f1697c6eb6376e056e320b8e66d59b0e0c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= <33755295+zavarin-michael@users.noreply.github.com> Date: Sun, 17 Dec 2023 20:23:56 +0500 Subject: [PATCH 1/6] first attempt to transport ports from database to redis --- example_script.sh | 1 + overhave/storage/emulation_storage.py | 39 ++++++++++--------- tests/integration/emulation/test_emulator.py | 4 +- .../storage/test_emulation_storage.py | 2 +- 4 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 example_script.sh diff --git a/example_script.sh b/example_script.sh new file mode 100644 index 00000000..6cfb3e7d --- /dev/null +++ b/example_script.sh @@ -0,0 +1 @@ +echo 'Hello world!' \ No newline at end of file diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index a7727a7c..6a6b92bf 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -1,14 +1,15 @@ import abc import logging +import pickle import socket from typing import cast import sqlalchemy as sa -import sqlalchemy.orm as so from overhave import db from overhave.entities.settings import OverhaveEmulationSettings from overhave.storage import EmulationRunModel +from overhave.transport.redis.deps import make_redis, get_redis_settings from overhave.utils import get_current_time logger = logging.getLogger(__name__) @@ -52,6 +53,8 @@ class EmulationStorage(IEmulationStorage): """Class for emulation runs storage.""" def __init__(self, settings: OverhaveEmulationSettings): + self._redis = redis = make_redis(get_redis_settings()) + self._redis.set('allocated_ports', pickle.dumps([])) self._settings = settings self._emulation_ports_len = len(self._settings.emulation_ports) @@ -65,20 +68,9 @@ def create_emulation_run(emulation_id: int, initiated_by: str) -> int: session.flush() return emulation_run.id - def _get_next_port(self, session: so.Session) -> int: - runs_with_allocated_ports = ( # noqa: ECE001 - session.query(db.EmulationRun) - .filter(db.EmulationRun.port.isnot(None)) - .order_by(db.EmulationRun.id.desc()) - .limit(self._emulation_ports_len) - .all() - ) - allocated_sorted_runs = sorted( - runs_with_allocated_ports, - key=lambda t: t.changed_at, - ) - - allocated_ports = {run.port for run in allocated_sorted_runs} + def _get_next_port(self) -> int: + allocated_ports = self.get_allocated_ports() + logger.debug("Allocated ports: %s", allocated_ports) not_allocated_ports = set(self._settings.emulation_ports).difference(allocated_ports) logger.debug("Not allocated ports: %s", not_allocated_ports) @@ -88,12 +80,20 @@ def _get_next_port(self, session: so.Session) -> int: continue return port logger.debug("All not allocated ports are busy!") - for run in allocated_sorted_runs: - if self._is_port_in_use(cast(int, run.port)): + for port in allocated_ports: + if self._is_port_in_use(cast(int, port)): continue - return cast(int, run.port) + return cast(int, port) raise AllPortsAreBusyError("All ports are busy - could not find free port!") + def get_allocated_ports(self): + return pickle.loads(self._redis.get('allocated_ports')) + + def allocate_port(self, port): + new_allocated_ports = self.get_allocated_ports() + new_allocated_ports.append(port) + self._redis.set('allocated_ports', pickle.dumps(sorted(new_allocated_ports))) + def _is_port_in_use(self, port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: return s.connect_ex((self._settings.emulation_bind_ip, port)) == 0 @@ -102,7 +102,8 @@ def get_requested_emulation_run(self, emulation_run_id: int) -> EmulationRunMode with db.create_session() as session: emulation_run = session.query(db.EmulationRun).filter(db.EmulationRun.id == emulation_run_id).one() emulation_run.status = db.EmulationStatus.REQUESTED - emulation_run.port = self._get_next_port(session) + emulation_run.port = self._get_next_port() + self.allocate_port(emulation_run.port) emulation_run.changed_at = get_current_time() return EmulationRunModel.model_validate(emulation_run) diff --git a/tests/integration/emulation/test_emulator.py b/tests/integration/emulation/test_emulator.py index a214737e..cf1b57ce 100644 --- a/tests/integration/emulation/test_emulator.py +++ b/tests/integration/emulation/test_emulator.py @@ -18,7 +18,7 @@ class TestEmulator: def test_start_emulation( self, emulator: Emulator, emulation_task: EmulationTask, mock_subprocess_popen: MagicMock ) -> None: - with count_queries(8): + with count_queries(6): emulator.start_emulation(task=emulation_task) with create_test_session() as session: emulation_run_db = session.get(db.EmulationRun, emulation_task.data.emulation_run_id) @@ -33,7 +33,7 @@ def test_start_emulation_with_error( emulation_task: EmulationTask, mock_subprocess_popen: MagicMock, ) -> None: - with count_queries(8): + with count_queries(6): emulator.start_emulation(task=emulation_task) with create_test_session() as session: emulation_run_db = session.get(db.EmulationRun, emulation_task.data.emulation_run_id) diff --git a/tests/integration/storage/test_emulation_storage.py b/tests/integration/storage/test_emulation_storage.py index 7d7b052f..1153b14e 100644 --- a/tests/integration/storage/test_emulation_storage.py +++ b/tests/integration/storage/test_emulation_storage.py @@ -36,7 +36,7 @@ def test_get_requested_emulation_run( test_emulation_storage: EmulationStorage, test_emulation_run: EmulationRunModel, ) -> None: - with count_queries(7): + with count_queries(5): requested_emulation_run = test_emulation_storage.get_requested_emulation_run(test_emulation_run.id) assert requested_emulation_run.status == EmulationStatus.REQUESTED assert requested_emulation_run.emulation_id == test_emulation_run.emulation_id From 8dc04d83502f15522f6a217f3be4a303e7a7c6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= <33755295+zavarin-michael@users.noreply.github.com> Date: Sun, 17 Dec 2023 20:26:39 +0500 Subject: [PATCH 2/6] black fix --- overhave/storage/emulation_storage.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index 6a6b92bf..0f48fc4e 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -9,7 +9,7 @@ from overhave import db from overhave.entities.settings import OverhaveEmulationSettings from overhave.storage import EmulationRunModel -from overhave.transport.redis.deps import make_redis, get_redis_settings +from overhave.transport.redis.deps import get_redis_settings, make_redis from overhave.utils import get_current_time logger = logging.getLogger(__name__) @@ -54,7 +54,7 @@ class EmulationStorage(IEmulationStorage): def __init__(self, settings: OverhaveEmulationSettings): self._redis = redis = make_redis(get_redis_settings()) - self._redis.set('allocated_ports', pickle.dumps([])) + self._redis.set("allocated_ports", pickle.dumps([])) self._settings = settings self._emulation_ports_len = len(self._settings.emulation_ports) @@ -87,12 +87,12 @@ def _get_next_port(self) -> int: raise AllPortsAreBusyError("All ports are busy - could not find free port!") def get_allocated_ports(self): - return pickle.loads(self._redis.get('allocated_ports')) + return pickle.loads(self._redis.get("allocated_ports")) def allocate_port(self, port): new_allocated_ports = self.get_allocated_ports() new_allocated_ports.append(port) - self._redis.set('allocated_ports', pickle.dumps(sorted(new_allocated_ports))) + self._redis.set("allocated_ports", pickle.dumps(sorted(new_allocated_ports))) def _is_port_in_use(self, port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: From 3a567ca8e87b07caf149367a5973a4c6887a4efe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= <33755295+zavarin-michael@users.noreply.github.com> Date: Sun, 17 Dec 2023 20:41:24 +0500 Subject: [PATCH 3/6] small fix --- overhave/storage/emulation_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index 0f48fc4e..e9a91ca6 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -53,7 +53,7 @@ class EmulationStorage(IEmulationStorage): """Class for emulation runs storage.""" def __init__(self, settings: OverhaveEmulationSettings): - self._redis = redis = make_redis(get_redis_settings()) + self._redis = make_redis(get_redis_settings()) self._redis.set("allocated_ports", pickle.dumps([])) self._settings = settings self._emulation_ports_len = len(self._settings.emulation_ports) From be0d51d747a9410f828ec653a2d58998ad118c41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= <33755295+zavarin-michael@users.noreply.github.com> Date: Thu, 4 Jan 2024 22:10:34 +0500 Subject: [PATCH 4/6] fixes --- .../scripts/example_script.sh | 0 overhave/api/deps.py | 2 +- overhave/entities/settings.py | 1 + overhave/factory/base_factory.py | 3 +- overhave/storage/emulation_storage.py | 21 ++--- poetry.lock | 83 ++++++++++++++++++- pyproject.toml | 1 + tests/integration/conftest.py | 3 +- 8 files changed, 100 insertions(+), 14 deletions(-) rename example_script.sh => demo/scripts/example_script.sh (100%) diff --git a/example_script.sh b/demo/scripts/example_script.sh similarity index 100% rename from example_script.sh rename to demo/scripts/example_script.sh diff --git a/overhave/api/deps.py b/overhave/api/deps.py index 36c6ad9a..4d4ccdb1 100644 --- a/overhave/api/deps.py +++ b/overhave/api/deps.py @@ -82,7 +82,7 @@ def get_test_run_storage() -> ITestRunStorage: @cache def get_emulation_storage() -> IEmulationStorage: - return EmulationStorage(OverhaveEmulationSettings()) + return EmulationStorage(OverhaveEmulationSettings(), make_redis(get_redis_settings())) @cache diff --git a/overhave/entities/settings.py b/overhave/entities/settings.py index 1b6d4a4c..91b5f5cd 100644 --- a/overhave/entities/settings.py +++ b/overhave/entities/settings.py @@ -139,6 +139,7 @@ class OverhaveEmulationSettings(BaseOverhavePrefix): emulation_bind_ip: str = "0.0.0.0" # noqa: S104 # Ports for emulation binding. Expects as string with format `["port1", "port2", ...]` emulation_ports: list[int] = [8080] + redis_ports_key: str = "allocated_ports" # As a real service, should be used follow path: `http://my-service.domain/mount` # where `emulation_service_url` = `http://my-service.domain` - URL for service, diff --git a/overhave/factory/base_factory.py b/overhave/factory/base_factory.py index 1321db29..0ea966ed 100644 --- a/overhave/factory/base_factory.py +++ b/overhave/factory/base_factory.py @@ -29,6 +29,7 @@ ) from overhave.test_execution import PytestRunner, StepCollector from overhave.transport import S3Manager +from overhave.transport.redis.deps import make_redis, get_redis_settings class IOverhaveFactory(Generic[TApplicationContext], abc.ABC): @@ -120,7 +121,7 @@ def draft_storage(self) -> IDraftStorage: @cached_property def _emulation_storage(self) -> EmulationStorage: - return EmulationStorage(settings=self.context.emulation_settings) + return EmulationStorage(settings=self.context.emulation_settings, redis=make_redis(get_redis_settings())) @property def emulation_storage(self) -> IEmulationStorage: diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index e9a91ca6..77bd19bf 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -1,15 +1,16 @@ import abc import logging -import pickle + import socket -from typing import cast +from typing import cast, List +import orjson import sqlalchemy as sa +from redis import Redis from overhave import db from overhave.entities.settings import OverhaveEmulationSettings from overhave.storage import EmulationRunModel -from overhave.transport.redis.deps import get_redis_settings, make_redis from overhave.utils import get_current_time logger = logging.getLogger(__name__) @@ -52,10 +53,10 @@ def get_emulation_runs_by_test_user_id(test_user_id: int) -> list[EmulationRunMo class EmulationStorage(IEmulationStorage): """Class for emulation runs storage.""" - def __init__(self, settings: OverhaveEmulationSettings): - self._redis = make_redis(get_redis_settings()) - self._redis.set("allocated_ports", pickle.dumps([])) + def __init__(self, settings: OverhaveEmulationSettings, redis: Redis): + self._redis = redis self._settings = settings + self._redis.set(self._settings.redis_ports_key, orjson.dumps([])) self._emulation_ports_len = len(self._settings.emulation_ports) @staticmethod @@ -86,13 +87,13 @@ def _get_next_port(self) -> int: return cast(int, port) raise AllPortsAreBusyError("All ports are busy - could not find free port!") - def get_allocated_ports(self): - return pickle.loads(self._redis.get("allocated_ports")) + def get_allocated_ports(self) -> List[int]: + return orjson.loads(self._redis.get(self._settings.redis_ports_key)) - def allocate_port(self, port): + def allocate_port(self, port: int) -> None: new_allocated_ports = self.get_allocated_ports() new_allocated_ports.append(port) - self._redis.set("allocated_ports", pickle.dumps(sorted(new_allocated_ports))) + self._redis.set(self._settings.redis_ports_key, orjson.dumps(sorted(new_allocated_ports))) def _is_port_in_use(self, port: int) -> bool: with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: diff --git a/poetry.lock b/poetry.lock index d98cb7f4..8bf2c256 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1197,6 +1197,7 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -1205,6 +1206,7 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -1234,6 +1236,7 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -1242,6 +1245,7 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -1678,6 +1682,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -1922,6 +1936,65 @@ files = [ {file = "nh3-0.2.14.tar.gz", hash = "sha256:a0c509894fd4dccdff557068e5074999ae3b75f4c5a2d6fb5415e782e25679c4"}, ] +[[package]] +name = "orjson" +version = "3.9.10" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.9.10-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:c18a4da2f50050a03d1da5317388ef84a16013302a5281d6f64e4a3f406aabc4"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5148bab4d71f58948c7c39d12b14a9005b6ab35a0bdf317a8ade9a9e4d9d0bd5"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4cf7837c3b11a2dfb589f8530b3cff2bd0307ace4c301e8997e95c7468c1378e"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c62b6fa2961a1dcc51ebe88771be5319a93fd89bd247c9ddf732bc250507bc2b"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deeb3922a7a804755bbe6b5be9b312e746137a03600f488290318936c1a2d4dc"}, + {file = "orjson-3.9.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1234dc92d011d3554d929b6cf058ac4a24d188d97be5e04355f1b9223e98bbe9"}, + {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:06ad5543217e0e46fd7ab7ea45d506c76f878b87b1b4e369006bdb01acc05a83"}, + {file = "orjson-3.9.10-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:4fd72fab7bddce46c6826994ce1e7de145ae1e9e106ebb8eb9ce1393ca01444d"}, + {file = "orjson-3.9.10-cp310-none-win32.whl", hash = "sha256:b5b7d4a44cc0e6ff98da5d56cde794385bdd212a86563ac321ca64d7f80c80d1"}, + {file = "orjson-3.9.10-cp310-none-win_amd64.whl", hash = "sha256:61804231099214e2f84998316f3238c4c2c4aaec302df12b21a64d72e2a135c7"}, + {file = "orjson-3.9.10-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:cff7570d492bcf4b64cc862a6e2fb77edd5e5748ad715f487628f102815165e9"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed8bc367f725dfc5cabeed1ae079d00369900231fbb5a5280cf0736c30e2adf7"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c812312847867b6335cfb264772f2a7e85b3b502d3a6b0586aa35e1858528ab1"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9edd2856611e5050004f4722922b7b1cd6268da34102667bd49d2a2b18bafb81"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:674eb520f02422546c40401f4efaf8207b5e29e420c17051cddf6c02783ff5ca"}, + {file = "orjson-3.9.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d0dc4310da8b5f6415949bd5ef937e60aeb0eb6b16f95041b5e43e6200821fb"}, + {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e99c625b8c95d7741fe057585176b1b8783d46ed4b8932cf98ee145c4facf499"}, + {file = "orjson-3.9.10-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:ec6f18f96b47299c11203edfbdc34e1b69085070d9a3d1f302810cc23ad36bf3"}, + {file = "orjson-3.9.10-cp311-none-win32.whl", hash = "sha256:ce0a29c28dfb8eccd0f16219360530bc3cfdf6bf70ca384dacd36e6c650ef8e8"}, + {file = "orjson-3.9.10-cp311-none-win_amd64.whl", hash = "sha256:cf80b550092cc480a0cbd0750e8189247ff45457e5a023305f7ef1bcec811616"}, + {file = "orjson-3.9.10-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:602a8001bdf60e1a7d544be29c82560a7b49319a0b31d62586548835bbe2c862"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f295efcd47b6124b01255d1491f9e46f17ef40d3d7eabf7364099e463fb45f0f"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92af0d00091e744587221e79f68d617b432425a7e59328ca4c496f774a356071"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c5a02360e73e7208a872bf65a7554c9f15df5fe063dc047f79738998b0506a14"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:858379cbb08d84fe7583231077d9a36a1a20eb72f8c9076a45df8b083724ad1d"}, + {file = "orjson-3.9.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666c6fdcaac1f13eb982b649e1c311c08d7097cbda24f32612dae43648d8db8d"}, + {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3fb205ab52a2e30354640780ce4587157a9563a68c9beaf52153e1cea9aa0921"}, + {file = "orjson-3.9.10-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7ec960b1b942ee3c69323b8721df2a3ce28ff40e7ca47873ae35bfafeb4555ca"}, + {file = "orjson-3.9.10-cp312-none-win_amd64.whl", hash = "sha256:3e892621434392199efb54e69edfff9f699f6cc36dd9553c5bf796058b14b20d"}, + {file = "orjson-3.9.10-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:8b9ba0ccd5a7f4219e67fbbe25e6b4a46ceef783c42af7dbc1da548eb28b6531"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2e2ecd1d349e62e3960695214f40939bbfdcaeaaa62ccc638f8e651cf0970e5f"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7f433be3b3f4c66016d5a20e5b4444ef833a1f802ced13a2d852c637f69729c1"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4689270c35d4bb3102e103ac43c3f0b76b169760aff8bcf2d401a3e0e58cdb7f"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4bd176f528a8151a6efc5359b853ba3cc0e82d4cd1fab9c1300c5d957dc8f48c"}, + {file = "orjson-3.9.10-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a2ce5ea4f71681623f04e2b7dadede3c7435dfb5e5e2d1d0ec25b35530e277b"}, + {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:49f8ad582da6e8d2cf663c4ba5bf9f83cc052570a3a767487fec6af839b0e777"}, + {file = "orjson-3.9.10-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2a11b4b1a8415f105d989876a19b173f6cdc89ca13855ccc67c18efbd7cbd1f8"}, + {file = "orjson-3.9.10-cp38-none-win32.whl", hash = "sha256:a353bf1f565ed27ba71a419b2cd3db9d6151da426b61b289b6ba1422a702e643"}, + {file = "orjson-3.9.10-cp38-none-win_amd64.whl", hash = "sha256:e28a50b5be854e18d54f75ef1bb13e1abf4bc650ab9d635e4258c58e71eb6ad5"}, + {file = "orjson-3.9.10-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:ee5926746232f627a3be1cc175b2cfad24d0170d520361f4ce3fa2fd83f09e1d"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a73160e823151f33cdc05fe2cea557c5ef12fdf276ce29bb4f1c571c8368a60"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c338ed69ad0b8f8f8920c13f529889fe0771abbb46550013e3c3d01e5174deef"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5869e8e130e99687d9e4be835116c4ebd83ca92e52e55810962446d841aba8de"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d2c1e559d96a7f94a4f581e2a32d6d610df5840881a8cba8f25e446f4d792df3"}, + {file = "orjson-3.9.10-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:81a3a3a72c9811b56adf8bcc829b010163bb2fc308877e50e9910c9357e78521"}, + {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7f8fb7f5ecf4f6355683ac6881fd64b5bb2b8a60e3ccde6ff799e48791d8f864"}, + {file = "orjson-3.9.10-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:c943b35ecdf7123b2d81d225397efddf0bce2e81db2f3ae633ead38e85cd5ade"}, + {file = "orjson-3.9.10-cp39-none-win32.whl", hash = "sha256:fb0b361d73f6b8eeceba47cd37070b5e6c9de5beaeaa63a1cb35c7e1a73ef088"}, + {file = "orjson-3.9.10-cp39-none-win_amd64.whl", hash = "sha256:b90f340cb6397ec7a854157fac03f0c82b744abdd1c0941a024c3c29d1340aff"}, + {file = "orjson-3.9.10.tar.gz", hash = "sha256:9ebbdbd6a046c304b1845e96fbcc5559cd296b4dfd3ad2509e33c4d9ce07d6a1"}, +] + [[package]] name = "packaging" version = "23.1" @@ -3162,6 +3235,14 @@ files = [ {file = "SQLAlchemy-2.0.21-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b69f1f754d92eb1cc6b50938359dead36b96a1dcf11a8670bff65fd9b21a4b09"}, {file = "SQLAlchemy-2.0.21-cp311-cp311-win32.whl", hash = "sha256:af520a730d523eab77d754f5cf44cc7dd7ad2d54907adeb3233177eeb22f271b"}, {file = "SQLAlchemy-2.0.21-cp311-cp311-win_amd64.whl", hash = "sha256:141675dae56522126986fa4ca713739d00ed3a6f08f3c2eb92c39c6dfec463ce"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:56628ca27aa17b5890391ded4e385bf0480209726f198799b7e980c6bd473bd7"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:db726be58837fe5ac39859e0fa40baafe54c6d54c02aba1d47d25536170b690f"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e7421c1bfdbb7214313919472307be650bd45c4dc2fcb317d64d078993de045b"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:632784f7a6f12cfa0e84bf2a5003b07660addccf5563c132cd23b7cc1d7371a9"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:f6f7276cf26145a888f2182a98f204541b519d9ea358a65d82095d9c9e22f917"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2a1f7ffac934bc0ea717fa1596f938483fb8c402233f9b26679b4f7b38d6ab6e"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-win32.whl", hash = "sha256:bfece2f7cec502ec5f759bbc09ce711445372deeac3628f6fa1c16b7fb45b682"}, + {file = "SQLAlchemy-2.0.21-cp312-cp312-win_amd64.whl", hash = "sha256:526b869a0f4f000d8d8ee3409d0becca30ae73f494cbb48801da0129601f72c6"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:7614f1eab4336df7dd6bee05bc974f2b02c38d3d0c78060c5faa4cd1ca2af3b8"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d59cb9e20d79686aa473e0302e4a82882d7118744d30bb1dfb62d3c47141b3ec"}, {file = "SQLAlchemy-2.0.21-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a95aa0672e3065d43c8aa80080cdd5cc40fe92dc873749e6c1cf23914c4b83af"}, @@ -3731,4 +3812,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "a2d9416046ca8f4e13b0aa1f1e4d5feae1598762336dc8d19cf693c54a1a1b63" +content-hash = "0d63d190a0685b9afcea4fa9ea71ba788c836a1cd523371fcef6c3d1557eb0e1" diff --git a/pyproject.toml b/pyproject.toml index 0ec02f5f..801329b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ pytz = "^2023.3.post1" fastapi = ">=0.99.0" sqlalchemy = "^2.0.17" flask-admin = "^1.6.1" +orjson = "^3.9.10" [tool.poetry.group.dev.dependencies] flake8-awesome = "<1.3" diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7ce2cb84..1a22e990 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,6 +29,7 @@ TestUserSpecification, TestUserStorage, ) +from overhave.transport.redis.deps import make_redis, get_redis_settings from overhave.utils import get_current_time from tests.db_utils import create_test_session @@ -70,7 +71,7 @@ def test_feature_storage() -> FeatureStorage: @pytest.fixture(scope="module") def test_emulation_storage(emulation_settings: OverhaveEmulationSettings) -> EmulationStorage: - return EmulationStorage(emulation_settings) + return EmulationStorage(emulation_settings, make_redis(get_redis_settings())) @pytest.fixture() From ab0e489284c681f03a937dc70a891a543267b5a8 Mon Sep 17 00:00:00 2001 From: zavarin-michael Date: Tue, 9 Jan 2024 20:54:07 +0500 Subject: [PATCH 5/6] kwargs + plint fixes --- overhave/api/deps.py | 2 +- overhave/factory/base_factory.py | 2 +- overhave/storage/emulation_storage.py | 11 +++++------ tests/integration/conftest.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/overhave/api/deps.py b/overhave/api/deps.py index 4d4ccdb1..a85cfb30 100644 --- a/overhave/api/deps.py +++ b/overhave/api/deps.py @@ -82,7 +82,7 @@ def get_test_run_storage() -> ITestRunStorage: @cache def get_emulation_storage() -> IEmulationStorage: - return EmulationStorage(OverhaveEmulationSettings(), make_redis(get_redis_settings())) + return EmulationStorage(settings=OverhaveEmulationSettings(), redis=make_redis(get_redis_settings())) @cache diff --git a/overhave/factory/base_factory.py b/overhave/factory/base_factory.py index 0ea966ed..d6ad5608 100644 --- a/overhave/factory/base_factory.py +++ b/overhave/factory/base_factory.py @@ -29,7 +29,7 @@ ) from overhave.test_execution import PytestRunner, StepCollector from overhave.transport import S3Manager -from overhave.transport.redis.deps import make_redis, get_redis_settings +from overhave.transport.redis.deps import get_redis_settings, make_redis class IOverhaveFactory(Generic[TApplicationContext], abc.ABC): diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index 77bd19bf..1a27b58e 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -1,8 +1,7 @@ import abc import logging - import socket -from typing import cast, List +from typing import Any, List, cast import orjson import sqlalchemy as sa @@ -53,7 +52,7 @@ def get_emulation_runs_by_test_user_id(test_user_id: int) -> list[EmulationRunMo class EmulationStorage(IEmulationStorage): """Class for emulation runs storage.""" - def __init__(self, settings: OverhaveEmulationSettings, redis: Redis): + def __init__(self, settings: OverhaveEmulationSettings, redis: "Redis[Any]"): self._redis = redis self._settings = settings self._redis.set(self._settings.redis_ports_key, orjson.dumps([])) @@ -82,13 +81,13 @@ def _get_next_port(self) -> int: return port logger.debug("All not allocated ports are busy!") for port in allocated_ports: - if self._is_port_in_use(cast(int, port)): + if self._is_port_in_use(port): continue - return cast(int, port) + return port raise AllPortsAreBusyError("All ports are busy - could not find free port!") def get_allocated_ports(self) -> List[int]: - return orjson.loads(self._redis.get(self._settings.redis_ports_key)) + return cast(List[int], orjson.loads(str(self._redis.get(self._settings.redis_ports_key)))) def allocate_port(self, port: int) -> None: new_allocated_ports = self.get_allocated_ports() diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 1a22e990..0342687d 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -29,7 +29,7 @@ TestUserSpecification, TestUserStorage, ) -from overhave.transport.redis.deps import make_redis, get_redis_settings +from overhave.transport.redis.deps import get_redis_settings, make_redis from overhave.utils import get_current_time from tests.db_utils import create_test_session @@ -71,7 +71,7 @@ def test_feature_storage() -> FeatureStorage: @pytest.fixture(scope="module") def test_emulation_storage(emulation_settings: OverhaveEmulationSettings) -> EmulationStorage: - return EmulationStorage(emulation_settings, make_redis(get_redis_settings())) + return EmulationStorage(settings=emulation_settings, redis=make_redis(get_redis_settings())) @pytest.fixture() From c9fdf638fcdcf291144a136804d8eae0b02685d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9C=D0=B8=D1=85=D0=B0=D0=B8=D0=BB?= Date: Tue, 9 Jan 2024 22:22:03 +0500 Subject: [PATCH 6/6] one more fix with orjson --- overhave/storage/emulation_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/overhave/storage/emulation_storage.py b/overhave/storage/emulation_storage.py index 1a27b58e..5f7fd8e0 100644 --- a/overhave/storage/emulation_storage.py +++ b/overhave/storage/emulation_storage.py @@ -87,7 +87,7 @@ def _get_next_port(self) -> int: raise AllPortsAreBusyError("All ports are busy - could not find free port!") def get_allocated_ports(self) -> List[int]: - return cast(List[int], orjson.loads(str(self._redis.get(self._settings.redis_ports_key)))) + return cast(List[int], orjson.loads(cast(bytes, self._redis.get(self._settings.redis_ports_key)))) def allocate_port(self, port: int) -> None: new_allocated_ports = self.get_allocated_ports()