Skip to content

Commit

Permalink
Merge branch 'enh/new-frontend-checker' of github.com:odeimaiz/osparc…
Browse files Browse the repository at this point in the history
…-simcore into enh/new-frontend-checker
  • Loading branch information
odeimaiz committed Nov 14, 2024
2 parents 7a3f8a8 + 4dce7f8 commit 061099b
Show file tree
Hide file tree
Showing 89 changed files with 1,168 additions and 533 deletions.
1 change: 1 addition & 0 deletions .env-devel
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ AGENT_VOLUMES_CLEANUP_S3_ENDPOINT=http://172.17.0.1:9001
AGENT_VOLUMES_CLEANUP_S3_PROVIDER=MINIO
AGENT_VOLUMES_CLEANUP_S3_REGION=us-east-1
AGENT_VOLUMES_CLEANUP_S3_SECRET_KEY=12345678
AGENT_TRACING={}

API_SERVER_DEV_FEATURES_ENABLED=0
API_SERVER_LOGLEVEL=INFO
Expand Down
2 changes: 0 additions & 2 deletions packages/aws-library/requirements/_base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -44,8 +44,6 @@ arrow==1.3.0
# -r requirements/../../../packages/service-library/requirements/../../../packages/models-library/requirements/_base.in
# -r requirements/../../../packages/service-library/requirements/_base.in
# -r requirements/_base.in
async-timeout==4.0.3
# via redis
attrs==24.2.0
# via
# aiohttp
Expand Down
27 changes: 27 additions & 0 deletions packages/notifications-library/requirements/_base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,10 @@ attrs==24.2.0
# referencing
click==8.1.7
# via typer
deprecated==1.2.14
# via
# opentelemetry-api
# opentelemetry-semantic-conventions
dnspython==2.6.1
# via email-validator
email-validator==2.2.0
Expand All @@ -26,6 +30,8 @@ idna==3.10
# via
# email-validator
# yarl
importlib-metadata==8.5.0
# via opentelemetry-api
jinja2==3.1.4
# via
# -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
Expand Down Expand Up @@ -54,13 +60,28 @@ mdurl==0.1.2
# via markdown-it-py
multidict==6.1.0
# via yarl
opentelemetry-api==1.28.1
# via
# opentelemetry-instrumentation
# opentelemetry-instrumentation-asyncpg
# opentelemetry-semantic-conventions
opentelemetry-instrumentation==0.49b1
# via opentelemetry-instrumentation-asyncpg
opentelemetry-instrumentation-asyncpg==0.49b1
# via -r requirements/../../../packages/postgres-database/requirements/_base.in
opentelemetry-semantic-conventions==0.49b1
# via
# opentelemetry-instrumentation
# opentelemetry-instrumentation-asyncpg
orjson==3.10.7
# via
# -c requirements/../../../packages/models-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/postgres-database/requirements/../../../requirements/constraints.txt
# -c requirements/../../../packages/settings-library/requirements/../../../requirements/constraints.txt
# -c requirements/../../../requirements/constraints.txt
# -r requirements/../../../packages/models-library/requirements/_base.in
packaging==24.2
# via opentelemetry-instrumentation
psycopg2-binary==2.9.9
# via sqlalchemy
pydantic==1.10.18
Expand Down Expand Up @@ -109,5 +130,11 @@ typing-extensions==4.12.2
# alembic
# pydantic
# typer
wrapt==1.16.0
# via
# deprecated
# opentelemetry-instrumentation
yarl==1.12.1
# via -r requirements/../../../packages/postgres-database/requirements/_base.in
zipp==3.21.0
# via importlib-metadata
3 changes: 2 additions & 1 deletion packages/notifications-library/requirements/_test.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ mypy==1.12.0
# via sqlalchemy
mypy-extensions==1.0.0
# via mypy
packaging==24.1
packaging==24.2
# via
# -c requirements/_base.txt
# pytest
# pytest-sugar
pluggy==1.5.0
Expand Down
3 changes: 2 additions & 1 deletion packages/notifications-library/requirements/_tools.txt
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,9 @@ mypy-extensions==1.0.0
# mypy
nodeenv==1.9.1
# via pre-commit
packaging==24.1
packaging==24.2
# via
# -c requirements/_base.txt
# -c requirements/_test.txt
# black
# build
Expand Down
2 changes: 0 additions & 2 deletions packages/service-library/requirements/_base.txt
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,6 @@ arrow==1.3.0
# via
# -r requirements/../../../packages/models-library/requirements/_base.in
# -r requirements/_base.in
async-timeout==4.0.3
# via redis
attrs==24.2.0
# via
# aiohttp
Expand Down
1 change: 1 addition & 0 deletions packages/service-library/requirements/_fastapi.in
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
fastapi
httpx
opentelemetry-instrumentation-fastapi
opentelemetry-instrumentation-httpx
prometheus-client
prometheus-fastapi-instrumentator
uvicorn
6 changes: 6 additions & 0 deletions packages/service-library/requirements/_fastapi.txt
Original file line number Diff line number Diff line change
Expand Up @@ -47,23 +47,29 @@ opentelemetry-api==1.27.0
# opentelemetry-instrumentation
# opentelemetry-instrumentation-asgi
# opentelemetry-instrumentation-fastapi
# opentelemetry-instrumentation-httpx
# opentelemetry-semantic-conventions
opentelemetry-instrumentation==0.48b0
# via
# opentelemetry-instrumentation-asgi
# opentelemetry-instrumentation-fastapi
# opentelemetry-instrumentation-httpx
opentelemetry-instrumentation-asgi==0.48b0
# via opentelemetry-instrumentation-fastapi
opentelemetry-instrumentation-fastapi==0.48b0
# via -r requirements/_fastapi.in
opentelemetry-instrumentation-httpx==0.48b0
# via -r requirements/_fastapi.in
opentelemetry-semantic-conventions==0.48b0
# via
# opentelemetry-instrumentation-asgi
# opentelemetry-instrumentation-fastapi
# opentelemetry-instrumentation-httpx
opentelemetry-util-http==0.48b0
# via
# opentelemetry-instrumentation-asgi
# opentelemetry-instrumentation-fastapi
# opentelemetry-instrumentation-httpx
prometheus-client==0.21.0
# via
# -r requirements/_fastapi.in
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,8 @@
from httpx import AsyncClient, ConnectError, HTTPError, PoolTimeout, Response
from httpx._types import TimeoutTypes, URLTypes
from pydantic.errors import PydanticErrorMixin
from servicelib.fastapi.tracing import setup_httpx_client_tracing
from settings_library.tracing import TracingSettings
from tenacity import RetryCallState
from tenacity.asyncio import AsyncRetrying
from tenacity.before_sleep import before_sleep_log
Expand Down Expand Up @@ -201,6 +203,7 @@ def __init__(
base_url: URLTypes | None = None,
default_http_client_timeout: TimeoutTypes | None = None,
extra_allowed_method_names: set[str] | None = None,
tracing_settings: TracingSettings | None,
) -> None:
_assert_public_interface(self, extra_allowed_method_names)

Expand All @@ -220,7 +223,10 @@ def __init__(
if default_http_client_timeout:
client_args["timeout"] = default_http_client_timeout

super().__init__(client=AsyncClient(**client_args))
client = AsyncClient(**client_args)
if tracing_settings:
setup_httpx_client_tracing(client)
super().__init__(client=client)

async def __aenter__(self):
await self.setup_client()
Expand Down
6 changes: 6 additions & 0 deletions packages/service-library/src/servicelib/fastapi/tracing.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,13 @@
import logging

from fastapi import FastAPI
from httpx import AsyncClient, Client
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
OTLPSpanExporter as OTLPSpanExporterHTTP,
)
from opentelemetry.instrumentation.fastapi import FastAPIInstrumentor
from opentelemetry.instrumentation.httpx import HTTPXClientInstrumentor
from opentelemetry.sdk.resources import Resource
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
Expand Down Expand Up @@ -121,3 +123,7 @@ def setup_tracing(
msg="Attempting to add requests opentelemetry autoinstrumentation...",
):
RequestsInstrumentor().instrument()


def setup_httpx_client_tracing(client: AsyncClient | Client):
HTTPXClientInstrumentor.instrument_client(client)
36 changes: 30 additions & 6 deletions packages/service-library/src/servicelib/redis_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import logging
from collections.abc import Awaitable, Callable
from datetime import timedelta
from typing import Any
from typing import Any, ParamSpec, TypeVar

import arrow

Expand All @@ -12,10 +12,16 @@

_logger = logging.getLogger(__file__)

P = ParamSpec("P")
R = TypeVar("R")


def exclusive(
redis: RedisClientSDK, *, lock_key: str, lock_value: bytes | str | None = None
):
redis: RedisClientSDK | Callable[..., RedisClientSDK],
*,
lock_key: str | Callable[..., str],
lock_value: bytes | str | None = None,
) -> Callable[[Callable[P, Awaitable[R]]], Callable[P, Awaitable[R]]]:
"""
Define a method to run exclusively across
processes by leveraging a Redis Lock.
Expand All @@ -24,12 +30,30 @@ def exclusive(
redis: the redis client SDK
lock_key: a string as the name of the lock (good practice: app_name:lock_name)
lock_value: some additional data that can be retrieved by another client
Raises:
- ValueError if used incorrectly
- CouldNotAcquireLockError if the lock could not be acquired
"""

def decorator(func):
if not lock_key:
msg = "lock_key cannot be empty string!"
raise ValueError(msg)

def decorator(func: Callable[P, Awaitable[R]]) -> Callable[P, Awaitable[R]]:
@functools.wraps(func)
async def wrapper(*args, **kwargs):
async with redis.lock_context(lock_key=lock_key, lock_value=lock_value):
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
redis_lock_key = (
lock_key(*args, **kwargs) if callable(lock_key) else lock_key
)
assert isinstance(redis_lock_key, str) # nosec

redis_client = redis(*args, **kwargs) if callable(redis) else redis
assert isinstance(redis_client, RedisClientSDK) # nosec

async with redis_client.lock_context(
lock_key=redis_lock_key, lock_value=lock_value
):
return await func(*args, **kwargs)

return wrapper
Expand Down
30 changes: 21 additions & 9 deletions packages/service-library/tests/fastapi/test_http_client_thin.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,9 @@ def request_timeout() -> int:

@pytest.fixture
async def thick_client(request_timeout: int) -> AsyncIterable[FakeThickClient]:
async with FakeThickClient(total_retry_interval=request_timeout) as client:
async with FakeThickClient(
total_retry_interval=request_timeout, tracing_settings=None
) as client:
yield client


Expand All @@ -95,7 +97,9 @@ async def test_retry_on_errors(
test_url: AnyHttpUrl,
caplog_info_level: pytest.LogCaptureFixture,
) -> None:
client = FakeThickClient(total_retry_interval=request_timeout)
client = FakeThickClient(
total_retry_interval=request_timeout, tracing_settings=None
)

with pytest.raises(ClientHttpError):
await client.get_provided_url(test_url)
Expand All @@ -119,7 +123,7 @@ async def raises_request_error(self) -> Response:
request=Request(method="GET", url=test_url),
)

client = ATestClient(total_retry_interval=request_timeout)
client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)

with pytest.raises(ClientHttpError):
await client.raises_request_error()
Expand All @@ -145,7 +149,7 @@ async def raises_http_error(self) -> Response:
msg = "mock_http_error"
raise HTTPError(msg)

client = ATestClient(total_retry_interval=request_timeout)
client = ATestClient(total_retry_interval=request_timeout, tracing_settings=None)

with pytest.raises(ClientHttpError):
await client.raises_http_error()
Expand All @@ -159,21 +163,25 @@ async def public_method_ok(self) -> Response: # type: ignore
"""this method will be ok even if no code is used"""

# OK
OKTestClient(total_retry_interval=request_timeout)
OKTestClient(total_retry_interval=request_timeout, tracing_settings=None)

class FailWrongAnnotationTestClient(BaseThinClient):
async def public_method_wrong_annotation(self) -> None:
"""this method will raise an error"""

with pytest.raises(AssertionError, match="should return an instance"):
FailWrongAnnotationTestClient(total_retry_interval=request_timeout)
FailWrongAnnotationTestClient(
total_retry_interval=request_timeout, tracing_settings=None
)

class FailNoAnnotationTestClient(BaseThinClient):
async def public_method_no_annotation(self):
"""this method will raise an error"""

with pytest.raises(AssertionError, match="should return an instance"):
FailNoAnnotationTestClient(total_retry_interval=request_timeout)
FailNoAnnotationTestClient(
total_retry_interval=request_timeout, tracing_settings=None
)


async def test_expect_state_decorator(
Expand All @@ -197,7 +205,9 @@ async def get_wrong_state(self) -> Response:
respx_mock.get(url_get_200_ok).mock(return_value=Response(codes.OK))
respx_mock.get(get_wrong_state).mock(return_value=Response(codes.OK))

test_client = ATestClient(total_retry_interval=request_timeout)
test_client = ATestClient(
total_retry_interval=request_timeout, tracing_settings=None
)

# OK
response = await test_client.get_200_ok()
Expand All @@ -218,7 +228,9 @@ async def test_retry_timeout_overwrite(
request_timeout: int,
caplog_info_level: pytest.LogCaptureFixture,
) -> None:
client = FakeThickClient(total_retry_interval=request_timeout)
client = FakeThickClient(
total_retry_interval=request_timeout, tracing_settings=None
)

caplog_info_level.clear()
start = arrow.utcnow()
Expand Down
Loading

0 comments on commit 061099b

Please sign in to comment.