Merge branch 'master' into buy-credits

packages/simcore-sdk/src/simcore_sdk/node_ports_common/_filemanager.py

@@ -23,6 +23,7 @@
 from tenacity.wait import wait_fixed
 
 from . import exceptions, storage_client
+from .storage_endpoint import get_basic_auth
 
 _logger = logging.getLogger(__name__)
 
@@ -57,6 +58,7 @@ async def _complete_upload(
     async with session.post(
         upload_completion_link,
         json=jsonable_encoder(FileUploadCompletionBody(parts=parts)),
+        auth=get_basic_auth(),
     ) as resp:
         resp.raise_for_status()
         # now poll for state
@@ -80,7 +82,7 @@ async def _complete_upload(
         before_sleep=before_sleep_log(_logger, logging.DEBUG),
     ):
         with attempt:
-            async with session.post(state_url) as resp:
+            async with session.post(state_url, auth=get_basic_auth()) as resp:
                 resp.raise_for_status()
                 future_enveloped = parse_obj_as(
                     Envelope[FileUploadCompleteFutureResponse], await resp.json()
@@ -127,7 +129,7 @@ async def _abort_upload(
 ) -> None:
     # abort the upload correctly, so it can revert back to last version
     try:
-        async with session.post(abort_upload_link) as resp:
+        async with session.post(abort_upload_link, auth=get_basic_auth()) as resp:
             resp.raise_for_status()
     except ClientError:
         _logger.warning("Error while aborting upload", exc_info=True)

packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_client.py

@@ -2,12 +2,12 @@
 import logging
 from collections.abc import AsyncIterator, Awaitable, Callable
 from contextlib import asynccontextmanager
-from functools import lru_cache, wraps
+from functools import wraps
 from json import JSONDecodeError
 from typing import Any, TypeAlias
 from urllib.parse import quote
 
-from aiohttp import BasicAuth, ClientResponse, ClientSession
+from aiohttp import ClientResponse, ClientSession
 from aiohttp import client as aiohttp_client_module
 from aiohttp.client_exceptions import ClientConnectionError, ClientResponseError
 from models_library.api_schemas_storage import (
@@ -25,7 +25,6 @@
 from pydantic import ByteSize
 from pydantic.networks import AnyUrl
 from servicelib.aiohttp import status
-from settings_library.node_ports import NodePortsSettings
 from tenacity import RetryCallState
 from tenacity._asyncio import AsyncRetrying
 from tenacity.before_sleep import before_sleep_log
@@ -34,6 +33,7 @@
 from tenacity.wait import wait_exponential
 
 from . import exceptions
+from .storage_endpoint import get_base_url, get_basic_auth
 
 _logger = logging.getLogger(__name__)
 
@@ -73,28 +73,6 @@ async def wrapped(*args, **kwargs):
     return wrapped
 
 
-@lru_cache
-def _base_url() -> str:
-    settings = NodePortsSettings.create_from_envs()
-    base_url: str = settings.NODE_PORTS_STORAGE_AUTH.api_base_url
-    return base_url
-
-
-@lru_cache
-def _get_basic_auth() -> BasicAuth | None:
-    settings = NodePortsSettings.create_from_envs()
-    node_ports_storage_auth = settings.NODE_PORTS_STORAGE_AUTH
-
-    if node_ports_storage_auth.auth_required:
-        assert node_ports_storage_auth.STORAGE_USERNAME is not None  # nosec
-        assert node_ports_storage_auth.STORAGE_PASSWORD is not None  # nosec
-        return BasicAuth(
-            login=node_ports_storage_auth.STORAGE_USERNAME,
-            password=node_ports_storage_auth.STORAGE_PASSWORD.get_secret_value(),
-        )
-    return None
-
-
 def _after_log(log: logging.Logger) -> Callable[[RetryCallState], None]:
     def log_it(retry_state: RetryCallState) -> None:
         assert retry_state.outcome  # nosec
@@ -111,7 +89,7 @@ def log_it(retry_state: RetryCallState) -> None:
 def _session_method(
     session: ClientSession, method: str, url: str, **kwargs
 ) -> RequestContextManager:
-    return session.request(method, url, auth=_get_basic_auth(), **kwargs)
+    return session.request(method, url, auth=get_basic_auth(), **kwargs)
 
 
 @asynccontextmanager
@@ -155,7 +133,7 @@ async def get_storage_locations(
     async with retry_request(
         session,
         "GET",
-        f"{_base_url()}/locations",
+        f"{get_base_url()}/locations",
         expected_status=status.HTTP_200_OK,
         params={"user_id": f"{user_id}"},
     ) as response:
@@ -184,7 +162,7 @@ async def get_download_file_link(
     async with retry_request(
         session,
         "GET",
-        f"{_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
+        f"{get_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
         expected_status=status.HTTP_200_OK,
         params={"user_id": f"{user_id}", "link_type": link_type.value},
     ) as response:
@@ -229,7 +207,7 @@ async def get_upload_file_links(
     async with retry_request(
         session,
         "PUT",
-        f"{_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
+        f"{get_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
         expected_status=status.HTTP_200_OK,
         params=query_params,
     ) as response:
@@ -253,7 +231,7 @@ async def get_file_metadata(
     async with retry_request(
         session,
         "GET",
-        f"{_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}/metadata",
+        f"{get_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}/metadata",
         expected_status=status.HTTP_200_OK,
         params={"user_id": f"{user_id}"},
     ) as response:
@@ -276,7 +254,7 @@ async def list_file_metadata(
     async with retry_request(
         session,
         "GET",
-        f"{_base_url()}/locations/{location_id}/files/metadata",
+        f"{get_base_url()}/locations/{location_id}/files/metadata",
         expected_status=status.HTTP_200_OK,
         params={"user_id": f"{user_id}", "uuid_filter": uuid_filter},
     ) as resp:
@@ -297,7 +275,7 @@ async def delete_file(
     async with retry_request(
         session,
         "DELETE",
-        f"{_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
+        f"{get_base_url()}/locations/{location_id}/files/{quote(file_id, safe='')}",
         expected_status=status.HTTP_204_NO_CONTENT,
         params={"user_id": f"{user_id}"},
     ):

packages/simcore-sdk/src/simcore_sdk/node_ports_common/storage_endpoint.py

@@ -0,0 +1,26 @@
+from functools import lru_cache
+
+from aiohttp import BasicAuth
+from settings_library.node_ports import NodePortsSettings
+
+
+@lru_cache
+def get_base_url() -> str:
+    settings = NodePortsSettings.create_from_envs()
+    base_url: str = settings.NODE_PORTS_STORAGE_AUTH.api_base_url
+    return base_url
+
+
+@lru_cache
+def get_basic_auth() -> BasicAuth | None:
+    settings = NodePortsSettings.create_from_envs()
+    node_ports_storage_auth = settings.NODE_PORTS_STORAGE_AUTH
+
+    if node_ports_storage_auth.auth_required:
+        assert node_ports_storage_auth.STORAGE_USERNAME is not None  # nosec
+        assert node_ports_storage_auth.STORAGE_PASSWORD is not None  # nosec
+        return BasicAuth(
+            login=node_ports_storage_auth.STORAGE_USERNAME,
+            password=node_ports_storage_auth.STORAGE_PASSWORD.get_secret_value(),
+        )
+    return None

packages/simcore-sdk/tests/unit/test_storage_client.py

@@ -27,20 +27,19 @@
 from simcore_sdk.node_ports_common import exceptions
 from simcore_sdk.node_ports_common.storage_client import (
     LinkType,
-    _base_url,
-    _get_basic_auth,
     delete_file,
     get_download_file_link,
     get_file_metadata,
     get_storage_locations,
     get_upload_file_links,
     list_file_metadata,
 )
+from simcore_sdk.node_ports_common.storage_endpoint import get_base_url, get_basic_auth
 
 
 def _clear_caches():
-    _base_url.cache_clear()
-    _get_basic_auth.cache_clear()
+    get_base_url.cache_clear()
+    get_basic_auth.cache_clear()
 
 
 @pytest.fixture
@@ -311,8 +310,8 @@ def test_mode_ports_storage_with_auth(
 ):
     setenvs_from_dict(monkeypatch, envs)
 
-    assert _base_url() == expected_base_url
-    assert _get_basic_auth() == aiohttp.BasicAuth(
+    assert get_base_url() == expected_base_url
+    assert get_basic_auth() == aiohttp.BasicAuth(
         login="user", password="passwd", encoding="latin1"
     )
 
@@ -344,5 +343,5 @@ def test_mode_ports_storage_without_auth(
 ):
     setenvs_from_dict(monkeypatch, envs)
 
-    assert _base_url() == expected_base_url
-    assert _get_basic_auth() is None
+    assert get_base_url() == expected_base_url
+    assert get_basic_auth() is None

services/api-server/src/simcore_service_api_server/api/errors/httpx_client_error.py

@@ -29,11 +29,7 @@ async def handle_httpx_client_exceptions(_: Request, exc: HTTPError):
         detail = f"{exc.request.url.host.capitalize()} service unexpectedly failed"
 
     if status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR:
-        _logger.exception(
-            "%s. host=%s",
-            detail,
-            exc.request.url.host,
-        )
+        _logger.exception("%s. host=%s. %s", detail, exc.request.url.host, f"{exc}")
     raise HTTPException(
         status_code=status_code, detail=detail, headers=headers
     ) from exc

services/api-server/src/simcore_service_api_server/services/_service_exception_handling.py

@@ -34,42 +34,56 @@ def backend_service_exception_handler(
     http_status_map: Mapping[int, tuple[int, Callable[[dict], str] | None]],
     **endpoint_kwargs,
 ):
+    status_code: int
+    error_code: str
+    detail: str
+    headers: dict[str, str] = {}
     try:
         yield
     except ValidationError as exc:
         error_code = create_error_code(exc)
+        status_code = status.HTTP_502_BAD_GATEWAY
+        detail = f"{service_name} service returned invalid response. {error_code}"
         _logger.exception(
-            "Invalid data exchanged with %s service [%s] ",
+            "Invalid data exchanged with %s service [%s]: %s",
             service_name,
             error_code,
+            f"{exc}",
             extra={"error_code": error_code},
         )
         raise HTTPException(
-            status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=f"{service_name} service returned invalid response. {error_code}",
+            status_code=status_code, detail=detail, headers=headers
         ) from exc
-
     except httpx.HTTPStatusError as exc:
+        error_code = create_error_code(exc)
         if status_detail_tuple := http_status_map.get(exc.response.status_code):
             status_code, detail_callback = status_detail_tuple
             if detail_callback is None:
-                detail = f"{exc}"
+                detail = f"{exc}. {error_code}"
             else:
-                detail = detail_callback(endpoint_kwargs)
-            raise HTTPException(status_code=status_code, detail=detail) from exc
-        if exc.response.status_code in {
-            status.HTTP_503_SERVICE_UNAVAILABLE,
+                detail = f"{detail_callback(endpoint_kwargs)}. {error_code}"
+        elif exc.response.status_code in {
             status.HTTP_429_TOO_MANY_REQUESTS,
+            status.HTTP_503_SERVICE_UNAVAILABLE,
         }:
-            headers = {}
-            if "Retry-After" in exc.response.headers:
-                headers["Retry-After"] = exc.response.headers["Retry-After"]
-            raise HTTPException(
-                status_code=exc.response.status_code,
-                detail=f"The {service_name} service was unavailable",
-                headers=headers,
-            ) from exc
+            status_code = exc.response.status_code
+            detail = f"The {service_name} service was unavailable. {error_code}"
+            if retry_after := exc.response.headers.get("Retry-After"):
+                headers["Retry-After"] = retry_after
+        else:
+            status_code = status.HTTP_502_BAD_GATEWAY
+            detail = f"Received unexpected response from {service_name}. {error_code}"
+
+        if status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR:
+            _logger.exception(
+                "Converted status code %s from %s service to %s [%s]: %s",
+                f"{exc.response.status_code}",
+                service_name,
+                f"{status_code}",
+                error_code,
+                f"{exc}",
+                extra={"error_code": error_code},
+            )
         raise HTTPException(
-            status_code=status.HTTP_502_BAD_GATEWAY,
-            detail=f"Received unexpected response from {service_name}",
+            status_code=status_code, detail=detail, headers=headers
         ) from exc

services/payments/src/simcore_service_payments/api/rpc/_payments_methods.py

@@ -22,6 +22,7 @@
 from ...db.payments_methods_repo import PaymentsMethodsRepo
 from ...db.payments_transactions_repo import PaymentsTransactionsRepo
 from ...services import payments, payments_methods
+from ...services.notifier import NotifierService
 from ...services.payments_gateway import PaymentsGatewayApi
 from ...services.resource_usage_tracker import ResourceUsageTrackerApi
 
@@ -176,6 +177,7 @@ async def pay_with_payment_method(  # noqa: PLR0913 # pylint: disable=too-many-a
             rut=ResourceUsageTrackerApi.get_from_app_state(app),
             repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine),
             repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine),
+            notifier=NotifierService.get_from_app_state(app),
             payment_method_id=payment_method_id,
             amount_dollars=amount_dollars,
             target_credits=target_credits,

services/payments/src/simcore_service_payments/services/auto_recharge_process_message.py

@@ -27,6 +27,7 @@
 
 from ..core.settings import ApplicationSettings
 from .auto_recharge import get_wallet_auto_recharge
+from .notifier import NotifierService
 from .payments import pay_with_payment_method
 from .payments_gateway import PaymentsGatewayApi
 from .rabbitmq import get_rabbitmq_rpc_client
@@ -146,15 +147,13 @@ async def _perform_auto_recharge(
     )
     credit_result = parse_obj_as(CreditResultGet, result)
 
-    payments_gateway = PaymentsGatewayApi.get_from_app_state(app)
-    payments_transactions_repo = PaymentsTransactionsRepo(db_engine=app.state.engine)
-    rut_api = ResourceUsageTrackerApi.get_from_app_state(app)
-
     await pay_with_payment_method(
-        gateway=payments_gateway,
-        rut=rut_api,
-        repo_transactions=payments_transactions_repo,
+        gateway=PaymentsGatewayApi.get_from_app_state(app),
+        rut=ResourceUsageTrackerApi.get_from_app_state(app),
+        repo_transactions=PaymentsTransactionsRepo(db_engine=app.state.engine),
         repo_methods=PaymentsMethodsRepo(db_engine=app.state.engine),
+        notifier=NotifierService.get_from_app_state(app),
+        #
         payment_method_id=cast(PaymentMethodID, wallet_auto_recharge.payment_method_id),
         amount_dollars=wallet_auto_recharge.top_up_amount_in_usd,
         target_credits=credit_result.credit_amount,