🎨 initiate proper error handling for backend clients in api server (I…

…TISFoundation#5362)

services/api-server/src/simcore_service_api_server/api/errors/custom_errors.py

@@ -1,7 +1,6 @@
 import logging
-from urllib.request import Request
 
-from fastapi import status
+from fastapi import Request, status
 from servicelib.error_codes import create_error_code
 from starlette.responses import JSONResponse
 

services/api-server/src/simcore_service_api_server/api/errors/httpx_client_error.py

@@ -1,50 +1,39 @@
 """ General handling of httpx-based exceptions
 
     - httpx-based clients are used to communicate with other backend services
-    - When those respond with 4XX, 5XX status codes, those are generally handled here
+    - any exception raised by a httpx client will be handled here.
 """
 import logging
+from typing import Any
 
-from fastapi import status
-from httpx import HTTPStatusError
-from starlette.requests import Request
-from starlette.responses import JSONResponse
-
-from .http_error import create_error_json_response
+from fastapi import HTTPException, Request, status
+from httpx import HTTPError, TimeoutException
 
 _logger = logging.getLogger(__file__)
 
 
-async def httpx_client_error_handler(_: Request, exc: HTTPStatusError) -> JSONResponse:
+async def handle_httpx_client_exceptions(_: Request, exc: HTTPError):
     """
-    This is called when HTTPStatusError was raised and reached the outermost handler
-
-    This handler is used as a "last resource" since it is recommended to handle these exceptions
-    closer to the raising point.
-
-    The response had an error HTTP status of 4xx or 5xx, and this is how is
-    transformed in the api-server API
+    Default httpx exception handler.
+    See https://www.python-httpx.org/exceptions/
+    With this in place only HTTPStatusErrors need to be customized closer to the httpx client itself.
     """
-    if exc.response.is_client_error:
-        assert exc.response.is_server_error  # nosec
-        # Forward api-server's client from backend client errors
-        status_code = exc.response.status_code
-        errors = exc.response.json()["errors"]
+    status_code: Any
+    detail: str
+    headers: dict[str, str] = {}
+    if isinstance(exc, TimeoutException):
+        status_code = status.HTTP_504_GATEWAY_TIMEOUT
+        detail = f"Request to {exc.request.url.host.capitalize()} timed out"
     else:
-        assert exc.response.is_server_error  # nosec
-        # Hide api-server's client from backend server errors
-        status_code = status.HTTP_503_SERVICE_UNAVAILABLE
-        message = f"{exc.request.url.host.capitalize()} service unexpectedly failed"
-        errors = [
-            message,
-        ]
+        status_code = status.HTTP_502_BAD_GATEWAY
+        detail = f"{exc.request.url.host.capitalize()} service unexpectedly failed"
 
+    if status_code >= status.HTTP_500_INTERNAL_SERVER_ERROR:
         _logger.exception(
-            "%s. host=%s status-code=%s msg=%s",
-            message,
+            "%s. host=%s",
+            detail,
             exc.request.url.host,
-            exc.response.status_code,
-            exc.response.text,
         )
-
-    return create_error_json_response(*errors, status_code=status_code)
+    raise HTTPException(
+        status_code=status_code, detail=detail, headers=headers
+    ) from exc

services/api-server/src/simcore_service_api_server/api/errors/validation_error.py

@@ -1,9 +1,9 @@
+from fastapi import Request
 from fastapi.encoders import jsonable_encoder
 from fastapi.exceptions import RequestValidationError
 from fastapi.openapi.constants import REF_PREFIX
 from fastapi.openapi.utils import validation_error_response_definition
 from pydantic import ValidationError
-from starlette.requests import Request
 from starlette.responses import JSONResponse
 from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY
 

services/api-server/src/simcore_service_api_server/api/routes/solvers.py

@@ -151,7 +151,7 @@ async def list_solver_releases(
     SEE get_solver_releases_page for a paginated version of this function
     """
     releases: list[Solver] = await catalog_client.list_solver_releases(
-        user_id, solver_key, product_name=product_name
+        user_id=user_id, solver_key=solver_key, product_name=product_name
     )
 
     for solver in releases:
@@ -189,7 +189,7 @@ async def get_solver_release(
 ) -> Solver:
     """Gets a specific release of a solver"""
     try:
-        solver = await catalog_client.get_service(
+        solver: Solver = await catalog_client.get_service(
             user_id=user_id,
             name=solver_key,
             version=version,

services/api-server/src/simcore_service_api_server/api/routes/studies.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Annotated, Any, Final
+from typing import Annotated, Final
 
 from fastapi import APIRouter, Depends, status
 from fastapi_pagination.api import create_page
@@ -124,11 +124,10 @@ async def list_study_ports(
     New in *version 0.5.0* (only with API_SERVER_DEV_FEATURES_ENABLED=1)
     """
     try:
-        project_ports: list[
-            dict[str, Any]
-        ] = await webserver_api.get_project_metadata_ports(project_id=study_id)
-
-        return OnePage[StudyPort](items=project_ports)  # type: ignore[arg-type]
+        project_ports: list[StudyPort] = await webserver_api.get_project_metadata_ports(
+            project_id=study_id
+        )
+        return OnePage[StudyPort](items=project_ports)
 
     except ProjectNotFoundError:
         return create_error_json_response(

services/api-server/src/simcore_service_api_server/api/routes/users.py

@@ -17,8 +17,8 @@
 async def get_my_profile(
     webserver_session: Annotated[AuthSession, Depends(get_webserver_session)],
 ) -> Profile:
-    data = await webserver_session.get("/me")
-    return Profile.parse_obj(data)
+    profile: Profile = await webserver_session.get_me()
+    return profile
 
 
 @router.put("", response_model=Profile)
@@ -28,6 +28,5 @@ async def update_my_profile(
         AuthSession, Security(get_webserver_session, scopes=["write"])
     ],
 ) -> Profile:
-    await webserver_session.put("/me", body=profile_update.dict(exclude_none=True))
-    profile: Profile = await get_my_profile(webserver_session)
+    profile: Profile = await webserver_session.update_me(profile_update)
     return profile

services/api-server/src/simcore_service_api_server/core/application.py

@@ -3,7 +3,7 @@
 from fastapi import FastAPI
 from fastapi.exceptions import RequestValidationError
 from fastapi_pagination import add_pagination
-from httpx import HTTPStatusError
+from httpx import HTTPError as HttpxException
 from models_library.basic_types import BootModeEnum
 from servicelib.fastapi.prometheus_instrumentation import (
     setup_prometheus_instrumentation,
@@ -24,7 +24,7 @@
     http_error_handler,
     make_http_error_handler_for_exception,
 )
-from ..api.errors.httpx_client_error import httpx_client_error_handler
+from ..api.errors.httpx_client_error import handle_httpx_client_exceptions
 from ..api.errors.validation_error import http422_error_handler
 from ..api.root import create_router
 from ..api.routes.health import router as health_router
@@ -102,8 +102,8 @@ def init_app(settings: ApplicationSettings | None = None) -> FastAPI:
     app.add_event_handler("shutdown", create_stop_app_handler(app))
 
     app.add_exception_handler(HTTPException, http_error_handler)
+    app.add_exception_handler(HttpxException, handle_httpx_client_exceptions)
     app.add_exception_handler(RequestValidationError, http422_error_handler)
-    app.add_exception_handler(HTTPStatusError, httpx_client_error_handler)
     app.add_exception_handler(LogDistributionBaseException, log_handling_error_handler)
     app.add_exception_handler(CustomBaseError, custom_error_handler)
 

services/api-server/src/simcore_service_api_server/services/_service_exception_handling.py

@@ -0,0 +1,75 @@
+import logging
+from contextlib import contextmanager
+from functools import wraps
+from typing import Any, Callable, Mapping
+
+import httpx
+from fastapi import HTTPException, status
+from pydantic import ValidationError
+from servicelib.error_codes import create_error_code
+
+_logger = logging.getLogger(__name__)
+
+
+def service_exception_mapper(
+    service_name: str,
+    http_status_map: Mapping[int, tuple[int, Callable[[Any], str] | None]],
+):
+    def decorator(func):
+        @wraps(func)
+        async def wrapper(*args, **kwargs):
+            with backend_service_exception_handler(
+                service_name, http_status_map, **kwargs
+            ):
+                return await func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+@contextmanager
+def backend_service_exception_handler(
+    service_name: str,
+    http_status_map: Mapping[int, tuple[int, Callable[[dict], str] | None]],
+    **endpoint_kwargs,
+):
+    try:
+        yield
+    except ValidationError as exc:
+        error_code = create_error_code(exc)
+        _logger.exception(
+            "Invalid data exchanged with %s service [%s] ",
+            service_name,
+            error_code,
+            extra={"error_code": error_code},
+        )
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"{service_name} service returned invalid response. {error_code}",
+        ) from exc
+
+    except httpx.HTTPStatusError as exc:
+        if status_detail_tuple := http_status_map.get(exc.response.status_code):
+            status_code, detail_callback = status_detail_tuple
+            if detail_callback is None:
+                detail = f"{exc}"
+            else:
+                detail = detail_callback(endpoint_kwargs)
+            raise HTTPException(status_code=status_code, detail=detail) from exc
+        if exc.response.status_code in {
+            status.HTTP_503_SERVICE_UNAVAILABLE,
+            status.HTTP_429_TOO_MANY_REQUESTS,
+        }:
+            headers = {}
+            if "Retry-After" in exc.response.headers:
+                headers["Retry-After"] = exc.response.headers["Retry-After"]
+            raise HTTPException(
+                status_code=exc.response.status_code,
+                detail=f"The {service_name} service was unavailable",
+                headers=headers,
+            ) from exc
+        raise HTTPException(
+            status_code=status.HTTP_502_BAD_GATEWAY,
+            detail=f"Received unexpected response from {service_name}",
+        ) from exc