Merge pull request #28 from bento-platform/refact/deps-service-info-c…

…onfig-override

refact: reconfigure app/config setup + update dependencies

Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.11.10
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.12.01
 
 # Use uvicorn (instead of hypercorn) in production since I've found
 # multiple benchmarks showing it to be faster - David L
@@ -13,7 +13,8 @@ COPY poetry.lock .
 # Install production dependencies
 # Without --no-root, we get errors related to the code not being copied in yet.
 # But we don't want the code here, otherwise Docker cache doesn't work well.
-RUN poetry config virtualenvs.create false && poetry install --without dev --no-root
+RUN poetry config virtualenvs.create false && \
+    poetry install --without dev --no-root
 
 # Manually copy only what's relevant
 # (Don't use .dockerignore, which allows us to have development containers too)

bento_service_registry/app.py

@@ -3,34 +3,41 @@
 from fastapi.exceptions import RequestValidationError
 from fastapi.middleware.cors import CORSMiddleware
 from starlette.exceptions import HTTPException as StarletteHTTPException
+from typing import Callable
 
 from .authz import authz_middleware
-from .config import get_config
+from .config import Config, get_config
 from .logger import get_logger
 from .routes import service_registry
 
 __all__ = [
-    "application",
+    "create_app",
 ]
 
 
-application = FastAPI()
-application.include_router(service_registry)
+def create_app(config_override: Callable[[], Config] | None = None) -> FastAPI:
+    app = FastAPI()
 
-# TODO: Find a way to DI this
-config_for_setup = get_config()
+    config_for_setup: Config = (config_override or get_config)()
+    if config_override:
+        # noinspection PyUnresolvedReferences
+        app.dependency_overrides[get_config] = config_override
 
-application.add_middleware(
-    CORSMiddleware,
-    allow_origins=config_for_setup.cors_origins,
-    allow_headers=["Authorization"],
-    allow_credentials=True,
-    allow_methods=["*"],
-)
+    app.include_router(service_registry)
 
-# Non-standard middleware setup so that we can import the instance and use it for dependencies too
-authz_middleware.attach(application)
+    app.add_middleware(
+        CORSMiddleware,
+        allow_origins=config_for_setup.cors_origins,
+        allow_headers=["Authorization"],
+        allow_credentials=True,
+        allow_methods=["*"],
+    )
 
-application.exception_handler(StarletteHTTPException)(
-    http_exception_handler_factory(get_logger(config_for_setup), authz_middleware))
-application.exception_handler(RequestValidationError)(validation_exception_handler_factory(authz_middleware))
+    # Non-standard middleware setup so that we can import the instance and use it for dependencies too
+    authz_middleware.attach(app)
+
+    app.exception_handler(StarletteHTTPException)(
+        http_exception_handler_factory(get_logger(config_for_setup), authz_middleware))
+    app.exception_handler(RequestValidationError)(validation_exception_handler_factory(authz_middleware))
+
+    return app

bento_service_registry/bento_services_json.py

@@ -1,5 +1,5 @@
 import aiofiles
-import json
+import orjson
 from fastapi import Depends
 
 from typing import Annotated
@@ -23,8 +23,8 @@
 
 
 async def get_bento_services_by_compose_id(config: ConfigDependency) -> BentoServicesByComposeID:
-    async with aiofiles.open(config.bento_services, "r") as fh:
-        bento_services_data: BentoServicesByComposeID = json.loads(await fh.read())
+    async with aiofiles.open(config.bento_services, "rb") as fh:
+        bento_services_data: BentoServicesByComposeID = orjson.loads(await fh.read())
 
     return {
         sk: BentoService(

bento_service_registry/config.py

@@ -1,11 +1,8 @@
-import json
-
+from bento_lib.config.pydantic import BentoBaseConfig
 from fastapi import Depends
 from functools import lru_cache
 from pathlib import Path
-from pydantic.fields import FieldInfo
-from pydantic_settings import BaseSettings, EnvSettingsSource, PydanticBaseSettingsSource, SettingsConfigDict
-from typing import Annotated, Any, Literal
+from typing import Annotated
 
 from .constants import SERVICE_TYPE
 
@@ -16,20 +13,12 @@
 ]
 
 
-class CorsOriginsParsingSource(EnvSettingsSource):
-    def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, value_is_complex: bool) -> Any:
-        if field_name == "cors_origins":
-            return tuple(x.strip() for x in value.split(";")) if value is not None else ()
-        return json.loads(value) if value_is_complex else value
-
-
 DEFAULT_SERVICE_ID = ":".join(list(SERVICE_TYPE.values())[:2])
 
 
-class Config(BaseSettings):
-    bento_debug: bool = False
-    bento_container_local: bool = False
-    bento_validate_ssl: bool = True
+class Config(BentoBaseConfig):
+    service_id: str = DEFAULT_SERVICE_ID
+    service_name: str = "Bento Service Registry"
 
     bento_services: Path
     contact_timeout: int = 5
@@ -38,29 +27,8 @@ class Config(BaseSettings):
     bento_public_url: str
     bento_portal_public_url: str
 
-    service_id: str = DEFAULT_SERVICE_ID
-
-    bento_authz_service_url: str  # Bento authorization service base URL
     authz_enabled: bool = True
 
-    cors_origins: tuple[str, ...] = ()
-
-    log_level: Literal["debug", "info", "warning", "error"] = "debug"
-
-    # Make Config instances hashable + immutable
-    model_config = SettingsConfigDict(frozen=True)
-
-    @classmethod
-    def settings_customise_sources(
-        cls,
-        settings_cls: type[BaseSettings],
-        init_settings: PydanticBaseSettingsSource,
-        env_settings: PydanticBaseSettingsSource,
-        dotenv_settings: PydanticBaseSettingsSource,
-        file_secret_settings: PydanticBaseSettingsSource,
-    ) -> tuple[PydanticBaseSettingsSource, ...]:
-        return (CorsOriginsParsingSource(settings_cls),)
-
 
 @lru_cache
 def get_config():

bento_service_registry/constants.py

@@ -1,21 +1,16 @@
-from bento_lib.service_info import GA4GHServiceType
-from bento_service_registry import __version__
+from bento_lib.service_info.types import GA4GHServiceType
 
 __all__ = [
     "BENTO_SERVICE_KIND",
-    "SERVICE_ARTIFACT",
     "SERVICE_TYPE",
-    "SERVICE_NAME",
 ]
 
 BENTO_SERVICE_KIND: str = "service-registry"
-SERVICE_ARTIFACT: str = "service-registry"
 
 # For exact implementations, this should be org.ga4gh/service-registry/1.0.0.
 # In our case, most of our services diverge or will at some point, so use ca.c3g.bento as the group.
 SERVICE_TYPE: GA4GHServiceType = {
     "group": "ca.c3g.bento",
-    "artifact": SERVICE_ARTIFACT,
-    "version": __version__,
+    "artifact": "service-registry",
+    "version": "1.0.0",
 }
-SERVICE_NAME: str = "Bento Service Registry"

bento_service_registry/service_info.py

@@ -1,13 +1,12 @@
-from bento_lib.service_info import SERVICE_ORGANIZATION_C3G, GA4GHServiceInfo, build_service_info
+from bento_lib.service_info.helpers import build_service_info_from_pydantic_config
+from bento_lib.service_info.types import GA4GHServiceInfo
 from bento_service_registry import __version__
 from fastapi import Depends
 from typing import Annotated
 
-from .bento_services_json import BentoServicesByKindDependency
 from .config import ConfigDependency
-from .constants import BENTO_SERVICE_KIND, SERVICE_NAME, SERVICE_TYPE
+from .constants import BENTO_SERVICE_KIND, SERVICE_TYPE
 from .logger import LoggerDependency
-from .utils import get_service_url
 
 
 __all__ = [
@@ -16,25 +15,14 @@
 ]
 
 
-async def get_service_info(
-    bento_services_by_kind: BentoServicesByKindDependency,
-    config: ConfigDependency,
-    logger: LoggerDependency,
-) -> GA4GHServiceInfo:
-    return await build_service_info({
-        "id": config.service_id,
-        "name": SERVICE_NAME,  # TODO: Should be globally unique?
-        "type": SERVICE_TYPE,
-        "description": "Service registry for a Bento platform node.",
-        "organization": SERVICE_ORGANIZATION_C3G,
-        "contactUrl": "mailto:[email protected]",
-        "version": __version__,
-        "url": get_service_url(bento_services_by_kind, BENTO_SERVICE_KIND),
-        "environment": "prod",
-        "bento": {
-            "serviceKind": BENTO_SERVICE_KIND,
-        },
-    }, debug=config.bento_debug, local=config.bento_container_local, logger=logger)
+async def get_service_info(config: ConfigDependency, logger: LoggerDependency) -> GA4GHServiceInfo:
+    return await build_service_info_from_pydantic_config(
+        config,
+        logger,
+        {"serviceKind": BENTO_SERVICE_KIND},
+        SERVICE_TYPE,
+        __version__,
+    )
 
 
 ServiceInfoDependency = Annotated[GA4GHServiceInfo, Depends(get_service_info)]

bento_service_registry/services.py

@@ -3,7 +3,7 @@
 import logging
 
 from aiohttp import ClientSession
-from bento_lib.service_info import GA4GHServiceInfo
+from bento_lib.service_info.types import GA4GHServiceInfo
 from datetime import datetime
 from fastapi import Depends, status
 from functools import lru_cache
@@ -41,19 +41,19 @@ async def get_service(
         service_metadata: BentoService,
     ) -> dict | None:
         kind = service_metadata["service_kind"]
+        s_url: str = service_metadata["url"]
 
         # special case: requesting info about the current service. Skip networking / self-connect;
         # instead, return pre-calculated /service-info contents.
         if kind == BENTO_SERVICE_KIND:
-            return service_info
+            return {**service_info, "url": s_url}
 
-        s_url: str = service_metadata["url"]
         service_info_url: str = urljoin(f"{s_url}/", "service-info")
 
         dt = datetime.now()
         self._logger.info(f"Contacting {service_info_url}{' with bearer token' if authz_header else ''}")
 
-        service_resp: dict[str, dict] = {}
+        service_resp: dict | None = None
 
         try:
             async with http_session.get(service_info_url, headers=authz_header) as r:
@@ -69,23 +69,23 @@ async def get_service(
                     return None
 
                 try:
-                    service_resp[kind] = {**(await r.json()), "url": s_url}
+                    service_resp = {**(await r.json()), "url": s_url}
+                    self._logger.info(f"{service_info_url}: Took {(datetime.now() - dt).total_seconds():.1f}s")
                 except (JSONDecodeError, aiohttp.ContentTypeError, TypeError) as e:
                     # JSONDecodeError can happen if the JSON is invalid
                     # ContentTypeError can happen if the Content-Type is not application/json
                     # TypeError can happen if None is received
                     self._logger.error(
-                        f"Encountered invalid response ({str(e)}) from {service_info_url}: {await r.text()}")
-
-                self._logger.info(f"{service_info_url}: Took {(datetime.now() - dt).total_seconds():.1f}s")
+                        f"{service_info_url}: Encountered invalid response ({str(e)}) - {await r.text()} "
+                        f"(Took {(datetime.now() - dt).total_seconds():.1f}s)")
 
         except asyncio.TimeoutError:
             self._logger.error(f"Encountered timeout with {service_info_url}")
 
         except aiohttp.ClientConnectionError as e:
             self._logger.error(f"Encountered connection error with {service_info_url}: {str(e)}")
 
-        return service_resp.get(kind)
+        return service_resp
 
     async def get_services(
         self,

dev.Dockerfile

@@ -1,4 +1,4 @@
-FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.11.10
+FROM ghcr.io/bento-platform/bento_base_image:python-debian-2023.12.01
 
 RUN pip install --no-cache-dir "uvicorn[standard]==0.24.0"
 
@@ -11,7 +11,8 @@ COPY poetry.lock .
 # Install production + development dependencies
 # Without --no-root, we get errors related to the code not being copied in yet.
 # But we don't want the code here, otherwise Docker cache doesn't work well.
-RUN poetry config virtualenvs.create false && poetry install --no-root
+RUN poetry config virtualenvs.create false && \
+    poetry install --no-root
 
 # Copy entrypoint and runner script in, so we have something to start with - even though it'll get
 # overwritten by volume mount.