Commitin

pyproject.toml

@@ -47,7 +47,6 @@ dependencies=[
     "deprecation",
     "dnspython >= 2",
     "ecl >= 2.14.1",
-    "ert-storage >= 0.3.16",
     "fastapi < 0.100.0",
     "filelock",
     "iterative_ensemble_smoother>=0.1.1",
@@ -142,7 +141,6 @@ markers = [
     "integration_test",
     "quick_only",
     "requires_eclipse",
-    "requires_ert_storage",
     "requires_window_manager",
     "script",
     "slow",

src/ert/dark_storage/app.py

@@ -1,15 +1,57 @@
-from ert_storage.app import JSONResponse
-from ert_storage.app import app as ert_storage_app
-from ert_storage.exceptions import ErtStorageError
+import json
+from enum import Enum
+from typing import Any
+
 from fastapi import FastAPI, Request, status
 from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
 
 from ert.dark_storage.endpoints import router as endpoints_router
 
+
+class ErtStorageError(RuntimeError):
+    """
+    Base error class for all the rest of errors
+    """
+
+    __status_code__ = status.HTTP_200_OK
+
+    def __init__(self, message: str, **kwargs: Any):
+        super().__init__(message, kwargs)
+
+
+class JSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder with support for Python 3.4 enums
+    """
+
+    def default(self, obj: Any) -> Any:
+        if isinstance(obj, Enum):
+            return obj.name
+        return super().default(obj)
+
+
+class JSONResponse(Response):
+    """A replacement for Starlette's JSONResponse that permits NaNs."""
+
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return (
+            JSONEncoder(
+                ensure_ascii=False,
+                allow_nan=True,
+                indent=None,
+                separators=(",", ":"),
+            )
+            .encode(content)
+            .encode("utf-8")
+        )
+
+
 app = FastAPI(
-    title=ert_storage_app.title,
-    version=ert_storage_app.version,
+    title="ERT Storage API",
+    version="0.1.2",
     debug=True,
     default_response_class=JSONResponse,
     # Disable documentation so we can replace it with ERT Storage's later
@@ -52,7 +94,7 @@ async def not_implemented_handler(
 
 @app.get("/openapi.json", include_in_schema=False)
 async def get_openapi() -> JSONResponse:
-    return JSONResponse(ert_storage_app.openapi())
+    return JSONResponse(app.openapi())
 
 
 @app.get("/docs", include_in_schema=False)

src/ert/dark_storage/client/__init__.py

@@ -0,0 +1,5 @@
+from ._session import ConnInfo
+from .async_client import AsyncClient
+from .client import Client
+
+__all__ = ["AsyncClient", "Client", "ConnInfo"]

src/ert/dark_storage/client/_session.py

@@ -0,0 +1,63 @@
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel, ValidationError
+
+
+class ConnInfo(BaseModel):
+    base_url: str
+    auth_token: Optional[str] = None
+
+
+ENV_VAR = "ERT_STORAGE_CONNECTION_STRING"
+
+# Avoid searching for the connection information on every request. We assume
+# that a single client process will only ever want to connect to a single ERT
+# Storage server during its lifetime, so we don't provide an API for managing
+# this cache.
+_CACHED_CONN_INFO: Optional[ConnInfo] = None
+
+
+def find_conn_info() -> ConnInfo:
+    """
+    The base url and auth token are read from either:
+    The file `storage_server.json`, starting from the current working directory
+    or the environment variable `ERT_STORAGE_CONNECTION_STRING`
+
+    In both cases the configuration is represented by JSON representation of the
+    `ConnInfo` pydantic model.
+
+    In the event that nothing is found, a RuntimeError is raised.
+    """
+    global _CACHED_CONN_INFO
+    if _CACHED_CONN_INFO is not None:
+        return _CACHED_CONN_INFO
+
+    conn_str = os.environ.get(ENV_VAR)
+
+    # This could be an empty string rather than None, as by the shell
+    # invocation: env ERT_STORAGE_CONNECTION_STRING= python
+    if not conn_str:
+        # Look for `storage_server.json` from cwd up to root.
+        root = Path("/")
+        path = Path.cwd()
+        while path != root:
+            try:
+                conn_str = (path / "storage_server.json").read_text()
+                break
+            except FileNotFoundError:
+                path = path.parent
+
+    if not conn_str:
+        raise RuntimeError("No Storage connection configuration found")
+
+    try:
+        conn_info = ConnInfo.parse_obj(json.loads(conn_str))
+        _CACHED_CONN_INFO = conn_info
+        return conn_info
+    except json.JSONDecodeError:
+        raise RuntimeError("Invalid storage conneciton configuration")
+    except ValidationError:
+        raise RuntimeError("Invalid storage conneciton configuration")

src/ert/dark_storage/client/async_client.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+import httpx
+
+from ._session import ConnInfo, find_conn_info
+
+
+class AsyncClient(httpx.AsyncClient):
+    """
+    Wrapper class for httpx.AsyncClient that provides a user-friendly way to
+    interact with ERT Storage's API
+    """
+
+    def __init__(self, conn_info: Optional[ConnInfo] = None) -> None:
+        if conn_info is None:
+            conn_info = find_conn_info()
+
+        headers = {}
+        if conn_info.auth_token is not None:
+            headers = {"Token": conn_info.auth_token}
+
+        super().__init__(base_url=conn_info.base_url, headers=headers)

src/ert/dark_storage/client/client.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+import httpx
+
+from ._session import ConnInfo, find_conn_info
+
+
+class Client(httpx.Client):
+    """
+    Wrapper class for httpx.Client that provides a user-friendly way to
+    interact with ERT Storage's API
+    """
+
+    def __init__(self, conn_info: Optional[ConnInfo] = None) -> None:
+        if conn_info is None:
+            conn_info = find_conn_info()
+
+        headers = {}
+        if conn_info.auth_token is not None:
+            headers = {"Token": conn_info.auth_token}
+
+        super().__init__(base_url=conn_info.base_url, headers=headers)

src/ert/dark_storage/compute/misfits.py

@@ -0,0 +1,35 @@
+from typing import List, Mapping
+
+import numpy as np
+import pandas as pd
+
+
+def _calculate_misfit(
+    obs_value: np.ndarray, response_value: np.ndarray, obs_std: np.ndarray
+) -> List[float]:
+    difference = response_value - obs_value
+    misfit = (difference / obs_std) ** 2
+    return (misfit * np.sign(difference)).tolist()
+
+
+def calculate_misfits_from_pandas(
+    reponses_dict: Mapping[int, pd.DataFrame],
+    observation: pd.DataFrame,
+    summary_misfits: bool = False,
+) -> pd.DataFrame:
+    """
+    Compute misfits from reponses_dict (real_id, values in dataframe)
+    and observation
+    """
+    misfits_dict = {}
+    for realization_index in reponses_dict:
+        misfits_dict[realization_index] = _calculate_misfit(
+            observation["values"],
+            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            observation["errors"],
+        )
+
+    df = pd.DataFrame(data=misfits_dict, index=observation.index)
+    if summary_misfits:
+        df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0])
+    return df.T

src/ert/dark_storage/endpoints/compute/misfits.py

@@ -4,12 +4,12 @@
 
 import pandas as pd
 from dateutil.parser import parse
-from ert_storage import exceptions as exc
-from ert_storage.compute import calculate_misfits_from_pandas
 from fastapi import APIRouter, Depends, status
 from fastapi.responses import Response
 
+from ert.dark_storage import exceptions as exc
 from ert.dark_storage.common import data_for_key, observations_for_obs_keys
+from ert.dark_storage.compute import calculate_misfits_from_pandas
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.storage import StorageReader
 

src/ert/dark_storage/endpoints/ensembles.py

@@ -1,9 +1,9 @@
 from typing import Any, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import ensemble_parameter_names, get_response_names
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.storage import StorageAccessor

src/ert/dark_storage/endpoints/experiments.py

@@ -1,9 +1,9 @@
 from typing import Any, List, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.shared.storage.extraction import create_priors
 from ert.storage import StorageReader

src/ert/dark_storage/endpoints/observations.py

@@ -1,9 +1,9 @@
 from typing import Any, List, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res
 from ert.shared.storage.extraction import create_observations
 

src/ert/dark_storage/endpoints/records.py

@@ -4,10 +4,10 @@
 from uuid import UUID, uuid4
 
 import pandas as pd
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status
 from fastapi.responses import Response
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import (
     data_for_key,
     ensemble_parameters,

src/ert/dark_storage/endpoints/updates.py

@@ -1,8 +1,8 @@
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res, reset_res
 
 router = APIRouter(tags=["ensemble"])

src/ert/dark_storage/enkf.py

@@ -3,10 +3,10 @@
 import os
 from typing import Optional
 
-from ert_storage.security import security
 from fastapi import Depends
 
 from ert.config import ErtConfig
+from ert.dark_storage.security import security
 from ert.enkf_main import EnKFMain
 from ert.libres_facade import LibresFacade
 from ert.storage import StorageReader, open_storage

src/ert/dark_storage/exceptions.py

@@ -0,0 +1,30 @@
+from typing import Any
+
+from fastapi import status
+
+
+class ErtStorageError(RuntimeError):
+    """
+    Base error class for all the rest of errors
+    """
+
+    __status_code__ = status.HTTP_200_OK
+
+    def __init__(self, message: str, **kwargs: Any):
+        super().__init__(message, kwargs)
+
+
+class NotFoundError(ErtStorageError):
+    __status_code__ = status.HTTP_404_NOT_FOUND
+
+
+class ConflictError(ErtStorageError):
+    __status_code__ = status.HTTP_409_CONFLICT
+
+
+class ExpectationError(ErtStorageError):
+    __status_code__ = status.HTTP_417_EXPECTATION_FAILED
+
+
+class UnprocessableError(ErtStorageError):
+    __status_code__ = status.HTTP_422_UNPROCESSABLE_ENTITY

src/ert/dark_storage/json_schema/ensemble.py

@@ -0,0 +1,37 @@
+from typing import Any, List, Mapping, Optional
+from uuid import UUID
+
+from pydantic import BaseModel, Field, model_validator
+
+
+class _Ensemble(BaseModel):
+    size: int
+    parameter_names: List[str]
+    response_names: List[str]
+    active_realizations: List[int] = []
+
+
+class EnsembleIn(_Ensemble):
+    update_id: Optional[UUID] = None
+    userdata: Mapping[str, Any] = {}
+
+    @model_validator
+    def _check_names_no_overlap(cls, values: Mapping[str, Any]) -> Mapping[str, Any]:
+        """
+        Verify that `parameter_names` and `response_names` don't overlap. Ie, no
+        record can be both a parameter and a response.
+        """
+        if not set(values["parameter_names"]).isdisjoint(set(values["response_names"])):
+            raise ValueError("parameters and responses cannot have a name in common")
+        return values
+
+
+class EnsembleOut(_Ensemble):
+    id: UUID
+    children: List[UUID] = Field(alias="child_ensemble_ids")
+    parent: Optional[UUID] = Field(alias="parent_ensemble_id")
+    experiment_id: Optional[UUID] = None
+    userdata: Mapping[str, Any]
+
+    class Config:
+        orm_mode = True