Remove dependency on ert-storage

Code taken from:
equinor/ert-storage@7137f62

pyproject.toml

@@ -47,7 +47,6 @@ dependencies=[
     "deprecation",
     "dnspython >= 2",
     "resdata",
-    "ert-storage >= 0.3.16",
     "fastapi < 0.100.0",
     "filelock",
     "iterative_ensemble_smoother>=0.1.1",
@@ -77,6 +76,8 @@ dependencies=[
     "sortedcontainers",
     "tables<3.9;python_version == '3.8'",
     "tables; python_version >= '3.9'",
+    "python-multipart",
+    "pyarrow",
 ]
 
 [project.scripts]
@@ -142,7 +143,6 @@ markers = [
     "integration_test",
     "quick_only",
     "requires_eclipse",
-    "requires_ert_storage",
     "requires_window_manager",
     "script",
     "slow",
@@ -181,6 +181,7 @@ ignore = ["PLW2901", # redefined-loop-name
 "F401", # unused-import, 
 "PLW0603" # global-statement
 ]
+"src/ert/dark_storage/json_schema/__init__.py" = ["F401"]
 
 [tool.ruff.pylint]
 max-args = 20

src/ert/__main__.py

@@ -45,9 +45,6 @@
 def run_ert_storage(args: Namespace, _: Optional[ErtPluginManager] = None) -> None:
     kwargs = {"ert_config": args.config, "verbose": True}
 
-    if args.database_url is not None:
-        kwargs["database_url"] = args.database_url
-
     with StorageService.start_server(**kwargs) as server:
         server.wait()
 
@@ -71,9 +68,6 @@ def run_webviz_ert(args: Namespace, _: Optional[ErtPluginManager] = None) -> Non
         kwargs["ert_config"] = os.path.basename(args.config)
         kwargs["project"] = os.path.abspath(ens_path)
 
-    if args.database_url is not None:
-        kwargs["database_url"] = args.database_url
-
     with StorageService.init_service(**kwargs) as storage:
         storage.wait_until_ready()
         print(

src/ert/dark_storage/__init__.py

@@ -1,11 +1,4 @@
 """
-Dark Storage is an implementation of a subset of ERT Storage's API
-(https://github.com/Equinor/ert-storage), where the data is provided by the
-legacy EnKFMain and 'storage/' directory.
-
-The purpose of this is to provide users with an API that they can use without
-requiring a dedicated PostgreSQL server, and that works with their existing
-data. It should be noted that it's unfeasible to implement the entire ERT
-Storage API, nor is it possible to have the same guarantees of data integrity
-that ERT Storage has.
+Dark Storage is an API towards data provided by the legacy EnKFMain object and
+the `storage/` directory.
 """

src/ert/dark_storage/app.py

@@ -1,21 +1,49 @@
-from ert_storage.app import JSONResponse
-from ert_storage.app import app as ert_storage_app
-from ert_storage.exceptions import ErtStorageError
+import json
+from enum import Enum
+from typing import Any
+
 from fastapi import FastAPI, Request, status
 from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html
-from fastapi.responses import HTMLResponse, RedirectResponse
+from fastapi.responses import HTMLResponse, RedirectResponse, Response
 
 from ert.dark_storage.endpoints import router as endpoints_router
+from ert.dark_storage.exceptions import ErtStorageError
+
+
+class JSONEncoder(json.JSONEncoder):
+    """
+    Custom JSON encoder with support for Python 3.4 enums
+    """
+
+    def default(self, obj: Any) -> Any:
+        if isinstance(obj, Enum):
+            return obj.name
+        return super().default(obj)
+
+
+class JSONResponse(Response):
+    """A replacement for Starlette's JSONResponse that permits NaNs."""
+
+    media_type = "application/json"
+
+    def render(self, content: Any) -> bytes:
+        return (
+            JSONEncoder(
+                ensure_ascii=False,
+                allow_nan=True,
+                indent=None,
+                separators=(",", ":"),
+            )
+            .encode(content)
+            .encode("utf-8")
+        )
+
 
 app = FastAPI(
-    title=ert_storage_app.title,
-    version=ert_storage_app.version,
+    title="Dark Storage API",
+    version="0.1.0",
     debug=True,
     default_response_class=JSONResponse,
-    # Disable documentation so we can replace it with ERT Storage's later
-    openapi_url=None,
-    docs_url=None,
-    redoc_url=None,
 )
 
 
@@ -52,7 +80,7 @@ async def not_implemented_handler(
 
 @app.get("/openapi.json", include_in_schema=False)
 async def get_openapi() -> JSONResponse:
-    return JSONResponse(ert_storage_app.openapi())
+    return JSONResponse(app.openapi())
 
 
 @app.get("/docs", include_in_schema=False)

src/ert/dark_storage/client/__init__.py

@@ -0,0 +1,5 @@
+from ._session import ConnInfo
+from .async_client import AsyncClient
+from .client import Client
+
+__all__ = ["AsyncClient", "Client", "ConnInfo"]

src/ert/dark_storage/client/_session.py

@@ -0,0 +1,63 @@
+import json
+import os
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseModel, ValidationError
+
+
+class ConnInfo(BaseModel):
+    base_url: str
+    auth_token: Optional[str] = None
+
+
+ENV_VAR = "ERT_STORAGE_CONNECTION_STRING"
+
+# Avoid searching for the connection information on every request. We assume
+# that a single client process will only ever want to connect to a single ERT
+# Storage server during its lifetime, so we don't provide an API for managing
+# this cache.
+_CACHED_CONN_INFO: Optional[ConnInfo] = None
+
+
+def find_conn_info() -> ConnInfo:
+    """
+    The base url and auth token are read from either:
+    The file `storage_server.json`, starting from the current working directory
+    or the environment variable `ERT_STORAGE_CONNECTION_STRING`
+
+    In both cases the configuration is represented by JSON representation of the
+    `ConnInfo` pydantic model.
+
+    In the event that nothing is found, a RuntimeError is raised.
+    """
+    global _CACHED_CONN_INFO  # noqa: PLW0603
+    if _CACHED_CONN_INFO is not None:
+        return _CACHED_CONN_INFO
+
+    conn_str = os.environ.get(ENV_VAR)
+
+    # This could be an empty string rather than None, as by the shell
+    # invocation: env ERT_STORAGE_CONNECTION_STRING= python
+    if not conn_str:
+        # Look for `storage_server.json` from cwd up to root.
+        root = Path("/")
+        path = Path.cwd()
+        while path != root:
+            try:
+                conn_str = (path / "storage_server.json").read_text()
+                break
+            except FileNotFoundError:
+                path = path.parent
+
+    if not conn_str:
+        raise RuntimeError("No Storage connection configuration found")
+
+    try:
+        conn_info = ConnInfo.parse_obj(json.loads(conn_str))
+        _CACHED_CONN_INFO = conn_info
+        return conn_info
+    except json.JSONDecodeError as e:
+        raise RuntimeError("Invalid storage conneciton configuration") from e
+    except ValidationError as e:
+        raise RuntimeError("Invalid storage conneciton configuration") from e

src/ert/dark_storage/client/async_client.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+import httpx
+
+from ._session import ConnInfo, find_conn_info
+
+
+class AsyncClient(httpx.AsyncClient):
+    """
+    Wrapper class for httpx.AsyncClient that provides a user-friendly way to
+    interact with ERT Storage's API
+    """
+
+    def __init__(self, conn_info: Optional[ConnInfo] = None) -> None:
+        if conn_info is None:
+            conn_info = find_conn_info()
+
+        headers = {}
+        if conn_info.auth_token is not None:
+            headers = {"Token": conn_info.auth_token}
+
+        super().__init__(base_url=conn_info.base_url, headers=headers)

src/ert/dark_storage/client/client.py

@@ -0,0 +1,22 @@
+from typing import Optional
+
+import httpx
+
+from ._session import ConnInfo, find_conn_info
+
+
+class Client(httpx.Client):
+    """
+    Wrapper class for httpx.Client that provides a user-friendly way to
+    interact with ERT Storage's API
+    """
+
+    def __init__(self, conn_info: Optional[ConnInfo] = None) -> None:
+        if conn_info is None:
+            conn_info = find_conn_info()
+
+        headers = {}
+        if conn_info.auth_token is not None:
+            headers = {"Token": conn_info.auth_token}
+
+        super().__init__(base_url=conn_info.base_url, headers=headers)

src/ert/dark_storage/compute/misfits.py

@@ -0,0 +1,38 @@
+from typing import List, Mapping
+
+import numpy as np
+import numpy.typing as npt
+import pandas as pd
+
+
+def _calculate_misfit(
+    obs_value: npt.NDArray[np.float_],
+    response_value: npt.NDArray[np.float_],
+    obs_std: npt.NDArray[np.float_],
+) -> List[float]:
+    difference = response_value - obs_value
+    misfit = (difference / obs_std) ** 2
+    return (misfit * np.sign(difference)).tolist()
+
+
+def calculate_misfits_from_pandas(
+    reponses_dict: Mapping[int, pd.DataFrame],
+    observation: pd.DataFrame,
+    summary_misfits: bool = False,
+) -> pd.DataFrame:
+    """
+    Compute misfits from reponses_dict (real_id, values in dataframe)
+    and observation
+    """
+    misfits_dict = {}
+    for realization_index in reponses_dict:
+        misfits_dict[realization_index] = _calculate_misfit(
+            observation["values"],
+            reponses_dict[realization_index].loc[:, observation.index].values.flatten(),
+            observation["errors"],
+        )
+
+    df = pd.DataFrame(data=misfits_dict, index=observation.index)
+    if summary_misfits:
+        df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0])
+    return df.T

src/ert/dark_storage/endpoints/compute/misfits.py

@@ -4,12 +4,12 @@
 
 import pandas as pd
 from dateutil.parser import parse
-from ert_storage import exceptions as exc
-from ert_storage.compute import calculate_misfits_from_pandas
 from fastapi import APIRouter, Depends, status
 from fastapi.responses import Response
 
+from ert.dark_storage import exceptions as exc
 from ert.dark_storage.common import data_for_key, observations_for_obs_keys
+from ert.dark_storage.compute.misfits import calculate_misfits_from_pandas
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.storage import StorageReader
 

src/ert/dark_storage/endpoints/ensembles.py

@@ -1,9 +1,9 @@
 from typing import Any, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import ensemble_parameter_names, get_response_names
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.storage import StorageAccessor

src/ert/dark_storage/endpoints/experiments.py

@@ -1,9 +1,9 @@
 from typing import Any, List, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res, get_storage
 from ert.shared.storage.extraction import create_priors
 from ert.storage import StorageReader

src/ert/dark_storage/endpoints/observations.py

@@ -1,9 +1,9 @@
 from typing import Any, List, Mapping
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res
 from ert.shared.storage.extraction import create_observations
 

src/ert/dark_storage/endpoints/records.py

@@ -4,10 +4,10 @@
 from uuid import UUID, uuid4
 
 import pandas as pd
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status
 from fastapi.responses import Response
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.common import (
     data_for_key,
     ensemble_parameters,

src/ert/dark_storage/endpoints/updates.py

@@ -1,8 +1,8 @@
 from uuid import UUID
 
-from ert_storage import json_schema as js
 from fastapi import APIRouter, Depends
 
+from ert.dark_storage import json_schema as js
 from ert.dark_storage.enkf import LibresFacade, get_res, reset_res
 
 router = APIRouter(tags=["ensemble"])

src/ert/dark_storage/enkf.py

@@ -3,10 +3,10 @@
 import os
 from typing import Optional
 
-from ert_storage.security import security
 from fastapi import Depends
 
 from ert.config import ErtConfig
+from ert.dark_storage.security import security
 from ert.enkf_main import EnKFMain
 from ert.libres_facade import LibresFacade
 from ert.storage import StorageReader, open_storage