From 2c0be5197895af1dd440bddb997f7c01608f02ba Mon Sep 17 00:00:00 2001 From: Feda Curic Date: Tue, 14 Nov 2023 09:14:08 +0100 Subject: [PATCH] Remove dependency on ert-storage Code taken from: https://github.com/equinor/ert-storage/commit/7137f62d00c0bb87d10def7e3b39a2dcaa494ce3 --- pyproject.toml | 5 +- src/ert/__main__.py | 6 - src/ert/dark_storage/__init__.py | 11 +- src/ert/dark_storage/app.py | 50 ++++-- src/ert/dark_storage/client/__init__.py | 5 + src/ert/dark_storage/client/_session.py | 63 ++++++++ src/ert/dark_storage/client/async_client.py | 22 +++ src/ert/dark_storage/client/client.py | 22 +++ src/ert/dark_storage/compute/__init__.py | 0 src/ert/dark_storage/compute/misfits.py | 38 +++++ .../dark_storage/endpoints/compute/misfits.py | 4 +- src/ert/dark_storage/endpoints/ensembles.py | 2 +- src/ert/dark_storage/endpoints/experiments.py | 2 +- .../dark_storage/endpoints/observations.py | 2 +- src/ert/dark_storage/endpoints/records.py | 2 +- src/ert/dark_storage/endpoints/updates.py | 2 +- src/ert/dark_storage/enkf.py | 2 +- src/ert/dark_storage/exceptions.py | 30 ++++ src/ert/dark_storage/json_schema/__init__.py | 11 ++ src/ert/dark_storage/json_schema/ensemble.py | 37 +++++ .../dark_storage/json_schema/experiment.py | 24 +++ .../dark_storage/json_schema/observation.py | 43 +++++ src/ert/dark_storage/json_schema/prior.py | 152 ++++++++++++++++++ src/ert/dark_storage/json_schema/record.py | 18 +++ src/ert/dark_storage/json_schema/update.py | 26 +++ src/ert/dark_storage/security.py | 25 +++ src/ert/namespace.py | 1 - src/ert/services/_storage_main.py | 18 +-- src/ert/services/storage_service.py | 5 +- src/ert/shared/storage/command.py | 17 +- tests/performance_tests/performance_utils.py | 1 - tests/unit_tests/dark_storage/conftest.py | 9 -- .../dark_storage/test_api_compatibility.py | 18 --- .../services/test_storage_service.py | 78 +-------- tests/unit_tests/shared/test_main_entry.py | 44 ----- tests/unit_tests/storage/conftest.py | 11 -- 36 files changed, 575 insertions(+), 231 deletions(-) create mode 100644 src/ert/dark_storage/client/__init__.py create mode 100644 src/ert/dark_storage/client/_session.py create mode 100644 src/ert/dark_storage/client/async_client.py create mode 100644 src/ert/dark_storage/client/client.py create mode 100644 src/ert/dark_storage/compute/__init__.py create mode 100644 src/ert/dark_storage/compute/misfits.py create mode 100644 src/ert/dark_storage/exceptions.py create mode 100644 src/ert/dark_storage/json_schema/__init__.py create mode 100644 src/ert/dark_storage/json_schema/ensemble.py create mode 100644 src/ert/dark_storage/json_schema/experiment.py create mode 100644 src/ert/dark_storage/json_schema/observation.py create mode 100644 src/ert/dark_storage/json_schema/prior.py create mode 100644 src/ert/dark_storage/json_schema/record.py create mode 100644 src/ert/dark_storage/json_schema/update.py create mode 100644 src/ert/dark_storage/security.py delete mode 100644 tests/unit_tests/dark_storage/test_api_compatibility.py diff --git a/pyproject.toml b/pyproject.toml index 58346b221d8..b784100f692 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,6 @@ dependencies=[ "deprecation", "dnspython >= 2", "resdata", - "ert-storage >= 0.3.16", "fastapi < 0.100.0", "filelock", "iterative_ensemble_smoother>=0.1.1", @@ -77,6 +76,8 @@ dependencies=[ "sortedcontainers", "tables<3.9;python_version == '3.8'", "tables; python_version >= '3.9'", + "python-multipart", + "pyarrow", ] [project.scripts] @@ -142,7 +143,6 @@ markers = [ "integration_test", "quick_only", "requires_eclipse", - "requires_ert_storage", "requires_window_manager", "script", "slow", @@ -181,6 +181,7 @@ ignore = ["PLW2901", # redefined-loop-name "F401", # unused-import, "PLW0603" # global-statement ] +"src/ert/dark_storage/json_schema/__init__.py" = ["F401"] [tool.ruff.pylint] max-args = 20 diff --git a/src/ert/__main__.py b/src/ert/__main__.py index 5ed448419ce..07360278869 100755 --- a/src/ert/__main__.py +++ b/src/ert/__main__.py @@ -45,9 +45,6 @@ def run_ert_storage(args: Namespace, _: Optional[ErtPluginManager] = None) -> None: kwargs = {"ert_config": args.config, "verbose": True} - if args.database_url is not None: - kwargs["database_url"] = args.database_url - with StorageService.start_server(**kwargs) as server: server.wait() @@ -71,9 +68,6 @@ def run_webviz_ert(args: Namespace, _: Optional[ErtPluginManager] = None) -> Non kwargs["ert_config"] = os.path.basename(args.config) kwargs["project"] = os.path.abspath(ens_path) - if args.database_url is not None: - kwargs["database_url"] = args.database_url - with StorageService.init_service(**kwargs) as storage: storage.wait_until_ready() print( diff --git a/src/ert/dark_storage/__init__.py b/src/ert/dark_storage/__init__.py index dacf2d6d5ae..4c850745a3e 100644 --- a/src/ert/dark_storage/__init__.py +++ b/src/ert/dark_storage/__init__.py @@ -1,11 +1,4 @@ """ -Dark Storage is an implementation of a subset of ERT Storage's API -(https://github.com/Equinor/ert-storage), where the data is provided by the -legacy EnKFMain and 'storage/' directory. - -The purpose of this is to provide users with an API that they can use without -requiring a dedicated PostgreSQL server, and that works with their existing -data. It should be noted that it's unfeasible to implement the entire ERT -Storage API, nor is it possible to have the same guarantees of data integrity -that ERT Storage has. +Dark Storage is an API towards data provided by the legacy EnKFMain object and +the `storage/` directory. """ diff --git a/src/ert/dark_storage/app.py b/src/ert/dark_storage/app.py index b108235dffe..d9b6620cb19 100644 --- a/src/ert/dark_storage/app.py +++ b/src/ert/dark_storage/app.py @@ -1,21 +1,49 @@ -from ert_storage.app import JSONResponse -from ert_storage.app import app as ert_storage_app -from ert_storage.exceptions import ErtStorageError +import json +from enum import Enum +from typing import Any + from fastapi import FastAPI, Request, status from fastapi.openapi.docs import get_redoc_html, get_swagger_ui_html -from fastapi.responses import HTMLResponse, RedirectResponse +from fastapi.responses import HTMLResponse, RedirectResponse, Response from ert.dark_storage.endpoints import router as endpoints_router +from ert.dark_storage.exceptions import ErtStorageError + + +class JSONEncoder(json.JSONEncoder): + """ + Custom JSON encoder with support for Python 3.4 enums + """ + + def default(self, obj: Any) -> Any: + if isinstance(obj, Enum): + return obj.name + return super().default(obj) + + +class JSONResponse(Response): + """A replacement for Starlette's JSONResponse that permits NaNs.""" + + media_type = "application/json" + + def render(self, content: Any) -> bytes: + return ( + JSONEncoder( + ensure_ascii=False, + allow_nan=True, + indent=None, + separators=(",", ":"), + ) + .encode(content) + .encode("utf-8") + ) + app = FastAPI( - title=ert_storage_app.title, - version=ert_storage_app.version, + title="Dark Storage API", + version="0.1.0", debug=True, default_response_class=JSONResponse, - # Disable documentation so we can replace it with ERT Storage's later - openapi_url=None, - docs_url=None, - redoc_url=None, ) @@ -52,7 +80,7 @@ async def not_implemented_handler( @app.get("/openapi.json", include_in_schema=False) async def get_openapi() -> JSONResponse: - return JSONResponse(ert_storage_app.openapi()) + return JSONResponse(app.openapi()) @app.get("/docs", include_in_schema=False) diff --git a/src/ert/dark_storage/client/__init__.py b/src/ert/dark_storage/client/__init__.py new file mode 100644 index 00000000000..839ae06609b --- /dev/null +++ b/src/ert/dark_storage/client/__init__.py @@ -0,0 +1,5 @@ +from ._session import ConnInfo +from .async_client import AsyncClient +from .client import Client + +__all__ = ["AsyncClient", "Client", "ConnInfo"] diff --git a/src/ert/dark_storage/client/_session.py b/src/ert/dark_storage/client/_session.py new file mode 100644 index 00000000000..b10178f559e --- /dev/null +++ b/src/ert/dark_storage/client/_session.py @@ -0,0 +1,63 @@ +import json +import os +from pathlib import Path +from typing import Optional + +from pydantic import BaseModel, ValidationError + + +class ConnInfo(BaseModel): + base_url: str + auth_token: Optional[str] = None + + +ENV_VAR = "ERT_STORAGE_CONNECTION_STRING" + +# Avoid searching for the connection information on every request. We assume +# that a single client process will only ever want to connect to a single ERT +# Storage server during its lifetime, so we don't provide an API for managing +# this cache. +_CACHED_CONN_INFO: Optional[ConnInfo] = None + + +def find_conn_info() -> ConnInfo: + """ + The base url and auth token are read from either: + The file `storage_server.json`, starting from the current working directory + or the environment variable `ERT_STORAGE_CONNECTION_STRING` + + In both cases the configuration is represented by JSON representation of the + `ConnInfo` pydantic model. + + In the event that nothing is found, a RuntimeError is raised. + """ + global _CACHED_CONN_INFO # noqa: PLW0603 + if _CACHED_CONN_INFO is not None: + return _CACHED_CONN_INFO + + conn_str = os.environ.get(ENV_VAR) + + # This could be an empty string rather than None, as by the shell + # invocation: env ERT_STORAGE_CONNECTION_STRING= python + if not conn_str: + # Look for `storage_server.json` from cwd up to root. + root = Path("/") + path = Path.cwd() + while path != root: + try: + conn_str = (path / "storage_server.json").read_text() + break + except FileNotFoundError: + path = path.parent + + if not conn_str: + raise RuntimeError("No Storage connection configuration found") + + try: + conn_info = ConnInfo.parse_obj(json.loads(conn_str)) + _CACHED_CONN_INFO = conn_info + return conn_info + except json.JSONDecodeError as e: + raise RuntimeError("Invalid storage conneciton configuration") from e + except ValidationError as e: + raise RuntimeError("Invalid storage conneciton configuration") from e diff --git a/src/ert/dark_storage/client/async_client.py b/src/ert/dark_storage/client/async_client.py new file mode 100644 index 00000000000..86cda55aa6a --- /dev/null +++ b/src/ert/dark_storage/client/async_client.py @@ -0,0 +1,22 @@ +from typing import Optional + +import httpx + +from ._session import ConnInfo, find_conn_info + + +class AsyncClient(httpx.AsyncClient): + """ + Wrapper class for httpx.AsyncClient that provides a user-friendly way to + interact with ERT Storage's API + """ + + def __init__(self, conn_info: Optional[ConnInfo] = None) -> None: + if conn_info is None: + conn_info = find_conn_info() + + headers = {} + if conn_info.auth_token is not None: + headers = {"Token": conn_info.auth_token} + + super().__init__(base_url=conn_info.base_url, headers=headers) diff --git a/src/ert/dark_storage/client/client.py b/src/ert/dark_storage/client/client.py new file mode 100644 index 00000000000..ad295878356 --- /dev/null +++ b/src/ert/dark_storage/client/client.py @@ -0,0 +1,22 @@ +from typing import Optional + +import httpx + +from ._session import ConnInfo, find_conn_info + + +class Client(httpx.Client): + """ + Wrapper class for httpx.Client that provides a user-friendly way to + interact with ERT Storage's API + """ + + def __init__(self, conn_info: Optional[ConnInfo] = None) -> None: + if conn_info is None: + conn_info = find_conn_info() + + headers = {} + if conn_info.auth_token is not None: + headers = {"Token": conn_info.auth_token} + + super().__init__(base_url=conn_info.base_url, headers=headers) diff --git a/src/ert/dark_storage/compute/__init__.py b/src/ert/dark_storage/compute/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/ert/dark_storage/compute/misfits.py b/src/ert/dark_storage/compute/misfits.py new file mode 100644 index 00000000000..6fd424e0f8a --- /dev/null +++ b/src/ert/dark_storage/compute/misfits.py @@ -0,0 +1,38 @@ +from typing import List, Mapping + +import numpy as np +import numpy.typing as npt +import pandas as pd + + +def _calculate_misfit( + obs_value: npt.NDArray[np.float_], + response_value: npt.NDArray[np.float_], + obs_std: npt.NDArray[np.float_], +) -> List[float]: + difference = response_value - obs_value + misfit = (difference / obs_std) ** 2 + return (misfit * np.sign(difference)).tolist() + + +def calculate_misfits_from_pandas( + reponses_dict: Mapping[int, pd.DataFrame], + observation: pd.DataFrame, + summary_misfits: bool = False, +) -> pd.DataFrame: + """ + Compute misfits from reponses_dict (real_id, values in dataframe) + and observation + """ + misfits_dict = {} + for realization_index in reponses_dict: + misfits_dict[realization_index] = _calculate_misfit( + observation["values"], + reponses_dict[realization_index].loc[:, observation.index].values.flatten(), + observation["errors"], + ) + + df = pd.DataFrame(data=misfits_dict, index=observation.index) + if summary_misfits: + df = pd.DataFrame([df.abs().sum(axis=0)], columns=df.columns, index=[0]) + return df.T diff --git a/src/ert/dark_storage/endpoints/compute/misfits.py b/src/ert/dark_storage/endpoints/compute/misfits.py index a9bae8386af..fa85b0b554e 100644 --- a/src/ert/dark_storage/endpoints/compute/misfits.py +++ b/src/ert/dark_storage/endpoints/compute/misfits.py @@ -4,12 +4,12 @@ import pandas as pd from dateutil.parser import parse -from ert_storage import exceptions as exc -from ert_storage.compute import calculate_misfits_from_pandas from fastapi import APIRouter, Depends, status from fastapi.responses import Response +from ert.dark_storage import exceptions as exc from ert.dark_storage.common import data_for_key, observations_for_obs_keys +from ert.dark_storage.compute.misfits import calculate_misfits_from_pandas from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.storage import StorageReader diff --git a/src/ert/dark_storage/endpoints/ensembles.py b/src/ert/dark_storage/endpoints/ensembles.py index 857ebb37442..8a05f0bdbce 100644 --- a/src/ert/dark_storage/endpoints/ensembles.py +++ b/src/ert/dark_storage/endpoints/ensembles.py @@ -1,9 +1,9 @@ from typing import Any, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.common import ensemble_parameter_names, get_response_names from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.storage import StorageAccessor diff --git a/src/ert/dark_storage/endpoints/experiments.py b/src/ert/dark_storage/endpoints/experiments.py index a23e72ba705..96f68711b7f 100644 --- a/src/ert/dark_storage/endpoints/experiments.py +++ b/src/ert/dark_storage/endpoints/experiments.py @@ -1,9 +1,9 @@ from typing import Any, List, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res, get_storage from ert.shared.storage.extraction import create_priors from ert.storage import StorageReader diff --git a/src/ert/dark_storage/endpoints/observations.py b/src/ert/dark_storage/endpoints/observations.py index 9aaf0452b2b..830049fe5f0 100644 --- a/src/ert/dark_storage/endpoints/observations.py +++ b/src/ert/dark_storage/endpoints/observations.py @@ -1,9 +1,9 @@ from typing import Any, List, Mapping from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res from ert.shared.storage.extraction import create_observations diff --git a/src/ert/dark_storage/endpoints/records.py b/src/ert/dark_storage/endpoints/records.py index bff872a3d18..29827f51887 100644 --- a/src/ert/dark_storage/endpoints/records.py +++ b/src/ert/dark_storage/endpoints/records.py @@ -4,10 +4,10 @@ from uuid import UUID, uuid4 import pandas as pd -from ert_storage import json_schema as js from fastapi import APIRouter, Body, Depends, File, Header, Request, UploadFile, status from fastapi.responses import Response +from ert.dark_storage import json_schema as js from ert.dark_storage.common import ( data_for_key, ensemble_parameters, diff --git a/src/ert/dark_storage/endpoints/updates.py b/src/ert/dark_storage/endpoints/updates.py index 174e410f545..c212b600b86 100644 --- a/src/ert/dark_storage/endpoints/updates.py +++ b/src/ert/dark_storage/endpoints/updates.py @@ -1,8 +1,8 @@ from uuid import UUID -from ert_storage import json_schema as js from fastapi import APIRouter, Depends +from ert.dark_storage import json_schema as js from ert.dark_storage.enkf import LibresFacade, get_res, reset_res router = APIRouter(tags=["ensemble"]) diff --git a/src/ert/dark_storage/enkf.py b/src/ert/dark_storage/enkf.py index a99b701b1d8..92dac671ef3 100644 --- a/src/ert/dark_storage/enkf.py +++ b/src/ert/dark_storage/enkf.py @@ -3,10 +3,10 @@ import os from typing import Optional -from ert_storage.security import security from fastapi import Depends from ert.config import ErtConfig +from ert.dark_storage.security import security from ert.enkf_main import EnKFMain from ert.libres_facade import LibresFacade from ert.storage import StorageReader, open_storage diff --git a/src/ert/dark_storage/exceptions.py b/src/ert/dark_storage/exceptions.py new file mode 100644 index 00000000000..dff48ae43e9 --- /dev/null +++ b/src/ert/dark_storage/exceptions.py @@ -0,0 +1,30 @@ +from typing import Any + +from fastapi import status + + +class ErtStorageError(RuntimeError): + """ + Base error class for all the rest of errors + """ + + __status_code__ = status.HTTP_200_OK + + def __init__(self, message: str, **kwargs: Any): + super().__init__(message, kwargs) + + +class NotFoundError(ErtStorageError): + __status_code__ = status.HTTP_404_NOT_FOUND + + +class ConflictError(ErtStorageError): + __status_code__ = status.HTTP_409_CONFLICT + + +class ExpectationError(ErtStorageError): + __status_code__ = status.HTTP_417_EXPECTATION_FAILED + + +class UnprocessableError(ErtStorageError): + __status_code__ = status.HTTP_422_UNPROCESSABLE_ENTITY diff --git a/src/ert/dark_storage/json_schema/__init__.py b/src/ert/dark_storage/json_schema/__init__.py new file mode 100644 index 00000000000..8692ecf41f2 --- /dev/null +++ b/src/ert/dark_storage/json_schema/__init__.py @@ -0,0 +1,11 @@ +from .ensemble import EnsembleIn, EnsembleOut +from .experiment import ExperimentIn, ExperimentOut +from .observation import ( + ObservationIn, + ObservationOut, + ObservationTransformationIn, + ObservationTransformationOut, +) +from .prior import Prior +from .record import RecordOut +from .update import UpdateIn, UpdateOut diff --git a/src/ert/dark_storage/json_schema/ensemble.py b/src/ert/dark_storage/json_schema/ensemble.py new file mode 100644 index 00000000000..347b34c28db --- /dev/null +++ b/src/ert/dark_storage/json_schema/ensemble.py @@ -0,0 +1,37 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel, Field, root_validator + + +class _Ensemble(BaseModel): + size: int + parameter_names: List[str] + response_names: List[str] + active_realizations: List[int] = [] + + +class EnsembleIn(_Ensemble): + update_id: Optional[UUID] = None + userdata: Mapping[str, Any] = {} + + @root_validator + def _check_names_no_overlap(cls, values: Mapping[str, Any]) -> Mapping[str, Any]: + """ + Verify that `parameter_names` and `response_names` don't overlap. Ie, no + record can be both a parameter and a response. + """ + if not set(values["parameter_names"]).isdisjoint(set(values["response_names"])): + raise ValueError("parameters and responses cannot have a name in common") + return values + + +class EnsembleOut(_Ensemble): + id: UUID + children: List[UUID] = Field(alias="child_ensemble_ids") + parent: Optional[UUID] = Field(alias="parent_ensemble_id") + experiment_id: Optional[UUID] = None + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/experiment.py b/src/ert/dark_storage/json_schema/experiment.py new file mode 100644 index 00000000000..39be8a515d4 --- /dev/null +++ b/src/ert/dark_storage/json_schema/experiment.py @@ -0,0 +1,24 @@ +from typing import Any, Dict, List, Mapping +from uuid import UUID + +from pydantic import BaseModel + +from .prior import Prior + + +class _Experiment(BaseModel): + name: str + + +class ExperimentIn(_Experiment): + priors: Mapping[str, Prior] = {} + + +class ExperimentOut(_Experiment): + id: UUID + ensemble_ids: List[UUID] + priors: Mapping[str, Dict[str, Any]] + userdata: Mapping[str, Any] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/observation.py b/src/ert/dark_storage/json_schema/observation.py new file mode 100644 index 00000000000..699a56d2d33 --- /dev/null +++ b/src/ert/dark_storage/json_schema/observation.py @@ -0,0 +1,43 @@ +from typing import Any, List, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class _ObservationTransformation(BaseModel): + name: str + active: List[bool] + scale: List[float] + observation_id: UUID + + +class ObservationTransformationIn(_ObservationTransformation): + pass + + +class ObservationTransformationOut(_ObservationTransformation): + id: UUID + + class Config: + orm_mode = True + + +class _Observation(BaseModel): + name: str + errors: List[float] + values: List[float] + x_axis: List[Any] + records: Optional[List[UUID]] = None + + +class ObservationIn(_Observation): + pass + + +class ObservationOut(_Observation): + id: UUID + transformation: Optional[ObservationTransformationOut] = None + userdata: Mapping[str, Any] = {} + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/prior.py b/src/ert/dark_storage/json_schema/prior.py new file mode 100644 index 00000000000..b22bfc5b735 --- /dev/null +++ b/src/ert/dark_storage/json_schema/prior.py @@ -0,0 +1,152 @@ +import sys +from typing import Union + +from pydantic import BaseModel + +if sys.version_info < (3, 8): + from typing_extensions import Literal +else: + from typing import Literal + + +class PriorConst(BaseModel): + """ + Constant parameter prior + """ + + function: Literal["const"] = "const" + value: float + + +class PriorTrig(BaseModel): + """ + Triangular distribution parameter prior + """ + + function: Literal["trig"] = "trig" + min: float + max: float + mode: float + + +class PriorNormal(BaseModel): + """ + Normal distribution parameter prior + """ + + function: Literal["normal"] = "normal" + mean: float + std: float + + +class PriorLogNormal(BaseModel): + """ + Log-normal distribution parameter prior + """ + + function: Literal["lognormal"] = "lognormal" + mean: float + std: float + + +class PriorErtTruncNormal(BaseModel): + """ + ERT Truncated normal distribution parameter prior + + ERT differs from the usual distribution by that it simply clamps on `min` + and `max`, which gives a bias towards the extremes. + + """ + + function: Literal["ert_truncnormal"] = "ert_truncnormal" + mean: float + std: float + min: float + max: float + + +class PriorStdNormal(BaseModel): + """ + Standard normal distribution parameter prior + + Normal distribution with mean of 0 and standard deviation of 1 + """ + + function: Literal["stdnormal"] = "stdnormal" + + +class PriorUniform(BaseModel): + """ + Uniform distribution parameter prior + """ + + function: Literal["uniform"] = "uniform" + min: float + max: float + + +class PriorErtDUniform(BaseModel): + """ + ERT Discrete uniform distribution parameter prior + + This discrete uniform distribution differs from the standard by using the + `bins` parameter. Normally, `a`, and `b` are integers, and the sample space + are the integers between. ERT allows `a` and `b` to be arbitrary floats, + where the sample space is binned. + + """ + + function: Literal["ert_duniform"] = "ert_duniform" + bins: int + min: float + max: float + + +class PriorLogUniform(BaseModel): + """ + Logarithmic uniform distribution parameter prior + """ + + function: Literal["loguniform"] = "loguniform" + min: float + max: float + + +class PriorErtErf(BaseModel): + """ + ERT Error function distribution parameter prior + """ + + function: Literal["ert_erf"] = "ert_erf" + min: float + max: float + skewness: float + width: float + + +class PriorErtDErf(BaseModel): + """ + ERT Discrete error function distribution parameter prior + """ + + function: Literal["ert_derf"] = "ert_derf" + bins: int + min: float + max: float + skewness: float + width: float + + +Prior = Union[ + PriorConst, + PriorTrig, + PriorNormal, + PriorLogNormal, + PriorErtTruncNormal, + PriorStdNormal, + PriorUniform, + PriorErtDUniform, + PriorLogUniform, + PriorErtErf, + PriorErtDErf, +] diff --git a/src/ert/dark_storage/json_schema/record.py b/src/ert/dark_storage/json_schema/record.py new file mode 100644 index 00000000000..b0b536bf459 --- /dev/null +++ b/src/ert/dark_storage/json_schema/record.py @@ -0,0 +1,18 @@ +from typing import Any, Mapping, Optional +from uuid import UUID + +from pydantic import BaseModel + + +class _Record(BaseModel): + pass + + +class RecordOut(_Record): + id: UUID + name: str + userdata: Mapping[str, Any] + has_observations: Optional[bool] + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/json_schema/update.py b/src/ert/dark_storage/json_schema/update.py new file mode 100644 index 00000000000..4b534b5c2ec --- /dev/null +++ b/src/ert/dark_storage/json_schema/update.py @@ -0,0 +1,26 @@ +from typing import List, Optional, Union +from uuid import UUID + +from pydantic import BaseModel + +from .observation import ( + ObservationTransformationIn, +) + + +class _Update(BaseModel): + algorithm: str + ensemble_result_id: Union[UUID, None] + ensemble_reference_id: Union[UUID, None] + + +class UpdateIn(_Update): + observation_transformations: Optional[List[ObservationTransformationIn]] = None + + +class UpdateOut(_Update): + id: UUID + experiment_id: UUID + + class Config: + orm_mode = True diff --git a/src/ert/dark_storage/security.py b/src/ert/dark_storage/security.py new file mode 100644 index 00000000000..f410b8047d9 --- /dev/null +++ b/src/ert/dark_storage/security.py @@ -0,0 +1,25 @@ +import os +from typing import Optional + +from fastapi import HTTPException, Security, status +from fastapi.security import APIKeyHeader + +DEFAULT_TOKEN = "hunter2" +_security_header = APIKeyHeader(name="Token", auto_error=False) + + +async def security(*, token: Optional[str] = Security(_security_header)) -> None: + if os.getenv("ERT_STORAGE_NO_TOKEN"): + return + if not token: + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, detail="Not authenticated" + ) + real_token = os.getenv("ERT_STORAGE_TOKEN", DEFAULT_TOKEN) + if token == real_token: + # Success + return + + # HTTP 403 is when the user has authorized themselves, but aren't allowed to + # access this resource + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="Invalid token") diff --git a/src/ert/namespace.py b/src/ert/namespace.py index a72541967df..1e791ce10ae 100644 --- a/src/ert/namespace.py +++ b/src/ert/namespace.py @@ -13,7 +13,6 @@ class Namespace(argparse.Namespace): mode: str config: str - database_url: str verbose: bool experimental_mode: bool logdir: str diff --git a/src/ert/services/_storage_main.py b/src/ert/services/_storage_main.py index 925845d91a0..f6f183d62c3 100644 --- a/src/ert/services/_storage_main.py +++ b/src/ert/services/_storage_main.py @@ -121,20 +121,10 @@ def run_server(args: Optional[argparse.Namespace] = None, debug: bool = False) - # Appropriated from uvicorn.main:run os.environ["ERT_STORAGE_NO_TOKEN"] = "1" - if args.enable_new_storage: - args.database_url = "sqlite:///ert.db" - if args.database_url: - os.environ["ERT_STORAGE_DATABASE_URL"] = args.database_url - config = uvicorn.Config("ert_storage.app:app", **config_args) - else: - # Dark Storage imports from ERT Storage, which connects to the database - # at startup. We set the database URL to an SQLite in-memory database so - # that the import succeeds. - os.environ["ERT_STORAGE_DATABASE_URL"] = "sqlite://" - os.environ["ERT_STORAGE_RES_CONFIG"] = ( - os.path.abspath(args.config) or find_ert_config() - ) - config = uvicorn.Config("ert.dark_storage.app:app", **config_args) + os.environ["ERT_STORAGE_RES_CONFIG"] = ( + os.path.abspath(args.config) or find_ert_config() + ) + config = uvicorn.Config("ert.dark_storage.app:app", **config_args) server = Server(config, json.dumps(connection_info)) logger = logging.getLogger("ert.shared.storage.info") diff --git a/src/ert/services/storage_service.py b/src/ert/services/storage_service.py index 0b4d5402096..e3cc7b10a1a 100644 --- a/src/ert/services/storage_service.py +++ b/src/ert/services/storage_service.py @@ -6,8 +6,8 @@ import httpx import requests -from ert_storage.client import Client, ConnInfo +from ert.dark_storage.client import Client, ConnInfo from ert.services._base_service import BaseService, _Context, local_exec_args @@ -17,7 +17,6 @@ class StorageService(BaseService): def __init__( self, ert_config: Optional[PathLike[str]] = None, - database_url: str = "sqlite:///ert.db", verbose: bool = False, *args: Any, **kwargs: Any, @@ -27,8 +26,6 @@ def __init__( exec_args = local_exec_args("storage") if ert_config: exec_args.append(str(ert_config)) - else: - exec_args.extend(("--database-url", database_url)) if verbose: exec_args.append("--verbose") if "project" in kwargs: diff --git a/src/ert/shared/storage/command.py b/src/ert/shared/storage/command.py index b930f4ef932..05bb63e9fbb 100644 --- a/src/ert/shared/storage/command.py +++ b/src/ert/shared/storage/command.py @@ -6,18 +6,9 @@ def add_parser_options(ap: ArgumentParser) -> None: ap.add_argument( "config", type=str, - help=( - "ERT config file to start the server from " - "(ignored if --database-url is set)" - ), + help=("ERT config file to start the server from "), nargs="?", # optional ) - ap.add_argument( - "--enable-new-storage", - action="store_true", - default=False, - help="Shorthand for --database-url=sqlite:///ert.db", - ) ap.add_argument( "--project", "-p", @@ -25,12 +16,6 @@ def add_parser_options(ap: ArgumentParser) -> None: help="Path to directory in which to create storage_server.json", default=os.getcwd(), ) - ap.add_argument( - "--database-url", - type=str, - default=None, - help="Database URL that the server should connect to. (eg. sqlite:///ert.db)", - ) ap.add_argument( "--host", type=str, default=os.environ.get("ERT_STORAGE_HOST", "127.0.0.1") ) diff --git a/tests/performance_tests/performance_utils.py b/tests/performance_tests/performance_utils.py index 0a0dbd84704..772a84c204a 100644 --- a/tests/performance_tests/performance_utils.py +++ b/tests/performance_tests/performance_utils.py @@ -137,7 +137,6 @@ def reset_enkf(): def dark_storage_app(monkeypatch): monkeypatch.setenv("ERT_STORAGE_NO_TOKEN", "yup") monkeypatch.setenv("ERT_STORAGE_RES_CONFIG", "poly.ert") - monkeypatch.setenv("ERT_STORAGE_DATABASE_URL", "sqlite://") from ert.dark_storage.app import app yield app diff --git a/tests/unit_tests/dark_storage/conftest.py b/tests/unit_tests/dark_storage/conftest.py index 3c1b7bb37e7..cce64baf8d2 100644 --- a/tests/unit_tests/dark_storage/conftest.py +++ b/tests/unit_tests/dark_storage/conftest.py @@ -60,17 +60,9 @@ def dark_storage_client(monkeypatch): @pytest.fixture def env(monkeypatch): - monkeypatch.setenv("ERT_STORAGE_DATABASE_URL", "sqlite:///:memory:") monkeypatch.setenv("ERT_STORAGE_NO_TOKEN", "yup") -@pytest.fixture -def ert_storage_app(env): - from ert_storage.app import app - - return app - - def reset_enkf(): enkf._config = None enkf._ert = None @@ -85,7 +77,6 @@ def reset_enkf(): def dark_storage_app_(monkeypatch): monkeypatch.setenv("ERT_STORAGE_NO_TOKEN", "yup") monkeypatch.setenv("ERT_STORAGE_RES_CONFIG", "poly.ert") - monkeypatch.setenv("ERT_STORAGE_DATABASE_URL", "sqlite://") from ert.dark_storage.app import app yield app diff --git a/tests/unit_tests/dark_storage/test_api_compatibility.py b/tests/unit_tests/dark_storage/test_api_compatibility.py deleted file mode 100644 index f3dc55c38e9..00000000000 --- a/tests/unit_tests/dark_storage/test_api_compatibility.py +++ /dev/null @@ -1,18 +0,0 @@ -def test_openapi(ert_storage_app, dark_storage_app): - """ - Test that the openapi.json of Dark Storage is identical to ERT Storage - """ - expect = ert_storage_app.openapi() - actual = dark_storage_app.openapi() - - # Remove textual data (descriptions and such) from ERT Storage's API. - def _remove_text(data): - if isinstance(data, dict): - return { - key: _remove_text(val) - for key, val in data.items() - if key not in ("description", "examples") - } - return data - - assert _remove_text(expect) == _remove_text(actual) diff --git a/tests/unit_tests/services/test_storage_service.py b/tests/unit_tests/services/test_storage_service.py index 145280634c8..767438faf19 100644 --- a/tests/unit_tests/services/test_storage_service.py +++ b/tests/unit_tests/services/test_storage_service.py @@ -1,52 +1,10 @@ import json import os -import pytest -import requests -from flaky import flaky - -from ert.services import StorageService, _storage_main +from ert.services import _storage_main from ert.shared import port_handler -@flaky -@pytest.mark.integration_test -@pytest.mark.requires_ert_storage -def test_integration(tmp_path, monkeypatch): - """Actually start the server, wait for it to be online and do a health check""" - monkeypatch.chdir(tmp_path) - - # Note: Sqlite needs at least 4-5 seconds to spin up even on - # an unloaded M1-based Mac using local disk. On the CI-server - # we have less control of available resources, so set timeout- - # value large to allow time for sqlite to get ready - with StorageService.start_server(timeout=120) as server: - resp = requests.get( - f"{server.fetch_url()}/healthcheck", auth=server.fetch_auth() - ) - assert "ALL OK!" in resp.json() - - with StorageService.session() as session: - session.get("/healthcheck") - - assert not (tmp_path / "storage_server.json").exists() - - -@pytest.mark.requires_ert_storage -def test_integration_timeout(tmp_path, monkeypatch): - """Try to start the server but give it too small time to get ready and - expect a timeout""" - monkeypatch.chdir(tmp_path) - - with pytest.raises(TimeoutError), StorageService.start_server( - timeout=0.01 - ) as server: - # Note timeout-value here in context of note above - requests.get(f"{server.fetch_url()}/healthcheck", auth=server.fetch_auth()) - - assert not (tmp_path / "storage_server.json").exists() - - def test_create_connection_string(monkeypatch): authtoken = "very_secret_token" _, _, sock = port_handler.find_available_port() @@ -60,37 +18,3 @@ def test_create_connection_string(monkeypatch): assert len(connection_string["urls"]) == 3 del os.environ["ERT_STORAGE_CONNECTION_STRING"] - - -@pytest.mark.requires_ert_storage -def test_init_service_no_server_start(tmpdir, mock_start_server, mock_connect): - with tmpdir.as_cwd(): - StorageService.init_service(project=str(tmpdir)) - mock_connect.assert_called_once_with(project=str(tmpdir), timeout=0) - mock_start_server.assert_not_called() - - -@pytest.mark.requires_ert_storage -def test_init_service_server_start_if_no_conf_file(tmpdir, mock_start_server): - with tmpdir.as_cwd(): - StorageService.init_service(project=str(tmpdir)) - mock_start_server.assert_called_once_with(project=str(tmpdir)) - - -@pytest.mark.requires_ert_storage -def test_init_service_server_start_conf_info_is_stale( - tmp_path, mock_start_server, monkeypatch -): - (tmp_path / f"{StorageService.service_name}_server.json").write_text( - json.dumps({"authtoken": "test123", "urls": ["http://127.0.0.1:51821"]}), - encoding="utf-8", - ) - - def raise_conerr(x, auth): - raise requests.ConnectionError() - - monkeypatch.setattr(requests, "get", raise_conerr) - - with tmp_path: - StorageService.init_service(project=tmp_path) - mock_start_server.assert_called_once_with(project=tmp_path) diff --git a/tests/unit_tests/shared/test_main_entry.py b/tests/unit_tests/shared/test_main_entry.py index b7c8d862773..e7df9afd520 100644 --- a/tests/unit_tests/shared/test_main_entry.py +++ b/tests/unit_tests/shared/test_main_entry.py @@ -34,47 +34,3 @@ def test_main_logging_argparse(monkeypatch, caplog): with caplog.at_level(logging.INFO): main.main() assert "mode='test_run'" in caplog.text - - -def test_api_database_default(monkeypatch): - monkeypatch.setattr(logging.config, "dictConfig", MagicMock()) - - monkeypatch.setattr(main, "start_ert_server", MagicMock()) - monkeypatch.setattr(main, "ErtPluginContext", MagicMock()) - mocked_start_server = MagicMock() - monkeypatch.setattr( - "ert.services.storage_service.BaseService.start_server", - mocked_start_server, - ) - monkeypatch.setattr( - sys, - "argv", - ["ert", "api"], - ) - - main.main() - # We expect default value from Storage class, validate that no explicit - # value is given for database_url - mocked_start_server.assert_called_once_with(ert_config=None, verbose=True) - - -def test_api_database_url_forwarded(monkeypatch): - monkeypatch.setattr(logging.config, "dictConfig", MagicMock()) - - monkeypatch.setattr(main, "start_ert_server", MagicMock()) - monkeypatch.setattr(main, "ErtPluginContext", MagicMock()) - mocked_start_server = MagicMock() - monkeypatch.setattr( - "ert.services.storage_service.BaseService.start_server", - mocked_start_server, - ) - monkeypatch.setattr( - sys, - "argv", - ["ert", "api", "--database-url", "TEST_DATABASE_URL"], - ) - - main.main() - mocked_start_server.assert_called_once_with( - ert_config=None, database_url="TEST_DATABASE_URL", verbose=True - ) diff --git a/tests/unit_tests/storage/conftest.py b/tests/unit_tests/storage/conftest.py index 4a6707f8e4b..ff8aead6623 100644 --- a/tests/unit_tests/storage/conftest.py +++ b/tests/unit_tests/storage/conftest.py @@ -1,17 +1,6 @@ import pytest -@pytest.fixture(autouse=True) -def _enable_new_storage(monkeypatch): - """ - All tests in this module assume --enable-new-storage is set - """ - from ert.shared.feature_toggling import FeatureToggling - - feature = FeatureToggling._conf["new-storage"] - monkeypatch.setattr(feature, "is_enabled", True) - - @pytest.fixture def client(monkeypatch, ert_storage_client): from ert.shared.storage import extraction