From 551e028a69bffe31e27cf44aa4b5b2fb494ef3c4 Mon Sep 17 00:00:00 2001 From: "Jan C. Rivenaes" Date: Thu, 8 Feb 2024 18:22:43 +0100 Subject: [PATCH 1/4] ENH, CLN: Apply ERT env variables in FMU runs This is a bigger PR where we go from parsing file paths in order to detect "running in FMU", to look for defined environment variables from ERT runs. In doing so, some refactorisering is done, in particular for the FmuProvider class to have a cleaner interface with the calling metadata.py module. --- src/fmu/dataio/_definitions.py | 41 ++ src/fmu/dataio/_filedata_provider.py | 26 +- src/fmu/dataio/_fmu_provider.py | 546 ++++++++++++++++----------- src/fmu/dataio/_metadata.py | 70 ++-- src/fmu/dataio/_utils.py | 4 +- src/fmu/dataio/dataio.py | 53 ++- tests/conftest.py | 125 ++++-- 7 files changed, 536 insertions(+), 329 deletions(-) diff --git a/src/fmu/dataio/_definitions.py b/src/fmu/dataio/_definitions.py index dd6f58447..d2bbf8589 100644 --- a/src/fmu/dataio/_definitions.py +++ b/src/fmu/dataio/_definitions.py @@ -2,6 +2,7 @@ from __future__ import annotations from dataclasses import dataclass, field +from enum import Enum, unique from typing import Final SCHEMA: Final = ( @@ -11,6 +12,10 @@ SOURCE: Final = "fmu" +class ValidationError(ValueError, KeyError): + """Raise error while validating.""" + + @dataclass class _ValidFormats: surface: dict = field( @@ -123,3 +128,39 @@ class _ValidFormats: "case_symlink_realization": "To case/share, with symlinks on realizations level", "preprocessed": "To share/preprocessed; from interactive runs but re-used later", } + + +@unique +class FmuContext(Enum): + """Use a Enum class for fmu_context entries.""" + + REALIZATION = "To realization-N/iter_M/share" + CASE = "To casename/share, but will also work on project disk" + CASE_SYMLINK_REALIZATION = "To case/share, with symlinks on realizations level" + PREPROCESSED = "To share/preprocessed; from interactive runs but re-used later" + NON_FMU = "Not ran in a FMU setting, e.g. interactive RMS" + + @classmethod + def has_key(cls, key: str) -> bool: + return key.upper() in cls._member_names_ + + @classmethod + def list_valid(cls) -> dict: + return {member.name: member.value for member in cls} + + @classmethod + def get(cls, key: FmuContext | str) -> FmuContext: + """Get the enum member with a case-insensitive key.""" + if isinstance(key, cls): + key_upper = key.name + elif isinstance(key, str): + key_upper = key.upper() + else: + raise ValidationError("The input must be a str or FmuContext instance") + + if not cls.has_key(key_upper): + raise ValidationError( + f"Invalid key <{key_upper}>. Valid keys: {cls.list_valid().keys()}" + ) + + return cls[key_upper] diff --git a/src/fmu/dataio/_filedata_provider.py b/src/fmu/dataio/_filedata_provider.py index a2aae4f6b..b43b0d12f 100644 --- a/src/fmu/dataio/_filedata_provider.py +++ b/src/fmu/dataio/_filedata_provider.py @@ -8,9 +8,10 @@ from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path -from typing import Any, Final, Optional +from typing import Any, Final, Literal, Optional from warnings import warn +from ._definitions import FmuContext from ._logging import null_logger logger: Final = null_logger(__name__) @@ -60,8 +61,6 @@ def __post_init__(self) -> None: self.forcefolder_is_absolute = False self.subfolder = self.dataio.subfolder - self.fmu_context = self.dataio._usecontext # may be None! - logger.info("Initialize %s", self.__class__) def derive_filedata(self) -> None: @@ -161,17 +160,24 @@ def _get_path(self) -> tuple[Path, Path | None]: """Construct and get the folder path(s).""" linkdest = None - dest = self._get_path_generic(mode=self.fmu_context, allow_forcefolder=True) + dest = self._get_path_generic( + mode=self.dataio.fmu_context, allow_forcefolder=True + ) - if self.fmu_context == "case_symlink_realization": + if self.dataio.fmu_context == FmuContext.CASE_SYMLINK_REALIZATION: linkdest = self._get_path_generic( - mode="realization", allow_forcefolder=False, info=self.fmu_context + mode=FmuContext.REALIZATION, + allow_forcefolder=False, + info=self.dataio.fmu_context.name, ) return dest, linkdest def _get_path_generic( - self, mode: str = "realization", allow_forcefolder: bool = True, info: str = "" + self, + mode: Literal[FmuContext.REALIZATION, FmuContext.PREPROCESSED], + allow_forcefolder: bool = True, + info: str = "", ) -> Path: """Generically construct and get the folder path and verify.""" dest = None @@ -179,7 +185,7 @@ def _get_path_generic( outroot = deepcopy(self.rootpath) logger.info("FMU context is %s", mode) - if mode == "realization": + if mode == FmuContext.REALIZATION: if self.realname: outroot = outroot / self.realname # TODO: if missing self.realname? @@ -188,14 +194,14 @@ def _get_path_generic( outroot = outroot / "share" - if mode == "preprocessed": + if mode == FmuContext.PREPROCESSED: outroot = outroot / "preprocessed" if self.dataio.forcefolder and self.dataio.forcefolder.startswith("/"): raise ValueError( "Cannot use absolute path to 'forcefolder' with preprocessed data" ) - if mode != "preprocessed": + if mode != FmuContext.PREPROCESSED: if self.dataio.is_observation: outroot = outroot / "observations" else: diff --git a/src/fmu/dataio/_fmu_provider.py b/src/fmu/dataio/_fmu_provider.py index 8d18d1303..627b246b1 100644 --- a/src/fmu/dataio/_fmu_provider.py +++ b/src/fmu/dataio/_fmu_provider.py @@ -1,292 +1,384 @@ -"""Module for DataIO _FmuProvider +"""Module for DataIO FmuProvider -The FmuProvider will from the run environment and eventually API's detect valid +The FmuProvider will from the environment and eventually API's detect valid fields to the FMU data block. -Note that FMU may potentially have different providers, e.g. ERT2 vs ERT3 -or it can detect that no providers are present (e.g. just ran from RMS interactive) +Note that FMU may potentially have different providers, e.g. ERT versions +or it can detect that no FMU providers are present (e.g. just ran from RMS interactive) + +Note that establishing the FMU case metadata for a run, is currently *not* done +here; this is done by code in the InitializeCase class. + +From ERT v. 5 (?), the following env variables are provided in startup (example): + +_ERT_EXPERIMENT_ID: 6a8e1e0f-9315-46bb-9648-8de87151f4c7 +_ERT_ENSEMBLE_ID: b027f225-c45d-477d-8f33-73695217ba14 +_ERT_SIMULATION_MODE: test_run + +and during a Forward model: + +_ERT_EXPERIMENT_ID: 6a8e1e0f-9315-46bb-9648-8de87151f4c7 +_ERT_ENSEMBLE_ID: b027f225-c45d-477d-8f33-73695217ba14 +_ERT_SIMULATION_MODE: test_run +_ERT_ITERATION_NUMBER: 0 +_ERT_REALIZATION_NUMBER: 0 +_ERT_RUNPATH: /scratch/fmu/jriv/01_drogon_ahm/realization-0/iter-0/ + """ from __future__ import annotations import json -import pathlib -import re +import os from copy import deepcopy from dataclasses import dataclass, field +from enum import Enum, auto from os import environ from pathlib import Path -from typing import Any, Final, Optional +from typing import Final, Optional from warnings import warn from fmu.config import utilities as ut from . import _utils +from ._definitions import FmuContext from ._logging import null_logger -# case metadata relative to rootpath -ERT2_RELATIVE_CASE_METADATA_FILE: Final = "share/metadata/fmu_case.yml" +# case metadata relative to casepath +ERT_RELATIVE_CASE_METADATA_FILE: Final = "share/metadata/fmu_case.yml" RESTART_PATH_ENVNAME: Final = "RESTART_FROM_PATH" logger: Final = null_logger(__name__) -def _get_folderlist(current: Path) -> list: - """Return a list of pure folder names incl. current casepath up to system root. +class FmuEnv(Enum): + EXPERIMENT_ID = auto() + ENSEMBLE_ID = auto() + SIMULATION_MODE = auto() + REALIZATION_NUMBER = auto() + ITERATION_NUMBER = auto() + RUNPATH = auto() - For example: current is /scratch/xfield/nn/case/realization-33/iter-1 - shall return ['', 'scratch', 'xfield', 'nn', 'case', 'realization-33', 'iter-1'] - """ - flist = [current.name] - for par in current.parents: - flist.append(par.name) + @property + def value(self) -> str | None: + # Fetch the environment variable; name of the enum member prefixed with _ERT_ + return os.getenv(f"_ERT_{self.name}") - flist.reverse() - return flist + @property + def keyname(self) -> str: + # Fetch the environment variable; name of the enum member prefixed with _ERT_ + return f"_ERT_{self.name}" @dataclass -class _FmuProvider: - """Class for detecting the run environment (e.g. an ERT2) and provide metadata.""" - - dataio: Any - - provider: Optional[str] = field(default=None, init=False) - is_fmurun: Optional[bool] = field(default=False, init=False) - iter_name: Optional[str] = field(default=None, init=False) - iter_id: Optional[int] = field(default=None, init=False) - iter_path: Optional[Path] = field(default=None, init=False) - real_name: Optional[str] = field(default=None, init=False) - real_id: int = field(default=0, init=False) - real_path: Optional[Path] = field(default=None, init=False) - case_name: Optional[str] = field(default=None, init=False) - user_name: Optional[str] = field(default=None, init=False) - ert2: dict = field(default_factory=dict, init=False) - case_metafile: Optional[Path] = field(default=None, init=False) - case_metadata: dict = field(default_factory=dict, init=False) - metadata: dict = field(default_factory=dict, init=False) - rootpath: Optional[Path] = field(default=None, init=False) +class FmuProvider: + """Class for detecting the run environment (e.g. an ERT) and provide metadata. + + Args: + model: Name of the model (usually from global config) + rootpath: .... + fmu_context: The FMU context this is ran in; see FmuContext enum class + casepath_proposed: Proposed casepath ... needed? + include_ertjobs: True if we want to include .... + forced_realization: If we want to force the realization (use case?) + workflow: Descriptive work flow info + """ + + model: str = "" + fmu_context: FmuContext = FmuContext.REALIZATION + include_ertjobs: bool = True + casepath_proposed: str | Path = "" + forced_realization: Optional[int] = None + workflow: str | dict = "" + + # private properties for this class + _stage: str = field(default="unset", init=False) + _runpath: Path | str = field(default="", init=False) + _casepath: Path | str = field(default="", init=False) # actual casepath + _provider: str = field(default="", init=False) + _iter_name: str = field(default="", init=False) + _iter_id: int = field(default=0, init=False) + _iter_path: Path | str = field(default="", init=False) + _real_name: str = field(default="", init=False) + _real_id: int = field(default=0, init=False) + _real_path: Path | str = field(default="", init=False) + _case_name: str = field(default="", init=False) + _user_name: str = field(default="", init=False) + _ert_info: dict = field(default_factory=dict, init=False) + _case_metadata: dict = field(default_factory=dict, init=False) + _metadata: dict = field(default_factory=dict, init=False) def __post_init__(self) -> None: - self.rootpath = Path(self.dataio._rootpath.absolute()) + logger.info("Initialize %s...", self.__class__) + logger.debug("Case path is initially <%s>...", self.casepath_proposed) + logger.debug("FMU context is <%s>...", self.fmu_context) + + if not FmuEnv.ENSEMBLE_ID.value: + return # not an FMU run + + self._provider = "ERT" + + self._detect_fmurun_stage() + self._detect_absolute_runpath() + self._detect_and_update_casepath() + self._parse_folder_info() + self._read_case_metadata() + + # the next ones will not be read if case metadata is empty, or stage is FMU CASE + self._read_optional_restart_data() + self._read_ert_information() + self._generate_ert_metadata() + + def get_iter_name(self) -> str: + """The client (metadata) will ask for iter_name""" + """Return the iter_name, e.g. 'iter-3' or 'pred'.""" + return self._iter_name + + def get_real_name(self) -> str: + """Return the real_name, e.g. 'realization-23'.""" + return self._real_name + + def get_casepath(self) -> str: + """Return updated casepath in a FMU run, will be updated if initially blank.""" + return "" if not self._casepath else str(self._casepath) + + def get_provider(self) -> str | None: + """Return the name of the FMU provider (so far 'ERT' only), or None.""" + return None if not self._provider else self._provider + + def get_metadata(self) -> dict: + """The client (metadata) will ask for complete metadata for FMU section""" + return {} if not self._metadata else self._metadata + + # private methods: + @staticmethod + def _get_folderlist_from_path(current: Path | str) -> list: + """Return a list of pure folder names incl. current casepath up to system root. + + For example: current is /scratch/xfield/nn/case/realization-33/iter-1 + shall return ['scratch', 'xfield', 'nn', 'case', 'realization-33', 'iter-1'] + """ + return [folder for folder in str(current).split("/") if folder] - self.rootpath_initial = self.rootpath + @staticmethod + def _get_folderlist_from_runpath_env() -> list: + """Return a list of pure folder names incl. current from RUNPATH environment, - logger.info("Initialize %s", self.__class__) + Derived from _ERT_RUNPATH. + + For example: runpath is /scratch/xfield/nn/case/realization-33/iter-1/ + shall return ['scratch', 'xfield', 'nn', 'case', 'realization-33', 'iter-1'] + """ + runpath = FmuEnv.RUNPATH.value + if runpath: + return [folder for folder in runpath.split("/") if folder] + return [] - def detect_provider(self) -> None: - """First order method to detect provider, ans also check fmu_context.""" - if self._detect_ert2provider() or self._detect_ert2provider_case_only(): - self.provider = "ERT2" - self.get_ert2_information() - self.get_ert2_case_metadata() - self.generate_ert2_metadata() + def _detect_fmurun_stage(self) -> None: + """Detect if ERT is in a PRE-HOOK or in a FORWARD MODEL stage + + Update self._stage = "case" | "forward" | "unset" + """ + if FmuEnv.EXPERIMENT_ID.value and not FmuEnv.RUNPATH.value: + self._stage = "case" + elif FmuEnv.EXPERIMENT_ID.value and FmuEnv.RUNPATH.value: + self._stage = "realization" else: - logger.info("Detecting FMU provider as None") - self.provider = None # e.g. an interactive RMS run - self.dataio._usecontext = None # e.g. an interactive RMS run - if self.dataio.fmu_context == "preprocessed": - self.dataio._usecontext = self.dataio.fmu_context - if self.dataio.fmu_context != self.dataio._usecontext: - logger.warning( - "Requested fmu_context is <%s> but since this is detected as a non " - "FMU run, the actual context is set to <%s>", - self.dataio.fmu_context, - self.dataio._usecontext, - ) + self._stage = "unset" + logger.debug("Detecting FMU stage as %s", self._stage) - def _detect_ert2provider(self) -> bool: - """Detect if ERT2 is provider and set itername, casename, etc. + def _detect_absolute_runpath(self) -> None: + """In case _ERT_RUNPATH is relative, an absolute runpath is detected.""" + if FmuEnv.RUNPATH.value: + self._runpath = Path(FmuEnv.RUNPATH.value).resolve() - This is the pattern in a forward model, where realization etc exists. + def _detect_and_update_casepath(self) -> None: + """If casepath is not given, then try update _casepath (if in realization). + + There is also a validation here that casepath contains case metadata, and if not + then a second guess is attempted, looking at `parent` insted of `parent.parent` + is case of unconventional structure. """ - logger.info("Try to detect ERT2 provider") + logger.debug("Try detect casepath, RUNPATH is %s", self._runpath) + logger.debug("Proposed casepath is now <%s>", self.casepath_proposed) + + self._casepath = Path(self.casepath_proposed) if self.casepath_proposed else "" + if self._stage == "case" and self._casepath: + try_casepath = Path(self._casepath) + logger.debug("Try casepath (stage is case): %s", try_casepath) + + elif not self._casepath: + try_casepath = Path(self._runpath).parent.parent + logger.debug("Try casepath (first attempt): %s", try_casepath) + + if not (try_casepath / ERT_RELATIVE_CASE_METADATA_FILE).exists(): + logger.debug("Cannot find metadata file, try just one parent...") + try_casepath = Path(self._runpath).parent + logger.debug("Try casepath (second attempt): %s", try_casepath) + self._casepath = try_casepath + + if not (Path(self._casepath) / ERT_RELATIVE_CASE_METADATA_FILE).exists(): + logger.debug("No case metadata, issue a warning!") + warn( + "Case metadata does not exist; will not update initial casepath", + UserWarning, + ) + self._casepath = "" - folders = _get_folderlist(self.rootpath_initial) - logger.info("Folders to evaluate: %s", folders) + def _parse_folder_info(self) -> None: + """Retreive the folders (id's and paths).""" + logger.debug("Parse folder info...") - for num, folder in enumerate(folders): - if folder and re.match("^realization-.", folder): - self.is_fmurun = True - realfolder = folders[num] - iterfolder = folders[num + 1] - casefolder = folders[num - 1] - userfolder = folders[num - 2] + folders = self._get_folderlist_from_runpath_env() + if self.fmu_context == FmuContext.CASE and self._casepath: + folders = self._get_folderlist_from_path(self._casepath) # override + logger.debug("Folders to evaluate (case): %s", folders) + + self._iter_path = "" + self._real_path = "" + self._case_name = folders[-1] + self._user_name = folders[-2] + + logger.debug( + "case_name, user_name: %s %s", self._case_name, self._user_name + ) + logger.debug("Detecting FMU provider as ERT (case only)") + else: + logger.debug("Folders to evaluate (realization): %s", folders) - case_path = Path("/".join(folders[0:num])) + self._case_name = folders[-3] + self._user_name = folders[-4] - # override: - if self.dataio.casepath: - case_path = Path(self.dataio.casepath) + self._iter_name = folders[-1] + self._real_name = folders[-2] - self.case_name = casefolder - self.user_name = userfolder + self._iter_path = Path("/" + "/".join(folders)) + self._real_path = Path("/" + "/".join(folders[:-1])) - # store findings - self.iter_name = iterfolder # name of the folder + self._iter_id = int(str(FmuEnv.ITERATION_NUMBER.value)) + self._real_id = int(str(FmuEnv.REALIZATION_NUMBER.value)) - # also derive the realization_id (realization number) from the folder - self.real_id = int(realfolder.replace("realization-", "")) - self.real_name = realfolder # name of the realization folder + def _read_case_metadata(self) -> None: + """Check if metadatafile file for CASE exists, and if so parse metadata. - # override realization if input key 'realization' is >= 0; only in rare - # cases - if self.dataio.realization and self.dataio.realization >= 0: - self.real_id = self.dataio.realization - self.real_name = "realization-" + str(self.real_id) + If file does not exist, still give a proposed file path, but the + self.casepath_proposed_metadata will be {} (empty) and the physical file + will not be made. + """ + logger.debug("Read case metadata, if any...") + if not self._casepath: + logger.info("No case path detected, hence FMU metadata will be empty.") + return - # also derive iteration_id from the folder - if "iter-" in str(iterfolder): - self.iter_id = int(iterfolder.replace("iter-", "")) - elif isinstance(iterfolder, str): - # any custom name of the iteration, like "pred" - self.iter_id = None - else: - raise ValueError("Could not derive iteration ID") + case_metafile = Path(self._casepath) / ERT_RELATIVE_CASE_METADATA_FILE + if case_metafile.exists(): + logger.debug("Case metadata file exists in file %s", str(case_metafile)) + self._case_metadata = ut.yaml_load(case_metafile, loader="standard") + logger.debug("Case metadata are: %s", self._case_metadata) + else: + logger.debug("Case metadata file does not exists as %s", str(case_metafile)) + warn( + "Cannot read case metadata, hence stop retrieving FMU data!", + UserWarning, + ) + self._case_metadata = {} - self.iter_path = pathlib.Path(case_path / realfolder / iterfolder) - self.real_path = pathlib.Path(case_path / realfolder) - self.rootpath = case_path + def _read_optional_restart_data(self) -> None: + # Load restart_from information + logger.debug("Read optional restart data, if any, and requested...") + if not self._case_metadata: + return - logger.info("Initial rootpath: %s", self.rootpath_initial) - logger.info("Updated rootpath: %s", self.rootpath) + if not environ.get(RESTART_PATH_ENVNAME): + return - logger.info("Detecting FMU provider as ERT2") - return True + logger.debug("Detected a restart run from environment variable") + restart_path = Path(self._iter_path) / environ[RESTART_PATH_ENVNAME] + restart_iter = self._get_folderlist_from_path(restart_path)[-1] + restart_case_metafile = ( + restart_path / "../.." / ERT_RELATIVE_CASE_METADATA_FILE + ).resolve() + if restart_case_metafile.exists(): + restart_metadata = ut.yaml_load(restart_case_metafile, loader="standard") + self._ert_info["restart_from"] = _utils.uuid_from_string( + restart_metadata["fmu"]["case"]["uuid"] + restart_iter + ) + else: + print( + f"{RESTART_PATH_ENVNAME} environment variable is set to " + f"{environ[RESTART_PATH_ENVNAME]} which is invalid. Metadata " + "restart_from will remain empty." + ) + logger.warning( + f"{RESTART_PATH_ENVNAME} environment variable is set to " + f"{environ[RESTART_PATH_ENVNAME]} which is invalid. Metadata " + "restart_from will remain empty." + ) - return False + def _read_ert_information(self) -> None: + """Retrieve information from an ERT (ver 5 and later) run.""" + logger.debug("Read ERT information, if any") - def _detect_ert2provider_case_only(self) -> bool: - """Detect ERT2 as provider when fmu_context is case'ish and casepath is given. + if not self._case_metadata: + return - This is typically found in ERT prehook work flows where case is establed by - fmu.dataio.InitialiseCase() but no iter and realization folders exist. So - only case-metadata are revelant here. - """ - logger.info("Try to detect ERT2 provider (case context)") - - if ( - self.dataio.fmu_context - and "case" in self.dataio.fmu_context - and self.dataio.casepath - ): - self.rootpath = Path(self.dataio.casepath) - - folders = _get_folderlist(self.rootpath) - logger.info("Folders to evaluate (case): %s", folders) - - self.iter_path = None - self.real_path = None - self.case_name = folders[-1] - self.user_name = folders[-2] - - logger.info("Initial rootpath: %s", self.rootpath_initial) - logger.info("Updated rootpath: %s", self.rootpath) - logger.info("case_name, user_name: %s %s", self.case_name, self.user_name) - - logger.info("Detecting FMU provider as ERT2 (case only)") - return True - return False - - def get_ert2_information(self) -> None: - """Retrieve information from an ERT2 run.""" - if not self.iter_path: + logger.debug("Read ERT information") + if not self._iter_path: + logger.debug("Not _iter_path!") return # store parameters.txt - parameters_file = self.iter_path / "parameters.txt" + logger.debug("Read ERT information, if any (continues)") + parameters_file = Path(self._iter_path) / "parameters.txt" if parameters_file.is_file(): params = _utils.read_parameters_txt(parameters_file) # BUG(?): value can contain Nones, loop in fn. below # does contains check, will fail. nested_params = _utils.nested_parameters_dict(params) # type: ignore - self.ert2["params"] = nested_params + self._ert_info["params"] = nested_params logger.debug("parameters.txt parsed.") else: - self.ert2["params"] = None - logger.debug("parameters.txt was not found") - - # Load restart_from information - if RESTART_PATH_ENVNAME in environ: - logger.info("Detected a restart run from environment variable") - restart_path = self.iter_path / environ[RESTART_PATH_ENVNAME] - restart_iter = _get_folderlist(restart_path)[-1] - restart_case_metafile = ( - restart_path / "../.." / ERT2_RELATIVE_CASE_METADATA_FILE - ).resolve() - if restart_case_metafile.exists(): - restart_metadata = ut.yaml_load( - restart_case_metafile, loader="standard" - ) - self.ert2["restart_from"] = _utils.uuid_from_string( - restart_metadata["fmu"]["case"]["uuid"] + restart_iter - ) - else: - print( - f"{RESTART_PATH_ENVNAME} environment variable is set to " - "{environ[RESTART_PATH_ENVNAME]} which is invalid. Metadata " - "restart_from will remain empty." - ) - logger.warning( - f"{RESTART_PATH_ENVNAME} environment variable is set to " - "{environ[RESTART_PATH_ENVNAME]} which is invalid. Metadata " - "restart_from will remain empty." - ) + self._ert_info["params"] = {} + warn("The parameters.txt file was not found", UserWarning) # store jobs.json if required! - if self.dataio.include_ert2jobs: - jobs_file = self.iter_path / "jobs.json" + if self.include_ertjobs: + jobs_file = Path(self._iter_path) / "jobs.json" if jobs_file.is_file(): with open(jobs_file) as stream: - self.ert2["jobs"] = json.load(stream) + self._ert_info["jobs"] = json.load(stream) logger.debug("jobs.json parsed.") - logger.debug("jobs.json was not found") + else: + logger.debug("jobs.json was not found") else: - self.ert2["jobs"] = None - logger.info("Storing jobs.json is disabled") + self._ert_info["jobs"] = None + logger.debug("Storing jobs.json is disabled") logger.debug("ERT files has been parsed.") - def get_ert2_case_metadata(self) -> None: - """Check if metadatafile file for CASE exists, and if so parse metadata. - - If file does not exist, still give a proposed file path, but the - self.case_metadata will be {} (empty) and the physical file will not be made. - """ - - assert self.rootpath is not None - self.case_metafile = self.rootpath / ERT2_RELATIVE_CASE_METADATA_FILE - self.case_metafile = self.case_metafile.resolve() - if self.case_metafile.exists(): - logger.info("Case metadata file exists at %s", str(self.case_metafile)) - self.case_metadata = ut.yaml_load(self.case_metafile, loader="standard") - logger.info("Case metadata are now read!") - else: - logger.info( - "Case metadata file does not exists as %s", str(self.case_metafile) - ) - - def generate_ert2_metadata(self) -> None: - """Construct the metadata FMU block for an ERT2 forward job.""" - logger.info("Generate ERT2 metadata...") + def _generate_ert_metadata(self) -> None: + """Construct the metadata FMU block for an ERT forward job.""" + if not self._case_metadata: + return - if not self.case_metadata: - logger.info("Trigger UserWarning!") + logger.debug("Generate ERT metadata...") + if not self._case_metadata: + logger.debug("Trigger UserWarning!") warn( - f"The fmu provider: {self.provider} is found but no case metadata!", + f"The fmu provider: {self._provider} is found but no case metadata!", UserWarning, ) - meta = self.metadata # shortform + meta = self._metadata # shortform - meta["model"] = self.dataio.config.get("model", None) + meta["model"] = self.model - meta["context"] = {"stage": self.dataio._usecontext} + meta["context"] = {"stage": self.fmu_context.name.lower()} - if self.dataio.workflow: - if isinstance(self.dataio.workflow, str): - meta["workflow"] = {"reference": self.dataio.workflow} - elif isinstance(self.dataio.workflow, dict): - if "reference" not in self.dataio.workflow: + if self.workflow: + if isinstance(self.workflow, str): + meta["workflow"] = {"reference": self.workflow} + elif isinstance(self.workflow, dict): + if "reference" not in self.workflow: raise ValueError( "When workflow is given as a dict, the 'reference' " "key must be included and be a string" @@ -297,39 +389,41 @@ def generate_ert2_metadata(self) -> None: PendingDeprecationWarning, ) - meta["workflow"] = {"reference": self.dataio.workflow["reference"]} + meta["workflow"] = {"reference": self.workflow["reference"]} else: raise TypeError("'workflow' should be string.") case_uuid = "not_present" # TODO! not allow missing case metadata? - if self.case_metadata and "fmu" in self.case_metadata: - meta["case"] = deepcopy(self.case_metadata["fmu"]["case"]) + if self._case_metadata and "fmu" in self._case_metadata: + meta["case"] = deepcopy(self._case_metadata["fmu"]["case"]) case_uuid = meta["case"]["uuid"] - if "realization" in self.dataio._usecontext: - iter_uuid = _utils.uuid_from_string(case_uuid + str(self.iter_name)) + if self.fmu_context == FmuContext.REALIZATION: + iter_uuid = _utils.uuid_from_string(case_uuid + str(self._iter_name)) meta["iteration"] = { - "id": self.iter_id, + "id": self._iter_id, "uuid": iter_uuid, - "name": self.iter_name, + "name": self._iter_name, **( - {"restart_from": self.ert2["restart_from"]} - if "restart_from" in self.ert2 + {"restart_from": self._ert_info["restart_from"]} + if "restart_from" in self._ert_info else {} ), } real_uuid = _utils.uuid_from_string( - case_uuid + str(iter_uuid) + str(self.real_id) + case_uuid + str(iter_uuid) + str(self._real_id) ) - logger.info( - "Generate ERT2 metadata continues, and real ID %s", self.real_id + logger.debug( + "Generate ERT metadata continues, and real ID %s", self._real_id ) mreal = meta["realization"] = {} - mreal["id"] = self.real_id + mreal["id"] = self._real_id mreal["uuid"] = real_uuid - mreal["name"] = self.real_name - mreal["parameters"] = self.ert2["params"] - mreal["jobs"] = self.ert2["jobs"] + mreal["name"] = self._real_name + mreal["parameters"] = self._ert_info["params"] + + if self.include_ertjobs: + mreal["jobs"] = self._ert_info["jobs"] diff --git a/src/fmu/dataio/_metadata.py b/src/fmu/dataio/_metadata.py index a98c10da7..72d0ef02d 100644 --- a/src/fmu/dataio/_metadata.py +++ b/src/fmu/dataio/_metadata.py @@ -19,16 +19,17 @@ from fmu import dataio from fmu.dataio._definitions import SCHEMA, SOURCE, VERSION from fmu.dataio._filedata_provider import _FileDataProvider -from fmu.dataio._fmu_provider import _FmuProvider +from fmu.dataio._fmu_provider import FmuProvider from fmu.dataio._objectdata_provider import _ObjectDataProvider from fmu.dataio._utils import ( drop_nones, export_file_compute_checksum_md5, glue_metadata_preprocessed, - read_metadata, + read_metadata_from_file, ) from fmu.dataio.datastructure.meta import meta +from ._definitions import FmuContext from ._logging import null_logger logger: Final = null_logger(__name__) @@ -222,6 +223,9 @@ class _MetaData: # storage state variables objdata: Any = field(default=None, init=False) fmudata: Any = field(default=None, init=False) + iter_name: str = field(default="", init=False) + real_name: str = field(default="", init=False) + meta_class: str = field(default="", init=False) meta_masterdata: dict = field(default_factory=dict, init=False) meta_objectdata: dict = field(default_factory=dict, init=False) @@ -247,7 +251,9 @@ def __post_init__(self) -> None: # according to rule described in string self.reuse_metadata_rule! if isinstance(self.obj, (str, Path)) and self.dataio.reuse_metadata_rule: logger.info("Partially reuse existing metadata from %s", self.obj) - self.meta_existing = read_metadata(self.obj) + self.meta_existing = read_metadata_from_file(self.obj) + + self.rootpath = str(self.dataio._rootpath.absolute()) def _populate_meta_objectdata(self) -> None: """Analyze the actual object together with input settings. @@ -261,32 +267,46 @@ def _populate_meta_objectdata(self) -> None: self.objdata.derive_metadata() self.meta_objectdata = self.objdata.metadata - def _get_case_metadata(self) -> object: - """Detect existing fmu CASE block in the metadata. + def _populate_meta_fmu(self) -> None: + """Populate the fmu block in the metadata. This block may be missing in case the client is not within a FMU run, e.g. it runs from RMS interactive - The _FmuDataProvider is ran first -> self.fmudata + The _FmuDataProvider is ran to provide this information """ - self.fmudata = _FmuProvider(self.dataio) - self.fmudata.detect_provider() - logger.info("FMU provider is %s", self.fmudata.provider) - return self.fmudata.case_metadata + fmudata = FmuProvider( + model=self.dataio.config.get("model", None), + fmu_context=self.dataio.fmu_context, + casepath_proposed=self.dataio.casepath, + include_ertjobs=self.dataio.include_ertjobs, + forced_realization=self.dataio.realization, + workflow=self.dataio.workflow, + ) + logger.info("FMU provider is %s", fmudata.get_provider()) - def _populate_meta_fmu(self) -> None: - """Populate the fmu block in the metadata. + if not fmudata.get_provider(): # e.g. run from RMS not in a FMU run + actual_context = ( + FmuContext.PREPROCESSED + if self.dataio.fmu_context == FmuContext.PREPROCESSED + else FmuContext.get("non_fmu") + ) + if self.dataio.fmu_context != actual_context: + logger.warning( + "Requested fmu_context is <%s> but since this is detected as a non " + "FMU run, the actual context is force set to <%s>", + self.dataio.fmu_context, + actual_context, + ) + self.dataio.fmu_context = actual_context - This block may be missing in case the client is not within a FMU run, e.g. - it runs from RMS interactive + else: + self.meta_fmu = fmudata.get_metadata() + self.rootpath = fmudata.get_casepath() + self.iter_name = fmudata.get_iter_name() + self.real_name = fmudata.get_real_name() - The _FmuDataProvider is ran first -> self.fmudata - """ - self.fmudata = _FmuProvider(self.dataio) - self.fmudata.detect_provider() - logger.info("FMU provider is %s", self.fmudata.provider) - self.meta_fmu = self.fmudata.metadata - self.rootpath = str(self.fmudata.rootpath if self.fmudata.rootpath else "") + logger.debug("Rootpath is now %s", self.rootpath) def _populate_meta_file(self) -> None: """Populate the file block in the metadata. @@ -308,8 +328,8 @@ def _populate_meta_file(self) -> None: self.dataio, self.objdata, Path(self.rootpath), - self.fmudata.iter_name, - self.fmudata.real_name, + self.iter_name, + self.real_name, ) fdata.derive_filedata() @@ -377,7 +397,7 @@ def _populate_meta_display(self) -> None: def _populate_meta_xpreprocessed(self) -> None: """Populate a few necessary 'tmp' metadata needed for preprocessed data.""" - if self.dataio.fmu_context == "preprocessed": + if self.dataio.fmu_context == FmuContext.PREPROCESSED: self.meta_xpreprocessed["name"] = self.dataio.name self.meta_xpreprocessed["tagname"] = self.dataio.tagname self.meta_xpreprocessed["subfolder"] = self.dataio.subfolder @@ -428,7 +448,7 @@ def generate_export_metadata( meta["access"] = self.meta_access meta["masterdata"] = self.meta_masterdata - if self.dataio.fmu_context == "preprocessed": + if self.dataio.fmu_context == FmuContext.PREPROCESSED: meta["_preprocessed"] = self.meta_xpreprocessed if skip_null: diff --git a/src/fmu/dataio/_utils.py b/src/fmu/dataio/_utils.py index 2b8b05d10..efc1ef15c 100644 --- a/src/fmu/dataio/_utils.py +++ b/src/fmu/dataio/_utils.py @@ -145,8 +145,8 @@ def export_file( if isinstance(obj, Table): from pyarrow import feather - # comment taken from equinor/webviz_subsurface/smry2arrow.py + # comment taken from equinor/webviz_subsurface/smry2arrow.py # Writing here is done through the feather import, but could also be # done using pa.RecordBatchFileWriter.write_table() with a few # pa.ipc.IpcWriteOptions(). It is convenient to use feather since it @@ -400,7 +400,7 @@ def generate_description(desc: str | list | None = None) -> list | None: raise ValueError("Description of wrong type, must be list of strings or string") -def read_metadata(filename: str | Path) -> dict: +def read_metadata_from_file(filename: str | Path) -> dict: """Read the metadata as a dictionary given a filename. If the filename is e.g. /some/path/mymap.gri, the assosiated metafile diff --git a/src/fmu/dataio/dataio.py b/src/fmu/dataio/dataio.py index bce1fee3c..641723133 100644 --- a/src/fmu/dataio/dataio.py +++ b/src/fmu/dataio/dataio.py @@ -10,7 +10,7 @@ from copy import deepcopy from dataclasses import dataclass, field from pathlib import Path -from typing import Any, ClassVar, Final, List, Literal, Optional, Union +from typing import Any, ClassVar, Dict, Final, List, Literal, Optional, Union from warnings import warn import pandas as pd @@ -19,28 +19,27 @@ from . import _metadata from ._definitions import ( ALLOWED_CONTENTS, - ALLOWED_FMU_CONTEXTS, CONTENTS_REQUIRED, DEPRECATED_CONTENTS, + FmuContext, ) from ._logging import null_logger from ._utils import ( create_symlink, - dataio_examples, - detect_inside_rms, + detect_inside_rms, # dataio_examples, drop_nones, export_file_compute_checksum_md5, export_metadata_file, filter_validate_metadata, generate_description, prettyprint_dict, - read_metadata as _utils_read_metadata, + read_metadata_from_file, some_config_from_env, uuid_from_string, ) from .datastructure.configuration import global_configuration -DATAIO_EXAMPLES: Final = dataio_examples() +# DATAIO_EXAMPLES: Final = dataio_examples() INSIDE_RMS: Final = detect_inside_rms() @@ -214,7 +213,7 @@ def read_metadata(filename: str | Path) -> dict: Returns: A dictionary with metadata read from the assiated metadata file. """ - return _utils_read_metadata(filename) + return read_metadata_from_file(filename) # ====================================================================================== @@ -261,7 +260,7 @@ class ExportData: /project/foo/resmod/ff/2022.1.0/share/results/maps/xx.gri << example absolute share/results/maps/xx.gri << example relative - When running an ERT2 forward job using a normal ERT job (e.g. a script):: + When running an ERT forward job using a normal ERT job (e.g. a script):: /scratch/nn/case/realization-44/iter-2 << pwd /scratch/nn/case << rootpath @@ -271,7 +270,7 @@ class ExportData: /scratch/nn/case/realization-44/iter-2/share/results/maps/xx.gri << absolute realization-44/iter-2/share/results/maps/xx.gri << relative - When running an ERT2 forward job but here executed from RMS:: + When running an ERT forward job but here executed from RMS:: /scratch/nn/case/realization-44/iter-2/rms/model << pwd /scratch/nn/case << rootpath @@ -448,7 +447,7 @@ class ExportData: cube_fformat: ClassVar[str] = "segy" filename_timedata_reverse: ClassVar[bool] = False # reverse order output file name grid_fformat: ClassVar[str] = "roff" - include_ert2jobs: ClassVar[bool] = False # if True, include jobs.json from ERT2 + include_ertjobs: ClassVar[bool] = False # if True, include jobs.json from ERT legacy_time_format: ClassVar[bool] = False meta_format: ClassVar[Literal["yaml", "json"]] = "yaml" polygons_fformat: ClassVar[str] = "csv" # or use "csv|xtgeo" @@ -469,7 +468,7 @@ class ExportData: depth_reference: str = "msl" description: Union[str, list] = "" display_name: Optional[str] = None - fmu_context: str = "realization" + fmu_context: Union[FmuContext, str] = "realization" forcefolder: str = "" grid_model: Optional[str] = None is_observation: bool = False @@ -491,7 +490,6 @@ class ExportData: # some keys that are modified version of input, prepended with _use _usecontent: dict = field(default_factory=dict, init=False) - _usecontext: str = field(default="", init=False) _usefmtflag: str = field(default="", init=False) # storing resulting state variables for instance, non-public: @@ -514,6 +512,8 @@ def __post_init__(self) -> None: logger.info("Running __post_init__ ExportData") logger.debug("Global config is %s", prettyprint_dict(self.config)) + self.fmu_context = FmuContext.get(self.fmu_context) + # set defaults for mutable keys self.vertical_domain = {"depth": "msl"} @@ -547,10 +547,7 @@ def __post_init__(self) -> None: self.config = global_configuration.roundtrip(theconfig) self._validate_content_key() - logger.info("Validate FMU context which is %s", self.fmu_context) - self._validate_fmucontext_key() self._update_globalconfig_from_settings() - # check state of global config self._config_is_valid = global_configuration.is_valid(self.config) if self._config_is_valid: @@ -585,14 +582,8 @@ def _validate_content_key(self) -> None: def _validate_fmucontext_key(self) -> None: """Validate the given 'fmu_context' input.""" - if self.fmu_context not in ALLOWED_FMU_CONTEXTS: - msg = "" - for key, value in ALLOWED_FMU_CONTEXTS.items(): - msg += f"{key}: {value}\n" - raise ValidationError( - "It seems like 'fmu_context' value is illegal! " - f"Allowed entries are: in list:\n{msg}" - ) + if isinstance(self.fmu_context, str): + self.fmu_context = FmuContext.get(self.fmu_context) def _update_fmt_flag(self) -> None: # treat special handling of "xtgeo" in format name: @@ -668,17 +659,14 @@ def _establish_pwd_rootpath(self) -> None: logger.info("The casepath is hard set as %s", self._rootpath) else: - if ExportData._inside_rms or INSIDE_RMS or DATAIO_EXAMPLES: + if ExportData._inside_rms or INSIDE_RMS: logger.info( - "Run from inside RMS: ExportData._inside_rms=%s, " - "INSIDE_RMS=%s, DATAIO_EXAMPLES=%s", + "Run from inside RMS: ExportData._inside_rms=%s, INSIDE_RMS=%s", ExportData._inside_rms, INSIDE_RMS, - DATAIO_EXAMPLES, ) self._rootpath = (self._pwd / "../../.").absolute().resolve() ExportData._inside_rms = True - self._usecontext = self.fmu_context # may change later! logger.info("pwd: %s", str(self._pwd)) logger.info("rootpath: %s", str(self._rootpath)) @@ -1029,13 +1017,14 @@ def generate_metadata( meta["fmu"] = {} meta["fmu"]["model"] = self.config["model"] - mcase = meta["fmu"]["case"] = {} + mcase: Dict[str, Any] = {} # needed for python < 3.10 + meta["fmu"]["case"] = mcase + mcase["name"] = self.casename mcase["uuid"] = str(uuid.uuid4()) + mcase["user"] = {"id": self.caseuser} - mcase["user"] = {"id": self.caseuser} # type: ignore - - mcase["description"] = generate_description(self.description) # type: ignore + mcase["description"] = generate_description(self.description) mcase["restart_from"] = self.restart_from meta["tracklog"] = _metadata.generate_meta_tracklog() diff --git a/tests/conftest.py b/tests/conftest.py index aa6547404..38d11c5a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ import xtgeo import yaml from fmu.config import utilities as ut +from fmu.dataio._fmu_provider import FmuEnv from fmu.dataio.dataio import ExportData, read_metadata from fmu.dataio.datastructure.configuration import global_configuration @@ -24,6 +25,25 @@ RUN2 = "tests/data/drogon/ertrun1" RUN_PRED = "tests/data/drogon/ertrun1/realization-0/pred" +RUN1_ENV_PREHOOK = { + f"_ERT_{FmuEnv.EXPERIMENT_ID.name}": "6a8e1e0f-9315-46bb-9648-8de87151f4c7", + f"_ERT_{FmuEnv.ENSEMBLE_ID.name}": "b027f225-c45d-477d-8f33-73695217ba14", + f"_ERT_{FmuEnv.SIMULATION_MODE.name}": "test_run", +} +RUN1_ENV_FORWARD = { + f"_ERT_{FmuEnv.ITERATION_NUMBER.name}": "0", + f"_ERT_{FmuEnv.REALIZATION_NUMBER.name}": "0", + f"_ERT_{FmuEnv.RUNPATH.name}": "---", # set dynamically due to pytest tmp rotation +} +RUN1_ENV_FULLRUN = {**RUN1_ENV_PREHOOK, **RUN1_ENV_FORWARD} + +ERT_RUNPATH = f"_ERT_{FmuEnv.RUNPATH.name}" + + +def _current_function_name(): + """Helper to retrieve current function name, e.g. for logging""" + return inspect.currentframe().f_back.f_code.co_name + def pytest_configure(): if "RMSVER" in os.environ and "INSIDE_RMS" not in os.environ: @@ -56,35 +76,70 @@ def fixture_testroot(): return ROOTPWD -@pytest.fixture(name="fmurun", scope="session") -def fixture_fmurun(tmp_path_factory): - """Create a tmp folder structure for testing; here a new fmurun.""" +def _fmu_run1_env_variables(monkeypatch, usepath="", case_only=False): + """Helper function for fixtures below. + + Will here monkeypatch the ENV variables, with a particular setting for RUNPATH + (trough `usepath`) which may vary dynamically due to pytest tmp area rotation. + """ + env = RUN1_ENV_FULLRUN if not case_only else RUN1_ENV_PREHOOK + for key, value in env.items(): + env_value = str(usepath) if "RUNPATH" in key else value + monkeypatch.setenv(key, env_value) + logger.debug("Setting env %s as %s", key, env_value) + + +@pytest.fixture(name="fmurun", scope="function") +def fixture_fmurun(tmp_path_factory, monkeypatch): + """A tmp folder structure for testing; here a new fmurun without case metadata.""" tmppath = tmp_path_factory.mktemp("data") newpath = tmppath / RUN1 shutil.copytree(ROOTPWD / RUN1, newpath) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + + _fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=False) + + logger.debug("Ran %s", _current_function_name()) + return newpath + + +@pytest.fixture(name="fmurun_prehook", scope="function") +def fixture_fmurun_prehook(tmp_path_factory, monkeypatch): + """A tmp folder structure for testing; here a new fmurun without case metadata.""" + tmppath = tmp_path_factory.mktemp("data") + newpath = tmppath / RUN2 + shutil.copytree(ROOTPWD / RUN2, newpath) + + _fmu_run1_env_variables(monkeypatch, usepath=newpath, case_only=True) + + logger.debug("Ran %s", _current_function_name()) return newpath -@pytest.fixture(name="fmurun_w_casemetadata", scope="session") -def fixture_fmurun_w_casemetadata(tmp_path_factory): +@pytest.fixture(name="fmurun_w_casemetadata", scope="function") +def fixture_fmurun_w_casemetadata(tmp_path_factory, monkeypatch): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") newpath = tmppath / RUN2 shutil.copytree(ROOTPWD / RUN2, newpath) rootpath = newpath / "realization-0/iter-0" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + + _fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False) + + logger.debug("Ran %s", _current_function_name()) return rootpath -@pytest.fixture(name="fmurun_w_casemetadata_pred", scope="session") -def fixture_fmurun_w_casemetadata_pred(tmp_path_factory): +@pytest.fixture(name="fmurun_w_casemetadata_pred", scope="function") +def fixture_fmurun_w_casemetadata_pred(tmp_path_factory, monkeypatch): """Create a tmp folder structure for testing; here existing fmurun w/ case meta!""" tmppath = tmp_path_factory.mktemp("data3") newpath = tmppath / RUN2 shutil.copytree(ROOTPWD / RUN2, newpath) rootpath = newpath / "realization-0/pred" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + + _fmu_run1_env_variables(monkeypatch, usepath=rootpath, case_only=False) + + logger.debug("Ran %s", _current_function_name()) return rootpath @@ -94,7 +149,7 @@ def fixture_fmurun_pred(tmp_path_factory): tmppath = tmp_path_factory.mktemp("data_pred") newpath = tmppath / RUN_PRED shutil.copytree(ROOTPWD / RUN_PRED, newpath) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return newpath @@ -110,8 +165,8 @@ def fixture_rmsrun_fmu_w_casemetadata(tmp_path_factory): shutil.copytree(ROOTPWD / RUN2, newpath) rmspath = newpath / "realization-0/iter-0/rms/model" rmspath.mkdir(parents=True, exist_ok=True) - logger.info("Active folder is %s", rmspath) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Active folder is %s", rmspath) + logger.debug("Ran %s", _current_function_name()) return rmspath @@ -128,7 +183,7 @@ def fixture_rmssetup(tmp_path_factory): ROOTPWD / "tests/data/drogon/global_config2/global_variables.yml", rmspath ) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return rmspath @@ -138,12 +193,12 @@ def fixture_rmsglobalconfig(rmssetup): """Read global config.""" # read the global config os.chdir(rmssetup) - logger.info("Global config is %s", str(rmssetup / "global_variables.yml")) + logger.debug("Global config is %s", str(rmssetup / "global_variables.yml")) with open("global_variables.yml", encoding="utf8") as stream: global_cfg = yaml.safe_load(stream) - logger.info("Ran setup for %s", "rmsglobalconfig") - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran setup for %s", "rmsglobalconfig") + logger.debug("Ran %s", _current_function_name()) return global_cfg @@ -177,7 +232,7 @@ def fixture_casesetup(tmp_path_factory): tmppath = tmppath / "realization-0/iter-0" tmppath.mkdir(parents=True, exist_ok=True) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return tmppath @@ -243,7 +298,7 @@ def fixture_globalconfig_asfile() -> str: @pytest.fixture(name="edataobj1", scope="module") def fixture_edataobj1(globalconfig1): """Combined globalconfig and settings to instance, for internal testing""" - logger.info("Establish edataobj1") + logger.debug("Establish edataobj1") eobj = dio.ExportData( config=globalconfig1, @@ -256,7 +311,7 @@ def fixture_edataobj1(globalconfig1): eobj.createfolder = False eobj.verifyfolder = False - logger.info( + logger.debug( "Ran %s returning %s", inspect.currentframe().f_code.co_name, type(eobj) ) return eobj @@ -272,7 +327,7 @@ def fixture_globalconfig2() -> dict: ) as stream: globvar = yaml.safe_load(stream) - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return globvar @@ -302,7 +357,7 @@ def fixture_edataobj2(globalconfig2): eobj._rootpath = Path(".") eobj._pwd = Path(".") - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return eobj @@ -352,14 +407,14 @@ def fixture_metadata_examples(): @pytest.fixture(name="regsurf", scope="module") def fixture_regsurf(): """Create an xtgeo surface.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.RegularSurface(ncol=12, nrow=10, xinc=20, yinc=20, values=1234.0) @pytest.fixture(name="polygons", scope="module") def fixture_polygons(): """Create an xtgeo polygons.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.Polygons( [ [1, 22, 3, 0], @@ -373,7 +428,7 @@ def fixture_polygons(): @pytest.fixture(name="points", scope="module") def fixture_points(): """Create an xtgeo points instance.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.Points( [ [1, 22, 3, "WELLA"], @@ -388,35 +443,35 @@ def fixture_points(): @pytest.fixture(name="cube", scope="module") def fixture_cube(): """Create an xtgeo cube instance.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.Cube(ncol=3, nrow=4, nlay=5, xinc=12, yinc=12, zinc=4, rotation=30) @pytest.fixture(name="grid", scope="module") def fixture_grid(): """Create an xtgeo grid instance.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.create_box_grid((3, 4, 5)) @pytest.fixture(name="gridproperty", scope="module") def fixture_gridproperty(): """Create an xtgeo gridproperty instance.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return xtgeo.GridProperty(ncol=3, nrow=7, nlay=3, values=123.0) @pytest.fixture(name="dataframe", scope="module") def fixture_dataframe(): """Create an pandas dataframe instance.""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return pd.DataFrame({"COL1": [1, 2, 3, 4], "COL2": [99.0, 98.0, 97.0, 96.0]}) @pytest.fixture(name="wellpicks", scope="module") def fixture_wellpicks(): """Create a pandas dataframe containing wellpicks""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) return pd.DataFrame( { "X_UTME": [ @@ -463,10 +518,10 @@ def fixture_arrowtable(): return None -@pytest.fixture(name="aggr_surfs_mean", scope="module") +@pytest.fixture(name="aggr_surfs_mean", scope="function") def fixture_aggr_surfs_mean(fmurun_w_casemetadata, rmsglobalconfig, regsurf): """Create aggregated surfaces, and return aggr. mean surface + lists of metadata""" - logger.info("Ran %s", inspect.currentframe().f_code.co_name) + logger.debug("Ran %s", _current_function_name()) origfolder = os.getcwd() os.chdir(fmurun_w_casemetadata) @@ -496,7 +551,9 @@ def fixture_aggr_surfs_mean(fmurun_w_casemetadata, rmsglobalconfig, regsurf): surfs.append([surf]) aggregated = surfs.statistics() - logger.info("Aggr. mean is %s", aggregated["mean"].values.mean()) # shall be 1238.5 + logger.debug( + "Aggr. mean is %s", aggregated["mean"].values.mean() + ) # shall be 1238.5 os.chdir(origfolder) @@ -506,7 +563,7 @@ def fixture_aggr_surfs_mean(fmurun_w_casemetadata, rmsglobalconfig, regsurf): @pytest.fixture(name="edataobj3") def fixture_edataobj3(globalconfig1): """Combined globalconfig and settings to instance, for internal testing""" - # logger.info("Establish edataobj1") + # logger.debug("Establish edataobj1") return ExportData( config=globalconfig1, From 02764c2b2483b1634256ef9e3ccc385d50843679 Mon Sep 17 00:00:00 2001 From: "Jan C. Rivenaes" Date: Thu, 8 Feb 2024 18:24:08 +0100 Subject: [PATCH 2/4] CLN: adjust examples accoring to redesign of FmuProvider The examples now fake ERT environment variables --- examples/run_examples.sh | 14 ++++++++++++-- .../realization-0/iter-0/any/bin/export_grid3d.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/examples/run_examples.sh b/examples/run_examples.sh index 2a56903e4..3eb6f8f6c 100755 --- a/examples/run_examples.sh +++ b/examples/run_examples.sh @@ -2,8 +2,6 @@ set -e # The examples scripts are ran and included in the documentation! -export RUN_DATAIO_EXAMPLES=1 # this is important when running examples! - current=$PWD # empty current results @@ -14,6 +12,14 @@ rm -rf examples/s/d/nn/xcase/iter-0/* # Note! run from RUNPATH, NOT being inside RMS but need RUN_DATAIO_EXAMPLES env! cd $current/examples/s/d/nn/xcase/realization-0/iter-0/rms/bin +# fake an ERT FMU run +export _ERT_EXPERIMENT_ID=6a8e1e0f-9315-46bb-9648-8de87151f4c7 +export _ERT_ENSEMBLE_ID=b027f225-c45d-477d-8f33-73695217ba14 +export _ERT_SIMULATION_MODE=test_run +export _ERT_ITERATION_NUMBER=0 +export _ERT_REALIZATION_NUMBER=0 +export _ERT_RUNPATH=$current/examples/s/d/nn/xcase/realization-0/iter-0 + python export_faultpolygons.py python export_propmaps.py @@ -25,6 +31,10 @@ python export_volumetables.py # Emulate FMU run with 3 realizations and export data to disk for num in 0 1 9; do cd $current/examples/s/d/nn/xcase/realization-${num}/iter-0/rms/bin + + export _ERT_REALIZATION_NUMBER=$num + export _ERT_RUNPATH=${current}/examples/s/d/nn/xcase/realization-${num}/iter-0 + python export_a_surface.py done diff --git a/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py b/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py index c77e19c92..1fd843462 100644 --- a/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py +++ b/examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_grid3d.py @@ -6,8 +6,8 @@ import xtgeo from fmu.config import utilities as ut +logging.basicConfig(level=logging.DEBUG) logger = logging.getLogger(__name__) -logger.setLevel(logging.WARNING) CFG = ut.yaml_load("../../fmuconfig/output/global_variables.yml") From c6e9a6e97e8df8b71d5a18caabf4e76c779e5615 Mon Sep 17 00:00:00 2001 From: "Jan C. Rivenaes" Date: Thu, 8 Feb 2024 18:24:59 +0100 Subject: [PATCH 3/4] BLD: let version.py to be excluded from ruff check --- pyproject.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8587ade4d..2d02a161f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -89,10 +89,9 @@ match = '(?!(test_|_)).*\.py' [tool.ruff] line-length = 88 +exclude = ["version.py"] [tool.ruff.lint] -ignore = [ - "C901", -] +ignore = ["C901"] select = [ "C", "E", From a522da7388678274b88c0d256883a8daa32f3e26 Mon Sep 17 00:00:00 2001 From: "Jan C. Rivenaes" Date: Thu, 8 Feb 2024 18:25:39 +0100 Subject: [PATCH 4/4] TST: adapt tests to redesign of FmuProvider --- tests/test_units/test_dictionary.py | 2 +- tests/test_units/test_enum_classes.py | 22 ++ ...st_ert2_context.py => test_ert_context.py} | 10 +- tests/test_units/test_fmuprovider_class.py | 306 +++++++++--------- tests/test_units/test_rms_context.py | 2 +- 5 files changed, 181 insertions(+), 161 deletions(-) create mode 100644 tests/test_units/test_enum_classes.py rename tests/test_units/{test_ert2_context.py => test_ert_context.py} (98%) diff --git a/tests/test_units/test_dictionary.py b/tests/test_units/test_dictionary.py index 41e10d31f..f50b5ae90 100644 --- a/tests/test_units/test_dictionary.py +++ b/tests/test_units/test_dictionary.py @@ -36,7 +36,7 @@ def _fixture_json(fmurun_w_casemetadata): return json.load(stream) -@pytest.fixture(name="simple_parameters", scope="session") +@pytest.fixture(name="simple_parameters", scope="function") def _fixture_simple_parameters(fmurun_w_casemetadata): """Return dictionary read from parameters.txt diff --git a/tests/test_units/test_enum_classes.py b/tests/test_units/test_enum_classes.py new file mode 100644 index 000000000..17cc295ac --- /dev/null +++ b/tests/test_units/test_enum_classes.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import pytest +from fmu.dataio._definitions import FmuContext + + +def test_fmu_context_validation() -> None: + """Test the FmuContext enum class.""" + rel = FmuContext.get("realization") + assert rel.name == "REALIZATION" + + with pytest.raises(KeyError, match="Invalid key"): + FmuContext.get("invalid_context") + + valid_types = FmuContext.list_valid() + assert list(valid_types.keys()) == [ + "REALIZATION", + "CASE", + "CASE_SYMLINK_REALIZATION", + "PREPROCESSED", + "NON_FMU", + ] diff --git a/tests/test_units/test_ert2_context.py b/tests/test_units/test_ert_context.py similarity index 98% rename from tests/test_units/test_ert2_context.py rename to tests/test_units/test_ert_context.py index 79eab47c0..ea6685501 100644 --- a/tests/test_units/test_ert2_context.py +++ b/tests/test_units/test_ert_context.py @@ -1,7 +1,9 @@ -"""Test the dataio running from ERT2 aka forward model as pretended context. +"""Test the dataio running from ERT aka forward model as pretended context. In this case a user sits in ERT. Hence the rootpath will be ./ """ +from __future__ import annotations + import logging import os import sys @@ -15,7 +17,7 @@ def test_regsurf_generate_metadata(fmurun_w_casemetadata, rmsglobalconfig, regsurf): - """Test generating metadata for a surface pretend ERT2 job""" + """Test generating metadata for a surface pretend ERT job""" logger.info("Active folder is %s", fmurun_w_casemetadata) os.chdir(fmurun_w_casemetadata) @@ -37,7 +39,7 @@ def test_regsurf_generate_metadata_incl_jobs( logger.info("Active folder is %s", fmurun_w_casemetadata) os.chdir(fmurun_w_casemetadata) - dataio.ExportData.include_ert2jobs = True + dataio.ExportData.include_ertjobs = True edata = dataio.ExportData( config=rmsglobalconfig, @@ -47,7 +49,7 @@ def test_regsurf_generate_metadata_incl_jobs( meta = edata.generate_metadata(regsurf) assert meta["fmu"]["realization"]["jobs"]["umask"] == "0002" - dataio.ExportData.include_ert2jobs = False + dataio.ExportData.include_ertjobs = False def test_regsurf_metadata_with_timedata( diff --git a/tests/test_units/test_fmuprovider_class.py b/tests/test_units/test_fmuprovider_class.py index 6d34c6e13..5a64d6e91 100644 --- a/tests/test_units/test_fmuprovider_class.py +++ b/tests/test_units/test_fmuprovider_class.py @@ -1,101 +1,130 @@ -"""Test the _MetaData class from the _metadata.py module""" +"""Test the FmuProvider class applied the _metadata.py module""" +import logging import os +from pathlib import Path -import fmu.dataio as dio +import fmu.dataio as dataio import pytest -from fmu.dataio._fmu_provider import RESTART_PATH_ENVNAME, _FmuProvider, _get_folderlist -FOLDERTREE = "scratch/myfield/case/realization-13/iter-2" +# from conftest import pretend_ert_env_run1 +from fmu.dataio._definitions import FmuContext +from fmu.dataio._fmu_provider import RESTART_PATH_ENVNAME, FmuEnv, FmuProvider +logger = logging.getLogger(__name__) -def test_get_folderlist(fmurun): - os.chdir(fmurun) - mylist = _get_folderlist(fmurun) - assert mylist[-1] == "iter-0" - assert mylist[-3] == "ertrun1" +FOLDERTREE = "/scratch/myfield/case/realization-13/iter-2/" -def test_fmuprovider_no_provider(testroot, globalconfig1): - """Testing the FmuProvider basics where no ERT context is found from folder tree.""" - os.chdir(testroot) +def test_get_folderlist_from_path(): + """Test static method on getting folders from a path""" + ftree = Path(FOLDERTREE) + mylist = FmuProvider._get_folderlist_from_path(ftree) + assert mylist[-1] == "iter-2" + assert mylist[-3] == "case" + assert mylist[0] == "scratch" - ex = dio.ExportData( - fmu_context="realization", config=globalconfig1, content="depth" - ) - myfmu = _FmuProvider(ex) - myfmu.detect_provider() - assert myfmu.is_fmurun is False - assert myfmu.case_name is None +def test_get_folderlist_from_ert_runpath(monkeypatch): + """Test static method on getting folders from a _ERT_RUNPATH env variable""" + logger.debug("Set ENV for RUNPATH as %s", FmuEnv.RUNPATH.keyname) + monkeypatch.setenv(FmuEnv.RUNPATH.keyname, FOLDERTREE) + mylist = FmuProvider._get_folderlist_from_runpath_env() + assert mylist[-1] == "iter-2" + assert mylist[-3] == "case" -def test_fmuprovider_ert2_provider(fmurun, globalconfig1): - """Testing the FmuProvider for an ERT2 case; case metadata are missing here""" - os.chdir(fmurun) +def test_fmuprovider_no_provider(): + """Testing the FmuProvider where no ERT context is found from env variables.""" - ex = dio.ExportData( - fmu_context="realization", config=globalconfig1, content="depth" + myfmu = FmuProvider( + model="Model2", + fmu_context=FmuContext.REALIZATION, + casepath_proposed="", + include_ertjobs=False, + forced_realization=None, + workflow="some work flow", ) - ex._rootpath = fmurun - - myfmu = _FmuProvider(ex) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - # since case name is missing - myfmu.detect_provider() - assert myfmu.case_name == "ertrun1" - assert myfmu.real_name == "realization-0" - assert myfmu.real_id == 0 + assert myfmu.get_provider() is None -def test_fmuprovider_ert2_provider_missing_parameter_txt(fmurun, globalconfig1): - """Test for an ERT2 case, when missing file parameter.txt (e.g. pred. run)""" +def test_fmuprovider_ert_provider_guess_casemeta_path(fmurun): + """The casepath input is empty, but try guess from ERT RUNPATH without success. + Since there are mot metadata here, this will issue a warning + """ os.chdir(fmurun) - (fmurun / "parameters.txt").unlink() + with pytest.warns(UserWarning, match="Case metadata does not exist"): + myfmu = FmuProvider( + model="Model2", + fmu_context=FmuContext.REALIZATION, + casepath_proposed="", # if casepath is undef, try deduce from, _ERT_RUNPATH + include_ertjobs=False, + forced_realization=None, + workflow="some work flow", + ) + + assert myfmu.get_provider() == "ERT" + assert myfmu._stage == "realization" # i.e. being a so-called forward model + assert not myfmu.get_metadata() + assert myfmu.get_casepath() == "" + + +def test_fmuprovider_ert_provider_missing_parameter_txt( + fmurun_w_casemetadata, globalconfig1 +): + """Test for an ERT case, when missing file parameter.txt (e.g. pred. run)""" - ex = dio.ExportData( - fmu_context="realization", content="depth", config=globalconfig1 - ) - ex._rootpath = fmurun + os.chdir(fmurun_w_casemetadata) - myfmu = _FmuProvider(ex) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - myfmu.detect_provider() - assert myfmu.case_name == "ertrun1" - assert myfmu.real_name == "realization-0" - assert myfmu.real_id == 0 + # delete the file for this test + (fmurun_w_casemetadata / "parameters.txt").unlink() + with pytest.warns(UserWarning, match="parameters.txt file was not found"): + myfmu = FmuProvider( + model="Model2", + fmu_context=FmuContext.REALIZATION, + include_ertjobs=True, + workflow="some work flow", + ) + assert myfmu._case_name == "ertrun1" + assert myfmu._real_name == "realization-0" + assert myfmu._real_id == 0 -def test_fmuprovider_arbitrary_iter_name(edataobj1, fmurun_w_casemetadata_pred): - """Test that iteration block correctly populated also with arbitrary iteration - names.""" - edataobj1._rootpath = fmurun_w_casemetadata_pred +def test_fmuprovider_arbitrary_iter_name(fmurun_w_casemetadata_pred): + """Test iteration block is correctly set, also with arbitrary iteration names.""" + os.chdir(fmurun_w_casemetadata_pred) - myfmu = _FmuProvider(edataobj1) - myfmu.detect_provider() - assert myfmu.case_name == "ertrun1" - assert myfmu.real_name == "realization-0" - assert myfmu.real_id == 0 - assert myfmu.iter_name == "pred" - assert myfmu.iter_id is None - assert "fmu_case" in str(myfmu.case_metafile) + myfmu = FmuProvider( + model="Model2", + fmu_context=FmuContext.REALIZATION, + include_ertjobs=True, + workflow="some work flow", + ) + assert myfmu._case_name == "ertrun1" + assert myfmu._real_name == "realization-0" + assert myfmu._real_id == 0 + assert myfmu._iter_name == "pred" + assert not myfmu._iter_id assert ( - myfmu.case_metadata["fmu"]["case"]["uuid"] + myfmu._case_metadata["fmu"]["case"]["uuid"] == "a40b05e8-e47f-47b1-8fee-f52a5116bd37" ) -def test_fmuprovider_prehook_case(globalconfig2, tmp_path): - """The fmu run case metadata is initialized with Initialize case; then fet provider. +def test_fmuprovider_prehook_case(tmp_path, globalconfig2, fmurun_prehook): + """The fmu run case metadata is initialized with Initialize case; then get provider. - A typical prehook section in a ERT2 run is to establish case metadata, and then - subsequent hook workflows should still recognize this as an ERT2 run, altough + A typical prehook section in a ERT run is to establish case metadata, and then + subsequent hook workflows should still recognize this as an ERT run, altough no iter and realization folders exists. This requires fmu_contact=case* and that - key casepath is given explicitly!. + key casepath_proposed is given explicitly!. + + I *may* be that this behaviour can be removed in near future, since the ERT env + variables now will tell us that this is an active ERT run. """ - icase = dio.InitializeCase(config=globalconfig2) + icase = dataio.InitializeCase(config=globalconfig2) caseroot = tmp_path / "prehook" caseroot.mkdir(parents=True) @@ -111,120 +140,96 @@ def test_fmuprovider_prehook_case(globalconfig2, tmp_path): casemetafile = caseroot / "share/metadata/fmu_case.yml" assert casemetafile.is_file() assert exp == str(casemetafile.resolve()) - - eobj = dio.ExportData( - config=globalconfig2, - name="TopWhatever", - content="depth", - tagname="mytag", - is_observation=True, - fmu_context="case", - casepath=caseroot, + os.chdir(fmurun_prehook) + + logger.debug("Case root proposed is: %s", caseroot) + myfmu = FmuProvider( + model="Model567", + fmu_context=FmuContext.CASE, + include_ertjobs=False, + workflow="some work flow", + casepath_proposed=caseroot, ) - myfmu = _FmuProvider(eobj) - myfmu.detect_provider() - assert myfmu.case_name == "prehook" - assert myfmu.real_name is None + assert myfmu._case_name == "prehook" + assert myfmu._real_name == "" -def test_fmuprovider_detect_no_case_metadata(fmurun, edataobj1): +def test_fmuprovider_detect_no_case_metadata(fmurun): """Testing the case metadata file which is not found here. That will still provide a file path but the metadata will be {} i.e. empty """ os.chdir(fmurun) - edataobj1._rootpath = fmurun - myfmu = _FmuProvider(edataobj1) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - myfmu.detect_provider() - assert myfmu.case_name == "ertrun1" - assert myfmu.real_name == "realization-0" - assert myfmu.real_id == 0 - assert "fmu_case" in str(myfmu.case_metafile) - assert not myfmu.case_metadata + with pytest.warns(UserWarning): + myfmu = FmuProvider( + model="Model567", + fmu_context=FmuContext.REALIZATION, + ) + assert myfmu._case_name == "ertrun1" + assert myfmu._real_name == "realization-0" + assert myfmu._real_id == 0 + assert not myfmu._case_metadata -def test_fmuprovider_valid_restart_env(fmurun_w_casemetadata, fmurun_pred, edataobj1): +def test_fmuprovider_valid_restart_env(monkeypatch, fmurun_w_casemetadata, fmurun_pred): """Testing the scenario given a valid RESTART_FROM_PATH environment variable - This will give the correct restart_from uuid + This shall give the correct restart_from uuid """ - - edataobj1._rootpath = fmurun_w_casemetadata os.chdir(fmurun_w_casemetadata) - fmu_restart_from = _FmuProvider(edataobj1) - fmu_restart_from.detect_provider() + fmu_restart_from = FmuProvider( + model="Model with restart", fmu_context=FmuContext.REALIZATION + ) + + monkeypatch.setenv(RESTART_PATH_ENVNAME, str(fmurun_w_casemetadata)) - os.environ[RESTART_PATH_ENVNAME] = str(fmurun_w_casemetadata) - edataobj1._rootpath = fmurun_pred os.chdir(fmurun_pred) - fmu_restart = _FmuProvider(edataobj1) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - fmu_restart.detect_provider() + fmu_restart = FmuProvider(model="Modelrun", fmu_context=FmuContext.REALIZATION) + assert ( - fmu_restart.metadata["iteration"]["restart_from"] - == fmu_restart_from.metadata["iteration"]["uuid"] + fmu_restart._metadata["iteration"]["restart_from"] + == fmu_restart_from._metadata["iteration"]["uuid"] ) -def test_fmuprovider_invalid_restart_env(fmurun_w_casemetadata, fmurun_pred, edataobj1): +def test_fmuprovider_invalid_restart_env( + monkeypatch, fmurun_w_casemetadata, fmurun_pred +): """Testing the scenario given invalid RESTART_FROM_PATH environment variable The iteration metadata will not contain restart_from key """ - - edataobj1._rootpath = fmurun_w_casemetadata os.chdir(fmurun_w_casemetadata) - fmu_restart_from = _FmuProvider(edataobj1) - fmu_restart_from.detect_provider() - os.environ[RESTART_PATH_ENVNAME] = "/path/to/somewhere/invalid" - edataobj1._rootpath = fmurun_pred + _ = FmuProvider(model="Model with restart", fmu_context=FmuContext.REALIZATION) + + monkeypatch.setenv(RESTART_PATH_ENVNAME, "/path/to/somewhere/invalid") + os.chdir(fmurun_pred) - fmu_restart = _FmuProvider(edataobj1) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - fmu_restart.detect_provider() - assert "restart_from" not in fmu_restart.metadata["iteration"] + fmu_restart = FmuProvider(model="Modelrun", fmu_context=FmuContext.REALIZATION) + assert "restart_from" not in fmu_restart._metadata["iteration"] -def test_fmuprovider_no_restart_env(fmurun_w_casemetadata, fmurun_pred, edataobj1): +def test_fmuprovider_no_restart_env(monkeypatch, fmurun_w_casemetadata, fmurun_pred): """Testing the scenario without RESTART_FROM_PATH environment variable The iteration metadata will not contain restart_from key """ - - edataobj1._rootpath = fmurun_w_casemetadata os.chdir(fmurun_w_casemetadata) - fmu_restart_from = _FmuProvider(edataobj1) - fmu_restart_from.detect_provider() - edataobj1._rootpath = fmurun_pred - os.chdir(fmurun_pred) - fmu_restart = _FmuProvider(edataobj1) - with pytest.warns(UserWarning, match="ERT2 is found but no case metadata"): - fmu_restart.detect_provider() - assert "restart_from" not in fmu_restart.metadata["iteration"] + _ = FmuProvider(model="Model with restart", fmu_context=FmuContext.REALIZATION) + monkeypatch.setenv(RESTART_PATH_ENVNAME, "/path/to/somewhere/invalid") + monkeypatch.delenv(RESTART_PATH_ENVNAME) -def test_fmuprovider_detect_case_has_metadata(fmurun_w_casemetadata, edataobj1): - """Testing the case metadata file which is found here""" - edataobj1._rootpath = fmurun_w_casemetadata - os.chdir(fmurun_w_casemetadata) - myfmu = _FmuProvider(edataobj1) - myfmu.detect_provider() - assert myfmu.case_name == "ertrun1" - assert myfmu.real_name == "realization-0" - assert myfmu.real_id == 0 - assert "fmu_case" in str(myfmu.case_metafile) - assert ( - myfmu.case_metadata["fmu"]["case"]["uuid"] - == "a40b05e8-e47f-47b1-8fee-f52a5116bd37" - ) + os.chdir(fmurun_pred) + fmu_restart = FmuProvider(model="Modelrun", fmu_context=FmuContext.REALIZATION) + assert "restart_from" not in fmu_restart._metadata["iteration"] -def test_fmuprovider_workflow_reference(fmurun_w_casemetadata, edataobj1): +def test_fmuprovider_workflow_reference(fmurun_w_casemetadata): """Testing the handling of workflow reference input. Metadata definitions of fmu.workflow is that it is a dictionary with 'reference' @@ -238,32 +243,23 @@ def test_fmuprovider_workflow_reference(fmurun_w_casemetadata, edataobj1): it shall always produce valid metadata. """ - edataobj1._rootpath = fmurun_w_casemetadata os.chdir(fmurun_w_casemetadata) # workflow input is a string - edataobj1.workflow = "my workflow" - myfmu = _FmuProvider(edataobj1) - myfmu.detect_provider() - assert "workflow" in myfmu.metadata - assert myfmu.metadata["workflow"] == {"reference": "my workflow"} + myfmu = FmuProvider(workflow="workflow as string") + assert "workflow" in myfmu._metadata + assert myfmu._metadata["workflow"] == {"reference": "workflow as string"} # workflow input is a correct dict - edataobj1.workflow = {"reference": "my workflow"} - myfmu = _FmuProvider(edataobj1) with pytest.warns(PendingDeprecationWarning, match="The 'workflow' argument"): - myfmu.detect_provider() - assert "workflow" in myfmu.metadata - assert myfmu.metadata["workflow"] == {"reference": "my workflow"} + myfmu = FmuProvider(workflow={"reference": "workflow as dict"}) + assert "workflow" in myfmu._metadata + assert myfmu._metadata["workflow"] == {"reference": "workflow as dict"} # workflow input is non-correct dict - edataobj1.workflow = {"something": "something"} - myfmu = _FmuProvider(edataobj1) with pytest.raises(ValueError): - myfmu.detect_provider() + myfmu = FmuProvider(workflow={"wrong": "workflow as dict"}) # workflow input is other types - shall fail - edataobj1.workflow = 123.4 - myfmu = _FmuProvider(edataobj1) with pytest.raises(TypeError): - myfmu.detect_provider() + myfmu = FmuProvider(workflow=123.4) diff --git a/tests/test_units/test_rms_context.py b/tests/test_units/test_rms_context.py index 0c8e5260c..637e8c8ef 100644 --- a/tests/test_units/test_rms_context.py +++ b/tests/test_units/test_rms_context.py @@ -187,7 +187,7 @@ def test_regsurf_metadata_with_timedata_legacy(rmssetup, rmsglobalconfig, regsur def test_regsurf_export_file_fmurun( rmsrun_fmu_w_casemetadata, rmsglobalconfig, regsurf ): - """Being in RMS and in an active FMU ERT2 run with case metadata present. + """Being in RMS and in an active FMU ERT run with case metadata present. Export the regular surface to file with correct metadata and name. """