Skip to content

Commit

Permalink
Improve handling of warning when missing content
Browse files Browse the repository at this point in the history
Address issue #379

The handling of content is moved to _object_data_provider, and also
check that  the fmu_context is not "case"
  • Loading branch information
jcrivenaes committed Oct 23, 2023
1 parent f63c593 commit 60fb519
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 19 deletions.
42 changes: 33 additions & 9 deletions src/fmu/dataio/_objectdata_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,14 +87,14 @@
from dataclasses import dataclass, field
from datetime import datetime as dt
from pathlib import Path
from typing import Any, Optional
from typing import Any, Optional, Tuple
from warnings import warn

import numpy as np
import pandas as pd # type: ignore
import xtgeo # type: ignore

from ._definitions import _ValidFormats, STANDARD_TABLE_INDEX_COLUMNS
from ._definitions import ALLOWED_CONTENTS, STANDARD_TABLE_INDEX_COLUMNS, _ValidFormats
from ._utils import generate_description, parse_timedata

try:
Expand Down Expand Up @@ -664,6 +664,33 @@ def _derive_from_existing(self):

self.time0, self.time1 = parse_timedata(self.meta_existing["data"])

def _process_content(self) -> Tuple[str, Optional[dict]]:
"""Work with the `content` metadata"""

# content == "unset" is not wanted, but for "case" fmu_context this check is not
# relevant, since "case" processing already shall have content from the
# preprocessed stage, and "case" fmu_contect scripts are more generic.
if self.dataio._usecontent == "unset" and self.dataio.fmu_context != "case":
warn(
"The <content> is not provided which defaults to 'unset'. "
"It is strongly recommended that content is given explicitly! "
f"\n\nValid contents are: {', '.join(ALLOWED_CONTENTS.keys())} "
"\n\nThis list can be extended upon request and need.",
UserWarning,
)

content = self.dataio._usecontent
content_spesific = None

# Outgoing content is always a string, but it can be given as a dict if content-
# specific information is to be included in the metadata.
# In that case, it shall be inserted in the data block as a key with name as the
# content, e.g. "seismic" or "field_outline"
if self.dataio._content_specific is not None:
content_spesific = self.dataio._content_specific

return content, content_spesific

def derive_metadata(self):
"""Main function here, will populate the metadata block for 'data'."""
logger.info("Derive all metadata for data object...")
Expand All @@ -683,14 +710,11 @@ def derive_metadata(self):
meta["alias"] = nameres.get("alias", None)
meta["top"] = nameres.get("top", None)
meta["base"] = nameres.get("base", None)
meta["content"] = self.dataio._usecontent

# Outgoing content is always a string, but it can be given as a dict if content-
# specific information is to be included in the metadata.
# In that case, it shall be inserted in the data block as a key with name as the
# content, e.g. "seismic" or "field_outline"
if self.dataio._content_specific is not None:
meta[self.dataio._usecontent] = self.dataio._content_specific
content, content_spesific = self._process_content()
meta["content"] = content
if content_spesific:
meta[self.dataio._usecontent] = content_spesific

meta["tagname"] = self.dataio.tagname
meta["format"] = objres["fmt"]
Expand Down
9 changes: 1 addition & 8 deletions src/fmu/dataio/dataio.py
Original file line number Diff line number Diff line change
Expand Up @@ -134,14 +134,7 @@ def _check_content(proposed: Union[str, dict]) -> Any:
content_specific = None
logger.debug("content is %s of type %s", str(content), type(content))
if content is None:
warn(
"The <content> is not provided which defaults to 'unset'. "
"It is strongly recommended that content is given explicitly! "
f"\n\nValid contents are: {', '.join(ALLOWED_CONTENTS.keys())} "
"\n\nThis list can be extended upon request and need.",
UserWarning,
)
usecontent = "unset"
usecontent = "unset" # user warnings on this will in _objectdata_provider

elif isinstance(content, str):
logger.debug("content is a string")
Expand Down
23 changes: 22 additions & 1 deletion tests/test_units/test_dataio.py
Original file line number Diff line number Diff line change
Expand Up @@ -121,13 +121,34 @@ def test_deprecated_keys(globalconfig1, regsurf, key, value, wtype, expected_msg

def test_content_not_given(globalconfig1, regsurf):
"""When content is not explicitly given, warning shall be issued."""
with pytest.warns(match="The <content> is not provided"):
with pytest.warns(UserWarning, match="The <content> is not provided"):
eobj = ExportData(config=globalconfig1)
mymeta = eobj.generate_metadata(regsurf)

assert mymeta["data"]["content"] == "unset"


def test_content_not_given_case(globalconfig1, regsurf):
"""When content is not explicitly given and fmu_conext = "case" -> no warning"""
eobj = ExportData(config=globalconfig1, fmu_context="case")
mymeta = eobj.generate_metadata(regsurf)

assert mymeta["data"]["content"] == "unset"


def test_content_given_init_or_later(globalconfig1, regsurf):
"""When content is not explicitly given, warning shall be issued."""
eobj = ExportData(config=globalconfig1, content="time")
mymeta = eobj.generate_metadata(regsurf)

assert mymeta["data"]["content"] == "time"

# override by adding content at generate_metadata
mymeta = eobj.generate_metadata(regsurf, content="depth")

assert mymeta["data"]["content"] == "depth" # last content shall win


def test_content_invalid_string(globalconfig1):
with pytest.raises(ValidationError, match=r"Invalid content"):
ExportData(config=globalconfig1, content="not_valid")
Expand Down
19 changes: 18 additions & 1 deletion tests/test_units/test_prerealization_surfaces.py
Original file line number Diff line number Diff line change
Expand Up @@ -125,15 +125,19 @@ def _run_case_fmu(fmurun_w_casemetadata, rmsglobalconfig, surfacepath):
os.chdir(fmurun_w_casemetadata)
logger.info("Active folder is %s", fmurun_w_casemetadata)

casepath = fmurun_w_casemetadata.parent.parent

edata = dataio.ExportData(
config=rmsglobalconfig, # read from global config
fmu_context="case",
content="depth",
content="time",
is_observation=True,
)
metadata = edata.generate_metadata(
surfacepath,
casepath=casepath,
)
logger.info("Casepath folder is now %s", casepath)
logger.debug("\n%s", utils.prettyprint_dict(metadata))
assert (
metadata["file"]["relative_path"]
Expand All @@ -144,6 +148,19 @@ def _run_case_fmu(fmurun_w_casemetadata, rmsglobalconfig, surfacepath):
assert "TopVolantis" in metadata["data"]["alias"]
assert "_preprocessed" not in metadata

# do the actual export (which will copy data to case/share/observations/...)
edata.export(
surfacepath,
casepath=casepath,
)
assert (
casepath
/ "share"
/ "observations"
/ "maps"
/ ".topvolantis--20240802_20200909.gri.yml"
).exists()

# run two stage process
mysurf = _export_data_from_rms(rmssetup, rmsglobalconfig, regsurf)
_run_case_fmu(fmurun_w_casemetadata, rmsglobalconfig, mysurf)
Expand Down

0 comments on commit 60fb519

Please sign in to comment.