Merge branch 'main' into 540-classification-arg

.gitignore

@@ -98,6 +98,8 @@ venv.bak/
 src/fmu/dataio/version.py
 # aggregated examples directory
 examples/s/d/nn/xcase/iter-0/
+# exported results directory
+share/
 
 # mypy
 .dmypy.json

src/fmu/dataio/dataio.py

@@ -387,20 +387,20 @@ class ExportData:
     allow_forcefolder_absolute: ClassVar[bool] = False
     arrow_fformat: ClassVar[str] = "arrow"
     case_folder: ClassVar[str] = "share/metadata"
-    createfolder: ClassVar[bool] = True
+    createfolder: ClassVar[bool] = True  # deprecated
     cube_fformat: ClassVar[str] = "segy"
     filename_timedata_reverse: ClassVar[bool] = False  # reverse order output file name
     grid_fformat: ClassVar[str] = "roff"
     include_ertjobs: ClassVar[bool] = False  # if True, include jobs.json from ERT
-    legacy_time_format: ClassVar[bool] = False
+    legacy_time_format: ClassVar[bool] = False  # deprecated
     meta_format: ClassVar[Literal["yaml", "json"]] = "yaml"
     polygons_fformat: ClassVar[str] = "csv"  # or use "csv|xtgeo"
     points_fformat: ClassVar[str] = "csv"  # or use "csv|xtgeo"
     surface_fformat: ClassVar[str] = "irap_binary"
     table_fformat: ClassVar[str] = "csv"
     dict_fformat: ClassVar[str] = "json"
     table_include_index: ClassVar[bool] = False
-    verifyfolder: ClassVar[bool] = True
+    verifyfolder: ClassVar[bool] = True  # deprecated
     _inside_rms: ClassVar[bool] = False  # developer only! if True pretend inside RMS
 
     # input keys (alphabetic)
@@ -564,6 +564,26 @@ def _show_deprecations_or_notimplemented(self) -> None:
                 )
                 self.access_ssdl["access_level"] = "restricted"
 
+        if self.legacy_time_format:
+            warn(
+                "Using the 'legacy_time_format=True' option to create metadata files "
+                "with the old format for time is now deprecated. This option has no "
+                "longer an effect and will be removed in the near future.",
+                UserWarning,
+            )
+        if not self.createfolder:
+            warn(
+                "Using the 'createfolder=False' option is now deprecated. "
+                "This option has no longer an effect and can safely be removed",
+                UserWarning,
+            )
+        if not self.verifyfolder:
+            warn(
+                "Using the 'verifyfolder=False' option to create metadata files "
+                "This option has no longer an effect and can safely be removed",
+                UserWarning,
+            )
+
     def _validate_content_key(self) -> None:
         """Validate the given 'content' input."""
         self._usecontent, self._content_specific = _check_content(self.content)
@@ -726,13 +746,7 @@ def generate_metadata(
             empty. If true, the MD5 checksum will be generated based on export to
             a temporary file, which may be time-consuming if the file is large.
         """
-        if self.legacy_time_format:
-            warn(
-                "Using the 'legacy_time_format=True' option to create metadata files "
-                "with the old format for time is now deprecated. This option has no "
-                "longer an effect and will be removed in the near future.",
-                UserWarning,
-            )
+
         logger.info("Generate metadata...")
         logger.info("KW args %s", kwargs)
 
@@ -798,6 +812,8 @@ def export(
         logger.info("Object type is: %s", type(self._object))  # from generate_metadata
 
         outfile = Path(metadata["file"]["absolute_path"])
+        # create output folders if they don't exist
+        outfile.parent.mkdir(parents=True, exist_ok=True)
         metafile = outfile.parent / ("." + str(outfile.name) + ".yml")
 
         useflag = (
@@ -827,8 +843,8 @@ def export(
         outfile_target = None
         if metadata["file"].get("absolute_path_symlink"):
             outfile_target = Path(metadata["file"]["absolute_path_symlink"])
-            outfile_source = Path(metadata["file"]["absolute_path"])
-            create_symlink(str(outfile_source), str(outfile_target))
+            outfile_target.parent.mkdir(parents=True, exist_ok=True)
+            create_symlink(str(outfile), str(outfile_target))
             metafile_target = outfile_target.parent / ("." + str(outfile.name) + ".yml")
             create_symlink(str(metafile), str(metafile_target))
 

src/fmu/dataio/providers/_filedata.py

@@ -235,14 +235,4 @@ def _get_path_generic(
                     f"You cannot use forcefolder in combination with fmucontext={info}"
                 )
 
-        if self.dataio.subfolder:
-            dest = dest / self.dataio.subfolder
-
-        if self.dataio.createfolder:
-            dest.mkdir(parents=True, exist_ok=True)
-
-        # check that destination actually exists if verifyfolder is True
-        if self.dataio.verifyfolder and not dest.exists():
-            raise OSError(f"Folder {str(dest)} is not present.")
-
-        return dest
+        return dest if not self.dataio.subfolder else dest / self.dataio.subfolder

tests/conftest.py

@@ -289,8 +289,6 @@ def fixture_edataobj1(globalconfig1):
         is_observation=False,
     )
     eobj.surface_fformat = "irap_binary"
-    eobj.createfolder = False
-    eobj.verifyfolder = False
 
     logger.debug(
         "Ran %s returning %s", inspect.currentframe().f_code.co_name, type(eobj)
@@ -324,8 +322,6 @@ def fixture_edataobj2(globalconfig2):
     )
 
     eobj.surface_fformat = "irap_binary"
-    eobj.createfolder = False
-    eobj.verifyfolder = False
     eobj.legacy_time_format = False
 
     eobj._rootpath = Path(".")

tests/test_units/test_dataio.py

@@ -49,9 +49,11 @@ def test_generate_metadata_simple(globalconfig1):
     ExportData.grid_fformat = default_fformat  # reset
 
 
-def test_missing_or_wrong_config_exports_with_warning(regsurf):
+def test_missing_or_wrong_config_exports_with_warning(monkeypatch, tmp_path, regsurf):
     """In case a config is missing, or is invalid, do export with warning."""
 
+    monkeypatch.chdir(tmp_path)
+
     with pytest.warns(UserWarning, match=pydantic_warning()):
         edata = ExportData(config={}, content="depth")
 
@@ -69,9 +71,11 @@ def test_missing_or_wrong_config_exports_with_warning(regsurf):
         read_metadata(out)
 
 
-def test_config_miss_required_fields(globalconfig1, regsurf):
+def test_config_miss_required_fields(monkeypatch, tmp_path, globalconfig1, regsurf):
     """Global config exists but missing critical data; export file but skip metadata."""
 
+    monkeypatch.chdir(tmp_path)
+
     cfg = globalconfig1.copy()
 
     del cfg["access"]

tests/test_units/test_dictionary.py

@@ -105,14 +105,15 @@ def read_dict_and_meta(path):
         ("nested_parameters"),
     ],
 )
-def test_export_dict_w_meta(globalconfig2, dictionary, request):
+def test_export_dict_w_meta(globalconfig2, dictionary, request, monkeypatch, tmp_path):
     """Test various dictionaries
 
     Args:
         globalconfig2 (dict): a global variables dictionary
         dictionary (str): name of fixture to use
         request (pytest.fixture): fixture for using fixtures in parameterize
     """
+    monkeypatch.chdir(tmp_path)
     name = dictionary
     in_dict = request.getfixturevalue(dictionary)
     print(f"{name}: {in_dict}")
@@ -122,13 +123,16 @@ def test_export_dict_w_meta(globalconfig2, dictionary, request):
     assert_dict_correct(out_dict, out_meta, name)
 
 
-def test_invalid_dict(globalconfig2, drogon_summary, drogon_volumes):
+def test_invalid_dict(
+    globalconfig2, drogon_summary, drogon_volumes, monkeypatch, tmp_path
+):
     """Test raising of error when dictionary is not serializable
     Args:
         globalconfig2 (_type_): _description_
         drogon_summary (pd.DataFrame): a dataframe
         drogon_volumes (pa.Table): a pyarrow table
     """
+    monkeypatch.chdir(tmp_path)
     in_dict = {"volumes": drogon_volumes, "summary": drogon_summary}
     exd = ExportData(config=globalconfig2, content="parameters")
     with pytest.raises(TypeError) as exc_info:

tests/test_units/test_filedataprovider_class.py

@@ -198,7 +198,6 @@ def test_get_paths_not_exists_so_create(regsurf, edataobj1, tmp_path):
     objdata.efolder = "efolder"
     cfg = edataobj1
 
-    cfg.createfolder = True
     cfg._rootpath = Path(".")
 
     fdata = FileDataProvider(cfg, objdata)
@@ -213,7 +212,6 @@ def test_filedata_provider(regsurf, edataobj1, tmp_path):
     os.chdir(tmp_path)
 
     cfg = edataobj1
-    cfg.createfolder = True
     cfg._rootpath = Path(".")
     cfg.name = ""
 
@@ -242,7 +240,6 @@ def test_filedata_has_nonascii_letters(regsurf, edataobj1, tmp_path):
     os.chdir(tmp_path)
 
     cfg = edataobj1
-    cfg.createfolder = True
     cfg._rootpath = Path(".")
     cfg.name = "mynõme"
 

tests/test_units/test_table.py

@@ -49,33 +49,39 @@ def assert_correct_table_index(dict_input, answer):
     assert_list_and_answer(index, answer, index)
 
 
-def test_inplace_volume_index(mock_volumes, globalconfig2):
+def test_inplace_volume_index(mock_volumes, globalconfig2, monkeypatch, tmp_path):
     """Test volumetric data
 
     Args:
         mock_volumes (pd.DataFrame): a volumetriclike dataset
         globalconfig2 (dict): one global variables dict
     """
+    monkeypatch.chdir(tmp_path)
     answer = ["ZONE", "LICENCE"]
     exd = ExportData(config=globalconfig2, content="volumes")
     path = exd.export(mock_volumes, name="baretull")
     assert_correct_table_index(path, answer)
 
 
-def test_derive_summary_index_pandas(mock_summary, globalconfig2):
+def test_derive_summary_index_pandas(
+    mock_summary, globalconfig2, monkeypatch, tmp_path
+):
     """Test summary data
 
     Args:
         mock_summary (pd.DataFrame): summary "like" dataframe
         globalconfig2 (dict): global variables dict
     """
+    monkeypatch.chdir(tmp_path)
     answer = ["DATE"]
     exd = ExportData(config=globalconfig2, content="timeseries")
     path = exd.export(mock_summary, name="baretull")
     assert_correct_table_index(path, answer)
 
 
-def test_derive_summary_index_pyarrow(mock_summary, globalconfig2):
+def test_derive_summary_index_pyarrow(
+    mock_summary, globalconfig2, monkeypatch, tmp_path
+):
     """Test summary data
 
     Args:
@@ -84,45 +90,51 @@ def test_derive_summary_index_pyarrow(mock_summary, globalconfig2):
     """
     from pyarrow import Table
 
+    monkeypatch.chdir(tmp_path)
     answer = ["DATE"]
     exd = ExportData(config=globalconfig2, content="timeseries")
     path = exd.export(Table.from_pandas(mock_summary), name="baretull")
     assert_correct_table_index(path, answer)
 
 
-def test_set_from_exportdata(mock_volumes, globalconfig2):
+def test_set_from_exportdata(mock_volumes, globalconfig2, monkeypatch, tmp_path):
     """Test setting of index from class ExportData
 
     Args:
         mock_volumes (pd.DataFrame): a volumetric like dataframe
         globalconfig2 (dict): one global variables dict
     """
+    monkeypatch.chdir(tmp_path)
     index = ["OTHER"]
     exd = ExportData(config=globalconfig2, table_index=index, content="timeseries")
     path = exd.export(mock_volumes, name="baretull")
     assert_correct_table_index(path, index)
 
 
-def test_set_from_export(mock_volumes, globalconfig2):
+def test_set_from_export(mock_volumes, globalconfig2, monkeypatch, tmp_path):
     """Test setting of index from method export on class ExportData
 
     Args:
         mock_volumes (pd.DataFrame): volumetric like data
         globalconfig2 (dict): one global variable dict
     """
+    monkeypatch.chdir(tmp_path)
     index = ["OTHER"]
     exd = ExportData(config=globalconfig2, content="timeseries")
     path = exd.export(mock_volumes, name="baretull", table_index=index)
     assert_correct_table_index(path, index)
 
 
-def test_set_table_index_not_in_table(mock_volumes, globalconfig2):
+def test_set_table_index_not_in_table(
+    mock_volumes, globalconfig2, monkeypatch, tmp_path
+):
     """Test when setting index with something that is not in data
 
     Args:
         mock_volumes (pd.DataFrame): volumetric like data
         globalconfig2 (dict): one global variables dict
     """
+    monkeypatch.chdir(tmp_path)
     index = ["banana"]
     exd = ExportData(config=globalconfig2, content="timeseries")
     with pytest.raises(KeyError) as k_err: