diff --git a/.gitignore b/.gitignore index c306fe326..54eb60275 100644 --- a/.gitignore +++ b/.gitignore @@ -98,6 +98,8 @@ venv.bak/ src/fmu/dataio/version.py # aggregated examples directory examples/s/d/nn/xcase/iter-0/ +# exported results directory +share/ # mypy .dmypy.json diff --git a/share/results/maps/mysurface.gri b/share/results/maps/mysurface.gri deleted file mode 100644 index 5e6eba26a..000000000 Binary files a/share/results/maps/mysurface.gri and /dev/null differ diff --git a/src/fmu/dataio/dataio.py b/src/fmu/dataio/dataio.py index 7c617daa4..889b9a04f 100644 --- a/src/fmu/dataio/dataio.py +++ b/src/fmu/dataio/dataio.py @@ -387,12 +387,12 @@ class ExportData: allow_forcefolder_absolute: ClassVar[bool] = False arrow_fformat: ClassVar[str] = "arrow" case_folder: ClassVar[str] = "share/metadata" - createfolder: ClassVar[bool] = True + createfolder: ClassVar[bool] = True # deprecated cube_fformat: ClassVar[str] = "segy" filename_timedata_reverse: ClassVar[bool] = False # reverse order output file name grid_fformat: ClassVar[str] = "roff" include_ertjobs: ClassVar[bool] = False # if True, include jobs.json from ERT - legacy_time_format: ClassVar[bool] = False + legacy_time_format: ClassVar[bool] = False # deprecated meta_format: ClassVar[Literal["yaml", "json"]] = "yaml" polygons_fformat: ClassVar[str] = "csv" # or use "csv|xtgeo" points_fformat: ClassVar[str] = "csv" # or use "csv|xtgeo" @@ -400,7 +400,7 @@ class ExportData: table_fformat: ClassVar[str] = "csv" dict_fformat: ClassVar[str] = "json" table_include_index: ClassVar[bool] = False - verifyfolder: ClassVar[bool] = True + verifyfolder: ClassVar[bool] = True # deprecated _inside_rms: ClassVar[bool] = False # developer only! if True pretend inside RMS # input keys (alphabetic) @@ -564,6 +564,26 @@ def _show_deprecations_or_notimplemented(self) -> None: ) self.access_ssdl["access_level"] = "restricted" + if self.legacy_time_format: + warn( + "Using the 'legacy_time_format=True' option to create metadata files " + "with the old format for time is now deprecated. This option has no " + "longer an effect and will be removed in the near future.", + UserWarning, + ) + if not self.createfolder: + warn( + "Using the 'createfolder=False' option is now deprecated. " + "This option has no longer an effect and can safely be removed", + UserWarning, + ) + if not self.verifyfolder: + warn( + "Using the 'verifyfolder=False' option to create metadata files " + "This option has no longer an effect and can safely be removed", + UserWarning, + ) + def _validate_content_key(self) -> None: """Validate the given 'content' input.""" self._usecontent, self._content_specific = _check_content(self.content) @@ -726,13 +746,7 @@ def generate_metadata( empty. If true, the MD5 checksum will be generated based on export to a temporary file, which may be time-consuming if the file is large. """ - if self.legacy_time_format: - warn( - "Using the 'legacy_time_format=True' option to create metadata files " - "with the old format for time is now deprecated. This option has no " - "longer an effect and will be removed in the near future.", - UserWarning, - ) + logger.info("Generate metadata...") logger.info("KW args %s", kwargs) @@ -798,6 +812,8 @@ def export( logger.info("Object type is: %s", type(self._object)) # from generate_metadata outfile = Path(metadata["file"]["absolute_path"]) + # create output folders if they don't exist + outfile.parent.mkdir(parents=True, exist_ok=True) metafile = outfile.parent / ("." + str(outfile.name) + ".yml") useflag = ( @@ -827,8 +843,8 @@ def export( outfile_target = None if metadata["file"].get("absolute_path_symlink"): outfile_target = Path(metadata["file"]["absolute_path_symlink"]) - outfile_source = Path(metadata["file"]["absolute_path"]) - create_symlink(str(outfile_source), str(outfile_target)) + outfile_target.parent.mkdir(parents=True, exist_ok=True) + create_symlink(str(outfile), str(outfile_target)) metafile_target = outfile_target.parent / ("." + str(outfile.name) + ".yml") create_symlink(str(metafile), str(metafile_target)) diff --git a/src/fmu/dataio/providers/_filedata.py b/src/fmu/dataio/providers/_filedata.py index 70343c1c5..663eca20a 100644 --- a/src/fmu/dataio/providers/_filedata.py +++ b/src/fmu/dataio/providers/_filedata.py @@ -235,14 +235,4 @@ def _get_path_generic( f"You cannot use forcefolder in combination with fmucontext={info}" ) - if self.dataio.subfolder: - dest = dest / self.dataio.subfolder - - if self.dataio.createfolder: - dest.mkdir(parents=True, exist_ok=True) - - # check that destination actually exists if verifyfolder is True - if self.dataio.verifyfolder and not dest.exists(): - raise OSError(f"Folder {str(dest)} is not present.") - - return dest + return dest if not self.dataio.subfolder else dest / self.dataio.subfolder diff --git a/tests/conftest.py b/tests/conftest.py index 58ee63707..3246ceaf9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -289,8 +289,6 @@ def fixture_edataobj1(globalconfig1): is_observation=False, ) eobj.surface_fformat = "irap_binary" - eobj.createfolder = False - eobj.verifyfolder = False logger.debug( "Ran %s returning %s", inspect.currentframe().f_code.co_name, type(eobj) @@ -324,8 +322,6 @@ def fixture_edataobj2(globalconfig2): ) eobj.surface_fformat = "irap_binary" - eobj.createfolder = False - eobj.verifyfolder = False eobj.legacy_time_format = False eobj._rootpath = Path(".") diff --git a/tests/test_units/test_dataio.py b/tests/test_units/test_dataio.py index 857df1db6..2d8a592ca 100644 --- a/tests/test_units/test_dataio.py +++ b/tests/test_units/test_dataio.py @@ -49,9 +49,11 @@ def test_generate_metadata_simple(globalconfig1): ExportData.grid_fformat = default_fformat # reset -def test_missing_or_wrong_config_exports_with_warning(regsurf): +def test_missing_or_wrong_config_exports_with_warning(monkeypatch, tmp_path, regsurf): """In case a config is missing, or is invalid, do export with warning.""" + monkeypatch.chdir(tmp_path) + with pytest.warns(UserWarning, match=pydantic_warning()): edata = ExportData(config={}, content="depth") @@ -69,9 +71,11 @@ def test_missing_or_wrong_config_exports_with_warning(regsurf): read_metadata(out) -def test_config_miss_required_fields(globalconfig1, regsurf): +def test_config_miss_required_fields(monkeypatch, tmp_path, globalconfig1, regsurf): """Global config exists but missing critical data; export file but skip metadata.""" + monkeypatch.chdir(tmp_path) + cfg = globalconfig1.copy() del cfg["access"] diff --git a/tests/test_units/test_dictionary.py b/tests/test_units/test_dictionary.py index aa84e4d49..133d3f594 100644 --- a/tests/test_units/test_dictionary.py +++ b/tests/test_units/test_dictionary.py @@ -105,7 +105,7 @@ def read_dict_and_meta(path): ("nested_parameters"), ], ) -def test_export_dict_w_meta(globalconfig2, dictionary, request): +def test_export_dict_w_meta(globalconfig2, dictionary, request, monkeypatch, tmp_path): """Test various dictionaries Args: @@ -113,6 +113,7 @@ def test_export_dict_w_meta(globalconfig2, dictionary, request): dictionary (str): name of fixture to use request (pytest.fixture): fixture for using fixtures in parameterize """ + monkeypatch.chdir(tmp_path) name = dictionary in_dict = request.getfixturevalue(dictionary) print(f"{name}: {in_dict}") @@ -122,13 +123,16 @@ def test_export_dict_w_meta(globalconfig2, dictionary, request): assert_dict_correct(out_dict, out_meta, name) -def test_invalid_dict(globalconfig2, drogon_summary, drogon_volumes): +def test_invalid_dict( + globalconfig2, drogon_summary, drogon_volumes, monkeypatch, tmp_path +): """Test raising of error when dictionary is not serializable Args: globalconfig2 (_type_): _description_ drogon_summary (pd.DataFrame): a dataframe drogon_volumes (pa.Table): a pyarrow table """ + monkeypatch.chdir(tmp_path) in_dict = {"volumes": drogon_volumes, "summary": drogon_summary} exd = ExportData(config=globalconfig2, content="parameters") with pytest.raises(TypeError) as exc_info: diff --git a/tests/test_units/test_filedataprovider_class.py b/tests/test_units/test_filedataprovider_class.py index 3a55defe9..557b9ab7b 100644 --- a/tests/test_units/test_filedataprovider_class.py +++ b/tests/test_units/test_filedataprovider_class.py @@ -198,7 +198,6 @@ def test_get_paths_not_exists_so_create(regsurf, edataobj1, tmp_path): objdata.efolder = "efolder" cfg = edataobj1 - cfg.createfolder = True cfg._rootpath = Path(".") fdata = FileDataProvider(cfg, objdata) @@ -213,7 +212,6 @@ def test_filedata_provider(regsurf, edataobj1, tmp_path): os.chdir(tmp_path) cfg = edataobj1 - cfg.createfolder = True cfg._rootpath = Path(".") cfg.name = "" @@ -242,7 +240,6 @@ def test_filedata_has_nonascii_letters(regsurf, edataobj1, tmp_path): os.chdir(tmp_path) cfg = edataobj1 - cfg.createfolder = True cfg._rootpath = Path(".") cfg.name = "mynõme" diff --git a/tests/test_units/test_table.py b/tests/test_units/test_table.py index 12c923a99..31fb86355 100644 --- a/tests/test_units/test_table.py +++ b/tests/test_units/test_table.py @@ -49,33 +49,39 @@ def assert_correct_table_index(dict_input, answer): assert_list_and_answer(index, answer, index) -def test_inplace_volume_index(mock_volumes, globalconfig2): +def test_inplace_volume_index(mock_volumes, globalconfig2, monkeypatch, tmp_path): """Test volumetric data Args: mock_volumes (pd.DataFrame): a volumetriclike dataset globalconfig2 (dict): one global variables dict """ + monkeypatch.chdir(tmp_path) answer = ["ZONE", "LICENCE"] exd = ExportData(config=globalconfig2, content="volumes") path = exd.export(mock_volumes, name="baretull") assert_correct_table_index(path, answer) -def test_derive_summary_index_pandas(mock_summary, globalconfig2): +def test_derive_summary_index_pandas( + mock_summary, globalconfig2, monkeypatch, tmp_path +): """Test summary data Args: mock_summary (pd.DataFrame): summary "like" dataframe globalconfig2 (dict): global variables dict """ + monkeypatch.chdir(tmp_path) answer = ["DATE"] exd = ExportData(config=globalconfig2, content="timeseries") path = exd.export(mock_summary, name="baretull") assert_correct_table_index(path, answer) -def test_derive_summary_index_pyarrow(mock_summary, globalconfig2): +def test_derive_summary_index_pyarrow( + mock_summary, globalconfig2, monkeypatch, tmp_path +): """Test summary data Args: @@ -84,45 +90,51 @@ def test_derive_summary_index_pyarrow(mock_summary, globalconfig2): """ from pyarrow import Table + monkeypatch.chdir(tmp_path) answer = ["DATE"] exd = ExportData(config=globalconfig2, content="timeseries") path = exd.export(Table.from_pandas(mock_summary), name="baretull") assert_correct_table_index(path, answer) -def test_set_from_exportdata(mock_volumes, globalconfig2): +def test_set_from_exportdata(mock_volumes, globalconfig2, monkeypatch, tmp_path): """Test setting of index from class ExportData Args: mock_volumes (pd.DataFrame): a volumetric like dataframe globalconfig2 (dict): one global variables dict """ + monkeypatch.chdir(tmp_path) index = ["OTHER"] exd = ExportData(config=globalconfig2, table_index=index, content="timeseries") path = exd.export(mock_volumes, name="baretull") assert_correct_table_index(path, index) -def test_set_from_export(mock_volumes, globalconfig2): +def test_set_from_export(mock_volumes, globalconfig2, monkeypatch, tmp_path): """Test setting of index from method export on class ExportData Args: mock_volumes (pd.DataFrame): volumetric like data globalconfig2 (dict): one global variable dict """ + monkeypatch.chdir(tmp_path) index = ["OTHER"] exd = ExportData(config=globalconfig2, content="timeseries") path = exd.export(mock_volumes, name="baretull", table_index=index) assert_correct_table_index(path, index) -def test_set_table_index_not_in_table(mock_volumes, globalconfig2): +def test_set_table_index_not_in_table( + mock_volumes, globalconfig2, monkeypatch, tmp_path +): """Test when setting index with something that is not in data Args: mock_volumes (pd.DataFrame): volumetric like data globalconfig2 (dict): one global variables dict """ + monkeypatch.chdir(tmp_path) index = ["banana"] exd = ExportData(config=globalconfig2, content="timeseries") with pytest.raises(KeyError) as k_err: