Skip to content

Commit

Permalink
Remove extension argument
Browse files Browse the repository at this point in the history
  • Loading branch information
JB Lovland committed Jan 10, 2024
1 parent 2c13a20 commit a894ab1
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 20 deletions.
8 changes: 6 additions & 2 deletions src/fmu/dataio/_metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -306,12 +306,16 @@ def _populate_meta_file(self) -> None:
self.meta_file["absolute_path_symlink"] = fdata.absolute_path_symlink

if self.compute_md5:
with NamedTemporaryFile(buffering=0) as tf:
if not self.objdata.extension.startswith("."):
raise ValueError("A extension must start with '.'")
with NamedTemporaryFile(
buffering=0,
suffix=self.objdata.extension,
) as tf:
logger.info("Compute MD5 sum for tmp file...: %s", tf.name)
self.meta_file["checksum_md5"] = export_file_compute_checksum_md5(
self.obj,
Path(tf.name),
self.objdata.extension,
flag=self.dataio._usefmtflag,
)
else:
Expand Down
29 changes: 13 additions & 16 deletions src/fmu/dataio/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,17 +112,16 @@ def export_metadata_file(
def export_file(
obj: object,
filename: Path,
extension: str,
flag: str | None = None,
) -> str:
"""Export a valid object to file"""

if isinstance(obj, Path):
# special case when processing data which already has metadata
shutil.copy(obj, filename)
elif extension == ".gri" and isinstance(obj, xtgeo.RegularSurface):
elif filename.suffix == ".gri" and isinstance(obj, xtgeo.RegularSurface):
obj.to_file(filename, fformat="irap_binary")
elif extension == ".csv" and isinstance(obj, (xtgeo.Polygons, xtgeo.Points)):
elif filename.suffix == ".csv" and isinstance(obj, (xtgeo.Polygons, xtgeo.Points)):
out = obj.copy() # to not modify incoming instance!
assert flag is not None
if "xtgeo" not in flag:
Expand All @@ -133,16 +132,18 @@ def export_file(
# out.pname = "ID" not working
out.dataframe.rename(columns={out.pname: "ID"}, inplace=True)
out.dataframe.to_csv(filename, index=False)
elif extension == ".pol" and isinstance(obj, (xtgeo.Polygons, xtgeo.Points)):
elif filename.suffix == ".pol" and isinstance(obj, (xtgeo.Polygons, xtgeo.Points)):
obj.to_file(filename)
elif extension == ".segy" and isinstance(obj, xtgeo.Cube):
elif filename.suffix == ".segy" and isinstance(obj, xtgeo.Cube):
obj.to_file(filename, fformat="segy")
elif extension == ".roff" and isinstance(obj, (xtgeo.Grid, xtgeo.GridProperty)):
elif filename.suffix == ".roff" and isinstance(
obj, (xtgeo.Grid, xtgeo.GridProperty)
):
obj.to_file(filename, fformat="roff")
elif extension == ".csv" and isinstance(obj, pd.DataFrame):
elif filename.suffix == ".csv" and isinstance(obj, pd.DataFrame):
includeindex = flag == "include_index"
obj.to_csv(filename, index=includeindex)
elif extension == ".arrow" and HAS_PYARROW and isinstance(obj, pa.Table):
elif filename.suffix == ".arrow" and HAS_PYARROW and isinstance(obj, pa.Table):
# comment taken from equinor/webviz_subsurface/smry2arrow.py

# Writing here is done through the feather import, but could also be done using
Expand All @@ -151,31 +152,27 @@ def export_file(
# actual file format is the same
# (https://arrow.apache.org/docs/python/feather.html)
feather.write_feather(obj, dest=filename)
elif extension == ".json":
elif filename.suffix == ".json":
with open(filename, "w") as stream:
json.dump(obj, stream)
else:
raise TypeError(f"Exporting {extension} for {type(obj)} is not supported")
raise TypeError(f"Exporting {filename.suffix} for {type(obj)} is not supported")

return str(filename)


def md5sum(fname: Path) -> str:
hash_md5 = hashlib.md5()
with open(fname, "rb") as fil:
for chunk in iter(lambda: fil.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
return hashlib.md5(fil.read()).hexdigest()


def export_file_compute_checksum_md5(
obj: object,
filename: Path,
extension: str,
flag: str | None = None,
) -> str:
"""Export and compute checksum"""
export_file(obj, filename, extension, flag=flag)
export_file(obj, filename, flag=flag)
return md5sum(filename)


Expand Down
3 changes: 1 addition & 2 deletions src/fmu/dataio/dataio.py
Original file line number Diff line number Diff line change
Expand Up @@ -910,7 +910,6 @@ def export(
metadata["file"]["checksum_md5"] = export_file_compute_checksum_md5(
obj,
outfile,
outfile.suffix,
flag=useflag, # type: ignore
# BUG(?): Looks buggy, if flag is bool export_file will blow up.
)
Expand Down Expand Up @@ -1510,7 +1509,7 @@ def export(self, obj: object, **kwargs: object) -> str:
logger.info("Export to file and compute MD5 sum")
# inject the computed md5 checksum in metadata
metadata["file"]["checksum_md5"] = export_file_compute_checksum_md5(
obj, outfile, outfile.suffix
obj, outfile
)

export_metadata_file(metafile, metadata, savefmt=self.meta_format)
Expand Down

0 comments on commit a894ab1

Please sign in to comment.