Update codebase to py38 (#404)

examples/s/d/nn/_project/aggregate_surfaces.py

@@ -1,12 +1,12 @@
 """Use fmu-dataio for aggregated surfaces created by an aggregation service."""
 
-from pathlib import Path
 import logging
+from pathlib import Path
 
-import yaml
 import numpy as np
-
 import xtgeo
+import yaml
+
 import fmu.dataio
 
 
@@ -67,7 +67,6 @@ def main():
     )
 
     for operation in operations:
-
         print(f"Running aggregation: {operation}")
 
         # Call the aggregation machine and create an aggregated surface
@@ -128,7 +127,7 @@ def _parse_yaml(fname):
         dict
     """
 
-    with open(fname, "r") as stream:
+    with open(fname) as stream:
         data = yaml.safe_load(stream)
     return data
 

examples/s/d/nn/xcase/realization-0/iter-0/any/bin/export_volumetables.py

@@ -6,10 +6,12 @@
 For the file case, CSV files are read from disk. The dataio function is the same.
 """
 import pathlib
+
 import pandas as pd
-import fmu.dataio
 from fmu.config import utilities as ut
 
+import fmu.dataio
+
 CFG = ut.yaml_load("../../fmuconfig/output/global_variables.yml")
 
 IN_ROXAR = False

examples/s/d/nn/xcase/realization-0/iter-0/rms/bin/export_faultpolygons.py

@@ -34,7 +34,6 @@ def export_faultlines():
     )
 
     for hname in HORISONNAMES:
-
         # RMS version for reading polygons from a project:
         # poly = xtgeo.polygons_from_roxar(project, hname, RMS_POL_CATEGORY)
 

src/fmu/dataio/__init__.py

@@ -4,7 +4,6 @@
 from fmu.dataio.dataio import AggregatedData  # noqa  # type: ignore
 from fmu.dataio.dataio import ExportData  # noqa  # type: ignore
 from fmu.dataio.dataio import InitializeCase  # noqa  # type: ignore
-
 from fmu.dataio.dataio import read_metadata  # noqa
 
 try:

src/fmu/dataio/_design_kw.py

@@ -7,9 +7,8 @@
 
 # pylint: disable=logging-fstring-interpolation
 import logging
-import shlex
 import re
-
+import shlex
 
 _STATUS_FILE_NAME = "DESIGN_KW.OK"
 
@@ -38,7 +37,7 @@ def run(
 
     key_vals.update(rm_genkw_prefix(key_vals))
 
-    with open(template_file_name, "r") as template_file:
+    with open(template_file_name) as template_file:
         template = template_file.readlines()
 
     if valid:
@@ -63,10 +62,8 @@ def all_matched(line, template_file_name, template):
     for unmatched in unmatched_templates(line):
         if is_perl(template_file_name, template):
             _logger.warning(  # pylint: disable=logging-fstring-interpolation
-                (
-                    f"{unmatched} not found in design matrix, "
-                    f"but this is probably a Perl file"
-                )
+                f"{unmatched} not found in design matrix, "
+                f"but this is probably a Perl file"
             )
         else:
             _logger.error(  # pylint: disable=logging-fstring-interpolation

src/fmu/dataio/_filedata_provider.py

@@ -232,6 +232,6 @@ def _get_path_generic(self, mode="realization", allow_forcefolder=True, info="")
 
         # check that destination actually exists if verifyfolder is True
         if self.dataio.verifyfolder and not dest.exists():
-            raise IOError(f"Folder {str(dest)} is not present.")
+            raise OSError(f"Folder {str(dest)} is not present.")
 
         return dest

src/fmu/dataio/_fmu_provider.py

@@ -235,7 +235,7 @@ def get_ert2_information(self):
         if self.dataio.include_ert2jobs:
             jobs_file = self.iter_path / "jobs.json"
             if jobs_file.is_file():
-                with open(jobs_file, "r") as stream:
+                with open(jobs_file) as stream:
                     self.ert2["jobs"] = json.load(stream)
                 logger.debug("jobs.json parsed.")
             logger.debug("jobs.json was not found")

src/fmu/dataio/_metadata.py

@@ -5,10 +5,10 @@
 # https://realpython.com/python-data-classes/#basic-data-classes
 
 import datetime
-from datetime import timezone
 import getpass
 import logging
 from dataclasses import dataclass, field
+from datetime import timezone
 from pathlib import Path
 from typing import Any, Optional
 from warnings import warn

src/fmu/dataio/_oyaml.py

@@ -31,10 +31,6 @@ def map_constructor(loader, node):
 pyyaml.add_representer(OrderedDict, map_representer, Dumper=DangerDumper)
 
 
-if sys.version_info < (3, 7):
-    pyyaml.add_constructor("tag:yaml.org,2002:map", map_constructor)
-
-
 del map_constructor, map_representer
 
 # cf. stackoverflow.com/questions/21695705/dump-an-python-object-as-yaml-file/51261042

src/fmu/dataio/_utils.py

@@ -187,17 +187,17 @@ def create_symlink(source, target):
 
     thesource = Path(source)
     if not thesource.exists():
-        raise IOError(f"Cannot symlink: Source file {thesource} does not exist.")
+        raise OSError(f"Cannot symlink: Source file {thesource} does not exist.")
 
     thetarget = Path(target)
 
     if thetarget.exists() and not thetarget.is_symlink():
-        raise IOError(f"Target file {thetarget} exists already as a normal file.")
+        raise OSError(f"Target file {thetarget} exists already as a normal file.")
 
     os.symlink(source, target)
 
     if not (thetarget.exists() and thetarget.is_symlink()):
-        raise IOError(f"Target file {thesource} does not exist or is not a symlink.")
+        raise OSError(f"Target file {thesource} does not exist or is not a symlink.")
 
 
 def size(fname):
@@ -421,13 +421,13 @@ def read_metadata(filename: Union[str, Path]) -> dict:
     """
     fname = Path(filename)
     if fname.stem.startswith("."):
-        raise IOError(f"The input is a hidden file, cannot continue: {fname.stem}")
+        raise OSError(f"The input is a hidden file, cannot continue: {fname.stem}")
 
     metafile = str(fname.parent) + "/." + fname.stem + fname.suffix + ".yml"
     metafilepath = Path(metafile)
     if not metafilepath.exists():
-        raise IOError(f"Cannot find requested metafile: {metafile}")
-    with open(metafilepath, "r") as stream:
+        raise OSError(f"Cannot find requested metafile: {metafile}")
+    with open(metafilepath) as stream:
         metacfg = yaml.safe_load(stream)
 
     return metacfg

src/fmu/dataio/dataio.py

@@ -1473,7 +1473,7 @@ def export(self, obj, **kwargs) -> str:
         abspath = metadata["file"].get("absolute_path", None)
 
         if not abspath:
-            raise IOError(
+            raise OSError(
                 "The absolute_path is None, hence no export is possible. "
                 "Use the ``casepath`` key to provide a valid absolute path."
             )

src/fmu/dataio/scripts/create_case_metadata.py

@@ -8,11 +8,12 @@
 pointed towards the produced global_variables, fmu-config should run
 before this script to make sure global_variables is updated."""
 
-import yaml
 import argparse
 import logging
 from pathlib import Path
 
+import yaml
+
 try:
     from ert.shared.plugins.plugin_manager import hook_implementation  # type: ignore
 except ModuleNotFoundError:
@@ -124,7 +125,7 @@ def register_on_sumo(args, case_metadata_path) -> str:
         return
 
     # lazy loading of Sumo dependencies
-    from fmu.sumo.uploader import SumoConnection, CaseOnDisk
+    from fmu.sumo.uploader import CaseOnDisk, SumoConnection
 
     # establish connection
     sumo_conn = SumoConnection(env=env)
@@ -144,7 +145,7 @@ def register_on_sumo(args, case_metadata_path) -> str:
 def _parse_yaml(path):
     """Parse the global variables, return as dict"""
 
-    with open(path, "r") as stream:
+    with open(path) as stream:
         data = yaml.safe_load(stream)
 
     return data

tests/conftest.py

@@ -1,21 +1,20 @@
 """The conftest.py, providing magical fixtures to tests."""
+import datetime
 import inspect
+import json
 import logging
 import os
 import shutil
 from functools import wraps
 from pathlib import Path
-import json
-import yaml
-import datetime
 
 import pandas as pd
 import pytest
 import xtgeo
+import yaml
 from fmu.config import utilities as ut
 from termcolor import cprint
 
-
 try:
     import pyarrow as pa
 except ImportError:
@@ -147,7 +146,7 @@ def fixture_rmsglobalconfig(rmssetup):
     # read the global config
     os.chdir(rmssetup)
     logger.info("Global config is %s", str(rmssetup / "global_variables.yml"))
-    with open("global_variables.yml", "r", encoding="utf8") as stream:
+    with open("global_variables.yml", encoding="utf8") as stream:
         global_cfg = yaml.safe_load(stream)
 
     logger.info("Ran setup for %s", "rmsglobalconfig")
@@ -284,7 +283,6 @@ def fixture_globalconfig2() -> dict:
     globvar = {}
     with open(
         ROOTPWD / "tests/data/drogon/global_config2/global_variables.yml",
-        "r",
         encoding="utf-8",
     ) as stream:
         globvar = yaml.safe_load(stream)
@@ -576,15 +574,15 @@ def fixture_drogon_volumes():
 
 def _parse_json(schema_path):
     """Parse the schema, return JSON"""
-    with open(schema_path, "r", encoding="utf-8") as stream:
+    with open(schema_path, encoding="utf-8") as stream:
         data = json.load(stream)
 
     return data
 
 
 def _parse_yaml(yaml_path):
     """Parse the filename as json, return data"""
-    with open(yaml_path, "r", encoding="utf-8") as stream:
+    with open(yaml_path, encoding="utf-8") as stream:
         data = yaml.safe_load(stream)
 
     data = _isoformat_all_datetimes(data)

tests/test_schema/test_schema_logic.py

@@ -3,7 +3,6 @@
 from copy import deepcopy
 
 import jsonschema
-
 import pytest
 
 from fmu.dataio._definitions import ALLOWED_CONTENTS

tests/test_units/test_dataio.py

@@ -3,7 +3,6 @@
 import os
 import pathlib
 import sys
-
 from copy import deepcopy
 
 import pytest
@@ -426,7 +425,7 @@ def test_norwegian_letters_globalconfig(globalvars_norw_letters, regsurf):
     # export to file and reread as raw
     result = pathlib.Path(edata.export(regsurf))
     metafile = result.parent / ("." + str(result.stem) + ".gri.yml")
-    with open(metafile, "r", encoding="utf-8") as stream:
+    with open(metafile, encoding="utf-8") as stream:
         stuff = stream.read()
     assert "DRÅGØN" in stuff
 
@@ -453,7 +452,7 @@ def test_norwegian_letters_globalconfig_as_json(globalvars_norw_letters, regsurf
 
     result = pathlib.Path(edata.export(regsurf))
     metafile = result.parent / ("." + str(result.stem) + ".gri.json")
-    with open(metafile, "r", encoding="utf-8") as stream:
+    with open(metafile, encoding="utf-8") as stream:
         stuff = stream.read()
     assert "DRÅGØN" in stuff
 

tests/test_units/test_dictionary.py

@@ -1,11 +1,13 @@
 """Test dictionary functionality"""
-import os
 import json
+import os
 from pathlib import Path
-import yaml
+
 import pytest
+import yaml
+
 from fmu.dataio import ExportData
-from fmu.dataio._utils import read_parameters_txt, nested_parameters_dict
+from fmu.dataio._utils import nested_parameters_dict, read_parameters_txt
 
 
 @pytest.fixture(name="direct_creation", scope="function")
@@ -31,9 +33,7 @@ def _fixture_json(fmurun_w_casemetadata):
     """
     os.chdir(fmurun_w_casemetadata)
     print(fmurun_w_casemetadata)
-    with open(
-        fmurun_w_casemetadata / "parameters.json", "r", encoding="utf-8"
-    ) as stream:
+    with open(fmurun_w_casemetadata / "parameters.json", encoding="utf-8") as stream:
         json_dict = json.load(stream)
     return json_dict
 
@@ -89,10 +89,10 @@ def read_dict_and_meta(path):
         tuple: the dictionary produced with corresponding metadata
     """
     result_dict = None
-    with open(path, "r", encoding="utf-8") as stream:
+    with open(path, encoding="utf-8") as stream:
         result_dict = json.load(stream)
     path = Path(path)
-    with open(path.parent / f".{path.name}.yml", "r", encoding="utf-8") as meta_stream:
+    with open(path.parent / f".{path.name}.yml", encoding="utf-8") as meta_stream:
         meta = yaml.load(meta_stream, Loader=yaml.Loader)
     return result_dict, meta
 

tests/test_units/test_fmuprovider_class.py

@@ -4,7 +4,7 @@
 import pytest
 
 import fmu.dataio as dio
-from fmu.dataio._fmu_provider import _FmuProvider, _get_folderlist, RESTART_PATH_ENVNAME
+from fmu.dataio._fmu_provider import RESTART_PATH_ENVNAME, _FmuProvider, _get_folderlist
 
 FOLDERTREE = "scratch/myfield/case/realization-13/iter-2"
 

tests/test_units/test_initialize_case.py

@@ -39,7 +39,7 @@ def test_inicase_barebone_with_export(globalconfig2, fmurun):
     casemetafile = caseroot / "share/metadata/fmu_case.yml"
 
     # check that special characters made it through
-    with open(casemetafile, "r") as stream:
+    with open(casemetafile) as stream:
         metadata = yaml.safe_load(stream)
 
     assert metadata["fmu"]["case"]["name"] == "MyCaseName_with_Æ"
@@ -48,7 +48,7 @@ def test_inicase_barebone_with_export(globalconfig2, fmurun):
     # Check that special characters are encoded properly in stored metadatafile.
     # yaml.safe_load() seems to sort this out, but we want files on disk to be readable.
     # Therefore check by reading the raw file content.
-    with open(casemetafile, "r") as stream:
+    with open(casemetafile) as stream:
         metadata_string = stream.read()
 
     assert "æøå" in metadata_string
@@ -161,7 +161,7 @@ def test_inicase_generate_case_metadata_exists_but_force(
     casemetafolder = fmurun_w_casemetadata.parent.parent
     old_metafile = casemetafolder / "share/metadata/fmu_case.yml"
 
-    with open(old_metafile, "r", encoding="utf-8") as stream:
+    with open(old_metafile, encoding="utf-8") as stream:
         old_content = yaml.safe_load(stream)
 
     icase = InitializeCase(globalconfig2, verbosity="INFO")
@@ -174,7 +174,7 @@ def test_inicase_generate_case_metadata_exists_but_force(
     )
 
     new_metafile = casemetafolder / "share/metadata/fmu_case.yml"
-    with open(new_metafile, "r", encoding="utf-8") as stream:
+    with open(new_metafile, encoding="utf-8") as stream:
         new_content = yaml.safe_load(stream)
 
     logger.debug("\n%s\n", prettyprint_dict(old_content))