read data on access only

metno · Nov 7, 2024 · 5c86d2b · 5c86d2b
1 parent 4e07711
commit 5c86d2b
Show file tree

Hide file tree

Showing 9 changed files with 22 additions and 20 deletions.
diff --git a/src/pyaro_readers/ascii2netcdf/Ascii2NetcdfTimeseries.py b/src/pyaro_readers/ascii2netcdf/Ascii2NetcdfTimeseries.py
@@ -73,7 +73,7 @@ def __init__(
 
         self._metadata = self.metadata()
 
-    def read(self):
+    # def read(self):
         self._variables = self._read_file_variables()
         station_file = "StationList.csv"
         station_filepath = os.path.join(self._directory, station_file)
@@ -287,7 +287,3 @@ def description(self) -> str:
 
     def url(self):
         return "https://github.com/metno/pyaro-readers"
-
-    def read(self):
-        return self.reader_class().read()
-
diff --git a/src/pyaro_readers/harpreader/harpreader.py b/src/pyaro_readers/harpreader/harpreader.py
@@ -75,7 +75,9 @@ def __init__(
 
     def read(self):
         """reading method"""
-
+        # check if the data has been read already
+        if len(self._data) != 0:
+            return
         bar = tqdm(total=len(self._files), disable=None)
 
         for f_idx, _file in enumerate(self._files):
@@ -217,12 +219,15 @@ def _unfiltered_variables(self) -> list[str]:
         list[str]
             The list of variable names.
         """
-        return self._data.keys()
+        self.read()
+        return list(self._data.keys())
 
     def _unfiltered_data(self, varname) -> Data:
+        self.read()
         return self._data[varname]
 
     def _unfiltered_stations(self) -> dict[str, Station]:
+        self.read()
         return self._stations
 
     def close(self):

diff --git a/src/pyaro_readers/netcdf_rw/Netcdf_RWTimeseries.py b/src/pyaro_readers/netcdf_rw/Netcdf_RWTimeseries.py
@@ -45,6 +45,8 @@ def __init__(
     ):
         self._set_filters(filters)
         self._mode = mode
+        self._metadata = self.metadata()
+
         if os.path.isdir(filename):
             self._directory = filename
         else:
@@ -55,9 +57,6 @@ def __init__(
             else:
                 os.path.makedirs(filename)
 
-    def read(self):
-        """standard read method"""
-
         dataglob = os.path.join(self._directory, f"{self.ncfile_prefix}.????.nc")
         self._years = set()
         for file in glob.iglob(dataglob):

diff --git a/src/pyaro_readers/nilupmfabsorptionreader/NILUPMFAbsorptionReader.py b/src/pyaro_readers/nilupmfabsorptionreader/NILUPMFAbsorptionReader.py
@@ -80,7 +80,9 @@ def __init__(
 
     def read(self):
         """ read method"""
-
+        # check if the data has been read already
+        if len(self._data) != 0:
+            return
         if Path(self._filename).is_file():
             self._process_file(self._filename, self._fill_country_flag)
 
@@ -200,12 +202,15 @@ def metadata(self):
         # return metadata
 
     def _unfiltered_data(self, varname) -> Data:
+        self.read()
         return self._data[varname]
 
     def _unfiltered_stations(self) -> dict[str, Station]:
+        self.read()
         return self._stations
 
     def _unfiltered_variables(self) -> list[str]:
+        self.read()
         return list(self._data.keys())
 
     def close(self):

diff --git a/src/pyaro_readers/nilupmfebas/EbasPmfReader.py b/src/pyaro_readers/nilupmfebas/EbasPmfReader.py
@@ -64,6 +64,9 @@ def __init__(
     def read(self):
         """read method"""
 
+        # check if the data has been read already
+        if len(self._data) != 0:
+            return
         if Path(self._realpath).is_dir():
             # search directory for files
             files = list(self._realpath.glob(self._filemask))
@@ -221,12 +224,15 @@ def read_file(self, filename: Path | str, vars_to_read: list[str] = None):
                 )
 
     def _unfiltered_data(self, varname) -> Data:
+        self.read()
         return self._data[varname]
 
     def _unfiltered_stations(self) -> dict[str, Station]:
+        self.read()
         return self._stations
 
     def _unfiltered_variables(self) -> list[str]:
+        self.read()
         return list(self._data.keys())
 
     def close(self):

diff --git a/tests/test_Ascii2NetcdfTimeseries.py b/tests/test_Ascii2NetcdfTimeseries.py
@@ -20,7 +20,6 @@ def test_1open(self):
         with pyaro.open_timeseries(
             self.engine, EBAS_URL, resolution="daily", filters=[]
         ) as ts:
-            ts.read()
             self.assertGreater(len(ts.variables()), 70)
             self.assertGreater(len(ts.stations()), 300)
 
@@ -30,7 +29,6 @@ def test_2read(self):
         with pyaro.open_timeseries(
             self.engine, EBAS_URL, resolution="daily", filters=[]
         ) as ts:
-            ts.read()
             data = ts.data("sulphur_dioxide_in_air")
             self.assertIn("AM0001", data.stations)
             self.assertGreater(np.sum(data.values), 10000)
@@ -47,7 +45,6 @@ def test_3read(self):
                 "stations": {"include": ["NO0002"]},
             },  # Birkenes2
         ) as ts:
-            ts.read()
             data = ts.data("sulphur_dioxide_in_air")
             self.assertIn("NO0002", data.stations)
             self.assertGreater(len(data), 360)

diff --git a/tests/test_EbasPmfTimeSeriesReader.py b/tests/test_EbasPmfTimeSeriesReader.py
@@ -54,7 +54,6 @@ def test_0engine(self):
 
     def test_1open_single_file(self):
         with pyaro.open_timeseries(self.engine, self.file, filters=[]) as ts:
-            ts.read()
             self.assertGreaterEqual(len(ts.variables()), 1)
             for var in ts.variables():
                 assert var in self.test_vars
@@ -64,7 +63,6 @@ def test_1open_single_file(self):
 
     def test_2open_directory(self):
         with pyaro.open_timeseries(self.engine, self.testdata_dir, filters=[]) as ts:
-            ts.read()
             self.assertGreaterEqual(len(ts.variables()), 1)
             for var in ts.variables():
                 assert var in self.test_vars

diff --git a/tests/test_HARPReader.py b/tests/test_HARPReader.py
@@ -26,7 +26,6 @@ def test_1read(self):
             self.file,
             vars_to_read=self.test_vars,
         ) as ts:
-            ts.read()
             for _v_idx, var in enumerate(self.test_vars):
                 data = ts.data(var)
                 self.assertGreater(len(data), 10000)
@@ -45,7 +44,6 @@ def test_2open_directory(self):
             with pyaro.open_timeseries(
                 self.engine, self.testdata_dir, filters=[], vars_to_read=self.test_vars
             ) as ts:
-                ts.read()
                 for _v_idx, var in enumerate(self.test_vars):
                     data = ts.data(var)
                     assert isinstance(data.units, str)

diff --git a/tests/test_NILUPMFAbsorptionReader.py b/tests/test_NILUPMFAbsorptionReader.py
@@ -26,7 +26,6 @@ def test_0engine(self):
 
     def test_1open_single_file(self):
         with pyaro.open_timeseries(self.engine, self.file, filters=[]) as ts:
-            ts.read()
             self.assertGreaterEqual(len(ts.variables()), 1)
             for var in ts.variables():
                 assert var in self.test_vars
@@ -37,7 +36,6 @@ def test_1open_single_file(self):
 
     def test_2open_directory(self):
         with pyaro.open_timeseries(self.engine, self.testdata_dir, filters=[]) as ts:
-            ts.read()
             self.assertGreaterEqual(len(ts.variables()), 1)
             for var in ts.variables():
                 assert var in self.test_vars