read data on access only

src/pyaro_readers/aeronetsdareader/AeronetSdaTimeseriesReader.py

@@ -110,6 +110,9 @@ def __init__(
     def read(self):
         # check if file is a URL
         _laststatstr = ""
+        # check if the data has been read already
+        if len(self._data) != 0:
+            return
         if self.is_valid_url(self._filename):
             # try to open as zipfile
             try:
@@ -134,7 +137,7 @@ def read(self):
                         lines = []
                         _fidx = 0
                         members = tf.getmembers()
-                        bar = tqdm(desc="extracting tar file...", total=len(members))
+                        bar = tqdm(desc="extracting tar file...", total=len(members), disable=None)
                         for _midx, member in enumerate(members):
                             if fnmatch(member.name, FILE_MASK):
                                 bar.update(1)
@@ -176,7 +179,7 @@ def read(self):
 
         gcd = Geocoder_Reverse_NE()
         crd = csv.DictReader(lines, fieldnames=self._fields, delimiter=DELIMITER)
-        bar = tqdm(desc=self.tqdm_desc, total=len(lines))
+        bar = tqdm(desc=self.tqdm_desc, total=len(lines), disable=None)
         for _ridx, row in enumerate(crd):
             bar.update(1)
             if row[SITE_NAME] != _laststatstr:
@@ -272,15 +275,19 @@ def read(self):
         bar.close()
 
     def metadata(self):
+        self.read()
         return dict(revision=datetime.datetime.strftime(self._revision, "%y%m%d%H%M%S"))
 
     def _unfiltered_data(self, varname) -> Data:
+        self.read()
         return self._data[varname]
 
     def _unfiltered_stations(self) -> dict[str, Station]:
+        self.read()
         return self._stations
 
     def _unfiltered_variables(self) -> list[str]:
+        self.read()
         return list(self._data.keys())
 
     def close(self):
@@ -333,7 +340,3 @@ def description(self):
 
     def url(self):
         return "https://github.com/metno/pyaro-readers"
-
-    def read(self):
-        return self.reader_class().read()
-

src/pyaro_readers/aeronetsunreader/AeronetSunTimeseriesReader.py

@@ -102,6 +102,9 @@ def read(
     ):
         # check if file is a URL
         _laststatstr = ""
+        # check if the data has been read already
+        if len(self._data) != 0:
+            return
         if self.is_valid_url(self._filename):
             # try to open as zipfile
             try:
@@ -127,7 +130,7 @@ def read(
 
         gcd = Geocoder_Reverse_NE()
         crd = csv.DictReader(lines, fieldnames=self._fields, delimiter=DELIMITER)
-        bar = tqdm(desc=tqdm_desc, total=len(lines))
+        bar = tqdm(desc=tqdm_desc, total=len(lines), disable=None)
         for _ridx, row in enumerate(crd):
             bar.update(1)
             if row[SITE_NAME] != _laststatstr:
@@ -209,15 +212,19 @@ def read(
         bar.close()
 
     def metadata(self):
+        self.read()
         return dict(revision=datetime.datetime.strftime(self._revision, "%y%m%d%H%M%S"))
 
     def _unfiltered_data(self, varname) -> Data:
+        self.read()
         return self._data[varname]
 
     def _unfiltered_stations(self) -> dict[str, Station]:
+        self.read()
         return self._stations
 
     def _unfiltered_variables(self) -> list[str]:
+        self.read()
         return list(self._data.keys())
 
     def close(self):
@@ -285,5 +292,3 @@ def description(self):
     def url(self):
         return "https://github.com/metno/pyaro-readers"
 
-    def read(self):
-        return self.reader_class().read()

tests/test_AERONETSDATimeSeriesReader.py

@@ -42,7 +42,6 @@ def test_dl_data_tared(self):
             fill_country_flag=False,
             tqdm_desc="test_sda_dl_data_tared",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -61,7 +60,6 @@ def test_dl_data_unzipped(self):
             fill_country_flag=False,
             tqdm_desc="test_sda_dl_data_unzipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -80,7 +78,6 @@ def test_dl_data_zipped(self):
             fill_country_flag=False,
             tqdm_desc="test_sda_dl_data_zipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -103,7 +100,6 @@ def test_aeronet_data_zipped(self):
             fill_country_flag=False,
             tqdm_desc="aeronet SDA data zipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -122,7 +118,6 @@ def test_init(self):
         with engine.open(
             self.file, filters=[], fill_country_flag=True, tqdm_desc="test_init"
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -135,7 +130,6 @@ def test_stationfilter(self):
         with engine.open(
             self.file, filters=[sfilter], tqdm_desc="test_stationfilter"
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -149,7 +143,6 @@ def test_wrappers(self):
             engine.open(self.file, filters=[]),
             {"Coarse_Mode_AOD_500nm[tau_c]": new_var_name},
         ) as ts:
-            ts.read()
             self.assertEqual(ts.data(new_var_name).variable, new_var_name)
         pass
 
@@ -162,7 +155,6 @@ def test_variables_filter(self):
         with engine.open(
             self.file, filters=[vfilter], tqdm_desc="test_sda_variables_filter"
         ) as ts:
-            ts.read()
             self.assertEqual(ts.data(new_var_name).variable, new_var_name)
 
 

tests/test_AERONETTimeSeriesReader.py

@@ -39,7 +39,6 @@ def test_dl_data_unzipped(self):
             fill_country_flag=False,
             tqdm_desc="test_dl_data_unzipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -74,7 +73,6 @@ def test_dl_data_zipped(self):
             fill_country_flag=False,
             tqdm_desc="test_dl_data_zipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -95,7 +93,6 @@ def test_aeronet_data_zipped(self):
             fill_country_flag=False,
             tqdm_desc="aeronet data zipped",
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -112,7 +109,6 @@ def test_init(self):
         with engine.open(
             self.file, filters=[], fill_country_flag=True, tqdm_desc="test_init"
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -125,7 +121,6 @@ def test_stationfilter(self):
         with engine.open(
             self.file, filters=[sfilter], tqdm_desc="test_stationfilter"
         ) as ts:
-            ts.read()
             count = 0
             for var in ts.variables():
                 count += len(ts.data(var))
@@ -138,7 +133,6 @@ def test_wrappers(self):
         with VariableNameChangingReader(
             engine.open(self.file, filters=[]), {"AOD_500nm": new_var_name}
         ) as ts:
-            ts.read()
             self.assertEqual(ts.data(new_var_name).variable, new_var_name)
         pass