WIP

src/pyaro_readers/actrisebas/ActrisEbasReader.py

@@ -126,6 +126,7 @@ def __init__(
         self._set_filters(filters)
         # self._header = []
         self._metadata = {}
+        self._tmp_metadata = {}
         # used for variable matching in the EBAS data files
         # gives a mapping between the EBAS or pyaerocom variable name
         # and the CF standard name found in the EBAS data files
@@ -134,6 +135,7 @@ def __init__(
         # _laststatstr = ""
         self._revision = datetime.datetime.now()
         self._metadata["revision"] = datetime.datetime.strftime(self._revision, "%y%m%d%H%M%S")
+        self._tmp_metadata["revision"] = datetime.datetime.strftime(self._revision, "%y%m%d%H%M%S")
         self.ebas_valid_flags = self.get_ebas_valid_flags()
         self.sites_to_read = []
         self.sites_to_exclude = []
@@ -210,7 +212,7 @@ def __init__(
         # of ACTRIS variables to read. values are a list
         self.actris_vars_to_read = {}
         for var in self.vars_to_read:
-            self._metadata[var] = {}
+            self._tmp_metadata[var] = {}
             # handle pyaerocom variables here:
             # if a given variable name is in the list of pyaerocom variable names in definitions.toml
             self.actris_vars_to_read[var] = []
@@ -283,7 +285,7 @@ def __init__(
                             json_resp_tmp = ""
                             continue
 
-                self._metadata[_pyaro_var][_actris_var] = json_resp
+                self._tmp_metadata[_pyaro_var][_actris_var] = json_resp
                 self.urls_to_dl[_actris_var] = self.extract_urls(
                     json_resp,
                     sites_to_read=self.sites_to_read,
@@ -295,7 +297,7 @@ def __init__(
                 # self.read_data(
                 #     actris_variable=_pyaro_var, urls_to_dl=self.urls_to_dl[_actris_var]
                 # )
-                assert self._metadata[_pyaro_var][_actris_var]
+                assert self._tmp_metadata[_pyaro_var][_actris_var]
                 # return _pyaro_var, self.urls_to_dl
 
     def metadata(self):
@@ -318,20 +320,24 @@ def _read(
                 urls_to_dl = self.urls_to_dl[actris_variable]
                 bar = tqdm(desc=tqdm_desc, total=len(urls_to_dl), disable=None)
                 for s_idx, site_name in enumerate(urls_to_dl):
+                    assert site_name
+                    self._metadata[site_name] = {}
                     for f_idx, thredds_url in enumerate(urls_to_dl[site_name]):
                         _local_file_flag = False
                         if self.cache_flag:
-                            _local_file = self.cache_dir / Path(thredds_url).name
+                            # _local_file = self.cache_dir / Path(thredds_url).name
+                            _local_file = self.cache_dir / "_".join(Path(thredds_url).parts[-4:])
                             if _local_file.exists():
                                 url = _local_file
                                 _local_file_flag = True
                             else:
                                 url = thredds_url
                         else:
                             url = thredds_url
-                        logger.info(f"reading file {url}")
                         try:
+                            logger.info(f"trying to read URL {url}")
                             tmp_data = xr.open_dataset(url)
+                            logger.info(f"Successfully read URL {url}")
                         except Exception as e:
                             logger.error(f"failed to read {url} with error {e}")
                             assert url
@@ -458,6 +464,8 @@ def _read(
                             )
                             break
                         if stat_code is not None:
+                            if site_name == "Schmücke":
+                                assert site_name
                             if not site_name in self._stations:
                                 self._stations[site_name] = Station(
                                     {

src/pyaro_readers/eeareader/EEATimeseriesReader.py

@@ -276,9 +276,9 @@ def _read_hourly_files(
         )
     )
 
-    assert (
-        joined.filter(polars.col("Longitude").is_null()).shape[0] == 0
-    ), "Some stations does not have a suitable left join"
+    assert joined.filter(polars.col("Longitude").is_null()).shape[0] == 0, (
+        "Some stations does not have a suitable left join"
+    )
 
     return joined
 
@@ -368,9 +368,9 @@ def _read_daily_files(
         .drop("Timezone")
     )
 
-    assert (
-        joined.filter(polars.col("Longitude").is_null()).shape[0] == 0
-    ), "Some stations does not have a suitable left join"
+    assert joined.filter(polars.col("Longitude").is_null()).shape[0] == 0, (
+        "Some stations does not have a suitable left join"
+    )
 
     return joined
 

src/pyaro_readers/nilupmfebas/ebas_nasa_ames.py

@@ -236,7 +236,7 @@ def __setitem__(self, key, val):
 
     def __str__(self):
         head = f"{type(self).__name__}"
-        s = f"{head}\n{len(head)*'-'}\n"
+        s = f"{head}\n{len(head) * '-'}\n"
         s += dict_to_str(self._head_fix)
         s += f"\n\n{str_underline('Column variable definitions', indent=3)}"
         for item in self._var_defs:

tests/test_AERONETTimeSeriesReader.py

@@ -145,7 +145,7 @@ def test_variables_filter(self):
         with engine.open(
             self.file, filters=[vfilter], tqdm_desc="test_variables_filter"
         ) as ts:
-            ts.read()
+            ts._read()
             self.assertEqual(ts.data(new_var_name).variable, new_var_name)
 
 

tests/test_ActrisEbasReader.py

@@ -23,13 +23,13 @@ class TestActrisEbasTimeSeriesReader(unittest.TestCase):
     # pyaerocom_vars_to_read = ["concso2"]
     # pyaerocom_vars_to_read = ["vmrso2"]
 
-    # pyaerocom_vars_to_read = ["concso4t"]
-    pyaerocom_vars_to_read = ["concpm10"]
+    pyaerocom_vars_to_read = ["concso4t"]
+    # pyaerocom_vars_to_read = ["concpm10"]
     # pyaerocom_vars_to_read = ["concpm25"]
     # pyaerocom_vars_to_read = ["concpm1"]
 
     station_filter = pyaro.timeseries.Filter.StationFilter(
-        ["Birkenes II", "Jungfraujoch", "Ispra", "Melpitz", "Westerland"], []
+        ["Schmücke", "Birkenes II", "Jungfraujoch", "Ispra", "Melpitz", "Westerland"], []
     )
 
     variable_filter_pyaerocom = pyaro.timeseries.Filter.VariableNameFilter(

tests/test_PyerocomBinding.py

@@ -10,8 +10,8 @@ class TestPyaroReaderPyaerocom(unittest.TestCase):
         },
     }
     AERONETVAR = "od440aer"
-    # ACTRISEBASVAR = "concso4t"
-    ACTRISEBASVAR = "vmro3"
+    ACTRISEBASVAR = "concso4t"
+    # ACTRISEBASVAR = "vmro3"
     ACTRISEBASVARLIST = ["concso4t", "concso4c"]
 
     def test_pyaerocom_aeronet(self):
@@ -50,15 +50,15 @@ def test_pyaerocom_actrisebas_single_var(self):
         data_name = "PYARO_actrisebas"
         data_id = "actrisebas"
         station_filter = {
-            "stations": {
-                "include": [
-                    "Birkenes II",
-                    "Jungfraujoch",
-                    "Ispra",
-                    "Melpitz",
-                    "Westerland",
-                ]
-            },
+            # "stations": {
+            #     "include": [
+            #         "Birkenes II",
+            #         "Jungfraujoch",
+            #         "Ispra",
+            #         "Melpitz",
+            #         "Westerland",
+            #     ]
+            # },
             "variables": {"include": [self.ACTRISEBASVAR, ]},
             "time_bounds":{"startend_include": [("2019-01-01 00:00:00", "2023-12-24 00:00:00")]}
         }