diff --git a/src/pyaro_readers/actrisebas/ActrisEbasReader.py b/src/pyaro_readers/actrisebas/ActrisEbasReader.py index ebf3ce7..c49604e 100644 --- a/src/pyaro_readers/actrisebas/ActrisEbasReader.py +++ b/src/pyaro_readers/actrisebas/ActrisEbasReader.py @@ -135,7 +135,8 @@ def __init__( self._revision = datetime.datetime.now() self._metadata["revision"] = datetime.datetime.strftime(self._revision, "%y%m%d%H%M%S") self.ebas_valid_flags = self.get_ebas_valid_flags() - self.sites_to_read = None + self.sites_to_read = [] + self.sites_to_exclude = [] self.vars_to_read = None self.times_to_read = (np.datetime64(1, "Y"), np.datetime64(120, "Y")) @@ -167,6 +168,7 @@ def __init__( np.max(filter._start_include), ) logger.info(f"applying time include filter {self.times_to_read}...") + # For some reason the filters come as dict from pyaerocom elif isinstance(filter, str): if filter == 'stations': assert filters[filter] @@ -185,9 +187,10 @@ def __init__( logger.info(f"applying variable include filter {self.vars_to_read}...") except KeyError: pass - elif filter == 'time': + elif filter == 'time_bounds': try: - self.times_to_read = filters[filter]["start_include"] + _time_bounds = np.array(filters[filter]["startend_include"], dtype=np.datetime64) + self.times_to_read = (np.min(_time_bounds), np.max(_time_bounds)) except Exception as e: pass assert filters[filter] @@ -196,13 +199,6 @@ def __init__( else: # pass on not reader supported filters pass - # for time filter: - # There's time_coverage_start and time_coverage_end in the global attributes with the - # time coverage as ISO string - # just looking at the time variable is not enough. It notes the middle time. The time_bnds variable - # has to be applied as well - # np.datetime64(tmp_data.attrs["time_coverage_start"].split()[0]) - # np.datetime64(tmp_data.attrs["time_coverage_end"].split()[0]) if self.vars_to_read is None: logger.info(f"No variable filter given, nothing to read...") @@ -310,7 +306,7 @@ def _read( tqdm_desc="reading stations", ): """ - read the data from EBAS thredds server + read the data from EBAS thredds server (or the local cache) """ # for actris vocabulary key and value of self.actris_vars_to_read are the same # for pyaerocom vocabulary they are not (key is pyaerocom variable name there)! @@ -319,8 +315,6 @@ def _read( logger.info(f"var {_var} already read") continue for actris_variable in self.actris_vars_to_read[_var]: - # actris_variable = self.actris_vars_to_read[_var][0] - urls_to_dl = self.urls_to_dl[actris_variable] bar = tqdm(desc=tqdm_desc, total=len(urls_to_dl), disable=None) for s_idx, site_name in enumerate(urls_to_dl): @@ -379,6 +373,7 @@ def _read( # some of the data files can't be read by xarray due to errors. So we # can't cache them (all data is only realized here) try: + tmp_data.load() tmp_data.to_netcdf(_local_file) logger.info(f"saved cache file {_local_file}") except Exception as e: @@ -476,7 +471,12 @@ def _read( "long_name": site_name, } ) + try: + tmp_data.close() + except Exception as e: + pass bar.update(1) + bar.close() assert True diff --git a/tests/test_PyerocomBinding.py b/tests/test_PyerocomBinding.py index c77e11b..a89c7e1 100644 --- a/tests/test_PyerocomBinding.py +++ b/tests/test_PyerocomBinding.py @@ -10,7 +10,8 @@ class TestPyaroReaderPyaerocom(unittest.TestCase): }, } AERONETVAR = "od440aer" - ACTRISEBASVAR = "concso4t" + # ACTRISEBASVAR = "concso4t" + ACTRISEBASVAR = "vmro3" ACTRISEBASVARLIST = ["concso4t", "concso4c"] def test_pyaerocom_aeronet(self): @@ -58,6 +59,8 @@ def test_pyaerocom_actrisebas_single_var(self): "Westerland", ] }, + "variables": {"include": [self.ACTRISEBASVAR, ]}, + "time_bounds":{"startend_include": [("2019-01-01 00:00:00", "2023-12-24 00:00:00")]} } # needs to be the variable name for actrisebas url = self.ACTRISEBASVAR @@ -68,7 +71,7 @@ def test_pyaerocom_actrisebas_single_var(self): filters=station_filter, ) reader = ReadUngridded(f"{data_name}") - data = reader._read(vars_to_retrieve=self.ACTRISEBASVAR, configs=obsconfig) + data = reader.read(vars_to_retrieve=self.ACTRISEBASVAR, configs=obsconfig) self.assertGreaterEqual(len(data.unique_station_names), 4) self.assertIn("Ispra", data.unique_station_names) self.assertIn(url, data.contains_vars)