From d564d7cf1adb36a2b6666c7c0725b7a720956e52 Mon Sep 17 00:00:00 2001
From: Jan Griesfeller <jang@met.no>
Date: Tue, 1 Oct 2024 16:41:18 +0200
Subject: [PATCH] WIP

---
 .../actrisebas/ActrisEbasReader.py            | 25 ++++++++++++++++---
 src/pyaro_readers/actrisebas/definitions.toml |  3 ++-
 tests/test_ActrisEbasReader.py                |  4 +--
 3 files changed, 25 insertions(+), 7 deletions(-)

diff --git a/src/pyaro_readers/actrisebas/ActrisEbasReader.py b/src/pyaro_readers/actrisebas/ActrisEbasReader.py
index 86a26b2..04bd3fe 100644
--- a/src/pyaro_readers/actrisebas/ActrisEbasReader.py
+++ b/src/pyaro_readers/actrisebas/ActrisEbasReader.py
@@ -24,8 +24,8 @@
 logger = logging.getLogger(__name__)
 
 # default API URL base
-# BASE_API_URL = "https://prod-actris-md.nilu.no/Vocabulary/categories"
 BASE_API_URL = "https://dev-actris-md.nilu.no/"
+# BASE_API_URL = "https://prod-actris-md.nilu.no/"
 # base URL to query for data for a certain variable
 VAR_QUERY_URL = f"{BASE_API_URL}metadata/content/"
 # basename of definitions.toml which connects the pyaerocom variable names with the ACTRIS variable names
@@ -181,9 +181,19 @@ def __init__(
                         retries = Retry(connect=5, read=2, redirect=5)
                         http = PoolManager(retries=retries)
                         response = http.request("GET", query_url)
-                        json_resp_tmp = json.loads(response.data.decode("utf-8"))
-                        json_resp.extend(json_resp_tmp)
-                        page_no += 1
+                        if len(response.data) > 0:
+                            try:
+                                json_resp_tmp = json.loads(
+                                    response.data.decode("utf-8")
+                                )
+                            except json.decoder.JSONDecodeError:
+                                json_resp_tmp = json.loads(response.data)
+
+                            json_resp.extend(json_resp_tmp)
+                            page_no += 1
+                        else:
+                            json_resp_tmp = ""
+                            continue
 
                 self._metadata[_pyaro_var][_actris_var] = json_resp
                 self._urls_to_dl[_actris_var] = self.extract_urls(
@@ -401,6 +411,7 @@ def extract_urls(
                 LOCATION_NAME_KEY
             ]
             if site_name in sites_to_exclude:
+                logger.info(f"site {site_name} excluded due to exclusion filter")
                 continue
             if site_name in sites_to_read or len(sites_to_read) == 0:
                 if site_name not in urls_to_dl:
@@ -415,11 +426,17 @@ def extract_urls(
                         distribution_data[DISTRIBUTION_PROTOCOL_KEY]
                         != DISTRIBUTION_PROTOCOL_NAME
                     ):
+                        logger.info(
+                            f"skipping site: {site_name} / proto: {distribution_data[DISTRIBUTION_PROTOCOL_KEY]}"
+                        )
                         continue
                     else:
                         urls_to_dl[site_name].append(
                             distribution_data[DISTRIBUTION_URL_KEY]
                         )
+                        logger.info(
+                            f"site: {site_name} / proto: {distribution_data[DISTRIBUTION_PROTOCOL_KEY]} included in URL list"
+                        )
                         break
         return urls_to_dl
 
diff --git a/src/pyaro_readers/actrisebas/definitions.toml b/src/pyaro_readers/actrisebas/definitions.toml
index cc198d9..0731e44 100644
--- a/src/pyaro_readers/actrisebas/definitions.toml
+++ b/src/pyaro_readers/actrisebas/definitions.toml
@@ -16,7 +16,8 @@ ebas_matrix = ["air"]
 standard_names = ["mass_concentration_of_ozone_in_air"]
 
 [variables.vmro3]
-actris_variable = ["ozone amount fraction"]
+#actris_variable = ["ozone amount fraction"]
+actris_variable = ["ozone mass concentration"]
 actris_matrix = ["gas phase"]
 ebas_component = ["ozone"]
 ebas_matrix = ["air"]
diff --git a/tests/test_ActrisEbasReader.py b/tests/test_ActrisEbasReader.py
index a600392..3c1f92b 100644
--- a/tests/test_ActrisEbasReader.py
+++ b/tests/test_ActrisEbasReader.py
@@ -21,8 +21,8 @@ class TestActrisEbasTimeSeriesReader(unittest.TestCase):
     # vars_to_read = ["aerosol particle sulphate mass concentration"]
     vars_to_read = ["aerosol particle elemental carbon mass concentration"]
     # pyaerocom_vars_to_read = ["conco3"]
-    # pyaerocom_vars_to_read = ["vmro3"]
-    pyaerocom_vars_to_read = ["concso4t"]
+    pyaerocom_vars_to_read = ["vmro3"]
+    # pyaerocom_vars_to_read = ["concso4t"]
 
     def test_api_online(self, url=TEST_URL):
         try: