Merge pull request #26 from metno/niluascii2netcdf

ascii2netcdf reader for EBAS and Airquip converted databases

.gitattributes

@@ -0,0 +1 @@
+*.nc filter=lfs diff=lfs merge=lfs -text

.github/workflows/CI.yml

@@ -11,6 +11,8 @@ jobs:
       PYTHON: 3.9
     steps:
     - uses: actions/checkout@v3
+      with:
+        lfs: 'true'
     - name: Set up Python ${{ env.PYTHON }}
       uses: actions/setup-python@v4
       with:
@@ -39,6 +41,8 @@ jobs:
         os: [ ubuntu-22.04 ]
     steps:
       - uses: actions/checkout@v3
+        with:
+          lfs: 'true'
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:

README.md

@@ -2,73 +2,112 @@
 implementations of readers for the pyaerocom project using pyaro as interface
 
 ## Installation
-`python -m pip install 'pyaro-readers@git+https://github.com/metno/pyaro-readers.git'`   
+`python -m pip install 'pyaro-readers@git+https://github.com/metno/pyaro-readers.git'`
 
 This will install pyaro and pyaro-readers and all their dependencies.
 
 ## Supported readers
-### aeronetsunreader  
-Reader for aeronet sun version 3 data (https://aeronet.gsfc.nasa.gov/new_web/download_all_v3_aod.html).  
+### aeronetsunreader
+Reader for aeronet sun version 3 data (https://aeronet.gsfc.nasa.gov/new_web/download_all_v3_aod.html).
 The reader supports reading from an uncompressed local file and from an URL providing a zip file or an
-uncompressed file.  
-If a zip file URL is provided, only the 1st file in there is used (since the 
+uncompressed file.
+If a zip file URL is provided, only the 1st file in there is used (since the
 Aeronet provided zip contains all data in a single file).
 
 ### aeronetsdareader
-Reader for aeronet SDA version 3 data (https://aeronet.gsfc.nasa.gov/new_web/download_all_v3_aod.html).  
+Reader for aeronet SDA version 3 data (https://aeronet.gsfc.nasa.gov/new_web/download_all_v3_aod.html).
 The reader supports reading from an uncompressed local file and from an URL providing a zip file, an
-uncompressed file or a tar file (including all common compression formats).  
-If a zip file URL is provided, only the 1st file in there is used (since the 
+uncompressed file or a tar file (including all common compression formats).
+If a zip file URL is provided, only the 1st file in there is used (since the
 Aeronet provided zip contains all data in a single file).
 
+### ascii2netcdf
+Reader for databases created with MSC-W tools niluNasaAmes2Netcdf or eea_airquip2emepdata.py.
+The database consists of a directory with a list of stations, i.e. `StationList.csv` and netcdf
+data-files per year with resolutions `hourly`, `daily`, `weekly`, `monthly` and `yearly` and a naming
+of `data_{resolution}.{YYYY}.nc`, e.g. `data_daily.2021.nc`. A test-database with daily data only
+can be found under `tests/testdata/NILU`.
+
+The MSC-W database contains the EBAS database for 1990-2021 and the EEA_Airquip database for
+2016-2018 as of yearly 2024. The data in the database is already aggregated, i.e. daily files
+contain already hourly data if enough hours have been measured. Therefore, `resolution` is a
+required parameter.
+
+
 ## Usage
 ### aeronetsunreader
 ```python
-import pyaro.timeseries
+import pyaro
 TEST_URL = "https://pyaerocom.met.no/pyaro-suppl/testdata/aeronetsun_testdata.csv"
-engine = pyaro.list_timeseries_engines()["aeronetsunreader"]
-ts = engine.open(TEST_URL, filters=[], fill_country_flag=False)
-print(ts.variables())
-# stations
-ts.data('AOD_550nm')['stations']
-# start_times
-ts.data('AOD_550nm')['start_times']
-# stop_times
-ts.data('AOD_550nm')['end_times']
-# latitudes
-ts.data('AOD_550nm')['latitudes']
-# longitudes
-ts.data('AOD_550nm')['longitudes']
-# altitudes
-ts.data('AOD_550nm')['altitudes']
-# values
-ts.data('AOD_550nm')['values']
+with pyro.open_timeseries("aeronetsunreader", TEST_URL, filters=[], fill_country_flag=False) as ts:
+    print(ts.variables())
+    data = ts.data('AOD_550nm')
+    # stations
+    data.stations
+    # start_times
+    data.start_times
+    # stop_times
+    data.end_times
+    # latitudes
+    data.latitudes
+    # longitudes
+    data.longitudes
+    # altitudes
+    data.altitudes
+    # values
+    data.values
 
 ```
 ### aeronetsdareader
 ```python
-import pyaro.timeseries
+import pyaro
 TEST_URL = "https://pyaerocom.met.no/pyaro-suppl/testdata/SDA_Level20_Daily_V3_testdata.tar.gz"
-engine = pyaro.list_timeseries_engines()["aeronetsdareader"]
-ts = engine.open(TEST_URL, filters=[], fill_country_flag=False)
-print(ts.variables())
-# stations
-ts.data('AODGT1_550nm')['stations']
-# start_times
-ts.data('AODGT1_550nm')['start_times']
-# stop_times
-ts.data('AODGT1_550nm')['end_times']
-# latitudes
-ts.data('AODGT1_550nm')['latitudes']
-# longitudes
-ts.data('AODGT1_550nm')['longitudes']
-# altitudes
-ts.data('AODGT1_550nm')['altitudes']
-# values
-ts.data('AODGT1_550nm')['values']
+with pyaro.open_timeseries("aeronetsdareader", TEST_URL, filters=[], fill_country_flag=False) as ts:
+    print(ts.variables())
+    data = ts.data('AODGT1_550nm')
+    # stations
+    data.stations
+    # start_times
+    data.start_times
+    # stop_times
+    data.end_times
+    # latitudes
+    data.latitudes
+    # longitudes
+    data.longitudes
+    # altitudes
+    data.altitudes
+    # values
+    data.values
+```
+
+### ascii2netcdf
+```python
+import pyaro
+TEST_URL = "/lustre/storeB/project/fou/kl/emep/Auxiliary/NILU/"
+with pyaro.open_timeseries(
+    'ascii2netcdf', EBAS_URL, resolution="daily", filters=[]
+) as ts:
+    data = ts.data("sulphur_dioxide_in_air")
+    data.units # ug
+    # stations
+    data.stations
+    # start_times
+    data.start_times
+    # stop_times
+    data.end_times
+    # latitudes
+    data.latitudes
+    # longitudes
+    data.longitudes
+    # altitudes
+    data.altitudes
+    # values
+    data.values
 
 ```
 
+
 ### geocoder_reverse_natural_earth
 geocoder_reverse_natural_earth is small helper to identify country codes for obs networks that don't mention the
 countrycode of a station in their location data
@@ -89,7 +128,7 @@ except Geocoder_Reverse_Exception as grex:
         print(f"error: {lat},{lon}")
     else:
         print(dummy["ISO_A2_EH"])
-
-
 
-```
+
+
+```

pyproject.toml → pyproject.toml_future

@@ -22,16 +22,17 @@ classifiers = [
 ]
 requires-python = ">=3.9"
 dependencies = [
-    "pyaro@git+https://github.com/metno/pyaro",
+    "pyaro >=0.0.6",
     "requests",
     "fiona",
+    "netCDF4",
     "shapely",
     "rtree",
     "tqdm",
 ]
 
 [tool.setuptools]
-packages = ["pyaro_readers.aeronetsunreader", "pyaro_readers.aeronetsdareader","geocoder_reverse_natural_earth"]
+packages = ["pyaro_readers.aeronetsunreader", "pyaro_readers.aeronetsdareader","geocoder_reverse_natural_earth", "pyaro_readers.ascii2netcdf"]
 package-dir = {"" = "src"}
 
 [tool.setuptools.package-data]
@@ -44,6 +45,7 @@ geocoder_reverse_natural_earth = ["*.zip"]
 [project.entry-points."pyaro.timeseries"]
 "aeronetsunreader" = "pyaro_readers.aeronetsunreader:AeronetSunTimeseriesEngine"
 "aeronetsdareader" = "pyaro_readers.aeronetsdareader:AeronetSdaTimeseriesEngine"
+"ascii2netcdf" = "pyaro_readers.ascii2netcdf:Ascii2NetcdfTimeseriesEngine"
 
 [tool.mypy]
 python_version = "3.9"

setup.cfg

@@ -0,0 +1,65 @@
+[metadata]
+name = pyaro_readers
+version = 0.0.3dev
+author = MET Norway
+description = implementations of pyaerocom reading plugings using pyaro as interface
+long_description = file: README.md
+long_description_content_type = text/markdown
+classifiers =
+    Programming Language :: Python :: 3 :: Only
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
+    Programming Language :: Python :: 3.11
+    License :: OSI Approved :: GNU Lesser General Public License v3 (LGPLv3)
+    Operating System :: OS Independent
+    Development Status :: 3 - Alpha
+    Intended Audience :: Science/Research
+    Intended Audience :: Education
+    Topic :: Scientific/Engineering :: Atmospheric Science
+url = https://github.com/metno/pyaro-readers
+
+[options]
+python_version = >=3.9
+install_requires =
+    pyaro >= 0.0.6
+    netCDF4
+    requests
+    fiona
+    shapely
+    rtree
+    tqdm
+
+package_dir =
+    =src
+packages = pyaro_readers.aeronetsunreader, pyaro_readers.aeronetsdareader, geocoder_reverse_natural_earth, pyaro_readers.ascii2netcdf
+test_require = tox:tox
+
+[options.package_data]
+geocoder_reverse_natural_earth =
+    *.zip
+
+[options.entry_points]
+pyaro.timeseries =
+    aeronetsunreader = pyaro_readers.aeronetsunreader:AeronetSunTimeseriesEngine
+    aeronetsdareader = pyaro_readers.aeronetsdareader:AeronetSdaTimeseriesEngine
+    ascii2netcdf = pyaro_readers.ascii2netcdf:Ascii2NetcdfTimeseriesEngine
+
+
+[tox:tox]
+min_version = 4.0
+skip_missing_interpreters = True
+isolated_build = True
+env_list =
+    py310
+    format
+#depends =
+
+[testenv]
+commands = python3 -m unittest discover -s tests
+
+[testenv:format]
+commands =
+    pre-commit run --all-files --show-diff-on-failure
+deps =
+    pre-commit
+

setup.py

@@ -0,0 +1,5 @@
+#!/usr/bin/env python3
+
+from setuptools import setup
+
+setup()