add geotiff read and write to compute tide programs (#32)

add long period equilibrium tide programs for ATL03 and ATL06

update documentation for geotiff files

.github/workflows/auto-update-files.yml

@@ -21,15 +21,13 @@ jobs:
     - name: Install dependencies
       run: |
         sudo apt-get update
-        sudo apt-get install libproj-dev proj-data proj-bin libgeos-dev octave
-        sudo apt-get install libhdf5-dev libnetcdf-dev
+        sudo apt-get install gdal-bin libgdal-dev libproj-dev proj-data proj-bin \
+            libgeos-dev libhdf5-dev libnetcdf-dev octave
+        export CPLUS_INCLUDE_PATH=/usr/include/gdal
+        export C_INCLUDE_PATH=/usr/include/gdal
         pip install --upgrade pip
+        pip install .
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Update required files
-      run: |
-        pip install --no-deps .
-        pytest test/test_leap_seconds.py
     - name: Test with pytest
       run: |
         pytest test/test_leap_seconds.py test/test_time.py test/test_eop.py

.github/workflows/python-package.yml

@@ -30,38 +30,47 @@ jobs:
       if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-        sudo apt-get install libproj-dev proj-data proj-bin libgeos-dev octave
-        sudo apt-get install libhdf5-dev libnetcdf-dev
+        sudo apt-get install gdal-bin libgdal-dev libproj-dev proj-data proj-bin \
+            libgeos-dev libhdf5-dev libnetcdf-dev octave
+        export CPLUS_INCLUDE_PATH=/usr/include/gdal
+        export C_INCLUDE_PATH=/usr/include/gdal
         pip install --upgrade pip
+        pip install zarr
+        pip install pandas
+        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
+        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
+        pip install .
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Install dependencies for MacOS
       if: matrix.os == 'macos-latest'
       run: |
         brew install proj
         brew install geos
+        brew install gdal
         brew install hdf5
         brew install netcdf
         brew install octave
         pip install --upgrade pip
+        pip install zarr
+        pip install pandas
+        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
+        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
+        pip install .
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Install from source
-      run: |
-        pip install zarr
-        pip install pandas
-        pip install --no-deps .
-        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
-        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
+    - name: Checkout Matlab TMD Toolbox
+      uses: actions/checkout@v2
+      with:
+        repository: tsutterley/TMD_Matlab_Toolbox_v2.5
+        path: TMD_Matlab_Toolbox
+        ref: inferminor
     - name: Test with pytest
       run: |
-        git clone ${{ secrets.TMD_MATLAB_TOOLBOX }}
         pytest --cov=./ --cov-report=xml \
             --username=${{ secrets.EARTHDATA_USERNAME }} \
             --password=${{ secrets.EARTHDATA_PASSWORD }} \

.github/workflows/python-request.yml

@@ -28,35 +28,39 @@ jobs:
       if: matrix.os == 'ubuntu-20.04'
       run: |
         sudo apt-get update
-        sudo apt-get install libproj-dev proj-data proj-bin libgeos-dev octave
-        sudo apt-get install libhdf5-dev libnetcdf-dev
+        sudo apt-get install gdal-bin libgdal-dev libproj-dev proj-data proj-bin \
+            libgeos-dev libhdf5-dev libnetcdf-dev octave
+        export CPLUS_INCLUDE_PATH=/usr/include/gdal
+        export C_INCLUDE_PATH=/usr/include/gdal
         pip install --upgrade pip
+        pip install zarr
+        pip install pandas
+        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
+        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
+        pip install .
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Install dependencies for MacOS
       if: matrix.os == 'macos-latest'
       run: |
         brew install proj
         brew install geos
+        brew install gdal
         brew install hdf5
         brew install netcdf
         brew install octave
         pip install --upgrade pip
+        pip install zarr
+        pip install pandas
+        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
+        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
+        pip install .
         if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
     - name: Lint with flake8
       run: |
         # stop the build if there are Python syntax errors or undefined names
         flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
         # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
         flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Install from source
-      run: |
-        pip install zarr
-        pip install pandas
-        pip install --no-deps .
-        pip install --no-deps git+https://github.com/tsutterley/read-ICESat-2.git
-        pip install --no-deps git+https://github.com/tsutterley/read-ATM1b-QFIT-binary.git
     - name: Checkout Matlab TMD Toolbox
       uses: actions/checkout@v2
       with:

README.md

@@ -36,6 +36,7 @@ pyTMD
  - [dateutil: powerful extensions to datetime](https://dateutil.readthedocs.io/en/stable/)  
  - [lxml: processing XML and HTML in Python](https://pypi.python.org/pypi/lxml)  
  - [PyYAML: YAML parser and emitter for Python](https://github.com/yaml/pyyaml)  
+ - [gdal: Pythonic interface to the Geospatial Data Abstraction Library (GDAL)](https://pypi.python.org/pypi/GDAL)  
  - [h5py: Python interface for Hierarchal Data Format 5 (HDF5)](https://www.h5py.org/)  
  - [netCDF4: Python interface to the netCDF C library](https://unidata.github.io/netcdf4-python/)  
  - [matplotlib: Python 2D plotting library](https://matplotlib.org/)  

doc/source/getting_started/Citations.md

@@ -25,7 +25,8 @@ This software is also dependent on other commonly used Python packages:
 - [pyproj: Python interface to PROJ library](https://pypi.org/project/pyproj/)  
 - [dateutil: powerful extensions to datetime](https://dateutil.readthedocs.io/en/stable/)  
 - [lxml: processing XML and HTML in Python](https://pypi.python.org/pypi/lxml)  
-- [PyYAML: YAML parser and emitter for Python](https://github.com/yaml/pyyaml)
+- [PyYAML: YAML parser and emitter for Python](https://github.com/yaml/pyyaml)  
+- [gdal: Pythonic interface to the Geospatial Data Abstraction Library (GDAL)](https://pypi.python.org/pypi/GDAL)  
 - [h5py: Python interface for Hierarchal Data Format 5 (HDF5)](https://www.h5py.org/)  
 - [netCDF4: Python interface to the netCDF C library](https://unidata.github.io/netcdf4-python/)  
 - [matplotlib: Python 2D plotting library](https://matplotlib.org/)  

doc/source/getting_started/Getting-Started.md

@@ -53,7 +53,7 @@ tide = compute_tide_corrections(x, y, delta_time, DIRECTORY=path_to_tide_models,
     METHOD='spline', FILL_VALUE=np.nan)
 ```
 
-For users wanting to calculate tidal elevations or currents for a series of files, the [`compute_tidal_elevations.py`](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_tidal_elevations.py) and [`compute_tidal_currents.py`](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_tidal_currents.py) programs cover most use cases.  They take an input file (in csv, netCDF4 or HDF5) and compute the tidal elevations or currents (zonal and meridonal) for each point.
+For users wanting to calculate tidal elevations or currents for a series of files, the [`compute_tidal_elevations.py`](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_tidal_elevations.py) and [`compute_tidal_currents.py`](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_tidal_currents.py) programs cover most use cases.  They take an input file (in csv, netCDF4, HDF5 or geotiff) and compute the tidal elevations or currents (zonal and meridional) for each point.
 ```bash
 python compute_tidal_elevations.py --directory <path_to_tide_models> --tide CATS2008 \
     --format HDF5 --variables t_sec lat lon h_cor --projection 4326 \

doc/source/getting_started/Install.md

@@ -1,6 +1,17 @@
-Installation
-============
+Setup and Installation
+======================
 
+#### Dependencies
+pyTMD is dependent on open source programs:
+- [GDAL](https://gdal.org/index.html)
+- [GEOS](https://trac.osgeo.org/geos)
+- [PROJ](https://proj.org/)
+- [HDF5](https://www.hdfgroup.org)
+- [netCDF](https://www.unidata.ucar.edu/software/netcdf)
+- [libxml2](http://xmlsoft.org/)
+- [libxslt](http://xmlsoft.org/XSLT/)
+
+#### Installation
 Presently pyTMD is available for use as a [GitHub repository](https://github.com/tsutterley/pyTMD) and from the [Python Package Index (pypi)](https://pypi.org/project/pyTMD/).
 The contents of the repository can be download from GitHub as a [zipped file](https://github.com/tsutterley/pyTMD/archive/main.zip) or cloned.
 To use this repository from GitHub, please fork into your own account and then clone onto your system.  

doc/source/index.rst

@@ -42,6 +42,8 @@ calculating radial pole tide displacements
     user_guide/compute_equilibrium_tide.md
     user_guide/compute_LPET_elevations.md
     user_guide/compute_LPET_icebridge_data.md
+    user_guide/compute_LPET_ICESat2_ATL03.md
+    user_guide/compute_LPET_ICESat2_ATL06.md
     user_guide/compute_LPT_displacements.md
     user_guide/compute_LPT_icebridge_data.md
     user_guide/compute_OPT_displacements.md

doc/source/user_guide/compute_LPET_ICESat2_ATL03.md

@@ -0,0 +1,20 @@
+compute_LPET_ICESat2_ATL03.py
+==============================
+
+- Calculates long-period equilibrium tidal elevations for correcting ICESat-2 geolocated photon height data
+- Calculated at ATL03 segment level using reference photon geolocation and time
+- Segment level corrections can be applied to the individual photon events (PEs)
+- Will calculate the long-period tides for all ATL03 segments and not just ocean segments defined by the ocean tide mask
+
+#### Calling Sequence
+```bash
+python compute_LPET_ICESat2_ATL03.py input_file
+```
+[Source code](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_LPET_ICESat2_ATL03.py)
+
+#### Inputs
+ 1. `input_file`: input ICESat-2 ATL03 file
+
+#### Command Line Options
+ - `-M X`, `--mode X`: Permission mode of output file
+ - `-V`, `--verbose`: Output information about each created file

doc/source/user_guide/compute_LPET_ICESat2_ATL06.md

@@ -0,0 +1,18 @@
+compute_LPET_ICESat2_ATL06.py
+==============================
+
+- Calculates long-period equilibrium tidal elevations for correcting ICESat-2 land ice elevation data
+- Will calculate the long-period tides for all ATL06 segments and not just ocean segments defined by the ocean tide mask
+
+#### Calling Sequence
+```bash
+python compute_LPET_ICESat2_ATL06.py input_file
+```
+[Source code](https://github.com/tsutterley/pyTMD/blob/main/scripts/compute_LPET_ICESat2_ATL06.py)
+
+#### Inputs
+ 1. `input_file`: input ICESat-2 ATL06 file
+
+#### Command Line Options
+ - `-M X`, `--mode X`: Permission mode of output file
+ - `-V`, `--verbose`: Output information about each created file

doc/source/user_guide/compute_LPET_elevations.md

@@ -19,11 +19,18 @@ python compute_LPET_elevation.py input_file output_file
     * `'csv'` (default)
     * `'netCDF4'`
     * `'HDF5'`
+    * `'geotiff'`
 - `--variables X`: variable names of data in csv, HDF5 or netCDF4 file
     * for csv files: the order of the columns within the file
     * for HDF5 and netCDF4 files: time, y, x and data variable names
-- `--epoch X`: Reference epoch of input time
+- `-H X`, `--header X`: number of header lines for csv files
+- `-t X`, `--type X`: input data type
+    * `'drift'`: drift buoys or satellite/airborne altimetry (time per data point)
+    * `'grid'`: spatial grids or images (single time for all data points)
+- `--epoch X`: reference epoch of input time
     * `'days since 1858-11-17T00:00:00'` (default Modified Julian Days)
+- `-d X`, `--deltatime X`: input delta time for files without date information
+    * can be set to 0 to use exact calendar date from epoch
 - `--projection X`: spatial projection as EPSG code or PROJ4 string
     * `4326`: latitude and longitude coordinates on WGS84 reference ellipsoid
 - `-V`, `--verbose`: Verbose output of processing run

doc/source/user_guide/compute_LPT_displacements.md

@@ -20,11 +20,18 @@ python compute_LPT_displacements.py input_file output_file
      * `'csv'` (default)
      * `'netCDF4'`
      * `'HDF5'`
+     * `'geotiff'`
  - `--variables X`: variable names of data in csv, HDF5 or netCDF4 file
      * for csv files: the order of the columns within the file
      * for HDF5 and netCDF4 files: time, y, x and data variable names
- - `--epoch X`: Reference epoch of input time
+ - `-H X`, `--header X`: number of header lines for csv files
+ - `-t X`, `--type X`: input data type
+     * `'drift'`: drift buoys or satellite/airborne altimetry (time per data point)
+     * `'grid'`: spatial grids or images (single time for all data points)
+ - `--epoch X`: reference epoch of input time
      * `'days since 1858-11-17T00:00:00'` (default Modified Julian Days)
+ - `-d X`, `--deltatime X`: input delta time for files without date information
+     * can be set to 0 to use exact calendar date from epoch
  - `--projection X`: spatial projection as EPSG code or PROJ4 string
      * `4326`: latitude and longitude coordinates on WGS84 reference ellipsoid
  - `-V`, `--verbose`: Verbose output of processing run

doc/source/user_guide/compute_OPT_displacements.md

@@ -20,11 +20,18 @@ python compute_OPT_displacements.py input_file output_file
      * `'csv'` (default)
      * `'netCDF4'`
      * `'HDF5'`
+     * `'geotiff'`
  - `--variables X`: variable names of data in csv, HDF5 or netCDF4 file
      * for csv files: the order of the columns within the file
      * for HDF5 and netCDF4 files: time, y, x and data variable names
- - `--epoch X`: Reference epoch of input time
+ - `-H X`, `--header X`: number of header lines for csv files
+ - `-t X`, `--type X`: input data type
+     * `'drift'`: drift buoys or satellite/airborne altimetry (time per data point)
+     * `'grid'`: spatial grids or images (single time for all data points)
+ - `--epoch X`: reference epoch of input time
      * `'days since 1858-11-17T00:00:00'` (default Modified Julian Days)
+ - `-d X`, `--deltatime X`: input delta time for files without date information
+     * can be set to 0 to use exact calendar date from epoch
  - `--projection X`: spatial projection as EPSG code or PROJ4 string
      * `4326`: latitude and longitude coordinates on WGS84 reference ellipsoid
  - `-I X`, `--interpolate X`: Interpolation method

doc/source/user_guide/compute_tidal_currents.md

@@ -32,11 +32,18 @@ python compute_tidal_currents.py --directory <path_to_directory> --tide <model>
      * `'csv'` (default)
      * `'netCDF4'`
      * `'HDF5'`
+     * `'geotiff'`
  - `--variables X`: variable names of data in csv, HDF5 or netCDF4 file
      * for csv files: the order of the columns within the file
      * for HDF5 and netCDF4 files: time, y, x and data variable names
- - `--epoch X`: Reference epoch of input time
+ - `-H X`, `--header X`: number of header lines for csv files
+ - `-t X`, `--type X`: input data type
+     * `'drift'`: drift buoys or satellite/airborne altimetry (time per data point)
+     * `'grid'`: spatial grids or images (single time for all data points)
+ - `--epoch X`: reference epoch of input time
      * `'days since 1858-11-17T00:00:00'` (default Modified Julian Days)
+ - `-d X`, `--deltatime X`: input delta time for files without date information
+     * can be set to 0 to use exact calendar date from epoch
  - `--projection X`: spatial projection as EPSG code or PROJ4 string
      * `4326`: latitude and longitude coordinates on WGS84 reference ellipsoid
  - `-I X`, `--interpolate X`: Interpolation method

doc/source/user_guide/compute_tidal_elevations.md

@@ -44,11 +44,18 @@ python compute_tidal_elevations.py --directory <path_to_directory> --tide <model
      * `'csv'` (default)
      * `'netCDF4'`
      * `'HDF5'`
+     * `'geotiff'`
  - `--variables X`: variable names of data in csv, HDF5 or netCDF4 file
      * for csv files: the order of the columns within the file
      * for HDF5 and netCDF4 files: time, y, x and data variable names
- - `--epoch X`: Reference epoch of input time
+ - `-H X`, `--header X`: number of header lines for csv files
+ - `-t X`, `--type X`: input data type
+     * `'drift'`: drift buoys or satellite/airborne altimetry (time per data point)
+     * `'grid'`: spatial grids or images (single time for all data points)
+ - `--epoch X`: reference epoch of input time
      * `'days since 1858-11-17T00:00:00'` (default Modified Julian Days)
+ - `-d X`, `--deltatime X`: input delta time for files without date information
+     * can be set to 0 to use exact calendar date from epoch
  - `--projection X`: spatial projection as EPSG code or PROJ4 string
      * `4326`: latitude and longitude coordinates on WGS84 reference ellipsoid
  - `-I X`, `--interpolate X`: Interpolation method

doc/source/user_guide/spatial.rst

@@ -4,8 +4,8 @@ spatial.py
 
 Utilities for reading and writing spatial data
 
- - Can read ascii, netCDF4, HDF5 files
- - Can output to ascii, netCDF4 or HDF5 files
+ - Can read ascii, netCDF4, HDF5 or geotiff files
+ - Can output to ascii, netCDF4, HDF5 or geotiff files
 
 Calling Sequence
 ================
@@ -93,6 +93,18 @@ General Methods
             `varname` input data variable units in HDF5 file
 
 
+    .. method:: pyTMD.spatial.from_geotiff(filename, compression=None, verbose=False)
+
+        Read data from a geotiff file
+
+        Inputs: full path of input geotiff file
+
+        Options:
+            `compression` geotiff file is compressed using gzip
+
+            `verbose` print geotiff filename
+
+
     .. method:: pyTMD.spatial.to_ascii(output, attributes, filename, delimiter=',', columns=['time','lat','lon','tide'], header=False, verbose=False)
 
         Write data to an ascii file
@@ -148,3 +160,34 @@ General Methods
         Options:
 
             `verbose` print HDF5 file information
+
+
+    .. method:: pyTMD.spatial.to_geotiff(output, attributes, filename, verbose=False, varname='data', dtype=osgeo.gdal.GDT_Float64)
+
+        Write data to a HDF5 file
+
+        Inputs:
+
+            `output` python dictionary of output data
+
+            `attributes` python dictionary of output attributes
+
+            `filename` full path of output HDF5 file
+
+
+        Options:
+
+            `verbose` print geotiff filename
+
+            `varname` output variable name
+
+            `dtype` GDAL data type
+
+
+    .. method:: pyTMD.spatial.expand_dims(obj, varname='data')
+
+        Add a singleton dimension to a spatial dictionary if non-existent
+
+        Options:
+
+            variable name to modify