feat: only download FES files if non-existent or updated (#308)

* refactor: add `_jd_j2000` variable instead of hard coded

README.rst

@@ -106,11 +106,11 @@ Credits
 #######
 
 This project contains work and contributions from the `scientific community <./CONTRIBUTORS.rst>`_.
-The Tidal Model Driver (TMD) Matlab Toolbox was developed by Laurie Padman, Lana Erofeeva and Susan Howard.
-An updated version of the TMD Matlab Toolbox (TMD3) was developed by Chad Greene.
-The OSU Tidal Inversion Software (OTIS) and OSU Tidal Prediction Software (OTPS) were developed by Lana Erofeeva and Gary Egbert (`copyright OSU <http://volkov.oce.orst.edu/tides/COPYRIGHT.pdf>`_, licensed for non-commercial use).
+The Tidal Model Driver (`TMD <https://github.com/EarthAndSpaceResearch/TMD_Matlab_Toolbox_v2.5>`_) Matlab Toolbox was developed by Laurie Padman, Lana Erofeeva and Susan Howard.
+An updated version of the TMD Matlab Toolbox (`TMD3 <https://github.com/chadagreene/Tide-Model-Driver>`_) was developed by Chad Greene.
+The OSU Tidal Inversion Software (OTIS) and OSU Tidal Prediction Software (`OTPS <https://www.tpxo.net/otps>`_) were developed by Lana Erofeeva and Gary Egbert (`copyright OSU <https://www.tpxo.net/tpxo-products-and-registration>`_, licensed for non-commercial use).
 The NASA Goddard Space Flight Center (GSFC) PREdict Tidal Heights (PERTH3) software was developed by Richard Ray and Remko Scharroo.
-An updated and more versatile version of the NASA GSFC tidal prediction software (PERTH5) was developed by Richard Ray.
+An updated and more versatile version of the NASA GSFC tidal prediction software (`PERTH5 <https://codeberg.org/rray/perth5>`_) was developed by Richard Ray.
 
 License
 #######

doc/source/getting_started/Overview.rst

@@ -6,8 +6,8 @@ pyTMD Overview
 It was developed with the goal of supporting science applications for
 airborne and satellite altimetry originally as part of a
 NASA Postdoctoral Program (NPP) Fellowship.
-``pyTMD`` provides data access utilities for ascii, netCDF4, HDF5, and geotiff
-formats.
+``pyTMD`` provides data access utilities for ascii, netCDF4, HDF5, parquet,
+and geotiff formats.
 High-level plotting programs are also provided through the
 use of `Jupyter Notebooks <../user_guide/Examples.html>`_.
 

pyTMD/astro.py

@@ -68,6 +68,8 @@
 _jd_mjd = 2400000.5
 # number of days between MJD and the J2000 epoch
 _mjd_j2000 = 51544.5
+# number of days between the Julian day epoch and J2000 epoch
+_jd_j2000 = _jd_mjd + _mjd_j2000
 # Julian century
 _century = 36525.0
 
@@ -589,8 +591,7 @@ def solar_ephemerides(MJD: np.ndarray, **kwargs):
     x, y, z = 1e3*(SSB_to_Sun.compute(ts.tt) - SSB_to_EMB.compute(ts.tt) -
         EMB_to_Earth.compute(ts.tt))
     # rotate to cartesian (ECEF) coordinates
-    # use UT1 time as input to itrs rotation function
-    rot_z = itrs((ts.ut1 - 2451545.0)/ts.century)
+    rot_z = itrs((ts.ut1 - _jd_j2000)/ts.century)
     X = rot_z[0,0,:]*x + rot_z[0,1,:]*y + rot_z[0,2,:]*z
     Y = rot_z[1,0,:]*x + rot_z[1,1,:]*y + rot_z[1,2,:]*z
     Z = rot_z[2,0,:]*x + rot_z[2,1,:]*y + rot_z[2,2,:]*z
@@ -768,7 +769,7 @@ def lunar_ephemerides(MJD: np.ndarray, **kwargs):
     x, y, z = 1e3*(EMB_to_Moon.compute(ts.tt) - EMB_to_Earth.compute(ts.tt))
     # rotate to cartesian (ECEF) coordinates
     # use UT1 time as input to itrs rotation function
-    rot_z = itrs((ts.ut1 - 2451545.0)/ts.century)
+    rot_z = itrs((ts.ut1 - _jd_j2000)/ts.century)
     X = rot_z[0,0,:]*x + rot_z[0,1,:]*y + rot_z[0,2,:]*z
     Y = rot_z[1,0,:]*x + rot_z[1,1,:]*y + rot_z[1,2,:]*z
     Z = rot_z[2,0,:]*x + rot_z[2,1,:]*y + rot_z[2,2,:]*z

scripts/aviso_fes_tides.py

@@ -40,6 +40,7 @@
 
 UPDATE HISTORY:
     Updated 07/2024: added list and download for FES2022 tide model
+        compare modification times with remote to not overwrite files
     Updated 05/2023: added option to change connection timeout
     Updated 04/2023: using pathlib to define and expand paths
         added option to include AVISO FTP password as argument
@@ -116,7 +117,6 @@ def aviso_fes_tides(MODEL: str,
             LOAD=LOAD,
             CURRENTS=CURRENTS,
             GZIP=GZIP,
-            LOG=LOG,
             MODE=MODE)
     elif MODEL in ('FES2022',):
         aviso_fes_list(MODEL, f, logger,
@@ -284,6 +284,8 @@ def ftp_download(logger, ftp, remote_path, local_dir,
     ):
     # remote and local directory for data product
     remote_file = posixpath.join('auxiliary','tide_model',*remote_path)
+    # if compressing the output file
+    opener = gzip.open if GZIP else open
 
     # Printing files transferred
     remote_ftp_url = posixpath.join('ftp://', ftp.host, remote_file)
@@ -299,74 +301,86 @@ def ftp_download(logger, ftp, remote_path, local_dir,
         member_files = [m for m in tar.getmembers() if tarfile.TarInfo.isfile(m)]
         for m in member_files:
             member = posixpath.basename(m.name) if FLATTEN else m.name
-            fileBasename, fileExtension = posixpath.splitext(m.name)
+            base, sfx = posixpath.splitext(m.name)
             # extract file contents to new file
-            if fileExtension in ('.asc','.nc') and GZIP:
-                local_file = local_dir.joinpath(*posixpath.split(f'{member}.gz'))
-                logger.info(f'\t{str(local_file)}')
-                # recursively create output directory if non-existent
-                local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
-                # extract file to compressed gzip format in local directory
-                with tar.extractfile(m) as fi,gzip.open(local_file, 'wb') as fo:
-                    shutil.copyfileobj(fi, fo)
-            else:
-                local_file = local_dir.joinpath(*posixpath.split(member))
-                logger.info(f'\t{str(local_file)}')
-                # recursively create output directory if non-existent
-                local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
-                # extract file to local directory
-                with tar.extractfile(m) as fi,open(local_file, 'wb') as fo:
-                    shutil.copyfileobj(fi, fo)
+            output = f'{member}.gz' if sfx in ('.asc','.nc') and GZIP else member
+            local_file = local_dir.joinpath(*posixpath.split(output))
+            # check if the local file exists
+            if local_file.exists() and newer(m.mtime, local_file.stat().st_mtime):
+                # check the modification time of the local file
+                # if remote file is newer: overwrite the local file
+                continue
+            # print the file being transferred
+            logger.info(f'\t{str(local_file)}')
+            # recursively create output directory if non-existent
+            local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
+            # extract file to local directory
+            with tar.extractfile(m) as fi,opener(local_file, 'wb') as fo:
+                shutil.copyfileobj(fi, fo)
             # get last modified date of remote file within tar file
             # keep remote modification time of file and local access time
             pathlib.os.utime(local_file, (local_file.stat().st_atime, m.mtime))
             local_file.chmod(mode=MODE)
     elif LZMA:
+        # get last modified date of remote file and convert into unix time
+        mdtm = ftp.sendcmd(f'MDTM {remote_file}')
+        mtime = calendar.timegm(time.strptime(mdtm[4:],"%Y%m%d%H%M%S"))
+        # output file name for compressed and uncompressed cases
+        stem = posixpath.basename(posixpath.splitext(remote_file)[0])
+        base, sfx = posixpath.splitext(stem)
+        # extract file contents to new file
+        output = f'{stem}.gz' if sfx in ('.asc','.nc') and GZIP else stem
+        local_file = local_dir.joinpath(output)
+        # check if the local file exists
+        if local_file.exists() and newer(mtime,local_file.stat().st_mtime):
+            # check the modification time of the local file
+            # if remote file is newer: overwrite the local file
+            return
+        # print the file being transferred
+        logger.info(f'\t{str(local_file)}')
+        # recursively create output directory if non-existent
+        local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
         # copy remote file contents to bytesIO object
         fileobj = io.BytesIO()
         ftp.retrbinary(f'RETR {remote_file}', fileobj.write, blocksize=CHUNK)
         fileobj.seek(0)
-        # decompress lzma file
-        stem = posixpath.basename(posixpath.splitext(remote_file)[0])
-        fileBasename, fileExtension = posixpath.splitext(stem)
-        # extract file contents to new file
-        if fileExtension in ('.asc','.nc') and GZIP:
-            local_file = local_dir.joinpath(f'{stem}.gz')
-            logger.info(f'\t{str(local_file)}')
-            # recursively create output directory if non-existent
-            local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
-            # extract file to compressed gzip format in local directory
-            with lzma.open(fileobj) as fi,gzip.open(local_file, 'wb') as fo:
-                shutil.copyfileobj(fi, fo)
-        else:
-            local_file = local_dir.joinpath(stem)
-            logger.info(f'\t{str(local_file)}')
-            # recursively create output directory if non-existent
-            local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
-            # extract file to local directory
-            with lzma.open(fileobj) as fi,open(local_file, 'wb') as fo:
-                shutil.copyfileobj(fi, fo)
-        # get last modified date of remote file and convert into unix time
-        mdtm = ftp.sendcmd(f'MDTM {remote_file}')
-        mtime = calendar.timegm(time.strptime(mdtm[4:],"%Y%m%d%H%M%S"))
+        # decompress lzma file and extract contents to local directory
+        with lzma.open(fileobj) as fi,opener(local_file, 'wb') as fo:
+            shutil.copyfileobj(fi, fo)
         # get last modified date of remote file within tar file
         # keep remote modification time of file and local access time
         pathlib.os.utime(local_file, (local_file.stat().st_atime, mtime))
         local_file.chmod(mode=MODE)
     else:
         # copy readme and uncompressed files directly
-        local_file = local_dir.joinpath(local_dir,remote_path[-1])
-        logger.info(f'\t{str(local_file)}\n')
-        # copy remote file contents to local file
-        with local_file.open('wb') as f:
-            ftp.retrbinary(f'RETR {remote_file}', f.write, blocksize=CHUNK)
+        stem = posixpath.basename(remote_file)
+        base, sfx = posixpath.splitext(stem)
+        # output file name for compressed and uncompressed cases
+        output = f'{stem}.gz' if sfx in ('.asc','.nc') and GZIP else stem
+        local_file = local_dir.joinpath(output)
         # get last modified date of remote file and convert into unix time
         mdtm = ftp.sendcmd(f'MDTM {remote_file}')
         mtime = calendar.timegm(time.strptime(mdtm[4:],"%Y%m%d%H%M%S"))
+        # check if the local file exists
+        if local_file.exists() and newer(mtime, local_file.stat().st_mtime):
+            # check the modification time of the local file
+            # if remote file is newer: overwrite the local file
+            return
+        # print the file being transferred
+        logger.info(f'\t{str(local_file)}\n')
+        # recursively create output directory if non-existent
+        local_file.parent.mkdir(mode=MODE, parents=True, exist_ok=True)
+        # copy remote file contents to local file
+        with opener(local_file, 'wb') as f:
+            ftp.retrbinary(f'RETR {remote_file}', f.write, blocksize=CHUNK)
         # keep remote modification time of file and local access time
         pathlib.os.utime(local_file, (local_file.stat().st_atime, mtime))
         local_file.chmod(mode=MODE)
 
+# PURPOSE: compare the modification time of two files
+def newer(t1: int, t2: int) -> bool:
+    return (pyTMD.utilities.even(t1) <= pyTMD.utilities.even(t2))
+
 # PURPOSE: create argument parser
 def arguments():
     parser = argparse.ArgumentParser(

test/test_solid_earth.py

@@ -365,3 +365,16 @@ def test_sidereal():
     # expected side real time in hours
     expected = 20.96154017401333
     assert np.isclose(expected, 24.0*ts.st).all()
+
+def test_epochs():
+    """Test that the epoch conversions match expected outputs
+    """
+    # Modified Julian Day (MJD)
+    assert np.isclose(pyTMD.astro._jd_mjd, 2400000.5) 
+    # J2000 time
+    mjd_j2000 = timescale.time.convert_calendar_dates(
+        *timescale.time._j2000_epoch,
+        epoch=timescale.time._mjd_epoch)
+    assert np.isclose(mjd_j2000, pyTMD.astro._mjd_j2000)
+    assert np.isclose(pyTMD.astro._mjd_j2000, 51544.5) 
+    assert np.isclose(pyTMD.astro._jd_j2000, 2451545.0)