fix: no gzip of TN files in cumulus program (#91)

fix: expose GSFC SLR 5x5 url as option
refactor: camel_case updates for #90

gravity_toolkit/geocenter.py

@@ -465,7 +465,7 @@ def from_UCI(self, geocenter_file, **kwargs):
             #-- file line at count
             line = file_contents[count]
             #--if End of YAML Header is found: set HEADER flag
-            HEADER = bool(re.search("\# End of YAML header",line))
+            HEADER = bool(re.search(r"\# End of YAML header",line))
             #-- add 1 to counter
             count += 1
 
@@ -485,7 +485,7 @@ def from_UCI(self, geocenter_file, **kwargs):
             Loader=yaml.BaseLoader))
 
         #-- compile numerical expression operator
-        regex_pattern = '[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?'
+        regex_pattern = r'[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?'
         rx = re.compile(regex_pattern, re.VERBOSE)
 
         #-- get names and columns of input variables

gravity_toolkit/mascons.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 u"""
 mascons.py
-Written by Tyler Sutterley (04/2022)
+Written by Tyler Sutterley (11/2022)
 Conversion routines for publicly available GRACE/GRACE-FO mascon solutions
 
 PYTHON DEPENDENCIES:
@@ -12,6 +12,7 @@
     mascon2grid.m written by Felix Landerer and David Wiese (JPL)
 
 UPDATE HISTORY:
+    Updated 11/2022: use lowercase keyword arguments
     Updated 04/2022: updated docstrings to numpy documentation format
     Updated 10/2021: publicly released version
     Updated 09/2019: check number of latitude points for using regional grids
@@ -23,6 +24,8 @@
     Updated 12/2015: added TRANSPOSE option to output spatial routines
     Written 07/2013
 """
+import copy
+import warnings
 import numpy as np
 
 def to_gsfc(gdata, lon, lat, lon_center, lat_center, lon_span, lat_span):
@@ -189,7 +192,7 @@ def to_jpl(gdata, lon, lat, lon_bound, lat_bound):
     return mascon_array
 
 def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span,
-    TRANSPOSE=False):
+    **kwargs):
     """
     Converts an input GSFC mascon array to an output gridded field
 
@@ -207,7 +210,7 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span,
         mascon longitudinal central angles
     lat_span: float
         mascon latitudinal central angles
-    TRANSPOSE: bool, default False
+    transpose: bool, default False
         transpose output matrix (lon/lat)
 
     Returns
@@ -224,6 +227,17 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span,
         *Journal of Glaciology*, 59(216), (2013).
         `doi: 10.3189/2013JoG12J147 <https://doi.org/10.3189/2013JoG12J147>`_
     """
+    #-- set default keyword arguments
+    kwargs.setdefault('transpose', False)
+    #-- raise warnings for deprecated keyword arguments
+    deprecated_keywords = dict(TRANSPOSE='transpose')
+    for old,new in deprecated_keywords.items():
+        if old in kwargs.keys():
+            warnings.warn(f"""Deprecated keyword argument {old}.
+                Changed to '{new}'""", DeprecationWarning)
+            #-- set renamed argument to not break workflows
+            kwargs[new] = copy.copy(kwargs[old])
+
     #-- number of mascons
     nmas = len(lon_center)
     #-- convert mascon centers to -180:180
@@ -266,12 +280,12 @@ def from_gsfc(mscdata, grid_spacing, lon_center, lat_center, lon_span, lat_span,
         mdata[I,J] = mscdata[k]
 
     #-- return array
-    if TRANSPOSE:
+    if kwargs['transpose']:
         return mdata.T
     else:
         return mdata
 
-def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False):
+def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, **kwargs):
     """
     Converts an input JPL mascon array to an output gridded field
 
@@ -285,7 +299,7 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False):
         mascon longitudinal bounds from coordinate file
     lat_bound: float
         mascon latitudinal bounds from coordinate file
-    TRANSPOSE: bool, default False
+    transpose: bool, default False
         transpose output matrix (lon/lat)
 
     Returns
@@ -301,14 +315,25 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False):
         *Journal of Geophysical Research: Solid Earth*, 120(4), 2648--2671,
         (2015). `doi: 10.1002/2014JB011547 <https://doi.org/10.1002/2014JB011547>`_
     """
+    #-- set default keyword arguments
+    kwargs.setdefault('transpose', False)
+    #-- raise warnings for deprecated keyword arguments
+    deprecated_keywords = dict(TRANSPOSE='transpose')
+    for old,new in deprecated_keywords.items():
+        if old in kwargs.keys():
+            warnings.warn(f"""Deprecated keyword argument {old}.
+                Changed to '{new}'""", DeprecationWarning)
+            #-- set renamed argument to not break workflows
+            kwargs[new] = copy.copy(kwargs[old])
+
     #-- mascon dimensions
     nmas,nvar = lat_bound.shape
 
     #-- Define latitude and longitude grids
     #-- output lon will not include 360
     #-- output lat will not include 90
-    lon = np.arange(grid_spacing/2.0,360.0+grid_spacing/2.0,grid_spacing)
-    lat = np.arange(-90.0+grid_spacing/2.0,90.0+grid_spacing/2.0,grid_spacing)
+    lon = np.arange(grid_spacing/2.0, 360.0+grid_spacing/2.0, grid_spacing)
+    lat = np.arange(-90.0+grid_spacing/2.0, 90.0+grid_spacing/2.0, grid_spacing)
     nlon,nlat = (len(lon),len(lat))
 
     #-- loop over each mascon bin and assign value to grid points inside bin:
@@ -320,7 +345,7 @@ def from_jpl(mscdata, grid_spacing, lon_bound, lat_bound, TRANSPOSE=False):
         mdata[I,J] = mscdata[k]
 
     #-- return array
-    if TRANSPOSE:
+    if kwargs['transpose']:
         return mdata.T
     else:
         return mdata

gravity_toolkit/time.py

@@ -709,8 +709,7 @@ def convert_julian(JD, **kwargs):
     for old,new in deprecated_keywords.items():
         if old in kwargs.keys():
             warnings.warn(f"""Deprecated keyword argument {old}.
-                Changed to '{new}'""",
-                DeprecationWarning)
+                Changed to '{new}'""", DeprecationWarning)
             #-- set renamed argument to not break workflows
             kwargs[new] = copy.copy(kwargs[old])
 

gravity_toolkit/utilities.py

@@ -10,6 +10,7 @@
 
 UPDATE HISTORY:
     Updated 11/2022: add CMR queries for collection metadata
+        exposed GSFC SLR url for weekly 5x5 harmonics as an option
     Updated 08/2022: add regular expression function for finding files
     Updated 07/2022: add s3 endpoints and buckets for Earthdata Cumulus
     Updated 05/2022: function for extracting bucket name from presigned url
@@ -1125,7 +1126,7 @@ def cmr_product_shortname(mission, center, release, level='L2', version='0'):
     #-- dictionary entries for GRACE-FO Level-2 products
     #-- for each data release
     for rl in ['RL06']:
-        rs = re.findall('\d+',rl).pop().zfill(3)
+        rs = re.findall(r'\d+',rl).pop().zfill(3)
         for c in ['CSR','GFZ','JPL']:
             shortname = gracefo_l2_format.format('L2',c,rs,version)
             cmr_shortname['grace-fo']['L2'][c][rl] = [shortname]
@@ -1578,9 +1579,9 @@ def compile_regex_pattern(PROC, DREL, DSET, mission=None,
 #-- PURPOSE: download geocenter files from Sutterley and Velicogna (2019)
 #-- https://doi.org/10.3390/rs11182108
 #-- https://doi.org/10.6084/m9.figshare.7388540
-def from_figshare(directory,article='7388540',timeout=None,
-    context=ssl.SSLContext(),chunk=16384,verbose=False,fid=sys.stdout,
-    pattern=r'(CSR|GFZ|JPL)_(RL\d+)_(.*?)_SLF_iter.txt$',mode=0o775):
+def from_figshare(directory, article='7388540', timeout=None,
+    context=ssl.SSLContext(), chunk=16384, verbose=False, fid=sys.stdout,
+    pattern=r'(CSR|GFZ|JPL)_(RL\d+)_(.*?)_SLF_iter.txt$', mode=0o775):
     """
     Download [Sutterley2019]_ geocenter files from
     `figshare <https://doi.org/10.6084/m9.figshare.7388540>`_
@@ -1637,9 +1638,9 @@ def from_figshare(directory,article='7388540',timeout=None,
             raise Exception('Checksum mismatch: {0}'.format(f['download_url']))
 
 #-- PURPOSE: send files to figshare using secure FTP uploader
-def to_figshare(files,username=None,password=None,directory=None,
-    timeout=None,context=ssl.SSLContext(ssl.PROTOCOL_TLS),
-    get_ca_certs=False,verbose=False,chunk=8192):
+def to_figshare(files, username=None, password=None, directory=None,
+    timeout=None, context=ssl.SSLContext(ssl.PROTOCOL_TLS),
+    get_ca_certs=False, verbose=False, chunk=8192):
     """
     Send files to figshare using secure `FTP uploader
     <https://help.figshare.com/article/upload-large-datasets-and-bulk-upload-using-the-ftp-uploader-desktop-uploader-or-api>`_
@@ -1697,8 +1698,8 @@ def to_figshare(files,username=None,password=None,directory=None,
 #-- PURPOSE: download satellite laser ranging files from CSR
 #-- http://download.csr.utexas.edu/pub/slr/geocenter/GCN_L1_L2_30d_CF-CM.txt
 #-- http://download.csr.utexas.edu/outgoing/cheng/gct2est.220_5s
-def from_csr(directory,timeout=None,context=ssl.SSLContext(),
-    chunk=16384,verbose=False,fid=sys.stdout,mode=0o775):
+def from_csr(directory, timeout=None, context=ssl.SSLContext(),
+    chunk=16384, verbose=False, fid=sys.stdout, mode=0o775):
     """
     Download `satellite laser ranging (SLR) <http://download.csr.utexas.edu/pub/slr/>`_
     files from the University of Texas Center for Space Research (UTCSR)
@@ -1755,8 +1756,8 @@ def from_csr(directory,timeout=None,context=ssl.SSLContext(),
 #-- PURPOSE: download GravIS and satellite laser ranging files from GFZ
 #-- ftp://isdcftp.gfz-potsdam.de/grace/Level-2/GFZ/RL06_SLR_C20/
 #-- ftp://isdcftp.gfz-potsdam.de/grace/GravIS/GFZ/Level-2B/aux_data/
-def from_gfz(directory,timeout=None,chunk=8192,verbose=False,fid=sys.stdout,
-    mode=0o775):
+def from_gfz(directory, timeout=None, chunk=8192, verbose=False,
+    fid=sys.stdout, mode=0o775):
     """
     Download GravIS and satellite laser ranging (SLR) files from the
         German Research Centre for Geosciences (GeoForschungsZentrum, GFZ)
@@ -1800,8 +1801,10 @@ def from_gfz(directory,timeout=None,chunk=8192,verbose=False,fid=sys.stdout,
 
 #-- PURPOSE: download satellite laser ranging files from GSFC
 #-- https://earth.gsfc.nasa.gov/geo/data/slr
-def from_gsfc(directory,timeout=None,context=ssl.SSLContext(),
-    chunk=16384,verbose=False,fid=sys.stdout,copy=False,mode=0o775):
+def from_gsfc(directory,
+    host='https://earth.gsfc.nasa.gov/sites/default/files/geo/slr-weekly',
+    timeout=None, context=ssl.SSLContext(), chunk=16384, verbose=False,
+    fid=sys.stdout, copy=False, mode=0o775):
     """
     Download `satellite laser ranging (SLR) <https://earth.gsfc.nasa.gov/geo/data/slr/>`_
     files from NASA Goddard Space Flight Center (GSFC)
@@ -1810,6 +1813,8 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(),
     ----------
     directory: str
         download directory
+    host: str, default 'https://earth.gsfc.nasa.gov/sites/default/files/geo/slr-weekly'
+        url for the GSFC SLR weekly fields
     timeout: int or NoneType, default None
         timeout in seconds for blocking operations
     context: obj, default ssl.SSLContext()
@@ -1825,20 +1830,19 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(),
     mode: oct, default 0o775
         permissions mode of output local file
     """
-    #-- GSFC download http server
-    HOST = ['https://earth.gsfc.nasa.gov/','sites','default','files','geo']
     #-- recursively create directory if non-existent
     directory = os.path.abspath(os.path.expanduser(directory))
     if not os.access(directory, os.F_OK):
         os.makedirs(directory, mode)
     #-- download GSFC SLR 5x5 file
     FILE = 'gsfc_slr_5x5c61s61.txt'
     original_md5 = get_hash(os.path.join(directory,FILE))
-    fileID = from_http([*HOST,FILE], timeout=timeout, context=context,
+    fileID = from_http(posixpath.join(host,FILE),
+        timeout=timeout, context=context,
         local=os.path.join(directory,FILE),
         hash=original_md5, chunk=chunk, verbose=verbose,
         fid=fid, mode=mode)
-    #-- can create a copy for archival purposes
+    #-- create a dated copy for archival purposes
     if copy:
         #-- create copy of file for archiving
         #-- read file and extract data date span
@@ -1856,8 +1860,8 @@ def from_gsfc(directory,timeout=None,context=ssl.SSLContext(),
 
 #-- PURPOSE: list a directory on the GFZ ICGEM https server
 #-- http://icgem.gfz-potsdam.de
-def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None,
-    parser=lxml.etree.HTMLParser()):
+def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',
+    timeout=None, parser=lxml.etree.HTMLParser()):
     """
     Parse the table of static gravity field models on the GFZ
     `International Centre for Global Earth Models (ICGEM) <http://icgem.gfz-potsdam.de/>`_
@@ -1889,5 +1893,5 @@ def icgem_list(host='http://icgem.gfz-potsdam.de/tom_longtime',timeout=None,
         colfiles = tree.xpath('//td[@class="tom-cell-modelfile"]//a/@href')
         #-- reduce list of files to find gfc files
         #-- return the dict of model files mapped by name
-        return {re.findall('(.*?).gfc',posixpath.basename(f)).pop():url_split(f)
-            for i,f in enumerate(colfiles) if re.search('gfc$',f)}
+        return {re.findall(r'(.*?).gfc',posixpath.basename(f)).pop():url_split(f)
+            for i,f in enumerate(colfiles) if re.search(r'gfc$',f)}

notebooks/GRACE-Harmonic-Plots.ipynb

@@ -363,7 +363,7 @@
     "RAD = widgets.gaussian.value\n",
     "if (RAD != 0):\n",
     "    wt = 2.0*np.pi*gauss_weights(RAD,LMAX)\n",
-    "    gw_str = '_r{0:0.0f}km'.format(RAD)\n",
+    "    gw_str = f'_r{RAD:0.0f}km'\n",
     "else:\n",
     "    # else = 1\n",
     "    wt = np.ones((LMAX+1))\n",

notebooks/GRACE-Spatial-Maps.ipynb

@@ -273,7 +273,7 @@
     "# number of time steps\n",
     "nt = len(months)\n",
     "# flag for spherical harmonic order\n",
-    "order_str = 'M{0:d}'.format(MMAX) if (MMAX != LMAX) else ''"
+    "order_str = f'M{MMAX:d}' if (MMAX != LMAX) else ''"
    ]
   },
   {
@@ -490,7 +490,7 @@
     "RAD = widgets.gaussian.value\n",
     "if (RAD != 0):\n",
     "    wt = 2.0*np.pi*gauss_weights(RAD,LMAX)\n",
-    "    gw_str = '_r{0:0.0f}km'.format(RAD)\n",
+    "    gw_str = f'_r{RAD:0.0f}km'\n",
     "else:\n",
     "    # else = 1\n",
     "    wt = np.ones((LMAX+1))\n",

scripts/podaac_cumulus.py

@@ -52,6 +52,7 @@
 
 UPDATE HISTORY:
     Updated 11/2022: added CMR queries for GRACE/GRACE-FO technical notes
+        recursively create geocenter directory if not in file system
     Updated 08/2022: moved regular expression function to utilities
         Dynamically select newest version of granules for index
     Updated 04/2022: added option for GRACE/GRACE-FO Level-2 data version
@@ -110,6 +111,9 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[],
     #-- compile regular expression operator for remote files
     R1 = re.compile(r'TN-13_GEOC_(CSR|GFZ|JPL)_(.*?).txt', re.VERBOSE)
     R2 = re.compile(r'TN-(14)_C30_C20_GSFC_SLR.txt', re.VERBOSE)
+    #-- check if geocenter directory exists and recursively create if not
+    local_dir = os.path.join(DIRECTORY,'geocenter')
+    os.makedirs(local_dir,MODE) if not os.path.exists(local_dir) else None
     #-- current time stamp to use for local files
     mtime = time.time()
     #-- for each processing center (CSR, GFZ, JPL)
@@ -131,14 +135,13 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[],
                 #-- access auxiliary data from endpoint
                 if (ENDPOINT == 'data'):
                     http_pull_file(url, mtime, local_file,
-                        GZIP=GZIP, TIMEOUT=TIMEOUT,
-                        CLOBBER=CLOBBER, MODE=MODE)
+                        TIMEOUT=TIMEOUT, CLOBBER=CLOBBER, MODE=MODE)
                 elif (ENDPOINT == 's3'):
                     bucket = gravity_toolkit.utilities.s3_bucket(url)
                     key = gravity_toolkit.utilities.s3_key(url)
                     response = client.get_object(Bucket=bucket, Key=key)
                     s3_pull_file(response, mtime, local_file,
-                        GZIP=GZIP, CLOBBER=CLOBBER, MODE=MODE)
+                        CLOBBER=CLOBBER, MODE=MODE)
 
                 #-- TN-14 SLR C2,0 and C3,0 files
                 url, = [url for url in urls if R2.search(url)]
@@ -147,14 +150,13 @@ def podaac_cumulus(client, DIRECTORY, PROC=[], DREL=[], VERSION=[],
                 #-- access auxiliary data from endpoint
                 if (ENDPOINT == 'data'):
                     http_pull_file(url, mtime, local_file,
-                        GZIP=GZIP, TIMEOUT=TIMEOUT,
-                        CLOBBER=CLOBBER, MODE=MODE)
+                        TIMEOUT=TIMEOUT, CLOBBER=CLOBBER, MODE=MODE)
                 elif (ENDPOINT == 's3'):
                     bucket = gravity_toolkit.utilities.s3_bucket(url)
                     key = gravity_toolkit.utilities.s3_key(url)
                     response = client.get_object(Bucket=bucket, Key=key)
                     s3_pull_file(response, mtime, local_file,
-                        GZIP=GZIP, CLOBBER=CLOBBER, MODE=MODE)
+                        CLOBBER=CLOBBER, MODE=MODE)
 
     #-- GRACE/GRACE-FO AOD1B dealiasing products
     if AOD1B: