Merge branch 'main' into deploy_site

pyproject.toml

@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "coastseg"
 dynamic = ["readme"]
-version = "1.2.1" 
+version = "1.2.3" 
 authors = [
   { name=" Sharon Fitzpatrick", email="[email protected]" },
 ]
@@ -18,7 +18,7 @@ description = "An interactive python package for satellite derived shoreline wor
 dependencies = [
   "h5py>=3.10.0",
   "pyTMD",
-  "coastsat-package>=0.1.62",
+  "coastsat-package>=0.1.65",
   "area",
   "aiohttp",
   "chardet",

scripts/apply_tidal_correction.py

@@ -11,7 +11,6 @@
 import numpy as np
 import pandas as pd
 import pyproj
-import pyTMD.constants
 import pyTMD.io
 import pyTMD.io.model
 import pyTMD.predict
@@ -1167,7 +1166,7 @@ def main():
     # Reset index if you want 'dates' back as a column
     pivot_df.reset_index(inplace=True)    
 
-    pivot_df.to_csv(TIDE_PREDICTIONS_FILE_NAME)
+    pivot_df.to_csv(TIDE_PREDICTIONS_FILE_NAME,index=False)
 
     print(f"Applying tide corrections to {RAW_TIMESERIES_FILE_PATH}")
     tide_corrected_timeseries_df = tidally_correct_timeseries(
@@ -1189,12 +1188,12 @@ def main():
                                                                                                 only_keep_points_on_transects=DROP_INTERSECTIONS,extension='tidally_corrected')
     # optionally save to session location in ROI the tide_corrected_timeseries_df to csv
     tide_corrected_timeseries_df.to_csv(
-        os.path.join(os.getcwd(), "tidally_corrected_transect_time_series_merged.csv")
+        os.path.join(os.getcwd(), "tidally_corrected_transect_time_series_merged.csv"),index=False
     )
     # Tidally correct the raw time series
     print(f"Tidally corrected data saved to {os.path.abspath(TIDALLY_CORRECTED_FILE_NAME)}")
     # Save the Tidally corrected time series
-    tide_corrected_timeseries_df.to_csv(TIDALLY_CORRECTED_FILE_NAME)
+    tide_corrected_timeseries_df.to_csv(TIDALLY_CORRECTED_FILE_NAME,index=False)
 
     # save the time series as a matrix of date vs transect id with the cross_distance as the values
     pivot_df = tide_corrected_timeseries_df.pivot_table(index='dates', columns='transect_id', values='cross_distance', aggfunc='first')
@@ -1204,7 +1203,7 @@ def main():
     # Tidally correct the raw time series
     print(f"Tidally corrected data saved to {os.path.abspath(TIDALLY_CORRECTED_MATRIX_FILE_NAME)}")
     # Save the Tidally corrected time series
-    pivot_df.to_csv(TIDALLY_CORRECTED_MATRIX_FILE_NAME)
+    pivot_df.to_csv(TIDALLY_CORRECTED_MATRIX_FILE_NAME,index=False)
 
 if __name__ == "__main__":
     main()

scripts/aviso_fes_tides.py

@@ -15,7 +15,8 @@
 COMMAND LINE OPTIONS:
     --help: list the command line options
     --directory X: working data directory
-    --user X: username for AVISO FTP servers (email)
+    -U X, --user: username for AVISO FTP servers (email)
+    -P X, --password: password for AVISO FTP servers
     -N X, --netrc X: path to .netrc file for authentication
     --tide X: FES tide model to download
         FES1999
@@ -24,6 +25,8 @@
         FES2014
     --load: download load tide model outputs (fes2014)
     --currents: download tide model current outputs (fes2012 and fes2014)
+    -G, --gzip: compress output ascii and netCDF4 tide files
+    -t X, --timeout X: timeout in seconds for blocking operations
     --log: output log of files downloaded
     -M X, --mode X: Local permissions mode of the files downloaded
 
@@ -35,6 +38,9 @@
     utilities.py: download and management utilities for syncing files
 
 UPDATE HISTORY:
+    Updated 05/2023: added option to change connection timeout
+    Updated 04/2023: using pathlib to define and expand paths
+        added option to include AVISO FTP password as argument
     Updated 11/2022: added encoding for writing ascii files
         use f-strings for formatting verbose or ascii output
     Updated 04/2022: use argparse descriptions within documentation
@@ -48,8 +54,8 @@
     Updated 05/2019: new authenticated ftp host (changed 2018-05-31)
     Written 09/2017
 """
-from __future__ import print_function
-from tqdm import tqdm
+from __future__ import print_function, annotations
+
 import sys
 import os
 import io
@@ -59,12 +65,15 @@
 import logging
 import tarfile
 import getpass
+import pathlib
 import argparse
 import builtins
 import posixpath
 import calendar, time
 import ftplib
 import pyTMD.utilities
+from tqdm import tqdm
+
 
 # FILE SIZES for files in these files
 load_tide_files = {
@@ -165,18 +174,19 @@ def check_files(directory_path: str, files_dict: dict) -> list:
 
 # PURPOSE: download local AVISO FES files with ftp server
 def aviso_fes_tides(
-    MODEL,
-    DIRECTORY=None,
-    USER="",
-    PASSWORD="",
-    LOAD=False,
-    CURRENTS=False,
-    GZIP=False,
-    LOG=False,
+    MODEL: str,
+    DIRECTORY: str | pathlib.Path | None = None,
+    USER: str = '',
+    PASSWORD: str = '',
+    LOAD: bool = False,
+    CURRENTS: bool = False,
+    GZIP: bool = False,
+    TIMEOUT: int | None = None,
+    LOG: bool = False,
     MODE=None,
 ):
     # connect and login to AVISO ftp server
-    f = ftplib.FTP("ftp-access.aviso.altimetry.fr", timeout=1000)
+    f = ftplib.FTP('ftp-access.aviso.altimetry.fr', timeout=TIMEOUT)
     f.login(USER, PASSWORD)
     # check if local directory exists and recursively create if not
     localpath = os.path.join(DIRECTORY, MODEL.lower())
@@ -435,75 +445,54 @@ def arguments():
             model from AVISO.  Decompresses the model tar files into the
             constituent files and auxiliary files.
             """,
-        fromfile_prefix_chars="@",
+        fromfile_prefix_chars="@"
     )
     parser.convert_arg_line_to_args = pyTMD.utilities.convert_arg_line_to_args
     # command line parameters
     # AVISO FTP credentials
-    parser.add_argument(
-        "--user", "-U", type=str, default="", help="Username for AVISO FTP servers"
-    )
-    parser.add_argument(
-        "--netrc",
-        "-N",
-        type=lambda p: os.path.abspath(os.path.expanduser(p)),
-        default=os.path.join(os.path.expanduser("~"), ".netrc"),
-        help="Path to .netrc file for authentication",
-    )
+    parser.add_argument('--user','-U',
+        type=str, default=os.environ.get('AVISO_USERNAME'),
+        help='Username for AVISO Login')
+    parser.add_argument('--password','-W',
+        type=str, default=os.environ.get('AVISO_PASSWORD'),
+        help='Password for AVISO Login')
+    parser.add_argument('--netrc','-N',
+        type=pathlib.Path, default=pathlib.Path().home().joinpath('.netrc'),
+        help='Path to .netrc file for authentication')
     # working data directory
-    parser.add_argument(
-        "--directory",
-        "-D",
-        type=lambda p: os.path.abspath(os.path.expanduser(p)),
-        default=os.getcwd(),
-        help="Working data directory",
-    )
+    parser.add_argument('--directory','-D',
+        type=pathlib.Path, default=pathlib.Path.cwd(),
+        help='Working data directory')
     # FES tide models
-    parser.add_argument(
-        "--tide",
-        "-T",
-        metavar="TIDE",
-        type=str,
-        nargs="+",
-        default=["FES2014"],
-        choices=["FES1999", "FES2004", "FES2012", "FES2014"],
-        help="FES tide model to download",
-    )
+    parser.add_argument('--tide','-T',
+        metavar='TIDE', type=str, nargs='+',
+        default=['FES2014'], choices=['FES1999','FES2004','FES2012','FES2014'],
+        help='FES tide model to download')
     # download FES load tides
-    parser.add_argument(
-        "--load",
-        default=False,
-        action="store_true",
-        help="Download load tide model outputs",
-    )
+    parser.add_argument('--load',
+        default=False, action='store_true',
+        help='Download load tide model outputs')
     # download FES tidal currents
-    parser.add_argument(
-        "--currents",
-        default=False,
-        action="store_true",
-        help="Download tide model current outputs",
-    )
+    parser.add_argument('--currents',
+        default=False, action='store_true',
+        help='Download tide model current outputs')
     # download FES tidal currents
-    parser.add_argument(
-        "--gzip",
-        "-G",
-        default=False,
-        action="store_true",
-        help="Compress output ascii and netCDF4 tide files",
-    )
+    parser.add_argument('--gzip','-G',
+        default=False, action='store_true',
+        help='Compress output ascii and netCDF4 tide files')
+    # connection timeout
+    parser.add_argument('--timeout','-t',
+        type=int, default=1000,
+        help='Timeout in seconds for blocking operations')
     # Output log file in form
     # AVISO_FES_tides_2002-04-01.log
-    parser.add_argument(
-        "--log", "-l", default=False, action="store_true", help="Output log file"
-    )
+    parser.add_argument('--log','-l',
+        default=False, action='store_true',
+        help='Output log file')
     # permissions mode of the local directories and files (number in octal)
-    parser.add_argument(
-        "--mode",
-        "-M",
-        type=lambda x: int(x, base=8),
-        default=0o775,
-        help="Permission mode of directories and files downloaded",
-    )
+    parser.add_argument('--mode','-M',
+        type=lambda x: int(x,base=8), default=0o775,
+        help='Permission mode of directories and files downloaded')
     # return the parser
     return parser
 
@@ -516,26 +505,28 @@ def main():
 
     # AVISO FTP Server hostname
     HOST = "ftp-access.aviso.altimetry.fr"
-    # get AVISO FTP Server credentials
-    try:
-        args.user, _, PASSWORD = netrc.netrc(args.netrc).authenticators(HOST)
-    except:
-        # check that AVISO FTP Server credentials were entered
-        if not args.user:
-            prompt = f"Username for {HOST}: "
-            args.user = builtins.input(prompt)
+    # AVISO FTP Server hostname
+    HOST = 'ftp-access.aviso.altimetry.fr'
+    # get authentication
+    if not args.user and not args.netrc.exists():
+        # check that AVISO credentials were entered
+        args.user = builtins.input(f'Username for {HOST}: ')
+        # enter password securely from command-line
+        args.password = getpass.getpass(f'Password for {args.user}@{HOST}: ')
+    elif args.netrc.exists():
+        args.user,_,args.password = netrc.netrc(args.netrc).authenticators(HOST)
+    elif args.user and not args.password:
         # enter password securely from command-line
-        prompt = f"Password for {args.user}@{HOST}: "
-        PASSWORD = getpass.getpass(prompt)
+        args.password = getpass.getpass(f'Password for {args.user}@{HOST}: ')
 
     # check internet connection before attempting to run program
-    if pyTMD.utilities.check_ftp_connection(HOST, args.user, PASSWORD):
+    if pyTMD.utilities.check_ftp_connection(HOST,args.user,args.password):
         for m in args.tide:
             aviso_fes_tides(
                 m,
                 DIRECTORY=args.directory,
                 USER=args.user,
-                PASSWORD=PASSWORD,
+                PASSWORD=args.password,
                 LOAD=args.load,
                 CURRENTS=args.currents,
                 GZIP=args.gzip,

scripts/gen_csv_per_transect_tide_corrected.py

@@ -12,7 +12,7 @@ def save_csv_per_id(
     for uid in unique_ids:
         new_filename = f"{uid}_{filename}"
         new_df = df[df[id_column_name] == uid]
-        new_df.to_csv(os.path.join(save_location, new_filename))
+        new_df.to_csv(os.path.join(save_location, new_filename),index=False)
 
 # 1. Enter the path to the csv file that contains the tide corrected time series
 # - replace the path below with the path to the csv file

scripts/preview_images_for_dates.py

@@ -12,6 +12,19 @@
 def get_roi_polygon(
     roi_gdf: gpd.GeoDataFrame, roi_id: Optional[int] = None, index: Optional[int] = None
 ) -> Optional[List[List[float]]]:
+    """
+    Get the polygon coordinates for a region of interest (ROI) in a GeoDataFrame.
+
+    Args:
+        roi_gdf (gpd.GeoDataFrame): The GeoDataFrame containing the ROI data.
+        roi_id (Optional[int], optional): The ID of the ROI to retrieve. Defaults to None.
+        index (Optional[int], optional): The index of the ROI to retrieve. Defaults to None.
+
+    Returns:
+        Optional[List[List[float]]]: The polygon coordinates of the ROI as a list of lists of floats,
+        or None if the ROI is not found.
+
+    """
     if roi_id is not None:
         geoseries = roi_gdf[roi_gdf["id"] == roi_id]["geometry"]
     elif index is not None:
@@ -157,7 +170,29 @@ def count_images_in_ee_collection(
     return image_counts
 
 
-def process_roi(polygon, identifier, settings):
+def get_image_counts_for_roi(polygon:list, identifier:str, settings:dict):
+    """
+    Retrieves the image counts for a given region of interest (ROI) within a specified date range.
+    Prints the image counts for each satellite in the specified date ranges for the ROI in the format:
+    
+        ROI ID: <ROI ID>
+        L7: 0 images
+        L8: 1 images
+        L9: 3 images
+        S2: 9 images
+
+    Args:
+        polygon (list): The coordinates of the polygon representing the ROI.
+        identifier (str): The ID for the ROI.
+        settings (dict): A dictionary containing the settings for image retrieval.
+            The settings should contain the following keys:
+            - dates: A list of date ranges to query.
+            - sat_list: A list of satellite names to query.
+            
+    Returns:
+        list: A list of dictionaries, where each dictionary contains the ROI ID, date range, image count, and polygon.
+
+    """
     results = []
     for start_date, end_date in tqdm(
         settings["dates"],
@@ -184,6 +219,20 @@ def process_roi(polygon, identifier, settings):
 def preview_images_for_rois(
     rois_gdf: gpd.GeoDataFrame, selected_ids: Set[int], settings: Dict
 ):
+    """
+    Preview the number images available for the given regions of interest (ROIs).
+    
+    Provides the number of images availabe for each satellite in the specified date ranges for each ROI.
+
+    Args:
+        rois_gdf (gpd.GeoDataFrame): A GeoDataFrame containing the ROIs.
+        selected_ids (Set[int]): A set of selected ROI IDs. If empty, all ROI IDs will be used.
+        settings (Dict): A dictionary containing the processing settings.
+
+    Returns:
+        results: The results of processing the ROIs.
+
+    """
     if "id" in rois_gdf.columns:
         if selected_ids:
             roi_ids = selected_ids
@@ -195,15 +244,15 @@ def preview_images_for_rois(
             pbar.set_description(f"Querying API for ROI: {roi_id}")
             polygon = get_roi_polygon(rois_gdf, roi_id=roi_id)
             if polygon:
-                results = process_roi(polygon, roi_id, settings)
+                results = get_image_counts_for_roi(polygon, roi_id, settings)
     else:
         pbar = tqdm(range(len(rois_gdf)), desc="Querying API", leave=False, unit="roi")
         for index in pbar:
             pbar.set_description(f"Querying API for ROI: {index}")
             # for index in tqdm(range(len(rois_gdf)), desc="Querying API", leave=False):
             polygon = get_roi_polygon(rois_gdf, index=index)
             if polygon:
-                results = process_roi(polygon, index, settings)
+                results = get_image_counts_for_roi(polygon, index, settings)
 
     return results