Merge branch 'master' of https://github.com/geodesymiami/rsmas_insar

minsar/create_insarmaps_jobfile.py

@@ -7,7 +7,7 @@
 import sys
 import glob
 import argparse
-import re
+import h5py
 from pathlib import Path
 from minsar.objects import message_rsmas
 from minsar.objects.auto_defaults import PathFind
@@ -17,6 +17,9 @@
 sys.path.insert(0, os.getenv('SSARAHOME'))
 import password_config as password
 
+REMOTEHOST_INSARMAPS = os.getenv('REMOTEHOST_INSARMAPS')
+REMOTEHOST_INSARMAPS2 = os.getenv('REMOTEHOST_INSARMAPS2')
+
 pathObj = PathFind()
 ############################################################
 EXAMPLE = """example:
@@ -30,7 +33,7 @@ def create_parser():
     epilog = EXAMPLE
     parser = argparse.ArgumentParser(description=synopsis, epilog=epilog, formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument('data_dir', nargs=1, help='Directory with hdf5eos file.\n')
-    parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'all'], default='PS', help='Plot data as image or scatter (default: %(default)s).')
+    parser.add_argument('--dataset', dest='dataset', choices=['PS', 'DS', 'PSDS', 'geo', 'filt*DS','all'], default='geo', help='Plot data as image or scatter (default: %(default)s).')
     parser.add_argument("--queue", dest="queue", metavar="QUEUE", default=os.getenv('QUEUENAME'), help="Name of queue to submit job to")
     parser.add_argument('--walltime', dest='wall_time', metavar="WALLTIME (HH:MM)", default='1:00', help='job walltime (default=1:00)')
 
@@ -48,21 +51,20 @@ def main(iargs=None):
     inps.num_data = 1
     job_obj = JOB_SUBMIT(inps)
 
-    files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5')
+    all_files = glob.glob(inps.work_dir + '/' + inps.data_dir[0] + '/*.he5')
 
-    file_DS = None
-    file_PS = None
+    file_geo = [file for file in all_files if 'DS'  not in file and 'PS' not in file]
+    file_PS = [file for file in all_files if 'PS'  in file]
+    file_DS = [file for file in all_files if 'DS'  in file and 'filt' not in file]
+    file_filtDS = [file for file in all_files if 'DS'  in file and 'filt' in file]
 
-    for file in files:
-        if re.search(r'filt.*DS', file):
-            file_DS = file
-        elif 'PS' in file:
-            file_PS = file
-        else:
-            file_geo = file
-
+
     job_name = f"insarmaps"
     job_file_name = job_name
+
+    with h5py.File(file_geo[0], 'r') as f:
+        ref_lat = float(f.attrs['REF_LAT'])
+        ref_lon = float(f.attrs['REF_LON'])
 
     files = []
     suffixes = []
@@ -75,6 +77,9 @@ def main(iargs=None):
     if inps.dataset == "DS":
         files.append(file_DS)
         suffixes.append("_DS")
+    if inps.dataset == "filt*DS":
+        files.append(file_filtDS)
+        suffixes.append("_filtDS")
     if inps.dataset == "PSDS" or inps.dataset == "DSPS":
         files.append(file_PS)
         files.append(file_DS)
@@ -90,29 +95,33 @@ def main(iargs=None):
 
     command = []
     for file, suffix in zip(files, suffixes):
-        command.append( f'rm -r {inps.data_dir[0]}/JSON{suffix}\n' )
-        command.append( f'hdfeos5_2json_mbtiles.py {file} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8\n' )
+        command.append( f'rm -rf {inps.data_dir[0]}/JSON{suffix}' )
+        command.append( f'hdfeos5_2json_mbtiles.py {file[0]} {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --num-workers 8' )
 
-    command.append('wait\n\n')
+    command.append('wait\n')
     for file, suffix in zip(files, suffixes):
-        path_obj = Path(file)
+        path_obj = Path(file[0])
         mbtiles_file = f"{path_obj.parent}/JSON{suffix}/{path_obj.name}"
         mbtiles_file = mbtiles_file.replace('he5','mbtiles')
-        command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host insarmaps.miami.edu -P rsmastest -U rsmas\@gmail.com --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' )
-        command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host 149.165.174.11 -P insarmaps -U [email protected] --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &\n' )
+        command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.insaruser} -p {password.insarpass} --host {REMOTEHOST_INSARMAPS} -P {password.databasepass} -U {password.databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' )
+        command.append( f'json_mbtiles2insarmaps.py --num-workers 8 -u {password.docker_insaruser} -p {password.docker_insarpass} --host {REMOTEHOST_INSARMAPS2} -P {password.docker_databasepass} -U {password.docker_databaseuser} --json_folder {inps.work_dir}/{inps.data_dir[0]}/JSON{suffix} --mbtiles_file {mbtiles_file} &' )
 
-    command.append('wait\n\n')
-    str = [f'cat >> insarmaps.log<<EOF\n']
+    command.append('wait\n')
+    str = [f'cat >> insarmaps.log<<EOF']
     for file in files:
-        base_name = os.path.basename(file)
+        base_name = os.path.basename(file[0])
         name_without_extension = os.path.splitext(base_name)[0]
-        str.append(f"https://insarmaps.miami.edu/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}\n")
-        str.append(f"149.165.174.11/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}\n")
+        # str.append(f"https://insarmaps.miami.edu/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
+        # str.append(f"149.165.174.11/start/25.78/-80.3/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
+
+        str.append(f"https://{REMOTEHOST_INSARMAPS}/start/{ref_lat:.1f}/{ref_lon:.1f}/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
+        str.append(f"{REMOTEHOST_INSARMAPS2}/start/{ref_lat:.1f}/{ref_lon:.1f}/11.0?flyToDatasetCenter=true&startDataset={name_without_extension}")
 
     str.append( 'EOF' ) 
-    command.append( "".join(str) )
+    command.append( '\n'.join(str) )
+    final_command =[ '\n'.join(command) ]
 
-    job_obj.submit_script(job_name, job_file_name, command, writeOnly='True')
+    job_obj.submit_script(job_name, job_file_name, final_command, writeOnly='True')
     print('jobfile created: ',job_file_name + '.job')
 
     return None

minsar/create_runfiles.py

@@ -95,13 +95,13 @@ def get_bbox_from_template(inps, delta_lat, delta_lon):
     """generates boundingBox string from miaplpy.subset.lalo, mintpy.subset.lalo or intersectsWith POLYGON string"""
 
     if 'miaplpy.subset.lalo' in inps.template.keys():
-        print("Creating topsStack.boundingBox using miaplpy.subset.lalo")
+        print("QQ0 Creating topsStack.boundingBox using miaplpy.subset.lalo: ",inps.template['miaplpy.subset.lalo'] )
         boundingBox_string = convert_subset_lalo_to_boundingBox_string(inps.template['miaplpy.subset.lalo'], delta_lat, delta_lon)
     elif 'mintpy.subset.lalo' in inps.template.keys():
-        print("Creating topsStack.boundingBox using mintpy.subset.lalo")
+        print("QQ0 Creating topsStack.boundingBox using mintpy.subset.lalo: ", inps.template['mintpy.subset.lalo'])
         boundingBox_string = convert_subset_lalo_to_boundingBox_string(inps.template['mintpy.subset.lalo'], delta_lat, delta_lon)
     else:
-        print("Creating topsStack.boundingBox using ssaraopt.intersectsWith")
+        print("QQ0 Creating topsStack.boundingBox using ssaraopt.intersectsWith: ", inps.template['ssaraopt.intersectsWith'])
         #boundingBox_string = convert_intersectsWith_string_to_boundingBox_string(inps.template['ssaraopt.intersectsWith'], delta_lat=0.0, delta_lon=0.0)
         boundingBox_string = convert_intersectsWith_string_to_boundingBox_string(inps.template['ssaraopt.intersectsWith'], delta_lat, delta_lon)
 
@@ -114,8 +114,8 @@ def main(iargs=None):
 
     if 'topsStack.boundingBox' in inps.template:
         if inps.template['topsStack.boundingBox'] == 'None':
-            inps.template['topsStack.boundingBox'] = get_bbox_from_template(inps, delta_lat=0.15, delta_lon=3)
-            print('QQ0',inps.template['topsStack.boundingBox'])
+            inps.template['topsStack.boundingBox'] = get_bbox_from_template(inps, delta_lat=0.10, delta_lon=3)
+            print('QQ0 New topsStack.boundingBox using delta_lat=0.10: ',inps.template['topsStack.boundingBox'])
 
     if not iargs is None:
         input_arguments = iargs

minsar/create_save_hdf5_jobfile.py

@@ -124,6 +124,7 @@ def main(iargs=None):
     command.append( 'source ' + os.path.dirname(os.path.abspath(__file__)) + '/utils/minsar_functions.bash' )
     command.append( f'h5file=`ls *_??????_??????_???????_???????*_{prefix}PS.he5` ; add_ref_lalo_to_file $h5file' )
     command.append( f'h5file=`ls *_??????_??????_???????_???????*_{prefix}DS.he5` ; add_ref_lalo_to_file $h5file' )
+    command.append( f'h5file=`ls *_??????_??????_???????_???????*_filt{prefix}DS.he5` ; add_ref_lalo_to_file $h5file' )
 
     command.append( f'view.py --dpi 150 --noverbose --nodisplay --update --memory 4.0 temporalCoherence_lowpass_gaussian.h5 -c gray -v 0 1 &')
     command.append( f'view.py --dpi 150 --noverbose --nodisplay --update --memory 4.0 maskTempCoh_lowpass_gaussian.h5 -c gray -v 0 1 &')

minsar/generate_download_command.py

@@ -2,8 +2,7 @@
 
 import os
 import sys
-import time
-import datetime
+import re
 import argparse
 from minsar.objects.dataset_template import Template
 from minsar.objects import message_rsmas
@@ -55,7 +54,7 @@ def create_parser():
 
 ###############################################
 def generate_intersects_string(dataset_template, delta_lat=0.0):
-    """generates intersectsWith polygon string from miaplpy.subset.lalo or *Stack.boundingBox"""
+    """generates intersectsWith polygon string from miaplpy.subset.lalo, mintpy.subset.lalo or *Stack.boundingBox"""
 
     if not 'acquisition_mode' in dataset_template.get_options():
         print('WARNING: "acquisition_mode" is not given --> default: tops  (available options: tops, stripmap)')
@@ -65,28 +64,28 @@ def generate_intersects_string(dataset_template, delta_lat=0.0):
 
 
     if 'miaplpy.subset.lalo' in dataset_template.get_options():
-       print("Creating intersectsWith string using miaplpy.subset.lalo")
-       intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo'])
+       print("QQ0 Creating intersectsWith string using miaplpy.subset.lalo: ", dataset_template.get_options()['miaplpy.subset.lalo'])
+       intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['miaplpy.subset.lalo'], delta_lat)
     elif 'mintpy.subset.lalo' in dataset_template.get_options():
-       print("Creating intersectsWith string using mintpy.subset.lalo")
-       intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo'])
+       print("QQ0 Creating intersectsWith string using mintpy.subset.lalo: dataset_template.get_options()['mintpy.subset.lalo']")
+       intersects_string = convert_subset_lalo_to_intersects_string(dataset_template.get_options()['mintpy.subset.lalo'], delta_lat)
     else:
-       print("Creating intersectsWith string using *Stack.boundingBox")
+       print("QQ0 Creating intersectsWith string using *Stack.boundingBox: ", dataset_template.get_options()[prefix + 'Stack.boundingBox'])
        intersects_string = convert_bounding_box_to_intersects_string(dataset_template.get_options()[prefix + 'Stack.boundingBox'], delta_lat)
 
     return intersects_string
 
 ###############################################
-def convert_subset_lalo_to_intersects_string(subset_lalo):
+def convert_subset_lalo_to_intersects_string(subset_lalo, delta_lat=0):
    """ Converts a subset.lalo string in S:N,E:W format (e.g., "2.7:2.8,125.3:125.4") to an intersectsWith polygon string."""
 
    lat_string = subset_lalo.split(',')[0] 
    lon_string = subset_lalo.split(',')[1] 
 
-   min_lat = float(lat_string.split(':')[0])
-   max_lat = float(lat_string.split(':')[1])
-   min_lon = float(lon_string.split(':')[0])
-   max_lon = float(lon_string.split(':')[1])
+   min_lat = float(lat_string.split(':')[0]) - delta_lat
+   max_lat = float(lat_string.split(':')[1]) + delta_lat
+   min_lon = float(lon_string.split(':')[0]) - delta_lat/2
+   max_lon = float(lon_string.split(':')[1]) + delta_lat/2
 
    intersects_string = '--intersectsWith=\'Polygon(({:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, {:.2f} {:.2f}, ' \
          '{:.2f} {:.2f}))\''.format(min_lon, min_lat, min_lon, max_lat, max_lon, max_lat, max_lon, min_lat, min_lon, min_lat)
@@ -119,36 +118,75 @@ def convert_bounding_box_to_intersects_string(string_bbox, delta_lat):
 
    return intersects_string
 
+###############################################
+def convert_intersects_string_to_extent_string(intersects_string):
+    """ Converts a intersectsWith string  to an extent string."""
+
+    match = re.search(r"Polygon\(\((.*?)\)\)", intersects_string)
+    if match:
+        polygon_str = match.group(1)
+    else:
+        polygon_str = None
+
+    lon_list = []
+    lat_list = []
+    bbox_list = polygon_str.split(',')
+    for bbox in bbox_list:
+        lon, lat = map(float, bbox.split())
+        lon_list.append(lon)
+        lat_list.append(lat)
+    lon_list.sort()
+    lat_list.sort()
+
+    extent_list = [lon_list[0], lat_list[0], lon_list [-1], lat_list[-1]]
+    extent_str = ' '.join(map(str, extent_list))
+
+    return extent_str, extent_list
+
 ###############################################
 def generate_download_command(template):
     """ generate ssara download options to use """
 
     dataset_template = Template(template)
     dataset_template.options.update(pathObj.correct_for_ssara_date_format(dataset_template.options))
 
-    ssaraopt_string = dataset_template.generate_ssaraopt_string()
+    ssaraopt_string, ssaraopt_dict = dataset_template.generate_ssaraopt_string()
     ssaraopt = ssaraopt_string.split(' ')
+    if 'end' not in ssaraopt_dict:
+        ssaraopt_dict['end'] = '2099-12-31'
 
     if not any(option.startswith('ssaraopt.intersectsWith') for option in dataset_template.get_options()):
-    # Your code here
-    # if 'ssaraopt.intersectsWith' not in dataset_template.get_options():
-       intersects_string = generate_intersects_string(dataset_template)
+       intersects_string = generate_intersects_string(dataset_template, delta_lat=0.1)
        ssaraopt.insert(2, intersects_string)
-       ssaraopt.append('--maxResults=20000')
+
+    extent_str, extent_list = convert_intersects_string_to_extent_string(intersects_string)
+    print('QQ0 New intersectsWith sting using delta_lat=0.1: ', intersects_string)
+    print('QQ0 New extent sting using delta_lat=0.1: ', extent_str)
 
-    ssara_cmd = ['ssara_federated_query.bash'] + ssaraopt 
-    ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--asfResponseTimeout=300', '--kml', '--print', '--download']
-    asf_cmd = ['asf_search_args.py'] + ssaraopt + ['--Product=SLC', '--print', '--download']
+    ssara_cmd_slc_download_bash = ['ssara_federated_query.bash'] + ssaraopt 
+    ssara_cmd_kml_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml']
+    ssara_cmd_slc_download_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print','--download']
 
-    #asf_cmd = [item for item in asf_cmd if "SENTINEL" not in item]
-    asf_cmd = [item for item in asf_cmd if "maxResults" not in item]
+    asf_cmd_slc_download = ['asf_search_args.py', '--product=SLC'] + ssaraopt + ['--print', '--download']
+    asf_cmd_burst_download = ['asf_search_args.py', '--product=BURST'] + ssaraopt + ['--print', '--download']
+    asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str]
+    asf_cmd_burst2safe = ['burst2stack','--rel-orbit',ssaraopt_dict['relativeOrbit'],'--start-date',ssaraopt_dict['start'],'--end-date',ssaraopt_dict['end'],'--extent',extent_str]
+    asf_cmd_burst2safe.insert(0, "srun -n1 -N1 -A $JOBSHEDULER_PROJECTNAME -p $QUEUENAME  -t 00:25:00 ")    #FA 8/24: it should create a burst2safe.job
 
     with open('ssara_command.txt', 'w') as f:
-        f.write(' '.join(ssara_cmd) + '\n')
-#    with open('ssara_command_python.txt', 'w') as f:
-#        f.write(' '.join(ssara_cmd_python) + '\n')
-    with open('asf_command.txt', 'w') as f:
-        f.write(' '.join(asf_cmd) + '\n')
+        f.write(' '.join(ssara_cmd_slc_download_bash) + '\n')
+
+    with open('asf_slc_download.txt', 'w') as f:
+        f.write(' '.join(asf_cmd_slc_download) + '\n')
+
+    with open('asf_burst_download.txt', 'w') as f:
+        f.write(' '.join(ssara_cmd_kml_download_python) + '\n')
+        f.write(' '.join(asf_cmd_burst_download) + '\n')
+        f.write(' '.join(asf_cmd_burst2safe ) + '\n')
+
+    ssara_cmd_python = ['ssara_federated_query.py'] + ssaraopt + ['--maxResults=20000','--asfResponseTimeout=300', '--kml', '--print']
+    with open('ssara_command_python.txt', 'w') as f:
+       f.write(' '.join(ssara_cmd_python) + '\n')
 
     return