STOFS-3D-Atl scripts: minor updates.

src/Utility/Pre-Processing/STOFS-3D-Atl-operation/Source_sink/v7/gen_sourcesink.py

@@ -1,76 +1,23 @@
-import os
-from datetime import datetime, timedelta
+"""
+Generate source and sink files for the NWM model for operational forecast.
+No dependency on pyschism.
+"""
+
+from datetime import datetime
 import glob
 import argparse
 import json
 
 import numpy as np
-import pandas as pd
 from netCDF4 import Dataset
 
-from relocate_source_feeder_lean import SourceSinkIn, relocate_sources
-
-def write_th_file(dataset, timeinterval, fname, issource=True):
-
-    data = []
-    for values, interval in zip(dataset, timeinterval):
-        if issource:
-            data.append(" ".join([f"{interval:G}", *[f'{x: .4f}' for x in values], '\n']))
-        else:
-            data.append(" ".join([f"{interval:G}", *[f'{-x: .4f}' for x in values], '\n']))
-
-    with open(fname, 'w+') as fid:
-        fid.writelines(data)
-
-def write_mth_file(temp, salinity, fname):
 
-    data = []
-    #first record
-    dt = 0.
-    line = [dt]
-    [line.append(x) for x in temp]
-    [line.append(x) for x in salinity]
-    data.append(line)
-
-    #last record
-    dt = 1555200.
-    line = [dt]
-    [line.append(x) for x in temp]
-    [line.append(x) for x in salinity]
-    data.append(line)
-    newset = np.array(data)
-    #data.append(" ".join([f"{dt}", *[f'{x: .1f}' for x in salinity], '\n']))
-
-    #with open(fname, 'w+') as fid:
-    #    fid.write(str(line))
-    np.savetxt(fname, newset, fmt='%.1f')
-
-def get_aggregated_features(nc_feature_id, features):
-
-    # get unique featureIDs from features
-    aggregated_features = []
-    for source_feats in features:
-        aggregated_features.extend(list(source_feats))
-
-    # get the indexes of aggregated_features in nc_feature_id
-    in_file=[]
-    for feature in aggregated_features:
-        idx=np.where(nc_feature_id == int(feature))[0]
-        in_file.append(idx.item())
-
-    in_file_2 = []
-    sidx = 0
-    for source_feats in features:
-        eidx = sidx + len(source_feats)
-        in_file_2.append(in_file[sidx:eidx])
-        sidx = eidx
-    return in_file_2
-
-def streamflow_lookup(file, indexes, threshold=-1e-5):
-    nc = Dataset(file)
+def streamflow_lookup(nwm_file, indexes, threshold=-1e-5):
+    '''look up streamflow in NWM's netcdf files'''
+    nc = Dataset(nwm_file)
     streamflow = nc["streamflow"][:]
     streamflow[np.where(streamflow < threshold)] = 0.0
-    #change masked value to zero
+    # change masked value to zero
     streamflow[np.where(streamflow.mask)] = 0.0
     data = []
     for indxs in indexes:
@@ -79,101 +26,66 @@ def streamflow_lookup(file, indexes, threshold=-1e-5):
     nc.close()
     return data
 
-if __name__ == '__main__':
+
+def main():
     '''
-    Usage: python gen_sourcesink.py "yyyy-mm-dd" or "yyyy-mm-dd hh:mm:ss"
+    Usage: python gen_sourcesink.py yyyy-mm-dd or yyyy-mm-dd hh:mm:ss
     '''
 
-    #cmd line input paramters 
     argparser = argparse.ArgumentParser()
-    argparser.add_argument('date', 
-        type=datetime.fromisoformat, 
-        help="The date format 'YYYY-MM-DD HH:MM:SS'", 
-    )
-    args=argparser.parse_args()
-    startdate=args.date
-    #startdate = datetime(2022, 3, 29, 0)
-
-    # ------------------------- hardwired inputs for operation--------------------------
-    # folder to save outputs (e.g., outdir='./')
-    outdir = './'
+    argparser.add_argument('date', type=datetime.fromisoformat,
+                           help="The date format 'YYYY-MM-DD HH:MM:SS'")
+    args = argparser.parse_args()
+    startdate = args.date
 
-    # folder holding outputs from the original source/sink files (e.g., outputs from hindcast's NWM/gen_source_sink.py):
-    # source_sink.in, sources.json
-    original_source_sink_folder = './original_source_sink'
-
-    # mapping from original to final source locations (e.g., from relocate_map.txt, which is generated from relocate_source_feeder.py)
-    relocate_map = np.loadtxt(f'./relocate_map.txt', dtype=int)
-
-    # source scaling file, containing element id and scale value, for temporary solution to correct bias
-    fname_scale = 'source_scale.txt'
-
-    # folder where nwm*.nc saved (
-    fdir = 'combine'
+    # startdate = datetime(2024, 3, 5)
 
+    # ------------------------- hardwired inputs for operation--------------------------
+    working_dir = '/sciclone/schism10/feiye/STOFS3D-v7/I12w/Source_sink/relocated_source_sink2/'
+    nwm_folder = '/sciclone/schism10/feiye/STOFS3D-v7/I12w/Source_sink/original_source_sink/20240305/'
     layer = 'conus'
     # ------------------------- end hardwired inputs--------------------------
 
-    eid_sources = []
-    times = []
-    dates = []
-
     # read source2fid_dict, note that the order of elements may not be the same as in source_sink.in
-    source2fid_dict = json.load(open(f'./{original_source_sink_folder}/sources.json'))
-
-    #add to the final list, use element order from source_sink.in
-    old_source_sink_in = SourceSinkIn(filename = f'{original_source_sink_folder}/source_sink.in')
-    eid_sources.extend(list(old_source_sink_in.ip_group[0]))
+    source2fid_dict = json.load(open(f'{working_dir}/sources.json', encoding='utf-8'))
 
-    #read nc files
-    files = glob.glob(f'./{fdir}/nwm*.{layer}.nc')
-    files.sort()
+    # read nc files
+    files = sorted(glob.glob(f'{nwm_folder}/nwm*.{layer}.nc'))
     print(f'file 0 is {files[0]}')
-    nc_fid0 = Dataset(files[0])["feature_id"][:]
-
-    # get the index of featureID in the netcdf file in the order of source_sink.in's source element order
-    source2fid = [source2fid_dict[str(x)] for x in eid_sources]
-    src_ncidxs = [[np.where(nc_fid0 == int(fid))[0].item() for fid in fids] for fids in source2fid]
-    # src_idxs = get_aggregated_features(nc_fid0, source2fid_dict.values())
-
-    sources = []
-    for fname in files:
+    nc_fid0 = np.array(Dataset(files[0])["feature_id"])
+
+    # Get the index of featureID in the netcdf file in the order of sources.json's keys
+    # "src_ncidxs" is similar to source2fid_dict's values,
+    # but with the fids replaced by their indexes in the netcdf file
+    src_ncidxs = []
+    for _, fids in source2fid_dict.items():
+        src_ncidxs.append([np.where(nc_fid0 == int(fid))[0].item() for fid in fids])
+
+    times = np.zeros((len(files), 1), dtype=float)  # ntimes x 1
+    sources = np.zeros((len(files), len(src_ncidxs)), dtype=float)  # ntimes x nsrc
+    for it, fname in enumerate(files):
         ds = Dataset(fname)
-        ncfeatureid=ds['feature_id'][:]
-        if not np.all(ncfeatureid == nc_fid0):  # accommodate for product switch (should be rare)
-            print(f'Indexes of feature_id are changed in  {fname}')
-            src_ncidxs = [[np.where(nc_fid0 == int(fid))[0].item() for fid in fids] for fids in source2fid]
-            # src_idxs=get_aggregated_features(ncfeatureid, source2fid_dict.values())
-            nc_fid0 = ncfeatureid
 
-        sources.append(streamflow_lookup(fname, src_ncidxs))
+        # accommodate for different nc_fid0 due to product switch (should be rare)
+        this_nc_fid0 = np.array(ds['feature_id'])
+        if not np.array_equal(nc_fid0, this_nc_fid0):
+            print(f'Indexes of feature_id are changed in {fname}, regenerating src_ncidxs ...')
+            nc_fid0 = this_nc_fid0  # update nc_fid0 for the rest of the files
+            src_ncidxs = []  # update src_ncidxs for the rest of the files
+            for _, fids in source2fid_dict.items():
+                src_ncidxs.append([np.where(nc_fid0 == int(fid))[0].item() for fid in fids])
 
+        sources[it, :] = streamflow_lookup(fname, src_ncidxs)
         model_time = datetime.strptime(ds.model_output_valid_time, "%Y-%m-%d_%H:%M:%S")
-        dates.append(str(model_time))
-        times.append((model_time - startdate).total_seconds())
+        times[it] = (model_time - startdate).total_seconds()
         ds.close()
 
-    #relocate sources; output: vsource.th and msources.th
-    df_vsources = relocate_sources(
-        old_source_sink_in=old_source_sink_in,
-        old_vsource=sources,
-        times=np.array(times),
-        outdir=outdir,
-        relocate_map=relocate_map
-    )
-
-    #source scaling (temporary solution to correct bias)
-    with open(fname_scale) as f:
-       total =  f.readline().split(' ')[0]
-       print(f'Total sources need to be scaled: {total}!')
-       for line in f.read().splitlines():
-           scale_idx = line.split(',')[-2].strip()
-           scale_value = float(line.split(',')[-1])
-           print(scale_idx)
-           print(scale_value)
-
-           print(f'Pre-scaling is {df_vsources[scale_idx]}')
-           df_vsources[scale_idx] = df_vsources[scale_idx]*scale_value
-           print(f'Post-scaling is {df_vsources[scale_idx]}')
-    #overwrite vsource.th with scaled values
-    df_vsources.to_csv(f'{outdir}/vsource.th', index=False, header=False, sep=' ', float_format='%.2f')
+    # sort source2fid_dict by keys (element ids) to conform with vsource generated by pyschism
+    idx = np.argsort(np.array([int(key) for key in source2fid_dict.keys()]))
+    # write source time history file
+    np.savetxt(f'{working_dir}/vsource.th',
+               np.c_[times, sources[:, idx]], fmt='%10.4f', delimiter=' ')
+
+
+if __name__ == '__main__':
+    main()

src/Utility/Pre-Processing/STOFS-3D-Atl-operation/pysh/Shapefiles

@@ -0,0 +1 @@
+../MaskLargeElevation/Shapefiles/

src/Utility/Pre-Processing/STOFS-3D-Atl-operation/pysh/mask_boundary_elev.py

@@ -0,0 +1 @@
+../MaskLargeElevation/mask_boundary_elev.py

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_CL_Survey_NAVD88_lonlat_feet.cpg

@@ -0,0 +1 @@
+UTF-8

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_CL_Survey_NAVD88_lonlat_feet.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_CL_Survey_NAVD88_lonlat_feet.qmd

@@ -0,0 +1,26 @@
+<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
+<qgis version="3.22.5-Bia?owie?a">
+  <identifier></identifier>
+  <parentidentifier></parentidentifier>
+  <language></language>
+  <type></type>
+  <title></title>
+  <abstract></abstract>
+  <links/>
+  <fees></fees>
+  <encoding></encoding>
+  <crs>
+    <spatialrefsys>
+      <wkt></wkt>
+      <proj4></proj4>
+      <srsid>0</srsid>
+      <srid>0</srid>
+      <authid></authid>
+      <description></description>
+      <projectionacronym></projectionacronym>
+      <ellipsoidacronym></ellipsoidacronym>
+      <geographicflag>false</geographicflag>
+    </spatialrefsys>
+  </crs>
+  <extent/>
+</qgis>

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_Centerline_102008.cpg

@@ -0,0 +1 @@
+UTF-8

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_Centerline_102008.prj

@@ -0,0 +1 @@
+PROJCS["North_America_Albers_Equal_Area_Conic",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-96.0],PARAMETER["Standard_Parallel_1",20.0],PARAMETER["Standard_Parallel_2",60.0],PARAMETER["Latitude_Of_Origin",40.0],UNIT["Meter",1.0]]

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2022_MTG_Centerline_102008.qmd

@@ -0,0 +1,43 @@
+<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
+<qgis version="3.22.5-Bia?owie?a">
+  <identifier></identifier>
+  <parentidentifier></parentidentifier>
+  <language></language>
+  <type></type>
+  <title></title>
+  <abstract></abstract>
+  <contact>
+    <name></name>
+    <organization></organization>
+    <position></position>
+    <voice></voice>
+    <fax></fax>
+    <email></email>
+    <role></role>
+  </contact>
+  <links/>
+  <fees></fees>
+  <encoding></encoding>
+  <crs>
+    <spatialrefsys>
+      <wkt></wkt>
+      <proj4></proj4>
+      <srsid>0</srsid>
+      <srid>0</srid>
+      <authid></authid>
+      <description></description>
+      <projectionacronym></projectionacronym>
+      <ellipsoidacronym></ellipsoidacronym>
+      <geographicflag>false</geographicflag>
+    </spatialrefsys>
+  </crs>
+  <extent>
+    <spatial crs="" miny="0" minz="0" maxz="0" maxx="0" minx="0" dimensions="2" maxy="0"/>
+    <temporal>
+      <period>
+        <start></start>
+        <end></end>
+      </period>
+    </temporal>
+  </extent>
+</qgis>

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2024-05-22_MTG Levee Elevations_NAVD88_lonlat_feet.cpg

@@ -0,0 +1 @@
+UTF-8

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2024-05-22_MTG Levee Elevations_NAVD88_lonlat_feet.prj

@@ -0,0 +1 @@
+GEOGCS["GCS_WGS_1984",DATUM["D_WGS_1984",SPHEROID["WGS_1984",6378137.0,298.257223563]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]]

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2024-05-22_MTG Levee Elevations_NAVD88_lonlat_feet.qmd

@@ -0,0 +1,43 @@
+<!DOCTYPE qgis PUBLIC 'http://mrcc.com/qgis.dtd' 'SYSTEM'>
+<qgis version="3.22.5-Bia?owie?a">
+  <identifier></identifier>
+  <parentidentifier></parentidentifier>
+  <language></language>
+  <type></type>
+  <title></title>
+  <abstract></abstract>
+  <contact>
+    <name></name>
+    <organization></organization>
+    <position></position>
+    <voice></voice>
+    <fax></fax>
+    <email></email>
+    <role></role>
+  </contact>
+  <links/>
+  <fees></fees>
+  <encoding></encoding>
+  <crs>
+    <spatialrefsys>
+      <wkt></wkt>
+      <proj4></proj4>
+      <srsid>0</srsid>
+      <srid>0</srid>
+      <authid></authid>
+      <description></description>
+      <projectionacronym></projectionacronym>
+      <ellipsoidacronym></ellipsoidacronym>
+      <geographicflag>false</geographicflag>
+    </spatialrefsys>
+  </crs>
+  <extent>
+    <spatial crs="" miny="0" minz="0" maxz="0" maxx="0" minx="0" dimensions="2" maxy="0"/>
+    <temporal>
+      <period>
+        <start></start>
+        <end></end>
+      </period>
+    </temporal>
+  </extent>
+</qgis>

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2024_MTG_Centerline_102008.cpg

@@ -0,0 +1 @@
+UTF-8

src/Utility/Pre-Processing/STOFS-3D-Atl-shadow-VIMS/Pre_processing/Bathy_edit/Levee/Levee_info/Polygons/2024_MTG_Centerline_102008.prj

@@ -0,0 +1 @@
+PROJCS["North_America_Albers_Equal_Area_Conic",GEOGCS["GCS_North_American_1983",DATUM["D_North_American_1983",SPHEROID["GRS_1980",6378137.0,298.257222101]],PRIMEM["Greenwich",0.0],UNIT["Degree",0.0174532925199433]],PROJECTION["Albers"],PARAMETER["False_Easting",0.0],PARAMETER["False_Northing",0.0],PARAMETER["Central_Meridian",-96.0],PARAMETER["Standard_Parallel_1",20.0],PARAMETER["Standard_Parallel_2",60.0],PARAMETER["Latitude_Of_Origin",40.0],UNIT["Meter",1.0]]