Minor fixes for EnKF cycling (#253)

* Minor fixes for EnKF cycling

* Get model specific nlevs for EnKF namelist

* More fixes for running EnKF on CONUS_3km domain

scripts/exregional_run_enkf.sh

@@ -97,7 +97,7 @@ case $MACHINE in
   ulimit -v unlimited
   ulimit -a
   export OMP_NUM_THREADS=1
-  export OMP_STACKSIZE=300M
+#  export OMP_STACKSIZE=300M
   APRUN="srun"
   ;;
 #
@@ -163,6 +163,16 @@ fixgriddir=$FIX_GSI/${PREDEF_GRID_NAME}
  cp_vrfy ${fixgriddir}/fv3_akbk           fv3sar_tile1_akbk.nc
  cp_vrfy ${fixgriddir}/fv3_grid_spec      fv3sar_tile1_grid_spec.nc
 
+#
+#-----------------------------------------------------------------------
+#
+# Get nlons (NX_RES) and nlats (NY_RES) from  fv3_grid_spec
+#
+#-----------------------------------------------------------------------
+#
+ NX_RES=`ncdump -h fv3sar_tile1_grid_spec.nc | grep "grid_xt =" | cut -f3 -d" " `
+ NY_RES=`ncdump -h fv3sar_tile1_grid_spec.nc | grep "grid_yt =" | cut -f3 -d" " `
+
  for imem in  $(seq 1 $nens) ensmean; do
 
      if [ ${imem} == "ensmean" ]; then
@@ -195,6 +205,7 @@ fixgriddir=$FIX_GSI/${PREDEF_GRID_NAME}
      if [ $imem == 1 ];then   
          ncvarlst_noaxis_time_new fv3_${memcharv0}_tracer > nck_tracer_list.txt
          ncvarlst_noaxis_time_new fv3_${memcharv0}_dynvars > nck_dynvar_list.txt
+         nlevs=`ncdump -h fv3_${memcharv0}_tracer | grep "zaxis_1 =" | cut -f3 -d" " `
      fi
      user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," -  | tr -d '[:space:]'`
      user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," -  | tr -d '[:space:]'` 

ush/config.sh.RRFS_CONUS_3km

@@ -46,10 +46,38 @@ DO_RETRO="TRUE"
 QUILTING="TRUE"
 CCPP_PHYS_SUITE="FV3_HRRR"
 
-EXTRN_MDL_ICS_OFFSET_HRS="3"
+if [[ ${DO_ENSEMBLE}  == "TRUE" ]]; then
+   NUM_ENS_MEMBERS=9
+#   DO_SPPT=TRUE
+#   SPPT_MAG=0.5
+   DO_DACYCLE="FALSE"
+   DO_SPINUP="FALSE"
+   DO_SURFACE_CYCLE="FALSE"
+   DO_NONVAR_CLDANAL="FALSE"
+   DO_REFL2TTEN="FALSE"
+#   DO_ENSCONTROL="TRUE"
+   DO_GSIOBSERVER="TRUE"
+   DO_ENKFUPDATE="TRUE"
+#   EXTRN_MDL_NAME_ICS="HRRRDAS"
+   EXTRN_MDL_NAME_ICS="GDASENKF"
+#   EXTRN_MDL_NAME_LBCS="GEFS"
+   EXTRN_MDL_NAME_LBCS="FV3GFS"
+   EXTRN_MDL_ICS_OFFSET_HRS="6"
+   EXTRN_MDL_LBCS_OFFSET_HRS="0"
+   LBC_SPEC_INTVL_HRS="1"
+   DA_CYCLE_INTERV="1"
+   ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens"
+else
+   EXTRN_MDL_NAME_ICS="FV3GFS"
+   EXTRN_MDL_NAME_LBCS="FV3GFS"
+   EXTRN_MDL_ICS_OFFSET_HRS="3"
+   EXTRN_MDL_LBCS_OFFSET_HRS="0"
+   LBC_SPEC_INTVL_HRS="1"
+   DA_CYCLE_INTERV="1"
+   ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1"
+fi
+
 BOUNDARY_LEN_HRS="21"
-LBC_SPEC_INTVL_HRS="1"
-EXTRN_MDL_LBCS_OFFSET_HRS="0"
 EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0"
 
 # avaialble retro period:
@@ -80,13 +108,13 @@ POSTPROC_LONG_LEN_HRS="12"
 #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18)
 for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=3; done
 for i in {0..23..3}; do FCST_LEN_HRS_CYCLES[$i]=12; done
-DA_CYCLE_INTERV="1"
 RESTART_INTERVAL="1"
 
 SST_update_hour=01
 GVF_update_hour=04
 
 NCORES_RUN_ANAL=240
+NCORES_RUN_ENKF=100
 HYBENSMEM_NMIN=66
 HALO_BLEND=20
 
@@ -117,7 +145,6 @@ USE_CUSTOM_POST_CONFIG_FILE="TRUE"
 TESTBED_FIELDS_FN="testbed_fields_bgdawp.txt"
 CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt"
 CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new"
-ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_dev1"
 NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447"
 NCL_HOME="/home/rtrr/RRFS/graphics"
 NCL_REGION="conus"

ush/config_defaults.sh

@@ -1525,6 +1525,7 @@ NNODES_RUN_FCST=""  # This is calculated in the workflow generation scripts, so
 NNODES_RUN_POST="2"
 NNODES_RUN_WGRIB2="1"
 NNODES_RUN_ANAL="16"
+NNODES_RUN_ENKF="10"
 NNODES_PROC_RADAR="2"
 NNODES_PROC_LIGHTNING="1"
 NNODES_PROC_BUFR="1"
@@ -1535,8 +1536,10 @@ NNODES_RUN_GRAPHICS="1"
 # Number of cores.
 #
 NCORES_RUN_ANAL="4"
+NCORES_RUN_ENKF="4"
 NATIVE_RUN_FCST="--cpus-per-task 2 --exclusive"
 NATIVE_RUN_ANAL="--cpus-per-task 2 --exclusive"
+NATIVE_RUN_ENKF="--cpus-per-task 4 --exclusive"
 #
 # Number of MPI processes per node.
 #
@@ -1552,6 +1555,7 @@ PPN_RUN_FCST="24"  # This may have to be changed depending on the number of thre
 PPN_RUN_POST="24"
 PPN_RUN_WGRIB2="1"
 PPN_RUN_ANAL="24"
+PPN_RUN_ENKF="24"
 PPN_PROC_RADAR="24"
 PPN_PROC_LIGHTNING="1"
 PPN_PROC_BUFR="1"
@@ -1569,10 +1573,12 @@ WTIME_GET_EXTRN_LBCS="00:45:00"
 WTIME_MAKE_ICS="00:30:00"
 WTIME_MAKE_LBCS="01:30:00"
 WTIME_RUN_PREPSTART="00:10:00"
+WTIME_RUN_PREPSTART_ENSMEAN="00:30:00"
 WTIME_RUN_FCST="04:30:00"
 WTIME_RUN_POST="00:15:00"
 WTIME_RUN_WGRIB2="00:40:00"
 WTIME_RUN_ANAL="00:30:00"
+WTIME_RUN_ENKF="00:45:00"
 WTIME_PROC_RADAR="00:25:00"
 WTIME_PROC_LIGHTNING="00:25:00"
 WTIME_PROC_BUFR="00:25:00"
@@ -1603,6 +1609,7 @@ MAXTRIES_ANAL_ENKF="1"
 MAXTRIES_RUN_POST="1"
 MAXTRIES_RUN_WGRIB2="1"
 MAXTRIES_RUN_ANAL="1"
+MAXTRIES_RUN_ENKF="1"
 MAXTRIES_PROCESS_RADARREF="1"
 MAXTRIES_PROCESS_LIGHTNING="1"
 MAXTRIES_PROCESS_BUFR="1"

ush/generate_FV3LAM_wflow.sh

@@ -229,6 +229,7 @@ settings="\
   'nnodes_run_prepstart': ${NNODES_RUN_PREPSTART}
   'nnodes_run_fcst': ${NNODES_RUN_FCST}
   'nnodes_run_anal': ${NNODES_RUN_ANAL}
+  'nnodes_run_enkf': ${NNODES_RUN_ENKF}
   'nnodes_run_post': ${NNODES_RUN_POST}
   'nnodes_run_wgrib2': ${NNODES_RUN_WGRIB2}
   'nnodes_proc_radar': ${NNODES_PROC_RADAR}
@@ -244,6 +245,8 @@ settings="\
   'native_run_fcst': ${NATIVE_RUN_FCST}
   'ncores_run_anal': ${NCORES_RUN_ANAL}
   'native_run_anal': ${NATIVE_RUN_ANAL}
+  'ncores_run_enkf': ${NCORES_RUN_ENKF}
+  'native_run_enkf': ${NATIVE_RUN_ENKF}
 #
 # Number of logical processes per node for each task.  If running without
 # threading, this is equal to the number of MPI processes per node.
@@ -258,6 +261,7 @@ settings="\
   'ppn_run_prepstart': ${PPN_RUN_PREPSTART}
   'ppn_run_fcst': ${PPN_RUN_FCST}
   'ppn_run_anal': ${PPN_RUN_ANAL}
+  'ppn_run_enkf': ${PPN_RUN_ENKF}
   'ppn_run_post': ${PPN_RUN_POST}
   'ppn_run_wgrib2': ${PPN_RUN_WGRIB2}
   'ppn_proc_radar': ${PPN_PROC_RADAR}
@@ -277,8 +281,10 @@ settings="\
   'wtime_make_ics': ${WTIME_MAKE_ICS}
   'wtime_make_lbcs': ${WTIME_MAKE_LBCS}
   'wtime_run_prepstart': ${WTIME_RUN_PREPSTART}
+  'wtime_run_prepstart_ensmean': ${WTIME_RUN_PREPSTART_ENSMEAN}
   'wtime_run_fcst': ${WTIME_RUN_FCST}
   'wtime_run_anal': ${WTIME_RUN_ANAL}
+  'wtime_run_enkf': ${WTIME_RUN_ENKF}
   'wtime_run_post': ${WTIME_RUN_POST}
   'wtime_run_wgrib2': ${WTIME_RUN_WGRIB2}
   'wtime_proc_radar': ${WTIME_PROC_RADAR}

ush/templates/FV3LAM_wflow.xml

@@ -1308,7 +1308,7 @@ MODULES_RUN_TASK_FP script.
     <command>&LOAD_MODULES_RUN_TASK_FP; "&RUN_PREPSTART_TN;" "&JOBSDIR;/JREGIONAL_RUN_PREPSTART_ENSMEAN"</command>
     <nodes>{{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }}</nodes>
     <nodesize>&NCORES_PER_NODE;</nodesize>
-    <walltime>{{ wtime_run_prepstart }}</walltime>
+    <walltime>{{ wtime_run_prepstart_ensmean }}</walltime>
     <jobname>&TAG;_&PREP_CYC_ENSMEAN_TN;</jobname>
     <join><cyclestr>&LOGDIR;/&PREP_CYC_ENSMEAN_TN;_@Y@m@[email protected]</cyclestr></join>
 
@@ -1440,13 +1440,13 @@ MODULES_RUN_TASK_FP script.
     <command>&LOAD_MODULES_RUN_TASK_FP; "&RUN_ENKFUPDT_TN;" &JOBSDIR;/JREGIONAL_RUN_ENKF</command>
 
     {% if machine in ["JET", "HERA"] -%}
-    <cores>{{ ncores_run_anal }}</cores>
-    <native>{{ native_run_anal }} &RRFS_RESERVE;</native>
+    <cores>{{ ncores_run_enkf }}</cores>
+    <native>{{ native_run_enkf }} &RRFS_RESERVE;</native>
     {% else -%}
-    <nodes>{{ nnodes_run_anal }}:ppn={{ ppn_run_anal }}</nodes>
+    <nodes>{{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }}</nodes>
     <nodesize>&NCORES_PER_NODE;</nodesize>
     {% endif -%}
-    <walltime>{{ wtime_run_anal }}</walltime>
+    <walltime>{{ wtime_run_enkf }}</walltime>
     <jobname><cyclestr>&TAG;_&RUN_ENKFUPDT_TN;</cyclestr></jobname>
     <join><cyclestr>&LOGDIR;/&RUN_ENKFUPDT_TN;_@Y@m@[email protected]</cyclestr></join>
     <envar><name>GLOBAL_VAR_DEFNS_FP</name><value>&GLOBAL_VAR_DEFNS_FP;</value></envar>