From f0c6eeb61100ac029b79322e44e8e2348b1cc578 Mon Sep 17 00:00:00 2001 From: chunhua zhou Date: Wed, 1 Dec 2021 14:12:51 -0700 Subject: [PATCH] Minor fixes for EnKF cycling (#253) * Minor fixes for EnKF cycling * Get model specific nlevs for EnKF namelist * More fixes for running EnKF on CONUS_3km domain --- scripts/exregional_run_enkf.sh | 13 +++++++++++- ush/config.sh.RRFS_CONUS_3km | 37 +++++++++++++++++++++++++++++----- ush/config_defaults.sh | 7 +++++++ ush/generate_FV3LAM_wflow.sh | 6 ++++++ ush/templates/FV3LAM_wflow.xml | 10 ++++----- 5 files changed, 62 insertions(+), 11 deletions(-) diff --git a/scripts/exregional_run_enkf.sh b/scripts/exregional_run_enkf.sh index 7e46b4d4e..dbd20d137 100755 --- a/scripts/exregional_run_enkf.sh +++ b/scripts/exregional_run_enkf.sh @@ -97,7 +97,7 @@ case $MACHINE in ulimit -v unlimited ulimit -a export OMP_NUM_THREADS=1 - export OMP_STACKSIZE=300M +# export OMP_STACKSIZE=300M APRUN="srun" ;; # @@ -163,6 +163,16 @@ fixgriddir=$FIX_GSI/${PREDEF_GRID_NAME} cp_vrfy ${fixgriddir}/fv3_akbk fv3sar_tile1_akbk.nc cp_vrfy ${fixgriddir}/fv3_grid_spec fv3sar_tile1_grid_spec.nc +# +#----------------------------------------------------------------------- +# +# Get nlons (NX_RES) and nlats (NY_RES) from fv3_grid_spec +# +#----------------------------------------------------------------------- +# + NX_RES=`ncdump -h fv3sar_tile1_grid_spec.nc | grep "grid_xt =" | cut -f3 -d" " ` + NY_RES=`ncdump -h fv3sar_tile1_grid_spec.nc | grep "grid_yt =" | cut -f3 -d" " ` + for imem in $(seq 1 $nens) ensmean; do if [ ${imem} == "ensmean" ]; then @@ -195,6 +205,7 @@ fixgriddir=$FIX_GSI/${PREDEF_GRID_NAME} if [ $imem == 1 ];then ncvarlst_noaxis_time_new fv3_${memcharv0}_tracer > nck_tracer_list.txt ncvarlst_noaxis_time_new fv3_${memcharv0}_dynvars > nck_dynvar_list.txt + nlevs=`ncdump -h fv3_${memcharv0}_tracer | grep "zaxis_1 =" | cut -f3 -d" " ` fi user_nck_dynvar_list=`cat nck_dynvar_list.txt|paste -sd "," - | tr -d '[:space:]'` user_nck_tracer_list=`cat nck_tracer_list.txt |paste -sd "," - | tr -d '[:space:]'` diff --git a/ush/config.sh.RRFS_CONUS_3km b/ush/config.sh.RRFS_CONUS_3km index f05bcf707..178bec107 100644 --- a/ush/config.sh.RRFS_CONUS_3km +++ b/ush/config.sh.RRFS_CONUS_3km @@ -46,10 +46,38 @@ DO_RETRO="TRUE" QUILTING="TRUE" CCPP_PHYS_SUITE="FV3_HRRR" -EXTRN_MDL_ICS_OFFSET_HRS="3" +if [[ ${DO_ENSEMBLE} == "TRUE" ]]; then + NUM_ENS_MEMBERS=9 +# DO_SPPT=TRUE +# SPPT_MAG=0.5 + DO_DACYCLE="FALSE" + DO_SPINUP="FALSE" + DO_SURFACE_CYCLE="FALSE" + DO_NONVAR_CLDANAL="FALSE" + DO_REFL2TTEN="FALSE" +# DO_ENSCONTROL="TRUE" + DO_GSIOBSERVER="TRUE" + DO_ENKFUPDATE="TRUE" +# EXTRN_MDL_NAME_ICS="HRRRDAS" + EXTRN_MDL_NAME_ICS="GDASENKF" +# EXTRN_MDL_NAME_LBCS="GEFS" + EXTRN_MDL_NAME_LBCS="FV3GFS" + EXTRN_MDL_ICS_OFFSET_HRS="6" + EXTRN_MDL_LBCS_OFFSET_HRS="0" + LBC_SPEC_INTVL_HRS="1" + DA_CYCLE_INTERV="1" + ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_ens" +else + EXTRN_MDL_NAME_ICS="FV3GFS" + EXTRN_MDL_NAME_LBCS="FV3GFS" + EXTRN_MDL_ICS_OFFSET_HRS="3" + EXTRN_MDL_LBCS_OFFSET_HRS="0" + LBC_SPEC_INTVL_HRS="1" + DA_CYCLE_INTERV="1" + ARCHIVEDIR="/1year/BMC/wrfruc/rrfs_dev1" +fi + BOUNDARY_LEN_HRS="21" -LBC_SPEC_INTVL_HRS="1" -EXTRN_MDL_LBCS_OFFSET_HRS="0" EXTRN_MDL_LBCS_SEARCH_OFFSET_HRS="0" # avaialble retro period: @@ -80,13 +108,13 @@ POSTPROC_LONG_LEN_HRS="12" #FCST_LEN_HRS_CYCLES=(48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18 48 18 18 18 18 18) for i in {0..23}; do FCST_LEN_HRS_CYCLES[$i]=3; done for i in {0..23..3}; do FCST_LEN_HRS_CYCLES[$i]=12; done -DA_CYCLE_INTERV="1" RESTART_INTERVAL="1" SST_update_hour=01 GVF_update_hour=04 NCORES_RUN_ANAL=240 +NCORES_RUN_ENKF=100 HYBENSMEM_NMIN=66 HALO_BLEND=20 @@ -117,7 +145,6 @@ USE_CUSTOM_POST_CONFIG_FILE="TRUE" TESTBED_FIELDS_FN="testbed_fields_bgdawp.txt" CUSTOM_POST_CONFIG_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/postxconfig-NT-fv3lam_rrfs.txt" CUSTOM_POST_PARAMS_FP="$(cd "$( dirname "${BASH_SOURCE[0]}" )/.." &>/dev/null&&pwd)/fix/upp/params_grib2_tbl_new" -ARCHIVEDIR="/5year/BMC/wrfruc/rrfs_dev1" NCARG_ROOT="/apps/ncl/6.5.0-CentOS6.10_64bit_nodap_gnu447" NCL_HOME="/home/rtrr/RRFS/graphics" NCL_REGION="conus" diff --git a/ush/config_defaults.sh b/ush/config_defaults.sh index c4d6b60c0..3c72f3f49 100644 --- a/ush/config_defaults.sh +++ b/ush/config_defaults.sh @@ -1525,6 +1525,7 @@ NNODES_RUN_FCST="" # This is calculated in the workflow generation scripts, so NNODES_RUN_POST="2" NNODES_RUN_WGRIB2="1" NNODES_RUN_ANAL="16" +NNODES_RUN_ENKF="10" NNODES_PROC_RADAR="2" NNODES_PROC_LIGHTNING="1" NNODES_PROC_BUFR="1" @@ -1535,8 +1536,10 @@ NNODES_RUN_GRAPHICS="1" # Number of cores. # NCORES_RUN_ANAL="4" +NCORES_RUN_ENKF="4" NATIVE_RUN_FCST="--cpus-per-task 2 --exclusive" NATIVE_RUN_ANAL="--cpus-per-task 2 --exclusive" +NATIVE_RUN_ENKF="--cpus-per-task 4 --exclusive" # # Number of MPI processes per node. # @@ -1552,6 +1555,7 @@ PPN_RUN_FCST="24" # This may have to be changed depending on the number of thre PPN_RUN_POST="24" PPN_RUN_WGRIB2="1" PPN_RUN_ANAL="24" +PPN_RUN_ENKF="24" PPN_PROC_RADAR="24" PPN_PROC_LIGHTNING="1" PPN_PROC_BUFR="1" @@ -1569,10 +1573,12 @@ WTIME_GET_EXTRN_LBCS="00:45:00" WTIME_MAKE_ICS="00:30:00" WTIME_MAKE_LBCS="01:30:00" WTIME_RUN_PREPSTART="00:10:00" +WTIME_RUN_PREPSTART_ENSMEAN="00:30:00" WTIME_RUN_FCST="04:30:00" WTIME_RUN_POST="00:15:00" WTIME_RUN_WGRIB2="00:40:00" WTIME_RUN_ANAL="00:30:00" +WTIME_RUN_ENKF="00:45:00" WTIME_PROC_RADAR="00:25:00" WTIME_PROC_LIGHTNING="00:25:00" WTIME_PROC_BUFR="00:25:00" @@ -1603,6 +1609,7 @@ MAXTRIES_ANAL_ENKF="1" MAXTRIES_RUN_POST="1" MAXTRIES_RUN_WGRIB2="1" MAXTRIES_RUN_ANAL="1" +MAXTRIES_RUN_ENKF="1" MAXTRIES_PROCESS_RADARREF="1" MAXTRIES_PROCESS_LIGHTNING="1" MAXTRIES_PROCESS_BUFR="1" diff --git a/ush/generate_FV3LAM_wflow.sh b/ush/generate_FV3LAM_wflow.sh index b632ac35f..03156e24e 100755 --- a/ush/generate_FV3LAM_wflow.sh +++ b/ush/generate_FV3LAM_wflow.sh @@ -229,6 +229,7 @@ settings="\ 'nnodes_run_prepstart': ${NNODES_RUN_PREPSTART} 'nnodes_run_fcst': ${NNODES_RUN_FCST} 'nnodes_run_anal': ${NNODES_RUN_ANAL} + 'nnodes_run_enkf': ${NNODES_RUN_ENKF} 'nnodes_run_post': ${NNODES_RUN_POST} 'nnodes_run_wgrib2': ${NNODES_RUN_WGRIB2} 'nnodes_proc_radar': ${NNODES_PROC_RADAR} @@ -244,6 +245,8 @@ settings="\ 'native_run_fcst': ${NATIVE_RUN_FCST} 'ncores_run_anal': ${NCORES_RUN_ANAL} 'native_run_anal': ${NATIVE_RUN_ANAL} + 'ncores_run_enkf': ${NCORES_RUN_ENKF} + 'native_run_enkf': ${NATIVE_RUN_ENKF} # # Number of logical processes per node for each task. If running without # threading, this is equal to the number of MPI processes per node. @@ -258,6 +261,7 @@ settings="\ 'ppn_run_prepstart': ${PPN_RUN_PREPSTART} 'ppn_run_fcst': ${PPN_RUN_FCST} 'ppn_run_anal': ${PPN_RUN_ANAL} + 'ppn_run_enkf': ${PPN_RUN_ENKF} 'ppn_run_post': ${PPN_RUN_POST} 'ppn_run_wgrib2': ${PPN_RUN_WGRIB2} 'ppn_proc_radar': ${PPN_PROC_RADAR} @@ -277,8 +281,10 @@ settings="\ 'wtime_make_ics': ${WTIME_MAKE_ICS} 'wtime_make_lbcs': ${WTIME_MAKE_LBCS} 'wtime_run_prepstart': ${WTIME_RUN_PREPSTART} + 'wtime_run_prepstart_ensmean': ${WTIME_RUN_PREPSTART_ENSMEAN} 'wtime_run_fcst': ${WTIME_RUN_FCST} 'wtime_run_anal': ${WTIME_RUN_ANAL} + 'wtime_run_enkf': ${WTIME_RUN_ENKF} 'wtime_run_post': ${WTIME_RUN_POST} 'wtime_run_wgrib2': ${WTIME_RUN_WGRIB2} 'wtime_proc_radar': ${WTIME_PROC_RADAR} diff --git a/ush/templates/FV3LAM_wflow.xml b/ush/templates/FV3LAM_wflow.xml index f9f37829f..ac25f2487 100644 --- a/ush/templates/FV3LAM_wflow.xml +++ b/ush/templates/FV3LAM_wflow.xml @@ -1308,7 +1308,7 @@ MODULES_RUN_TASK_FP script. &LOAD_MODULES_RUN_TASK_FP; "&RUN_PREPSTART_TN;" "&JOBSDIR;/JREGIONAL_RUN_PREPSTART_ENSMEAN" {{ nnodes_run_prepstart }}:ppn={{ ppn_run_prepstart }} &NCORES_PER_NODE; - {{ wtime_run_prepstart }} + {{ wtime_run_prepstart_ensmean }} &TAG;_&PREP_CYC_ENSMEAN_TN; &LOGDIR;/&PREP_CYC_ENSMEAN_TN;_@Y@m@d@H.log @@ -1440,13 +1440,13 @@ MODULES_RUN_TASK_FP script. &LOAD_MODULES_RUN_TASK_FP; "&RUN_ENKFUPDT_TN;" &JOBSDIR;/JREGIONAL_RUN_ENKF {% if machine in ["JET", "HERA"] -%} - {{ ncores_run_anal }} - {{ native_run_anal }} &RRFS_RESERVE; + {{ ncores_run_enkf }} + {{ native_run_enkf }} &RRFS_RESERVE; {% else -%} - {{ nnodes_run_anal }}:ppn={{ ppn_run_anal }} + {{ nnodes_run_enkf }}:ppn={{ ppn_run_enkf }} &NCORES_PER_NODE; {% endif -%} - {{ wtime_run_anal }} + {{ wtime_run_enkf }} &TAG;_&RUN_ENKFUPDT_TN; &LOGDIR;/&RUN_ENKFUPDT_TN;_@Y@m@d@H.log GLOBAL_VAR_DEFNS_FP&GLOBAL_VAR_DEFNS_FP;