Merge remote-tracking branch 'origin/develop' into fix/early_run_configs

* origin/develop:
  Add 3 and 9 hr increment files to IC staging (NOAA-EMC#2876)
  Add diffusion/diag B for aerosol DA and some other needed changes (NOAA-EMC#2738)
  Correct ocean `MOM.res_#` stage copy (NOAA-EMC#2868)
  Support coupling on AWS (NOAA-EMC#2859)
  Add JEDI ATM lgetkf observer and solver jobs (NOAA-EMC#2833)

ci/cases/pr/C96C48_ufs_hybatmDA.yaml

@@ -18,9 +18,7 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml
 
 skip_ci_on_hosts:
-  - hera
   - gaea
   - orion
   - hercules
-  - wcoss2
 

env/HERA.env

@@ -72,6 +72,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -82,13 +92,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

env/HERCULES.env

@@ -76,6 +76,16 @@ case ${step} in
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}
     export APRUN_ATMANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMANLFV3INC}"
  ;;
+ "atmensanlobs")
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+ ;;
+ "atmensanlsol")
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
+ ;;
  "atmensanlletkf")
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -86,12 +96,17 @@ case ${step} in
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
  ;;
- "aeroanlrun")
+ "aeroanlvar")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
+ ;;
+  "aeroanlgenb")
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
  ;;
  "prepobsaero")
 

env/JET.env

@@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -70,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
-
-    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

env/ORION.env

@@ -68,6 +68,15 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN} --cpus-per-task=${NTHREADS_ATMANLVAR}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLOBS}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLSOL}"
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -78,13 +87,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

env/S4.env

@@ -60,6 +60,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs" ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -70,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

env/WCOSS2.env

@@ -53,6 +53,16 @@ elif [[ "${step}" = "atmanlvar" ]]; then
     export NTHREADS_ATMANLVAR=${NTHREADSmax}
     export APRUN_ATMANLVAR="${APRUN}"
 
+elif [[ "${step}" = "atmensanlobs"  ]]; then
+
+    export NTHREADS_ATMENSANLOBS=${NTHREADSmax}
+    export APRUN_ATMENSANLOBS="${APRUN}"
+
+elif [[ "${step}" = "atmensanlsol" ]]; then
+
+    export NTHREADS_ATMENSANLSOL=${NTHREADSmax}
+    export APRUN_ATMENSANLSOL="${APRUN}"
+
 elif [[ "${step}" = "atmensanlletkf" ]]; then
 
     export NTHREADS_ATMENSANLLETKF=${NTHREADSmax}
@@ -63,13 +73,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}
@@ -89,6 +104,33 @@ elif [[ "${step}" = "esnowrecen" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "marinebmat" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+    export APRUN_MARINEBMAT="${APRUN}"
+
+elif [[ "${step}" = "ocnanalrun" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalchkpt" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd --multi-prog"
+
+    export APRUN_OCNANAL="${APRUN}"
+
+elif [[ "${step}" = "ocnanalecen" ]]; then
+
+    export NTHREADS_OCNANALECEN=${NTHREADSmax}
+    export APRUN_OCNANALECEN="${APRUN} --cpus-per-task=${NTHREADS_OCNANALECEN}"
+
+elif [[ "${step}" = "marineanalletkf" ]]; then
+
+    export NTHREADS_MARINEANALLETKF=${NTHREADSmax}
+    export APRUN_MARINEANALLETKF="${APRUN} --cpus-per-task=${NTHREADS_MARINEANALLETKF}"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX

@@ -0,0 +1,46 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \
+    COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
+
+mkdir -p "${COMOUT_CHEM_BMAT}"
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0

jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE

@@ -8,25 +8,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan
 ##############################################
 # Set variables used in the script
 ##############################################
-# shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-gPDY=${GDATE:0:8}
-gcyc=${GDATE:8:2}
-GDUMP="gdas"
-
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS
-
-RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
-    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
-
-mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+    COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
 ###############################################################
 # Run relevant script

jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE

@@ -19,13 +19,13 @@ GDUMP="gdas"
 ##############################################
 
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COM_OBS:COM_OBS_TMPL \
+    COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
-    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
-
-mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
+    COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \
+    COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL
 
 ###############################################################
 # Run relevant script

jobs/JGLOBAL_AERO_ANALYSIS_RUN → jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL

@@ -3,7 +3,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}}
-source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar"
 
 ##############################################
 # Set variables used in the script
@@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py}
+EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py}
 ${EXSCRIPT}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"