Add diffusion/diag B for aerosol DA and some other needed changes (#2738

) This PR adds in support for computing files needed for the aerosol analysis **B**. This includes a new task, `aeroanlgenb`. This work was performed by both me and @andytangborn Resolves #2501 Resolves #2737 --------- Co-authored-by: Andrew.Tangborn <[email protected]> Co-authored-by: Walter Kolczynski - NOAA <[email protected]>
NOAA-EMC · Aug 29, 2024 · 7f160f6 · 7f160f6
1 parent 5ccdd8b
commit 7f160f6
Show file tree

Hide file tree

Showing 36 changed files with 809 additions and 201 deletions.
diff --git a/env/HERA.env b/env/HERA.env
@@ -92,13 +92,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -96,12 +96,17 @@ case ${step} in
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
  ;;
- "aeroanlrun")
+ "aeroanlvar")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
+ ;;
+  "aeroanlgenb")
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
  ;;
  "prepobsaero")
 

diff --git a/env/JET.env b/env/JET.env
@@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
-
-    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

diff --git a/env/ORION.env b/env/ORION.env
@@ -87,13 +87,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

diff --git a/env/S4.env b/env/S4.env
@@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -73,13 +73,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
     export APRUN_ATMENSANLFV3INC="${APRUN}"
 
-elif [[ "${step}" = "aeroanlrun" ]]; then
+elif [[ "${step}" = "aeroanlvar" ]]; then
 
     export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}"
 
     export NTHREADS_AEROANL=${NTHREADSmax}
     export APRUN_AEROANL="${APRUN}"
 
+elif [[ "${step}" = "aeroanlgenb" ]]; then
+
+    export NTHREADS_AEROANLGENB=${NTHREADSmax}
+    export APRUN_AEROANLGENB="${APRUN}"
+
 elif [[ "${step}" = "prepobsaero" ]]; then
 
     export NTHREADS_PREPOBSAERO=${NTHREADS1}

diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX
@@ -0,0 +1,46 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb"
+
+##############################################
+# Set variables used in the script
+##############################################
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \
+    COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
+
+mkdir -p "${COMOUT_CHEM_BMAT}"
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py}
+${EXSCRIPT}
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0
diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE
@@ -8,25 +8,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan
 ##############################################
 # Set variables used in the script
 ##############################################
-# shellcheck disable=SC2153
-GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours")
-gPDY=${GDATE:0:8}
-gcyc=${GDATE:8:2}
-GDUMP="gdas"
-
 
 ##############################################
 # Begin JOB SPECIFIC work
 ##############################################
 
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS
-
-RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
-    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
-
-mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \
+    COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE
@@ -19,13 +19,13 @@ GDUMP="gdas"
 ##############################################
 
 # Generate COM variables from templates
-YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COM_OBS:COM_OBS_TMPL \
+    COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL
 
 RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \
-    COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \
-    COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL
-
-mkdir -m 775 -p "${COM_CHEM_ANALYSIS}"
+    COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \
+    COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL
 
 ###############################################################
 # Run relevant script

diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN → jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL b/jobs/JGLOBAL_AERO_ANALYSIS_RUN → jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL
@@ -3,7 +3,7 @@
 source "${HOMEgfs}/ush/preamble.sh"
 export WIPE_DATA="NO"
 export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}}
-source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar"
 
 ##############################################
 # Set variables used in the script
@@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr
 ###############################################################
 # Run relevant script
 
-EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py}
+EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py}
 ${EXSCRIPT}
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"

diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh
@@ -0,0 +1,19 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="aeroanlgenb"
+export jobid="${job}.$$"
+
+###############################################################
+
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX"
+status=$?
+exit "${status}"
diff --git a/jobs/rocoto/aeroanlrun.sh → jobs/rocoto/aeroanlvar.sh b/jobs/rocoto/aeroanlrun.sh → jobs/rocoto/aeroanlvar.sh
@@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh"
 status=$?
 [[ ${status} -ne 0 ]] && exit "${status}"
 
-export job="aeroanlrun"
+export job="aeroanlvar"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN"
+"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL"
 status=$?
 exit "${status}"
diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl
@@ -5,20 +5,36 @@
 
 echo "BEGIN: config.aeroanl"
 
-export CASE_ANL=${CASE}
+# define analysis resolution based on deterministic res
+case ${CASE} in
+  "C1152" | "C768" | "C384" | "C192")
+    CASE_ANL="C192"
+    ;;
+  "C96" | "C48")
+    CASE_ANL=${CASE}
+    ;;
+  *)
+    echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution"
+    exit 4
+esac
+export CASE_ANL
 export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2"
-export STATICB_TYPE='identity'
+export STATICB_TYPE='diffusion'
 export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2"
-export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/"
-export BERROR_DATE="20160630.000000"
+export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b"
 
 export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2"
 export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2"
 
+export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2"
+export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2"
+
 export io_layout_x=@IO_LAYOUT_X@
 export io_layout_y=@IO_LAYOUT_Y@
 
 export JEDIEXE="${EXECgfs}/gdas.x"
+export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x"
+export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x"
 
 if [[ "${DOIAU}" == "YES" ]]; then
   export aero_bkg_times="3,6,9"

diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb
@@ -0,0 +1,29 @@
+#!/bin/bash -x
+
+########## config.aeroanlgenb ##########
+# Aerosol Variance specific
+
+echo "BEGIN: config.aeroanlgenb"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" aeroanlgenb
+
+export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2"
+export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2"
+export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2"
+export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2"
+export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2"
+export aero_diffusion_iter=10
+export aero_diffusion_horiz_len=2500e3
+export aero_diffusion_fixed_val=1.0
+export npx_clim_b=97
+export npy_clim_b=97
+export aero_diagb_weight=0.9
+export aero_staticb_rescaling_factor=2.0
+export aero_diagb_rescale=20.0
+export aero_diagb_n_halo=4
+export aero_diagb_n_neighbors=16
+export aero_diagb_smooth_horiz_iter=0
+export aero_diagb_smooth_vert_iter=0
+
+echo "END: config.aeroanlgenb"
diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun
diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar
@@ -0,0 +1,11 @@
+#!/bin/bash -x
+
+########## config.aeroanlvar ##########
+# Aerosol Analysis specific
+
+echo "BEGIN: config.aeroanlvar"
+
+# Get task specific resources
+source "${EXPDIR}/config.resources" aeroanlvar
+
+echo "END: config.aeroanlvar"
diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com
@@ -98,5 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}'
 
 declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history'
 declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem'
+declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix'
 
 declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart'