diff --git a/env/HERA.env b/env/HERA.env index 8d59c870cc..272c6773f9 100755 --- a/env/HERA.env +++ b/env/HERA.env @@ -92,13 +92,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "atmanlfv3inc" ]]; then export NTHREADS_ATMANLFV3INC=${NTHREADSmax} diff --git a/env/HERCULES.env b/env/HERCULES.env index 79ff8391bd..62b579dda3 100755 --- a/env/HERCULES.env +++ b/env/HERCULES.env @@ -96,12 +96,17 @@ case ${step} in export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" ;; - "aeroanlrun") + "aeroanlvar") export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" + ;; + "aeroanlgenb") + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" ;; "prepobsaero") diff --git a/env/JET.env b/env/JET.env index 4294a00de1..52730fc74c 100755 --- a/env/JET.env +++ b/env/JET.env @@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${launcher} ${ntasks}" -elif [[ "${step}" = "aeroanlrun" ]]; then - - export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" +elif [[ "${step}" = "aeroanlvar" ]]; then export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/ORION.env b/env/ORION.env index 7838d28640..638764908f 100755 --- a/env/ORION.env +++ b/env/ORION.env @@ -87,13 +87,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN} --cpus-per-task=${NTHREADS_AEROANL}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/S4.env b/env/S4.env index a29ec49fb4..dd852afa0f 100755 --- a/env/S4.env +++ b/env/S4.env @@ -80,13 +80,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN} --cpus-per-task=${NTHREADS_AEROANLGENB}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/env/WCOSS2.env b/env/WCOSS2.env index adff54d636..cfc6cb4097 100755 --- a/env/WCOSS2.env +++ b/env/WCOSS2.env @@ -73,13 +73,18 @@ elif [[ "${step}" = "atmensanlfv3inc" ]]; then export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax} export APRUN_ATMENSANLFV3INC="${APRUN}" -elif [[ "${step}" = "aeroanlrun" ]]; then +elif [[ "${step}" = "aeroanlvar" ]]; then export APRUNCFP="${launcher} -np \$ncmd ${mpmd_opt}" export NTHREADS_AEROANL=${NTHREADSmax} export APRUN_AEROANL="${APRUN}" +elif [[ "${step}" = "aeroanlgenb" ]]; then + + export NTHREADS_AEROANLGENB=${NTHREADSmax} + export APRUN_AEROANLGENB="${APRUN}" + elif [[ "${step}" = "prepobsaero" ]]; then export NTHREADS_PREPOBSAERO=${NTHREADS1} diff --git a/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX new file mode 100755 index 0000000000..81c89e9155 --- /dev/null +++ b/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlgenb" -c "base aeroanl aeroanlgenb" + +############################################## +# Set variables used in the script +############################################## + +############################################## +# Begin JOB SPECIFIC work +############################################## + +# Generate COM variables from templates +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COMIN_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_BMAT:COM_CHEM_BMAT_TMPL \ + COMIN_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL + +mkdir -p "${COMOUT_CHEM_BMAT}" + +############################################################### +# Run relevant script + +EXSCRIPT=${GDASAEROBMATPY:-${SCRgfs}/exgdas_aero_analysis_generate_bmatrix.py} +${EXSCRIPT} +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +############################################## +# End JOB SPECIFIC work +############################################## + +############################################## +# Final processing +############################################## +if [[ -e "${pgmout}" ]] ; then + cat "${pgmout}" +fi + +########################################## +# Remove the Temporary working directory +########################################## +cd "${DATAROOT}" || exit 1 +[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}" + +exit 0 diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE index 455f572da5..9c68d1fed6 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_FINALIZE @@ -8,25 +8,15 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlfinal" -c "base aeroanl aeroan ############################################## # Set variables used in the script ############################################## -# shellcheck disable=SC2153 -GDATE=$(date --utc +%Y%m%d%H -d "${PDY} ${cyc} - ${assim_freq} hours") -gPDY=${GDATE:0:8} -gcyc=${GDATE:8:2} -GDUMP="gdas" - ############################################## # Begin JOB SPECIFIC work ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS - -RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL \ + COMOUT_ATMOS_RESTART:COM_ATMOS_RESTART_TMPL ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE index b2a2893bc0..921b1458b2 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE +++ b/jobs/JGLOBAL_AERO_ANALYSIS_INITIALIZE @@ -19,13 +19,13 @@ GDUMP="gdas" ############################################## # Generate COM variables from templates -YMD=${PDY} HH=${cyc} declare_from_tmpl -rx COM_OBS COM_CHEM_ANALYSIS +YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \ + COM_OBS:COM_OBS_TMPL \ + COMOUT_CHEM_ANALYSIS:COM_CHEM_ANALYSIS_TMPL RUN=${GDUMP} YMD=${gPDY} HH=${gcyc} declare_from_tmpl -rx \ - COM_CHEM_ANALYSIS_PREV:COM_CHEM_ANALYSIS_TMPL \ - COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL - -mkdir -m 775 -p "${COM_CHEM_ANALYSIS}" + COMIN_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL \ + COMIN_CHEM_BMAT_PREV:COM_CHEM_BMAT_TMPL ############################################################### # Run relevant script diff --git a/jobs/JGLOBAL_AERO_ANALYSIS_RUN b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL similarity index 83% rename from jobs/JGLOBAL_AERO_ANALYSIS_RUN rename to jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL index 43749b78c5..290d7225dd 100755 --- a/jobs/JGLOBAL_AERO_ANALYSIS_RUN +++ b/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL @@ -3,7 +3,7 @@ source "${HOMEgfs}/ush/preamble.sh" export WIPE_DATA="NO" export DATA=${DATA:-${DATAROOT}/${RUN}aeroanl_${cyc}} -source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlrun" +source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlvar" -c "base aeroanl aeroanlvar" ############################################## # Set variables used in the script @@ -16,7 +16,7 @@ source "${HOMEgfs}/ush/jjob_header.sh" -e "aeroanlrun" -c "base aeroanl aeroanlr ############################################################### # Run relevant script -EXSCRIPT=${GDASAERORUNSH:-${SCRgfs}/exglobal_aero_analysis_run.py} +EXSCRIPT=${GDASAEROVARSH:-${SCRgfs}/exglobal_aero_analysis_variational.py} ${EXSCRIPT} status=$? [[ ${status} -ne 0 ]] && exit "${status}" diff --git a/jobs/rocoto/aeroanlgenb.sh b/jobs/rocoto/aeroanlgenb.sh new file mode 100755 index 0000000000..d0bc5dda9b --- /dev/null +++ b/jobs/rocoto/aeroanlgenb.sh @@ -0,0 +1,19 @@ +#! /usr/bin/env bash + +source "${HOMEgfs}/ush/preamble.sh" + +############################################################### +# Source UFSDA workflow modules +. "${HOMEgfs}/ush/load_ufsda_modules.sh" +status=$? +[[ ${status} -ne 0 ]] && exit "${status}" + +export job="aeroanlgenb" +export jobid="${job}.$$" + +############################################################### + +# Execute the JJOB +"${HOMEgfs}/jobs/JGDAS_AERO_ANALYSIS_GENERATE_BMATRIX" +status=$? +exit "${status}" diff --git a/jobs/rocoto/aeroanlrun.sh b/jobs/rocoto/aeroanlvar.sh similarity index 83% rename from jobs/rocoto/aeroanlrun.sh rename to jobs/rocoto/aeroanlvar.sh index 529bb2d7d1..7aa7d831f9 100755 --- a/jobs/rocoto/aeroanlrun.sh +++ b/jobs/rocoto/aeroanlvar.sh @@ -8,11 +8,11 @@ source "${HOMEgfs}/ush/preamble.sh" status=$? [[ ${status} -ne 0 ]] && exit "${status}" -export job="aeroanlrun" +export job="aeroanlvar" export jobid="${job}.$$" ############################################################### # Execute the JJOB -"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_RUN" +"${HOMEgfs}/jobs/JGLOBAL_AERO_ANALYSIS_VARIATIONAL" status=$? exit "${status}" diff --git a/parm/config/gfs/config.aeroanl b/parm/config/gfs/config.aeroanl index a1b7e1d44b..5ac03bd7ee 100644 --- a/parm/config/gfs/config.aeroanl +++ b/parm/config/gfs/config.aeroanl @@ -5,20 +5,36 @@ echo "BEGIN: config.aeroanl" -export CASE_ANL=${CASE} +# define analysis resolution based on deterministic res +case ${CASE} in + "C1152" | "C768" | "C384" | "C192") + CASE_ANL="C192" + ;; + "C96" | "C48") + CASE_ANL=${CASE} + ;; + *) + echo "FATAL ERROR: Aerosol DA not supported at ${CASE} resolution" + exit 4 +esac +export CASE_ANL export OBS_LIST="${PARMgfs}/gdas/aero/obs/lists/gdas_aero.yaml.j2" -export STATICB_TYPE='identity' +export STATICB_TYPE='diffusion' export BERROR_YAML="${PARMgfs}/gdas/aero/berror/staticb_${STATICB_TYPE}.yaml.j2" -export BERROR_DATA_DIR="${FIXgfs}/gdas/bump/aero/${CASE_ANL}/" -export BERROR_DATE="20160630.000000" +export BERROR_DATA_DIR="${FIXgfs}/gdas/aero/clim_b" export CRTM_FIX_YAML="${PARMgfs}/gdas/aero_crtm_coeff.yaml.j2" export JEDI_FIX_YAML="${PARMgfs}/gdas/aero_jedi_fix.yaml.j2" +export AERO_STAGE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_stage_variational.yaml.j2" +export AERO_FINALIZE_VARIATIONAL_TMPL="${PARMgfs}/gdas/aero_finalize_variational.yaml.j2" + export io_layout_x=@IO_LAYOUT_X@ export io_layout_y=@IO_LAYOUT_Y@ export JEDIEXE="${EXECgfs}/gdas.x" +export BMATEXE="${EXECgfs}/gdasapp_chem_diagb.x" +export DIFFUSIONEXE="${EXECgfs}/gdas_fv3jedi_error_covariance_toolbox.x" if [[ "${DOIAU}" == "YES" ]]; then export aero_bkg_times="3,6,9" diff --git a/parm/config/gfs/config.aeroanlgenb b/parm/config/gfs/config.aeroanlgenb new file mode 100644 index 0000000000..b41b22a524 --- /dev/null +++ b/parm/config/gfs/config.aeroanlgenb @@ -0,0 +1,29 @@ +#!/bin/bash -x + +########## config.aeroanlgenb ########## +# Aerosol Variance specific + +echo "BEGIN: config.aeroanlgenb" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlgenb + +export BMATYAML="${PARMgfs}/gdas/aero/berror/aero_diagb.yaml.j2" +export DIFFUSIONYAML="${PARMgfs}/gdas/aero/berror/aero_diffusionparm.yaml.j2" +export INTERPYAML="${PARMgfs}/gdas/aero/berror/aero_interp.yaml.j2" +export AERO_BMATRIX_STAGE_TMPL="${PARMgfs}/gdas/aero_stage_bmatrix_bkg.yaml.j2" +export AERO_BMATRIX_FINALIZE_TMPL="${PARMgfs}/gdas/aero_finalize_bmatrix_bkg.yaml.j2" +export aero_diffusion_iter=10 +export aero_diffusion_horiz_len=2500e3 +export aero_diffusion_fixed_val=1.0 +export npx_clim_b=97 +export npy_clim_b=97 +export aero_diagb_weight=0.9 +export aero_staticb_rescaling_factor=2.0 +export aero_diagb_rescale=20.0 +export aero_diagb_n_halo=4 +export aero_diagb_n_neighbors=16 +export aero_diagb_smooth_horiz_iter=0 +export aero_diagb_smooth_vert_iter=0 + +echo "END: config.aeroanlgenb" diff --git a/parm/config/gfs/config.aeroanlrun b/parm/config/gfs/config.aeroanlrun deleted file mode 100644 index 012e5b79f3..0000000000 --- a/parm/config/gfs/config.aeroanlrun +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -x - -########## config.aeroanlrun ########## -# Aerosol Analysis specific - -echo "BEGIN: config.aeroanlrun" - -# Get task specific resources -source "${EXPDIR}/config.resources" aeroanlrun - -echo "END: config.aeroanlrun" diff --git a/parm/config/gfs/config.aeroanlvar b/parm/config/gfs/config.aeroanlvar new file mode 100644 index 0000000000..4282b6c840 --- /dev/null +++ b/parm/config/gfs/config.aeroanlvar @@ -0,0 +1,11 @@ +#!/bin/bash -x + +########## config.aeroanlvar ########## +# Aerosol Analysis specific + +echo "BEGIN: config.aeroanlvar" + +# Get task specific resources +source "${EXPDIR}/config.resources" aeroanlvar + +echo "END: config.aeroanlvar" diff --git a/parm/config/gfs/config.com b/parm/config/gfs/config.com index 02a5b1edf5..61d592561d 100644 --- a/parm/config/gfs/config.com +++ b/parm/config/gfs/config.com @@ -98,5 +98,6 @@ declare -rx COM_ICE_GRIB_GRID_TMPL=${COM_ICE_GRIB_TMPL}'/${GRID}' declare -rx COM_CHEM_HISTORY_TMPL=${COM_BASE}'/model/chem/history' declare -rx COM_CHEM_ANALYSIS_TMPL=${COM_BASE}'/analysis/chem' +declare -rx COM_CHEM_BMAT_TMPL=${COM_CHEM_ANALYSIS_TMPL}'/bmatrix' declare -rx COM_MED_RESTART_TMPL=${COM_BASE}'/model/med/restart' diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources index 2f541ff945..4bba9a1795 100644 --- a/parm/config/gfs/config.resources +++ b/parm/config/gfs/config.resources @@ -16,7 +16,7 @@ if (( $# != 1 )); then echo "atmanlinit atmanlvar atmanlfv3inc atmanlfinal" echo "atmensanlinit atmensanlobs atmensanlsol atmensanlletkf atmensanlfv3inc atmensanlfinal" echo "snowanl esnowrecen" - echo "prepobsaero aeroanlinit aeroanlrun aeroanlfinal" + echo "prepobsaero aeroanlinit aeroanlvar aeroanlfinal aeroanlgenb" echo "anal sfcanl analcalc analdiag fcst echgres" echo "upp atmos_products" echo "tracker genesis genesis_fsu" @@ -401,12 +401,12 @@ case ${step} in layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") - layout_x=8 - layout_y=8 + layout_x=4 + layout_y=4 ;; "C48" ) # this case is for testing only @@ -427,27 +427,61 @@ case ${step} in memory="3072M" ;; - "aeroanlrun") + "aeroanlvar") case ${CASE} in "C768") layout_x=8 layout_y=8 ;; "C384") - layout_x=8 - layout_y=8 + layout_x=6 + layout_y=6 ;; "C192" | "C96") + layout_x=4 + layout_y=4 + ;; + "C48" ) + # this case is for testing only + layout_x=1 + layout_y=1 + ;; + *) + echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + exit 4 + esac + + export layout_x + export layout_y + + walltime="00:30:00" + ntasks=$(( layout_x * layout_y * 6 )) + threads_per_task=1 + tasks_per_node=$(( max_tasks_per_node / threads_per_task )) + export is_exclusive=True + ;; + + "aeroanlgenb") + case ${CASE} in + "C768") layout_x=8 layout_y=8 ;; + "C384") + layout_x=6 + layout_y=6 + ;; + "C192" | "C96") + layout_x=4 + layout_y=4 + ;; "C48" ) # this case is for testing only layout_x=1 layout_y=1 ;; *) - echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}" + echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}" exit 4 esac @@ -459,8 +493,10 @@ case ${step} in threads_per_task=1 tasks_per_node=$(( max_tasks_per_node / threads_per_task )) export is_exclusive=True + ;; + "aeroanlfinal") walltime="00:10:00" ntasks=1 diff --git a/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 new file mode 100644 index 0000000000..b33f280945 --- /dev/null +++ b/parm/gdas/aero_finalize_bmatrix_bkg.yaml.j2 @@ -0,0 +1,19 @@ +{% set cycle_HH = current_cycle | strftime("%H") %} +{% set HEAD = RUN + ".t" + cycle_HH + "z." %} +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +copy: +### copy YAMLs used +{% set yaml_list = ['chem_diagb.yaml', 'chem_diffusion.yaml'] %} +{% for fname in yaml_list %} +- ["{{ DATA }}/{{ HEAD }}{{ fname }}", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}{{ fname }}"] +{% endfor %} +### copy stddev files to ROTDIR +{% for tile in range(1, ntiles+1) %} +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ DATA }}/stddev/{{ background_time | to_fv3time }}.stddev.coupler.res", "{{ COMOUT_CHEM_BMAT }}/{{ background_time | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ DATA }}/diffusion/diffusion_hz.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_hz.nc"] +- ["{{ DATA }}/diffusion/diffusion_vt.nc", "{{ COMOUT_CHEM_BMAT }}/{{ HEAD }}aero_diffusion_vt.nc"] diff --git a/parm/gdas/aero_finalize_variational.yaml.j2 b/parm/gdas/aero_finalize_variational.yaml.j2 new file mode 100644 index 0000000000..7dadd36291 --- /dev/null +++ b/parm/gdas/aero_finalize_variational.yaml.j2 @@ -0,0 +1,24 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkgtime = AERO_WINDOW_BEGIN %} +{% else %} + {% set bkgtime = current_cycle %} +{% endif %} +###################################### +mkdir: +- "{{ COMOUT_CHEM_ANALYSIS }}" +- "{{ COMOUT_ATMOS_RESTART }}" +copy: +## copy variational YAML to ROTDIR +- ["{{ DATA }}/{{ APREFIX }}aerovar.yaml", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aerovar.yaml"] +## copy increments +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_CHEM_ANALYSIS }}/aeroinc.{{ current_cycle | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +- ["{{ DATA }}/anl/aeroinc_gauss.{{ current_cycle | to_isotime }}.gaussian.modelLevels.nc", "{{ COMOUT_CHEM_ANALYSIS }}/{{ APREFIX }}aeroinc.nc"] +## copy analysis +{% for tile in range(1,ntiles+1) %} +- ["{{ DATA }}/anl/{{ bkgtime | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ COMOUT_ATMOS_RESTART }}/{{ bkgtime | to_fv3time }}.aeroanl_fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} diff --git a/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 new file mode 100644 index 0000000000..9005b9ff12 --- /dev/null +++ b/parm/gdas/aero_stage_bmatrix_bkg.yaml.j2 @@ -0,0 +1,38 @@ +###################################### +# set some variables +###################################### +{% set offset_td = "+6H" | to_timedelta %} +{% set background_time = current_cycle | add_to_datetime(offset_td) %} +{% set ftype_list = ['fv_core.res', 'fv_tracer.res'] %} +###################################### +# create working directories +###################################### +mkdir: +- "{{ DATA }}/bkg" +- "{{ DATA }}/stddev" +- "{{ DATA }}/clm_stddev" +- "{{ DATA }}/diffusion" +copy: +###################################### +# copy deterministic background files +###################################### +# define variables +# Declare a dict of search and replace terms to run on each template +{% set tmpl_dict = {'${ROTDIR}':ROTDIR, + '${RUN}':RUN, + '${YMD}':current_cycle | to_YMD, + '${HH}':current_cycle | strftime("%H"), + '${MEMDIR}':""} %} + +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.coupler.res"] +{% for ftype in ftype_list %} + {% for tile in range(1, ntiles+1) %} +- ["{{ COM_ATMOS_RESTART_TMPL | replace_tmpl(tmpl_dict) }}/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ background_time | to_fv3time }}.{{ ftype }}.tile{{ tile }}.nc"] + {% endfor %} +{% endfor %} +# copy climatological stddev files +###################################### +{% for tile in range(1, ntiles+1) %} +- ["{{ BERROR_DATA_DIR }}/stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/clm_stddev/stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + diff --git a/parm/gdas/aero_stage_variational.yaml.j2 b/parm/gdas/aero_stage_variational.yaml.j2 new file mode 100644 index 0000000000..afd0e1b946 --- /dev/null +++ b/parm/gdas/aero_stage_variational.yaml.j2 @@ -0,0 +1,50 @@ +###################################### +# set some variables +###################################### +{% if DOIAU == True %} + {% set bkg_times = [] %} + {% for fh in range(0, 7, 3) %} + {% set offset = fh | string + "H" %} + {% set fcst_timedelta = offset | to_timedelta %} + {% set fcst_time = AERO_WINDOW_BEGIN | add_to_datetime(fcst_timedelta) %} + {% do bkg_times.append(fcst_time) %} + {% endfor %} +{% else %} + {% set bkg_times = [] %} + {% do bkg_times.append(current_cycle) %} +{% endif %} +{% set fvfiles = ['fv_core.res.', 'fv_tracer.res.'] %} +###################################### +mkdir: +- "{{ DATA }}/anl" +- "{{ DATA }}/diags" +- "{{ DATA }}/berror" +- "{{ DATA }}/bkg" +copy: +###################################### +## copy backgrounds +{% for bkgtime in bkg_times %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.coupler.res", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.coupler.res"] + {% for fvfile in fvfiles %} + {% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc", "{{ DATA }}/bkg/{{ bkgtime | to_fv3time }}.{{ fvfile }}tile{{ tile }}.nc"] + {% endfor %} + {% endfor %} +{% endfor %} +###################################### +## copy backgrounds again for fv_tracer to create analysis files later +{% for tile in range(1,ntiles+1) %} +- ["{{ COMIN_ATMOS_RESTART_PREV }}/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/anl/{{ bkg_times[0] | to_fv3time }}.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} + +###################################### +## copy berror files from COMIN_CHEM_BMAT_PREV +## stddev files +{% for tile in range(1, ntiles+1) %} +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.fv_tracer.res.tile{{ tile }}.nc"] +{% endfor %} +### copy coupler file +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ current_cycle | to_fv3time }}.stddev.coupler.res", "{{ DATA }}/berror/{{ current_cycle | to_fv3time }}.stddev.coupler.res"] +### copy diffusion files +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_hz.nc", "{{ DATA }}/berror/diffusion_hz.nc"] +- ["{{ COMIN_CHEM_BMAT_PREV }}/{{ GPREFIX }}aero_diffusion_vt.nc", "{{ DATA }}/berror/diffusion_vt.nc"] diff --git a/parm/ufs/gocart/ExtData.other b/parm/ufs/gocart/ExtData.other index 7a0d63d6ca..5d2ddc5102 100644 --- a/parm/ufs/gocart/ExtData.other +++ b/parm/ufs/gocart/ExtData.other @@ -17,12 +17,12 @@ DU_UTHRES '1' Y E - none none uthres ExtData/n #====== Sulfate Sources ================================================= # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -SU_ANTHROL1 NA N Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_ANTHROL2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL1 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2 ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_ANTHROL2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Ship emissions -SU_SHIPSO2 NA N Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -SU_SHIPSO4 NA N Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO2 NA Y Y %y4-%m2-%d2t12:00:00 none none SO2_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +SU_SHIPSO4 NA Y Y %y4-%m2-%d2t12:00:00 none none SO4_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption SU_AIRCRAFT NA Y Y %y4-%m2-%d2t12:00:00 none none none /dev/null @@ -63,11 +63,11 @@ OC_MTPO NA Y Y %y4-%m2-%d2t12:00:00 none none mtpo ExtData/nexus/MEGAN_ OC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -OC_ANTEOC1 NA N Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -OC_ANTEOC2 NA N Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC1 NA Y Y %y4-%m2-%d2t12:00:00 none none OC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_ANTEOC2 NA Y Y %y4-%m2-%d2t12:00:00 none none OC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -OC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +OC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none OC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption OC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none oc_aviation /dev/null @@ -88,11 +88,11 @@ pSOA_ANTHRO_VOC NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null BC_BIOFUEL NA Y Y %y4-%m2-%d2t12:00:00 none none biofuel /dev/null # Anthropogenic (BF & FF) emissions -- allowed to input as two layers -BC_ANTEBC1 NA N Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc -BC_ANTEBC2 NA N Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC1 NA Y Y %y4-%m2-%d2t12:00:00 none none BC ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_ANTEBC2 NA Y Y %y4-%m2-%d2t12:00:00 none none BC_elev ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # EDGAR based ship emissions -BC_SHIP NA N Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc +BC_SHIP NA Y Y %y4-%m2-%d2t12:00:00 none none BC_ship ExtData/nexus/CEDS/v2019/monthly/%y4/CEDS_2019_monthly.%y4%m2.nc # Aircraft fuel consumption BC_AIRCRAFT NA N Y %y4-%m2-%d2t12:00:00 none none bc_aviation /dev/null diff --git a/scripts/exgdas_aero_analysis_generate_bmatrix.py b/scripts/exgdas_aero_analysis_generate_bmatrix.py new file mode 100755 index 0000000000..0d8389c40d --- /dev/null +++ b/scripts/exgdas_aero_analysis_generate_bmatrix.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +# exgdas_aero_analysis_generate_bmatrix.py +# This script creates an AerosolBMatrix object +# and runs the methods needed +# to stage files, compute the variance, and write to com +# files needed for the variational solver +import os + +from wxflow import Logger, cast_strdict_as_dtypedict +from pygfs.task.aero_bmatrix import AerosolBMatrix + +# Initialize root logger +logger = Logger(level='DEBUG', colored_log=True) + + +if __name__ == '__main__': + + # Take configuration from environment and cast it as python dictionary + config = cast_strdict_as_dtypedict(os.environ) + + # Instantiate the aerosol variance and diffusion correlation tasks + AeroB = AerosolBMatrix(config) + AeroB.initialize() + AeroB.interpBackground() + AeroB.computeVariance() + AeroB.computeDiffusion() + AeroB.finalize() diff --git a/scripts/exglobal_aero_analysis_run.py b/scripts/exglobal_aero_analysis_variational.py similarity index 84% rename from scripts/exglobal_aero_analysis_run.py rename to scripts/exglobal_aero_analysis_variational.py index 85f4b963a4..dd5bb4f65a 100755 --- a/scripts/exglobal_aero_analysis_run.py +++ b/scripts/exglobal_aero_analysis_variational.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 -# exglobal_aero_analysis_run.py +# exglobal_aero_analysis_variational.py # This script creates an AerosolAnalysis object -# and runs the execute method +# and runs the variational method # which executes the global aerosol variational analysis import os @@ -19,4 +19,4 @@ # Instantiate the aerosol analysis task AeroAnl = AerosolAnalysis(config) - AeroAnl.execute() + AeroAnl.variational() diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh index 92cc1d50b1..d86ca0d68d 100755 --- a/sorc/link_workflow.sh +++ b/sorc/link_workflow.sh @@ -238,7 +238,7 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd" ]]; then cd "${HOMEgfs}/fix" || exit 1 [[ ! -d gdas ]] && mkdir -p gdas cd gdas || exit 1 - for gdas_sub in fv3jedi gsibec obs soca; do + for gdas_sub in fv3jedi gsibec obs soca aero; do if [[ -d "${gdas_sub}" ]]; then rm -rf "${gdas_sub}" fi @@ -368,9 +368,11 @@ if [[ -d "${HOMEgfs}/sorc/gdas.cd/build" ]]; then declare -a JEDI_EXE=("gdas.x" \ "gdas_soca_gridgen.x" \ "gdas_soca_error_covariance_toolbox.x" \ + "gdas_fv3jedi_error_covariance_toolbox.x" \ "gdas_soca_setcorscales.x" \ "gdas_soca_diagb.x" \ "fv3jedi_plot_field.x" \ + "gdasapp_chem_diagb.x" \ "fv3jedi_fv3inc.x" \ "gdas_ens_handler.x" \ "gdas_incr_handler.x" \ diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh index 8af9054972..d13cb0df0c 100755 --- a/ush/forecast_postdet.sh +++ b/ush/forecast_postdet.sh @@ -37,7 +37,7 @@ FV3_postdet() { fi # Get list of FV3 restart files - local file_list + local file_list file_list=$(FV3_restarts) echo "Copying FV3 restarts for 'RUN=${RUN}' at '${restart_date}' from '${restart_dir}'" local fv3_file restart_file @@ -60,6 +60,24 @@ FV3_postdet() { break fi done + # Replace fv_tracer with aeroanl_fv_tracer restart files from current cycle (if found) + local nn + local use_anl_aero="YES" + for (( nn = 1; nn <= ntiles; nn++ )); do + test_tracer_file="${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" + if [[ ! -f "${test_tracer_file}" ]]; then + use_anl_aero="NO" + echo "WARNING: File ${test_tracer_file} does not exist, will not replace any files from the aerosol analysis" + break + fi + done + if [[ ${use_anl_aero} == "YES" ]]; then + for (( nn = 1; nn <= ntiles; nn++ )); do + rm -f "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + ${NCP} "${COMOUT_ATMOS_RESTART}/${restart_date:0:8}.${restart_date:8:2}0000.aeroanl_fv_tracer.res.tile${nn}.nc" \ + "${DATA}/INPUT/fv_tracer.res.tile${nn}.nc" + done + fi # if [[ ${use_anl_aero} == "YES" ]]; then fi # if [[ "${RERUN}" != "YES" ]]; then fi # if [[ "${warm_start}" == ".true." ]]; then @@ -265,7 +283,7 @@ FV3_out() { ${NCP} "${DATA}/model_configure" "${COMOUT_CONF}/ufs.model_configure" ${NCP} "${DATA}/ufs.configure" "${COMOUT_CONF}/ufs.ufs.configure" ${NCP} "${DATA}/diag_table" "${COMOUT_CONF}/ufs.diag_table" - + # Determine the dates for restart files to be copied to COM local restart_date restart_dates diff --git a/ush/python/pygfs/__init__.py b/ush/python/pygfs/__init__.py index b87bba0c2c..9f290fafd3 100644 --- a/ush/python/pygfs/__init__.py +++ b/ush/python/pygfs/__init__.py @@ -2,8 +2,10 @@ import os from .task.analysis import Analysis +from .task.bmatrix import BMatrix from .task.aero_emissions import AerosolEmissions from .task.aero_analysis import AerosolAnalysis +from .task.aero_bmatrix import AerosolBMatrix from .task.atm_analysis import AtmAnalysis from .task.atmens_analysis import AtmEnsAnalysis from .task.marine_bmat import MarineBMat diff --git a/ush/python/pygfs/task/aero_analysis.py b/ush/python/pygfs/task/aero_analysis.py index ccc5fb601a..0389e109a1 100644 --- a/ush/python/pygfs/task/aero_analysis.py +++ b/ush/python/pygfs/task/aero_analysis.py @@ -82,29 +82,18 @@ def initialize(self: Analysis) -> None: jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) FileHandler(jedi_fix_list).sync() - # stage berror files - # copy BUMP files, otherwise it will assume ID matrix - if self.task_config.get('STATICB_TYPE', 'identity') in ['bump']: - FileHandler(self.get_berror_dict(self.task_config)).sync() - - # stage backgrounds - FileHandler(self.get_bkg_dict(AttrDict(self.task_config, **self.task_config))).sync() + # stage files from COM and create working directories + logger.info(f"Staging files prescribed from {self.task_config.AERO_STAGE_VARIATIONAL_TMPL}") + aero_var_stage_list = parse_j2yaml(self.task_config.AERO_STAGE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_stage_list).sync() # generate variational YAML file logger.debug(f"Generate variational YAML file: {self.task_config.jedi_yaml}") save_as_yaml(self.task_config.jedi_config, self.task_config.jedi_yaml) logger.info(f"Wrote variational YAML to: {self.task_config.jedi_yaml}") - # need output dir for diags and anl - logger.debug("Create empty output [anl, diags] directories to receive output from executable") - newdirs = [ - os.path.join(self.task_config['DATA'], 'anl'), - os.path.join(self.task_config['DATA'], 'diags'), - ] - FileHandler({'mkdir': newdirs}).sync() - @logit(logger) - def execute(self: Analysis) -> None: + def variational(self: Analysis) -> None: chdir(self.task_config.DATA) @@ -140,59 +129,33 @@ def finalize(self: Analysis) -> None: """ # ---- tar up diags # path of output tar statfile - aerostat = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") + logger.info('Preparing observation space diagnostics for archiving') + aerostat = os.path.join(self.task_config.COMOUT_CHEM_ANALYSIS, f"{self.task_config['APREFIX']}aerostat") # get list of diag files to put in tarball diags = glob.glob(os.path.join(self.task_config['DATA'], 'diags', 'diag*nc4')) # gzip the files first for diagfile in diags: + logger.info(f'Adding {diagfile} to tar file') with open(diagfile, 'rb') as f_in, gzip.open(f"{diagfile}.gz", 'wb') as f_out: f_out.writelines(f_in) + # ---- add increments to RESTART files + logger.info('Adding increments to RESTART files') + self._add_fms_cube_sphere_increments() + + # copy files back to COM + logger.info(f"Copying files to COM based on {self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL}") + aero_var_final_list = parse_j2yaml(self.task_config.AERO_FINALIZE_VARIATIONAL_TMPL, self.task_config) + FileHandler(aero_var_final_list).sync() + # open tar file for writing with tarfile.open(aerostat, "w") as archive: for diagfile in diags: diaggzip = f"{diagfile}.gz" archive.add(diaggzip, arcname=os.path.basename(diaggzip)) - - # copy full YAML from executable to ROTDIR - src = os.path.join(self.task_config['DATA'], f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f"{self.task_config['RUN']}.t{self.task_config['cyc']:02d}z.aerovar.yaml") - yaml_copy = { - 'mkdir': [self.task_config.COM_CHEM_ANALYSIS], - 'copy': [[src, dest]] - } - FileHandler(yaml_copy).sync() - - # ---- copy RESTART fv_tracer files for future reference - if self.task_config.DOIAU: - bkgtime = self.task_config.AERO_WINDOW_BEGIN - else: - bkgtime = self.task_config.current_cycle - template = '{}.fv_tracer.res.tile{}.nc'.format(to_fv3time(bkgtime), '{tilenum}') - bkglist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, f'aeroges.{tracer}') - bkglist.append([src, dest]) - FileHandler({'copy': bkglist}).sync() - - # ---- add increments to RESTART files - logger.info('Adding increments to RESTART files') - self._add_fms_cube_sphere_increments() - - # ---- move increments to ROTDIR - logger.info('Moving increments to ROTDIR') - template = f'aeroinc.{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' - inclist = [] - for itile in range(1, self.task_config.ntiles + 1): - tracer = template.format(tilenum=itile) - src = os.path.join(self.task_config.DATA, 'anl', tracer) - dest = os.path.join(self.task_config.COM_CHEM_ANALYSIS, tracer) - inclist.append([src, dest]) - FileHandler({'copy': inclist}).sync() + logger.info(f'Saved diags to {aerostat}') def clean(self): super().clean() @@ -209,7 +172,7 @@ def _add_fms_cube_sphere_increments(self: Analysis) -> None: restart_template = f'{to_fv3time(bkgtime)}.fv_tracer.res.tile{{tilenum}}.nc' increment_template = f'{to_fv3time(self.task_config.current_cycle)}.fv_tracer.res.tile{{tilenum}}.nc' inc_template = os.path.join(self.task_config.DATA, 'anl', 'aeroinc.' + increment_template) - bkg_template = os.path.join(self.task_config.COM_ATMOS_RESTART_PREV, restart_template) + bkg_template = os.path.join(self.task_config.DATA, 'anl', restart_template) # get list of increment vars incvars_list_path = os.path.join(self.task_config['PARMgfs'], 'gdas', 'aeroanl_inc_vars.yaml') incvars = YAMLFile(path=incvars_list_path)['incvars'] @@ -232,38 +195,7 @@ def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: bkg_dict: Dict a dictionary containing the list of model background files to copy for FileHandler """ - # NOTE for now this is FV3 RESTART files and just assumed to be fh006 - - # get FV3 RESTART files, this will be a lot simpler when using history files - rst_dir = task_config.COM_ATMOS_RESTART_PREV - run_dir = os.path.join(task_config['DATA'], 'bkg') - - # Start accumulating list of background files to copy - bkglist = [] - - # if using IAU, we can use FGAT - bkgtimes = [] - begintime = task_config.previous_cycle - for fcsthr in task_config.aero_bkg_fhr: - bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) - - # now loop over background times - for bkgtime in bkgtimes: - # aerosol DA needs coupler - basename = f'{to_fv3time(bkgtime)}.coupler.res' - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - # aerosol DA only needs core/tracer - for ftype in ['core', 'tracer']: - template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' - for itile in range(1, task_config.ntiles + 1): - basename = template.format(tilenum=itile) - bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) - - bkg_dict = { - 'mkdir': [run_dir], - 'copy': bkglist, - } + bkg_dict = {} return bkg_dict @logit(logger) @@ -285,34 +217,5 @@ def get_berror_dict(self, config: Dict[str, Any]) -> Dict[str, List[str]]: berror_dict: Dict a dictionary containing the list of background error files to copy for FileHandler """ - # aerosol static-B needs nicas, cor_rh, cor_rv and stddev files. - b_dir = config.BERROR_DATA_DIR - b_datestr = to_fv3time(config.BERROR_DATE) - berror_list = [] - - for ftype in ['stddev']: - coupler = f'{b_datestr}.{ftype}.coupler.res' - berror_list.append([ - os.path.join(b_dir, coupler), os.path.join(config.DATA, 'berror', coupler) - ]) - template = f'{b_datestr}.{ftype}.fv_tracer.res.tile{{tilenum}}.nc' - for itile in range(1, config.ntiles + 1): - tracer = template.format(tilenum=itile) - berror_list.append([ - os.path.join(b_dir, tracer), os.path.join(config.DATA, 'berror', tracer) - ]) - radius = 'cor_aero_universe_radius' - berror_list.append([ - os.path.join(b_dir, radius), os.path.join(config.DATA, 'berror', radius) - ]) - nproc = config.ntiles * config.layout_x * config.layout_y - for nn in range(1, nproc + 1): - berror_list.append([ - os.path.join(b_dir, f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc'), - os.path.join(config.DATA, 'berror', f'nicas_aero_nicas_local_{nproc:06}-{nn:06}.nc') - ]) - berror_dict = { - 'mkdir': [os.path.join(config.DATA, 'berror')], - 'copy': berror_list, - } + berror_dict = {} return berror_dict diff --git a/ush/python/pygfs/task/aero_bmatrix.py b/ush/python/pygfs/task/aero_bmatrix.py new file mode 100644 index 0000000000..c652bad558 --- /dev/null +++ b/ush/python/pygfs/task/aero_bmatrix.py @@ -0,0 +1,294 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (AttrDict, FileHandler, rm_p, + add_to_datetime, to_fv3time, to_timedelta, + to_fv3time, chdir, Executable, WorkflowException, + parse_j2yaml, save_as_yaml, logit) +from pygfs.task.bmatrix import BMatrix + +logger = getLogger(__name__.split('.')[-1]) + + +class AerosolBMatrix(BMatrix): + """ + Class for global aerosol BMatrix tasks + """ + @logit(logger, name="AerosolBMatrix") + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + + _res = int(self.task_config['CASE'][1:]) + _res_anl = int(self.task_config['CASE_ANL'][1:]) + + _bmat_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diagb.yaml") + _diffusion_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_diffusion.yaml") + _convertstate_yaml = os.path.join(self.task_config.DATA, f"{self.task_config.RUN}.t{self.task_config['cyc']:02d}z.chem_convertstate.yaml") + + # Create a local dictionary that is repeatedly used across this class + local_dict = AttrDict( + { + 'npx_ges': _res + 1, + 'npy_ges': _res + 1, + 'npz_ges': self.task_config.LEVS - 1, + 'npz': self.task_config.LEVS - 1, + 'npx_anl': _res_anl + 1, + 'npy_anl': _res_anl + 1, + 'npz_anl': self.task_config['LEVS'] - 1, + 'aero_bkg_fhr': map(int, str(self.task_config['aero_bkg_times']).split(',')), + 'OPREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'APREFIX': f"{self.task_config.RUN}.t{self.task_config.cyc:02d}z.", + 'GPREFIX': f"gdas.t{self.task_config.previous_cycle.hour:02d}z.", + 'bmat_yaml': _bmat_yaml, + 'diffusion_yaml': _diffusion_yaml, + 'convertstate_yaml': _convertstate_yaml, + } + ) + + # task_config is everything that this task should need + self.task_config = AttrDict(**self.task_config, **local_dict) + + @logit(logger) + def initialize(self: BMatrix) -> None: + super().initialize() + # stage fix files + logger.info(f"Staging JEDI fix files from {self.task_config.JEDI_FIX_YAML}") + jedi_fix_list = parse_j2yaml(self.task_config.JEDI_FIX_YAML, self.task_config) + FileHandler(jedi_fix_list).sync() + + # stage backgrounds + logger.info(f"Staging backgrounds prescribed from {self.task_config.AERO_BMATRIX_STAGE_TMPL}") + aero_bmat_stage_list = parse_j2yaml(self.task_config.AERO_BMATRIX_STAGE_TMPL, self.task_config) + FileHandler(aero_bmat_stage_list).sync() + + # generate convert state YAML file + logger.info(f"Generate convert state YAML file: {self.task_config.convertstate_yaml}") + self.task_config.convertstate_config = parse_j2yaml(self.task_config.INTERPYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.convertstate_config, self.task_config.convertstate_yaml) + logger.info(f"Wrote convert state YAML to: {self.task_config.convertstate_yaml}") + + # generate diagb YAML file + logger.info(f"Generate bmat YAML file: {self.task_config.bmat_yaml}") + self.task_config.bmat_config = parse_j2yaml(self.task_config.BMATYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.bmat_config, self.task_config.bmat_yaml) + logger.info(f"Wrote bmat YAML to: {self.task_config.bmat_yaml}") + + # generate diffusion parameters YAML file + logger.info(f"Generate diffusion YAML file: {self.task_config.diffusion_yaml}") + self.task_config.diffusion_config = parse_j2yaml(self.task_config.DIFFUSIONYAML, + self.task_config, + searchpath=self.gdasapp_j2tmpl_dir) + save_as_yaml(self.task_config.diffusion_config, self.task_config.diffusion_yaml) + logger.info(f"Wrote diffusion YAML to: {self.task_config.diffusion_yaml}") + + # link executable to run directory + self.link_bmatexe() + self.link_diffusion_exe() + self.link_jediexe() + + @logit(logger) + def interpBackground(self) -> None: + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdas.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg('fv3jedi') + exec_cmd.add_default_arg('convertstate') + exec_cmd.add_default_arg(self.task_config.convertstate_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeVariance(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name = os.path.join(self.task_config.DATA, 'gdasapp_chem_diagb.x') + exec_cmd.add_default_arg(exec_name) + exec_cmd.add_default_arg(self.task_config.bmat_yaml) + + try: + logger.debug(f"Executing {exec_cmd}") + exec_cmd() + except OSError: + raise OSError(f"Failed to execute {exec_cmd}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd}") + + pass + + @logit(logger) + def computeDiffusion(self) -> None: + + chdir(self.task_config.DATA) + + exec_cmd_diffusion = Executable(self.task_config.APRUN_AEROANLGENB) + exec_name_diffusion = os.path.join(self.task_config.DATA, 'gdas_fv3jedi_error_covariance_toolbox.x') + exec_cmd_diffusion.add_default_arg(exec_name_diffusion) + exec_cmd_diffusion.add_default_arg(self.task_config.diffusion_yaml) + + try: + logger.debug(f"Executing {exec_cmd_diffusion}") + exec_cmd_diffusion() + except OSError: + raise OSError(f"Failed to execute {exec_cmd_diffusion}") + except Exception: + raise WorkflowException(f"An error occured during execution of {exec_cmd_diffusion}") + + pass + + @logit(logger) + def finalize(self) -> None: + super().finalize() + # save files to COMOUT + logger.info(f"Saving files to COMOUT based on {self.task_config.AERO_BMATRIX_FINALIZE_TMPL}") + aero_bmat_finalize_list = parse_j2yaml(self.task_config.AERO_BMATRIX_FINALIZE_TMPL, self.task_config) + FileHandler(aero_bmat_finalize_list).sync() + + @logit(logger) + def link_jediexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.JEDIEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return exe_dest + + @logit(logger) + def link_bmatexe(self) -> None: + """ + + This method links a JEDI executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + exe_src = self.task_config.BMATEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest = os.path.join(self.task_config.DATA, os.path.basename(exe_src)) + if os.path.exists(exe_dest): + rm_p(exe_dest) + os.symlink(exe_src, exe_dest) + + return + + @logit(logger) + def link_diffusion_exe(self) -> None: + """ + + This method links a JEDI (fv3jedi_error_covariance_toolbox.x) + executable to the run directory + + Parameters + ---------- + Task: GDAS task + + Returns + ---------- + None + """ + + exe_src_diffusion = self.task_config.DIFFUSIONEXE + + # TODO: linking is not permitted per EE2. Needs work in JEDI to be able to copy the exec. + logger.info(f"Link executable {exe_src_diffusion} to DATA/") + logger.warn("Linking is not permitted per EE2.") + exe_dest_diffusion = os.path.join(self.task_config.DATA, os.path.basename(exe_src_diffusion)) + if os.path.exists(exe_dest_diffusion): + rm_p(exe_dest_diffusion) + os.symlink(exe_src_diffusion, exe_dest_diffusion) + + return + + @logit(logger) + def get_bkg_dict(self, task_config: Dict[str, Any]) -> Dict[str, List[str]]: + """Compile a dictionary of model background files to copy + + This method constructs a dictionary of FV3 RESTART files (coupler, core, tracer) + that are needed for global aerosol DA and returns said dictionary for use by the FileHandler class. + + Parameters + ---------- + task_config: Dict + a dictionary containing all of the configuration needed for the task + + Returns + ---------- + bkg_dict: Dict + a dictionary containing the list of model background files to copy for FileHandler + """ + # NOTE for now this is FV3 RESTART files and just assumed to be fh006 + + # get FV3 RESTART files, this will be a lot simpler when using history files + rst_dir = task_config.COM_ATMOS_RESTART_PREV + run_dir = os.path.join(task_config['DATA'], 'bkg') + + # Start accumulating list of background files to copy + bkglist = [] + + # if using IAU, we can use FGAT + bkgtimes = [] + begintime = task_config.previous_cycle + for fcsthr in task_config.aero_bkg_fhr: + bkgtimes.append(add_to_datetime(begintime, to_timedelta(f"{fcsthr}H"))) + + # now loop over background times + for bkgtime in bkgtimes: + # aerosol DA needs coupler + basename = f'{to_fv3time(bkgtime)}.coupler.res' + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + # aerosol DA only needs core/tracer + for ftype in ['core', 'tracer']: + template = f'{to_fv3time(bkgtime)}.fv_{ftype}.res.tile{{tilenum}}.nc' + for itile in range(1, task_config.ntiles + 1): + basename = template.format(tilenum=itile) + bkglist.append([os.path.join(rst_dir, basename), os.path.join(run_dir, basename)]) + + bkg_dict = { + 'mkdir': [run_dir], + 'copy': bkglist, + } + return bkg_dict diff --git a/ush/python/pygfs/task/analysis.py b/ush/python/pygfs/task/analysis.py index bf47b9a950..6f7d3dfc68 100644 --- a/ush/python/pygfs/task/analysis.py +++ b/ush/python/pygfs/task/analysis.py @@ -198,7 +198,7 @@ def add_fv3_increments(self, inc_file_tmpl: str, bkg_file_tmpl: str, incvars: Li @logit(logger) def link_jediexe(self) -> None: - """Compile a dictionary of background error files to copy + """ This method links a JEDI executable to the run directory diff --git a/ush/python/pygfs/task/bmatrix.py b/ush/python/pygfs/task/bmatrix.py new file mode 100644 index 0000000000..d0edba2358 --- /dev/null +++ b/ush/python/pygfs/task/bmatrix.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python3 + +import os +from logging import getLogger +from typing import List, Dict, Any, Union + +from wxflow import (parse_j2yaml, FileHandler, logit, + Task, Executable, WorkflowException) + +logger = getLogger(__name__.split('.')[-1]) + + +class BMatrix(Task): + """Parent class for GDAS BMatrix tasks + + The BMatrix class is the parent class for all + Global Data Assimilation System (GDAS) BMatrix tasks + """ + def __init__(self, config: Dict[str, Any]) -> None: + super().__init__(config) + # Store location of GDASApp jinja2 templates + self.gdasapp_j2tmpl_dir = os.path.join(self.task_config.PARMgfs, 'gdas') + + def initialize(self) -> None: + super().initialize() + + def finalize(self) -> None: + super().finalize() diff --git a/versions/fix.ver b/versions/fix.ver index 3f85a45fee..7c18bea081 100644 --- a/versions/fix.ver +++ b/versions/fix.ver @@ -12,6 +12,7 @@ export gdas_fv3jedi_ver=20220805 export gdas_soca_ver=20240802 export gdas_gsibec_ver=20240416 export gdas_obs_ver=20240213 +export gdas_aero_ver=20240806 export glwu_ver=20220805 export gsi_ver=20240208 export lut_ver=20220805 diff --git a/workflow/applications/gfs_cycled.py b/workflow/applications/gfs_cycled.py index 8c3bca0fd7..b8aa2dba3a 100644 --- a/workflow/applications/gfs_cycled.py +++ b/workflow/applications/gfs_cycled.py @@ -107,7 +107,7 @@ def _get_app_configs(self): configs += ['waveawipsbulls', 'waveawipsgridded'] if self.do_aero: - configs += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + configs += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: configs += ['prepobsaero'] @@ -184,7 +184,7 @@ def get_task_names(self): gdas_tasks += wave_prep_tasks if self.do_aero and 'gdas' in self.aero_anl_runs: - gdas_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gdas_tasks += ['aeroanlgenb', 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gdas_tasks += ['prepobsaero'] @@ -223,7 +223,7 @@ def get_task_names(self): gfs_tasks += wave_prep_tasks if self.do_aero and 'gfs' in self.aero_anl_runs: - gfs_tasks += ['aeroanlinit', 'aeroanlrun', 'aeroanlfinal'] + gfs_tasks += ['aeroanlinit', 'aeroanlvar', 'aeroanlfinal'] if self.do_prep_obs_aero: gfs_tasks += ['prepobsaero'] diff --git a/workflow/rocoto/gfs_tasks.py b/workflow/rocoto/gfs_tasks.py index 23f334549a..76db3db88e 100644 --- a/workflow/rocoto/gfs_tasks.py +++ b/workflow/rocoto/gfs_tasks.py @@ -455,13 +455,41 @@ def prepobsaero(self): return task + def aeroanlgenb(self): + + deps = [] + dep_dict = {'type': 'metatask', 'name': f'{self.run}fcst'} + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep=deps) + + resources = self.get_resource('aeroanlgenb') + task_name = f'{self.run}aeroanlgenb' + task_dict = {'task_name': task_name, + 'resources': resources, + 'dependency': dependencies, + 'envars': self.envars, + 'cycledef': 'gdas_half,gdas', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlgenb.sh', + 'job_name': f'{self.pslot}_{task_name}_@H', + 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', + 'maxtries': '&MAXTRIES;' + } + + task = rocoto.create_task(task_dict) + + return task + def aeroanlinit(self): deps = [] + dep_dict = {'type': 'task', 'name': 'gdasaeroanlgenb', 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}"} + deps.append(rocoto.add_dependency(dep_dict)) dep_dict = {'type': 'task', 'name': f'{self.run}prep'} + deps.append(rocoto.add_dependency(dep_dict)) + if self.app_config.do_prep_obs_aero: dep_dict = {'type': 'task', 'name': f'{self.run}prepobsaero'} - deps.append(rocoto.add_dependency(dep_dict)) + deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) resources = self.get_resource('aeroanlinit') @@ -481,21 +509,28 @@ def aeroanlinit(self): return task - def aeroanlrun(self): + def aeroanlvar(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlinit'} + dep_dict = { + 'type': 'task', 'name': f'gdasaeroanlgenb', + 'offset': f"-{timedelta_to_HMS(self._base['cycle_interval'])}", + } deps.append(rocoto.add_dependency(dep_dict)) - dependencies = rocoto.create_dependency(dep=deps) + dep_dict = { + 'type': 'task', 'name': f'{self.run}aeroanlinit', + } + deps.append(rocoto.add_dependency(dep_dict)) + dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) - resources = self.get_resource('aeroanlrun') - task_name = f'{self.run}aeroanlrun' + resources = self.get_resource('aeroanlvar') + task_name = f'{self.run}aeroanlvar' task_dict = {'task_name': task_name, 'resources': resources, 'dependency': dependencies, 'envars': self.envars, 'cycledef': self.run.replace('enkf', ''), - 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlrun.sh', + 'command': f'{self.HOMEgfs}/jobs/rocoto/aeroanlvar.sh', 'job_name': f'{self.pslot}_{task_name}_@H', 'log': f'{self.rotdir}/logs/@Y@m@d@H/{task_name}.log', 'maxtries': '&MAXTRIES;' @@ -508,7 +543,7 @@ def aeroanlrun(self): def aeroanlfinal(self): deps = [] - dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlrun'} + dep_dict = {'type': 'task', 'name': f'{self.run}aeroanlvar'} deps.append(rocoto.add_dependency(dep_dict)) dependencies = rocoto.create_dependency(dep_condition='and', dep=deps) diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py index d943cd130c..d8d5edb5e6 100644 --- a/workflow/rocoto/tasks.py +++ b/workflow/rocoto/tasks.py @@ -20,7 +20,7 @@ class Tasks: 'earc', 'ecen', 'echgres', 'ediag', 'efcs', 'eobs', 'eomg', 'epos', 'esfc', 'eupd', 'atmensanlinit', 'atmensanlobs', 'atmensanlsol', 'atmensanlletkf', 'atmensanlfv3inc', 'atmensanlfinal', - 'aeroanlinit', 'aeroanlrun', 'aeroanlfinal', + 'aeroanlinit', 'aeroanlvar', 'aeroanlfinal', 'aeroanlgenb', 'prepsnowobs', 'snowanl', 'esnowrecen', 'fcst', 'atmanlupp', 'atmanlprod', 'atmupp', 'goesupp',