sync with develop

.gitignore

@@ -53,11 +53,8 @@ parm/gdas/soca
 parm/gdas/jcb-gdas
 parm/gdas/jcb-algorithms
 parm/monitor
-parm/post/AEROSOL_LUTS.dat
 parm/post/nam_micro_lookup.dat
 parm/post/optics_luts_DUST.dat
-parm/post/gtg.config.gfs
-parm/post/gtg_imprintings.txt
 parm/post/optics_luts_DUST_nasa.dat
 parm/post/optics_luts_NITR_nasa.dat
 parm/post/optics_luts_SALT.dat
@@ -70,47 +67,14 @@ parm/post/optics_luts_WASO.dat
 parm/post/optics_luts_WASO_nasa.dat
 parm/post/params_grib2_tbl_new
 parm/post/post_tag_gfs128
-parm/post/post_tag_gfs65
-parm/post/postcntrl_gefs.xml
-parm/post/postcntrl_gefs_aerosol.xml
-parm/post/postcntrl_gefs_anl.xml
-parm/post/postcntrl_gefs_chem.xml
-parm/post/postcntrl_gefs_f00.xml
-parm/post/postcntrl_gfs.xml
-parm/post/postcntrl_gfs_anl.xml
-parm/post/postcntrl_gfs_f00.xml
-parm/post/postcntrl_gfs_f00_two.xml
-parm/post/postcntrl_gfs_flux.xml
-parm/post/postcntrl_gfs_flux_f00.xml
-parm/post/postcntrl_gfs_goes.xml
-parm/post/postcntrl_gfs_goes.xml-new
-parm/post/postcntrl_gfs_two.xml
-parm/post/postcntrl_gfs_wafs.xml
-parm/post/postcntrl_gfs_wafs_anl.xml
-parm/post/postxconfig-NT-GEFS-ANL.txt
-parm/post/postxconfig-NT-GEFS-F00.txt
-parm/post/postxconfig-NT-GEFS-F00-aerosol.txt
-parm/post/postxconfig-NT-GEFS-WAFS.txt
-parm/post/postxconfig-NT-GEFS-aerosol.txt
-parm/post/postxconfig-NT-GEFS.txt
-parm/post/postxconfig-NT-GFS-ANL.txt
-parm/post/postxconfig-NT-GFS-F00-TWO.txt
-parm/post/postxconfig-NT-GFS-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX-F00.txt
-parm/post/postxconfig-NT-GFS-FLUX.txt
-parm/post/postxconfig-NT-GFS-GOES.txt
-parm/post/postxconfig-NT-GFS-TWO.txt
-parm/post/postxconfig-NT-GFS-WAFS-ANL.txt
-parm/post/postxconfig-NT-GFS-WAFS.txt
-parm/post/postxconfig-NT-GFS.txt
-parm/post/postxconfig-NT-gefs-aerosol.txt
-parm/post/postxconfig-NT-gefs-chem.txt
+parm/post/gfs
+parm/post/gefs
 parm/post/ocean.csv
 parm/post/ice.csv
 parm/post/ocnicepost.nml.jinja2
 parm/ufs/noahmptable.tbl
 parm/ufs/model_configure.IN
-parm/ufs/model_configure_nest.IN
+parm/ufs/input_global_nest.nml.IN
 parm/ufs/MOM_input_*.IN
 parm/ufs/MOM6_data_table.IN
 parm/ufs/ice_in.IN

docs/source/errors_faq.rst

@@ -2,6 +2,14 @@
 Common Errors Known Issues
 ==========================
 
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+Error: Reserved Variables Causing Workflow Issues
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+Several variables are reserved in the workflow and should not be used as environment variables in your shell. Some of the common ones include (but are not limited to):
+``HOMEgfs``, ``machine``, ``ROTDIR``, ``COMROT``, ``COMROOT``, ``COMOUT``, ``COMIN``, ``STMP``, ``PTMP``, ``DATAROOT``, ``DATA``, ``ACCOUNT``, ``PDY``, ``cyc``, ``RUN``, etc.
+If you are using any of these variables in your shell, you may encounter unexpected behavior in the workflow.
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Error: "ImportError" message when running setup script
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -42,4 +50,3 @@ Issue: Directory name change for EnKF folder in ROTDIR
 **Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
 
 **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle.
-

docs/source/index.rst

@@ -1,6 +1,6 @@
 
 ###############
-Global Workflow 
+Global Workflow
 ###############
 
 **Global-workflow** is the end-to-end workflow designed to run global configurations of medium range weather forecasting for the UFS weather model. It supports both development and operational implementations. In its current format it supports the Global Forecast System (GFS) and the Global Ensemble Forecast System (GEFS) configurations
@@ -31,7 +31,7 @@ GitHub updates: Users should adjust their "Watch" settings for this repo so they
 Table of Contents
 =================
 
-.. toctree:: 
+.. toctree::
    :numbered:
    :maxdepth: 3
 
@@ -42,3 +42,4 @@ Table of Contents
    output.rst
    run.rst
    noaa_csp.rst
+   errors_faq.rst

docs/source/run.rst

@@ -13,4 +13,3 @@ Here we will show how you can run an experiment using the Global Workflow. The G
    start.rst
    monitor_rocoto.rst
    view.rst
-   errors_faq.rst

env/GAEA.env

@@ -34,12 +34,42 @@ else
     exit 2
 fi
 
-if [[ "${step}" = "waveinit" ]]; then
+if [[ "${step}" = "prep" ]]; then
 
-    export CFP_MP="YES"
-    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
-    export wavempexec=${launcher}
-    export wave_mpmd=${mpmd_opt}
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="GAEA"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
+
+    export MKL_NUM_THREADS=4
+    export MKL_CBWR=AUTO
+
+    export CFP_MP=${CFP_MP:-"YES"}
+    export USE_CFP=${USE_CFP:-"YES"}
+    export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"
+
+    export NTHREADS_GSI=${NTHREADSmax}
+    export APRUN_GSI="${APRUN} --cpus-per-task=${NTHREADS_GSI}"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-12}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    ntasks_cycle=${ntiles:-6}
+    export APRUN_CYCLE="${launcher} -n ${ntasks_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
+
+    export NTHREADS_GAUSFCANL=1
+    ntasks_gausfcanl=${ntasks_gausfcanl:-1}
+    export APRUN_GAUSFCANL="${launcher} -n ${ntasks_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
+
+elif [[ "${step}" = "sfcanl" ]]; then
+
+    export NTHREADS_CYCLE=${threads_per_task:-14}
+    export APRUN_CYCLE="${APRUN} --cpus-per-task=${NTHREADS_CYCLE}"
 
 elif [[ "${step}" = "fcst" ]]; then
 
@@ -49,9 +79,14 @@ elif [[ "${step}" = "fcst" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "upp" ]]; then
+
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN} --cpus-per-task=${NTHREADS_UPP}"
+
 elif [[ "${step}" = "atmos_products" ]]; then
 
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Gaea
 
 elif [[ "${step}" = "oceanice_products" ]]; then
 

env/GOOGLEPW.env

@@ -43,6 +43,17 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,4 +63,45 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
+elif [[ "${step}" = "ecen" ]]; then
+
+    export NTHREADS_ECEN=${NTHREADSmax}
+    export APRUN_ECEN="${APRUN}"
+
+    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
+    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
+    export APRUN_CHGRES="time"
+
+    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
+    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
+    export APRUN_CALCINC="${APRUN}"
+
+elif [[ "${step}" = "esfc" ]]; then
+
+    export NTHREADS_ESFC=${NTHREADSmax}
+    export APRUN_ESFC="${APRUN}"
+
+    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
+    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
+    export APRUN_CYCLE="${APRUN}"
+
+elif [[ "${step}" = "epos" ]]; then
+
+    export NTHREADS_EPOS=${NTHREADSmax}
+    export APRUN_EPOS="${APRUN}"
+
+elif [[ "${step}" = "fit2obs" ]]; then
+
+    export NTHREADS_FIT2OBS=${NTHREADS1}
+    export MPIRUN="${APRUN}"
+
 fi

env/WCOSS2.env

@@ -272,7 +272,7 @@ elif [[ "${step}" = "postsnd" ]]; then
     export OMP_NUM_THREADS=1
 
     export NTHREADS_POSTSND=${NTHREADS1}
-    export APRUN_POSTSND="${APRUN} --depth=${NTHREADS_POSTSND} --cpu-bind depth"
+    export mpmd_opt="-ppn 21 ${mpmd_opt}"
 
     export NTHREADS_POSTSNDCFP=${threads_per_task_postsndcfp:-1}
     [[ ${NTHREADS_POSTSNDCFP} -gt ${max_threads_per_task} ]] && export NTHREADS_POSTSNDCFP=${max_threads_per_task}

modulefiles/module_base.gaea.lua

@@ -15,6 +15,7 @@ load(pathJoin("cdo", (os.getenv("cdo_ver") or "None")))
 load(pathJoin("hdf5", (os.getenv("hdf5_ver") or "None")))
 load(pathJoin("netcdf-c", (os.getenv("netcdf_c_ver") or "None")))
 load(pathJoin("netcdf-fortran", (os.getenv("netcdf_fortran_ver") or "None")))
+load(pathJoin("perlbrew", (os.getenv("perl_ver") or "None")))
 
 load(pathJoin("nco", (os.getenv("nco_ver") or "None")))
 load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
@@ -25,6 +26,7 @@ load(pathJoin("crtm", (os.getenv("crtm_ver") or "None")))
 load(pathJoin("bufr", (os.getenv("bufr_ver") or "None")))
 load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
 load(pathJoin("py-netcdf4", (os.getenv("py_netcdf4_ver") or "None")))
+load(pathJoin("py-f90nml", (os.getenv("py_f90nml_ver") or "None")))
 load(pathJoin("py-pyyaml", (os.getenv("py_pyyaml_ver") or "None")))
 load(pathJoin("py-jinja2", (os.getenv("py_jinja2_ver") or "None")))
 load(pathJoin("py-pandas", (os.getenv("py_pandas_ver") or "None")))
@@ -36,4 +38,13 @@ load(pathJoin("py-xarray", (os.getenv("py_xarray_ver") or "None")))
 setenv("WGRIB2","wgrib2")
 setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
+--prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+--load(pathJoin("prepobs", (os.getenv("prepobs_run_ver") or "None")))
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/prepobs/v1.1.0", "modulefiles"))
+load(pathJoin("prepobs", "1.1.0"))
+
+prepend_path("MODULEPATH", pathJoin("/gpfs/f5/ufs-ard/world-shared/global/glopara/data/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
+load(pathJoin("fit2obs", (os.getenv("fit2obs_ver") or "None")))
+
+
 whatis("Description: GFS run setup environment")

parm/archive/enkf.yaml.j2

@@ -54,12 +54,14 @@ enkf:
                            "radstat.ensmean"] %}
         {% else %}
         {% if lobsdiag_forenkf %}
-        {% set da_files = ["atmens_observer.yaml",
-                           "atmens_solver.yaml",
+        {% set da_files = ["atmensanlobs.yaml",
+                           "atmensanlsol.yaml",
+                           "atmensanlfv3inc.yaml",
                            "atminc.ensmean.nc",
                            "atmensstat"] %}
         {% else %}
-        {% set da_files = ["atmens.yaml",
+        {% set da_files = ["atmensanlletkf.yaml",
+                           "atmensanlfv3inc.yaml",
                            "atminc.ensmean.nc",
                            "atmensstat"] %}
         {% endif %}

parm/archive/gdas.yaml.j2

@@ -58,7 +58,8 @@ gdas:
 
         # Analysis state
             {% if DO_JEDIATMVAR %}
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat"
             {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat"

parm/archive/gefs_arcdir.yaml.j2

@@ -0,0 +1,38 @@
+{% set cycle_HH = current_cycle | strftime("%H") %}
+{% set cycle_YMDH = current_cycle | to_YMDH %}
+{% set cycle_YMD = current_cycle | to_YMD %}
+{% set head = RUN + ".t" + cycle_HH + "z." %}
+
+# Declare the GEFS_ARCH where atmos data will be sent
+{% set GEFS_ARCH = ROTDIR ~ "/gefsarch" %}
+
+{% set file_set = [] %}
+
+{% set tmpl_dict = ({ '${ROTDIR}':ROTDIR,
+                      '${RUN}':RUN,
+                      '${YMD}':cycle_YMD,
+                      '${HH}':cycle_HH,
+                      '${GRID}': '1p00',
+                      '${MEMDIR}': 'ensstat' }) %}
+
+{% set COMIN_ATMOS_ENSSTAT_1p00 = COM_ATMOS_GRIB_GRID_TMPL | replace_tmpl(tmpl_dict) %}
+
+# Select ensstat files to copy to the arcdir
+{% if RUN == "gefs" %}
+    {% set ensstat_files = [] %}
+    {% if path_exists(COMIN_ATMOS_ENSSTAT_1p00) %}
+        {% for fhr in range(FHMIN_GFS, FHMAX_GFS + FHOUT_GFS, FHOUT_GFS) %}
+            {% do ensstat_files.append([COMIN_ATMOS_ENSSTAT_1p00 ~ "/" ~ head ~ "mean.pres_." ~
+                                        "1p00" ~ ".f" ~ '%03d'|format(fhr) ~ ".grib2",
+                                        GEFS_ARCH]) %}
+        {% endfor %}
+    {% endif %}
+{% endif %}
+{% set file_set = ensstat_files %}
+# Actually write the yaml
+mkdir:
+    - "{{ GEFS_ARCH }}"
+copy:
+   {% for source_dest_pair in file_set %}
+   - {{ source_dest_pair }}
+   {% endfor %}

parm/archive/gfsa.yaml.j2

@@ -32,7 +32,8 @@ gfsa:
 
         # State data
         {% if DO_JEDIATMVAR %}
-        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlvar.yaml"
+        - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmanlfv3inc.yaml"
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}atmstat"
         {% else %}
         - "{{ COMIN_ATMOS_ANALYSIS | relpath(ROTDIR) }}/{{ head }}gsistat"

parm/config/gefs/config.base

@@ -268,10 +268,6 @@ export OUTPUT_GRID="gaussian_grid"
 export WRITE_DOPOST=".true." # WRITE_DOPOST=true, use inline POST
 export WRITE_NSFLIP=".true."
 
-# Override normal post flat files for GEFS
-export FLTFILEGFS="${PARMgfs}/post/postxconfig-NT-GEFS.txt"
-export FLTFILEGFSF00="${PARMgfs}/post/postxconfig-NT-GEFS-F00.txt"
-
 # Microphysics Options: 99-ZhaoCarr, 8-Thompson; 6-WSM6, 10-MG, 11-GFDL
 export imp_physics=8
 

parm/config/gefs/config.resources

@@ -46,6 +46,9 @@ case ${machine} in
   "AZUREPW")
     export PARTITION_BATCH="compute"
     max_tasks_per_node=24
+  "GOOGLEPW")
+    export PARTITION_BATCH="compute"
+    max_tasks_per_node=32
     ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"

parm/config/gefs/config.resources.GOOGLEPW

@@ -0,0 +1,11 @@
+#! /usr/bin/env bash
+
+# GOOGLE-specific job resources
+
+export is_exclusive="True"
+unset memory
+
+# shellcheck disable=SC2312
+for mem_var in $(env | grep '^memory_' | cut -d= -f1); do
+  unset "${mem_var}"
+done