From 3c86873acfc2afed19dd38e72c47782ace60f610 Mon Sep 17 00:00:00 2001
From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
Date: Fri, 20 Sep 2024 07:16:55 -0400
Subject: [PATCH] Adjust C768 resources for Hera (#2819)

This modifies the resources for gdasfcst (everywhere) and enkfgdaseupd
(Hera only). For the fcst job, the number of write tasks is increased to
prevent out of memory errors from the inline post. For the eupd, the
number of tasks is decreased to prevent out of memory errors. The
runtime for the eupd job was just over 10 minutes.

Resolves #2506
Resolves #2498
Resolves #2916

---------

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 jobs/JGFS_ATMOS_CYCLONE_GENESIS       |  5 -----
 jobs/JGFS_ATMOS_CYCLONE_TRACKER       |  5 -----
 parm/config/gfs/config.base           |  7 +++++++
 parm/config/gfs/config.resources.HERA | 16 ++++++++++++++--
 parm/config/gfs/config.resources.S4   |  2 +-
 parm/config/gfs/config.ufs            |  2 +-
 sorc/gsi_utils.fd                     |  2 +-
 7 files changed, 24 insertions(+), 15 deletions(-)

diff --git a/jobs/JGFS_ATMOS_CYCLONE_GENESIS b/jobs/JGFS_ATMOS_CYCLONE_GENESIS
index de130bf9aa..a80bcc1153 100755
--- a/jobs/JGFS_ATMOS_CYCLONE_GENESIS
+++ b/jobs/JGFS_ATMOS_CYCLONE_GENESIS
@@ -3,11 +3,6 @@
 source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis" -c "base genesis"
 
-# Hack to temporary skip this as the tracker has not been build
-#   on Hercules Rocky 9 yet
-# TODO: Remove this after tracker has been built for Rocky 9 #2639
-if [[ "${machine}" == 'HERCULES' ]]; then exit 0; fi
-
 ##############################################
 # Set variables used in the exglobal script
 ##############################################
diff --git a/jobs/JGFS_ATMOS_CYCLONE_TRACKER b/jobs/JGFS_ATMOS_CYCLONE_TRACKER
index 067de2c4aa..24fe33f8ca 100755
--- a/jobs/JGFS_ATMOS_CYCLONE_TRACKER
+++ b/jobs/JGFS_ATMOS_CYCLONE_TRACKER
@@ -3,11 +3,6 @@
 source "${HOMEgfs}/ush/preamble.sh"
 source "${HOMEgfs}/ush/jjob_header.sh" -e "tracker" -c "base tracker"
 
-# Hack to temporary skip this as the tracker has not been build
-#   on Hercules Rocky 9 yet
-# TODO: Remove this after tracker has been built for Rocky 9 #2639
-if [[ "${machine}" == 'HERCULES' ]]; then exit 0; fi
-
 export COMPONENT="atmos"
 
 
diff --git a/parm/config/gfs/config.base b/parm/config/gfs/config.base
index 784c334d82..27fcbdd055 100644
--- a/parm/config/gfs/config.base
+++ b/parm/config/gfs/config.base
@@ -492,4 +492,11 @@ if [[ "${machine}" =~ "PW" ]]; then
    export DO_WAVE="NO"
 fi
 
+# The tracker and genesis are not installed on Orion/Hercules yet; this requires spack-stack builds of the package.
+# TODO: we should place these in workflow/hosts/[orion|hercules].yaml.
+if [[ "${machine}" == "ORION" || "${machine}" == "HERCULES" ]]; then
+   export DO_TRACKER="NO"
+   export DO_GENESIS="NO"
+fi
+
 echo "END: config.base"
diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA
index e79d4c5b0a..d1b09fcc32 100644
--- a/parm/config/gfs/config.resources.HERA
+++ b/parm/config/gfs/config.resources.HERA
@@ -5,8 +5,9 @@
 case ${step} in
   "anal")
     if [[ "${CASE}" == "C384" ]]; then
-      export ntasks=270
-      export threads_per_task_anal=8
+      export ntasks_gdas=270
+      export ntasks_gfs=270
+      export threads_per_task=8
       export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     fi
     ;;
@@ -26,6 +27,10 @@ case ${step} in
 
   "eupd")
     case ${CASE} in
+      "C768")
+        export ntasks=80
+        export threads_per_task=20
+        ;;
       "C384")
         export ntasks=80
         ;;
@@ -43,6 +48,13 @@ case ${step} in
     export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
+  "upp")
+    if (( "${CASE:1}" >= 768 )); then
+       # Run fewer tasks per node for memory
+       tasks_per_node=20
+    fi
+    ;;
+
   *)
     ;;
 esac
diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4
index 1af64bf250..817494c7cd 100644
--- a/parm/config/gfs/config.resources.S4
+++ b/parm/config/gfs/config.resources.S4
@@ -32,7 +32,7 @@ case ${step} in
       *)
         ;;
     esac
-    export tasks_node=$(( max_tasks_per_node / threads_per_task ))
+    export tasks_per_node=$(( max_tasks_per_node / threads_per_task ))
     ;;
 
   "eobs")
diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
index babbe1f2dd..6309c4073b 100644
--- a/parm/config/gfs/config.ufs
+++ b/parm/config/gfs/config.ufs
@@ -281,7 +281,7 @@ case "${fv3_res}" in
           export rf_cutoff=100.0
           export fv_sg_adj=450
           export WRITE_GROUP_GDAS=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=15
           export WRITE_GROUP_GFS=4
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2
         fi
diff --git a/sorc/gsi_utils.fd b/sorc/gsi_utils.fd
index 9382fd01c2..a6ea311e5c 160000
--- a/sorc/gsi_utils.fd
+++ b/sorc/gsi_utils.fd
@@ -1 +1 @@
-Subproject commit 9382fd01c2a626c8934c3f553d420a45de2b4dec
+Subproject commit a6ea311e5c82369d255e3afdc99c1bce0c9a3014