From 03e58b64d22decc6aebab0a1eacc80e1b3ba29c2 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 26 Feb 2025 17:06:15 +0000 Subject: [PATCH 1/5] modulefiles to use spack-stack-1.6.0, fms-2024.1 environment that contains g2/3.5.1 --- modulefiles/build.gaeac5.intel.lua | 4 ++-- modulefiles/build.gaeac6.intel.lua | 4 ++-- modulefiles/build.hera.gnu.lua | 10 ++++++---- modulefiles/build.hera.intel.lua | 4 ++-- modulefiles/build.hera.intelllvm.lua | 4 ++-- modulefiles/build.hercules.intel.lua | 4 ++-- modulefiles/build.hercules.intelllvm.lua | 4 ++-- modulefiles/build.jet.intel.lua | 4 ++-- modulefiles/build.jet.intelllvm.lua | 4 ++-- modulefiles/build.noaacloud.intel.lua | 2 +- modulefiles/build.orion.intel.lua | 4 ++-- modulefiles/build.orion.intelllvm.lua | 4 ++-- modulefiles/build.s4.intel.lua | 4 ++-- modulefiles/build.wcoss2.intel.lua | 4 ++-- modulefiles/common4noaacloud.lua | 2 +- 15 files changed, 32 insertions(+), 30 deletions(-) diff --git a/modulefiles/build.gaeac5.intel.lua b/modulefiles/build.gaeac5.intel.lua index e9dd614b3..45cfdeddf 100644 --- a/modulefiles/build.gaeac5.intel.lua +++ b/modulefiles/build.gaeac5.intel.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Gaea using Intel prepend_path("MODULEPATH", "/sw/rdtn/modulefiles") load("hsi") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -19,7 +19,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.gaeac6.intel.lua b/modulefiles/build.gaeac6.intel.lua index e1f1cbfec..86bb00f65 100644 --- a/modulefiles/build.gaeac6.intel.lua +++ b/modulefiles/build.gaeac6.intel.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Gaea C6 using Intel prepend_path("MODULEPATH", "/sw/rdtn/modulefiles") load("hsi") -prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2023.2.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -19,7 +19,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.hera.gnu.lua b/modulefiles/build.hera.gnu.lua index 70daf13ae..e858ac350 100644 --- a/modulefiles/build.hera.gnu.lua +++ b/modulefiles/build.hera.gnu.lua @@ -5,10 +5,12 @@ Load environment to compile UFS_UTILS on Hera using Gnu hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8-ompi416/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/modulefiles") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/installs/openmpi/modulefiles") +prepend_path("MODULEPATH", "/scratch2/NCEPDEV/stmp1/role.epic/spack-stack/spack-stack-1.6.0_gnu13/envs/fms-2024.01/install/modulefiles/Core") -stack_gcc_ver=os.getenv("stack_gcc_ver") or "9.2" -load(pathJoin("stack-gcc", gnu_ver)) +stack_gnu_ver=os.getenv("stack_gnu_ver") or "13.3.0" +load(pathJoin("stack-gcc", stack_gnu_ver)) stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.6" load(pathJoin("stack-openmpi", stack_openmpi_ver)) @@ -19,7 +21,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.hera.intel.lua b/modulefiles/build.hera.intel.lua index 377b0dda6..f79d67adb 100644 --- a/modulefiles/build.hera.intel.lua +++ b/modulefiles/build.hera.intel.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Hera using Intel hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -19,7 +19,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.hera.intelllvm.lua b/modulefiles/build.hera.intelllvm.lua index 64de60839..dc59c40f5 100644 --- a/modulefiles/build.hera.intelllvm.lua +++ b/modulefiles/build.hera.intelllvm.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Hera using Intel hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -22,7 +22,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.hercules.intel.lua b/modulefiles/build.hercules.intel.lua index 23314b7bd..277909359 100644 --- a/modulefiles/build.hercules.intel.lua +++ b/modulefiles/build.hercules.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Hercules using Intel Classic ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -16,7 +16,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.hercules.intelllvm.lua b/modulefiles/build.hercules.intelllvm.lua index 5253076b2..eeddf7966 100644 --- a/modulefiles/build.hercules.intelllvm.lua +++ b/modulefiles/build.hercules.intelllvm.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Hercules using IntelLLVM ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -16,7 +16,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.jet.intel.lua b/modulefiles/build.jet.intel.lua index 85296a7fc..69052aebf 100644 --- a/modulefiles/build.jet.intel.lua +++ b/modulefiles/build.jet.intel.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Jet using Intel Classic hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -19,7 +19,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.jet.intelllvm.lua b/modulefiles/build.jet.intelllvm.lua index ac8a6d7eb..1c740c45e 100644 --- a/modulefiles/build.jet.intelllvm.lua +++ b/modulefiles/build.jet.intelllvm.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on Jet using Intel LLVM hpss_ver=os.getenv("hpss_ver") or "" load(pathJoin("hpss", hpss_ver)) -prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -19,7 +19,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.noaacloud.intel.lua b/modulefiles/build.noaacloud.intel.lua index cfed65dd8..806e36d59 100644 --- a/modulefiles/build.noaacloud.intel.lua +++ b/modulefiles/build.noaacloud.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on NOAA CSPs using Intel ]]) -prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/ue-intel/install/modulefiles/Core") +prepend_path("MODULEPATH", "/contrib/spack-stack-rocky8/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") prepend_path("MODULEPATH", "/apps/modules/modulefiles") load("gnu") load("stack-intel") diff --git a/modulefiles/build.orion.intel.lua b/modulefiles/build.orion.intel.lua index 1d51da87a..b7fc2befc 100644 --- a/modulefiles/build.orion.intel.lua +++ b/modulefiles/build.orion.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Orion using Intel classic ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -16,7 +16,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.orion.intelllvm.lua b/modulefiles/build.orion.intelllvm.lua index 9b2464b11..e32178289 100644 --- a/modulefiles/build.orion.intelllvm.lua +++ b/modulefiles/build.orion.intelllvm.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Orion using IntelLLVM ]]) -prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/unified-env-rocky9/install/modulefiles/Core") +prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0" load(pathJoin("stack-intel", stack_intel_ver)) @@ -16,7 +16,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.s4.intel.lua b/modulefiles/build.s4.intel.lua index e2ab40feb..468647d26 100644 --- a/modulefiles/build.s4.intel.lua +++ b/modulefiles/build.s4.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on S4 using Intel ]]) -prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.6.0/envs/upp-addon-env/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", hpc_intel_ver)) @@ -16,7 +16,7 @@ load(pathJoin("cmake", cmake_ver)) bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" diff --git a/modulefiles/build.wcoss2.intel.lua b/modulefiles/build.wcoss2.intel.lua index 6e0659bdd..e5f179e73 100644 --- a/modulefiles/build.wcoss2.intel.lua +++ b/modulefiles/build.wcoss2.intel.lua @@ -39,7 +39,7 @@ load(pathJoin("bacio", bacio_ver)) --sfcio_ver=os.getenv("sfcio_ver") or "1.4.1" --load(pathJoin("sfcio", sfcio_ver)) -w3emc_ver=os.getenv("w3emc_ver") or "2.9.2" +w3emc_ver=os.getenv("w3emc_ver") or "2.12.0" load(pathJoin("w3emc", w3emc_ver)) nemsio_ver=os.getenv("nemsio_ver") or "2.5.4" @@ -54,7 +54,7 @@ load(pathJoin("sp", sp_ver)) ip_ver=os.getenv("ip_ver") or "4.0.0" load(pathJoin("ip", ip_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) -- Needed for mpiexec command. diff --git a/modulefiles/common4noaacloud.lua b/modulefiles/common4noaacloud.lua index 8e24390c6..7bf7729d6 100644 --- a/modulefiles/common4noaacloud.lua +++ b/modulefiles/common4noaacloud.lua @@ -5,7 +5,7 @@ Load environment to compile UFS_UTILS on NOAA CSPs using Intel bacio_ver=os.getenv("bacio_ver") or "2.4.1" load(pathJoin("bacio", bacio_ver)) -g2_ver=os.getenv("g2_ver") or "3.4.5" +g2_ver=os.getenv("g2_ver") or "3.5.1" load(pathJoin("g2", g2_ver)) ip_ver=os.getenv("ip_ver") or "4.3.0" From d72e37e7cbbae3e9b31b8fff21163dbb10195fb3 Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Wed, 26 Feb 2025 18:50:44 +0000 Subject: [PATCH 2/5] updated modulefiles/build.hera.gnu.lua --- modulefiles/build.hera.gnu.lua | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/modulefiles/build.hera.gnu.lua b/modulefiles/build.hera.gnu.lua index e858ac350..c64009afc 100644 --- a/modulefiles/build.hera.gnu.lua +++ b/modulefiles/build.hera.gnu.lua @@ -43,13 +43,17 @@ load(pathJoin("w3emc", w3emc_ver)) sigio_ver=os.getenv("sigio_ver") or "2.3.2" load(pathJoin("sigio", sigio_ver)) -nccmp_ver=os.getenv("nccmp_ver") or "1.9.1" +nccmp_ver=os.getenv("nccmp_ver") or "1.9.0.1" load(pathJoin("nccmp", nccmp_ver)) -esmf_ver=os.getenv("esmf_ver") or "8.5.0" +esmf_ver=os.getenv("esmf_ver") or "8.6.0" load(pathJoin("esmf", esmf_ver)) -nco_ver=os.getenv("nco_ver") or "5.1.6" +nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) +setenv("CC", "mpicc") +setenv("CXX", "mpic++") +setenv("FC", "mpif90") + whatis("Description: UFS_UTILS build environment") From 2d42c425b3e1fc5fbb63edc7307348be888a8caa Mon Sep 17 00:00:00 2001 From: Natalie Perlin Date: Fri, 7 Mar 2025 14:23:51 +0000 Subject: [PATCH 3/5] changes to build.hera.gnu.lua modulefile to fix linking issues and runtime errors --- modulefiles/build.hera.gnu.lua | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/modulefiles/build.hera.gnu.lua b/modulefiles/build.hera.gnu.lua index c64009afc..54dcb55de 100644 --- a/modulefiles/build.hera.gnu.lua +++ b/modulefiles/build.hera.gnu.lua @@ -1,5 +1,5 @@ help([[ -Load environment to compile UFS_UTILS on Hera using Gnu +Load environment to compile UFS_UTILS on Hera using Gnu 13.3 ]]) hpss_ver=os.getenv("hpss_ver") or "" @@ -52,6 +52,11 @@ load(pathJoin("esmf", esmf_ver)) nco_ver=os.getenv("nco_ver") or "5.0.6" load(pathJoin("nco", nco_ver)) + +prepend_path("CPPFLAGS", " -I/apps/slurm_hera/23.11.3/include/slurm"," ") +prepend_path("LD_LIBRARY_PATH", "/apps/slurm_hera/23.11.3/lib") +setenv("LD_PRELOAD", "/scratch2/NCEPDEV/stmp1/role.epic/installs/gnu/13.3.0/lib64/libstdc++.so.6") + setenv("CC", "mpicc") setenv("CXX", "mpic++") setenv("FC", "mpif90") From c87310700b435f36d17b3343a871857eac77ce08 Mon Sep 17 00:00:00 2001 From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:18:28 -0500 Subject: [PATCH 4/5] Update hsi path on Gaea clusters (#1031) Fixes #1030. --- modulefiles/build.gaeac5.intel.lua | 2 +- modulefiles/build.gaeac6.intel.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/modulefiles/build.gaeac5.intel.lua b/modulefiles/build.gaeac5.intel.lua index 45cfdeddf..b2132f3fd 100644 --- a/modulefiles/build.gaeac5.intel.lua +++ b/modulefiles/build.gaeac5.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Gaea using Intel ]]) -prepend_path("MODULEPATH", "/sw/rdtn/modulefiles") +prepend_path("MODULEPATH", "/usw/hpss/modulefiles") load("hsi") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") diff --git a/modulefiles/build.gaeac6.intel.lua b/modulefiles/build.gaeac6.intel.lua index 86bb00f65..bc0064518 100644 --- a/modulefiles/build.gaeac6.intel.lua +++ b/modulefiles/build.gaeac6.intel.lua @@ -2,7 +2,7 @@ help([[ Load environment to compile UFS_UTILS on Gaea C6 using Intel ]]) -prepend_path("MODULEPATH", "/sw/rdtn/modulefiles") +prepend_path("MODULEPATH", "/usw/hpss/modulefiles") load("hsi") prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/c6/spack-stack-1.6.0/envs/fms-2024.01/install/modulefiles/Core") From 7f77d18e08bc5105537e08271d12a0c92cf3d54c Mon Sep 17 00:00:00 2001 From: GeorgeGayno-NOAA <52789452+GeorgeGayno-NOAA@users.noreply.github.com> Date: Mon, 10 Mar 2025 09:59:31 -0400 Subject: [PATCH 5/5] cpld_gridgen - Update regression tests to run in parallel (#1027) Previously, the tests ran in serial and some parts were run from the command line. This speeds up the processing and places all the test functions in the batch queue. The functionality was not changed. Fixes #1016. --- reg_tests/cpld_gridgen/cpld_gridgen.sh | 190 ++++++++++++++ reg_tests/cpld_gridgen/rt.sh | 340 ++++++++++--------------- reg_tests/cpld_gridgen/rt.summary.sh | 55 ++++ reg_tests/rt.sh | 2 +- ush/cpld_gridgen.sh | 117 --------- 5 files changed, 377 insertions(+), 327 deletions(-) create mode 100755 reg_tests/cpld_gridgen/cpld_gridgen.sh create mode 100755 reg_tests/cpld_gridgen/rt.summary.sh delete mode 100755 ush/cpld_gridgen.sh diff --git a/reg_tests/cpld_gridgen/cpld_gridgen.sh b/reg_tests/cpld_gridgen/cpld_gridgen.sh new file mode 100755 index 000000000..75c2be511 --- /dev/null +++ b/reg_tests/cpld_gridgen/cpld_gridgen.sh @@ -0,0 +1,190 @@ +#!/bin/bash +set -eux + +SECONDS=0 + +function edit_namelist { + + sed -e "s/NI_GLB/$NI/g" \ + -e "s/NJ_GLB/$NJ/g" \ + -e "s|FIXDIR|$FIXDIR_PATH|g" \ + -e "s|OUTDIR|$OUTDIR_PATH|g" \ + -e "s|MOSAICDIR|$MOSAICDIR_PATH|g" \ + -e "s/TOPOGFILE/$TOPOGFILE/g" \ + -e "s/EDITSFILE/$EDITSFILE/g" \ + -e "s/RESNAME/$RESNAME/g" \ + -e "s/DO_MASKEDIT/$MASKEDIT/g" \ + -e "s/DO_DEBUG/$DEBUG/g" \ + -e "s/DO_POSTWGTS/$DO_POSTWGTS/g" \ + -e "s/ATMRESLIST/$ATMRESLIST/g" +} + +check_results() { + + [ -o xtrace ] && set_x='set -x' || set_x='set +x' + set +x + + local test_status=PASS + # verification run + if [[ $CREATE_BASELINE = false ]]; then + + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo "Working dir = $RUNDIR" | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo "Baseline dir = $BASELINE" | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo "Checking test $TEST_NAME results ...." | tee -a $PATHRT/$REGRESSIONTEST_LOG + + for file in $BASELINE/*.nc; do + printf %s "Comparing " $(basename ${file}) "...." | tee -a $PATHRT/$REGRESSIONTEST_LOG + + if [[ ! -f $RUNDIR/$(basename ${file}) ]]; then + echo "....MISSING file" | tee -a $PATHRT/$REGRESSIONTEST_LOG + test_status=FAIL + else + $NCCMP -dmfqS -w format $(basename ${file}) $file >>${PATHRT}/nccmp_${TEST_NAME}.log 2>&1 && d=$? || d=$? + if [[ $d -ne 0 ]]; then + echo "....NOT OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG + test_status=FAIL + else + echo "....OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG + fi + fi + done + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + # baseline creation run + else + + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo "Working dir = $RUNDIR" | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo "Moving baseline files to $NEW_BASELINE ...." | tee -a $PATHRT/$REGRESSIONTEST_LOG + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + + mkdir -p $NEW_BASELINE + + for file in *.nc; do + printf %s "Moving " $file "...." | tee -a $PATHRT/$REGRESSIONTEST_LOG + + cp $file $NEW_BASELINE/$file && d=$? || d=$? + if [[ $d -ne 0 ]]; then + echo "....NOT OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG + test_status=FAIL + else + echo "....OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG + fi + done + echo | tee -a $PATHRT/$REGRESSIONTEST_LOG + + fi + + if [[ $test_status == FAIL ]]; then + echo "$TEST_NAME failed" >> $PATHRT/fail_test_$TEST_NAME + fi +} + +set +x +echo BEGIN cpld_gridgen.sh +set -x + +cd $RUNDIR + +RESNAME=${RESNAME:-$1} +TEST_NAME=$RESNAME +ATMLIST=${ATMLIST:-$2} +DEBUG=.false. +MASKEDIT=.false. +DO_POSTWGTS=.true. +MOSAICDIR_PATH=${MOSAICDIR_PATH:-$PATHTR/fix/orog} +FIXDIR_PATH=${MOM6_FIXDIR}/${RESNAME} +if [[ ${ATMLIST} -eq -1 ]]; then + ATMRESLIST=12,24,48,96,192,384,768,1152,3072 +else + ATMRESLIST=${ATMLIST} +fi + +APRUN=${APRUN:-"srun"} + +if [ $RESNAME = 900 ]; then + NI=40 + NJ=20 + TOPOGFILE=topog.nc + EDITSFILE='none' + if [ $DO_POSTWGTS == .true. ]; then + #pre-generate SCRIP files for dst rectilinear grids using NCO + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap + fi +fi + +if [ $RESNAME = 500 ]; then + NI=72 + NJ=35 + TOPOGFILE=ocean_topog.nc + EDITSFILE='none' + if [ $DO_POSTWGTS == .true. ]; then + #pre-generate SCRIP files for dst rectilinear grids using NCO + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap + fi +fi + +if [ $RESNAME = 100 ]; then + NI=360 + NJ=320 + MASKEDIT=.T. + TOPOGFILE=topog.nc + EDITSFILE=topo_edits_011818.nc + if [ $DO_POSTWGTS == .true. ]; then + #pre-generate SCRIP files for dst rectilinear grids using NCO + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap + fi +fi + +if [ $RESNAME = 050 ]; then + NI=720 + NJ=576 + TOPOGFILE=ocean_topog.nc + EDITSFILE='none' + if [ $DO_POSTWGTS == .true. ]; then + #pre-generate SCRIP files for dst rectilinear grids using NCO + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p50_SCRIP.nc -G latlon=361,720#lon_typ=grn_ctr#lat_typ=cap + fi +fi + +if [ $RESNAME = 025 ]; then + NI=1440 + NJ=1080 + TOPOGFILE=ocean_topog.nc + EDITSFILE=All_edits.nc + if [ $DO_POSTWGTS == .true. ]; then + #pre-generate SCRIP files for dst rectilinear grids using NCO + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p50_SCRIP.nc -G latlon=361,720#lon_typ=grn_ctr#lat_typ=cap + $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p25_SCRIP.nc -G latlon=721,1440#lon_typ=grn_ctr#lat_typ=cap + fi +fi + +edit_namelist < grid.nml.IN > grid.nml +$APRUN ./cpld_gridgen + +# generate ice mesh +FSRC=${OUTDIR_PATH}/Ct.mx${RESNAME}_SCRIP_land.nc +FDST=${OUTDIR_PATH}/mesh.mx${RESNAME}.nc +$APRUN -n 1 ESMF_Scrip2Unstruct ${FSRC} ${FDST} 0 + +# generate kmt file for CICE +FSRC=${OUTDIR_PATH}/grid_cice_NEMS_mx${RESNAME}.nc +FDST=${OUTDIR_PATH}/kmtu_cice_NEMS_mx${RESNAME}.nc +ncks -O -v kmt ${FSRC} ${FDST} + +check_results + +elapsed_time=$( printf '%02dh:%02dm:%02ds\n' $((SECONDS%86400/3600)) $((SECONDS%3600/60)) $((SECONDS%60)) ) +echo "Elapsed time: ${elapsed_time}. Have a nice day!" >> $PATHRT/${REGRESSIONTEST_LOG} +set +x +echo "Elapsed time: ${elapsed_time}. Have a nice day!" diff --git a/reg_tests/cpld_gridgen/rt.sh b/reg_tests/cpld_gridgen/rt.sh index 6fede1dc3..0ca97556a 100755 --- a/reg_tests/cpld_gridgen/rt.sh +++ b/reg_tests/cpld_gridgen/rt.sh @@ -1,9 +1,9 @@ #!/bin/bash -set -eu -SECONDS=0 +set -ux error() { + set +x echo echo "$@" 1>&2 exit 1 @@ -30,73 +30,11 @@ usage() { } usage_and_exit() { + set +x usage exit $1 } -check_results() { - - [ -o xtrace ] && set_x='set -x' || set_x='set +x' - set +x - - local test_status=PASS - # verification run - if [[ $CREATE_BASELINE = false ]]; then - - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo "Working dir = $RUNDIR" | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo "Baseline dir = $BASELINE" | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo "Checking test $TEST_NAME results ...." | tee -a $PATHRT/$REGRESSIONTEST_LOG - - for file in $BASELINE/*.nc; do - printf %s "Comparing " $(basename ${file}) "...." | tee -a $PATHRT/$REGRESSIONTEST_LOG - - if [[ ! -f $RUNDIR/$(basename ${file}) ]]; then - echo "....MISSING file" | tee -a $PATHRT/$REGRESSIONTEST_LOG - test_status=FAIL - else - $NCCMP -dmfqS -w format $(basename ${file}) $file >>${PATHRT}/nccmp_${TEST_NAME}.log 2>&1 && d=$? || d=$? - if [[ $d -ne 0 ]]; then - echo "....NOT OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG - test_status=FAIL - else - echo "....OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG - fi - fi - done - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - - # baseline creation run - else - - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo "Working dir = $RUNDIR" | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo "Moving baseline files to $NEW_BASELINE ...." | tee -a $PATHRT/$REGRESSIONTEST_LOG - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - - mkdir -p $NEW_BASELINE - - for file in *.nc; do - printf %s "Moving " $file "...." | tee -a $PATHRT/$REGRESSIONTEST_LOG - - cp $file $NEW_BASELINE/$file && d=$? || d=$? - if [[ $d -ne 0 ]]; then - echo "....NOT OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG - test_status=FAIL - else - echo "....OK" | tee -a $PATHRT/$REGRESSIONTEST_LOG - fi - done - echo | tee -a $PATHRT/$REGRESSIONTEST_LOG - - fi - - if [[ $test_status == FAIL ]]; then - echo "$TEST_NAME failed" >> $PATHRT/fail_test_$TEST_NAME - fi -} - readonly program=$(basename $0) # PATHRT - Path to regression tests directory readonly PATHRT="$(cd $(dirname $0) && pwd -P)" @@ -104,84 +42,67 @@ export PATHRT # PATHTR - Path to the UFS UTILS directory readonly PATHTR="$(cd $PATHRT/../.. && pwd)" export PATHTR -TESTS_FILE="$PATHRT/rt.conf" -export TEST_NAME= -export ATMLIST= - -# for C3072 on hera, use WLCLK=60 and MEM="--exclusive" -WLCLK_dflt=60 -export WLCLK=$WLCLK_dflt -MEM_dflt="--mem=16g" -export MEM=$MEM_dflt -export MOM6_version=20250128 - -cd $PATHRT -export compiler=${compiler:-intelllvm} + source $PATHTR/sorc/machine-setup.sh >/dev/null 2>&1 -if [[ "$compiler" == "intelllvm" ]]; then - if [[ ! -f ${PATHTR}/modulefiles/build.$target.$compiler.lua ]];then - echo "IntelLLVM not available. Will use Intel Classic." - compiler=intel - fi -fi +set +x echo "Machine: $target" -echo "Compiler: $compiler" +set -x -COMPILE_LOG=compile.log -REGRESSIONTEST_LOG=RegressionTests_$target.$compiler.log -rm -f fail_test* $COMPILE_LOG run_*.log nccmp_*.log summary.log +MOM6_version=20250128 -if [[ $target = wcoss2 ]]; then - STMP=${STMP:-/lfs/h2/emc/stmp/$USER} - export MOM6_FIXDIR=/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/mom6/${MOM6_version} - BASELINE_ROOT=/lfs/h2/emc/nems/noscrub/emc.nems/UFS_UTILS/reg_tests/cpld_gridgen/baseline_data - ACCOUNT=${ACCOUNT:-GFS-DEV} - export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core" - QUEUE=${QUEUE:-dev} - SBATCH_COMMAND="./cpld_gridgen.sh" - NCCMP=nccmp -elif [[ $target = hera ]]; then - STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER} - export MOM6_FIXDIR=/scratch1/NCEPDEV/global/glopara/fix/mom6/${MOM6_version} - BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data - ACCOUNT=${ACCOUNT:-nems} - QUEUE=${QUEUE:-batch} - NCCMP=nccmp - PARTITION=hera - SBATCH_COMMAND="./cpld_gridgen.sh" +# Adjust STMP, ACCOUNT and QUEUE as needed. + +if [[ $target = hera ]]; then + STMP=${STMP:-/scratch2/NCEPDEV/stmp1/$USER} + ACCOUNT=${ACCOUNT:-fv3-cpu} + QUEUE=${QUEUE:-batch} + WLCLK=40 + export MOM6_FIXDIR=/scratch1/NCEPDEV/global/glopara/fix/mom6/${MOM6_version} + export NCCMP=nccmp + BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data + PARTITION=hera elif [[ $target = orion ]]; then - STMP=${STMP:-/work/noaa/stmp/$USER} - export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} - BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils/reg_tests/cpld_gridgen/baseline_data - ACCOUNT=${ACCOUNT:-nems} - QUEUE=${QUEUE:-batch} - NCCMP=nccmp - PARTITION=orion - ulimit -s unlimited - SBATCH_COMMAND="./cpld_gridgen.sh" + STMP=${STMP:-/work/noaa/stmp/$USER} + ACCOUNT=${ACCOUNT:-fv3-cpu} + QUEUE=${QUEUE:-batch} + WLCLK=85 + export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} + export NCCMP=nccmp + BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils/reg_tests/cpld_gridgen/baseline_data + PARTITION=orion + ulimit -s unlimited elif [[ $target = hercules ]]; then - STMP=${STMP:-/work2/noaa/stmp/$USER} - export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} - BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils.hercules/reg_tests/cpld_gridgen/baseline_data - ACCOUNT=${ACCOUNT:-nems} - QUEUE=${QUEUE:-batch} - NCCMP=nccmp - PARTITION=hercules - ulimit -s unlimited - SBATCH_COMMAND="./cpld_gridgen.sh" + STMP=${STMP:-/work2/noaa/stmp/$USER} + ACCOUNT=${ACCOUNT:-fv3-cpu} + QUEUE=${QUEUE:-batch} + WLCLK=45 + export MOM6_FIXDIR=/work/noaa/global/glopara/fix/mom6/${MOM6_version} + BASELINE_ROOT=/work/noaa/nems/role-nems/ufs_utils.hercules/reg_tests/cpld_gridgen/baseline_data + export NCCMP=nccmp + PARTITION=hercules + ulimit -s unlimited elif [[ $target = jet ]]; then - STMP=${STMP:-/lfs5/HFIP/h-nems/$USER} - export MOM6_FIXDIR=/lfs5/HFIP/hfv3gfs/glopara/FIX/fix/mom6/${MOM6_version} - BASELINE_ROOT=/lfs5/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data - ACCOUNT=${ACCOUNT:-h-nems} - QUEUE=${QUEUE:-batch} - NCCMP=nccmp - PARTITION=xjet - ulimit -s unlimited - SBATCH_COMMAND="./cpld_gridgen.sh" + STMP=${STMP:-/lfs5/HFIP/emcda/$USER/stmp} + ACCOUNT=${ACCOUNT:-hfv3gfs} + QUEUE=${QUEUE:-batch} + WLCLK=60 + export MOM6_FIXDIR=/lfs5/HFIP/hfv3gfs/glopara/FIX/fix/mom6/${MOM6_version} + BASELINE_ROOT=/lfs5/HFIP/hfv3gfs/emc.nemspara/role.ufsutils/ufs_utils/reg_tests/cpld_gridgen/baseline_data + export NCCMP=nccmp + PARTITION=xjet + ulimit -s unlimited +elif [[ $target = wcoss2 ]]; then + STMP=${STMP:-/lfs/h2/emc/stmp/$USER} + ACCOUNT=${ACCOUNT:-GFS-DEV} + QUEUE=${QUEUE:-dev} + WLCLK=40 + export MOM6_FIXDIR=/lfs/h2/emc/global/noscrub/emc.global/FIX/fix/mom6/${MOM6_version} + BASELINE_ROOT=/lfs/h2/emc/nems/noscrub/emc.nems/UFS_UTILS/reg_tests/cpld_gridgen/baseline_data + export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core" + export NCCMP=nccmp fi + NEW_BASELINE_ROOT=$STMP/CPLD_GRIDGEN/BASELINE -RUNDIR_ROOT=$STMP/CPLD_GRIDGEN/rt_$$ BUILD_EXE=false CREATE_BASELINE=false @@ -205,22 +126,44 @@ while getopts :bcmh opt; do esac done +export CREATE_BASELINE +if [[ $CREATE_BASELINE = true ]]; then + rm -rf $NEW_BASELINE_ROOT + mkdir -p $NEW_BASELINE_ROOT +fi + +compiler=${compiler:-intelllvm} +if [[ "$compiler" == "intelllvm" ]]; then + if [[ ! -f ${PATHTR}/modulefiles/build.$target.$compiler.lua ]];then + set +x + echo "IntelLLVM not available. Will use Intel Classic." + set -x + compiler=intel + fi +fi +export compiler +set +x +echo "Compiler: $compiler" +set -x + # Build the executable file if [[ $BUILD_EXE = true ]]; then + COMPILE_LOG=compile.log cd $PATHTR - rm -rf $PATHTR/build $PATHTR/exec $PATHTR/lib + rm -rf $COMPILE_LOG $PATHTR/build $PATHTR/exec $PATHTR/lib ./build_all.sh >$PATHRT/$COMPILE_LOG 2>&1 && d=$? || d=$? if [[ d -ne 0 ]]; then error "Build did not finish successfully. Check $COMPILE_LOG" else + set +x echo "Build was successful" + set -x + cd $PATHRT fi -fi - -if [[ ! -f $PATHTR/exec/cpld_gridgen ]]; then - error "cpld_gridgen exe file is not found in $PATHTR/exe/. Try -b to build or -h for help." else - echo "cpld_gridgen exe file is found in $PATHTR/exec/" + if [[ ! -f $PATHTR/exec/cpld_gridgen ]]; then + error "cpld_gridgen exe file is not found in $PATHTR/exe/. Try -b to build or -h for help." + fi fi module use $PATHTR/modulefiles @@ -233,90 +176,69 @@ set +x module list set -x -if [[ $CREATE_BASELINE = true ]]; then - rm -rf $NEW_BASELINE_ROOT - mkdir -p $NEW_BASELINE_ROOT -fi - -date > $PATHRT/$REGRESSIONTEST_LOG -echo "Start Regression test" | tee -a $PATHRT/$REGRESSIONTEST_LOG +RUNDIR_ROOT=$STMP/CPLD_GRIDGEN/rt_$$ -# Run tests specified in $TESTS_FILE -while read -r line || [ "$line" ]; do +declare -A tests +all_tests="" - line="${line#"${line%%[![:space:]]*}"}" - [[ ${#line} == 0 ]] && continue - [[ $line =~ \# ]] && continue +rm -f fail_test* nccmp_*.log summary.log run_*log RegressionTests_$target.$compiler.*.log - TEST_NAME=$(echo $line | cut -d'|' -f1 | sed -e 's/^ *//' -e 's/ *$//') - TEST_NAME=${TEST_NAME##mx} - ATMLIST=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') - if [[ -z ${ATMLIST} ]]; then - ATMLIST=-1 - fi +# Kick off all tests. - cd $PATHRT - RUNDIR=$RUNDIR_ROOT/$TEST_NAME - BASELINE=$BASELINE_ROOT/$TEST_NAME - NEW_BASELINE=$NEW_BASELINE_ROOT/$TEST_NAME - mkdir -p $RUNDIR +i=0 +while read -r line || [ "$line" ]; do - # OUTDIR_PATH is passed down to $PATHTR/ush/cpld_gridgen.sh - # It MUST be set - export OUTDIR_PATH=$RUNDIR + line="${line#"${line%%[![:space:]]*}"}" + [[ ${#line} == 0 ]] && continue + [[ $line =~ \# ]] && continue - cp $PATHTR/exec/cpld_gridgen $RUNDIR - cp $PATHTR/ush/cpld_gridgen.sh $RUNDIR - cp $PATHRT/parm/grid.nml.IN $RUNDIR - cd $RUNDIR + TEST_NAME=$(echo $line | cut -d'|' -f1 | sed -e 's/^ *//' -e 's/ *$//') + TEST_NAME=${TEST_NAME##mx} + ATMLIST=$(echo $line | cut -d'|' -f2 | sed -e 's/^ *//' -e 's/ *$//') + if [[ -z ${ATMLIST} ]]; then + ATMLIST=-1 + fi - if [[ $target = wcoss2 ]]; then + export NEW_BASELINE=${NEW_BASELINE_ROOT}/$TEST_NAME + RUNDIR=$RUNDIR_ROOT/$TEST_NAME + mkdir -p $RUNDIR + export RUNDIR + export OUTDIR_PATH=$RUNDIR + export BASELINE=$BASELINE_ROOT/$TEST_NAME + export REGRESSIONTEST_LOG=RegressionTests_$target.$compiler.${TEST_NAME}.log + + cp $PATHRT/parm/grid.nml.IN $RUNDIR + cp $PATHTR/exec/cpld_gridgen $RUNDIR + + if [[ $target = wcoss2 ]]; then + tests[$i]=$(qsub -V -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ + -l walltime=00:${WLCLK}:00 -N $TEST_NAME -l select=1:ncpus=1:mem=12GB -v RESNAME=$TEST_NAME,ATMLIST="'$ATMLIST'" ./cpld_gridgen.sh) + + else + tests[$i]=$(sbatch --parsable --ntasks-per-node=1 --nodes=1 -t 00:${WLCLK}:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ + --partition=$PARTITION -o run_${TEST_NAME}.log -e run_${TEST_NAME}.log ./cpld_gridgen.sh "$TEST_NAME" "$ATMLIST") + fi - TEST=$(qsub -V -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log -q $QUEUE -A $ACCOUNT \ - -Wblock=true -l walltime=00:${WLCLK}:00 -N $TEST_NAME -l select=1:ncpus=1:mem=12GB -v RESNAME=$TEST_NAME $SBATCH_COMMAND) + all_tests=${all_tests}":"${tests[$i]} - else - sbatch --wait --ntasks-per-node=1 --nodes=1 ${MEM} -t 00:${WLCLK}:00 -A $ACCOUNT -q $QUEUE -J $TEST_NAME \ - --partition=$PARTITION -o $PATHRT/run_${TEST_NAME}.log -e $PATHRT/run_${TEST_NAME}.log \ - --wrap "time $SBATCH_COMMAND $TEST_NAME $ATMLIST" && d=$? || d=$? + ((i=i+1)) - if [[ d -ne 0 ]]; then - error "Batch job for test $TEST_NAME did not finish successfully. Refer to run_${TEST_NAME}.log" - fi +done < ./rt.conf - fi +export target - check_results +# Once all the jobs are finished, this summary job will run. -done <$TESTS_FILE -if [[ $? -ne 0 ]]; then - error "Run test while loop did not finish properly" -fi +if [[ $target = wcoss2 ]]; then -cd $PATHRT -FAIL_FILES="fail_test_*" -for file in $FAIL_FILES; do - if [[ -f "$file" ]]; then - cat "$file" >> fail_test - fi -done + qsub -V -o /dev/null -e /dev/null -q $QUEUE -A $ACCOUNT -l walltime=00:01:00 \ + -N summary -l select=1:ncpus=1:mem=100MB \ + -W depend=afterok${all_tests} ./rt.summary.sh +else -if [[ -e fail_test ]]; then - echo | tee -a $REGRESSIONTEST_LOG - for file in fail_test_*; do - cat $file >>$REGRESSIONTEST_LOG - cat $file >>summary.log - done + sbatch --nodes=1 -t 0:01:00 -A $ACCOUNT -J summary -o /dev/null -e /dev/null \ + --partition=$PARTITION --open-mode=append -q $QUEUE -d afterok${all_tests} ./rt.summary.sh - echo | tee -a $REGRESSIONTEST_LOG - echo "REGRESSION TEST FAILED" | tee -a $REGRESSIONTEST_LOG -else - echo | tee -a $REGRESSIONTEST_LOG - echo "REGRESSION TEST WAS SUCCESSFUL" | tee -a $REGRESSIONTEST_LOG - echo "All tests passed" >>summary.log fi -date >> $REGRESSIONTEST_LOG -elapsed_time=$( printf '%02dh:%02dm:%02ds\n' $((SECONDS%86400/3600)) $((SECONDS%3600/60)) $((SECONDS%60)) ) -echo "Elapsed time: ${elapsed_time}. Have a nice day!" >> ${REGRESSIONTEST_LOG} -echo "Elapsed time: ${elapsed_time}. Have a nice day!" +exit diff --git a/reg_tests/cpld_gridgen/rt.summary.sh b/reg_tests/cpld_gridgen/rt.summary.sh new file mode 100755 index 000000000..7e4d48b59 --- /dev/null +++ b/reg_tests/cpld_gridgen/rt.summary.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +set -eux + +#-------------------------------------------------------------------------- +# This script runs once all tests have completed. It checks for +# failed tests and creates some final log files. +#-------------------------------------------------------------------------- + +cd $PATHRT + +#-------------------------------------------------------------------------- +# If there are any failed tests, combine the log files into one log +# called 'fail_test'. +#-------------------------------------------------------------------------- + +rm -f failed_tests +FAIL_FILES="fail_test_*" +for file in $FAIL_FILES; do + if [[ -f "$file" ]]; then + cat "$file" >> failed_tests + rm -f $file + fi +done + +#-------------------------------------------------------------------------- +# Combine the run logs for each test into one log. +#-------------------------------------------------------------------------- + +LOG_FILE=RegressionTests_${target}.${compiler}.log +rm -f $LOG_FILE +for file in RegressionTests_$target.${compiler}.*.log +do + if [[ -f "$file" ]]; then + cat "$file" >> $LOG_FILE + rm -f $file + fi +done + +#-------------------------------------------------------------------------- +# Summarize the results in summary.log. +#-------------------------------------------------------------------------- + +rm -f summary.log +if [[ -e "failed_tests" ]]; then + echo | tee -a $LOG_FILE + cat failed_tests >>$LOG_FILE + cat failed_tests >>summary.log +else + echo | tee -a $LOG_FILE + echo "REGRESSION TEST WAS SUCCESSFUL" | tee -a $LOG_FILE + echo "All tests passed" >>summary.log +fi + +exit 0 diff --git a/reg_tests/rt.sh b/reg_tests/rt.sh index 72eb7587b..eaf6bc47a 100755 --- a/reg_tests/rt.sh +++ b/reg_tests/rt.sh @@ -100,7 +100,7 @@ if [[ $target == "orion" ]] || [[ $target == "jet" ]] || [[ $target == "hera" ]] export ACCOUNT=$PROJECT_CODE export STMP=$WORK_DIR/reg-tests - ./rt.sh 2>/dev/null & + ./rt.sh set -x diff --git a/ush/cpld_gridgen.sh b/ush/cpld_gridgen.sh deleted file mode 100755 index 8b0a16e25..000000000 --- a/ush/cpld_gridgen.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -set -eux - -function edit_namelist { - - sed -e "s/NI_GLB/$NI/g" \ - -e "s/NJ_GLB/$NJ/g" \ - -e "s|FIXDIR|$FIXDIR_PATH|g" \ - -e "s|OUTDIR|$OUTDIR_PATH|g" \ - -e "s|MOSAICDIR|$MOSAICDIR_PATH|g" \ - -e "s/TOPOGFILE/$TOPOGFILE/g" \ - -e "s/EDITSFILE/$EDITSFILE/g" \ - -e "s/RESNAME/$RESNAME/g" \ - -e "s/DO_MASKEDIT/$MASKEDIT/g" \ - -e "s/DO_DEBUG/$DEBUG/g" \ - -e "s/DO_POSTWGTS/$DO_POSTWGTS/g" \ - -e "s/ATMRESLIST/$ATMRESLIST/g" -} -export RESNAME=${RESNAME:-$1} -export ATMLIST=${ATMLIST:-$2} -export DEBUG=.false. -export MASKEDIT=.false. -export DO_POSTWGTS=.true. -export MOSAICDIR_PATH=${MOSAICDIR_PATH:-$PATHTR/fix/orog} -export FIXDIR_PATH=${MOM6_FIXDIR}/${RESNAME} -if [[ ${ATMLIST} -eq -1 ]]; then - export ATMRESLIST=12,24,48,96,192,384,768,1152,3072 -else - export ATMRESLIST=${ATMLIST} -fi - -APRUN=${APRUN:-"srun"} - -if [ $RESNAME = 900 ]; then - export NI=40 - export NJ=20 - export TOPOGFILE=topog.nc - export EDITSFILE='none' - if [ $DO_POSTWGTS == .true. ]; then - #pre-generate SCRIP files for dst rectilinear grids using NCO - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap - fi -fi - -if [ $RESNAME = 500 ]; then - export NI=72 - export NJ=35 - export TOPOGFILE=ocean_topog.nc - export EDITSFILE='none' - if [ $DO_POSTWGTS == .true. ]; then - #pre-generate SCRIP files for dst rectilinear grids using NCO - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap - fi -fi - -if [ $RESNAME = 100 ]; then - export NI=360 - export NJ=320 - export MASKEDIT=.T. - export TOPOGFILE=topog.nc - export EDITSFILE=topo_edits_011818.nc - if [ $DO_POSTWGTS == .true. ]; then - #pre-generate SCRIP files for dst rectilinear grids using NCO - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap - fi -fi - -if [ $RESNAME = 050 ]; then - export NI=720 - export NJ=576 - export TOPOGFILE=ocean_topog.nc - export EDITSFILE='none' - if [ $DO_POSTWGTS == .true. ]; then - #pre-generate SCRIP files for dst rectilinear grids using NCO - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p50_SCRIP.nc -G latlon=361,720#lon_typ=grn_ctr#lat_typ=cap - fi -fi - -if [ $RESNAME = 025 ]; then - export NI=1440 - export NJ=1080 - export TOPOGFILE=ocean_topog.nc - export EDITSFILE=All_edits.nc - if [ $DO_POSTWGTS == .true. ]; then - #pre-generate SCRIP files for dst rectilinear grids using NCO - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.9p00_SCRIP.nc -G latlon=20,40#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.5p00_SCRIP.nc -G latlon=36,72#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.1p00_SCRIP.nc -G latlon=181,360#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p50_SCRIP.nc -G latlon=361,720#lon_typ=grn_ctr#lat_typ=cap - $APRUN -n 1 ncremap -g ${OUTDIR_PATH}/rect.0p25_SCRIP.nc -G latlon=721,1440#lon_typ=grn_ctr#lat_typ=cap - fi -fi - -if [ ! -d ${OUTDIR_PATH} ]; then - mkdir -p ${OUTDIR_PATH} -fi - -cd ${OUTDIR_PATH} - -edit_namelist < grid.nml.IN > grid.nml -$APRUN ./cpld_gridgen - -# generate ice mesh -export FSRC=${OUTDIR_PATH}/Ct.mx${RESNAME}_SCRIP_land.nc -export FDST=${OUTDIR_PATH}/mesh.mx${RESNAME}.nc -$APRUN -n 1 ESMF_Scrip2Unstruct ${FSRC} ${FDST} 0 - -# generate kmt file for CICE -export FSRC=${OUTDIR_PATH}/grid_cice_NEMS_mx${RESNAME}.nc -export FDST=${OUTDIR_PATH}/kmtu_cice_NEMS_mx${RESNAME}.nc -ncks -O -v kmt ${FSRC} ${FDST}