From d576098cbaf28f1b3c675c316798827e56d5fc2c Mon Sep 17 00:00:00 2001 From: FernandoAndrade-NOAA Date: Thu, 14 Mar 2024 17:30:04 +0000 Subject: [PATCH 01/12] hera rocky8 and hercules python --- ci/rt.sh | 1 + modulefiles/hera.lua | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/rt.sh b/ci/rt.sh index cf839e660..65937b020 100755 --- a/ci/rt.sh +++ b/ci/rt.sh @@ -85,6 +85,7 @@ elif [ $mac3 = herc ] ; then module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load prod_util/2.1.1 + module load python/3.10.8 fi #set working directory diff --git a/modulefiles/hera.lua b/modulefiles/hera.lua index 5bd691da6..2231ad1c1 100644 --- a/modulefiles/hera.lua +++ b/modulefiles/hera.lua @@ -3,7 +3,7 @@ Load environment to build UPP on hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) From fe72b17b3713a9bddf96c7facf4fc384f644b0da Mon Sep 17 00:00:00 2001 From: Wen Meng Date: Sun, 17 Mar 2024 01:52:30 +0000 Subject: [PATCH 02/12] Add changes for Rocky8 on Hera. --- modulefiles/hera.lua | 2 +- modulefiles/hera_c.lua | 23 +++++++++++++++++++++++ tests/detect_machine.sh | 16 ++++++++-------- 3 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 modulefiles/hera_c.lua diff --git a/modulefiles/hera.lua b/modulefiles/hera.lua index 5bd691da6..2231ad1c1 100644 --- a/modulefiles/hera.lua +++ b/modulefiles/hera.lua @@ -3,7 +3,7 @@ Load environment to build UPP on hera ]]) -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/hera_c.lua b/modulefiles/hera_c.lua new file mode 100644 index 000000000..5bd691da6 --- /dev/null +++ b/modulefiles/hera_c.lua @@ -0,0 +1,23 @@ +help([[ +Load environment to build UPP on hera +]]) + + +prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("upp_common") + +setenv("CC","mpiicc") +setenv("CXX","mpiicpc") +setenv("FC","mpiifort") + +whatis("Description: UPP build environment") diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 997e550dc..23b0f4a87 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -44,14 +44,14 @@ case $(hostname -f) in gaea5[0-9]*) MACHINE_ID=gaea-c5 ;; ### GAEA C5 login node - hfe01) MACHINE_ID=hera ;; ### hera01 - hfe02) MACHINE_ID=hera ;; ### hera02 - hfe03) MACHINE_ID=hera ;; ### hera03 - hfe04) MACHINE_ID=hera ;; ### hera04 - hfe05) MACHINE_ID=hera ;; ### hera05 - hfe06) MACHINE_ID=hera ;; ### hera06 - hfe07) MACHINE_ID=hera ;; ### hera07 - hfe08) MACHINE_ID=hera ;; ### hera08 + hfe01) MACHINE_ID=hera_c ;; ### hera01 + hfe02) MACHINE_ID=hera_c ;; ### hera02 + hfe03) MACHINE_ID=hera_c ;; ### hera03 + hfe04) MACHINE_ID=hera_c ;; ### hera04 + hfe05) MACHINE_ID=hera_c ;; ### hera05 + hfe06) MACHINE_ID=hera_c ;; ### hera06 + hfe07) MACHINE_ID=hera_c ;; ### hera07 + hfe08) MACHINE_ID=hera_c ;; ### hera08 hfe09) MACHINE_ID=hera ;; ### hera09 hfe10) MACHINE_ID=hera ;; ### hera10 hfe11) MACHINE_ID=hera ;; ### hera11 From 3f31db8c3c96a450a67bd1aac64fdf6a06e7da72 Mon Sep 17 00:00:00 2001 From: Wen Meng Date: Sun, 17 Mar 2024 02:24:30 +0000 Subject: [PATCH 03/12] Add changes for rocky8 on jet. --- modulefiles/jet.lua | 2 +- modulefiles/jet_c.lua | 22 ++++++++++++++++++++++ tests/detect_machine.sh | 8 ++++---- 3 files changed, 27 insertions(+), 5 deletions(-) create mode 100644 modulefiles/jet_c.lua diff --git a/modulefiles/jet.lua b/modulefiles/jet.lua index dcd54d7b5..a118c88aa 100644 --- a/modulefiles/jet.lua +++ b/modulefiles/jet.lua @@ -2,7 +2,7 @@ help([[ Load environment to build UPP on Jet ]]) -prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core") stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" load(pathJoin("stack-intel", stack_intel_ver)) diff --git a/modulefiles/jet_c.lua b/modulefiles/jet_c.lua new file mode 100644 index 000000000..dcd54d7b5 --- /dev/null +++ b/modulefiles/jet_c.lua @@ -0,0 +1,22 @@ +help([[ +Load environment to build UPP on Jet +]]) + +prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" +load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) + +cmake_ver=os.getenv("cmake_ver") or "3.23.1" +load(pathJoin("cmake", cmake_ver)) + +load("upp_common") + +setenv("CC","mpiicc") +setenv("CXX","mpiicpc") +setenv("FC","mpiifort") + +whatis("Description: UPP build environment") diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 23b0f4a87..cbc6a96bc 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -57,10 +57,10 @@ case $(hostname -f) in hfe11) MACHINE_ID=hera ;; ### hera11 hfe12) MACHINE_ID=hera ;; ### hera12 - fe1) MACHINE_ID=jet ;; ### jet01 - fe2) MACHINE_ID=jet ;; ### jet02 - fe3) MACHINE_ID=jet ;; ### jet03 - fe4) MACHINE_ID=jet ;; ### jet04 + fe1) MACHINE_ID=jet_c ;; ### jet01 + fe2) MACHINE_ID=jet_c ;; ### jet02 + fe3) MACHINE_ID=jet_c ;; ### jet03 + fe4) MACHINE_ID=jet_c ;; ### jet04 fe5) MACHINE_ID=jet ;; ### jet05 fe6) MACHINE_ID=jet ;; ### jet06 fe7) MACHINE_ID=jet ;; ### jet07 From fe14ec7c4338d8887babe170bc6f269ce1ac8b16 Mon Sep 17 00:00:00 2001 From: FernandoAndrade-NOAA Date: Mon, 18 Mar 2024 15:28:01 -0400 Subject: [PATCH 04/12] gaea f5 & hercules python revert gaea_c5 to just gaea, update to spack stack 1.6.0, add python load to hercules --- modulefiles/gaea-c5.lua | 58 --------------------------------- modulefiles/gaea.lua | 70 ++++++++++++++-------------------------- modulefiles/hercules.lua | 3 ++ tests/compile_upp.sh | 24 ++++---------- tests/detect_machine.sh | 20 ++---------- 5 files changed, 35 insertions(+), 140 deletions(-) delete mode 100644 modulefiles/gaea-c5.lua diff --git a/modulefiles/gaea-c5.lua b/modulefiles/gaea-c5.lua deleted file mode 100644 index 09c7e6593..000000000 --- a/modulefiles/gaea-c5.lua +++ /dev/null @@ -1,58 +0,0 @@ -help([[ - This module loads libraries required for building and running UPP - on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. -]]) - -whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) - -load("PrgEnv-intel/8.3.3") -load("intel-classic/2023.1.0") -load("cray-mpich/8.1.25") -load("python/3.9.12") - -prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/spack-stack-dev-20230717/envs/unified-env/install/modulefiles/Core") -prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c5/modulefiles") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" -load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) - -stack_python_ver=os.getenv("stack_python_ver") or "3.9.12" -load(pathJoin("stack-python", stack_python_ver)) - -local ufs_modules = { - {["jasper"] = "2.0.32"}, - {["zlib"] = "1.2.13"}, - {["libpng"] = "1.6.37"}, - {["hdf5"] = "1.14.0"}, - {["netcdf-c"] = "4.9.2"}, - {["netcdf-fortran"] = "4.6.0"}, - {["bacio"] = "2.4.1"}, - {["crtm"] = "2.4.0"}, - {["g2"] = "3.4.5"}, - {["g2tmpl"] = "1.10.2"}, - {["ip"] = "3.3.3"}, - {["nemsio"] = "2.5.4"}, - {["sfcio"] = "1.4.1"}, - {["sigio"] = "2.3.2"}, - {["sp"] = "2.3.3"}, - {["w3emc"] = "2.9.2"}, - {["wrf-io"] = "1.2.0"}, -} - -for i = 1, #ufs_modules do - for name, default_version in pairs(ufs_modules[i]) do - local env_version_name = string.gsub(name, "-", "_") .. "_ver" - load(pathJoin(name, os.getenv(env_version_name) or default_version)) - end -end - -unload("darshan-runtime") -unload("cray-libsci") - -setenv("CC","cc") -setenv("CXX","CC") -setenv("FC","ftn") -setenv("CMAKE_Platform","gaea_c5.intel") diff --git a/modulefiles/gaea.lua b/modulefiles/gaea.lua index 7a87da323..0e610cfd7 100644 --- a/modulefiles/gaea.lua +++ b/modulefiles/gaea.lua @@ -1,56 +1,34 @@ help([[ This module loads libraries required for building and running UPP - on the NOAA RDHPC machine Gaea using Intel-2022.0.2 + on the NOAA RDHPC machine Gaea C5 using Intel-2023.1.0. ]]) whatis([===[Loads libraries needed for building the UPP on Gaea ]===]) -load_any(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"),"cmake") - -prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack") -load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0")) - -load(pathJoin("intel-classic", os.getenv("intel_classic_ver") or "2022.0.2")) -load(pathJoin("cray-mpich", os.getenv("cray_mpich_ver") or "7.7.20")) -load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2")) -load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20")) -load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37")) - --- Needed at runtime: -load("alps") - -local ufs_modules = { - {["jasper"] = "2.0.25"}, - {["zlib"] = "1.2.11"}, - {["libpng"] = "1.6.37"}, - {["hdf5"] = "1.10.6"}, - {["netcdf"] = "4.7.4"}, - {["pio"] = "2.5.7"}, - {["esmf"] = "8.3.0b09"}, - {["fms"] = "2022.04"}, - {["bacio"] = "2.4.1"}, - {["crtm"] = "2.4.0"}, - {["g2"] = "3.4.5"}, - {["g2tmpl"] = "1.10.2"}, - {["ip"] = "3.3.3"}, - {["sp"] = "2.3.3"}, - {["w3emc"] = "2.9.2"}, - {["gftl-shared"] = "v1.5.0"}, - {["mapl"] = "2.22.0-esmf-8.3.0b09"}, - {["nemsio"] = "2.5.4"}, - {["sigio"] = "2.3.2"}, - {["sfcio"] = "1.4.1"}, - {["wrf_io"] = "1.2.0"}, -} - -for i = 1, #ufs_modules do - for name, default_version in pairs(ufs_modules[i]) do - local env_version_name = string.gsub(name, "-", "_") .. "_ver" - load(pathJoin(name, os.getenv(env_version_name) or default_version)) - end -end +load("PrgEnv-intel/8.3.3") +load("intel-classic/2023.1.0") +load("cray-mpich/8.1.25") +load("python/3.9.12") + +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") +prepend_path("MODULEPATH", "/ncrc/proj/epic/spack-stack/modulefiles") + +stack_intel_ver=os.getenv("stack_intel_ver") or "2023.1.0" +load(pathJoin("stack-intel", stack_intel_ver)) + +stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "8.1.25" +load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver)) + +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" +load(pathJoin("stack-python", stack_python_ver)) + +load("upp_common") + +unload("darshan-runtime") +unload("cray-libsci") setenv("CC","cc") -setenv("FC","ftn") setenv("CXX","CC") +setenv("FC","ftn") + setenv("CMAKE_Platform","gaea.intel") diff --git a/modulefiles/hercules.lua b/modulefiles/hercules.lua index b057df81e..30163691b 100644 --- a/modulefiles/hercules.lua +++ b/modulefiles/hercules.lua @@ -14,6 +14,9 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) +stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +load(pathJoin("stack-python", stack_python_ver)) + load("upp_common") setenv("CC","mpiicc") diff --git a/tests/compile_upp.sh b/tests/compile_upp.sh index 2ecfe9f5e..c1cb507a1 100755 --- a/tests/compile_upp.sh +++ b/tests/compile_upp.sh @@ -83,24 +83,12 @@ source ${PATHTR}/tests/detect_machine.sh if [[ $MACHINE_ID != "unknown" ]]; then if [ $MACHINE_ID == "wcoss2" -o $MACHINE_ID == "wcoss2_a" ]; then module reset - elif [[ "$MACHINE_ID" =~ gaea.* ]] ; then - if [[ $( hostname ) =~ gaea5[0-9] ]] ; then - module purge - # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV - gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ - -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ - --pid=$$ --batch - # Reload system default modules: - set +eu - source /etc/bash.bashrc.local - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - set -eu - else - # There is no safe "module purge" on GAEA compute nodes. - set +eu - source /lustre/f2/dev/role.epic/contrib/Lmod_init.sh - set -eu - fi + elif [[ "$MACHINE_ID" =~ gaea* ]] ; then + module reset + # Unset the read-only variables $PELOCAL_PRGENV and $RCLOCAL_PRGENV + gdb -ex 'call (int) unbind_variable("PELOCAL_PRGENV")' \ + -ex 'call (int) unbind_variable("RCLOCAL_PRGENV")' \ + --pid=$$ --batch else module purge fi diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index cbc6a96bc..6df5d7c3b 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -25,24 +25,8 @@ case $(hostname -f) in clogin08.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus clogin09.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - gaea9) MACHINE_ID=gaea ;; ### gaea9 - gaea10) MACHINE_ID=gaea ;; ### gaea10 - gaea11) MACHINE_ID=gaea ;; ### gaea11 - gaea12) MACHINE_ID=gaea ;; ### gaea12 - gaea13) MACHINE_ID=gaea ;; ### gaea13 - gaea14) MACHINE_ID=gaea ;; ### gaea14 - gaea15) MACHINE_ID=gaea ;; ### gaea15 - gaea16) MACHINE_ID=gaea ;; ### gaea16 - gaea9.ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 - gaea10.ncrc.gov) MACHINE_ID=gaea ;; ### gaea10 - gaea11.ncrc.gov) MACHINE_ID=gaea ;; ### gaea11 - gaea12.ncrc.gov) MACHINE_ID=gaea ;; ### gaea12 - gaea13.ncrc.gov) MACHINE_ID=gaea ;; ### gaea13 - gaea14.ncrc.gov) MACHINE_ID=gaea ;; ### gaea14 - gaea15.ncrc.gov) MACHINE_ID=gaea ;; ### gaea15 - gaea16.ncrc.gov) MACHINE_ID=gaea ;; ### gaea16 - - gaea5[0-9]*) MACHINE_ID=gaea-c5 ;; ### GAEA C5 login node + gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea9 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 hfe01) MACHINE_ID=hera_c ;; ### hera01 hfe02) MACHINE_ID=hera_c ;; ### hera02 From c688c43798a6d610be46dc8297a3eb6ac87d6800 Mon Sep 17 00:00:00 2001 From: FernandoAndrade-NOAA Date: Mon, 18 Mar 2024 20:25:58 +0000 Subject: [PATCH 05/12] update hercules python & cleanup detect_machine --- modulefiles/hercules.lua | 2 +- tests/detect_machine.sh | 84 +++++++--------------------------------- 2 files changed, 15 insertions(+), 71 deletions(-) diff --git a/modulefiles/hercules.lua b/modulefiles/hercules.lua index 30163691b..da1722e68 100644 --- a/modulefiles/hercules.lua +++ b/modulefiles/hercules.lua @@ -14,7 +14,7 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) cmake_ver=os.getenv("cmake_ver") or "3.23.1" load(pathJoin("cmake", cmake_ver)) -stack_python_ver=os.getenv("stack_python_ver") or "3.10.8" +stack_python_ver=os.getenv("stack_python_ver") or "3.10.13" load(pathJoin("stack-python", stack_python_ver)) load("upp_common") diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index e2b5b0d8c..43ff12fd4 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -2,83 +2,27 @@ case $(hostname -f) in - alogin01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - alogin02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - adecflow01.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - adecflow02.acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### acorn - dlogin01.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin02.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin03.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin04.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin05.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin06.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin07.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin08.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - dlogin09.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - clogin01.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin02.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin03.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin04.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin05.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin06.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin07.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin08.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - clogin09.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus + alogin0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn + adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn + dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood + clogin0[1-9].cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - gaea5[1-8]*) MACHINE_ID=gaea ;; ### gaea9 + gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea9 gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 - hfe01) MACHINE_ID=hera_c ;; ### hera01 - hfe02) MACHINE_ID=hera_c ;; ### hera02 - hfe03) MACHINE_ID=hera_c ;; ### hera03 - hfe04) MACHINE_ID=hera_c ;; ### hera04 - hfe05) MACHINE_ID=hera_c ;; ### hera05 - hfe06) MACHINE_ID=hera_c ;; ### hera06 - hfe07) MACHINE_ID=hera_c ;; ### hera07 - hfe08) MACHINE_ID=hera_c ;; ### hera08 - hfe09) MACHINE_ID=hera ;; ### hera09 - hfe10) MACHINE_ID=hera ;; ### hera10 - hfe11) MACHINE_ID=hera ;; ### hera11 - hfe12) MACHINE_ID=hera ;; ### hera12 + hfe0[1-8]) MACHINE_ID=hera_c ;; ### hera01 + hfe09) MACHINE_ID=hera ;; ### hera09 + hfe1[0-2]) MACHINE_ID=hera ;; ### hera10 - fe1) MACHINE_ID=jet_c ;; ### jet01 - fe2) MACHINE_ID=jet_c ;; ### jet02 - fe3) MACHINE_ID=jet_c ;; ### jet03 - fe4) MACHINE_ID=jet_c ;; ### jet04 - fe5) MACHINE_ID=jet ;; ### jet05 - fe6) MACHINE_ID=jet ;; ### jet06 - fe7) MACHINE_ID=jet ;; ### jet07 - fe8) MACHINE_ID=jet ;; ### jet08 - tfe1) MACHINE_ID=jet ;; ### jet09 - tfe2) MACHINE_ID=jet ;; ### jet10 + fe[1-4]) MACHINE_ID=jet_c ;; ### jet01-4 + fe[5-8]) MACHINE_ID=jet ;; ### jet05-8 + tfe[12]) MACHINE_ID=jet ;; ### tjet1-2 - Orion-login-1.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1 - Orion-login-2.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion2 - Orion-login-3.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion3 - Orion-login-4.HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion4 + Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 - Hercules-login-1.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules1 - Hercules-login-2.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules2 - Hercules-login-3.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules3 - Hercules-login-4.HPC.MsState.Edu) MACHINE_ID=hercules ;; ### hercules4 + [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4 - cheyenne1.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - cheyenne1.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne1 - cheyenne2.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne2 - cheyenne3.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne3 - cheyenne4.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne4 - cheyenne5.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne5 - cheyenne6.ib0.cheyenne.ucar.edu) MACHINE_ID=cheyenne ;; ### cheyenne6 - - login1.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1 - login2.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede2 - login3.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede3 - login4.stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede4 + login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### S4 From d29762f25ddb6a1287a7429f740bf9677def2a4c Mon Sep 17 00:00:00 2001 From: Fernando Andrade - NOAA <121964418+FernandoAndrade-NOAA@users.noreply.github.com> Date: Mon, 18 Mar 2024 14:46:57 -0700 Subject: [PATCH 06/12] Update rt.sh --- ci/rt.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/rt.sh b/ci/rt.sh index 65937b020..cf839e660 100755 --- a/ci/rt.sh +++ b/ci/rt.sh @@ -85,7 +85,6 @@ elif [ $mac3 = herc ] ; then module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load prod_util/2.1.1 - module load python/3.10.8 fi #set working directory From c775516b047ced9583dfcb6f1525b109817a4179 Mon Sep 17 00:00:00 2001 From: Fernando Andrade - NOAA <121964418+FernandoAndrade-NOAA@users.noreply.github.com> Date: Mon, 18 Mar 2024 15:00:46 -0700 Subject: [PATCH 07/12] Update detect_machine.sh comment cleanup --- tests/detect_machine.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 43ff12fd4..19b725d2b 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -7,15 +7,15 @@ case $(hostname -f) in dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood clogin0[1-9].cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea9 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea9 + gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 - hfe0[1-8]) MACHINE_ID=hera_c ;; ### hera01 + hfe0[1-8]) MACHINE_ID=hera_c ;; ### hera01-08 hfe09) MACHINE_ID=hera ;; ### hera09 - hfe1[0-2]) MACHINE_ID=hera ;; ### hera10 + hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 - fe[1-4]) MACHINE_ID=jet_c ;; ### jet01-4 - fe[5-8]) MACHINE_ID=jet ;; ### jet05-8 + fe[1-4]) MACHINE_ID=jet_c ;; ### jet1-4 + fe[5-8]) MACHINE_ID=jet ;; ### jet5-8 tfe[12]) MACHINE_ID=jet ;; ### tjet1-2 Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 From 154422629ec70931a0fbec2a36c0c90fb749bfcf Mon Sep 17 00:00:00 2001 From: Fernando Andrade - NOAA <121964418+FernandoAndrade-NOAA@users.noreply.github.com> Date: Wed, 20 Mar 2024 08:10:17 -0700 Subject: [PATCH 08/12] Update detect_machine.sh --- tests/detect_machine.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 19b725d2b..4bcbdbfc6 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -10,8 +10,8 @@ case $(hostname -f) in gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 - hfe0[1-8]) MACHINE_ID=hera_c ;; ### hera01-08 - hfe09) MACHINE_ID=hera ;; ### hera09 + hfe0[1-4]) MACHINE_ID=hera_c ;; ### hera01-04 + hfe0[5-9]) MACHINE_ID=hera ;; ### hera05-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 fe[1-4]) MACHINE_ID=jet_c ;; ### jet1-4 From 92014fe2edb338f0163a156d1e1a61934c5bbfea Mon Sep 17 00:00:00 2001 From: Fernando Andrade - NOAA <121964418+FernandoAndrade-NOAA@users.noreply.github.com> Date: Wed, 20 Mar 2024 10:56:53 -0700 Subject: [PATCH 09/12] Update detect_machine.sh (ufs weather model) Adopted detect_machine.sh from the ufs-weather-model which should be more flexible in detecting the current machine. --- tests/detect_machine.sh | 98 ++++++++++++++++++++++++++++++++--------- 1 file changed, 77 insertions(+), 21 deletions(-) diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index 4bcbdbfc6..f8529c872 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -1,40 +1,96 @@ #!/bin/bash +# The authoritative copy of this script lives in the ufs-weather-model at: +# https://github.com/ufs-community/ufs-weather-model/blob/develop/tests/detect_machine.sh +# If any local modifications are made or new platform support added, +# please consider opening an issue and a PR to the ufs-weather-model +# so that this copy remains in sync with its authoritative source +# +# Thank you for your contribution + +# If the MACHINE_ID variable is set, skip this script. +[[ -n ${MACHINE_ID:-} ]] && return + +# First detect w/ hostname case $(hostname -f) in - alogin0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2_a ;; ### acorn - dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dodwood - clogin0[1-9].cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus + adecflow0[12].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=acorn ;; ### acorn + alogin0[123].acorn.wcoss2.ncep.noaa.gov) MACHINE_ID=acorn ;; ### acorn + clogin0[1-9].cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus01-9 + clogin10.cactus.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### cactus10 + dlogin0[1-9].dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood01-9 + dlogin10.dogwood.wcoss2.ncep.noaa.gov) MACHINE_ID=wcoss2 ;; ### dogwood10 - gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 - gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 + gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 - hfe0[1-4]) MACHINE_ID=hera_c ;; ### hera01-04 - hfe0[5-9]) MACHINE_ID=hera ;; ### hera05-09 - hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 + hfe0[1-4]) MACHINE_ID=hera_c ;; ### hera01-04 + hfe0[5-9]) MACHINE_ID=hera ;; ### hera01-09 + hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 + hecflow01) MACHINE_ID=hera ;; ### heraecflow01 - fe[1-4]) MACHINE_ID=jet_c ;; ### jet1-4 - fe[5-8]) MACHINE_ID=jet ;; ### jet5-8 - tfe[12]) MACHINE_ID=jet ;; ### tjet1-2 + s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### s4 + + fe[1-4]) MACHINE_ID=jet_c ;; ### jet1-4 + fe[5-8]) MACHINE_ID=jet ;; ### jet1-8 + tfe[12]) MACHINE_ID=jet ;; ### tjet1-2 Orion-login-[1-4].HPC.MsState.Edu) MACHINE_ID=orion ;; ### orion1-4 [Hh]ercules-login-[1-4].[Hh][Pp][Cc].[Mm]s[Ss]tate.[Ee]du) MACHINE_ID=hercules ;; ### hercules1-4 + derecho[1-8].hsn.de.hpc.ucar.edu) MACHINE_ID=derecho ;; ### derecho1-8 + login[1-4].stampede2.tacc.utexas.edu) MACHINE_ID=stampede ;; ### stampede1-4 - s4-submit.ssec.wisc.edu) MACHINE_ID=s4 ;; ### S4 + login0[1-2].expanse.sdsc.edu) MACHINE_ID=expanse ;; ### expanse1-2 - *) MACHINE_ID=unknown + discover3[1-5].prv.cube) MACHINE_ID=discover ;; ### discover31-35 + *) MACHINE_ID=UNKNOWN ;; # Unknown platform esac -# Overwrite auto-detect with RT_MACHINE if set -MACHINE_ID=${RT_MACHINE:-${MACHINE_ID}} +if [[ ${MACHINE_ID} == "UNKNOWN" ]]; then + case ${PW_CSP:-} in + "aws" | "google" | "azure") MACHINE_ID=noaacloud ;; + *) PW_CSP="UNKNOWN" + esac +fi + +# Overwrite auto-detect with MACHINE if set +MACHINE_ID=${MACHINE:-${MACHINE_ID}} -# Append compiler -#if [ $MACHINE_ID = orion ] || [ $MACHINE_ID = hera ] || [ $MACHINE_ID = cheyenne ] || [ $MACHINE_ID = jet ] || [ $MACHINE_ID = gaea ] || [ $MACHINE_ID = stampede ] || [ $MACHINE_ID = s4 ]; then -# MACHINE_ID=${MACHINE_ID}.${RT_COMPILER} -#fi +# If MACHINE_ID is no longer UNKNNOWN, return it +if [[ "${MACHINE_ID}" != "UNKNOWN" ]]; then + return +fi -echo "Machine: " $MACHINE_ID +# Try searching based on paths since hostname may not match on compute nodes +if [[ -d /lfs/h3 ]]; then + # We are on NOAA Cactus or Dogwood + MACHINE_ID=wcoss2 +elif [[ -d /lfs/h1 && ! -d /lfs/h3 ]]; then + # We are on NOAA TDS Acorn + MACHINE_ID=acorn +elif [[ -d /mnt/lfs1 ]]; then + # We are on NOAA Jet + MACHINE_ID=jet +elif [[ -d /scratch1 ]]; then + # We are on NOAA Hera + MACHINE_ID=hera +elif [[ -d /work ]]; then + # We are on MSU Orion or Hercules + if [[ -d /apps/other ]]; then + # We are on Hercules + MACHINE_ID=hercules + else + MACHINE_ID=orion + fi +elif [[ -d /gpfs && -d /ncrc ]]; then + # We are on GAEA. + MACHINE_ID=gaea +elif [[ -d /data/prod ]]; then + # We are on SSEC's S4 + MACHINE_ID=s4 +else + echo WARNING: UNKNOWN PLATFORM 1>&2 +fi From 7d2ff0680293a7e2a32a5e7748a3dee839518b45 Mon Sep 17 00:00:00 2001 From: FernandoAndrade-NOAA Date: Wed, 20 Mar 2024 13:08:40 -0500 Subject: [PATCH 10/12] rm hera_c & fail message --- modulefiles/hera_c.lua | 23 ----------------------- tests/compile_upp.sh | 5 +++++ 2 files changed, 5 insertions(+), 23 deletions(-) delete mode 100644 modulefiles/hera_c.lua diff --git a/modulefiles/hera_c.lua b/modulefiles/hera_c.lua deleted file mode 100644 index 5bd691da6..000000000 --- a/modulefiles/hera_c.lua +++ /dev/null @@ -1,23 +0,0 @@ -help([[ -Load environment to build UPP on hera -]]) - - -prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core") - -stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0" -load(pathJoin("stack-intel", stack_intel_ver)) - -stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1" -load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver)) - -cmake_ver=os.getenv("cmake_ver") or "3.23.1" -load(pathJoin("cmake", cmake_ver)) - -load("upp_common") - -setenv("CC","mpiicc") -setenv("CXX","mpiicpc") -setenv("FC","mpiifort") - -whatis("Description: UPP build environment") diff --git a/tests/compile_upp.sh b/tests/compile_upp.sh index c1cb507a1..70d5fc566 100755 --- a/tests/compile_upp.sh +++ b/tests/compile_upp.sh @@ -79,6 +79,11 @@ fi PATHTR=${PATHTR:-$( cd ${MYDIR}/.. && pwd )} source ${PATHTR}/tests/detect_machine.sh +if [[ $MACHINE_ID == "hera_c" ]]; then + echo "CentOS will soon be decommissioned on Hera, please utilize Rocky 8 Nodes (hfe05-9 + hfe10-12)." + exit 1 +fi + #Load required modulefiles if [[ $MACHINE_ID != "unknown" ]]; then if [ $MACHINE_ID == "wcoss2" -o $MACHINE_ID == "wcoss2_a" ]; then From 22f3d35333668d51d6a63ca94f010d5a15650873 Mon Sep 17 00:00:00 2001 From: FernandoAndrade-NOAA Date: Thu, 21 Mar 2024 10:26:11 -0500 Subject: [PATCH 11/12] rm hera_c & hercules python load --- ci/rt.sh | 1 + tests/compile_upp.sh | 5 ----- tests/detect_machine.sh | 3 +-- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/ci/rt.sh b/ci/rt.sh index cf839e660..e17f4d804 100755 --- a/ci/rt.sh +++ b/ci/rt.sh @@ -85,6 +85,7 @@ elif [ $mac3 = herc ] ; then module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load prod_util/2.1.1 + module load stack-python/3.10.13 fi #set working directory diff --git a/tests/compile_upp.sh b/tests/compile_upp.sh index 70d5fc566..c1cb507a1 100755 --- a/tests/compile_upp.sh +++ b/tests/compile_upp.sh @@ -79,11 +79,6 @@ fi PATHTR=${PATHTR:-$( cd ${MYDIR}/.. && pwd )} source ${PATHTR}/tests/detect_machine.sh -if [[ $MACHINE_ID == "hera_c" ]]; then - echo "CentOS will soon be decommissioned on Hera, please utilize Rocky 8 Nodes (hfe05-9 + hfe10-12)." - exit 1 -fi - #Load required modulefiles if [[ $MACHINE_ID != "unknown" ]]; then if [ $MACHINE_ID == "wcoss2" -o $MACHINE_ID == "wcoss2_a" ]; then diff --git a/tests/detect_machine.sh b/tests/detect_machine.sh index f8529c872..ba157a0eb 100755 --- a/tests/detect_machine.sh +++ b/tests/detect_machine.sh @@ -24,8 +24,7 @@ case $(hostname -f) in gaea5[1-8]) MACHINE_ID=gaea ;; ### gaea51-58 gaea5[1-8].ncrc.gov) MACHINE_ID=gaea ;; ### gaea51-58 - hfe0[1-4]) MACHINE_ID=hera_c ;; ### hera01-04 - hfe0[5-9]) MACHINE_ID=hera ;; ### hera01-09 + hfe0[1-9]) MACHINE_ID=hera ;; ### hera01-09 hfe1[0-2]) MACHINE_ID=hera ;; ### hera10-12 hecflow01) MACHINE_ID=hera ;; ### heraecflow01 From f8a1088e95cd7759b510f7b587cd259f3c083324 Mon Sep 17 00:00:00 2001 From: Fernando Andrade - NOAA <121964418+FernandoAndrade-NOAA@users.noreply.github.com> Date: Thu, 21 Mar 2024 08:46:29 -0700 Subject: [PATCH 12/12] Update rt.sh hera rocky8 --- ci/rt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/rt.sh b/ci/rt.sh index e17f4d804..57cea7942 100755 --- a/ci/rt.sh +++ b/ci/rt.sh @@ -65,7 +65,7 @@ if [ $mac2 = hf ]; then # for HERA export machine=HERA export homedir=${homedir:-"/scratch2/NAGAPE/epic/UPP/test_suite"} export rundir=${rundir:-"/scratch1/NCEPDEV/stmp2/${USER}"} - module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env/install/modulefiles/Core + module use /scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.6.0/envs/unified-env-rocky8/install/modulefiles/Core module load stack-intel/2021.5.0 module load stack-intel-oneapi-mpi/2021.5.1 module load prod_util/2.1.1