Update to spack-stack-1.5.0 (update to gocart hash f7b34e)

CICE-interface/CMakeLists.txt

@@ -2,6 +2,9 @@
 if(CMAKE_Fortran_COMPILER_ID MATCHES "GNU")
   set(CMAKE_Fortran_FLAGS "-g -fbacktrace")
   set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fconvert=big-endian -ffree-line-length-none ")
+  if(${CMAKE_Fortran_COMPILER_VERSION} VERSION_GREATER_EQUAL 10)
+    set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fallow-argument-mismatch -fallow-invalid-boz")
+  endif()
   set(CMAKE_Fortran_FLAGS_RELEASE "-O2")
   set(CMAKE_Fortran_FLAGS_DEBUG "-O0 -Wall -Wextra -fcheck=bounds -ffpe-trap=invalid,zero,overflow,underflow" )
   set(CMAKE_Fortran_LINK_FLAGS "" )

modulefiles/ufs_acorn.intel.lua

@@ -2,11 +2,10 @@ help([[
 Load environment to build UFS on Acorn with Intel compiler
 ]])
 
-prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/lfs/h1/emc/nceplibs/noscrub/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 load("stack-intel")
 load("stack-cray-mpich")
-load("stack-python")
 
 --Avoid production installations; use spack-stack only:
 remove_path("MODULEPATH", "/apps/ops/prod/libs/modulefiles/compiler/intel/19.1.3.304")

modulefiles/ufs_common.lua

@@ -16,12 +16,12 @@ local ufs_modules = {
   {["crtm"]            = "2.4.0"},
   {["g2"]              = "3.4.5"},
   {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
+  {["ip"]              = "4.3.0"},
   {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
+  {["w3emc"]           = "2.10.0"},
   {["gftl-shared"]     = "1.5.0"},
   {["mapl"]            = "2.35.2-esmf-8.4.2"},
-  {["scotch"]          = "7.0.3"},
+  {["scotch"]          = "7.0.4"},
 }
 
 for i = 1, #ufs_modules do

modulefiles/ufs_gaea.intel.lua

@@ -10,18 +10,14 @@ unload("cray-mpich")
 unload("cray-python")
 unload("darshan")
 
-prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/spack-stack-1.4.1-c4/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/lustre/f2/pdata/esrl/gsd/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/lustre/f2/dev/wpo/role.epic/contrib/spack-stack/c4/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_cray_mpich_ver=os.getenv("stack_cray_mpich_ver") or "7.7.20"
 load(pathJoin("stack-cray-mpich", stack_cray_mpich_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 

modulefiles/ufs_hera.gnu.lua

@@ -2,7 +2,8 @@ help([[
 loads UFS Model prerequisites for Hera/GNU
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
+-- for openmpi, need
 prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
 
 stack_gnu_ver=os.getenv("stack_gnu_ver") or "9.2.0"
@@ -11,9 +12,6 @@ load(pathJoin("stack-gcc", stack_gnu_ver))
 stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
 load(pathJoin("stack-openmpi", stack_openmpi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 

modulefiles/ufs_hera.intel.lua

@@ -2,18 +2,14 @@ help([[
 loads UFS Model prerequisites for Hera/Intel
 ]])
 
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/scratch1/NCEPDEV/jcsda/jedipara/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env-noavx512/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 

modulefiles/ufs_hercules.gnu.lua

@@ -2,49 +2,20 @@ help([[
 loads UFS Model prerequisites for Hercules/GNU
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230825/envs/ufs-pio-2.5.10/install/modulefiles/Core")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env-mvap2/install/modulefiles/Core")
+-- for mvapich2, need:
 prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles")
 
 stack_gnu_ver=os.getenv("stack_gnu_ver") or "11.3.1"
 load(pathJoin("stack-gcc", stack_gnu_ver))
 
-stack_openmpi_ver=os.getenv("stack_openmpi_ver") or "4.1.5"
-load(pathJoin("stack-openmpi", stack_openmpi_ver))
-
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.14"
-load(pathJoin("stack-python", stack_python_ver))
+stack_mvapich2_ver=os.getenv("stack_mvapich2_ver") or "2.3.7"
+load(pathJoin("stack-mvapich2", stack_mvapich2_ver))
 
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
-local ufs_modules = {
-  {["jasper"]          = "2.0.32"},
-  {["zlib"]            = "1.2.13"},
-  {["libpng"]          = "1.6.37"},
-  {["hdf5"]            = "1.14.0"},
-  {["netcdf-c"]        = "4.9.2"},
-  {["netcdf-fortran"]  = "4.6.0"},
-  {["parallelio"]      = "2.5.10"},
-  {["esmf"]            = "8.4.2"},
-  {["fms"]             = "2023.01"},
-  {["bacio"]           = "2.4.1"},
-  {["crtm"]            = "2.4.0"},
-  {["g2"]              = "3.4.5"},
-  {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
-  {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
-  {["gftl-shared"]     = "1.5.0"},
-  {["mapl"]            = "2.35.2-esmf-8.4.2"},
-  {["scotch"]          = "7.0.4"},
-}
-
-for i = 1, #ufs_modules do
-  for name, default_version in pairs(ufs_modules[i]) do
-    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
-    load(pathJoin(name, os.getenv(env_version_name) or default_version))
-  end
-end
+load("ufs_common")
 
 setenv("CC", "mpicc")
 setenv("CXX", "mpic++")

modulefiles/ufs_hercules.intel.lua

@@ -2,49 +2,18 @@ help([[
 loads UFS Model prerequisites for Hercules/Intel
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230825/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/modulefiles")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.9.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.9.0"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.14"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
-local ufs_modules = {
-  {["jasper"]          = "2.0.32"},
-  {["zlib"]            = "1.2.13"},
-  {["libpng"]          = "1.6.37"},
-  {["hdf5"]            = "1.14.0"},
-  {["netcdf-c"]        = "4.9.2"},
-  {["netcdf-fortran"]  = "4.6.0"},
-  {["parallelio"]      = "2.5.10"},
-  {["esmf"]            = "8.4.2"},
-  {["fms"]             = "2023.01"},
-  {["bacio"]           = "2.4.1"},
-  {["crtm"]            = "2.4.0"},
-  {["g2"]              = "3.4.5"},
-  {["g2tmpl"]          = "1.10.2"},
-  {["ip"]              = "3.3.3"},
-  {["sp"]              = "2.3.3"},
-  {["w3emc"]           = "2.9.2"},
-  {["gftl-shared"]     = "1.5.0"},
-  {["mapl"]            = "2.35.2-esmf-8.4.2"},
-  {["scotch"]          = "7.0.4"},
-}
-
-for i = 1, #ufs_modules do
-  for name, default_version in pairs(ufs_modules[i]) do
-    local env_version_name = string.gsub(name, "-", "_") .. "_ver"
-    load(pathJoin(name, os.getenv(env_version_name) or default_version))
-  end
-end
+load("ufs_common")
 
 setenv("CC", "mpiicc")
 setenv("CXX", "mpiicpc")

modulefiles/ufs_jet.intel.lua

@@ -2,18 +2,14 @@ help([[
 loads UFS Model prerequisites for Jet/Intel
 ]])
 
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 

modulefiles/ufs_linux.intel

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+#%Module######################################################################
+##
+##    UFS prerequisites: Linux (tested: Ubuntu) with gcc/gfortran compilers
+
+echo "Setting environment variables for UFS Model on Linux with gcc/gfortran"
+
+##
+## load programming environment: compiler, flags, paths
+##
+export CC=${CC:-mpicc}
+export CXX=${CXX:-mpicxx}
+export F77=${F77:-mpif77}
+export F90=${F90:-mpif90}
+export FC=${FC:-mpif90}
+
+##
+## set up variables for ../cmake/configure_linux.intel.cmake
+##
+export CMAKE_Platform=linux.intel
+
+##
+## use own NetCDF library
+##
+export NETCDF=${NETCDF:-/home/builder/opt}
+
+##
+## use pre-compiled EMSF library for above compiler / MPI combination
+##
+export ESMFMKFILE=${ESMFMKFILE:-/home/builder/opt/lib/esmf.mk}
+
+##
+## NCEP libraries (need to download and build manually, see doc/README_{UBUNTU,CENTOS,...}.txt and https://github.com/NCAR/NCEPlibs)
+##
+export NCEPLIBS_DIR=${NCEPLIBS_DIR:-/home/builder/opt}
+export CMAKE_PREFIX_PATH=${NCEPLIBS_DIR}

modulefiles/ufs_noaacloud.intel.lua

@@ -3,8 +3,7 @@ loads UFS Model prerequisites for NOAA Parallelworks/Intel
 ]])
 
 
-prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/contrib/spack-stack/modulefiles/core")
+prepend_path("MODULEPATH", "/contrib/EPIC/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.3.0"
 load(pathJoin("stack-intel", stack_intel_ver))
@@ -15,9 +14,6 @@ load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 load("ufs_common")
 
 setenv("CC", "mpiicc")

modulefiles/ufs_orion.intel.lua

@@ -2,18 +2,14 @@ help([[
 loads UFS Model prerequisites for Orion/Intel
 ]])
 
-prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/work/noaa/da/role-da/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.1"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.7"
-load(pathJoin("stack-python", stack_python_ver))
-
 cmake_ver=os.getenv("cmake_ver") or "3.23.1"
 load(pathJoin("cmake", cmake_ver))
 

modulefiles/ufs_s4.intel.lua

@@ -2,18 +2,14 @@ help([[
 loads UFS Model prerequisites for S4/Intel
 ]])
 
-prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.4.1/envs/ufs-pio-2.5.10/install/modulefiles/Core")
-prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/modulefiles")
+prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core")
 
 stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
 load(pathJoin("stack-intel", stack_intel_ver))
 
 stack_impi_ver=os.getenv("stack_impi_ver") or "2021.5.0"
 load(pathJoin("stack-intel-oneapi-mpi", stack_impi_ver))
 
-stack_python_ver=os.getenv("stack_python_ver") or "3.9.12"
-load(pathJoin("stack-python", stack_python_ver))
-
 load("ufs_common")
 
 setenv("CC", "mpiicc")

tests/fv3_conf/fv3_slurm.IN_hercules

@@ -36,14 +36,15 @@ export OMP_NUM_THREADS=@[THRD]
 export ESMF_RUNTIME_PROFILE=ON
 export ESMF_RUNTIME_PROFILE_OUTPUT="SUMMARY"
 
+# For mvapich2
+if [[ @[RT_COMPILER] == gnu ]]; then
+  export MV2_SHMEM_COLL_NUM_COMM=128
+fi
+
 # Avoid job errors because of filesystem synchronization delays
 sync && sleep 1
 
-if [[ @[RT_COMPILER] == intel ]]; then
-  srun --label -n @[TASKS] ./fv3.exe
-elif [[ @[RT_COMPILER] == gnu ]]; then
-  mpirun -n @[TASKS] -N @[TPN] ./fv3.exe
-fi
+srun --label -n @[TASKS] ./fv3.exe
 
 echo "Model ended:    " `date`
 echo -n " $( date +%s )," >> job_timestamp.txt