diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 7c994ce45..8fe637fb6 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -136,6 +136,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 551c12e65..3275aee44 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -138,6 +138,8 @@ modules: ^esmf@8.7.0b04+debug snapshot=b04: 'esmf-8.7.0b04-debug' ^esmf@8.7.0b11~debug snapshot=b11: 'esmf-8.7.0b11' ^esmf@8.7.0b11+debug snapshot=b11: 'esmf-8.7.0b11-debug' + ^esmf@8.7.0~debug snapshot=none: 'esmf-8.7.0' + ^esmf@8.7.0+debug snapshot=none: 'esmf-8.7.0-debug' openmpi: environment: set: diff --git a/configs/common/packages b/configs/common/packages new file mode 100644 index 000000000..e69de29bb diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index fc7089741..99f17b060 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -54,17 +54,19 @@ packages: # Also, check the acorn and derecho site configs which have esmf modifications. esmf: require: - - '~xerces ~pnetcdf +shared +external-parallelio +python' - - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0b11 snapshot=b11'] - - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] - when: "%intel" - message: "Extra ESMF compile options for Intel" - #- any_of: [''] - # when: "%gcc" - # message: "Extra ESMF compile options for GCC" - #- any_of: [''] - # when: "%apple-clang" - # message: "Extra ESMF compile options for GCC" + - '~xerces ~pnetcdf +shared +external-parallelio' + - any_of: ['@=8.6.1 snapshot=none', '@=8.7.0 snapshot=none'] + - any_of: ['fflags="-fp-model precise" cxxflags="-fp-model precise"'] + when: "%intel" + message: "Extra ESMF compile options for Intel" + #- any_of: [''] + # when: "%gcc" + # message: "Extra ESMF compile options for GCC" + #- any_of: [''] + # when: "%apple-clang" + # message: "Extra ESMF compile options for GCC" + prefer: + - '+python' fckit: require: '@0.11.0 +eckit' fftw: diff --git a/configs/sites/tier1/sandy/mirrors.yaml b/configs/sites/tier1/atlantis/mirrors.yaml similarity index 67% rename from configs/sites/tier1/sandy/mirrors.yaml rename to configs/sites/tier1/atlantis/mirrors.yaml index c48f33bf1..4d363e009 100644 --- a/configs/sites/tier1/sandy/mirrors.yaml +++ b/configs/sites/tier1/atlantis/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///gpfs/fs1/neptune/spack-stack/source-cache + url: file:///neptune_diagnostics/spack-stack/source-cache/ access_pair: - null - null diff --git a/configs/sites/tier1/atlantis/packages_gcc.yaml b/configs/sites/tier1/atlantis/packages_gcc.yaml new file mode 100644 index 000000000..bc10f8b75 --- /dev/null +++ b/configs/sites/tier1/atlantis/packages_gcc.yaml @@ -0,0 +1,13 @@ +packages: + all: + compiler:: [gcc@11.2.0] + providers: + mpi:: [openmpi@4.1.5] + mpi: + buildable: False + openmpi: + buildable: False + externals: + - spec: openmpi@4.1.5%gcc@=11.2.0~cuda~cxx~cxx_exceptions~java~memchecker+pmi~static~wrapper-rpath fabrics=ucx schedulers=slurm + modules: + - openmpi/mlnx/gcc/64/4.1.5a1 diff --git a/configs/sites/tier1/atlantis/packages_intel.yaml b/configs/sites/tier1/atlantis/packages_intel.yaml index ea2f4de32..4271f3f10 100644 --- a/configs/sites/tier1/atlantis/packages_intel.yaml +++ b/configs/sites/tier1/atlantis/packages_intel.yaml @@ -15,6 +15,6 @@ packages: intel-oneapi-mkl: externals: - spec: intel-oneapi-mkl@2022.1.0 + prefix: /cm/shared/apps/intel/oneapi modules: - mkl/2022.1.0 - prefix: /cm/shared/apps/intel/oneapi diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index 1b7c23dcf..838a8a1dd 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -21,6 +21,30 @@ compilers:: set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] + - compiler: + spec: oneapi@2024.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-intel/8.4.0 + - intel/2024.2 + - cray-libsci/23.05.1.4 + - libfabric/1.12.1.2.2.1 + environment: + prepend_path: + PATH: '/opt/cray/pe/gcc/10.3.0/snos/bin' + CPATH: '/opt/cray/pe/gcc/10.3.0/snos/include' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/INTEL/2022.2/x86_64/lib:/opt/cray/pe/gcc/10.3.0/snos/lib:/opt/cray/pe/gcc/10.3.0/snos/lib64' + append_path: + CPATH: '/opt/intel/oneapi_2024.2.0.634/compiler/2024.2/opt/compiler/include/intel64' + set: + CRAYPE_LINK_TYPE: 'dynamic' + extra_rpaths: [] - compiler: spec: gcc@10.3.0 paths: diff --git a/configs/sites/tier1/narwhal/mirrors.yaml b/configs/sites/tier1/narwhal/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/narwhal/mirrors.yaml +++ b/configs/sites/tier1/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/narwhal/packages_oneapi.yaml b/configs/sites/tier1/narwhal/packages_oneapi.yaml new file mode 100644 index 000000000..3bcf99f38 --- /dev/null +++ b/configs/sites/tier1/narwhal/packages_oneapi.yaml @@ -0,0 +1,27 @@ +packages: + all: + compiler:: [oneapi@2024.2.0,gcc@10.3.0] + providers: + mpi:: [cray-mpich@8.1.21] + mpi: + buildable: False + cray-mpich: + externals: + - spec: cray-mpich@8.1.21%oneapi@2024.2.0 ~wrappers + modules: + - cray-mpich-ucx/8.1.21 + - craype-network-ucx + intel-oneapi-mkl: + externals: + - spec: intel-oneapi-mkl@2024.2%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + intel-oneapi-tbb: + externals: + - spec: intel-oneapi-tbb@2021.13%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 + modules: + - tbb/2021.13 + intel-oneapi-runtime: + externals: + - spec: intel-oneapi-runtime@2024.2.0%oneapi@2024.2.0 + prefix: /opt/intel/oneapi_2024.2.0.634 diff --git a/configs/sites/tier1/nautilus/mirrors.yaml b/configs/sites/tier1/nautilus/mirrors.yaml index 709f9096c..ec3db4a00 100644 --- a/configs/sites/tier1/nautilus/mirrors.yaml +++ b/configs/sites/tier1/nautilus/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache + url: file:///p/cwfs/projects/NEPTUNE/spack-stack/build-cache access_pair: - null - null diff --git a/configs/sites/tier1/sandy/compilers.yaml b/configs/sites/tier1/sandy/compilers.yaml deleted file mode 100644 index b86ff0a33..000000000 --- a/configs/sites/tier1/sandy/compilers.yaml +++ /dev/null @@ -1,14 +0,0 @@ -compilers: -- compiler: - spec: gcc@=9.3.0 - paths: - cc: /software/depot/gcc-9.3.0/bin/gcc - cxx: /software/depot/gcc-9.3.0/bin/g++ - f77: /software/depot/gcc-9.3.0/bin/gfortran - fc: /software/depot/gcc-9.3.0/bin/gfortran - flags: {} - operating_system: centos7 - target: x86_64 - modules: [] - environment: {} - extra_rpaths: [] diff --git a/configs/sites/tier1/sandy/config.yaml b/configs/sites/tier1/sandy/config.yaml deleted file mode 100644 index 84fe05ab9..000000000 --- a/configs/sites/tier1/sandy/config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -config: - build_jobs: 8 diff --git a/configs/sites/tier1/sandy/modules.yaml b/configs/sites/tier1/sandy/modules.yaml deleted file mode 100644 index aeb254c2e..000000000 --- a/configs/sites/tier1/sandy/modules.yaml +++ /dev/null @@ -1,8 +0,0 @@ -modules: - default: - enable:: - - tcl - tcl: - include: - # List of packages for which we need modules that are blacklisted by default - - python diff --git a/configs/sites/tier1/sandy/packages.yaml b/configs/sites/tier1/sandy/packages.yaml deleted file mode 100644 index 1af1ab029..000000000 --- a/configs/sites/tier1/sandy/packages.yaml +++ /dev/null @@ -1,113 +0,0 @@ -packages: - all: - compiler:: [gcc@9.3.0] - providers: - mpi:: [openmpi@4.0.5] - -### MPI, Python, MKL - mpi: - buildable: False - openmpi: - externals: - - spec: openmpi@4.0.5%gcc@9.3.0 - prefix: /software7/depot/openmpi-4.0.5 - -### All other external packages listed alphabetically - autoconf: - externals: - - spec: autoconf@2.69 - prefix: /usr - automake: - externals: - - spec: automake@1.16.1 - prefix: /usr - binutils: - externals: - - spec: binutils@2.30.117 - prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr - coreutils: - externals: - - spec: coreutils@8.30 - prefix: /usr - curl: - externals: - - spec: curl@7.61.1 - prefix: /usr - diffutils: - externals: - - spec: diffutils@3.6 - prefix: /usr - findutils: - externals: - - spec: findutils@4.6.0 - prefix: /usr - flex: - externals: - - spec: flex@2.6.1+lex - prefix: /usr - gawk: - externals: - - spec: gawk@4.2.1 - prefix: /usr - git: - externals: - - spec: git@2.39.2+tcltk - prefix: /software8/depot/git-2.39.2 - git-lfs: - externals: - - spec: git-lfs@3.3.0 - prefix: /software8/depot/git-2.39.2 - gmake: - externals: - - spec: gmake@4.2.1 - prefix: /usr - groff: - externals: - - spec: groff@1.22.3 - prefix: /usr - libtool: - externals: - - spec: libtool@2.4.6 - prefix: /usr - m4: - externals: - - spec: m4@1.4.18 - prefix: /usr - openssh: - externals: - - spec: openssh@8.0p1 - prefix: /usr - # Don't use external openssl, too old - #openssl: - # externals: - # - spec: openssl@1.1.1k - # prefix: /usr - # Can't use with py-xnrl - #perl: - # externals: - # - spec: perl@5.26.3~cpanm+shared+threads - # prefix: /usr - pkgconf: - externals: - - spec: pkgconf@1.4.2 - prefix: /usr - subversion: - externals: - - spec: subversion@1.10.2 - prefix: /usr - tar: - externals: - - spec: tar@1.30 - prefix: /usr - texinfo: - externals: - - spec: texinfo@6.5 - prefix: /usr - wget: - externals: - - spec: wget@1.19.5 - prefix: /usr diff --git a/configs/templates/neptune-dev/spack.yaml b/configs/templates/neptune-dev/spack.yaml index 564cf6d86..9a9e3dfec 100644 --- a/configs/templates/neptune-dev/spack.yaml +++ b/configs/templates/neptune-dev/spack.yaml @@ -8,8 +8,8 @@ spack: definitions: - compilers: ['%aocc', '%apple-clang', '%gcc', '%intel', '%oneapi'] - packages: - - neptune-env +espc ^esmf@8.7.0b11 snapshot=b11 - - neptune-python-env +xnrl ^esmf@8.7.0b11 snapshot=b11 + - neptune-env +espc ^esmf@8.7.0 + - neptune-python-env +xnrl ^esmf@8.7.0 specs: - matrix: diff --git a/configs/templates/skylab-dev/spack.yaml b/configs/templates/skylab-dev/spack.yaml index 4616138ca..be901ec7e 100644 --- a/configs/templates/skylab-dev/spack.yaml +++ b/configs/templates/skylab-dev/spack.yaml @@ -14,11 +14,11 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env # Various crtm tags (list all to avoid duplicate packages) @@ -27,7 +27,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none specs: - matrix: diff --git a/configs/templates/unified-dev/spack.yaml b/configs/templates/unified-dev/spack.yaml index 608b4462a..370a03e7c 100644 --- a/configs/templates/unified-dev/spack.yaml +++ b/configs/templates/unified-dev/spack.yaml @@ -17,12 +17,12 @@ spack: - jedi-fv3-env - jedi-geos-env ^esmf@=8.6.1 - jedi-mpas-env - - jedi-neptune-env ^esmf@=8.7.0b11 snapshot=b11 + - jedi-neptune-env ^esmf@=8.7.0 - jedi-tools-env - jedi-ufs-env ^esmf@=8.6.1 - jedi-um-env - - neptune-env ^esmf@=8.7.0b11 snapshot=b11 - - neptune-python-env ^esmf@=8.7.0b11 snapshot=b11 + - neptune-env ^esmf@=8.7.0 + - neptune-python-env ^esmf@=8.7.0 - soca-env - ufs-srw-app-env ^esmf@=8.6.1 - ufs-weather-model-env ^esmf@=8.6.1 @@ -33,7 +33,7 @@ spack: # Various esmf tags (list all to avoid duplicate packages) - esmf@=8.6.1 snapshot=none - - esmf@=8.7.0b11 snapshot=b11 + - esmf@=8.7.0 snapshot=none # MADIS for WCOSS2 decoders. - madis@4.5 diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 138fa604c..1fcf8ab7b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -65,7 +65,7 @@ Pre-configured sites (tier 1) +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | | Narwhal | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | | U.S. Navy (HPCMP) +-----------------------+--------------------+--------------------------------------------------------+-----------------+ -| | Nautilus | Intel | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +| | Nautilus | GCC, Intel, oneAPI | ``/p/app/projects/NEPTUNE/spack-stack/`` | NRL | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ | Univ. of Wisconsin | S4 | Intel | ``/data/prod/jedi/spack-stack/`` | JCSDA | +---------------------+-----------------------+--------------------+--------------------------------------------------------+-----------------+ @@ -167,6 +167,30 @@ The following is required for building new spack environments with Intel on this module unload cray-libsci module load cray-libsci/23.05.1.4 +The following is required for building new spack environments with Intel oneAPI on this platform.. Don't use ``module purge`` on Narwhal! + +.. code-block:: console + + umask 0022 + module unload PrgEnv-cray + module load PrgEnv-intel/8.4.0 + module unload intel + module load intel/2024.2 + module unload cray-mpich + module unload craype-network-ofi + # Warning. Do not load craype-network-ucx + # or cray-mpich-ucx/8.1.21! + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load craype-network-ucx + #module load cray-mpich-ucx/8.1.21 + module load libfabric/1.12.1.2.2.1 + module unload cray-libsci + module load cray-libsci/23.05.1.4 + The following is required for building new spack environments with GNU on this platform.. Don't use ``module purge`` on Narwhal! .. code-block:: console diff --git a/spack b/spack index 6f9a6ba4e..b9f2dd225 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 6f9a6ba4e5850494ffb7e7d592fe982036c32fb9 +Subproject commit b9f2dd2254cfb1086040f1a100746bd22cb12702 diff --git a/spack-ext/repos/spack-stack/packages/neptune-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-env/package.py index ddd732692..ff7aef9d4 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-env/package.py @@ -47,5 +47,6 @@ class NeptuneEnv(BundlePackage): # Basic Python dependencies that are always needed depends_on("py-f90nml", type="run") depends_on("py-python-dateutil", type="run") + depends_on("py-pyyaml", type="run") # There is no need for install() since there is no code. diff --git a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py index 5b6f243b8..2e20105b8 100644 --- a/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py +++ b/spack-ext/repos/spack-stack/packages/neptune-python-env/package.py @@ -22,6 +22,8 @@ class NeptunePythonEnv(BundlePackage): variant("xnrl", default=False, description="Build non-pulic XNRL") depends_on("neptune-env", type="run") + # Enable the Python variant for ESMF + depends_on("esmf +python", type="run") depends_on("py-h5py", type="run") depends_on("py-netcdf4", type="run")