diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index deacbae25..d1ed33d65 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -105,14 +105,13 @@ jobs: #echo " - spec: intel-oneapi-runtime@2023.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml #echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - # Add external ecflow for Intel + # Add external qt@5 to build ecflow echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " ecflow::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " qt::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " buildable: False" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: ecflow@5.8.4+ui+static_boost" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " modules:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - ecflow/5.11.4" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: qt@5.15.3" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /usr" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml export -n SPACK_SYSTEM_CONFIG_PATH @@ -121,13 +120,13 @@ jobs: # Set compiler and MPI specs spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]" - spack config add "packages:all:compiler:[intel@2021.10.0]" + spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - # Don't generate ecflow module when using external package - spack config add "modules:default:tcl:exclude:[ecflow]" + # Don't generate qt module when using external package + spack config add "modules:default:tcl:exclude:[qt]" # Concretize and check for duplicates spack concretize 2>&1 | tee log.concretize.intel-2021.10.0-buildcache diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 1d20750ce..e2bf9e604 100644 --- a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -111,7 +111,7 @@ jobs: # Set compiler and MPI specs spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" - spack config add "packages:all:compiler:[oneapi@2024.2.0]" + spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index cd1d42635..4e78711ea 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -17,7 +17,8 @@ packages: require: '@3.4:' # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' boost: - require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' + require: + - '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: require: '@12.1.0 +python' bufr-query: @@ -37,7 +38,8 @@ packages: eccodes: require: '@2.33.0 +png +tools' ecflow: - require: '@5.11.4 +ui' + require: + - '@5.11.4 +ui' eckit: require: '@1.27.0 linalg=eigen,lapack compression=lz4,bzip2' ecmwf-atlas: diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 79fa77090..15c38b784 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -18,3 +18,9 @@ packages: py-numpy: require: - '^intel-oneapi-mkl' + boost: + require: + - '%gcc' + ecflow: + require: + - '%gcc' diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index a85b80871..e118f32c3 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -70,3 +70,25 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Needed for Intel 19/2022 backend +- compiler: + spec: gcc@10.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-gnu/8.3.3 + - craype/2.7.13 + - gcc/10.2.0 + - libfabric + environment: + set: + # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + extra_rpaths: [] diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 13c5246fd..34065c870 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@19.1.3.304,intel@2022.0.2.262] + compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@10.2.0] providers: mpi:: [cray-mpich@8.1.9] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml index 05b5731e6..3c9b262ba 100644 --- a/configs/sites/tier1/atlantis/compilers.yaml +++ b/configs/sites/tier1/atlantis/compilers.yaml @@ -16,3 +16,4 @@ compilers: prepend_path: LD_LIBRARY_PATH: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin extra_rpaths: [] +# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 0fb097d39..2e0c5976a 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.6.0] + compiler:: [intel@2021.6.0] # todo: add gcc here providers: mpi:: [intel-oneapi-mpi@2021.6.0] diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index bc5f464d4..1363ff58f 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@11.4.0] providers: mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml index edf995c8f..f37ddcacf 100644 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ b/configs/sites/tier1/derecho/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@12.2.0] providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/discover-scu16/compilers.yaml b/configs/sites/tier1/discover-scu16/compilers.yaml index 84cfc9b9a..949f2b8b5 100644 --- a/configs/sites/tier1/discover-scu16/compilers.yaml +++ b/configs/sites/tier1/discover-scu16/compilers.yaml @@ -34,3 +34,18 @@ compilers: - comp/gcc/12.1.0 environment: {} extra_rpaths: [] +# Needed for Intel backend +- compiler: + spec: gcc@=11.2.0 + paths: + cc: /usr/local/other/gcc/11.2.0/bin/gcc + cxx: /usr/local/other/gcc/11.2.0/bin/g++ + f77: /usr/local/other/gcc/11.2.0/bin/gfortran + fc: /usr/local/other/gcc/11.2.0/bin/gfortran + flags: {} + operating_system: sles12 + target: x86_64 + modules: + - comp/gcc/11.2.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/discover-scu16/packages_intel.yaml b/configs/sites/tier1/discover-scu16/packages_intel.yaml index 139fd869c..8bd48da65 100644 --- a/configs/sites/tier1/discover-scu16/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu16/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.6.0] + compiler:: [intel@2021.6.0,gcc@11.2.0] providers: mpi:: [intel-oneapi-mpi@2021.6.0] mpi: diff --git a/configs/sites/tier1/discover-scu17/compilers.yaml b/configs/sites/tier1/discover-scu17/compilers.yaml index 4ea0fa10d..989c471c2 100644 --- a/configs/sites/tier1/discover-scu17/compilers.yaml +++ b/configs/sites/tier1/discover-scu17/compilers.yaml @@ -51,3 +51,18 @@ compilers: - comp/gcc/12.3.0 environment: {} extra_rpaths: [] +# Needed for Intel backend +- compiler: + spec: gcc@=11.4.0 + paths: + cc: /usr/local/other/gcc/11.4.0/bin/gcc + cxx: /usr/local/other/gcc/11.4.0/bin/g++ + f77: /usr/local/other/gcc/11.4.0/bin/gfortran + fc: /usr/local/other/gcc/11.4.0/bin/gfortran + flags: {} + operating_system: sles15 + target: x86_64 + modules: + - comp/gcc/11.4.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index 0595c84a7..3512790ab 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@11.4.0] providers: mpi:: [intel-oneapi-mpi@2021.10.0] mpi: diff --git a/configs/sites/tier1/gaea-c5/compilers.yaml b/configs/sites/tier1/gaea-c5/compilers.yaml index 8aba0ea69..e60d7fece 100644 --- a/configs/sites/tier1/gaea-c5/compilers.yaml +++ b/configs/sites/tier1/gaea-c5/compilers.yaml @@ -23,3 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index 99b4bae6a..da5173fad 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2023.1.0] + compiler:: [intel@2023.1.0] # todo: add gcc here providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index f34608197..1bca116f0 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -23,3 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 321b33a7b..92f7d0a7c 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2023.2.0] + compiler:: [intel@2023.2.0] # todo: add gcc here providers: mpi:: [cray-mpich@8.1.29] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 983a808db..4f8025c5f 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.5.1] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index 1919c5d1d..78339c1c5 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.9.0] + compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index a31345813..d9cdd7209 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.5.1] mpi: diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index eb74a5993..1b7c23dcf 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -6,10 +6,7 @@ compilers:: cxx: CC f77: ftn fc: ftn - flags: - cflags: -diag-disable=10441 - cxxflags: -diag-disable=10441 - fflags: -diag-disable=10448 + flags: {} operating_system: sles15 modules: - PrgEnv-intel/8.3.3 @@ -27,20 +24,20 @@ compilers:: - compiler: spec: gcc@10.3.0 paths: - cc: /opt/cray/pe/gcc/10.3.0/bin/gcc - cxx: /opt/cray/pe/gcc/10.3.0/bin/g++ - f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran - fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran + cc: cc + cxx: CC + f77: ftn + fc: ftn flags: {} operating_system: sles15 modules: - PrgEnv-gnu/8.3.3 - gcc/10.3.0 - - cray-libsci/22.11.1.2 + - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml index 0cc712415..3261960c6 100644 --- a/configs/sites/tier1/narwhal/packages_gcc.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%gcc@10.3.0 +wrappers + - spec: cray-mpich@8.1.21%gcc@10.3.0 ~wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml index b8effd96d..0d631f58b 100644 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ b/configs/sites/tier1/narwhal/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.21] mpi: diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml index c3d08c859..c4460ac05 100644 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ b/configs/sites/tier1/nautilus/compilers.yaml @@ -35,6 +35,25 @@ compilers: CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' extra_rpaths: [] +- compiler: + spec: oneapi@2024.1.2 + paths: + cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx + f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + prepend_path: + PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' + CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' + LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' + append_path: + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + extra_rpaths: [] - compiler: spec: gcc@12.2.1 paths: @@ -49,22 +68,18 @@ compilers: - scl/gcc-toolset-12 environment: {} extra_rpaths: [] +# Required for Intel backend - compiler: - spec: oneapi@2024.1.2 + spec: gcc@11.2.1 paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + cc: /opt/rh/gcc-toolset-11/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++ + f77: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran + fc: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran flags: {} operating_system: rhel8 target: x86_64 - modules: [] - environment: - prepend_path: - PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' - CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' - LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + modules: + - scl/gcc-toolset-11 + environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/nautilus/packages_intel.yaml b/configs/sites/tier1/nautilus/packages_intel.yaml index 625387414..7516471db 100644 --- a/configs/sites/tier1/nautilus/packages_intel.yaml +++ b/configs/sites/tier1/nautilus/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@11.2.1] providers: mpi:: [openmpi@4.1.6] mpi: diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml index 40d0378e7..ce40a89e7 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2] + compiler:: [oneapi@2024.1.2,gcc@11.2.1] providers: mpi:: [intel-oneapi-mpi@2021.12] mpi: diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 193d5aac0..88131d85c 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.9.0] + compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] # https://github.com/JCSDA/spack-stack/issues/1055 diff --git a/configs/sites/tier1/s4/modules.yaml b/configs/sites/tier1/s4/modules.yaml index 5b7d734eb..e06812257 100644 --- a/configs/sites/tier1/s4/modules.yaml +++ b/configs/sites/tier1/s4/modules.yaml @@ -3,8 +3,6 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - mpich - python diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index a5e542ffd..3650e0678 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.3.0] providers: mpi:: [intel-oneapi-mpi@2021.5.0] # https://github.com/JCSDA/spack-stack/issues/1055 @@ -68,13 +68,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /data/prod/jedi/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.11 @@ -161,6 +154,7 @@ packages: - spec: pkg-config@0.27.1 prefix: /usr qt: + buildable: False externals: - spec: qt@5.9.7 prefix: /usr diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml index 8466a209c..f4c243c1b 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2] + compiler:: [oneapi@2024.1.2,gcc@11.4.1] providers: mpi:: [intel-oneapi-mpi] diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index b80cd7164..319b53b4a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -255,7 +255,13 @@ With Intel, the following is required for building new spack environments and fo module unload cray-mpich module unload craype-network-ofi module load craype-network-ucx - module load cray-mpich-ucx/8.1.21 + # Warning. Do not load cray-mpich-ucx/8.1.21 + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci module load cray-libsci/23.05.1.4 @@ -285,10 +291,16 @@ With GNU, the following is required for building new spack environments and for module unload cray-mpich module unload craype-network-ofi module load craype-network-ucx - module load cray-mpich-ucx/8.1.21 + # Warning. Do not load cray-mpich-ucx/8.1.21 + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci - module load cray-libsci/22.11.1.2 + module load cray-libsci/23.05.1.4 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 305bcaa8c..e2d75b651 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -39,7 +39,7 @@ SUBSTITUTES_TEMPLATE = { "MODULELOADS": "", "MODULEPREREQS": "", - "MODULEPATH": "", + "MODULEPATHS": "", "CC": "", "CXX": "", "F77": "", @@ -198,6 +198,13 @@ def module_prereq_command(module_choice, module): return "" +def modulepath_prepend_command(module_choice, modulepath): + if module_choice == "lmod": + return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) + else: + return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) + + def substitute_config_vars(config_str): """ Substitute spack-specific and environment variables that may be present @@ -299,7 +306,7 @@ def setup_meta_modules(): "Compiler candidate list is not unique: {}".format(compiler_candidate_list) ) if not len(mpi_candidate_list) == len(set(mpi_candidate_list)): - raise Exception("Compiler candidate list is not unique: {}".format(mpi_candidate_list)) + raise Exception("MPI candidate list is not unique: {}".format(mpi_candidate_list)) # Parse the directory tree under the top-level module directory logging.debug(os.listdir(module_dir)) @@ -308,6 +315,7 @@ def setup_meta_modules(): if not compiler_dict: raise Exception("No matching compilers found") logging.info(" ... stack compilers: '{}'".format(compiler_dict)) + # Then, check for mpi providers - recursively for compilers mpi_dict = get_matched_dict(module_dir, mpi_candidate_list, compiler_candidate_list) if not mpi_dict: @@ -332,6 +340,7 @@ def setup_meta_modules(): for version in compiler_dict[name] ] + # Determine core compiler(s) and make sure they are not used (usually something ancient) core_compilers = module_config["default"][module_choice]["core_compilers"] logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler @@ -349,14 +358,85 @@ def setup_meta_modules(): os.mkdir(meta_module_dir) logging.info(" ... meta module directory : {}".format(meta_module_dir)) + # Determine the preferred compiler and sort the flattened list of compilers + # such that the preferred compiler comes last. This is so that all other + # compilers populate the MODULEPATHS_SAVE list before the preferred compiler + # takes it and adds it to the stack-COMPILER metamodule. + try: + preferred_compilers = spack.config.get("packages")["all"]["prefer"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:all:prefer?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Not supported: more than one preferred compiler: {preferred_compilers}") + preferred_compiler = preferred_compilers[0].replace("%","") + logging.info(" ... preferred compiler: {}".format(preferred_compiler)) + del preferred_compilers + # Sort the list using a custom key + def custom_sort_key(entry): + # Return a tuple where the first element is 1 if the entry contains the word, else 0 + # The second element is the entry itself for natural sorting within groups + return (1 if preferred_compiler in entry else 0, entry) + sorted_flattened_compiler_list = sorted(flattened_compiler_list, key=custom_sort_key) + # Create compiler modules logging.info("Creating compiler modules ...") compiler_config = spack.config.get("compilers") - for compiler in compiler_config: - # On macOS, since July 2023, spack compiler find creates 'apple-clang@=14.0.0' etc. - compiler_spec = compiler["compiler"]["spec"].replace("@=", "@") - if compiler_spec in flattened_compiler_list: - (compiler_name, compiler_version) = compiler_spec.split("@") + # Collect and save modulepaths for the preferred compiler + MODULEPATHS_SAVE = [] + for compiler_identifier in sorted_flattened_compiler_list: + (compiler_name, compiler_version) = compiler_identifier.replace("@=","@").split("@") + # Loop through all configured compilers and find the correct match + compiler = False + for candidate_compiler in compiler_config: + if compiler_identifier.replace("@=","@") == \ + candidate_compiler["compiler"]["spec"].replace("@=", "@"): + compiler = candidate_compiler + break + if not compiler: + raise Exception("No matching compiler for {} found in spack compiler config".format(compiler)) + continue + + # This is only to keep the current level of indentation for easier reviewing + if True: + modulepath_save = os.path.join(module_dir, compiler_name, compiler_version) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) + MODULEPATHS_SAVE.append(modulepath_save) + + # For tcl modules remove the compiler prefices from the module contents + if module_choice == "tcl": + logging.info( + " ... ... removing compiler prefices from tcl modulefiles in {}".format( + modulepath_save + ) + ) + for root, ddir, files in os.walk(modulepath_save): + for ffile in files: + filepath = os.path.join(root, ffile) + logging.debug( + " ... ... ... removing compiler prefices in {}".format(filepath) + ) + for pattern in module_replace_patterns: + for tmp_compiler_spec in sorted_flattened_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Paranoid, but better safe than sorry + del pattern + del tmp_compiler_spec + + # The remainder of the loop is only needed for the preferred compiler + if not compiler_name in preferred_compiler: + continue + logging.info( " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) ) @@ -423,34 +503,11 @@ def setup_meta_modules(): substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) - # Spack compiler module hierarchy - substitutes["MODULEPATH"] = os.path.join(module_dir, compiler_name, compiler_version) - logging.debug(" ... ... MODULEPATH : {}".format(substitutes["MODULEPATH"])) - # If the environment doesn't have compiler-only dependent modules - # then simply create the placeholder directory - if not os.path.isdir(substitutes["MODULEPATH"]): - os.makedirs(substitutes["MODULEPATH"]) - - # For tcl modules remove the compiler prefices from the module contents - if module_choice == "tcl": - logging.info( - " ... ... removing compiler prefices from tcl modulefiles in {}".format( - substitutes["MODULEPATH"] - ) - ) - for root, ddir, files in os.walk(substitutes["MODULEPATH"]): - for ffile in files: - filepath = os.path.join(root, ffile) - logging.debug( - " ... ... ... removing compiler prefices in {}".format(filepath) - ) - for pattern in module_replace_patterns: - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, compiler_name, compiler_version, filepath, sed_syntax_fix - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) + # Spack compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) # Read compiler template into module_content string with open(COMPILER_TEMPLATES[module_choice]) as f: @@ -466,6 +523,7 @@ def setup_meta_modules(): with open(compiler_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(compiler_module_file)) + del MODULEPATHS_SAVE # Create mpi modules for mpi_name in mpi_dict.keys(): @@ -638,24 +696,25 @@ def setup_meta_modules(): substitutes["MPIF77"] = os.path.join("mpif77") substitutes["MPIF90"] = os.path.join("mpif90") - # Spack compiler module hierarchy - substitutes["MODULEPATH"] = os.path.join( + # Spack mpi+compiler module hierarchy + modulepath = os.path.join( module_dir, mpi_name, mpi_version, compiler_name, compiler_version ) - logging.debug(" ... ... MODULEPATH : {}".format(substitutes["MODULEPATH"])) # If the environment doesn't have mpi dependent modules # then simply create the placeholder directory - if not os.path.isdir(substitutes["MODULEPATH"]): - os.makedirs(substitutes["MODULEPATH"]) + if not os.path.isdir(modulepath): + os.makedirs(modulepath) + substitutes["MODULEPATHS"] = modulepath_prepend_command(module_choice, modulepath).rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) # For tcl modules remove the compiler/mpi prefices from the module contents if module_choice == "tcl": logging.info( " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - substitutes["MODULEPATH"] + modulepath ) ) - for root, ddir, files in os.walk(substitutes["MODULEPATH"]): + for root, ddir, files in os.walk(modulepath): for ffile in files: filepath = os.path.join(root, ffile) logging.debug( @@ -666,18 +725,23 @@ def setup_meta_modules(): # Search patterns patterns = ["is-loaded", "module load", "depends-on"] # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all for pattern in module_replace_patterns: - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, - compiler_name, - compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) + for tmp_compiler_spec in sorted_flattened_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Paranoid, but better safe than sorry + del pattern + del tmp_compiler_spec # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers for pattern in module_replace_patterns: cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( pattern, @@ -731,9 +795,6 @@ def setup_meta_modules(): not compiler_name in python_dict.keys() or not compiler_version in python_dict[compiler_name].keys() ): - logging.warn( - f"No Python version found for compiler {compiler_name}@{compiler_version}, skipping Python modules" - ) continue spec = python_dict[compiler_name][compiler_version] python_version = str(spec.version) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler index 24725a82c..78891450f 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler @@ -4,18 +4,19 @@ # # conflicts +conflict stack-aocc conflict stack-apple-clang conflict stack-clang -conflict stack-gnu +conflict stack-gcc conflict stack-intel -conflict stack-intel-oneapi-compilers +conflict stack-oneapi # prerequisite modules @MODULELOADS@ @MODULEPREREQS@ # spack compiler module hierarchy -prepend-path {MODULEPATH} {@MODULEPATH@} +@MODULEPATHS@ # compiler environment variables setenv {F77} {@F77@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 21daea763..2f741ddbc 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -8,18 +8,19 @@ local pkgNameVer = myModuleFullName() family("MetaCompiler") -- conflicts +conflict("stack-aocc") conflict("stack-apple-clang") conflict("stack-clang") -conflict("stack-gnu") +conflict("stack-gcc") conflict("stack-intel") -conflict("stack-intel-oneapi-compilers") +conflict("stack-oneapi") -- prerequisite modules @MODULELOADS@ @MODULEPREREQS@ -- spack compiler module hierarchy -prepend_path("MODULEPATH", "@MODULEPATH@") +@MODULEPATHS@ -- compiler environment variables setenv("F77", "@F77@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi index d239fd5d5..188d50946 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi @@ -15,7 +15,7 @@ conflict stack-mpt @MODULEPREREQS@ # spack compiler module hierarchy -prepend-path {MODULEPATH} {@MODULEPATH@} +@MODULEPATHS@ # mpi wrapper environment variables setenv {MPICC} {@MPICC@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index bf805321c..c84007f84 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -19,7 +19,7 @@ conflict("stack-mpt") @MODULEPREREQS@ -- spack compiler module hierarchy -prepend_path("MODULEPATH", "@MODULEPATH@") +@MODULEPATHS@ -- mpi wrapper environment variables setenv("MPICC", "@MPICC@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py index 36f08c1aa..3be5afa38 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py @@ -50,6 +50,21 @@ def test_setup_meta_modules(): # Setup env and pretend that a build exists # by creating the module directory structure. scope = env.scope_name + # Mock compiler config + compiler_def = { + "compiler": { + "flags": {}, + "modules": [], + "paths": {"cc": "gcc", "cxx": "g++", "f77": "gfortran", "fc": "gfortran"}, + "extra_rpaths": [], + "operating_system": "test", + "target": "test", + "environment": {}, + "spec": "gcc@9.4.0", + } + } + compiler_config = [compiler_def] + spack.config.set("compilers", compiler_config, scope=scope) spack.config.add("packages:all:compiler:[{}]".format(comp), scope=scope) spack.config.add("packages:all:providers:mpi:[{}]".format(mpi), scope=scope) spack.config.add("packages:openmpi:version:[{}]".format(mpi_ver))