diff --git a/.gitmodules b/.gitmodules index be1fdad253..df1ec05f41 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ [submodule "spack"] path = spack - url = https://github.com/dominichofer/spack.git + url = https://github.com/spack/spack.git diff --git a/README.md b/README.md index aeab8042a6..4fe7e4e464 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ Spack is the package manager used by C2SM and MeteoSwiss to install and deploy s * [spack-c2sm v0.18.1.1](https://C2SM.github.io/spack-c2sm/v0.18.1.1) [deprecated] **General infos about spack** + * [Official spack v0.21.1](https://spack.readthedocs.io/en/v0.21.1/) * [Official spack v0.20.1](https://spack.readthedocs.io/en/v0.20.1/) * [Official spack v0.18.1](https://spack.readthedocs.io/en/v0.18.1/) diff --git a/repos/c2sm/packages/cdo/package.py b/repos/c2sm/packages/cdo/package.py deleted file mode 100644 index b623a5c495..0000000000 --- a/repos/c2sm/packages/cdo/package.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -from spack.pkg.builtin.cdo import Cdo as SpackCdo - - -class Cdo(SpackCdo): - conflicts('%nvhpc@22.7') diff --git a/repos/c2sm/packages/fdb/package.py b/repos/c2sm/packages/fdb/package.py index d11e599e5c..459f692f5a 100644 --- a/repos/c2sm/packages/fdb/package.py +++ b/repos/c2sm/packages/fdb/package.py @@ -1,28 +1,21 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - from spack.package import * + from spack.pkg.builtin.fdb import Fdb as SpackFdb class Fdb(SpackFdb): - """FDB (Fields DataBase) is a domain-specific object store developed at - ECMWF for storing, indexing and retrieving GRIB data.""" - version("5.11.17", - sha256= - "375c6893c7c60f6fdd666d2abaccb2558667bd450100817c0e1072708ad5591e") - - depends_on("ecbuild@3.7:", type="build", when="@5.11.6:") + # This section can be removed when the following commit + # https://github.com/spack/spack/commit/8871bd5ba5c58562b8c20baa00f125aeccba586f + # is included in a release on spack and this is used by spack-c2sm. + depends_on("eckit@1.24.4:", when="@5.11.22:") + # This section can be removed when the following PR + # https://github.com/spack/spack/pull/42874 + # is included in a release on spack and this is used by spack-c2sm. @property def libs(self): return find_libraries("libfdb5", root=self.prefix, shared=True, recursive=True) - - def setup_build_environment(self, env): - env.set('CTEST_OUTPUT_ON_FAILURE', 1) diff --git a/repos/c2sm/packages/infero/package.py b/repos/c2sm/packages/infero/package.py index 86758ed41a..d5e159dbe8 100644 --- a/repos/c2sm/packages/infero/package.py +++ b/repos/c2sm/packages/infero/package.py @@ -29,8 +29,8 @@ class Infero(CMakePackage): variant('tf_c', description='Enable tensorflow-c backend', default=False) variant('onnx', description='Enable ONNX backend', default=False) - depends_on('eckit@1.20.0:') - depends_on('fckit') + depends_on('eckit@1.20.2') + depends_on('fckit@0.9.0') depends_on('ecbuild', type=('build')) depends_on('tensorflowc', when='+tf_c') depends_on('onnx-runtime', when='+onnx') diff --git a/repos/c2sm/packages/metkit/package.py b/repos/c2sm/packages/metkit/package.py new file mode 100644 index 0000000000..42733610db --- /dev/null +++ b/repos/c2sm/packages/metkit/package.py @@ -0,0 +1,10 @@ +from spack.package import * + +from spack.pkg.builtin.metkit import Metkit as SpackMetkit + + +class Metkit(SpackMetkit): + + # This file can be removed when this PR https://github.com/spack/spack/pull/42871 + # is included in a release on spack and this is used by spack-c2sm. + depends_on("eckit@:1.21", when="@:1.10") diff --git a/repos/c2sm/packages/py-cmake/package.py b/repos/c2sm/packages/py-cmake/package.py index 110087ac7a..b43601df0f 100644 --- a/repos/c2sm/packages/py-cmake/package.py +++ b/repos/c2sm/packages/py-cmake/package.py @@ -25,6 +25,8 @@ class PyCmake(PythonPackage): sha256= '52b98c5ee70b5fa30a8623e96482227e065292f78794eb085fdf0fecb204b79b') + # in newer pip versions --install-option does not exist + depends_on("py-pip@:23.0", type="build") depends_on('ninja', type='build') depends_on('py-scikit-build@0.12:', type='build') depends_on('py-setuptools@42:', type='build') diff --git a/repos/c2sm/packages/py-cytoolz/package.py b/repos/c2sm/packages/py-cytoolz/package.py index e995177256..b3e8f72b83 100644 --- a/repos/c2sm/packages/py-cytoolz/package.py +++ b/repos/c2sm/packages/py-cytoolz/package.py @@ -17,6 +17,9 @@ class PyCytoolz(PythonPackage): maintainers = ['samkellerhals'] + version('0.12.3', + sha256= + '4503dc59f4ced53a54643272c61dc305d1dbbfbd7d6bdf296948de9f34c3a282') version('0.12.0', sha256= 'c105b05f85e03fbcd60244375968e62e44fe798c15a3531c922d531018d22412') @@ -24,6 +27,10 @@ class PyCytoolz(PythonPackage): depends_on('py-setuptools', type='build') depends_on('py-cython', type='build') + # py-cytoolz@0.12.0 not compatible with py-cython@3:, see + # https://www.layerzrozero.network/?_=%2Fpytoolz%2Fcytoolz%2Fissues%2F202%23w2n%2BddnGqHIZTHHkluHJC3Vn + depends_on('py-cython@:2', when='@0.12.0', type='build') + depends_on('python@3.5:', type=('build', 'run')) depends_on('py-toolz@0.8.0:', type=('build', 'run')) diff --git a/repos/c2sm/packages/py-frozendict/package.py b/repos/c2sm/packages/py-frozendict/package.py index 7c8f731ea3..54f3a85f2b 100644 --- a/repos/c2sm/packages/py-frozendict/package.py +++ b/repos/c2sm/packages/py-frozendict/package.py @@ -4,22 +4,19 @@ # SPDX-License-Identifier: (Apache-2.0 OR MIT) from spack import * +from spack.pkg.builtin.py_frozendict import PyFrozendict as SpackPyFrozendict -class PyFrozendict(PythonPackage): - """frozendict is a simple immutable dictionary.""" +class PyFrozendict(SpackPyFrozendict): - homepage = "https://github.com/Marco-Sulla/python-frozendict" - - pypi = 'frozendict/frozendict-2.3.4.tar.gz' - - maintainers = ['samkellerhals'] - - # FIXME: Add proper versions and checksums here. - version('2.3.4', + version('2.4.0', sha256= - '15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78') - - depends_on('python@3.6:', type=('build', 'run')) - - depends_on('py-setuptools', type='build') + 'c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a') + + # TODO: remove this extension once we have a more recent + # version than v0.21.1 + def setup_build_environment(self, env): + # C extension is not supported for 3.11+. See also + # https://github.com/Marco-Sulla/python-frozendict/issues/68 + if self.spec.satisfies("^python@3.11:"): + env.set("FROZENDICT_PURE_PY", "1") diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index a7c593374c..a768c62434 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -51,12 +51,17 @@ class PyGt4py(PythonPackage): depends_on('py-lark@1.1.2:', type=('build', 'run')) depends_on('py-mako@1.1:', type=('build', 'run')) depends_on('py-ninja@1.10:', type=('build', 'run')) - depends_on('py-numpy@1.24.2: ~blas ~lapack', type=('build', 'run')) + depends_on('py-numpy@1.24.2:', type=('build', 'run')) depends_on('py-packaging@20.0:', type=('build', 'run')) depends_on('py-pybind11@2.10.1:', type=('build', 'run')) depends_on('py-nanobind@1.4.0:', when="@1.0.1.3:", type=('build', 'run')) depends_on('py-tabulate@0.8.10:', type=('build', 'run')) - depends_on('py-typing-extensions@4.2:4.6', type=('build', 'run')) + depends_on('py-typing-extensions@4.5.0', + when="@:1.0.3.5", + type=('build', 'run')) + depends_on('py-typing-extensions@4.10.0', + when="@1.0.3.6:", + type=('build', 'run')) depends_on('py-toolz@0.12.0:', type=('build', 'run')) depends_on('py-xxhash@1.4.4:3.0.9', type=('build', 'run')) depends_on('py-hypothesis@6.0.0:', type=('build', 'run')) diff --git a/repos/c2sm/packages/py-numpy/package.py b/repos/c2sm/packages/py-numpy/package.py deleted file mode 100644 index d1591a9ca5..0000000000 --- a/repos/c2sm/packages/py-numpy/package.py +++ /dev/null @@ -1,24 +0,0 @@ -from spack.package import * -from spack.pkg.builtin.py_numpy import PyNumpy as SpackPyNumpy - - -class PyNumpy(SpackPyNumpy): - """NumPy is the fundamental package for scientific computing with Python. - It contains among other things: a powerful N-dimensional array object, - sophisticated (broadcasting) functions, tools for integrating C/C++ and - Fortran code, and useful linear algebra, Fourier transform, and random - number capabilities""" - - version('1.24.2', - sha256= - '003a9f530e880cb2cd177cba1af7220b9aa42def9c4afc2a2fc3ee6be7eb2b22') - - conflicts("python@3.10.1", when="@1.24.2") # metadata generation fails. - - def url_for_version(self, version): - url = "https://files.pythonhosted.org/packages/source/n/numpy/numpy-{}.{}" - if version >= Version("1.23"): - ext = "tar.gz" - else: - ext = "zip" - return url.format(version, ext) diff --git a/repos/c2sm/packages/py-toolz/package.py b/repos/c2sm/packages/py-toolz/package.py deleted file mode 100644 index 9d78cf434f..0000000000 --- a/repos/c2sm/packages/py-toolz/package.py +++ /dev/null @@ -1,11 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.py_toolz import PyToolz as SpackPyToolz - - -class PyToolz(SpackPyToolz): - """A set of utility functions for iterators, functions, and dictionaries.""" - - version("0.12.0", - sha256= - "88c570861c440ee3f2f6037c4654613228ff40c93a6c25e0eba70d17282c6194") diff --git a/repos/c2sm/packages/py-typing-extensions/package.py b/repos/c2sm/packages/py-typing-extensions/package.py index 136f30de18..d3b0981791 100644 --- a/repos/c2sm/packages/py-typing-extensions/package.py +++ b/repos/c2sm/packages/py-typing-extensions/package.py @@ -5,6 +5,9 @@ class PyTypingExtensions(SpackPyTypingExtensions): """Backported and Experimental Type Hints for Python 3.7+""" + version("4.10.0", + sha256= + "b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb") version("4.5.0", sha256= "5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb") diff --git a/repos/c2sm/packages/python/package.py b/repos/c2sm/packages/python/package.py new file mode 100644 index 0000000000..b3c4e15a08 --- /dev/null +++ b/repos/c2sm/packages/python/package.py @@ -0,0 +1,5 @@ +from spack.pkg.builtin.python import Python as SpackPython + + +class Python(SpackPython): + variant("ssl", default=False, description="Build ssl module") diff --git a/repos/c2sm/packages/xpmem/package.py b/repos/c2sm/packages/xpmem/package.py new file mode 100644 index 0000000000..d1c19da4f2 --- /dev/null +++ b/repos/c2sm/packages/xpmem/package.py @@ -0,0 +1,120 @@ +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +from spack.package import * + + +# Identical to spack v0.21.1 except for the part labeled as WORKAROUND +class Xpmem(AutotoolsPackage): + """XPMEM is a Linux kernel module that enables a process to map the memory + of another process into its virtual address space.""" + + # The README file of the repository says that the development was + # transferred to a new repository on GitLab: https://gitlab.com/hjelmn/xpmem + # However, it looks like that the repository on GitHub has a more recent + # version of the codebase. + homepage = "https://github.com/hjelmn/xpmem" + url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz" + git = "https://github.com/hjelmn/xpmem.git" + + maintainers("skosukhin") + + version("master", branch="master") + + # Versions starting 2.6.4 are neither tagged nor released in the repo + # (the choice of commits is based on the commit history of + # 'kernel/xpmem_private.h'): + version("2.6.5-36", commit="0d0bad4e1d07b38d53ecc8f20786bb1328c446da") + version("2.6.5", commit="4efeed9cbaabe971f3766d67cb108e2c3316d4b8") + version("2.6.4", commit="522054850e4d1479d69f50f7190d1548bf9749fd") + + # Released versions: + version("2.6.3", + sha256= + "ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54") + version("2.6.2", + sha256= + "2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b") + + variant("kernel-module", + default=True, + description="Enable building the kernel module") + + # Added RHEL 8.3 kernel support + # Here 2.6.5-36 referes to 2.6.5 version and 36th commit id + patch("xpmem_v2.6.5-36.patch", when="@2.6.5-36", level=1) + patch( + "https://github.com/hjelmn/xpmem/commit/cbd6e5bd3d2a1d3823c335ddcd3c57b94474f578.patch?full_index=1", + sha256= + "75299398b6c15546479bfbb8aa972431f58637fe2f0328196a26738bd7148140", + when="@2.6.5-36", + level=1, + ) + patch( + "https://github.com/hjelmn/xpmem/commit/7d346aaf1fdfc24d38cebb4ad107b7f5c43769e9.patch?full_index=1", + sha256= + "6be8c5f33d55c611924d8412253740f6f4b738e6d98e32981fa300d2ccbe99cc", + when="@2.6.5-36", + level=1, + ) + + depends_on("autoconf", type="build") + depends_on("automake", type="build") + depends_on("libtool", type="build") + depends_on("m4", type="build") + + # It will become possible to disable the kernel module only starting 2.6.6: + # https://github.com/hjelmn/xpmem/pull/24 + conflicts("~kernel-module", when="@:2.6.5") + + # Ideally, we should list all non-Linux-based platforms here: + conflicts("+kernel-module", when="platform=darwin") + + # All compilers except for gcc are in conflict with +kernel-module: + #requires("%gcc", when="+kernel-module", msg="Linux kernel module must be compiled with gcc") + + #WORKAROUND: The above line is not working as expected. + # The below line tires to express the same thing. + # But it is only true in a context with nvhpc and gcc as the only compilers. + conflicts("%nvhpc", when="+kernel-module") + + def autoreconf(self, spec, prefix): + Executable("./autogen.sh")() + + @run_before("build") + def override_kernel_compiler(self): + # Override the compiler for kernel module source files. We need + # this additional argument for all installation phases. + if "+kernel-module" in self.spec: + make.add_default_arg("CC={0}".format(spack_cc)) + + def configure_args(self): + args = [] + + if "~kernel-module" in self.spec: + # The kernel module is enabled by default. An attempt of explicit + # enabling with '--enable-kernel-module' disables the module. + args.append("--disable-kernel-module") + + if self.spec.satisfies("@:2.6.5"): + fmt = self.spec.format + # The following arguments will not be needed starting 2.6.6: + # https://github.com/hjelmn/xpmem/pull/18 + args.extend([ + fmt("--with-default-prefix={prefix}"), + fmt("--with-module={prefix.share}/Modules/{name}/{version}"), + ]) + + return args + + @when("@:2.6.5") + def install(self, spec, prefix): + with working_dir(self.build_directory): + # Override the hardcoded prefix for 'cray-xpmem.conf' + make( + "ldsoconfdir={0}".format( + self.spec.prefix.etc.join("ld.so.conf.d")), + *self.install_targets, + ) diff --git a/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch b/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch new file mode 100644 index 0000000000..a17fba4ad1 --- /dev/null +++ b/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch @@ -0,0 +1,15 @@ +--- xpmem/kernel/xpmem_pfn.c 2021-02-08 22:49:05.321501753 -0800 ++++ xpmem.patch/kernel/xpmem_pfn.c 2021-02-08 23:30:30.875309634 -0800 +@@ -251,6 +251,12 @@ + cpu_to_node(task_cpu(current)) != cpu_to_node(task_cpu(src_task))) { + #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0) + saved_mask = current->cpus_mask; ++#elif LINUX_VERSION_CODE == KERNEL_VERSION(4,18, 0) ++ #ifdef RHEL_RELEASE_CODE ++ #if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3) ++ saved_mask = current->cpus_mask; ++ #endif ++ #endif + #else + saved_mask = current->cpus_allowed; + #endif diff --git a/spack b/spack index f0fcd2734a..e30fedab10 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit f0fcd2734a3a0d82a1a0f69fef7b8cc301ae48a8 +Subproject commit e30fedab102f9281a220fb4fae82e3f8c43a82ac diff --git a/src/report_tests.py b/src/report_tests.py index 87629f3c69..f6ec5ac6dc 100644 --- a/src/report_tests.py +++ b/src/report_tests.py @@ -41,8 +41,6 @@ def __str__(self) -> str: # Trigger phrases that cause a test to get a special icon and comment. # List[(trigger, icon, comment)] triggers = [ - ('AssertionError exception when releasing read lock', ':lock:', - 'spack locking problem'), ('Timed out waiting for a write lock', ':lock:', 'spack write lock problem'), ('Timed out waiting for a read lock', ':lock:', diff --git a/sysconfigs/balfrin/config.yaml b/sysconfigs/balfrin/config.yaml index c9fdfde0d5..d242e424c6 100644 --- a/sysconfigs/balfrin/config.yaml +++ b/sysconfigs/balfrin/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/daint/config.yaml b/sysconfigs/daint/config.yaml index fec1c3dfb2..2533771b03 100644 --- a/sysconfigs/daint/config.yaml +++ b/sysconfigs/daint/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/daint/modules.yaml b/sysconfigs/daint/modules.yaml deleted file mode 100644 index 30eb2a4a46..0000000000 --- a/sysconfigs/daint/modules.yaml +++ /dev/null @@ -1,78 +0,0 @@ -modules: - default: - enable: - - tcl - tcl: - all: - autoload: 'direct' - naming_scheme: '${PACKAGE}/${VERSION}' - hash_length: 7 - blacklist: ['cmake', 'slurm', 'boost', 'automake', 'autoconf', 'jasper', 'libtool', 'm4', 'ncurses', 'netcdf-c', 'netcdf-fortran', 'perl', 'libjpeg'] - boost: - environment: - set: - BOOST_ROOT: '${PREFIX}' - gridtools: - environment: - set: - GRIDTOOLS_DIR: '${PREFIX}' - GRIDTOOLS_ROOT: '${PREFIX}' - cosmo-dycore: - environment: - set: - DYCOREGT_DIR: '${PREFIX}' - UCX_MEMTYPE_CACHE: 'n' - UCX_TLS: 'rc_x,ud_x,mm,shm,cuda_copy,cuda_ipc,cma' - MPICH_G2G_PIPELINE: '64' - MPICH_RDMA_ENABLED_CUDA: '1' - cosmo-grib-api-definitions: - environment: - prepend_path: - GRIB_DEFINITION_PATH: '${PREFIX}/cosmoDefinitions/definitions/' - GRIB_SAMPLES_PATH: '${PREFIX}/cosmoDefinitions/samples' - cosmo-grib-api: - environment: - set: - GRIBAPI_DIR: '${PREFIX}' - prepend_path: - GRIB_DEFINITION_PATH: '{PREFIX}/share/grib_api/definitions' - cosmo-eccodes-definitions: - environment: - prepend_path: - GRIB_DEFINITION_PATH: '${PREFIX}/cosmoDefinitions/definitions/' - GRIB_SAMPLES_PATH: '${PREFIX}/cosmoDefinitions/samples' - eccodes: - environment: - set: - GRIBAPI_DIR: '${PREFIX}' - prepend_path: - GRIB_DEFINITION_PATH: '{PREFIX}/share/eccodes/definitions' - serialbox: - environment: - set: - SERIALBOX_DIR: '${PREFIX}' - SERIALBOX_ROOT: '${PREFIX}' - ant: - environment: - set: - ANT_HOME: '${PREFIX}' - bison: - environment: - set: - YACC: 'bison -y' - cosmo cosmo_target=cpu: - environment: - set: - UCX_MEMTYPE_CACHE: 'n' - UCX_TLS: 'rc_x,ud_x,mm,shm,cma' - mpi: - environment: - set: - MPI_ROOT: '${PREFIX}' - cuda: - environment: - prepend_path: - CPATH: '${PREFIX}/extras/CUPTI/include:${PREFIX}/nvvm/include' - LD_LIBRARY_PATH: '${PREFIX}/extras/CUPTI/lib64:${PREFIX}/nvvm/lib64:/cm/local/apps/cuda/libs/current/lib64' - LIBRARY_PATH: '${PREFIX}/lib64/stubs' - PATH: '${PREFIX}:${PREFIX}/nvvm/bin' diff --git a/sysconfigs/daint/packages.yaml b/sysconfigs/daint/packages.yaml index 3652848f8c..808f2ac00b 100755 --- a/sysconfigs/daint/packages.yaml +++ b/sysconfigs/daint/packages.yaml @@ -58,7 +58,6 @@ packages: cosmo: # manually added variants: slave=daint cuda_arch=60 cosmo-dycore: # manually added - compiler: [gcc@8.3.0] variants: slave=daint cuda_arch=60 data_path=/scratch/snx3000/jenkins/data/cosmo/ cuda: # manually added buildable: false diff --git a/sysconfigs/daint/upstreams.yaml b/sysconfigs/daint/upstreams.yaml index e372e85752..11e4dd1b0f 100644 --- a/sysconfigs/daint/upstreams.yaml +++ b/sysconfigs/daint/upstreams.yaml @@ -1,7 +1,7 @@ upstreams: base: - install_tree: /project/g110/spack/upstream/daint_v0.20.1.0/base + install_tree: /project/g110/spack/upstream/daint_v0.21.1.0/base icon-dsl: - install_tree: /project/g110/spack/upstream/daint_v0.20.1.3/icon-dsl + install_tree: /project/g110/spack/upstream/daint_v0.21.1.0/icon-dsl icon-rttov: - install_tree: /project/g110/spack/upstream/daint_v0.20.1.0/icon-rttov + install_tree: /project/g110/spack/upstream/daint_v0.21.1.0/icon-rttov diff --git a/sysconfigs/dom/config.yaml b/sysconfigs/dom/config.yaml index de80985dd8..4e136592d6 100644 --- a/sysconfigs/dom/config.yaml +++ b/sysconfigs/dom/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/eiger/config.yaml b/sysconfigs/eiger/config.yaml index c9fdfde0d5..d242e424c6 100644 --- a/sysconfigs/eiger/config.yaml +++ b/sysconfigs/eiger/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/tasna/config.yaml b/sysconfigs/tasna/config.yaml index c9fdfde0d5..d242e424c6 100644 --- a/sysconfigs/tasna/config.yaml +++ b/sysconfigs/tasna/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/tsa/config.yaml b/sysconfigs/tsa/config.yaml index 6c98d3afed..e9b1aae628 100644 --- a/sysconfigs/tsa/config.yaml +++ b/sysconfigs/tsa/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' misc_cache: '$spack/../spack-misc_cache' diff --git a/sysconfigs/tsa/modules.yaml b/sysconfigs/tsa/modules.yaml deleted file mode 100644 index 68ebd5237e..0000000000 --- a/sysconfigs/tsa/modules.yaml +++ /dev/null @@ -1,9 +0,0 @@ -modules: - default: - enable: - - tcl - tcl: - naming_scheme: '${PACKAGE}/${VERSION}' - hash_length: 7 - blacklist: ['%gcc', '%pgi'] - whitelist: ['cosmo', 'cosmo-dycore', 'cosmo-eccodes-definitions', 'cosmo-gribapi-definitions'] \ No newline at end of file diff --git a/sysconfigs/unknown/config.yaml b/sysconfigs/unknown/config.yaml index 5d90f0a405..53ef97f15f 100644 --- a/sysconfigs/unknown/config.yaml +++ b/sysconfigs/unknown/config.yaml @@ -1,6 +1,4 @@ config: - extensions: - - '$spack/../tools/spack-scripting' build_stage: '$spack/../spack-build_stage' test_stage: '$spack/../spack-test_stage' - misc_cache: '$spack/../spack-misc_cache' \ No newline at end of file + misc_cache: '$spack/../spack-misc_cache' diff --git a/test/integration_test.py b/test/integration_test.py index b87da5eee6..7eee322fca 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -37,9 +37,6 @@ def spack_spec(spec: str, log_filename: str = None): class InfoTest(unittest.TestCase): """Tests that the command 'spack info ' works for all spack-c2sm packages.""" - def test_cdo(self): - spack_info('cdo') - def test_clang_format(self): spack_info('clang-format') @@ -58,9 +55,6 @@ def test_eccodes(self): def test_fckit(self): spack_info('fckit') - def test_fdb(self): - spack_info('fdb') - def test_fdb_fortran(self): spack_info('fdb-fortran') @@ -205,9 +199,6 @@ def test_py_sphinxcontrib_jquery(self): def test_py_tabulate(self): spack_info('py-tabulate') - def test_py_toolz(self): - spack_info('py-toolz') - def test_py_typing_extensions(self): spack_info('py-typing-extensions') @@ -236,9 +227,6 @@ class SpecTest(unittest.TestCase): Ideally all conditional dependencies are triggered at least once. """ - def test_cdo(self): - spack_spec('cdo') - def test_clang_format(self): spack_spec('clang-format') @@ -246,7 +234,6 @@ def test_clang_format(self): def test_cosmo(self): spack_spec('cosmo') spack_spec('cosmo cosmo_target=gpu ~cppdycore') - spack_spec('cosmo cosmo_target=gpu +serialize +zlib_ng +oasis') def test_cosmo_dycore(self): spack_spec('cosmo-dycore') @@ -406,9 +393,6 @@ def test_py_sphinxcontrib_jquery(self): def test_py_tabulate(self): spack_spec('py-tabulate') - def test_py_toolz(self): - spack_spec('py-toolz') - def test_py_typing_extensions(self): spack_spec('py-typing-extensions') diff --git a/test/system_test.py b/test/system_test.py index 7c7f0f4714..4d29e947b8 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -206,36 +206,12 @@ def spack_env_dev_install_and_test(spack_env: str, }[machine_name()] -@pytest.mark.no_tsa # proj-8.2.1 fails with "./.libs/libproj.so: error: undefined reference to 'curl_easy_setopt'" -class CdoTest(unittest.TestCase): - - def test_install_default(self): - spack_install_and_test('cdo') - - class ClangFormatTest(unittest.TestCase): def test_install_default(self): spack_install_and_test('clang-format') -@pytest.mark.no_balfrin # cosmo-dycore does not support the cuda arch of balfrin -@pytest.mark.no_tsa # irrelevant -class CosmoTest(unittest.TestCase): - - def test_install_c2sm_master_cpu(self): - spack_env_dev_install_and_test( - 'cosmo/ACC/spack/v0.20.1.0/nvhpc_cpu_double', - 'git@github.com:C2SM-RCM/cosmo.git', '6.1_2023.11', - 'cosmo-c2sm-master') - - def test_install_c2sm_master_gpu(self): - spack_env_dev_install_and_test( - 'cosmo/ACC/spack/v0.20.1.0/nvhpc_gpu_double', - 'git@github.com:C2SM-RCM/cosmo.git', '6.1_2023.11', - 'cosmo-c2sm-master') - - @pytest.mark.no_balfrin # cuda arch is not supported @pytest.mark.no_tsa # irrelevant class CosmoDycoreTest(unittest.TestCase): @@ -274,17 +250,6 @@ def test_install_0_9_0(self): spack_install_and_test('fckit@0.9.0') -@pytest.mark.no_tsa # FDB tests fail on tsa due to 'ucp_context' -class FdbTest(unittest.TestCase): - - def test_install_5_11_17_gcc(self): - spack_install_and_test('fdb @5.11.17 %gcc') - - def test_install_5_11_17_nvhpc(self): - # tests fail because compiler emitted warnings. - spack_install(f'fdb @5.11.17 %{nvidia_compiler}') - - class FdbFortranTest(unittest.TestCase): def test_install(self): @@ -317,6 +282,17 @@ def test_install_version_1_1_3_nvhpc(self): spack_install_and_test(f'gridtools @1.1.3 %{nvidia_compiler}') +@pytest.mark.no_tsa # FDB tests fail on tsa due to 'ucp_context' +class FdbTest(unittest.TestCase): + + def test_install_5_11_17_gcc(self): + spack_install_and_test('fdb @5.11.17 %gcc') + + def test_install_5_11_17_nvhpc(self): + # tests fail because compiler emitted warnings. + spack_install(f'fdb @5.11.17 %{nvidia_compiler}') + + @pytest.mark.no_tsa # Icon does not run on Tsa class IconTest(unittest.TestCase): @@ -326,7 +302,8 @@ def test_install_2024_1_gcc(self): @pytest.mark.no_daint def test_install_2024_1_nvhpc(self): - spack_install_and_test('icon @2024.1-1 %nvhpc') + #WORKAROUND: ^libxml2%gcc works around a problem in the concretizer of spack v0.21.1 and /mch-environment/v6 + spack_install_and_test('icon @2024.1-1 %nvhpc ^libxml2%gcc') @pytest.mark.no_daint # libxml2 %nvhpc fails to build def test_install_conditional_dependencies(self): @@ -338,25 +315,28 @@ def test_install_conditional_dependencies(self): # +eccodes-definitions triggers cosmo-eccodes-definitions # +mpi triggers mpi # gpu=openacc+cuda triggers cuda + + #WORKAROUND: ^libxml2%gcc works around a problem in the concretizer of spack v0.21.1 and /mch-environment/v6 spack_install_and_test( - 'icon @2024.1-1 %nvhpc +coupling +rttov serialization=create +emvorado +mpi gpu=openacc+cuda' + 'icon @2024.1-1 %nvhpc +coupling +rttov serialization=create +emvorado +mpi gpu=openacc+cuda ^libxml2%gcc' ) + @pytest.mark.no_daint # no time for that @pytest.mark.no_balfrin # config file does not exist for this machine def test_install_c2sm_test_cpu_gcc(self): spack_env_dev_install_and_test( - 'config/cscs/spack/v0.20.1.4/daint_cpu_gcc', + 'config/cscs/spack/v0.21.1.0/daint_cpu_gcc', 'git@github.com:C2SM/icon.git', - '2024.01', + '2024.01.1', 'icon', build_on_login_node=True) @pytest.mark.no_balfrin # config file does not exist for this machine def test_install_c2sm_test_cpu_nvhpc_out_of_source(self): spack_env_dev_install_and_test( - 'config/cscs/spack/v0.20.1.4/daint_cpu_nvhpc', + 'config/cscs/spack/v0.21.1.0/daint_cpu_nvhpc', 'git@github.com:C2SM/icon.git', - '2024.01', + '2024.01.1', 'icon', out_of_source=True, build_on_login_node=True) @@ -364,43 +344,21 @@ def test_install_c2sm_test_cpu_nvhpc_out_of_source(self): @pytest.mark.no_balfrin # config file does not exist for this machine def test_install_c2sm_test_cpu(self): spack_env_dev_install_and_test( - 'config/cscs/spack/v0.20.1.4/daint_cpu_nvhpc', + 'config/cscs/spack/v0.21.1.0/daint_cpu_nvhpc', 'git@github.com:C2SM/icon.git', - '2024.01', + '2024.01.1', 'icon', build_on_login_node=True) @pytest.mark.no_balfrin # config file does not exist for this machine def test_install_c2sm_test_gpu(self): spack_env_dev_install_and_test( - 'config/cscs/spack/v0.20.1.4/daint_gpu_nvhpc', + 'config/cscs/spack/v0.21.1.0/daint_gpu_nvhpc', 'git@github.com:C2SM/icon.git', - '2024.01', + '2024.01.1', 'icon', build_on_login_node=True) - @pytest.mark.no_balfrin # config file does not exist for this machine - def test_install_nwp_test_cpu_cce(self): - spack_env_dev_install_and_test( - 'config/cscs/spack/v0.18.1.10/daint_cpu_cce', - 'git@github.com:C2SM/icon.git', - 'icon-2.6.6.2', - 'icon', - build_on_login_node=True) - - @pytest.mark.no_balfrin # config file does not exist for this machine - def test_install_exclaim_test_gpu_dsl(self): - spack_env_dev_install_and_test( - 'config/cscs/spack/v0.20.1.3/daint_dsl_nvhpc', - 'git@github.com:C2SM/icon-exclaim.git', - 'v0.2.0', - 'icon', - build_on_login_node=True) - - -class IconHamTest(unittest.TestCase): - pass - @pytest.mark.no_tsa # This test is flaky and sometimes fails with: icondelaunay.cpp:29:10: fatal error: version.c: No such file or directory. See issue #781. class IconToolsTest(unittest.TestCase): @@ -517,7 +475,7 @@ class PytorchFortranTest(unittest.TestCase): def test_install_version_0_4(self): spack_install( - 'pytorch-fortran@0.4%nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10' + 'pytorch-fortran@0.4%nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc' ) @@ -648,12 +606,6 @@ def test_install_default(self): spack_install_and_test('py-nanobind') -class PyNumpyTest(unittest.TestCase): - - def test_install_default(self): - spack_install('py-numpy') - - class PyPathspecTest(unittest.TestCase): def test_install_default(self): @@ -690,12 +642,6 @@ def test_install_default(self): spack_install_and_test('py-tabulate') -class PyToolzTest(unittest.TestCase): - - def test_install_default(self): - spack_install_and_test('py-toolz') - - class PyTypingExtensionsTest(unittest.TestCase): def test_install_default(self): diff --git a/test/unit_test.py b/test/unit_test.py index a9e2d11057..9d74d2cfcd 100644 --- a/test/unit_test.py +++ b/test/unit_test.py @@ -145,7 +145,7 @@ def test_upstream_from_config(self): upstream_base = read_upstream_from_spack_yaml( os.path.join(os.path.normpath(spack_c2sm_path), 'upstreams/daint/base')) - self.assertEqual('/project/g110/spack/upstream/daint_v0.20.1.0/base', + self.assertEqual('/project/g110/spack/upstream/daint_v0.21.1.0/base', upstream_base) def test_upstream_from_another_tag(self): diff --git a/upstreams/daint/base/spack.yaml b/upstreams/daint/base/spack.yaml index 4ba93b9cef..be2971eb02 100644 --- a/upstreams/daint/base/spack.yaml +++ b/upstreams/daint/base/spack.yaml @@ -3,17 +3,18 @@ spack: # ICON - matrix: - [eccodes@2.19.0, cosmo-eccodes-definitions@2.19.0.7, serialbox@2.6.1+fortran] - - ['%nvhpc@21.3', '%cce@12.0.3', '%gcc@9.3.0'] - - libxml2@2.9.13%gcc + - ['%nvhpc@21.3', '%gcc@9.3.0','%gcc@11.2.0'] + - libxml2@2.9.13%gcc@11.2.0 + - libxml2@2.9.13%gcc@9.3.0 - boost@1.82.0%gcc@9.3.0 +atomic +chrono +container +contract +coroutine +date_time +exception +fiber +filesystem +graph +graph_parallel +iostreams +json +locale +log +math +mpi +program_options +random +regex +serialization +signals +stacktrace cxxstd=11 + - boost@1.82.0%gcc@11.2.0 +atomic +chrono +container +contract +coroutine +date_time + +exception +fiber +filesystem +graph +graph_parallel +iostreams +json +locale + +log +math +mpi +program_options +random +regex +serialization +signals +stacktrace + cxxstd=11 - # COSMO - - jasper @1.900.1 %nvhpc - - eccodes@2.19.0 %nvhpc +fortran - # boost as needed to compile COSMO-Dycore - boost@1.79.0%gcc@8.3.0~atomic~chrono~clanglibcpp~container~context~contract~coroutine ~date_time~debug~exception~fiber~filesystem~graph~graph_parallel~icu~iostreams~json @@ -26,4 +27,4 @@ spack: view: false config: install_tree: - root: /project/g110/spack/upstream/daint_v0.20.1.0/base + root: /project/g110/spack/upstream/daint_v0.21.1.0/base diff --git a/upstreams/daint/icon-dsl/spack.yaml b/upstreams/daint/icon-dsl/spack.yaml index 28c163e98c..bf1ec2a5e3 100644 --- a/upstreams/daint/icon-dsl/spack.yaml +++ b/upstreams/daint/icon-dsl/spack.yaml @@ -3,7 +3,7 @@ spack: - cmake@3.26.3%gcc@9.3.0 - cmake@3.22.2%gcc@9.3.0 - python@3.10%gcc@9.3.0 - - py-numpy@1.24.3%gcc@9.3.0~blas~lapack + - py-numpy@1.24.3%gcc@9.3.0 - py-black@23.3.0%gcc@9.3.0 - py-ninja@1.10.2%gcc@9.3.0 - py-pytest@7.2.1%gcc@9.3.0 @@ -37,4 +37,4 @@ spack: view: false config: install_tree: - root: /project/g110/spack/upstream/daint_v0.20.1.3/icon-dsl + root: /project/g110/spack/upstream/daint_v0.21.1.0/icon-dsl diff --git a/upstreams/daint/icon-rttov/spack.yaml b/upstreams/daint/icon-rttov/spack.yaml index eff3139837..0f860f1335 100644 --- a/upstreams/daint/icon-rttov/spack.yaml +++ b/upstreams/daint/icon-rttov/spack.yaml @@ -12,4 +12,4 @@ spack: view: false config: install_tree: - root: /project/g110/spack/upstream/daint_v0.20.1.0/icon-rttov + root: /project/g110/spack/upstream/daint_v0.21.1.0/icon-rttov