Update spack to v0.21.1 (#909)

* inital update

* remove blas variant, remove serialbox

* remove old extensions

* Delete sysconfigs/tsa/modules.yaml

* do not use upstreams

* py-toolz available in upstream spack

* GitHub Action: Apply Pep8-formatting

* remove upstreams

* add version 0.12.3

* cython@2 for [email protected]

* GitHub Action: Apply Pep8-formatting

* inherit from upstream, introduce v 2.4.0

* GitHub Action: Apply Pep8-formatting

* lock pip to :23.0 for py-cmake

* lock eckit-version for infero

* temporary deactivate testsuite for int2lm

* Revert "temporary deactivate testsuite for int2lm"

This reverts commit 2834f46.

* remove gmake on Daint

* remove deprecated config

* add gmake again

* Remove faulty classification as "spack locking problem"

* [python] no default ssl package

* [fdb] Remove package and tests

* [fckit] Remove package and tests

* [numpy] Remove package and tests.

* Revert "[fckit] Remove package and tests"

This reverts commit fe41fc2.

* Fix FDB related tests (#920)

* test:fix(pytorch-fortran): try explicit dependency on (c/g)make%gcc

* upd: point to icon-c2sm branch spack_v0.21.1

* Update system_test.py

* GitHub Action: Apply Pep8-formatting

* upd: cosmo spack env

* skip failing tests

* GitHub Action: Apply Pep8-formatting

* add link to latest spack version

* downgrade icon4py,gt4py mostly to build-tests, add fdb again

* GitHub Action: Apply Pep8-formatting

* add upstreams for Daint

* add upstream to config

* adapt unit-test

* Use single-thread make for flexpart-cosmo

* Update system_test.py

* Fixes to gt4py in v0.21.1 (#937)

* locking typing extensions version

* Update system_test.py

* Update system_test.py

* Update system_test.py

* GitHub Action: Apply Pep8-formatting

---------

Co-authored-by: Abishek Gopal <[email protected]>
Co-authored-by: github-actions <[email protected]>

* Update system_test.py (#938)

* drop test for nvhpc-cpu ICON

* cleanup

* [py-gt4py] fixes to typing-extensions versions (#939)

* fixes to typing-extensions versions

* GitHub Action: Apply Pep8-formatting

* Adding typing-extensions v4.10.0

* fixing typo in versions

---------

Co-authored-by: github-actions <[email protected]>

* remove irrelevant test

* [cdo] Remove conflict

* add patched xpmem, libxml%gcc to nudge concretizer

* GitHub Action: Apply Pep8-formatting

* fix typo

* deactivate failing test

* Revert "add patched xpmem, libxml%gcc to nudge concretizer"

This reverts commit 1d7a27a.

* GitHub Action: Apply Pep8-formatting

* Add workarounds for concretizer (#961)

---------

Co-authored-by: Jonas Jucker <[email protected]>
Co-authored-by: Dominic Hofer <[email protected]>
Co-authored-by: github-actions <[email protected]>
Co-authored-by: Victoria Cherkas <[email protected]>
Co-authored-by: Matthieu <[email protected]>
Co-authored-by: Michael Jähn <[email protected]>
Co-authored-by: abishekg7 <[email protected]>
Co-authored-by: Abishek Gopal <[email protected]>
Co-authored-by: Dominic Hofer <[email protected]>

.gitmodules

@@ -1,3 +1,3 @@
 [submodule "spack"]
 	path = spack
-	url = https://github.com/dominichofer/spack.git
+	url = https://github.com/spack/spack.git

README.md

@@ -24,6 +24,7 @@ Spack is the package manager used by C2SM and MeteoSwiss to install and deploy s
   * [spack-c2sm v0.18.1.1](https://C2SM.github.io/spack-c2sm/v0.18.1.1) [deprecated]
 
 **General infos about spack**
+  * [Official spack v0.21.1](https://spack.readthedocs.io/en/v0.21.1/) 
   * [Official spack v0.20.1](https://spack.readthedocs.io/en/v0.20.1/) 
   * [Official spack v0.18.1](https://spack.readthedocs.io/en/v0.18.1/) 
 

repos/c2sm/packages/fdb/package.py

@@ -1,28 +1,21 @@
-# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other
-# Spack Project Developers. See the top-level COPYRIGHT file for details.
-#
-# SPDX-License-Identifier: (Apache-2.0 OR MIT)
-
 from spack.package import *
+
 from spack.pkg.builtin.fdb import Fdb as SpackFdb
 
 
 class Fdb(SpackFdb):
-    """FDB (Fields DataBase) is a domain-specific object store developed at
-    ECMWF for storing, indexing and retrieving GRIB data."""
 
-    version("5.11.17",
-            sha256=
-            "375c6893c7c60f6fdd666d2abaccb2558667bd450100817c0e1072708ad5591e")
-
-    depends_on("[email protected]:", type="build", when="@5.11.6:")
+    # This section can be removed when the following commit
+    # https://github.com/spack/spack/commit/8871bd5ba5c58562b8c20baa00f125aeccba586f
+    # is included in a release on spack and this is used by spack-c2sm.
+    depends_on("[email protected]:", when="@5.11.22:")
 
+    # This section can be removed when the following PR
+    # https://github.com/spack/spack/pull/42874
+    # is included in a release on spack and this is used by spack-c2sm.
     @property
     def libs(self):
         return find_libraries("libfdb5",
                               root=self.prefix,
                               shared=True,
                               recursive=True)
-
-    def setup_build_environment(self, env):
-        env.set('CTEST_OUTPUT_ON_FAILURE', 1)

repos/c2sm/packages/infero/package.py

@@ -29,8 +29,8 @@ class Infero(CMakePackage):
     variant('tf_c', description='Enable tensorflow-c backend', default=False)
     variant('onnx', description='Enable ONNX backend', default=False)
 
-    depends_on('[email protected].0:')
-    depends_on('fckit')
+    depends_on('[email protected].2')
+    depends_on('fckit@0.9.0')
     depends_on('ecbuild', type=('build'))
     depends_on('tensorflowc', when='+tf_c')
     depends_on('onnx-runtime', when='+onnx')

repos/c2sm/packages/metkit/package.py

@@ -0,0 +1,10 @@
+from spack.package import *
+
+from spack.pkg.builtin.metkit import Metkit as SpackMetkit
+
+
+class Metkit(SpackMetkit):
+
+    # This file can be removed when this PR https://github.com/spack/spack/pull/42871
+    # is included in a release on spack and this is used by spack-c2sm.
+    depends_on("eckit@:1.21", when="@:1.10")

repos/c2sm/packages/py-cmake/package.py

@@ -25,6 +25,8 @@ class PyCmake(PythonPackage):
             sha256=
             '52b98c5ee70b5fa30a8623e96482227e065292f78794eb085fdf0fecb204b79b')
 
+    # in newer pip versions --install-option does not exist
+    depends_on("py-pip@:23.0", type="build")
     depends_on('ninja', type='build')
     depends_on('[email protected]:', type='build')
     depends_on('py-setuptools@42:', type='build')

repos/c2sm/packages/py-cytoolz/package.py

@@ -17,13 +17,20 @@ class PyCytoolz(PythonPackage):
 
     maintainers = ['samkellerhals']
 
+    version('0.12.3',
+            sha256=
+            '4503dc59f4ced53a54643272c61dc305d1dbbfbd7d6bdf296948de9f34c3a282')
     version('0.12.0',
             sha256=
             'c105b05f85e03fbcd60244375968e62e44fe798c15a3531c922d531018d22412')
 
     depends_on('py-setuptools', type='build')
     depends_on('py-cython', type='build')
 
+    # [email protected] not compatible with py-cython@3:, see
+    # https://www.layerzrozero.network/?_=%2Fpytoolz%2Fcytoolz%2Fissues%2F202%23w2n%2BddnGqHIZTHHkluHJC3Vn
+    depends_on('py-cython@:2', when='@0.12.0', type='build')
+
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
 

repos/c2sm/packages/py-frozendict/package.py

@@ -4,22 +4,19 @@
 # SPDX-License-Identifier: (Apache-2.0 OR MIT)
 
 from spack import *
+from spack.pkg.builtin.py_frozendict import PyFrozendict as SpackPyFrozendict
 
 
-class PyFrozendict(PythonPackage):
-    """frozendict is a simple immutable dictionary."""
+class PyFrozendict(SpackPyFrozendict):
 
-    homepage = "https://github.com/Marco-Sulla/python-frozendict"
-
-    pypi = 'frozendict/frozendict-2.3.4.tar.gz'
-
-    maintainers = ['samkellerhals']
-
-    # FIXME: Add proper versions and checksums here.
-    version('2.3.4',
+    version('2.4.0',
             sha256=
-            '15b4b18346259392b0d27598f240e9390fafbff882137a9c48a1e0104fb17f78')
-
-    depends_on('[email protected]:', type=('build', 'run'))
-
-    depends_on('py-setuptools', type='build')
+            'c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a')
+
+    # TODO: remove this extension once we have a more recent
+    # version than v0.21.1
+    def setup_build_environment(self, env):
+        # C extension is not supported for 3.11+. See also
+        # https://github.com/Marco-Sulla/python-frozendict/issues/68
+        if self.spec.satisfies("^[email protected]:"):
+            env.set("FROZENDICT_PURE_PY", "1")

repos/c2sm/packages/py-gt4py/package.py

@@ -51,12 +51,17 @@ class PyGt4py(PythonPackage):
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
-    depends_on('[email protected]: ~blas ~lapack', type=('build', 'run'))
+    depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:', when="@1.0.1.3:", type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
-    depends_on('[email protected]:4.6', type=('build', 'run'))
+    depends_on('[email protected]',
+               when="@:1.0.3.5",
+               type=('build', 'run'))
+    depends_on('[email protected]',
+               when="@1.0.3.6:",
+               type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))
     depends_on('[email protected]:3.0.9', type=('build', 'run'))
     depends_on('[email protected]:', type=('build', 'run'))

repos/c2sm/packages/py-typing-extensions/package.py

@@ -5,6 +5,9 @@
 class PyTypingExtensions(SpackPyTypingExtensions):
     """Backported and Experimental Type Hints for Python 3.7+"""
 
+    version("4.10.0",
+            sha256=
+            "b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb")
     version("4.5.0",
             sha256=
             "5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb")

repos/c2sm/packages/python/package.py

@@ -0,0 +1,5 @@
+from spack.pkg.builtin.python import Python as SpackPython
+
+
+class Python(SpackPython):
+    variant("ssl", default=False, description="Build ssl module")

repos/c2sm/packages/xpmem/package.py

@@ -0,0 +1,120 @@
+# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other
+# Spack Project Developers. See the top-level COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (Apache-2.0 OR MIT)
+
+from spack.package import *
+
+
+# Identical to spack v0.21.1 except for the part labeled as WORKAROUND
+class Xpmem(AutotoolsPackage):
+    """XPMEM is a Linux kernel module that enables a process to map the memory
+    of another process into its virtual address space."""
+
+    # The README file of the repository says that the development was
+    # transferred to a new repository on GitLab: https://gitlab.com/hjelmn/xpmem
+    # However, it looks like that the repository on GitHub has a more recent
+    # version of the codebase.
+    homepage = "https://github.com/hjelmn/xpmem"
+    url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz"
+    git = "https://github.com/hjelmn/xpmem.git"
+
+    maintainers("skosukhin")
+
+    version("master", branch="master")
+
+    # Versions starting 2.6.4 are neither tagged nor released in the repo
+    # (the choice of commits is based on the commit history of
+    # 'kernel/xpmem_private.h'):
+    version("2.6.5-36", commit="0d0bad4e1d07b38d53ecc8f20786bb1328c446da")
+    version("2.6.5", commit="4efeed9cbaabe971f3766d67cb108e2c3316d4b8")
+    version("2.6.4", commit="522054850e4d1479d69f50f7190d1548bf9749fd")
+
+    # Released versions:
+    version("2.6.3",
+            sha256=
+            "ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54")
+    version("2.6.2",
+            sha256=
+            "2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b")
+
+    variant("kernel-module",
+            default=True,
+            description="Enable building the kernel module")
+
+    # Added RHEL 8.3 kernel support
+    # Here 2.6.5-36 referes to 2.6.5 version and 36th commit id
+    patch("xpmem_v2.6.5-36.patch", when="@2.6.5-36", level=1)
+    patch(
+        "https://github.com/hjelmn/xpmem/commit/cbd6e5bd3d2a1d3823c335ddcd3c57b94474f578.patch?full_index=1",
+        sha256=
+        "75299398b6c15546479bfbb8aa972431f58637fe2f0328196a26738bd7148140",
+        when="@2.6.5-36",
+        level=1,
+    )
+    patch(
+        "https://github.com/hjelmn/xpmem/commit/7d346aaf1fdfc24d38cebb4ad107b7f5c43769e9.patch?full_index=1",
+        sha256=
+        "6be8c5f33d55c611924d8412253740f6f4b738e6d98e32981fa300d2ccbe99cc",
+        when="@2.6.5-36",
+        level=1,
+    )
+
+    depends_on("autoconf", type="build")
+    depends_on("automake", type="build")
+    depends_on("libtool", type="build")
+    depends_on("m4", type="build")
+
+    # It will become possible to disable the kernel module only starting 2.6.6:
+    # https://github.com/hjelmn/xpmem/pull/24
+    conflicts("~kernel-module", when="@:2.6.5")
+
+    # Ideally, we should list all non-Linux-based platforms here:
+    conflicts("+kernel-module", when="platform=darwin")
+
+    # All compilers except for gcc are in conflict with +kernel-module:
+    #requires("%gcc", when="+kernel-module", msg="Linux kernel module must be compiled with gcc")
+
+    #WORKAROUND: The above line is not working as expected.
+    #            The below line tires to express the same thing.
+    #            But it is only true in a context with nvhpc and gcc as the only compilers.
+    conflicts("%nvhpc", when="+kernel-module")
+
+    def autoreconf(self, spec, prefix):
+        Executable("./autogen.sh")()
+
+    @run_before("build")
+    def override_kernel_compiler(self):
+        # Override the compiler for kernel module source files. We need
+        # this additional argument for all installation phases.
+        if "+kernel-module" in self.spec:
+            make.add_default_arg("CC={0}".format(spack_cc))
+
+    def configure_args(self):
+        args = []
+
+        if "~kernel-module" in self.spec:
+            # The kernel module is enabled by default. An attempt of explicit
+            # enabling with '--enable-kernel-module' disables the module.
+            args.append("--disable-kernel-module")
+
+        if self.spec.satisfies("@:2.6.5"):
+            fmt = self.spec.format
+            # The following arguments will not be needed starting 2.6.6:
+            # https://github.com/hjelmn/xpmem/pull/18
+            args.extend([
+                fmt("--with-default-prefix={prefix}"),
+                fmt("--with-module={prefix.share}/Modules/{name}/{version}"),
+            ])
+
+        return args
+
+    @when("@:2.6.5")
+    def install(self, spec, prefix):
+        with working_dir(self.build_directory):
+            # Override the hardcoded prefix for 'cray-xpmem.conf'
+            make(
+                "ldsoconfdir={0}".format(
+                    self.spec.prefix.etc.join("ld.so.conf.d")),
+                *self.install_targets,
+            )

repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch

@@ -0,0 +1,15 @@
+--- xpmem/kernel/xpmem_pfn.c	2021-02-08 22:49:05.321501753 -0800
++++ xpmem.patch/kernel/xpmem_pfn.c	2021-02-08 23:30:30.875309634 -0800
+@@ -251,6 +251,12 @@
+ 	    cpu_to_node(task_cpu(current)) != cpu_to_node(task_cpu(src_task))) {
+ #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0)
+ 		saved_mask = current->cpus_mask;
++#elif LINUX_VERSION_CODE == KERNEL_VERSION(4,18, 0)
++	#ifdef RHEL_RELEASE_CODE
++		#if  RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3)
++			saved_mask = current->cpus_mask;
++		#endif
++	#endif
+ #else
+ 		saved_mask = current->cpus_allowed;
+ #endif

src/report_tests.py

@@ -41,8 +41,6 @@ def __str__(self) -> str:
     # Trigger phrases that cause a test to get a special icon and comment.
     # List[(trigger, icon, comment)]
     triggers = [
-        ('AssertionError exception when releasing read lock', ':lock:',
-         'spack locking problem'),
         ('Timed out waiting for a write lock', ':lock:',
          'spack write lock problem'),
         ('Timed out waiting for a read lock', ':lock:',

sysconfigs/balfrin/config.yaml

@@ -1,6 +1,4 @@
 config:
-  extensions:
-    - '$spack/../tools/spack-scripting'
   build_stage: '$spack/../spack-build_stage'
   test_stage: '$spack/../spack-test_stage'
   misc_cache: '$spack/../spack-misc_cache'

sysconfigs/daint/config.yaml

@@ -1,6 +1,4 @@
 config:
-  extensions:
-    - '$spack/../tools/spack-scripting'
   build_stage: '$spack/../spack-build_stage'
   test_stage: '$spack/../spack-test_stage'
   misc_cache: '$spack/../spack-misc_cache'