From 58dd5585b7a70daad429a0259d431c4a30cdd6a2 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:05:48 +0100 Subject: [PATCH 001/104] Set submodule spack to v0.22.2 --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index e30fedab10..594a376c52 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit e30fedab102f9281a220fb4fae82e3f8c43a82ac +Subproject commit 594a376c521cc746978571b1181a47bbcff30a21 From e95aaae4e32e967429cb0d200dc4a3787f2c65d7 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:05:58 +0100 Subject: [PATCH 002/104] fdb: Remove package --- repos/c2sm/packages/fdb/package.py | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 repos/c2sm/packages/fdb/package.py diff --git a/repos/c2sm/packages/fdb/package.py b/repos/c2sm/packages/fdb/package.py deleted file mode 100644 index 459f692f5a..0000000000 --- a/repos/c2sm/packages/fdb/package.py +++ /dev/null @@ -1,21 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.fdb import Fdb as SpackFdb - - -class Fdb(SpackFdb): - - # This section can be removed when the following commit - # https://github.com/spack/spack/commit/8871bd5ba5c58562b8c20baa00f125aeccba586f - # is included in a release on spack and this is used by spack-c2sm. - depends_on("eckit@1.24.4:", when="@5.11.22:") - - # This section can be removed when the following PR - # https://github.com/spack/spack/pull/42874 - # is included in a release on spack and this is used by spack-c2sm. - @property - def libs(self): - return find_libraries("libfdb5", - root=self.prefix, - shared=True, - recursive=True) From 8592711d1a7f2f697616f221a04c4769dd79576a Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:17:19 +0100 Subject: [PATCH 003/104] metkit: Remove package --- repos/c2sm/packages/metkit/package.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 repos/c2sm/packages/metkit/package.py diff --git a/repos/c2sm/packages/metkit/package.py b/repos/c2sm/packages/metkit/package.py deleted file mode 100644 index 42733610db..0000000000 --- a/repos/c2sm/packages/metkit/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.metkit import Metkit as SpackMetkit - - -class Metkit(SpackMetkit): - - # This file can be removed when this PR https://github.com/spack/spack/pull/42871 - # is included in a release on spack and this is used by spack-c2sm. - depends_on("eckit@:1.21", when="@:1.10") From 4f1ce653dcb77274d36ddc2f92fdfc00f7724a5d Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:17:30 +0100 Subject: [PATCH 004/104] py-frozendict: Remove package --- repos/c2sm/packages/py-frozendict/package.py | 22 -------------------- 1 file changed, 22 deletions(-) delete mode 100644 repos/c2sm/packages/py-frozendict/package.py diff --git a/repos/c2sm/packages/py-frozendict/package.py b/repos/c2sm/packages/py-frozendict/package.py deleted file mode 100644 index 54f3a85f2b..0000000000 --- a/repos/c2sm/packages/py-frozendict/package.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -from spack.pkg.builtin.py_frozendict import PyFrozendict as SpackPyFrozendict - - -class PyFrozendict(SpackPyFrozendict): - - version('2.4.0', - sha256= - 'c26758198e403337933a92b01f417a8240c954f553e1d4b5e0f8e39d9c8e3f0a') - - # TODO: remove this extension once we have a more recent - # version than v0.21.1 - def setup_build_environment(self, env): - # C extension is not supported for 3.11+. See also - # https://github.com/Marco-Sulla/python-frozendict/issues/68 - if self.spec.satisfies("^python@3.11:"): - env.set("FROZENDICT_PURE_PY", "1") From 0116e61265578ea786042b01f192b2f34e512bd3 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:18:13 +0100 Subject: [PATCH 005/104] xpmem: Remove package --- repos/c2sm/packages/xpmem/package.py | 120 ------------------ .../c2sm/packages/xpmem/xpmem_v2.6.5-36.patch | 15 --- 2 files changed, 135 deletions(-) delete mode 100644 repos/c2sm/packages/xpmem/package.py delete mode 100644 repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch diff --git a/repos/c2sm/packages/xpmem/package.py b/repos/c2sm/packages/xpmem/package.py deleted file mode 100644 index d1c19da4f2..0000000000 --- a/repos/c2sm/packages/xpmem/package.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * - - -# Identical to spack v0.21.1 except for the part labeled as WORKAROUND -class Xpmem(AutotoolsPackage): - """XPMEM is a Linux kernel module that enables a process to map the memory - of another process into its virtual address space.""" - - # The README file of the repository says that the development was - # transferred to a new repository on GitLab: https://gitlab.com/hjelmn/xpmem - # However, it looks like that the repository on GitHub has a more recent - # version of the codebase. - homepage = "https://github.com/hjelmn/xpmem" - url = "https://github.com/hjelmn/xpmem/archive/v2.6.3.tar.gz" - git = "https://github.com/hjelmn/xpmem.git" - - maintainers("skosukhin") - - version("master", branch="master") - - # Versions starting 2.6.4 are neither tagged nor released in the repo - # (the choice of commits is based on the commit history of - # 'kernel/xpmem_private.h'): - version("2.6.5-36", commit="0d0bad4e1d07b38d53ecc8f20786bb1328c446da") - version("2.6.5", commit="4efeed9cbaabe971f3766d67cb108e2c3316d4b8") - version("2.6.4", commit="522054850e4d1479d69f50f7190d1548bf9749fd") - - # Released versions: - version("2.6.3", - sha256= - "ee239a32269f33234cdbdb94db29c12287862934c0784328d34aff82a9fa8b54") - version("2.6.2", - sha256= - "2c1a93b4cb20ed73c2093435a7afec513e0e797aa1e49d4d964cc6bdae89d65b") - - variant("kernel-module", - default=True, - description="Enable building the kernel module") - - # Added RHEL 8.3 kernel support - # Here 2.6.5-36 referes to 2.6.5 version and 36th commit id - patch("xpmem_v2.6.5-36.patch", when="@2.6.5-36", level=1) - patch( - "https://github.com/hjelmn/xpmem/commit/cbd6e5bd3d2a1d3823c335ddcd3c57b94474f578.patch?full_index=1", - sha256= - "75299398b6c15546479bfbb8aa972431f58637fe2f0328196a26738bd7148140", - when="@2.6.5-36", - level=1, - ) - patch( - "https://github.com/hjelmn/xpmem/commit/7d346aaf1fdfc24d38cebb4ad107b7f5c43769e9.patch?full_index=1", - sha256= - "6be8c5f33d55c611924d8412253740f6f4b738e6d98e32981fa300d2ccbe99cc", - when="@2.6.5-36", - level=1, - ) - - depends_on("autoconf", type="build") - depends_on("automake", type="build") - depends_on("libtool", type="build") - depends_on("m4", type="build") - - # It will become possible to disable the kernel module only starting 2.6.6: - # https://github.com/hjelmn/xpmem/pull/24 - conflicts("~kernel-module", when="@:2.6.5") - - # Ideally, we should list all non-Linux-based platforms here: - conflicts("+kernel-module", when="platform=darwin") - - # All compilers except for gcc are in conflict with +kernel-module: - #requires("%gcc", when="+kernel-module", msg="Linux kernel module must be compiled with gcc") - - #WORKAROUND: The above line is not working as expected. - # The below line tires to express the same thing. - # But it is only true in a context with nvhpc and gcc as the only compilers. - conflicts("%nvhpc", when="+kernel-module") - - def autoreconf(self, spec, prefix): - Executable("./autogen.sh")() - - @run_before("build") - def override_kernel_compiler(self): - # Override the compiler for kernel module source files. We need - # this additional argument for all installation phases. - if "+kernel-module" in self.spec: - make.add_default_arg("CC={0}".format(spack_cc)) - - def configure_args(self): - args = [] - - if "~kernel-module" in self.spec: - # The kernel module is enabled by default. An attempt of explicit - # enabling with '--enable-kernel-module' disables the module. - args.append("--disable-kernel-module") - - if self.spec.satisfies("@:2.6.5"): - fmt = self.spec.format - # The following arguments will not be needed starting 2.6.6: - # https://github.com/hjelmn/xpmem/pull/18 - args.extend([ - fmt("--with-default-prefix={prefix}"), - fmt("--with-module={prefix.share}/Modules/{name}/{version}"), - ]) - - return args - - @when("@:2.6.5") - def install(self, spec, prefix): - with working_dir(self.build_directory): - # Override the hardcoded prefix for 'cray-xpmem.conf' - make( - "ldsoconfdir={0}".format( - self.spec.prefix.etc.join("ld.so.conf.d")), - *self.install_targets, - ) diff --git a/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch b/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch deleted file mode 100644 index a17fba4ad1..0000000000 --- a/repos/c2sm/packages/xpmem/xpmem_v2.6.5-36.patch +++ /dev/null @@ -1,15 +0,0 @@ ---- xpmem/kernel/xpmem_pfn.c 2021-02-08 22:49:05.321501753 -0800 -+++ xpmem.patch/kernel/xpmem_pfn.c 2021-02-08 23:30:30.875309634 -0800 -@@ -251,6 +251,12 @@ - cpu_to_node(task_cpu(current)) != cpu_to_node(task_cpu(src_task))) { - #if LINUX_VERSION_CODE >= KERNEL_VERSION(5, 3, 0) - saved_mask = current->cpus_mask; -+#elif LINUX_VERSION_CODE == KERNEL_VERSION(4,18, 0) -+ #ifdef RHEL_RELEASE_CODE -+ #if RHEL_RELEASE_CODE >= RHEL_RELEASE_VERSION(8,3) -+ saved_mask = current->cpus_mask; -+ #endif -+ #endif - #else - saved_mask = current->cpus_allowed; - #endif From 40eb0a7348913dae3a9862d65f393f8fcca27d76 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:18:44 +0100 Subject: [PATCH 006/104] py-hatchling: Update from upstream --- repos/c2sm/packages/py-hatchling/package.py | 59 ++++++++++----------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/repos/c2sm/packages/py-hatchling/package.py b/repos/c2sm/packages/py-hatchling/package.py index eca6c6e181..3efb07e283 100644 --- a/repos/c2sm/packages/py-hatchling/package.py +++ b/repos/c2sm/packages/py-hatchling/package.py @@ -1,38 +1,33 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other +# Copyright 2013-2024 Lawrence Livermore National Security, LLC and other # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) -from spack import * - - class PyHatchling(PythonPackage): - """This is the extensible, standards compliant - build backend used by Hatch. - """ - - # FIXME: Add a proper url for your package's homepage here. + """Modern, extensible Python build backend.""" homepage = "https://hatch.pypa.io/latest/" - - pypi = "hatchling/hatchling-1.11.1.tar.gz" - - maintainers = ['samkellerhals'] - - version('1.11.1', - sha256= - '9f84361f70cf3a7ab9543b0c3ecc64211ed2ba8a606a71eb6a473c1c9b08e1d0') - - depends_on('py-setuptools', type='build') - - depends_on('python@3.7:', type=('build', 'run')) - - # Be less strict with py-pluggy, otherwise concretizer - # fails due to 0.12 and 1.0 required at the same time - #depends_on('py-pluggy@1.0.0:', type=('build', 'run')) - - depends_on('py-pluggy@0.12:1', type=('build', 'run')) - depends_on('py-pathspec@0.10.1:', type=('build', 'run')) - depends_on('py-tomli@1.2.2:', type=('build', 'run')) - depends_on('py-packaging@21.3:', type=('build', 'run')) - depends_on('py-editables@0.3:', type=('build', 'run')) - depends_on('py-importlib-metadata', type=('build', 'run')) + pypi = "hatchling/hatchling-1.4.1.tar.gz" + git = "https://github.com/pypa/hatch" + + license("MIT", checked_by="tgamblin") + + version("1.21.0", sha256="5c086772357a50723b825fd5da5278ac7e3697cdf7797d07541a6c90b6ff754c") + version("1.18.0", sha256="50e99c3110ce0afc3f7bdbadff1c71c17758e476731c27607940cfa6686489ca") + version("1.17.0", sha256="b1244db3f45b4ef5a00106a46612da107cdfaf85f1580b8e1c059fefc98b0930") + version("1.14.0", sha256="462ea91df03ff5d52813b5613fec1313a1a2059d2e37343e572b3f979867c5da") + version("1.13.0", sha256="f8d275a2cc720735286b7c2e2bc35da05761e6d3695c2fa416550395f10c53c7") + version("1.10.0", sha256="5d31f43dffaf6265c808e1b5353662ffa5146d844278b55caa6c7f74f427ec50") + version("1.8.1", sha256="448b04b23faed669b2b565b998ac955af4feea66c5deed3a1212ac9399d2e1cd") + version("1.4.1", sha256="13461b42876ade4f75ee5d2a2c656b288ca0aab7f048ef66657ef166996b2118") + + depends_on("py-editables@0.3:", type=("build", "run")) + depends_on("py-packaging@21.3:", type=("build", "run")) + depends_on("py-pathspec@0.10.1:", when="@1.9:", type=("build", "run")) + depends_on("py-pathspec@0.9:", type=("build", "run")) + + #WORKAROUND: py-pluggy@1: and py-tox' py-pluggy@0.12.0:0 are incompatible + #depends_on("py-pluggy@1:", type=("build", "run")) + depends_on("py-pluggy@0.12:", type=("build", "run")) + + depends_on("py-tomli@1.2.2:", when="^python@:3.10", type=("build", "run")) + depends_on("py-trove-classifiers", when="@1.14:", type=("build", "run")) From 698bd7f374c016fc4530386304e5f6da20f891b1 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:18:57 +0100 Subject: [PATCH 007/104] py-tabulate: Update from upstream --- repos/c2sm/packages/py-tabulate/package.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repos/c2sm/packages/py-tabulate/package.py b/repos/c2sm/packages/py-tabulate/package.py index f3b0593285..b0d3b07ef2 100644 --- a/repos/c2sm/packages/py-tabulate/package.py +++ b/repos/c2sm/packages/py-tabulate/package.py @@ -10,6 +10,6 @@ class PyTabulate(SpackPyTabulate): """Pretty-print tabular data""" - version("0.8.10", - sha256= - "6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519") + #BACKPORT: Add missing versions + version("0.9.0", sha256="0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c") + version("0.8.10", sha256="6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519") From 561c57131ec64ec34f731b14e023b6925caf7900 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:19:20 +0100 Subject: [PATCH 008/104] icon-ham: Use spack/icon --- repos/c2sm/packages/icon-ham/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon-ham/package.py b/repos/c2sm/packages/icon-ham/package.py index 038cc18a5b..af1dfc495e 100644 --- a/repos/c2sm/packages/icon-ham/package.py +++ b/repos/c2sm/packages/icon-ham/package.py @@ -1,8 +1,8 @@ from spack import * -from spack.pkg.c2sm.icon import Icon as C2SMIcon +from spack.pkg.builtin.icon import Icon as SpackIcon -class IconHam(C2SMIcon): +class IconHam(SpackIcon): @run_before('build') def generate_hammoz_nml(self): From 2c6231e3199fb8ba0b3524ba95e83a30bdb524d1 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:20:32 +0100 Subject: [PATCH 009/104] icon: Split package --- .../packages/{icon => icon-c2sm}/package.py | 2 +- repos/c2sm/packages/icon-mch/package.py | 58 +++++++++++++++++++ test/system_test.py | 12 ++-- 3 files changed, 65 insertions(+), 7 deletions(-) rename repos/c2sm/packages/{icon => icon-c2sm}/package.py (99%) create mode 100644 repos/c2sm/packages/icon-mch/package.py diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon-c2sm/package.py similarity index 99% rename from repos/c2sm/packages/icon/package.py rename to repos/c2sm/packages/icon-c2sm/package.py index 3bd28894e5..fa699d3824 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -37,7 +37,7 @@ def check_variant_extra_config_args(extra_config_arg): return False -class Icon(AutotoolsPackage, CudaPackage): +class IconC2SM(AutotoolsPackage, CudaPackage): """Icosahedral Nonhydrostatic Weather and Climate Model.""" homepage = "https://www.icon-model.org" diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py new file mode 100644 index 0000000000..bcf0bdae71 --- /dev/null +++ b/repos/c2sm/packages/icon-mch/package.py @@ -0,0 +1,58 @@ +from spack.pkg.builtin.icon import Icon as SpackIcon + + +class IconMch(SpackIcon): + git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' + + maintainers("dominichofer") + + version('master', submodules=True) + version('icon-2.6.6-mch2b', submodules=True) + version('icon-2.6.6-mch2a', submodules=True) + + # Model Features: + variant('dace', default=False, description='Enable the DACE modules for data assimilation') + requires("+mpi", when="+dace") + + variant('emvorado', default=False, description='Enable the radar forward operator EMVORADO') + requires("+mpi", when="+emvorado") + + # Infrastructural Features: + variant('async-io-rma', default=True, description='Enable remote memory access (RMA) for async I/O') + variant('realloc-buf', default=False, description='Enable reallocatable communication buffer') + + # Optimization Features: + variant('pgi-inlib', default=False, description='Enable PGI/NVIDIA cross-file function inlining via an inline library') + + # MCH specific features: + variant('eccodes-definitions', default=False, description='Enable extension of eccodes with center specific definition files') + + depends_on('cosmo-eccodes-definitions', type='run', when='+eccodes-definitions') + + with when('+emvorado'): + depends_on('eccodes +fortran') + depends_on('hdf5 +szip +hl +fortran') + depends_on('zlib') + + def configure_args(self): + args = super().configure_args() + super_libs = args.pop() + + libs = LibraryList([]) + + for x in [ + 'dace', + 'emvorado', + 'async-io-rma', + 'realloc-buf', + 'pgi-inlib', + ]: + args += self.enable_or_disable(x) + + if '+emvorado' in self.spec: + libs += self.spec['eccodes:fortran'].libs + libs += self.spec['hdf5:fortran,hl'].libs + libs += self.spec['zlib'].libs + + args.append(f"{super_libs} {libs.link_flags}") + return args diff --git a/test/system_test.py b/test/system_test.py index 4f9e6afee8..00fb8ca3fb 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -45,15 +45,15 @@ def test_install_fdb_5_11_17_nvhpc(): spack_install('fdb @5.11.17 %nvhpc', test_root=False) -def test_install_icon_2024_1_gcc(): - spack_install('icon @2024.1-1 %gcc') +def test_install_icon_mch_2024_01_gcc(): + spack_install('icon-mch @2024.01 %gcc') -def test_install_icon_2024_1_nvhpc(): - spack_install('icon @2024.1-1 %nvhpc') +def test_install_icon_mch_2024_01_nvhpc(): + spack_install('icon-mch @2024.01 %nvhpc') -def test_install_conditional_dependencies(): +def test_install_icon_mch_conditional_dependencies(): # +coupling triggers libfyaml, libxml2, netcdf-c # serialization=create triggers serialbox # +emvorado triggers eccodes, hdf5, zlib @@ -62,7 +62,7 @@ def test_install_conditional_dependencies(): # gpu=openacc+cuda triggers cuda spack_install( - 'icon @2024.1-1 %nvhpc +coupling serialization=create +emvorado +mpi gpu=openacc+cuda cuda_arch=80' + 'icon-mch @2024.01 %nvhpc +coupling serialization=create +emvorado +mpi gpu=openacc+cuda cuda_arch=80' ) From 6cf047ab035c6174b499194baee8130f2aaeff9a Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:24:21 +0100 Subject: [PATCH 010/104] py-lark: Remove package --- repos/c2sm/packages/py-lark/package.py | 48 -------------------------- 1 file changed, 48 deletions(-) delete mode 100644 repos/c2sm/packages/py-lark/package.py diff --git a/repos/c2sm/packages/py-lark/package.py b/repos/c2sm/packages/py-lark/package.py deleted file mode 100644 index 53cb7fe0ee..0000000000 --- a/repos/c2sm/packages/py-lark/package.py +++ /dev/null @@ -1,48 +0,0 @@ -# Copyright 2013-2022 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.package import * - - -# TODO: This package is taken from a more recent version of Spack -# Remove this package once Spack in spack-c2sm has been updated -class PyLark(PythonPackage): - """Lark is a modern general-purpose parsing library for Python.""" - - homepage = "https://github.com/lark-parser/lark/" - pypi = "lark/lark-1.0.0.tar.gz" - - maintainers = ['samkellerhals'] - - version("1.1.2", - sha256= - "7a8d0c07d663da9391d7faee1bf1d7df4998c47ca43a593cbef5c7566acd057a") - version("1.1.1", - sha256= - "5115193433051f079374c4f81059fa4bf2afa78cc87dd87817ed4435e8647c82") - version("1.1.0", - sha256= - "669eab99a9627b2b9e0c6fb97f23113c64d673c93d804bca40b05b2a765f13c0") - version("1.0.0", - sha256= - "2269dee215e6c689d5ce9d34fdc6e749d0c1c763add3fc7935938ebd7da159cb") - version("0.12.0", - sha256= - "7da76fcfddadabbbbfd949bbae221efd33938451d90b1fefbbc423c3cccf48ef") - version("0.11.3", - sha256= - "3100d9749b5a85735ec428b83100876a5da664804579e729c23a36341f961e7e") - version("0.11.1", - sha256= - "f2c6ed79ae128a89714bbaa4a6ecb61b6eec84d1b5d63b9195ad461762f96298") - version("0.11.0", - sha256= - "29868417eb190fe7d6b1ff6bcd9446903e0c73a1ca69cec58c92a01cae0abc24") - version("0.10.1", - sha256= - "98f2c6f8e41fe601fd103476eb759ac1ad4d3dc8094633133a16cef5a32b0f65") - - depends_on("python@3.6:", when="@1.0.0:") - depends_on("py-setuptools", type="build") From 118109d341b5906d24d1295ab1d06402b580bb8a Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 31 Oct 2024 11:27:34 +0100 Subject: [PATCH 011/104] Remove unnecessary tests --- test/system_test.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 00fb8ca3fb..2d77a8810a 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -36,15 +36,6 @@ def test_install_flexpart_cosmo(): spack_install('flexpart-cosmo @V8C4.0') -def test_install_fdb_5_11_17_gcc(): - spack_install('fdb @5.11.17 %gcc') - - -def test_install_fdb_5_11_17_nvhpc(): - # tests fail because compiler emitted warnings. - spack_install('fdb @5.11.17 %nvhpc', test_root=False) - - def test_install_icon_mch_2024_01_gcc(): spack_install('icon-mch @2024.01 %gcc') @@ -109,10 +100,6 @@ def test_install_py_factory_boy_install_default(): spack_install('py-factory-boy') -def test_install_py_frozendict_install_default(): - spack_install('py-frozendict') - - def test_install_py_gridtools_cpp_install_default(): spack_install('py-gridtools-cpp') From 695ac57d95a5bced06103be1658dae54e0bcfeb7 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:56:02 +0100 Subject: [PATCH 012/104] Migrate unit tests to pytest --- test/unit_test.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/test/unit_test.py b/test/unit_test.py index e024555fe9..b3c02628da 100644 --- a/test/unit_test.py +++ b/test/unit_test.py @@ -1,10 +1,6 @@ -import unittest from spack_commands import time_format - -class TimeFormatTest(unittest.TestCase): - - def test_example(self): - self.assertEqual(time_format(0.123), "0.12s") - self.assertEqual(time_format(123.456), "2m 3.46s") - self.assertEqual(time_format(123456.789), "34h 17m 36.79s") +def test_example(): + assert time_format(0.123) == "0.12s" + assert time_format(123.456) == "2m 3.46s" + assert time_format(123456.789) == "34h 17m 36.79s" From 491591d24639e4fba21c9099ae01de5d4cc142da Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:57:27 +0100 Subject: [PATCH 013/104] Rename icon to icon-c2sm --- repos/c2sm/packages/icon-c2sm/package.py | 2 +- test/integration_test.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon-c2sm/package.py index fa699d3824..fe4de03afa 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -37,7 +37,7 @@ def check_variant_extra_config_args(extra_config_arg): return False -class IconC2SM(AutotoolsPackage, CudaPackage): +class IconC2sm(AutotoolsPackage, CudaPackage): """Icosahedral Nonhydrostatic Weather and Climate Model.""" homepage = "https://www.icon-model.org" diff --git a/test/integration_test.py b/test/integration_test.py index 3dca6c709e..7c339b5edd 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -14,17 +14,17 @@ def test_spec(package: str): spack_spec(package) -def test_icon_serialization(): - spack_spec("icon serialization=create") +def test_icon_c2sm_serialization(): + spack_spec("icon-c2sm serialization=create") -def test_icon_fcgroup(): - spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") +def test_icon_c2sm_fcgroup(): + spack_spec("icon-c2sm fcgroup=DACE.externals/dace_icon.-O1") def test_icon_extra_config_args(): spack_spec( - "icon extra-config-args=--disable-new_feature,--enable-old_config_arg") + "icon-c2sm extra-config-args=--disable-new_feature,--enable-old_config_arg") def test_int2lm_parallel(): From 2a7a3371f06e362d4ffba9a666eb127623e5db86 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:58:37 +0100 Subject: [PATCH 014/104] cosmo: Remove package --- repos/c2sm/packages/cosmo/package.py | 329 ------------------ .../packages/cosmo/patches/5.09a.mch1.2.p2 | 1 - .../spec_as_yaml/patch.serialize_cosmo | 20 -- .../apn-mch/spec_as_yaml/patch.test_cosmo | 21 -- .../spec_as_yaml/patch.serialize_cosmo | 1 - repos/c2sm/packages/cosmo/patches/c2sm-master | 1 - .../spec_as_yaml/patch.serialize_cosmo | 1 - .../spec_as_yaml/patch.serialize_cosmo | 19 - .../org-master/spec_as_yaml/patch.test_cosmo | 22 -- test/system_test.py | 4 - 10 files changed, 419 deletions(-) delete mode 100644 repos/c2sm/packages/cosmo/package.py delete mode 120000 repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 delete mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo delete mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo delete mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-master delete mode 120000 repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo diff --git a/repos/c2sm/packages/cosmo/package.py b/repos/c2sm/packages/cosmo/package.py deleted file mode 100644 index 2abb680196..0000000000 --- a/repos/c2sm/packages/cosmo/package.py +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import subprocess, itertools, os -from spack import * - - -class Cosmo(MakefilePackage): - """COSMO: Numerical Weather Prediction Model. Needs access to private GitHub.""" - - homepage = "http://www.cosmo-model.org" - url = "https://github.com/COSMO-ORG/cosmo/archive/6.0.tar.gz" - git = 'git@github.com:COSMO-ORG/cosmo.git' - apn_git = 'git@github.com:MeteoSwiss-APN/cosmo.git' - c2sm_git = 'git@github.com:C2SM-RCM/cosmo.git' - empa_git = 'git@github.com:C2SM-RCM/cosmo-ghg.git' - maintainers = ['mjaehn', 'juckerj'] - - version('org-master', branch='master') - version('6.0', tag='6.0') - - version('apn-mch', git=apn_git, branch='mch') - version('5.09a.mch1.2.p2', git=apn_git, tag='5.09a.mch1.2.p2') - - version('6.1_2023.11', git=c2sm_git, tag='6.1_2023.11') - version('c2sm-master', git=c2sm_git, branch='master') - version('c2sm-features', git=c2sm_git, branch='c2sm-features') - version('empa-ghg', git=empa_git, branch='c2sm') - - # pass spec from spec to test_cosmo.py in yaml-format - # There are three different types of test_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', - when='@org-master') - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', when='@6.0') - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.test_cosmo', when='@apn-mch') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.test_cosmo', - when='@5.09a.mch1.2.p2') - - # pass spec from spec to serialize_cosmo.py in yaml-format - - # There are two different types of serialize_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@org-master +serialize') - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@6.0 +serialize') - patch('patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo', - when='@c2sm-features +serialize') - patch('patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo', - when='@empa-ghg +serialize') - - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.serialize_cosmo', - when='@apn-mch +serialize') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.serialize_cosmo', - when='@5.09a.mch1.2.p2 +serialize') - - # build dependency - depends_on('perl@5.16.3:', type='build') - depends_on('libgrib1', type='build') - - # build and link dependency - depends_on('mpi +fortran') - depends_on('netcdf-fortran') - depends_on('netcdf-c +mpi') - depends_on('jasper@1.900.1') - depends_on('eccodes +fortran') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('eccodes %nvhpc', when='%nvhpc') - depends_on('eccodes %gcc', when='%gcc') - - # run dependency - depends_on('slurm', type='run') - - depends_on('cosmo-eccodes-definitions', type=('build', 'run')) - depends_on('serialbox +fortran ^python@2:2.9', - when='+serialize', - type=('build', 'link', 'run')) - depends_on('oasis', when='+oasis', type=('build', 'link', 'run')) - - variant('serialize', - default=False, - description='Build with serialization enabled') - variant('parallel', default=True, description='Build parallel COSMO') - variant('debug', default=False, description='Build debug mode') - variant('real_type', - default='double', - description='Build with double or single precision enabled', - values=('double', 'float'), - multi=False) - variant('slave', default='none', description='Build on slave') - variant('pollen', default=False, description='Build with pollen enabled') - variant('verbose', - default=False, - description='Build cosmo with verbose enabled') - variant('set_version', - default=False, - description='Pass cosmo tag version to Makefile') - variant('gt1', - default=False, - description='Build dycore with gridtools 1.1.3') - variant('cuda_arch', - default='none', - description='Build with cuda_arch', - values=('80', '70', '60', '37'), - multi=False) - variant('oasis', - default=False, - description='Build with the unified oasis interface') - - conflicts( - '@dev-build', - msg= - "Please use only official versions listed with 'spack info cosmo'. Even when using 'devbuildcosmo'. C2SM introduced 'dev-build' to avoid name conflicts with the upstream instance. Since spack-c2sm v0.18.1.0 this is not relevant anymore." - ) - conflicts('+pollen', when='@org-master,master') - # - ML - A conflict should be added there if the oasis variant is - # chosen and the version is neither c2sm-features nor - # dev-build. The problem is that this doesn't seem possible in a - # native spack way. Hence the dirty check at the beginning of - # setup_build_environment method - - build_directory = 'cosmo/ACC' - - # v0.20.1 does not fully support builds of COSMO - # users should continue using v0.18.1.x on Daint - # once support on Alps has been clarified finalize - # recipe and remove line below. - manual_download = True - - def setup_build_environment(self, env): - - # - ML - Dirty conflict check (see above) - if self.spec.variants['oasis'].value and self.spec.version not in ( - Version('c2sm-features')): - raise InstallError( - '+oasis variant only compatible with the @c2sm-features versions' - ) - - self.setup_run_environment(env) - - # Check mpi provider - self.mpi_spec = self.spec['mpi'] - - grib_prefix = self.spec['eccodes'].prefix - env.set( - 'GRIBAPIL', - str(self.spec['eccodes:fortran'].libs.ld_flags) + ' ' + - str(self.spec['jasper'].libs.ld_flags)) - grib_inc_dir_path = os.path.join(grib_prefix, 'include') - if os.path.exists(grib_inc_dir_path): - env.set('GRIBAPII', '-I' + grib_inc_dir_path) - else: - env.set('GRIBAPII', '') - - # libaec - if 'libaec' in self.spec: - env.set('AECL', '-L' + self.spec['libaec'].prefix + '/lib64 -laec') - env.set('AECI', '-I' + self.spec['libaec'].prefix + '/include') - - # Netcdf library - env.set( - 'NETCDFL', - '-L' + self.spec['netcdf-fortran'].prefix + '/lib -lnetcdff -L' + - self.spec['netcdf-c'].prefix + '/lib -lnetcdf') - env.set('NETCDFI', - '-I' + self.spec['netcdf-fortran'].prefix + '/include') - - # Grib1 library - if self.compiler.name == 'gcc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_gnu') - elif self.compiler.name == 'cce': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_cray') - elif self.compiler.name == 'nvhpc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_pgi') - else: - env.set( - 'GRIBDWDL', '-L' + self.spec['libgrib1'].prefix + - '/lib -lgrib1_' + self.compiler.name) - env.set('GRIBDWDI', '-I' + self.spec['libgrib1'].prefix + '/include') - - # MPI library - if self.mpi_spec.name == 'openmpi': - env.set('MPIL', '-L' + self.mpi_spec.prefix + ' -lmpi') - - else: - env.set('MPIL', '-L' + self.spec['mpi'].prefix + ' -lmpich') - - env.set('MPII', '-I' + self.mpi_spec.prefix + '/include') - - # Serialbox library - if '+serialize' in self.spec: - env.set('SERIALBOX', self.spec['serialbox'].prefix) - env.set('SERIALBOXL', - self.spec['serialbox:fortran,c'].libs.ld_flags) - env.set('SERIALBOXI', - '-I' + self.spec['serialbox'].prefix + '/include') - - # OASIS library - if '+oasis' in self.spec: - oasis_prefix = self.spec['oasis'].prefix - env.set( - 'PSMILEL', - '-L{:s}/lib -lpsmile.MPI1 -lscrip -lmct -lmpeu'.format( - oasis_prefix)) - env.set( - 'PSMILEI', - '-I{0:s}/build/lib/psmile.MPI1 -I{0:s}/build/lib/mct'.format( - oasis_prefix)) - env.set('MPPIOI', '-I{:s}/build/lib/mct'.format(oasis_prefix)) - - # Linker flags - if self.compiler.name in ('pgi', - 'nvhpc') and '~cppdycore' in self.spec: - env.set('LFLAGS', '-lstdc++') - - # Compiler & linker variables - if self.compiler.name in ('pgi', 'nvhpc'): - env.set('F90', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - env.set('LD', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - else: - env.set('F90', self.mpi_spec.mpifc) - env.set('LD', self.mpi_spec.mpifc) - - @property - def build_targets(self): - build = [] - if self.spec.variants['pollen'].value: - build.append('POLLEN=1') - if self.spec.variants['oasis'].value: - build.append('COUP_OAS=1') - if self.spec.variants['real_type'].value == 'float': - build.append('SINGLEPRECISION=1') - if '+serialize' in self.spec: - build.append('SERIALIZE=1') - if self.spec.variants['verbose'].value: - build.append('VERBOSE=1') - if '+set_version' in self.spec: - build.append('COSMO_VERSION=' + self.spec.format('{version}')) - MakeFileTarget = '' - if '+parallel' in self.spec: - MakeFileTarget += 'par' - else: - MakeFileTarget += 'seq' - if '+debug' in self.spec: - MakeFileTarget += 'debug' - else: - MakeFileTarget += 'opt' - build.append(MakeFileTarget) - - return build - - def edit(self, spec, prefix): - with working_dir(self.build_directory): - OptionsFileName = 'Options' - if self.compiler.name == 'gcc': - OptionsFileName += '.gnu' - elif self.compiler.name in ('pgi', 'nvhpc'): - OptionsFileName += '.pgi' - elif self.compiler.name == 'cce': - OptionsFileName += '.cray' - OptionsFileName += '.cpu' - OptionsFile = FileFilter(OptionsFileName) - - makefile = FileFilter('Makefile') - makefile.filter('/Options.*', '/' + OptionsFileName) - if self.spec.version == Version('empa-ghg'): - if '~serialize' in spec: - makefile.filter( - 'TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg')) - else: - if '~serialize' in spec: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo')) - - # Pre-processor flags - if self.mpi_spec.name == 'mpich': - OptionsFile.filter( - 'PFLAGS = -Mpreprocess.*', - 'PFLAGS = -Mpreprocess -DNO_MPI_HOST_DATA') - - def install(self, spec, prefix): - - with working_dir(self.build_directory): - mkdir(prefix.bin) - if self.spec.version == Version('empa-ghg'): - if '+serialize' in spec: - install('cosmo-ghg_serialize', prefix.bin) - else: - install('cosmo-ghg_cpu', prefix.bin) - install('cosmo-ghg_cpu', 'test/testsuite') - else: - if '+serialize' in spec: - install('cosmo_serialize', prefix.bin) - else: - install('cosmo_cpu', prefix.bin) - install('cosmo_cpu', 'test/testsuite') - - @run_after('install') - @on_package_attributes(run_tests=True) - def test(self): - with open('spec.yaml', mode='w') as f: - f.write(self.spec.to_yaml()) - try: - subprocess.run([ - self.build_directory + '/test/tools/test_cosmo.py', '-s', - 'spec.yaml', '-b', - str('.') - ], - stderr=subprocess.STDOUT, - check=True) - except: - raise InstallError('Testsuite failed') diff --git a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 b/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 deleted file mode 120000 index 34c563540d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 +++ /dev/null @@ -1 +0,0 @@ -apn-mch/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 39aa5e1fc2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..fc15c712 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,14 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 40bc6a90d1..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..2a166bbd 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,15 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-master b/repos/c2sm/packages/cosmo/patches/c2sm-master deleted file mode 120000 index 71b99193f9..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-master +++ /dev/null @@ -1 +0,0 @@ -org-master/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 8c72160de2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..e93b96f6 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,13 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 8e91d15f45..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..c2c60f2b 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,16 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - diff --git a/test/system_test.py b/test/system_test.py index 2d77a8810a..d37341b4ba 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -15,10 +15,6 @@ def test_install_cosmo_eccodes_definitions_version(version): spack_install(f'cosmo-eccodes-definitions @{version}', test_root=False) -def test_install_cosmo_6_0(): - spack_install('cosmo @6.0 %nvhpc', test_root=False) - - def test_install_eccodes_2_19_0(): spack_install('eccodes @2.19.0', test_root=False) From a1e05dddb821bd0ac64a810f0d550334a92081b1 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 15:58:51 +0100 Subject: [PATCH 015/104] Jenkins: Use mch/prgenv-icon:rc1 --- Jenkinsfile | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 4bda2c7252..31b75f56b5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -20,7 +20,7 @@ pipeline { } } stages { - stage('Create environment') { + stage('Create python environment') { steps { sh """ python3 -m venv env @@ -29,6 +29,17 @@ pipeline { """ } } + stage('Create uenv') { + steps { + sh """ + git clone -b fix/jenkins https://github.com/eth-cscs/uenv.git + ./uenv/install --yes --destdir=$WORKSPACE + source $WORKSPACE/etc/profile.d/uenv.sh + uenv repo create + uenv image pull mch/prgenv-icon:rc1 + """ + } + } stage('Bootstrap spack') { // Bootstrapping spack is a separate stage to avoid problems with concurrently bootstrapping spack in the tests. steps { @@ -51,7 +62,8 @@ pipeline { steps { sh """ source env/bin/activate - source ./setup-env.sh $USER_ENV_ROOT + source $WORKSPACE/etc/profile.d/uenv.sh + source ./setup-env.sh /user-environment pytest -v -n auto test/integration_test.py """ } @@ -60,7 +72,8 @@ pipeline { steps { sh """ source env/bin/activate - source ./setup-env.sh $USER_ENV_ROOT + source $WORKSPACE/etc/profile.d/uenv.sh + source ./setup-env.sh /user-environment pytest -v -n auto test/system_test.py """ } From 9b5c5be0f41e97b6339f1c1cd462daebc0c74f2c Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 5 Nov 2024 14:59:49 +0000 Subject: [PATCH 016/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon-mch/package.py | 34 ++++++++++++++++----- repos/c2sm/packages/py-hatchling/package.py | 33 +++++++++++++++----- repos/c2sm/packages/py-tabulate/package.py | 8 +++-- test/integration_test.py | 3 +- test/unit_test.py | 1 + 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index bcf0bdae71..2a33a45670 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -11,23 +11,41 @@ class IconMch(SpackIcon): version('icon-2.6.6-mch2a', submodules=True) # Model Features: - variant('dace', default=False, description='Enable the DACE modules for data assimilation') + variant('dace', + default=False, + description='Enable the DACE modules for data assimilation') requires("+mpi", when="+dace") - variant('emvorado', default=False, description='Enable the radar forward operator EMVORADO') + variant('emvorado', + default=False, + description='Enable the radar forward operator EMVORADO') requires("+mpi", when="+emvorado") # Infrastructural Features: - variant('async-io-rma', default=True, description='Enable remote memory access (RMA) for async I/O') - variant('realloc-buf', default=False, description='Enable reallocatable communication buffer') + variant('async-io-rma', + default=True, + description='Enable remote memory access (RMA) for async I/O') + variant('realloc-buf', + default=False, + description='Enable reallocatable communication buffer') # Optimization Features: - variant('pgi-inlib', default=False, description='Enable PGI/NVIDIA cross-file function inlining via an inline library') + variant( + 'pgi-inlib', + default=False, + description= + 'Enable PGI/NVIDIA cross-file function inlining via an inline library') # MCH specific features: - variant('eccodes-definitions', default=False, description='Enable extension of eccodes with center specific definition files') - - depends_on('cosmo-eccodes-definitions', type='run', when='+eccodes-definitions') + variant( + 'eccodes-definitions', + default=False, + description= + 'Enable extension of eccodes with center specific definition files') + + depends_on('cosmo-eccodes-definitions', + type='run', + when='+eccodes-definitions') with when('+emvorado'): depends_on('eccodes +fortran') diff --git a/repos/c2sm/packages/py-hatchling/package.py b/repos/c2sm/packages/py-hatchling/package.py index 3efb07e283..2afe34325d 100644 --- a/repos/c2sm/packages/py-hatchling/package.py +++ b/repos/c2sm/packages/py-hatchling/package.py @@ -3,6 +3,7 @@ # # SPDX-License-Identifier: (Apache-2.0 OR MIT) + class PyHatchling(PythonPackage): """Modern, extensible Python build backend.""" homepage = "https://hatch.pypa.io/latest/" @@ -11,14 +12,30 @@ class PyHatchling(PythonPackage): license("MIT", checked_by="tgamblin") - version("1.21.0", sha256="5c086772357a50723b825fd5da5278ac7e3697cdf7797d07541a6c90b6ff754c") - version("1.18.0", sha256="50e99c3110ce0afc3f7bdbadff1c71c17758e476731c27607940cfa6686489ca") - version("1.17.0", sha256="b1244db3f45b4ef5a00106a46612da107cdfaf85f1580b8e1c059fefc98b0930") - version("1.14.0", sha256="462ea91df03ff5d52813b5613fec1313a1a2059d2e37343e572b3f979867c5da") - version("1.13.0", sha256="f8d275a2cc720735286b7c2e2bc35da05761e6d3695c2fa416550395f10c53c7") - version("1.10.0", sha256="5d31f43dffaf6265c808e1b5353662ffa5146d844278b55caa6c7f74f427ec50") - version("1.8.1", sha256="448b04b23faed669b2b565b998ac955af4feea66c5deed3a1212ac9399d2e1cd") - version("1.4.1", sha256="13461b42876ade4f75ee5d2a2c656b288ca0aab7f048ef66657ef166996b2118") + version("1.21.0", + sha256= + "5c086772357a50723b825fd5da5278ac7e3697cdf7797d07541a6c90b6ff754c") + version("1.18.0", + sha256= + "50e99c3110ce0afc3f7bdbadff1c71c17758e476731c27607940cfa6686489ca") + version("1.17.0", + sha256= + "b1244db3f45b4ef5a00106a46612da107cdfaf85f1580b8e1c059fefc98b0930") + version("1.14.0", + sha256= + "462ea91df03ff5d52813b5613fec1313a1a2059d2e37343e572b3f979867c5da") + version("1.13.0", + sha256= + "f8d275a2cc720735286b7c2e2bc35da05761e6d3695c2fa416550395f10c53c7") + version("1.10.0", + sha256= + "5d31f43dffaf6265c808e1b5353662ffa5146d844278b55caa6c7f74f427ec50") + version("1.8.1", + sha256= + "448b04b23faed669b2b565b998ac955af4feea66c5deed3a1212ac9399d2e1cd") + version("1.4.1", + sha256= + "13461b42876ade4f75ee5d2a2c656b288ca0aab7f048ef66657ef166996b2118") depends_on("py-editables@0.3:", type=("build", "run")) depends_on("py-packaging@21.3:", type=("build", "run")) diff --git a/repos/c2sm/packages/py-tabulate/package.py b/repos/c2sm/packages/py-tabulate/package.py index b0d3b07ef2..b6a8f08b4b 100644 --- a/repos/c2sm/packages/py-tabulate/package.py +++ b/repos/c2sm/packages/py-tabulate/package.py @@ -11,5 +11,9 @@ class PyTabulate(SpackPyTabulate): """Pretty-print tabular data""" #BACKPORT: Add missing versions - version("0.9.0", sha256="0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c") - version("0.8.10", sha256="6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519") + version("0.9.0", + sha256= + "0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c") + version("0.8.10", + sha256= + "6c57f3f3dd7ac2782770155f3adb2db0b1a269637e42f27599925e64b114f519") diff --git a/test/integration_test.py b/test/integration_test.py index 7c339b5edd..2bd1083fc5 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -24,7 +24,8 @@ def test_icon_c2sm_fcgroup(): def test_icon_extra_config_args(): spack_spec( - "icon-c2sm extra-config-args=--disable-new_feature,--enable-old_config_arg") + "icon-c2sm extra-config-args=--disable-new_feature,--enable-old_config_arg" + ) def test_int2lm_parallel(): diff --git a/test/unit_test.py b/test/unit_test.py index b3c02628da..49eef857b7 100644 --- a/test/unit_test.py +++ b/test/unit_test.py @@ -1,5 +1,6 @@ from spack_commands import time_format + def test_example(): assert time_format(0.123) == "0.12s" assert time_format(123.456) == "2m 3.46s" From 6ff2517a38a22eede538f1188321a4529feac4ef Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:24:22 +0100 Subject: [PATCH 017/104] Add tests for 'spack find' and 'spack compilers'. --- test/integration_test.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/test/integration_test.py b/test/integration_test.py index 2bd1083fc5..abf1f4e3c6 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -1,5 +1,11 @@ import pytest -from spack_commands import ALL_PACKAGES, spack_info, spack_spec +from spack_commands import ( + ALL_PACKAGES, + spack_info, + spack_spec, + run_with_spack, + log_file, +) @pytest.mark.parametrize("package", ALL_PACKAGES) @@ -34,3 +40,11 @@ def test_int2lm_parallel(): def test_int2lm_no_parallel(): spack_spec("int2lm ~parallel") + + +def test_compilers(): + run_with_spack("spack compilers", log=log_file("compilers")) + + +def test_find(): + run_with_spack("spack find", log=log_file("find")) From de903c8f26f503e3c52e28c4dccc63ff4bd31109 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 16:41:25 +0100 Subject: [PATCH 018/104] Start uenv in test stages --- Jenkinsfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Jenkinsfile b/Jenkinsfile index 31b75f56b5..05591107c3 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -63,6 +63,7 @@ pipeline { sh """ source env/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh + uenv start mch/prgenv-icon:rc1 source ./setup-env.sh /user-environment pytest -v -n auto test/integration_test.py """ @@ -73,6 +74,7 @@ pipeline { sh """ source env/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh + uenv start mch/prgenv-icon:rc1 source ./setup-env.sh /user-environment pytest -v -n auto test/system_test.py """ From 1059f74aefde45bbbf8f7750a1b1b2d276ff4f89 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:06:04 +0100 Subject: [PATCH 019/104] deactivate --- test/spack_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/spack_commands.py b/test/spack_commands.py index 8259ebacc7..6f748a18fe 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -54,12 +54,10 @@ def run_with_spack(command: str, log: Path) -> None: start = time.time() # Direct stream to avoid buffering. - # 'env -i' clears the environment. - # 'bash -l' makes it a login shell. - # 'bash -c' reads commands from string. + # 'deactivate' deactivates the python virtual environment. # '2>&1' redirects stderr to stdout. ret = subprocess.run( - f'env -i bash -l -c "(. {REPO_DIR}/setup-env.sh {uenv}; {command}) >> {log} 2>&1"', + f'deactivate; . {REPO_DIR}/setup-env.sh {uenv}; {command}) >> {log} 2>&1', check=False, shell=True, ) From 4a3cf6d1d661e7891caa77be450d6f8de8d12514 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 5 Nov 2024 17:09:08 +0100 Subject: [PATCH 020/104] Simplify run_with_spack --- setup-env.sh | 1 - test/spack_commands.py | 8 +------- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/setup-env.sh b/setup-env.sh index b939ea95c0..4f77023fff 100644 --- a/setup-env.sh +++ b/setup-env.sh @@ -4,7 +4,6 @@ parent_dir=$( cd "$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")" ; pwd -P ) if [[ "$#" == 1 ]]; then uenv="$1" - export SPACK_UENV_PATH="$uenv" export SPACK_SYSTEM_CONFIG_PATH="$uenv"/config if [[ $uenv == "euler" ]]; then diff --git a/test/spack_commands.py b/test/spack_commands.py index 6f748a18fe..cf4da8ffc7 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -46,18 +46,12 @@ def run_with_spack(command: str, log: Path) -> None: with log.open("a") as f: f.write(f"{command}\n\n") - # setup-env.sh may define SPACK_UENV_PATH. - if "SPACK_UENV_PATH" in os.environ: - uenv = os.environ["SPACK_UENV_PATH"] - else: - uenv = "" - start = time.time() # Direct stream to avoid buffering. # 'deactivate' deactivates the python virtual environment. # '2>&1' redirects stderr to stdout. ret = subprocess.run( - f'deactivate; . {REPO_DIR}/setup-env.sh {uenv}; {command}) >> {log} 2>&1', + f'deactivate; {command} >> {log} 2>&1', check=False, shell=True, ) From 8b5222c4cf2b5834d29f7958d07ad2c338b53fc9 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 6 Nov 2024 14:15:06 +0100 Subject: [PATCH 021/104] Keep venv --- test/spack_commands.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/spack_commands.py b/test/spack_commands.py index cf4da8ffc7..52ae675ba3 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -51,7 +51,7 @@ def run_with_spack(command: str, log: Path) -> None: # 'deactivate' deactivates the python virtual environment. # '2>&1' redirects stderr to stdout. ret = subprocess.run( - f'deactivate; {command} >> {log} 2>&1', + f'{command} >> {log} 2>&1', check=False, shell=True, ) From 77df0b95901416fdcf2a815b18c5c40ae2882315 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:10:08 +0100 Subject: [PATCH 022/104] Bring back devirtualization --- test/spack_commands.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/test/spack_commands.py b/test/spack_commands.py index 52ae675ba3..b1ea7a5560 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -47,13 +47,23 @@ def run_with_spack(command: str, log: Path) -> None: f.write(f"{command}\n\n") start = time.time() + # Remove python virtual environment paths for this subprocess call + env = os.environ.copy() + if 'VIRTUAL_ENV' in env: + del env['VIRTUAL_ENV'] + if 'PATH' in env: + # Filter out any paths related to the virtual environment + env['PATH'] = ':'.join( + p for p in env['PATH'].split(':') + if 'venv' not in p + ) # Direct stream to avoid buffering. - # 'deactivate' deactivates the python virtual environment. # '2>&1' redirects stderr to stdout. ret = subprocess.run( f'{command} >> {log} 2>&1', check=False, shell=True, + env=env, ) end = time.time() From e073f3c325b452aca30fb8610a66991b005d3f06 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 6 Nov 2024 16:11:17 +0100 Subject: [PATCH 023/104] Update uenv --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 05591107c3..9e7c327e7a 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ pipeline { ./uenv/install --yes --destdir=$WORKSPACE source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create - uenv image pull mch/prgenv-icon:rc1 + uenv image pull mch/v8:rc1 """ } } @@ -63,7 +63,7 @@ pipeline { sh """ source env/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh - uenv start mch/prgenv-icon:rc1 + uenv start mch/v8:rc1 source ./setup-env.sh /user-environment pytest -v -n auto test/integration_test.py """ @@ -74,7 +74,7 @@ pipeline { sh """ source env/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh - uenv start mch/prgenv-icon:rc1 + uenv start mch/v8:rc1 source ./setup-env.sh /user-environment pytest -v -n auto test/system_test.py """ From c827041e07189129e975a12f869a4cc52827bf62 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 6 Nov 2024 17:28:14 +0100 Subject: [PATCH 024/104] Use .venv --- Jenkinsfile | 12 ++++++------ test/spack_commands.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 9e7c327e7a..f93f684fdc 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -23,8 +23,8 @@ pipeline { stage('Create python environment') { steps { sh """ - python3 -m venv env - source env/bin/activate + python3 -m venv .venv + source .venv/bin/activate pip install -r requirements.txt """ } @@ -44,7 +44,7 @@ pipeline { // Bootstrapping spack is a separate stage to avoid problems with concurrently bootstrapping spack in the tests. steps { sh """ - source env/bin/activate + source .venv/bin/activate source ./setup-env.sh spack spec gnuconfig """ @@ -53,7 +53,7 @@ pipeline { stage('Unit Tests') { steps { sh """ - source env/bin/activate + source .venv/bin/activate python3 test/unit_test.py """ } @@ -61,7 +61,7 @@ pipeline { stage('Integration Tests') { steps { sh """ - source env/bin/activate + source .venv/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh uenv start mch/v8:rc1 source ./setup-env.sh /user-environment @@ -72,7 +72,7 @@ pipeline { stage('System Tests') { steps { sh """ - source env/bin/activate + source .venv/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh uenv start mch/v8:rc1 source ./setup-env.sh /user-environment diff --git a/test/spack_commands.py b/test/spack_commands.py index b1ea7a5560..96952169da 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -55,7 +55,7 @@ def run_with_spack(command: str, log: Path) -> None: # Filter out any paths related to the virtual environment env['PATH'] = ':'.join( p for p in env['PATH'].split(':') - if 'venv' not in p + if '.venv' not in p ) # Direct stream to avoid buffering. # '2>&1' redirects stderr to stdout. From bce9413f065d796e82d886deae56b2b12e5c8727 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 6 Nov 2024 16:33:52 +0000 Subject: [PATCH 025/104] GitHub Action: Apply Pep8-formatting --- test/spack_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test/spack_commands.py b/test/spack_commands.py index 96952169da..d767ce4018 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -53,10 +53,8 @@ def run_with_spack(command: str, log: Path) -> None: del env['VIRTUAL_ENV'] if 'PATH' in env: # Filter out any paths related to the virtual environment - env['PATH'] = ':'.join( - p for p in env['PATH'].split(':') - if '.venv' not in p - ) + env['PATH'] = ':'.join(p for p in env['PATH'].split(':') + if '.venv' not in p) # Direct stream to avoid buffering. # '2>&1' redirects stderr to stdout. ret = subprocess.run( From 5c42d6eb42e95f12d86c92a2b99dfd4875430118 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 10:19:35 +0100 Subject: [PATCH 026/104] Shuffle Jenkinsfile --- Jenkinsfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f93f684fdc..2e9d4a7cba 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,7 +32,7 @@ pipeline { stage('Create uenv') { steps { sh """ - git clone -b fix/jenkins https://github.com/eth-cscs/uenv.git + git clone -b fix_jenkins https://github.com/dominichofer/uenv.git ./uenv/install --yes --destdir=$WORKSPACE source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create @@ -44,7 +44,6 @@ pipeline { // Bootstrapping spack is a separate stage to avoid problems with concurrently bootstrapping spack in the tests. steps { sh """ - source .venv/bin/activate source ./setup-env.sh spack spec gnuconfig """ @@ -61,10 +60,11 @@ pipeline { stage('Integration Tests') { steps { sh """ - source .venv/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh uenv start mch/v8:rc1 + ls /user-environment source ./setup-env.sh /user-environment + source .venv/bin/activate pytest -v -n auto test/integration_test.py """ } @@ -72,10 +72,10 @@ pipeline { stage('System Tests') { steps { sh """ - source .venv/bin/activate source $WORKSPACE/etc/profile.d/uenv.sh uenv start mch/v8:rc1 source ./setup-env.sh /user-environment + source .venv/bin/activate pytest -v -n auto test/system_test.py """ } From e352780b7260af86cb3d086090728379fca3aee3 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:14:24 +0100 Subject: [PATCH 027/104] revert --- Jenkinsfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2e9d4a7cba..7ed8c3c43b 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -32,11 +32,13 @@ pipeline { stage('Create uenv') { steps { sh """ - git clone -b fix_jenkins https://github.com/dominichofer/uenv.git + git clone -b fix/jenkins https://github.com/eth-cscs/uenv.git ./uenv/install --yes --destdir=$WORKSPACE source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create uenv image pull mch/v8:rc1 + uenv start mch/v8:rc1 + ls /user-environment """ } } @@ -62,7 +64,6 @@ pipeline { sh """ source $WORKSPACE/etc/profile.d/uenv.sh uenv start mch/v8:rc1 - ls /user-environment source ./setup-env.sh /user-environment source .venv/bin/activate pytest -v -n auto test/integration_test.py From 20fe9af00096616f49bff09795a66ea62f8b8f5f Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:49:40 +0100 Subject: [PATCH 028/104] uenv --- Jenkinsfile | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 7ed8c3c43b..a9d08e6ecb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,8 +37,6 @@ pipeline { source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create uenv image pull mch/v8:rc1 - uenv start mch/v8:rc1 - ls /user-environment """ } } @@ -63,10 +61,9 @@ pipeline { steps { sh """ source $WORKSPACE/etc/profile.d/uenv.sh - uenv start mch/v8:rc1 source ./setup-env.sh /user-environment source .venv/bin/activate - pytest -v -n auto test/integration_test.py + uenv run mch/v8:rc1 -- pytest -v -n auto test/integration_test.py """ } } @@ -74,10 +71,9 @@ pipeline { steps { sh """ source $WORKSPACE/etc/profile.d/uenv.sh - uenv start mch/v8:rc1 source ./setup-env.sh /user-environment source .venv/bin/activate - pytest -v -n auto test/system_test.py + uenv run mch/v8:rc1 -- pytest -v -n auto test/system_test.py """ } } From 6fb5890a504a7a81463130eb976ca68a58895858 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 12:59:26 +0100 Subject: [PATCH 029/104] revert clean env --- setup-env.sh | 1 + test/spack_commands.py | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/setup-env.sh b/setup-env.sh index 4f77023fff..b939ea95c0 100644 --- a/setup-env.sh +++ b/setup-env.sh @@ -4,6 +4,7 @@ parent_dir=$( cd "$(dirname "${BASH_SOURCE[0]:-${(%):-%x}}")" ; pwd -P ) if [[ "$#" == 1 ]]; then uenv="$1" + export SPACK_UENV_PATH="$uenv" export SPACK_SYSTEM_CONFIG_PATH="$uenv"/config if [[ $uenv == "euler" ]]; then diff --git a/test/spack_commands.py b/test/spack_commands.py index d767ce4018..8259ebacc7 100644 --- a/test/spack_commands.py +++ b/test/spack_commands.py @@ -46,22 +46,22 @@ def run_with_spack(command: str, log: Path) -> None: with log.open("a") as f: f.write(f"{command}\n\n") + # setup-env.sh may define SPACK_UENV_PATH. + if "SPACK_UENV_PATH" in os.environ: + uenv = os.environ["SPACK_UENV_PATH"] + else: + uenv = "" + start = time.time() - # Remove python virtual environment paths for this subprocess call - env = os.environ.copy() - if 'VIRTUAL_ENV' in env: - del env['VIRTUAL_ENV'] - if 'PATH' in env: - # Filter out any paths related to the virtual environment - env['PATH'] = ':'.join(p for p in env['PATH'].split(':') - if '.venv' not in p) # Direct stream to avoid buffering. + # 'env -i' clears the environment. + # 'bash -l' makes it a login shell. + # 'bash -c' reads commands from string. # '2>&1' redirects stderr to stdout. ret = subprocess.run( - f'{command} >> {log} 2>&1', + f'env -i bash -l -c "(. {REPO_DIR}/setup-env.sh {uenv}; {command}) >> {log} 2>&1"', check=False, shell=True, - env=env, ) end = time.time() From 3bd74dd5aecfe9a4c946aea96fddf7f7a55c7dcf Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 15:59:45 +0100 Subject: [PATCH 030/104] Remove fdb-fortran --- repos/c2sm/packages/fdb-fortran/package.py | 33 --------------------- repos/c2sm/packages/flexpart-ifs/package.py | 10 +------ test/system_test.py | 9 ++---- 3 files changed, 3 insertions(+), 49 deletions(-) delete mode 100644 repos/c2sm/packages/fdb-fortran/package.py diff --git a/repos/c2sm/packages/fdb-fortran/package.py b/repos/c2sm/packages/fdb-fortran/package.py deleted file mode 100644 index 422f21b261..0000000000 --- a/repos/c2sm/packages/fdb-fortran/package.py +++ /dev/null @@ -1,33 +0,0 @@ -from spack import * - - -class FdbFortran(CMakePackage): - """An experimental Fortran interface to ECMWF's FDB (Fields DataBase).""" - - homepage = 'https://github.com/MeteoSwiss/fdb-fortran' - git = 'https://github.com/MeteoSwiss/fdb-fortran.git' - maintainers = ['victoria-cherkas'] - - version('0.1.0', tag='0.1.0') - - depends_on('cmake@3.10:', type='build') - depends_on('eckit') - depends_on('metkit') - depends_on('eccodes +fortran') - depends_on('fdb@5.11.0:') - - @property - def libs(self): - return find_libraries("libfdbf", - root=self.prefix, - shared=False, - recursive=True) - - def cmake_args(self): - args = [ - self.define('Deckit_DIR', self.spec['eckit'].prefix), - self.define('Dmetkit_DIR', self.spec['metkit'].prefix), - self.define('Deccodes_DIR', self.spec['eccodes'].prefix), - self.define('Dfdb5_DIR', self.spec['fdb'].prefix), - ] - return args diff --git a/repos/c2sm/packages/flexpart-ifs/package.py b/repos/c2sm/packages/flexpart-ifs/package.py index cc3f5c7751..6a4d3a4c45 100644 --- a/repos/c2sm/packages/flexpart-ifs/package.py +++ b/repos/c2sm/packages/flexpart-ifs/package.py @@ -10,12 +10,10 @@ class FlexpartIfs(MakefilePackage): maintainers = ['pirmink'] version('main', branch='main') - version('fdb', tag='10.4.4_fdb') version('10.4.4', tag='10.4.4') depends_on('eccodes +fortran') - depends_on('netcdf-fortran') - depends_on('fdb-fortran', when='@fdb') + depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. conflicts('%nvhpc') conflicts('%pgi') @@ -29,12 +27,6 @@ def setup_build_environment(self, env): '-I' + self.spec['netcdf-fortran'].prefix.include) env.set('NETCDF_FORTRAN_LD_FLAGS', self.spec['netcdf-fortran'].libs.ld_flags) - if self.spec.satisfies('@fdb'): - env.set('FDB_DIR', self.spec['fdb'].prefix) - env.set('FDB_LD_FLAGS', self.spec['fdb'].libs.ld_flags) - env.set('FDB_FORTRAN_DIR', self.spec['fdb-fortran'].prefix) - env.set('FDB_FORTRAN_LD_FLAGS', - self.spec['fdb-fortran'].libs.ld_flags) def build(self, spec, prefix): with working_dir(self.build_directory): diff --git a/test/system_test.py b/test/system_test.py index d37341b4ba..528112c6a9 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -19,13 +19,8 @@ def test_install_eccodes_2_19_0(): spack_install('eccodes @2.19.0', test_root=False) -def test_install_fdb_fortran(): - spack_install('fdb-fortran') - - -@pytest.mark.parametrize("version", ['10.4.4', 'fdb']) -def test_install_flexpart_ifs_version(version): - spack_install(f'flexpart-ifs @{version}', test_root=False) +def test_install_flexpart_ifs(version): + spack_install('flexpart-ifs', test_root=False) def test_install_flexpart_cosmo(): From 31fe66b8b3543fde167447f216abe2846d43a46e Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:13:09 +0100 Subject: [PATCH 031/104] flexpart-cosmo: Add workaround --- repos/c2sm/packages/flexpart-cosmo/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/c2sm/packages/flexpart-cosmo/package.py b/repos/c2sm/packages/flexpart-cosmo/package.py index 79df4300b2..f0f5ce61f8 100644 --- a/repos/c2sm/packages/flexpart-cosmo/package.py +++ b/repos/c2sm/packages/flexpart-cosmo/package.py @@ -19,7 +19,7 @@ class FlexpartCosmo(MakefilePackage): version('main', branch='main') depends_on('eccodes +fortran') - depends_on('netcdf-fortran') + depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. depends_on('makedepf90') conflicts('%gcc@:10') From 1b6690126b52557c185b262f0b84afa8ffbb7bb2 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:13:21 +0100 Subject: [PATCH 032/104] Update icon-mch tests --- test/system_test.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 528112c6a9..a853d055ac 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -27,12 +27,12 @@ def test_install_flexpart_cosmo(): spack_install('flexpart-cosmo @V8C4.0') -def test_install_icon_mch_2024_01_gcc(): - spack_install('icon-mch @2024.01 %gcc') +def test_install_icon_mch_2_6_6_mch2b_gcc(): + spack_install('icon-mch @icon-2.6.6-mch2b %gcc') -def test_install_icon_mch_2024_01_nvhpc(): - spack_install('icon-mch @2024.01 %nvhpc') +def test_install_icon_mch_2_6_6_mch2b_nvhpc(): + spack_install('icon-mch @icon-2.6.6-mch2b %nvhpc') def test_install_icon_mch_conditional_dependencies(): @@ -41,10 +41,10 @@ def test_install_icon_mch_conditional_dependencies(): # +emvorado triggers eccodes, hdf5, zlib # +eccodes-definitions triggers cosmo-eccodes-definitions # +mpi triggers mpi - # gpu=openacc+cuda triggers cuda + # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @2024.01 %nvhpc +coupling serialization=create +emvorado +mpi gpu=openacc+cuda cuda_arch=80' + 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' ) From d219b6edcd54577296222e77c21edfd5748db0f2 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:04:53 +0100 Subject: [PATCH 033/104] Typo --- test/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index a853d055ac..abaa531369 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -19,7 +19,7 @@ def test_install_eccodes_2_19_0(): spack_install('eccodes @2.19.0', test_root=False) -def test_install_flexpart_ifs(version): +def test_install_flexpart_ifs(): spack_install('flexpart-ifs', test_root=False) From 9353b9b752257d290a4caf3a76bd53560afc68c9 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 17:05:05 +0100 Subject: [PATCH 034/104] icontools: Workaround --- repos/c2sm/packages/icontools/package.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repos/c2sm/packages/icontools/package.py b/repos/c2sm/packages/icontools/package.py index 8ad698c619..37cfaaa340 100644 --- a/repos/c2sm/packages/icontools/package.py +++ b/repos/c2sm/packages/icontools/package.py @@ -39,15 +39,15 @@ class Icontools(AutotoolsPackage): depends_on('libtool', type='build') depends_on('m4', type='build') - depends_on('netcdf-fortran', type=('build', 'link')) - depends_on('netcdf-c ~mpi', type=('build', 'link')) - depends_on('hdf5 ~mpi +hl', type=('build', 'link')) + depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-c ~mpi') + depends_on('hdf5 ~mpi +hl') depends_on( 'mpi', type=('build', 'link', 'run'), ) depends_on('eccodes@2.19.0 +fortran ~aec', type=('build', 'link', 'run')) - depends_on('jasper@1.900.1', type=('build', 'link')) + depends_on('jasper@1.900.1') variant( 'slave', From c94ed34ec4121fcca36685237d7f7f8163d6088f Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:09:52 +0100 Subject: [PATCH 035/104] Fix some icon tests --- test/system_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index abaa531369..39b7961a25 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -32,7 +32,7 @@ def test_install_icon_mch_2_6_6_mch2b_gcc(): def test_install_icon_mch_2_6_6_mch2b_nvhpc(): - spack_install('icon-mch @icon-2.6.6-mch2b %nvhpc') + spack_install('icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc') def test_install_icon_mch_conditional_dependencies(): @@ -44,7 +44,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' + 'icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' ) From 7362726b70683f99e29a0fa182032680b5b5ccc4 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:04:40 +0100 Subject: [PATCH 036/104] Fix type --- test/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 39b7961a25..8d423aa5b2 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -44,7 +44,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' + 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc' ) From ea79b332052fcb2b6ff5bf5cd146cfe210449273 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:05:49 +0100 Subject: [PATCH 037/104] Remove icon-mch gcc test --- test/system_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 8d423aa5b2..9dd4411c62 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -27,10 +27,6 @@ def test_install_flexpart_cosmo(): spack_install('flexpart-cosmo @V8C4.0') -def test_install_icon_mch_2_6_6_mch2b_gcc(): - spack_install('icon-mch @icon-2.6.6-mch2b %gcc') - - def test_install_icon_mch_2_6_6_mch2b_nvhpc(): spack_install('icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc') From 32677b1ed064cfae438b45be34e2e32a97a2b4d2 Mon Sep 17 00:00:00 2001 From: github-actions Date: Mon, 11 Nov 2024 16:36:21 +0000 Subject: [PATCH 038/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/flexpart-cosmo/package.py | 4 +++- repos/c2sm/packages/flexpart-ifs/package.py | 4 +++- repos/c2sm/packages/icontools/package.py | 4 +++- test/system_test.py | 4 +++- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/repos/c2sm/packages/flexpart-cosmo/package.py b/repos/c2sm/packages/flexpart-cosmo/package.py index f0f5ce61f8..ae09f08455 100644 --- a/repos/c2sm/packages/flexpart-cosmo/package.py +++ b/repos/c2sm/packages/flexpart-cosmo/package.py @@ -19,7 +19,9 @@ class FlexpartCosmo(MakefilePackage): version('main', branch='main') depends_on('eccodes +fortran') - depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on( + 'netcdf-fortran %gcc' + ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. depends_on('makedepf90') conflicts('%gcc@:10') diff --git a/repos/c2sm/packages/flexpart-ifs/package.py b/repos/c2sm/packages/flexpart-ifs/package.py index 6a4d3a4c45..637aa97bc4 100644 --- a/repos/c2sm/packages/flexpart-ifs/package.py +++ b/repos/c2sm/packages/flexpart-ifs/package.py @@ -13,7 +13,9 @@ class FlexpartIfs(MakefilePackage): version('10.4.4', tag='10.4.4') depends_on('eccodes +fortran') - depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on( + 'netcdf-fortran %gcc' + ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. conflicts('%nvhpc') conflicts('%pgi') diff --git a/repos/c2sm/packages/icontools/package.py b/repos/c2sm/packages/icontools/package.py index 37cfaaa340..4af2485897 100644 --- a/repos/c2sm/packages/icontools/package.py +++ b/repos/c2sm/packages/icontools/package.py @@ -39,7 +39,9 @@ class Icontools(AutotoolsPackage): depends_on('libtool', type='build') depends_on('m4', type='build') - depends_on('netcdf-fortran %gcc') # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on( + 'netcdf-fortran %gcc' + ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. depends_on('netcdf-c ~mpi') depends_on('hdf5 ~mpi +hl') depends_on( diff --git a/test/system_test.py b/test/system_test.py index 9dd4411c62..7c0de81596 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -28,7 +28,9 @@ def test_install_flexpart_cosmo(): def test_install_icon_mch_2_6_6_mch2b_nvhpc(): - spack_install('icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc') + spack_install( + 'icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc' + ) def test_install_icon_mch_conditional_dependencies(): From fe7348820e85a75da3a79dbbd2e85dfeee17e2cb Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 11 Nov 2024 17:50:45 +0100 Subject: [PATCH 039/104] Revert "cosmo: Remove package" This reverts commit 2a7a3371f06e362d4ffba9a666eb127623e5db86. --- repos/c2sm/packages/cosmo/package.py | 329 ++++++++++++++++++ .../packages/cosmo/patches/5.09a.mch1.2.p2 | 1 + .../spec_as_yaml/patch.serialize_cosmo | 20 ++ .../apn-mch/spec_as_yaml/patch.test_cosmo | 21 ++ .../spec_as_yaml/patch.serialize_cosmo | 1 + repos/c2sm/packages/cosmo/patches/c2sm-master | 1 + .../spec_as_yaml/patch.serialize_cosmo | 1 + .../spec_as_yaml/patch.serialize_cosmo | 19 + .../org-master/spec_as_yaml/patch.test_cosmo | 22 ++ test/system_test.py | 4 + 10 files changed, 419 insertions(+) create mode 100644 repos/c2sm/packages/cosmo/package.py create mode 120000 repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 create mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo create mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo create mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo create mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-master create mode 120000 repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo create mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo create mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo diff --git a/repos/c2sm/packages/cosmo/package.py b/repos/c2sm/packages/cosmo/package.py new file mode 100644 index 0000000000..2abb680196 --- /dev/null +++ b/repos/c2sm/packages/cosmo/package.py @@ -0,0 +1,329 @@ +# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import subprocess, itertools, os +from spack import * + + +class Cosmo(MakefilePackage): + """COSMO: Numerical Weather Prediction Model. Needs access to private GitHub.""" + + homepage = "http://www.cosmo-model.org" + url = "https://github.com/COSMO-ORG/cosmo/archive/6.0.tar.gz" + git = 'git@github.com:COSMO-ORG/cosmo.git' + apn_git = 'git@github.com:MeteoSwiss-APN/cosmo.git' + c2sm_git = 'git@github.com:C2SM-RCM/cosmo.git' + empa_git = 'git@github.com:C2SM-RCM/cosmo-ghg.git' + maintainers = ['mjaehn', 'juckerj'] + + version('org-master', branch='master') + version('6.0', tag='6.0') + + version('apn-mch', git=apn_git, branch='mch') + version('5.09a.mch1.2.p2', git=apn_git, tag='5.09a.mch1.2.p2') + + version('6.1_2023.11', git=c2sm_git, tag='6.1_2023.11') + version('c2sm-master', git=c2sm_git, branch='master') + version('c2sm-features', git=c2sm_git, branch='c2sm-features') + version('empa-ghg', git=empa_git, branch='c2sm') + + # pass spec from spec to test_cosmo.py in yaml-format + # There are three different types of test_cosmo.py around: + + # COSMO-ORG + patch('patches/org-master/spec_as_yaml/patch.test_cosmo', + when='@org-master') + patch('patches/org-master/spec_as_yaml/patch.test_cosmo', when='@6.0') + # APN-MCH + patch('patches/apn-mch/spec_as_yaml/patch.test_cosmo', when='@apn-mch') + patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.test_cosmo', + when='@5.09a.mch1.2.p2') + + # pass spec from spec to serialize_cosmo.py in yaml-format + + # There are two different types of serialize_cosmo.py around: + + # COSMO-ORG + patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', + when='@org-master +serialize') + patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', + when='@6.0 +serialize') + patch('patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo', + when='@c2sm-features +serialize') + patch('patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo', + when='@empa-ghg +serialize') + + # APN-MCH + patch('patches/apn-mch/spec_as_yaml/patch.serialize_cosmo', + when='@apn-mch +serialize') + patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.serialize_cosmo', + when='@5.09a.mch1.2.p2 +serialize') + + # build dependency + depends_on('perl@5.16.3:', type='build') + depends_on('libgrib1', type='build') + + # build and link dependency + depends_on('mpi +fortran') + depends_on('netcdf-fortran') + depends_on('netcdf-c +mpi') + depends_on('jasper@1.900.1') + depends_on('eccodes +fortran') + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('eccodes %nvhpc', when='%nvhpc') + depends_on('eccodes %gcc', when='%gcc') + + # run dependency + depends_on('slurm', type='run') + + depends_on('cosmo-eccodes-definitions', type=('build', 'run')) + depends_on('serialbox +fortran ^python@2:2.9', + when='+serialize', + type=('build', 'link', 'run')) + depends_on('oasis', when='+oasis', type=('build', 'link', 'run')) + + variant('serialize', + default=False, + description='Build with serialization enabled') + variant('parallel', default=True, description='Build parallel COSMO') + variant('debug', default=False, description='Build debug mode') + variant('real_type', + default='double', + description='Build with double or single precision enabled', + values=('double', 'float'), + multi=False) + variant('slave', default='none', description='Build on slave') + variant('pollen', default=False, description='Build with pollen enabled') + variant('verbose', + default=False, + description='Build cosmo with verbose enabled') + variant('set_version', + default=False, + description='Pass cosmo tag version to Makefile') + variant('gt1', + default=False, + description='Build dycore with gridtools 1.1.3') + variant('cuda_arch', + default='none', + description='Build with cuda_arch', + values=('80', '70', '60', '37'), + multi=False) + variant('oasis', + default=False, + description='Build with the unified oasis interface') + + conflicts( + '@dev-build', + msg= + "Please use only official versions listed with 'spack info cosmo'. Even when using 'devbuildcosmo'. C2SM introduced 'dev-build' to avoid name conflicts with the upstream instance. Since spack-c2sm v0.18.1.0 this is not relevant anymore." + ) + conflicts('+pollen', when='@org-master,master') + # - ML - A conflict should be added there if the oasis variant is + # chosen and the version is neither c2sm-features nor + # dev-build. The problem is that this doesn't seem possible in a + # native spack way. Hence the dirty check at the beginning of + # setup_build_environment method + + build_directory = 'cosmo/ACC' + + # v0.20.1 does not fully support builds of COSMO + # users should continue using v0.18.1.x on Daint + # once support on Alps has been clarified finalize + # recipe and remove line below. + manual_download = True + + def setup_build_environment(self, env): + + # - ML - Dirty conflict check (see above) + if self.spec.variants['oasis'].value and self.spec.version not in ( + Version('c2sm-features')): + raise InstallError( + '+oasis variant only compatible with the @c2sm-features versions' + ) + + self.setup_run_environment(env) + + # Check mpi provider + self.mpi_spec = self.spec['mpi'] + + grib_prefix = self.spec['eccodes'].prefix + env.set( + 'GRIBAPIL', + str(self.spec['eccodes:fortran'].libs.ld_flags) + ' ' + + str(self.spec['jasper'].libs.ld_flags)) + grib_inc_dir_path = os.path.join(grib_prefix, 'include') + if os.path.exists(grib_inc_dir_path): + env.set('GRIBAPII', '-I' + grib_inc_dir_path) + else: + env.set('GRIBAPII', '') + + # libaec + if 'libaec' in self.spec: + env.set('AECL', '-L' + self.spec['libaec'].prefix + '/lib64 -laec') + env.set('AECI', '-I' + self.spec['libaec'].prefix + '/include') + + # Netcdf library + env.set( + 'NETCDFL', + '-L' + self.spec['netcdf-fortran'].prefix + '/lib -lnetcdff -L' + + self.spec['netcdf-c'].prefix + '/lib -lnetcdf') + env.set('NETCDFI', + '-I' + self.spec['netcdf-fortran'].prefix + '/include') + + # Grib1 library + if self.compiler.name == 'gcc': + env.set('GRIBDWDL', + '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_gnu') + elif self.compiler.name == 'cce': + env.set('GRIBDWDL', + '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_cray') + elif self.compiler.name == 'nvhpc': + env.set('GRIBDWDL', + '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_pgi') + else: + env.set( + 'GRIBDWDL', '-L' + self.spec['libgrib1'].prefix + + '/lib -lgrib1_' + self.compiler.name) + env.set('GRIBDWDI', '-I' + self.spec['libgrib1'].prefix + '/include') + + # MPI library + if self.mpi_spec.name == 'openmpi': + env.set('MPIL', '-L' + self.mpi_spec.prefix + ' -lmpi') + + else: + env.set('MPIL', '-L' + self.spec['mpi'].prefix + ' -lmpich') + + env.set('MPII', '-I' + self.mpi_spec.prefix + '/include') + + # Serialbox library + if '+serialize' in self.spec: + env.set('SERIALBOX', self.spec['serialbox'].prefix) + env.set('SERIALBOXL', + self.spec['serialbox:fortran,c'].libs.ld_flags) + env.set('SERIALBOXI', + '-I' + self.spec['serialbox'].prefix + '/include') + + # OASIS library + if '+oasis' in self.spec: + oasis_prefix = self.spec['oasis'].prefix + env.set( + 'PSMILEL', + '-L{:s}/lib -lpsmile.MPI1 -lscrip -lmct -lmpeu'.format( + oasis_prefix)) + env.set( + 'PSMILEI', + '-I{0:s}/build/lib/psmile.MPI1 -I{0:s}/build/lib/mct'.format( + oasis_prefix)) + env.set('MPPIOI', '-I{:s}/build/lib/mct'.format(oasis_prefix)) + + # Linker flags + if self.compiler.name in ('pgi', + 'nvhpc') and '~cppdycore' in self.spec: + env.set('LFLAGS', '-lstdc++') + + # Compiler & linker variables + if self.compiler.name in ('pgi', 'nvhpc'): + env.set('F90', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') + env.set('LD', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') + else: + env.set('F90', self.mpi_spec.mpifc) + env.set('LD', self.mpi_spec.mpifc) + + @property + def build_targets(self): + build = [] + if self.spec.variants['pollen'].value: + build.append('POLLEN=1') + if self.spec.variants['oasis'].value: + build.append('COUP_OAS=1') + if self.spec.variants['real_type'].value == 'float': + build.append('SINGLEPRECISION=1') + if '+serialize' in self.spec: + build.append('SERIALIZE=1') + if self.spec.variants['verbose'].value: + build.append('VERBOSE=1') + if '+set_version' in self.spec: + build.append('COSMO_VERSION=' + self.spec.format('{version}')) + MakeFileTarget = '' + if '+parallel' in self.spec: + MakeFileTarget += 'par' + else: + MakeFileTarget += 'seq' + if '+debug' in self.spec: + MakeFileTarget += 'debug' + else: + MakeFileTarget += 'opt' + build.append(MakeFileTarget) + + return build + + def edit(self, spec, prefix): + with working_dir(self.build_directory): + OptionsFileName = 'Options' + if self.compiler.name == 'gcc': + OptionsFileName += '.gnu' + elif self.compiler.name in ('pgi', 'nvhpc'): + OptionsFileName += '.pgi' + elif self.compiler.name == 'cce': + OptionsFileName += '.cray' + OptionsFileName += '.cpu' + OptionsFile = FileFilter(OptionsFileName) + + makefile = FileFilter('Makefile') + makefile.filter('/Options.*', '/' + OptionsFileName) + if self.spec.version == Version('empa-ghg'): + if '~serialize' in spec: + makefile.filter( + 'TARGET :=.*', + 'TARGET := {0}'.format('cosmo-ghg_cpu')) + else: + makefile.filter('TARGET :=.*', + 'TARGET := {0}'.format('cosmo-ghg')) + else: + if '~serialize' in spec: + makefile.filter('TARGET :=.*', + 'TARGET := {0}'.format('cosmo_cpu')) + else: + makefile.filter('TARGET :=.*', + 'TARGET := {0}'.format('cosmo')) + + # Pre-processor flags + if self.mpi_spec.name == 'mpich': + OptionsFile.filter( + 'PFLAGS = -Mpreprocess.*', + 'PFLAGS = -Mpreprocess -DNO_MPI_HOST_DATA') + + def install(self, spec, prefix): + + with working_dir(self.build_directory): + mkdir(prefix.bin) + if self.spec.version == Version('empa-ghg'): + if '+serialize' in spec: + install('cosmo-ghg_serialize', prefix.bin) + else: + install('cosmo-ghg_cpu', prefix.bin) + install('cosmo-ghg_cpu', 'test/testsuite') + else: + if '+serialize' in spec: + install('cosmo_serialize', prefix.bin) + else: + install('cosmo_cpu', prefix.bin) + install('cosmo_cpu', 'test/testsuite') + + @run_after('install') + @on_package_attributes(run_tests=True) + def test(self): + with open('spec.yaml', mode='w') as f: + f.write(self.spec.to_yaml()) + try: + subprocess.run([ + self.build_directory + '/test/tools/test_cosmo.py', '-s', + 'spec.yaml', '-b', + str('.') + ], + stderr=subprocess.STDOUT, + check=True) + except: + raise InstallError('Testsuite failed') diff --git a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 b/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 new file mode 120000 index 0000000000..34c563540d --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 @@ -0,0 +1 @@ +apn-mch/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo new file mode 100644 index 0000000000..39aa5e1fc2 --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo @@ -0,0 +1,20 @@ +diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py +index 624fd295..fc15c712 100755 +--- a/cosmo/ACC/test/tools/serialize_cosmo.py ++++ b/cosmo/ACC/test/tools/serialize_cosmo.py +@@ -15,7 +15,14 @@ def main(): + parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) + args=parser.parse_args() + +- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] ++ try: ++ # assume spec is passed as a yaml-file ++ with open(args.spec,"r") as f: ++ cosmo_spec = Spec.from_yaml(f) ++ except: ++ # assume spec is passed as a string ++ cosmo_spec = Spec(args.spec) ++ + cosmo_spec.concretize() + + serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo new file mode 100644 index 0000000000..40bc6a90d1 --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo @@ -0,0 +1,21 @@ +diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py +index 733bde4a..2a166bbd 100755 +--- a/cosmo/ACC/test/tools/test_cosmo.py ++++ b/cosmo/ACC/test/tools/test_cosmo.py +@@ -16,7 +16,15 @@ def main(): + parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') + args=parser.parse_args() + +- test_cosmo(Spec(args.spec), args.base_directory) ++ try: ++ # assume spec is passed as a yaml-file ++ with open(args.spec,"r") as f: ++ cosmo_spec = Spec.from_yaml(f) ++ except: ++ # assume spec is passed as a string ++ cosmo_spec = Spec(args.spec) ++ ++ test_cosmo(cosmo_spec, args.base_directory) + + def test_cosmo(cosmo_spec, base_directory): + diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo new file mode 120000 index 0000000000..bfd9c3d65d --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo @@ -0,0 +1 @@ +../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-master b/repos/c2sm/packages/cosmo/patches/c2sm-master new file mode 120000 index 0000000000..71b99193f9 --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/c2sm-master @@ -0,0 +1 @@ +org-master/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo new file mode 120000 index 0000000000..bfd9c3d65d --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo @@ -0,0 +1 @@ +../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo new file mode 100644 index 0000000000..8c72160de2 --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo @@ -0,0 +1,19 @@ +diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py +index 624fd295..e93b96f6 100755 +--- a/cosmo/ACC/test/tools/serialize_cosmo.py ++++ b/cosmo/ACC/test/tools/serialize_cosmo.py +@@ -15,7 +15,13 @@ def main(): + parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) + args=parser.parse_args() + +- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] ++ try: ++ # assume spec is passed as a yaml-file ++ with open(args.spec,"r") as f: ++ cosmo_spec = Spec.from_yaml(f) ++ except: ++ # assume spec is passed as a string ++ cosmo_spec = Spec(args.spec) + cosmo_spec.concretize() + + serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo new file mode 100644 index 0000000000..8e91d15f45 --- /dev/null +++ b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo @@ -0,0 +1,22 @@ +diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py +index 733bde4a..c2c60f2b 100755 +--- a/cosmo/ACC/test/tools/test_cosmo.py ++++ b/cosmo/ACC/test/tools/test_cosmo.py +@@ -16,7 +16,16 @@ def main(): + parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') + args=parser.parse_args() + +- test_cosmo(Spec(args.spec), args.base_directory) ++ try: ++ # assume spec is passed as a yaml-file ++ with open(args.spec,"r") as f: ++ cosmo_spec = Spec.from_yaml(f) ++ except: ++ # assume spec is passed as a string ++ cosmo_spec = Spec(args.spec) ++ ++ ++ test_cosmo(cosmo_spec, args.base_directory) + + def test_cosmo(cosmo_spec, base_directory): + diff --git a/test/system_test.py b/test/system_test.py index 7c0de81596..0e220d3d40 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -15,6 +15,10 @@ def test_install_cosmo_eccodes_definitions_version(version): spack_install(f'cosmo-eccodes-definitions @{version}', test_root=False) +def test_install_cosmo_6_0(): + spack_install('cosmo @6.0 %nvhpc', test_root=False) + + def test_install_eccodes_2_19_0(): spack_install('eccodes @2.19.0', test_root=False) From 6a6e53c14322d81bfced113f1a84cc55ca4c3d30 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 10:39:24 +0100 Subject: [PATCH 040/104] cosmo: remove package --- repos/c2sm/packages/cosmo/package.py | 329 ------------------ .../packages/cosmo/patches/5.09a.mch1.2.p2 | 1 - .../spec_as_yaml/patch.serialize_cosmo | 20 -- .../apn-mch/spec_as_yaml/patch.test_cosmo | 21 -- .../spec_as_yaml/patch.serialize_cosmo | 1 - repos/c2sm/packages/cosmo/patches/c2sm-master | 1 - .../spec_as_yaml/patch.serialize_cosmo | 1 - .../spec_as_yaml/patch.serialize_cosmo | 19 - .../org-master/spec_as_yaml/patch.test_cosmo | 22 -- 9 files changed, 415 deletions(-) delete mode 100644 repos/c2sm/packages/cosmo/package.py delete mode 120000 repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 delete mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo delete mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo delete mode 120000 repos/c2sm/packages/cosmo/patches/c2sm-master delete mode 120000 repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo delete mode 100644 repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo diff --git a/repos/c2sm/packages/cosmo/package.py b/repos/c2sm/packages/cosmo/package.py deleted file mode 100644 index 2abb680196..0000000000 --- a/repos/c2sm/packages/cosmo/package.py +++ /dev/null @@ -1,329 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -import subprocess, itertools, os -from spack import * - - -class Cosmo(MakefilePackage): - """COSMO: Numerical Weather Prediction Model. Needs access to private GitHub.""" - - homepage = "http://www.cosmo-model.org" - url = "https://github.com/COSMO-ORG/cosmo/archive/6.0.tar.gz" - git = 'git@github.com:COSMO-ORG/cosmo.git' - apn_git = 'git@github.com:MeteoSwiss-APN/cosmo.git' - c2sm_git = 'git@github.com:C2SM-RCM/cosmo.git' - empa_git = 'git@github.com:C2SM-RCM/cosmo-ghg.git' - maintainers = ['mjaehn', 'juckerj'] - - version('org-master', branch='master') - version('6.0', tag='6.0') - - version('apn-mch', git=apn_git, branch='mch') - version('5.09a.mch1.2.p2', git=apn_git, tag='5.09a.mch1.2.p2') - - version('6.1_2023.11', git=c2sm_git, tag='6.1_2023.11') - version('c2sm-master', git=c2sm_git, branch='master') - version('c2sm-features', git=c2sm_git, branch='c2sm-features') - version('empa-ghg', git=empa_git, branch='c2sm') - - # pass spec from spec to test_cosmo.py in yaml-format - # There are three different types of test_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', - when='@org-master') - patch('patches/org-master/spec_as_yaml/patch.test_cosmo', when='@6.0') - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.test_cosmo', when='@apn-mch') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.test_cosmo', - when='@5.09a.mch1.2.p2') - - # pass spec from spec to serialize_cosmo.py in yaml-format - - # There are two different types of serialize_cosmo.py around: - - # COSMO-ORG - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@org-master +serialize') - patch('patches/org-master/spec_as_yaml/patch.serialize_cosmo', - when='@6.0 +serialize') - patch('patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo', - when='@c2sm-features +serialize') - patch('patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo', - when='@empa-ghg +serialize') - - # APN-MCH - patch('patches/apn-mch/spec_as_yaml/patch.serialize_cosmo', - when='@apn-mch +serialize') - patch('patches/5.09a.mch1.2.p2/spec_as_yaml/patch.serialize_cosmo', - when='@5.09a.mch1.2.p2 +serialize') - - # build dependency - depends_on('perl@5.16.3:', type='build') - depends_on('libgrib1', type='build') - - # build and link dependency - depends_on('mpi +fortran') - depends_on('netcdf-fortran') - depends_on('netcdf-c +mpi') - depends_on('jasper@1.900.1') - depends_on('eccodes +fortran') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('eccodes %nvhpc', when='%nvhpc') - depends_on('eccodes %gcc', when='%gcc') - - # run dependency - depends_on('slurm', type='run') - - depends_on('cosmo-eccodes-definitions', type=('build', 'run')) - depends_on('serialbox +fortran ^python@2:2.9', - when='+serialize', - type=('build', 'link', 'run')) - depends_on('oasis', when='+oasis', type=('build', 'link', 'run')) - - variant('serialize', - default=False, - description='Build with serialization enabled') - variant('parallel', default=True, description='Build parallel COSMO') - variant('debug', default=False, description='Build debug mode') - variant('real_type', - default='double', - description='Build with double or single precision enabled', - values=('double', 'float'), - multi=False) - variant('slave', default='none', description='Build on slave') - variant('pollen', default=False, description='Build with pollen enabled') - variant('verbose', - default=False, - description='Build cosmo with verbose enabled') - variant('set_version', - default=False, - description='Pass cosmo tag version to Makefile') - variant('gt1', - default=False, - description='Build dycore with gridtools 1.1.3') - variant('cuda_arch', - default='none', - description='Build with cuda_arch', - values=('80', '70', '60', '37'), - multi=False) - variant('oasis', - default=False, - description='Build with the unified oasis interface') - - conflicts( - '@dev-build', - msg= - "Please use only official versions listed with 'spack info cosmo'. Even when using 'devbuildcosmo'. C2SM introduced 'dev-build' to avoid name conflicts with the upstream instance. Since spack-c2sm v0.18.1.0 this is not relevant anymore." - ) - conflicts('+pollen', when='@org-master,master') - # - ML - A conflict should be added there if the oasis variant is - # chosen and the version is neither c2sm-features nor - # dev-build. The problem is that this doesn't seem possible in a - # native spack way. Hence the dirty check at the beginning of - # setup_build_environment method - - build_directory = 'cosmo/ACC' - - # v0.20.1 does not fully support builds of COSMO - # users should continue using v0.18.1.x on Daint - # once support on Alps has been clarified finalize - # recipe and remove line below. - manual_download = True - - def setup_build_environment(self, env): - - # - ML - Dirty conflict check (see above) - if self.spec.variants['oasis'].value and self.spec.version not in ( - Version('c2sm-features')): - raise InstallError( - '+oasis variant only compatible with the @c2sm-features versions' - ) - - self.setup_run_environment(env) - - # Check mpi provider - self.mpi_spec = self.spec['mpi'] - - grib_prefix = self.spec['eccodes'].prefix - env.set( - 'GRIBAPIL', - str(self.spec['eccodes:fortran'].libs.ld_flags) + ' ' + - str(self.spec['jasper'].libs.ld_flags)) - grib_inc_dir_path = os.path.join(grib_prefix, 'include') - if os.path.exists(grib_inc_dir_path): - env.set('GRIBAPII', '-I' + grib_inc_dir_path) - else: - env.set('GRIBAPII', '') - - # libaec - if 'libaec' in self.spec: - env.set('AECL', '-L' + self.spec['libaec'].prefix + '/lib64 -laec') - env.set('AECI', '-I' + self.spec['libaec'].prefix + '/include') - - # Netcdf library - env.set( - 'NETCDFL', - '-L' + self.spec['netcdf-fortran'].prefix + '/lib -lnetcdff -L' + - self.spec['netcdf-c'].prefix + '/lib -lnetcdf') - env.set('NETCDFI', - '-I' + self.spec['netcdf-fortran'].prefix + '/include') - - # Grib1 library - if self.compiler.name == 'gcc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_gnu') - elif self.compiler.name == 'cce': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_cray') - elif self.compiler.name == 'nvhpc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_pgi') - else: - env.set( - 'GRIBDWDL', '-L' + self.spec['libgrib1'].prefix + - '/lib -lgrib1_' + self.compiler.name) - env.set('GRIBDWDI', '-I' + self.spec['libgrib1'].prefix + '/include') - - # MPI library - if self.mpi_spec.name == 'openmpi': - env.set('MPIL', '-L' + self.mpi_spec.prefix + ' -lmpi') - - else: - env.set('MPIL', '-L' + self.spec['mpi'].prefix + ' -lmpich') - - env.set('MPII', '-I' + self.mpi_spec.prefix + '/include') - - # Serialbox library - if '+serialize' in self.spec: - env.set('SERIALBOX', self.spec['serialbox'].prefix) - env.set('SERIALBOXL', - self.spec['serialbox:fortran,c'].libs.ld_flags) - env.set('SERIALBOXI', - '-I' + self.spec['serialbox'].prefix + '/include') - - # OASIS library - if '+oasis' in self.spec: - oasis_prefix = self.spec['oasis'].prefix - env.set( - 'PSMILEL', - '-L{:s}/lib -lpsmile.MPI1 -lscrip -lmct -lmpeu'.format( - oasis_prefix)) - env.set( - 'PSMILEI', - '-I{0:s}/build/lib/psmile.MPI1 -I{0:s}/build/lib/mct'.format( - oasis_prefix)) - env.set('MPPIOI', '-I{:s}/build/lib/mct'.format(oasis_prefix)) - - # Linker flags - if self.compiler.name in ('pgi', - 'nvhpc') and '~cppdycore' in self.spec: - env.set('LFLAGS', '-lstdc++') - - # Compiler & linker variables - if self.compiler.name in ('pgi', 'nvhpc'): - env.set('F90', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - env.set('LD', self.mpi_spec.mpifc + ' -D__PGI_FORTRAN__') - else: - env.set('F90', self.mpi_spec.mpifc) - env.set('LD', self.mpi_spec.mpifc) - - @property - def build_targets(self): - build = [] - if self.spec.variants['pollen'].value: - build.append('POLLEN=1') - if self.spec.variants['oasis'].value: - build.append('COUP_OAS=1') - if self.spec.variants['real_type'].value == 'float': - build.append('SINGLEPRECISION=1') - if '+serialize' in self.spec: - build.append('SERIALIZE=1') - if self.spec.variants['verbose'].value: - build.append('VERBOSE=1') - if '+set_version' in self.spec: - build.append('COSMO_VERSION=' + self.spec.format('{version}')) - MakeFileTarget = '' - if '+parallel' in self.spec: - MakeFileTarget += 'par' - else: - MakeFileTarget += 'seq' - if '+debug' in self.spec: - MakeFileTarget += 'debug' - else: - MakeFileTarget += 'opt' - build.append(MakeFileTarget) - - return build - - def edit(self, spec, prefix): - with working_dir(self.build_directory): - OptionsFileName = 'Options' - if self.compiler.name == 'gcc': - OptionsFileName += '.gnu' - elif self.compiler.name in ('pgi', 'nvhpc'): - OptionsFileName += '.pgi' - elif self.compiler.name == 'cce': - OptionsFileName += '.cray' - OptionsFileName += '.cpu' - OptionsFile = FileFilter(OptionsFileName) - - makefile = FileFilter('Makefile') - makefile.filter('/Options.*', '/' + OptionsFileName) - if self.spec.version == Version('empa-ghg'): - if '~serialize' in spec: - makefile.filter( - 'TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo-ghg')) - else: - if '~serialize' in spec: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo_cpu')) - else: - makefile.filter('TARGET :=.*', - 'TARGET := {0}'.format('cosmo')) - - # Pre-processor flags - if self.mpi_spec.name == 'mpich': - OptionsFile.filter( - 'PFLAGS = -Mpreprocess.*', - 'PFLAGS = -Mpreprocess -DNO_MPI_HOST_DATA') - - def install(self, spec, prefix): - - with working_dir(self.build_directory): - mkdir(prefix.bin) - if self.spec.version == Version('empa-ghg'): - if '+serialize' in spec: - install('cosmo-ghg_serialize', prefix.bin) - else: - install('cosmo-ghg_cpu', prefix.bin) - install('cosmo-ghg_cpu', 'test/testsuite') - else: - if '+serialize' in spec: - install('cosmo_serialize', prefix.bin) - else: - install('cosmo_cpu', prefix.bin) - install('cosmo_cpu', 'test/testsuite') - - @run_after('install') - @on_package_attributes(run_tests=True) - def test(self): - with open('spec.yaml', mode='w') as f: - f.write(self.spec.to_yaml()) - try: - subprocess.run([ - self.build_directory + '/test/tools/test_cosmo.py', '-s', - 'spec.yaml', '-b', - str('.') - ], - stderr=subprocess.STDOUT, - check=True) - except: - raise InstallError('Testsuite failed') diff --git a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 b/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 deleted file mode 120000 index 34c563540d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/5.09a.mch1.2.p2 +++ /dev/null @@ -1 +0,0 @@ -apn-mch/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 39aa5e1fc2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,20 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..fc15c712 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,14 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 40bc6a90d1..0000000000 --- a/repos/c2sm/packages/cosmo/patches/apn-mch/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,21 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..2a166bbd 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,15 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-features/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/c2sm-master b/repos/c2sm/packages/cosmo/patches/c2sm-master deleted file mode 120000 index 71b99193f9..0000000000 --- a/repos/c2sm/packages/cosmo/patches/c2sm-master +++ /dev/null @@ -1 +0,0 @@ -org-master/ \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo deleted file mode 120000 index bfd9c3d65d..0000000000 --- a/repos/c2sm/packages/cosmo/patches/empa-ghg/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1 +0,0 @@ -../../org-master/spec_as_yaml/patch.serialize_cosmo \ No newline at end of file diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo deleted file mode 100644 index 8c72160de2..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.serialize_cosmo +++ /dev/null @@ -1,19 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/serialize_cosmo.py b/cosmo/ACC/test/tools/serialize_cosmo.py -index 624fd295..e93b96f6 100755 ---- a/cosmo/ACC/test/tools/serialize_cosmo.py -+++ b/cosmo/ACC/test/tools/serialize_cosmo.py -@@ -15,7 +15,13 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory', required=True) - args=parser.parse_args() - -- cosmo_spec = spack.cmd.parse_specs(args.spec)[0] -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) - cosmo_spec.concretize() - - serialize_cosmo(cosmo_spec, args.base_directory) diff --git a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo b/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo deleted file mode 100644 index 8e91d15f45..0000000000 --- a/repos/c2sm/packages/cosmo/patches/org-master/spec_as_yaml/patch.test_cosmo +++ /dev/null @@ -1,22 +0,0 @@ -diff --git a/cosmo/ACC/test/tools/test_cosmo.py b/cosmo/ACC/test/tools/test_cosmo.py -index 733bde4a..c2c60f2b 100755 ---- a/cosmo/ACC/test/tools/test_cosmo.py -+++ b/cosmo/ACC/test/tools/test_cosmo.py -@@ -16,7 +16,16 @@ def main(): - parser.add_argument('-b', '--base_directory', type=str, help='Cosmo base directory') - args=parser.parse_args() - -- test_cosmo(Spec(args.spec), args.base_directory) -+ try: -+ # assume spec is passed as a yaml-file -+ with open(args.spec,"r") as f: -+ cosmo_spec = Spec.from_yaml(f) -+ except: -+ # assume spec is passed as a string -+ cosmo_spec = Spec(args.spec) -+ -+ -+ test_cosmo(cosmo_spec, args.base_directory) - - def test_cosmo(cosmo_spec, base_directory): - From 7c06aafabbdde12ef8ff29905495f58a83d61ba7 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:19:36 +0100 Subject: [PATCH 041/104] cosmo: remove tests --- test/system_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 0e220d3d40..7c0de81596 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -15,10 +15,6 @@ def test_install_cosmo_eccodes_definitions_version(version): spack_install(f'cosmo-eccodes-definitions @{version}', test_root=False) -def test_install_cosmo_6_0(): - spack_install('cosmo @6.0 %nvhpc', test_root=False) - - def test_install_eccodes_2_19_0(): spack_install('eccodes @2.19.0', test_root=False) From d99581c4fcd186da8abc532511dc98a49f95b1e5 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:23:00 +0100 Subject: [PATCH 042/104] Fix icon-mch test --- test/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 7c0de81596..97b23ab4b1 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -42,7 +42,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc' + 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc ^eccodes%nvhpc' ) From 5a1ebe83e9f62625e4761843cc777d2670057e83 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 14:48:54 +0100 Subject: [PATCH 043/104] fix icon test --- test/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 97b23ab4b1..e3135ad11b 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -42,7 +42,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc ^eccodes%nvhpc' + 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc ^eccodes%nvhpc ^serialbox%nvhpc' ) From 94826e1ab8dc951e63c9b2976cc25ab3b07a5215 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:03:44 +0100 Subject: [PATCH 044/104] eccodes: remove version 2.19.0 --- repos/c2sm/packages/eccodes/package.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 repos/c2sm/packages/eccodes/package.py diff --git a/repos/c2sm/packages/eccodes/package.py b/repos/c2sm/packages/eccodes/package.py deleted file mode 100644 index f27782f4b8..0000000000 --- a/repos/c2sm/packages/eccodes/package.py +++ /dev/null @@ -1,10 +0,0 @@ -from spack.package import * - -from spack.pkg.builtin.eccodes import Eccodes as SpackEccodes - - -class Eccodes(SpackEccodes): - - version('2.19.0', - sha256= - 'a1d080aed1b17a9d4e3aecccc5a328c057830cd4d54f451f5498b80b24c46404') From 0ce30535e2cacf20722042bde4bb03cdc65e5416 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 15:03:52 +0100 Subject: [PATCH 045/104] Clean up system tests --- test/system_test.py | 84 ++++++++++++++++++++++++--------------------- 1 file changed, 45 insertions(+), 39 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index e3135ad11b..3616c1aebc 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -2,29 +2,29 @@ from spack_commands import spack_install -def test_install_libfyaml_default(): - spack_install('libfyaml', test_root=False) - - -def test_install_libtorch_default(): - spack_install('libtorch', test_root=False) +def test_install_clang_format(): + spack_install('clang-format') @pytest.mark.parametrize("version", ['2.25.0.1', '2.19.0.7']) def test_install_cosmo_eccodes_definitions_version(version): - spack_install(f'cosmo-eccodes-definitions @{version}', test_root=False) + spack_install(f'cosmo-eccodes-definitions @{version}') + + +def test_install_ecbuild(): + spack_install('ecbuild @3.7.2') -def test_install_eccodes_2_19_0(): - spack_install('eccodes @2.19.0', test_root=False) +def test_install_flexpart_cosmo(): + spack_install('flexpart-cosmo') def test_install_flexpart_ifs(): - spack_install('flexpart-ifs', test_root=False) + spack_install('flexpart-ifs') -def test_install_flexpart_cosmo(): - spack_install('flexpart-cosmo @V8C4.0') +#TODO: Add test for icon-c2sm +#TODO: Add test for icon-ham def test_install_icon_mch_2_6_6_mch2b_nvhpc(): @@ -47,54 +47,51 @@ def test_install_icon_mch_conditional_dependencies(): def test_install_icontools(): - spack_install('icontools @2.5.2') + spack_install('icontools') def test_install_int2lm_3_00_nvhpc(): - spack_install('int2lm @int2lm-3.00 %nvhpc', test_root=False) + spack_install('int2lm @int2lm-3.00 %nvhpc') + +def test_install_libfyaml(): + spack_install('libfyaml') -def test_install_libgrib1_22_01_2020_nvhpc(): - spack_install('libgrib1 @22-01-2020 %nvhpc') - -def test_install_makedepf90(): - spack_install('makedepf90 @3.0.1', test_root=False) +def test_install_libgrib1_22_01_2020_nvhpc(): + spack_install('libgrib1 %nvhpc') -def test_install_oasis_version_4_0_nvhpc(): - spack_install('oasis @4.0 %nvhpc') +def test_install_libtorch(): + spack_install('libtorch', test_root=False) -def test_install_pytorch_fortran_version_0_4(): - spack_install( - 'pytorch-fortran@0.4%nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc', - test_root=False) +def test_install_makedepf90(): + spack_install('makedepf90') -def test_install_pytorch_fortran_proxy_version_0_4(): - spack_install('pytorch-fortran-proxy@0.4%gcc ^python@3.10', - test_root=False) +def test_install_oasis_nvhpc(): + spack_install('oasis %nvhpc') -def test_install_py_cytoolz_install_default(): +def test_install_py_cytoolz(): spack_install('py-cytoolz') -def test_install_py_devtools_install_default(): +def test_install_py_devtools(): spack_install('py-devtools') -def test_install_py_factory_boy_install_default(): +def test_install_py_factory_boy(): spack_install('py-factory-boy') -def test_install_py_gridtools_cpp_install_default(): +def test_install_py_gridtools_cpp(): spack_install('py-gridtools-cpp') @pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9']) -def test_install_py_gt4py_for_version(version): +def test_install_py_gt4py_version(version): spack_install(f'py-gt4py @{version}') @@ -102,25 +99,34 @@ def test_install_py_icon4py(): spack_install('py-icon4py') -def test_install_py_hatchling_default(): +def test_install_py_hatchling(): spack_install('py-hatchling') -def test_install_py_inflection_default(): +def test_install_py_inflection(): spack_install('py-inflection') -def test_install_py_pytest_factoryboy_default(): +def test_install_py_pytest_factoryboy(): spack_install('py-pytest-factoryboy') -def test_install_py_tabulate_default(): +def test_install_py_tabulate(): spack_install('py-tabulate') -def test_install_py_typing_extensions_default(): +def test_install_py_typing_extensions(): spack_install('py-typing-extensions') -def test_install_yaxt_default(): +def test_install_pytorch_fortran(): + spack_install( + 'pytorch-fortran %nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc') + + +def test_install_pytorch_fortran_proxy(): + spack_install('pytorch-fortran-proxy %gcc ^python@3.10') + + +def test_install_yaxt(): spack_install('yaxt') From c5706308a60065b8aa57ed692597d7bbdf8066c0 Mon Sep 17 00:00:00 2001 From: github-actions Date: Tue, 12 Nov 2024 14:04:19 +0000 Subject: [PATCH 046/104] GitHub Action: Apply Pep8-formatting --- test/system_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 3616c1aebc..5dc19a53a6 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -52,7 +52,7 @@ def test_install_icontools(): def test_install_int2lm_3_00_nvhpc(): spack_install('int2lm @int2lm-3.00 %nvhpc') - + def test_install_libfyaml(): spack_install('libfyaml') @@ -121,7 +121,8 @@ def test_install_py_typing_extensions(): def test_install_pytorch_fortran(): spack_install( - 'pytorch-fortran %nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc') + 'pytorch-fortran %nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc' + ) def test_install_pytorch_fortran_proxy(): From 59f1b025d491bdb5a280641525fceb6c8dd80b1d Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:32:43 +0100 Subject: [PATCH 047/104] Update uenv --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index a9d08e6ecb..f3865140e4 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ pipeline { ./uenv/install --yes --destdir=$WORKSPACE source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create - uenv image pull mch/v8:rc1 + uenv image pull mch/v8:rc2 """ } } @@ -63,7 +63,7 @@ pipeline { source $WORKSPACE/etc/profile.d/uenv.sh source ./setup-env.sh /user-environment source .venv/bin/activate - uenv run mch/v8:rc1 -- pytest -v -n auto test/integration_test.py + uenv run mch/v8:rc2 -- pytest -v -n auto test/integration_test.py """ } } @@ -73,7 +73,7 @@ pipeline { source $WORKSPACE/etc/profile.d/uenv.sh source ./setup-env.sh /user-environment source .venv/bin/activate - uenv run mch/v8:rc1 -- pytest -v -n auto test/system_test.py + uenv run mch/v8:rc2 -- pytest -v -n auto test/system_test.py """ } } From 1ca68b1a86c957359bc3716c8493809cb87fd744 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:58:24 +0100 Subject: [PATCH 048/104] cosmo-eccodes-definitions: Remove versions 2.19.0.x --- .../cosmo-eccodes-definitions/package.py | 32 ------------------- test/system_test.py | 5 ++- 2 files changed, 2 insertions(+), 35 deletions(-) diff --git a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py index 4179b1e1ca..d398f81a8e 100644 --- a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py +++ b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py @@ -1,25 +1,3 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -# If you submit this package back to Spack as a pull request, -# please first remove this boilerplate and all FIXME comments. -# -# This is a template package file for Spack. We've put "FIXME" -# next to all the things you'll want to change. Once you've handled -# them, you can save this file and test your package like this: -# -# spack install cosmo-eccodes-definitions -# -# You can edit this file again by typing: -# -# spack edit cosmo-eccodes-definitions -# -# See the Spack documentation for more information on packaging. -# ---------------------------------------------------------------------------- - from spack import * @@ -35,22 +13,12 @@ class CosmoEccodesDefinitions(Package): version('2.25.0.3', tag='v2.25.0.3') version('2.25.0.2', tag='v2.25.0.2') version('2.25.0.1', tag='v2.25.0.1') - version('2.19.0.7', tag='v2.19.0.7') - version('2.19.0.6', tag='v2.19.0.6') - version('2.19.0.5', tag='v2.19.0.5') - version('2.19.0.4', tag='v2.19.0.4') - version('2.19.0.3', tag='v2.19.0.3') - version('2.19.0.2', tag='v2.19.0.2') - version('2.19.0.1', tag='v2.19.0.1') version('2.18.0.1', tag='v2.18.0.1') depends_on('eccodes') depends_on('eccodes@2.25.0', type=('build', 'link', 'run'), when='@2.25.0.1:') - depends_on('eccodes@2.19.0', - type=('build', 'link', 'run'), - when='@2.19.0.1:2.19.0.7') depends_on('eccodes@2.18.0', type=('build', 'link', 'run'), when='@2.18.0.1') diff --git a/test/system_test.py b/test/system_test.py index 5dc19a53a6..2a081bebf6 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -6,9 +6,8 @@ def test_install_clang_format(): spack_install('clang-format') -@pytest.mark.parametrize("version", ['2.25.0.1', '2.19.0.7']) -def test_install_cosmo_eccodes_definitions_version(version): - spack_install(f'cosmo-eccodes-definitions @{version}') +def test_install_cosmo_eccodes_definitions(): + spack_install('cosmo-eccodes-definitions') def test_install_ecbuild(): From 9627760fdcb02706cd734b317347b00eff7e0843 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:03:19 +0100 Subject: [PATCH 049/104] icon-mch: Add workarounds --- repos/c2sm/packages/icon-mch/package.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index 2a33a45670..2f4d56b637 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -48,9 +48,27 @@ class IconMch(SpackIcon): when='+eccodes-definitions') with when('+emvorado'): - depends_on('eccodes +fortran') depends_on('hdf5 +szip +hl +fortran') depends_on('zlib') + depends_on('eccodes +fortran') + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('eccodes %nvhpc', when='%nvhpc') + depends_on('eccodes %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + for __x in SpackIcon.serialization_values: + with when("serialization={0}".format(__x)): + depends_on('serialbox %nvhpc', when='%nvhpc') + depends_on('serialbox %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('netcdf-fortran %nvhpc', when='%nvhpc') + depends_on('netcdf-fortran %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + with when('+mpi'): + depends_on('mpi %nvhpc', when='%nvhpc') + depends_on('mpi %gcc', when='%gcc') def configure_args(self): args = super().configure_args() From 3c01785773f0712127483324e125f1463803e605 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:04:05 +0100 Subject: [PATCH 050/104] icon-c2sm: Inherit from icon --- repos/c2sm/packages/icon-c2sm/package.py | 465 ++--------------------- 1 file changed, 28 insertions(+), 437 deletions(-) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon-c2sm/package.py index fe4de03afa..fbebeb1a96 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -1,9 +1,10 @@ -import os, subprocess, glob, re +from spack.pkg.builtin.icon import Icon as SpackIcon +import os, glob, re from collections import defaultdict -from llnl.util import lang, filesystem, tty -from spack.util.environment import is_system_path, dump_environment -from spack.util.executable import which_string, which +from llnl.util import tty +from spack.util.environment import is_system_path +from spack.util.executable import which import spack.error as error @@ -37,14 +38,10 @@ def check_variant_extra_config_args(extra_config_arg): return False -class IconC2sm(AutotoolsPackage, CudaPackage): - """Icosahedral Nonhydrostatic Weather and Climate Model.""" - - homepage = "https://www.icon-model.org" - url = "https://gitlab.dkrz.de/icon/icon-model/-/archive/icon-2024.01-public/icon-model-icon-2024.01-public.tar.gz" +class IconC2sm(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon.git' - maintainers = ['jonasjucker', 'dominichofer'] + maintainers('jonasjucker', 'dominichofer') version('develop', submodules=True) version("2024.01-1", tag="icon-2024.01-1", submodules=True) @@ -65,45 +62,19 @@ class IconC2sm(AutotoolsPackage, CudaPackage): # https://gitlab.dkrz.de/icon/icon/-/blob/icon-2024.01/configure?ref_type=tags#L1492-1638 # Model Features: - variant('atmo', - default=True, - description='Enable the atmosphere component') - variant('les', - default=True, - description='Enable the Large-Eddy Simulation component') - variant('upatmo', - default=True, - description='Enable the upper atmosphere component') - variant('ocean', default=True, description='Enable the ocean component') - variant('jsbach', default=True, description='Enable the land component') - variant('waves', - default=False, - description='Enable the surface wave component') - variant('coupling', default=True, description='Enable the coupling') - variant('aes', default=True, description='Enable the AES physics package') - variant('nwp', default=True, description='Enable the NWP physics package') - variant('ecrad', - default=False, - description='Enable usage of the ECMWF radiation scheme') - variant('rte-rrtmgp', - default=True, - description='Enable usage of the RTE+RRTMGP toolbox ' - 'for radiation calculations') variant('dace', default=False, description='Enable the DACE modules for data assimilation') + requires("+mpi", when="+dace") + variant('emvorado', default=False, description='Enable the radar forward operator EMVORADO') - variant('art', - default=False, - description='Enable the aerosols and reactive trace component ART') + requires("+mpi", when="+emvorado") + variant('art-gpl', default=False, description='Enable GPL-licensed code parts of the ART component') - variant('comin', - default=False, - description='Enable the ICON community interfaces') variant( 'acm-license', default=False, @@ -111,9 +82,6 @@ class IconC2sm(AutotoolsPackage, CudaPackage): 'Enable code parts that require accepting the ACM Software License') # Infrastructural Features: - variant('mpi', - default=True, - description='Enable MPI (parallelization) support') variant( 'active-target-sync', default=False, @@ -123,29 +91,10 @@ class IconC2sm(AutotoolsPackage, CudaPackage): variant('async-io-rma', default=True, description='Enable remote memory access (RMA) for async I/O') - variant('mpi-gpu', - default=False, - description='Enable usage of the GPU-aware MPI features') - variant('openmp', default=False, description='Enable OpenMP support') - variant('gpu', - default='no', - values=('openacc+cuda', 'no'), - description='Enable GPU support') variant('realloc-buf', default=False, description='Enable reallocatable communication buffer') - variant('grib2', default=False, description='Enable GRIB2 I/O') - variant('parallel-netcdf', - default=False, - description='Enable usage of the parallel features of NetCDF') variant('sct', default=False, description='Enable the SCT timer') - variant('yaxt', default=False, description='Enable the YAXT data exchange') - - serialization_values = ('read', 'perturb', 'create') - variant('serialization', - default='none', - values=('none', ) + serialization_values, - description='Enable the Serialbox2 serialization') variant('testbed', default=False, description='Enable ICON Testbed infrastructure') @@ -158,26 +107,22 @@ class IconC2sm(AutotoolsPackage, CudaPackage): description= 'Inject any configure argument not yet available as variant\nUse this feature cautiously, as injecting non-variant configure arguments may potentially disrupt the build process' ) - variant('comin', - default=False, - description='Enable usage of ComIn toolbox ' - 'for building plugins.') # Optimization Features: variant('loop-exchange', default=True, description='Enable loop exchange') variant('vectorized-lrtm', default=False, description='Enable the parallelization-invariant version of LRTM') - variant('mixed-precision', - default=False, - description='Enable mixed precision dycore') variant( 'pgi-inlib', default=False, description= 'Enable PGI/NVIDIA cross-file function inlining via an inline library') variant('nccl', default=False, description='Enable NCCL for communication') + variant('cuda-graphs', default=False, description='Enable CUDA graphs.') + requires('%nvhpc@23.3:', when='+cuda-graphs') + variant( 'fcgroup', default='none', @@ -191,14 +136,12 @@ class IconC2sm(AutotoolsPackage, CudaPackage): variant('silent-rules', default=True, description='Enable silent-rules for build-process') - variant( 'pytorch', default=False, description= 'Build with pytorch for inference with machine-learning models. Experimental, needs non-standard codebase!' ) - variant( 'eccodes-definitions', default=False, @@ -220,61 +163,18 @@ class IconC2sm(AutotoolsPackage, CudaPackage): depends_on('boost', when='dsl={0}'.format(x)) conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) - depends_on('pytorch-fortran', when='+pytorch') - - depends_on('libfyaml', when='+coupling') - depends_on('libxml2', when='+art') - - for x in serialization_values: - depends_on('serialbox+fortran', when='serialization={0}'.format(x)) - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('serialbox %nvhpc', when='%nvhpc') - depends_on('serialbox %gcc', when='%gcc') - - depends_on('eccodes +fortran', when='+emvorado') - depends_on('eccodes', when='+grib2') depends_on('cosmo-eccodes-definitions', - type=('build', 'run'), + type='run', when='+eccodes-definitions') - depends_on('lapack') - depends_on('blas') - depends_on('netcdf-fortran') + with when('+emvorado'): + depends_on('eccodes +fortran') + depends_on('hdf5 +szip +hl +fortran') + depends_on('zlib') - depends_on('netcdf-c') - depends_on('netcdf-c+mpi', when='+parallel-netcdf') - - depends_on('hdf5 +szip +hl +fortran', when='+emvorado') + depends_on('pytorch-fortran', when='+pytorch') depends_on('hdf5 +szip', when='+sct') - depends_on('zlib', when='+emvorado') - depends_on('mpi', when='+mpi') - - depends_on('python', type='build') - depends_on('perl', type='build') - depends_on('cmake@3.18:', type='build') - - conflicts('+dace', when='~mpi') - conflicts('+emvorado', when='~mpi') - conflicts('+cuda', when='%gcc') - - # The gpu=openacc+cuda relies on the cuda variant - conflicts('~cuda', when='gpu=openacc+cuda') - conflicts('+cuda', when='gpu=no') - - conflicts('+cuda-graphs', when='%cce') - conflicts('+cuda-graphs', when='%gcc') - conflicts('+cuda-graphs', when='%intel') - conflicts('+cuda-graphs', when='%pgi') - conflicts('+cuda-graphs', when='%nvhpc@:23.2') - - conflicts('+loop-exchange', when='gpu=openacc+cuda') - - # Flag to mark if we build out-of-source - # Needed to trigger sync of input files for experiments - out_of_source_build = False - out_of_source_configure_directory = '' - # patch_libtool is a function from Autotoolspackage. # For BB we cannot use it because it finds all files # named "libtool". spack-c2sm is cloned into icon-repo, @@ -290,217 +190,39 @@ def setup_build_environment(self, env): env.set("BOOST_ROOT", self.spec['boost'].prefix) def configure_args(self): - args = ['--disable-rpaths'] - flags = defaultdict(list) + args = super().configure_args() + super_libs = args.pop() + libs = LibraryList([]) + flags = defaultdict(list) for x in [ - 'atmo', - 'les', - 'upatmo', - 'ocean', - 'jsbach', - 'waves', - 'coupling', - 'aes', - 'nwp', - 'ecrad', - 'rte-rrtmgp', 'dace', 'emvorado', - 'art', 'art-gpl', - 'comin', 'acm-license', - 'mpi', 'active-target-sync', 'async-io-rma', - 'mpi-gpu', - 'openmp', 'realloc-buf', - 'grib2', 'parallel-netcdf', 'sct', - 'yaxt', 'testbed', 'loop-exchange', 'vectorized-lrtm', - 'mixed-precision', 'pgi-inlib', 'nccl', 'cuda-graphs', 'silent-rules', - 'comin', ]: args += self.enable_or_disable(x) - if self.compiler.name == 'gcc': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - flags['ICON_BUNDLED_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-g', - '-fmodule-private', - '-fimplicit-none', - '-fmax-identifier-length=63', - '-Wall', - '-Wcharacter-truncation', - '-Wconversion', - '-Wunderflow', - '-Wunused-parameter', - '-Wno-surprising', - '-fall-intrinsics', - ]) - flags['ICON_FCFLAGS'].extend([ - '-O2', '-fbacktrace', '-fbounds-check', - '-fstack-protector-all', '-finit-real=nan', - '-finit-integer=-2147483648', '-finit-character=127' - ]) - flags['ICON_OCEAN_FCFLAGS'].append('-O3') - - # Version-specific workarounds: - fc_version = self.compiler.version - if fc_version >= ver(10): - flags['ICON_FCFLAGS'].append('-fallow-argument-mismatch') - flags['ICON_OCEAN_FCFLAGS'].append('-fallow-argument-mismatch') - if '+ecrad' in self.spec: - # For externals/ecrad/ifsaux/random_numbers_mix.F90: - flags['ICON_ECRAD_FCFLAGS'].append('-fallow-invalid-boz') - elif self.compiler.name == 'intel': - flags['CFLAGS'].extend( - ['-g', '-gdwarf-4', '-O3', '-qno-opt-dynamic-align', '-ftz']) - flags['FCFLAGS'].extend( - ['-g', '-gdwarf-4', '-traceback', '-fp-model source']) - flags['ICON_FCFLAGS'].extend( - ['-O2', '-assume realloc_lhs', '-ftz']) - flags['ICON_OCEAN_FCFLAGS'].extend([ - '-O3', '-assume norealloc_lhs', '-reentrancy threaded', - '-qopt-report-file=stdout', '-qopt-report=0', - '-qopt-report-phase=vec' - ]) - args.append('--enable-intel-consistency') - elif self.compiler.name == 'nag': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - flags['ICON_BUNDLED_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-g', '-Wc,-g', '-O0', '-colour', '-f2008', '-w=uep', - '-float-store', '-nan' - ]) - if '~openmp' in self.spec: - # The -openmp option is incompatible with the -gline option: - flags['FCFLAGS'].append('-gline') - flags['ICON_FCFLAGS'].extend([ - '-Wc,-pipe', - '-Wc,--param,max-vartrack-size=200000000', - '-Wc,-mno-fma', - # Spack compiler wrapper (see the respective compilers.yaml) - # injects '-mismatch', which is incompatible with '-C=calls' - # Therefore, we specify the following flags instead of a single - # '-C=all', which implies '-C=calls'. - '-C=alias', - '-C=array', - '-C=bits', - '-C=dangling', - '-C=do', - '-C=intovf', - '-C=present', - '-C=pointer', - '-C=recursion' - ]) - flags['ICON_BUNDLED_FCFLAGS'] = [] - elif self.compiler.name in ['pgi', 'nvhpc']: - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend( - ['-g', '-O', '-Mrecursive', '-Mallocatable=03', '-Mbackslash']) - - if self.spec.variants['gpu'].value == 'openacc+cuda': - flags['FCFLAGS'].extend([ - '-acc=verystrict', '-Minfo=accel,inline', - '-gpu=cc{0}'.format( - self.spec.variants['cuda_arch'].value[0]) - ]) - elif self.compiler.name == 'cce': - flags['CFLAGS'].append('-g') - flags['ICON_CFLAGS'].append('-O3') - if self.spec.satisfies('%cce@13.0.0+coupling'): - # For externals/yac/tests/test_interpolation_method_conserv.c: - flags['ICON_YAC_CFLAGS'].append('-O2') - flags['FCFLAGS'].extend([ - '-hadd_paren', '-r am', '-Ktrap=divz,ovf,inv', - '-hflex_mp=intolerant', '-hfp0', '-O0' - ]) - if self.spec.variants['gpu'].value == 'openacc+cuda': - flags['FCFLAGS'].extend(['-hacc']) - elif self.compiler.name == 'aocc': - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend(['-g', '-O2']) - if self.spec.satisfies('+yaxt'): - # Enable the PGI/Cray (NO_2D_PARAM) workaround for the test - # suite of the bundled YAXT (apply also when not self.run_tests - # to make sure we get identical installations): - flags['ICON_YAXT_FCFLAGS'].append('-DNO_2D_PARAM') - else: - flags['CFLAGS'].extend(['-g', '-O2']) - flags['FCFLAGS'].extend(['-g', '-O2']) - - if '+coupling' in self.spec or '+art' in self.spec: - xml2_spec = self.spec['libxml2'] - libs += xml2_spec.libs - # Account for the case when libxml2 is an external package installed - # to a system directory, which means that Spack will not inject the - # required -I flag with the compiler wrapper: - if is_system_path(xml2_spec.prefix): - xml2_headers = xml2_spec.headers - # We, however, should filter the pure system directories out: - xml2_headers.directories = [ - d for d in xml2_headers.directories - if not is_system_path(d) - ] - flags['CPPFLAGS'].append(xml2_headers.include_flags) - - if '+coupling' in self.spec: - libs += self.spec['libfyaml'].libs - - serialization = self.spec.variants['serialization'].value - if serialization == 'none': - args.append('--disable-serialization') - else: - args.extend([ - '--enable-serialization={0}'.format(serialization), - 'SB2PP={0}'.format(self.spec['serialbox'].pp_ser) - ]) - libs += self.spec['serialbox:fortran'].libs - if '+emvorado' in self.spec: libs += self.spec['eccodes:fortran'].libs - - if '+grib2' in self.spec: - libs += self.spec['eccodes:c'].libs - - libs += self.spec['lapack:fortran'].libs - libs += self.spec['blas:fortran'].libs - libs += self.spec['netcdf-fortran'].libs - - if '+coupling' in self.spec: - libs += self.spec['netcdf-c'].libs - - if '+emvorado' in self.spec: libs += self.spec['hdf5:fortran,hl'].libs - elif '+sct' in self.spec: - libs += self.spec['hdf5'].libs - - if '+emvorado' in self.spec: libs += self.spec['zlib'].libs - if '+mpi' in self.spec: - args.extend([ - 'CC=' + self.spec['mpi'].mpicc, - 'FC=' + self.spec['mpi'].mpifc, - # We cannot provide a universal value for MPI_LAUNCH, therefore - # we have to disable the MPI checks: - '--disable-mpi-checks' - ]) + if '+sct' in self.spec: + libs += self.spec['hdf5'].libs if '+pytorch' in self.spec: libs += self.spec['pytorch-fortran'].libs @@ -511,32 +233,6 @@ def configure_args(self): args.extend(self.fcgroup_to_config_arg()) flags.update(self.fcgroup_to_config_var()) - gpu = self.spec.variants['gpu'].value - if gpu == 'no': - args.append('--disable-gpu') - else: - args.extend([ - '--enable-gpu={0}'.format(gpu), - 'NVCC={0}'.format(self.spec['cuda'].prefix.bin.nvcc) - ]) - - # -cuda is an umbrella flag for NVHPC - # that pulls in all the needed CUDA libraries and wrappers - # It can't go into libs as spack prepends -l - flags['LDFLAGS'].append('-cuda') - - cuda_host_compiler = self.compiler.cxx - cuda_host_compiler_stdcxx_libs = self.compiler.stdcxx_libs - - flags['NVCFLAGS'].extend([ - '-g', '-O3', - '-arch=sm_{0}'.format(self.spec.variants['cuda_arch'].value[0]) - ]) - # cuda_host_compiler_stdcxx_libs might contain compiler-specific - # flags (i.e. not the linker -l flags), therefore we put - # the value to the config_flags directly. - flags['LIBS'].extend(cuda_host_compiler_stdcxx_libs) - # Check for DSL variants and set corresponding Liskov options dsl = self.spec.variants['dsl'].value if dsl != ('none', ): @@ -610,10 +306,8 @@ def configure_args(self): if not is_system_path(d) ]) - args.extend([ - '{0}={1}'.format(var, ' '.join(val)) for var, val in flags.items() - ]) - + args.extend(["{0}={1}".format(name, " ".join(value)) for name, value in flags.items()]) + args.append(f"{super_libs} {libs.link_flags}") return args def fcgroup_to_config_arg(self): @@ -684,109 +378,6 @@ def checksuite(self): else: tty.warn('Cannot find test.py -> skipping tests') - @property - def archive_files(self): - # Archive files that are normally archived for AutotoolsPackage: - archive = list(super(Icon, self).archive_files) - # Archive makefiles: - archive.extend( - [join_path(self.build_directory, f) for f in ['Makefile', '*.mk']]) - return archive - - @property - def build_directory(self): - """Overrides function from spack.build_system.autotools - - By default build_directory is identical as configure_directory - To enable out-of-source builds this is not the case anymore - """ - - return self.stage.source_path - - @property - def configure_directory(self): - """Returns the directory where 'configure' resides. - - Overides function from spack.build_systems.autotools - - """ - - source_path = self.build_directory - - # dev_path is indicator for dev-build or develop - # only case when out-of-source build are possible - if "dev_path" in self.spec.variants: - Git = which('git', required=True) - git_root = Git('rev-parse', - '--show-toplevel', - output=str, - fail_on_error=True).replace("\n", "") - if git_root != source_path: - # mark out-of-source build for function - # copy_runscript_related_input_files - self.out_of_source_build = True - self.out_of_source_configure_directory = git_root - return git_root - - return source_path - - @run_before('configure') - def report_out_of_source_directories(self): - if self.out_of_source_build: - tty.info(f'build-directory: {self.build_directory}') - tty.info( - f'configure-directory: {self.out_of_source_configure_directory}' - ) - - def configure(self, spec, prefix): - if os.path.exists( - os.path.join(self.build_directory, - 'icon.mk')) and self.build_uses_same_spec(): - tty.warn( - 'icon.mk already present -> skip configure stage', - '\t delete "icon.mk" or run "make distclean" to not skip configure' - ) - return - - # Call configure of Autotools - super().configure(spec, prefix) - - def build_uses_same_spec(self): - """ - Ensure that configure is rerun in case spec has changed, - otherwise for the case below - - $ spack dev-build icon @develop ~dace - $ spack dev-build icon @develop +dace - - configure is skipped for the latter. - """ - - is_same_spec = False - - previous_spec = os.path.join(self.build_directory, - '.previous_spec.yaml') - - # not the first build in self.build_directory - if os.path.exists(previous_spec): - with open(previous_spec, mode='r') as f: - if self.spec == Spec.from_yaml(f): - is_same_spec = True - else: - is_same_spec = False - tty.warn( - 'Cannot skip configure phase because spec changed') - - # first build in self.build_directory, no worries - else: - is_same_spec = True - - # dump spec of new build - with open(previous_spec, mode='w') as f: - f.write(self.spec.to_yaml()) - - return is_same_spec - @run_after('configure') def copy_runscript_related_input_files(self): if self.out_of_source_build: From 76bfb04c8ca3f5768a27c6c3beb619cbf26d1354 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 12 Nov 2024 17:04:19 +0100 Subject: [PATCH 051/104] Improve tests --- test/system_test.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 2a081bebf6..5ae95c9677 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -11,7 +11,15 @@ def test_install_cosmo_eccodes_definitions(): def test_install_ecbuild(): - spack_install('ecbuild @3.7.2') + # Tests are disabled because they fail: + # The following tests FAILED: + # 1 - ECBUILD-359 (Failed) + # 2 - ECBUILD-401 (Failed) + # 8 - ECBUILD-511 (Failed) + # 11 - bundle-subdir-std (Failed) + # 12 - bundle-subdir-ecbfind (Failed) + # 17 - test_ecbuild_find_package (Failed) + spack_install('ecbuild @3.7.2', test_root=False) def test_install_flexpart_cosmo(): @@ -22,14 +30,17 @@ def test_install_flexpart_ifs(): spack_install('flexpart-ifs') -#TODO: Add test for icon-c2sm +@pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) +def test_install_icon_c2sm(compiler): + spack_install(f'icon-c2sm %{compiler} ^cray-mpich%{compiler} ^netcdf-fortran%{compiler} ^eccodes%{compiler} ^serialbox%{compiler}') + + #TODO: Add test for icon-ham -def test_install_icon_mch_2_6_6_mch2b_nvhpc(): - spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc' - ) +@pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) +def test_install_icon_mch(compiler): + spack_install(f'icon-mch %{compiler}') def test_install_icon_mch_conditional_dependencies(): @@ -41,7 +52,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch @icon-2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc ^netcdf-fortran%nvhpc ^eccodes%nvhpc ^serialbox%nvhpc' + 'icon-mch %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' ) @@ -62,7 +73,7 @@ def test_install_libgrib1_22_01_2020_nvhpc(): def test_install_libtorch(): - spack_install('libtorch', test_root=False) + spack_install('libtorch') def test_install_makedepf90(): From e4d3aacc9b8f91be24932179cd803a2d9268e3c9 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:32:27 +0100 Subject: [PATCH 052/104] Disable failing package tests. --- test/system_test.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 5ae95c9677..af7937a1b5 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -11,7 +11,7 @@ def test_install_cosmo_eccodes_definitions(): def test_install_ecbuild(): - # Tests are disabled because they fail: + # Tests are disabled because they fail with: # The following tests FAILED: # 1 - ECBUILD-359 (Failed) # 2 - ECBUILD-401 (Failed) @@ -61,7 +61,10 @@ def test_install_icontools(): def test_install_int2lm_3_00_nvhpc(): - spack_install('int2lm @int2lm-3.00 %nvhpc') + # Tests are disabled because they fail with: + # Error: cmake is a duplicate dependency, with conflicting dependency types + # which stems from the package's 'test_int2lm.py'. + spack_install('int2lm @int2lm-3.00 %nvhpc', test_root=False) def test_install_libfyaml(): @@ -77,7 +80,9 @@ def test_install_libtorch(): def test_install_makedepf90(): - spack_install('makedepf90') + # Tests are disabled because they fail with: + # test1.sh: No such file or directory + spack_install('makedepf90', test_root=False) def test_install_oasis_nvhpc(): From 7b68f2ee4b1dfa0a485547d6a94860fbe0733e1f Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 13 Nov 2024 16:32:47 +0100 Subject: [PATCH 053/104] Move mpi workaround --- repos/c2sm/packages/icon-mch/package.py | 5 ----- test/system_test.py | 4 ++-- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index 2f4d56b637..9ef8b50338 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -65,11 +65,6 @@ class IconMch(SpackIcon): depends_on('netcdf-fortran %nvhpc', when='%nvhpc') depends_on('netcdf-fortran %gcc', when='%gcc') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - with when('+mpi'): - depends_on('mpi %nvhpc', when='%nvhpc') - depends_on('mpi %gcc', when='%gcc') - def configure_args(self): args = super().configure_args() super_libs = args.pop() diff --git a/test/system_test.py b/test/system_test.py index af7937a1b5..19c5e9e8b8 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -40,7 +40,7 @@ def test_install_icon_c2sm(compiler): @pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) def test_install_icon_mch(compiler): - spack_install(f'icon-mch %{compiler}') + spack_install(f'icon-mch %{compiler} ^cray-mpich%{compiler}') def test_install_icon_mch_conditional_dependencies(): @@ -52,7 +52,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80' + 'icon-mch %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' ) From ca88007aac44bd38d61b65d100b93e81701852fd Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 13 Nov 2024 15:33:30 +0000 Subject: [PATCH 054/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon-c2sm/package.py | 5 ++++- repos/c2sm/packages/icon-mch/package.py | 2 +- test/system_test.py | 4 +++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon-c2sm/package.py index fbebeb1a96..3bcf541b11 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -306,7 +306,10 @@ def configure_args(self): if not is_system_path(d) ]) - args.extend(["{0}={1}".format(name, " ".join(value)) for name, value in flags.items()]) + args.extend([ + "{0}={1}".format(name, " ".join(value)) + for name, value in flags.items() + ]) args.append(f"{super_libs} {libs.link_flags}") return args diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index 9ef8b50338..ea7faca210 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -54,7 +54,7 @@ class IconMch(SpackIcon): # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. depends_on('eccodes %nvhpc', when='%nvhpc') depends_on('eccodes %gcc', when='%gcc') - + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. for __x in SpackIcon.serialization_values: with when("serialization={0}".format(__x)): diff --git a/test/system_test.py b/test/system_test.py index 19c5e9e8b8..b3f6d1f618 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -32,7 +32,9 @@ def test_install_flexpart_ifs(): @pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) def test_install_icon_c2sm(compiler): - spack_install(f'icon-c2sm %{compiler} ^cray-mpich%{compiler} ^netcdf-fortran%{compiler} ^eccodes%{compiler} ^serialbox%{compiler}') + spack_install( + f'icon-c2sm %{compiler} ^cray-mpich%{compiler} ^netcdf-fortran%{compiler} ^eccodes%{compiler} ^serialbox%{compiler}' + ) #TODO: Add test for icon-ham From 7376c3c1bd9644cdfabf8c9e2147fe4a3b01e656 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:12:42 +0100 Subject: [PATCH 055/104] icon-mch: Remove gcc test and add version tests --- repos/c2sm/packages/icon-mch/package.py | 6 +++--- test/system_test.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index ea7faca210..20aa89059b 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -6,9 +6,9 @@ class IconMch(SpackIcon): maintainers("dominichofer") - version('master', submodules=True) - version('icon-2.6.6-mch2b', submodules=True) - version('icon-2.6.6-mch2a', submodules=True) + version('master', branch='master', submodules=True) + version('2.6.6-mch2b', tag='icon-2.6.6-mch2b', submodules=True) + version('2.6.6-mch2a', tag='icon-2.6.6-mch2a', submodules=True) # Model Features: variant('dace', diff --git a/test/system_test.py b/test/system_test.py index b3f6d1f618..17001427c1 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -40,9 +40,9 @@ def test_install_icon_c2sm(compiler): #TODO: Add test for icon-ham -@pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) -def test_install_icon_mch(compiler): - spack_install(f'icon-mch %{compiler} ^cray-mpich%{compiler}') +@pytest.mark.parametrize('version', ['2.6.6-mch2a', '2.6.6-mch2b']) +def test_install_icon_mch(version): + spack_install(f'icon-mch @{version} %nvhpc ^cray-mpich%nvhpc') def test_install_icon_mch_conditional_dependencies(): @@ -54,7 +54,7 @@ def test_install_icon_mch_conditional_dependencies(): # gpu=nvidia-80 triggers cuda spack_install( - 'icon-mch %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' + 'icon-mch @2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' ) From cdc5b67945f71747838bc52401f6a79a23d46ca2 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:13:10 +0100 Subject: [PATCH 056/104] icon-c2sm: Remove copy_runscript_related_input_files because there's no self.out_of_source_build --- repos/c2sm/packages/icon-c2sm/package.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon-c2sm/package.py index 3bcf541b11..5063b79704 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -380,24 +380,3 @@ def checksuite(self): os.environ['PYTHONHOME'] = PYTHONHOME else: tty.warn('Cannot find test.py -> skipping tests') - - @run_after('configure') - def copy_runscript_related_input_files(self): - if self.out_of_source_build: - with working_dir(self.build_directory): - Rsync = which('rsync', required=True) - icon_dir = self.configure_directory - Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", - "--exclude=.*", "--exclude=standard_*") - Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", - "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", - "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", - "--exclude=*.mod", "--exclude=*.o") - Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") - - Ln = which('ln', required=True) - dirs = glob.glob(f"{icon_dir}/run/standard_*") - for dir in dirs: - Ln("-sf", "-t", "run/", f"{dir}") - Ln("-sf", f"{icon_dir}/data") - Ln("-sf", f"{icon_dir}/vertical_coord_tables") From c0f83dc67e0b9be972bbe0d5dfbb96a689209320 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Mon, 25 Nov 2024 18:20:49 +0100 Subject: [PATCH 057/104] icon-mch: fix tags --- repos/c2sm/packages/icon-mch/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index 20aa89059b..545b31a39e 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -7,8 +7,8 @@ class IconMch(SpackIcon): maintainers("dominichofer") version('master', branch='master', submodules=True) - version('2.6.6-mch2b', tag='icon-2.6.6-mch2b', submodules=True) - version('2.6.6-mch2a', tag='icon-2.6.6-mch2a', submodules=True) + version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) + version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) # Model Features: variant('dace', From 789844471e7010ebb7ea07c6c7ea760aa0664489 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Tue, 26 Nov 2024 17:13:47 +0100 Subject: [PATCH 058/104] icon: Set new maintainer --- repos/c2sm/packages/icon-c2sm/package.py | 2 +- repos/c2sm/packages/icon-mch/package.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon-c2sm/package.py index 5063b79704..212ada81c2 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon-c2sm/package.py @@ -41,7 +41,7 @@ def check_variant_extra_config_args(extra_config_arg): class IconC2sm(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon.git' - maintainers('jonasjucker', 'dominichofer') + maintainers('jonasjucker', 'huppd') version('develop', submodules=True) version("2024.01-1", tag="icon-2024.01-1", submodules=True) diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py index 545b31a39e..025d3b194d 100644 --- a/repos/c2sm/packages/icon-mch/package.py +++ b/repos/c2sm/packages/icon-mch/package.py @@ -4,7 +4,7 @@ class IconMch(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' - maintainers("dominichofer") + maintainers("huppd") version('master', branch='master', submodules=True) version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) From 83da9477e3df85913eea8be860ab984b22e19c12 Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:43:35 +0100 Subject: [PATCH 059/104] Rename icon-c2sm to icon --- .../packages/{icon-c2sm => icon}/package.py | 22 ++++++++++--- test/system_test.py | 32 +++++++++++-------- 2 files changed, 36 insertions(+), 18 deletions(-) rename repos/c2sm/packages/{icon-c2sm => icon}/package.py (93%) diff --git a/repos/c2sm/packages/icon-c2sm/package.py b/repos/c2sm/packages/icon/package.py similarity index 93% rename from repos/c2sm/packages/icon-c2sm/package.py rename to repos/c2sm/packages/icon/package.py index 212ada81c2..730cea2e96 100755 --- a/repos/c2sm/packages/icon-c2sm/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -1,10 +1,9 @@ from spack.pkg.builtin.icon import Icon as SpackIcon -import os, glob, re +import os +import re from collections import defaultdict - from llnl.util import tty from spack.util.environment import is_system_path -from spack.util.executable import which import spack.error as error @@ -38,13 +37,15 @@ def check_variant_extra_config_args(extra_config_arg): return False -class IconC2sm(SpackIcon): +class Icon(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon.git' maintainers('jonasjucker', 'huppd') version('develop', submodules=True) version("2024.01-1", tag="icon-2024.01-1", submodules=True) + version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) + version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) version('exclaim-master', branch='master', git='git@github.com:C2SM/icon-exclaim.git', @@ -171,6 +172,19 @@ class IconC2sm(SpackIcon): depends_on('eccodes +fortran') depends_on('hdf5 +szip +hl +fortran') depends_on('zlib') + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('eccodes %nvhpc', when='%nvhpc') + depends_on('eccodes %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + for __x in SpackIcon.serialization_values: + with when("serialization={0}".format(__x)): + depends_on('serialbox %nvhpc', when='%nvhpc') + depends_on('serialbox %gcc', when='%gcc') + + # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. + depends_on('netcdf-fortran %nvhpc', when='%nvhpc') + depends_on('netcdf-fortran %gcc', when='%gcc') depends_on('pytorch-fortran', when='+pytorch') depends_on('hdf5 +szip', when='+sct') diff --git a/test/system_test.py b/test/system_test.py index 17001427c1..a53502f624 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -30,22 +30,15 @@ def test_install_flexpart_ifs(): spack_install('flexpart-ifs') -@pytest.mark.parametrize("compiler", ['gcc', 'nvhpc']) -def test_install_icon_c2sm(compiler): +@pytest.mark.parametrize('version', ['2024.01-1', '2.6.6-mch2a', '2.6.6-mch2b']) +def test_install_icon(version): + # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install( - f'icon-c2sm %{compiler} ^cray-mpich%{compiler} ^netcdf-fortran%{compiler} ^eccodes%{compiler} ^serialbox%{compiler}' + f'icon @{version} %nvhpc ^cray-mpich%nvhpc' ) -#TODO: Add test for icon-ham - - -@pytest.mark.parametrize('version', ['2.6.6-mch2a', '2.6.6-mch2b']) -def test_install_icon_mch(version): - spack_install(f'icon-mch @{version} %nvhpc ^cray-mpich%nvhpc') - - -def test_install_icon_mch_conditional_dependencies(): +def test_install_icon_conditional_dependencies(): # +coupling triggers libfyaml, libxml2, netcdf-c # serialization=create triggers serialbox # +emvorado triggers eccodes, hdf5, zlib @@ -53,11 +46,22 @@ def test_install_icon_mch_conditional_dependencies(): # +mpi triggers mpi # gpu=nvidia-80 triggers cuda + # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. spack_install( - 'icon-mch @2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' + 'icon @2.6.6-mch2b %nvhpc +coupling serialization=create +emvorado +mpi gpu=nvidia-80 ^cray-mpich%nvhpc' ) +def test_install_icon_ham(version): + spack_install('icon-ham') + + +@pytest.mark.parametrize('version', ['2.6.6-mch2a', '2.6.6-mch2b']) +def test_install_icon_mch(version): + # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. + spack_install(f'icon-mch @{version} %nvhpc ^cray-mpich%nvhpc') + + def test_install_icontools(): spack_install('icontools') @@ -73,7 +77,7 @@ def test_install_libfyaml(): spack_install('libfyaml') -def test_install_libgrib1_22_01_2020_nvhpc(): +def test_install_libgrib1_nvhpc(): spack_install('libgrib1 %nvhpc') From c58351d3f9afdee2b1482ed76ff53c934a95a09c Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 27 Nov 2024 09:50:02 +0100 Subject: [PATCH 060/104] Use require --- repos/c2sm/packages/flexpart-cosmo/package.py | 10 +++------- repos/c2sm/packages/flexpart-ifs/package.py | 8 +++----- repos/c2sm/packages/pytorch-fortran-proxy/package.py | 2 ++ test/system_test.py | 6 ++---- 4 files changed, 10 insertions(+), 16 deletions(-) diff --git a/repos/c2sm/packages/flexpart-cosmo/package.py b/repos/c2sm/packages/flexpart-cosmo/package.py index ae09f08455..5b380ed7ba 100644 --- a/repos/c2sm/packages/flexpart-cosmo/package.py +++ b/repos/c2sm/packages/flexpart-cosmo/package.py @@ -19,15 +19,11 @@ class FlexpartCosmo(MakefilePackage): version('main', branch='main') depends_on('eccodes +fortran') - depends_on( - 'netcdf-fortran %gcc' - ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-fortran %gcc') depends_on('makedepf90') - conflicts('%gcc@:10') - conflicts('%nvhpc') - conflicts('%pgi') - conflicts('%cce') + requires('%gcc@11:') build_directory = 'src' diff --git a/repos/c2sm/packages/flexpart-ifs/package.py b/repos/c2sm/packages/flexpart-ifs/package.py index 637aa97bc4..2eea6ddb43 100644 --- a/repos/c2sm/packages/flexpart-ifs/package.py +++ b/repos/c2sm/packages/flexpart-ifs/package.py @@ -13,12 +13,10 @@ class FlexpartIfs(MakefilePackage): version('10.4.4', tag='10.4.4') depends_on('eccodes +fortran') - depends_on( - 'netcdf-fortran %gcc' - ) # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + # WORKAROUND: '%gcc' should not be necessary, but without it, spack concretizes to nvhpc. + depends_on('netcdf-fortran %gcc') - conflicts('%nvhpc') - conflicts('%pgi') + requires('%gcc') build_directory = 'src' diff --git a/repos/c2sm/packages/pytorch-fortran-proxy/package.py b/repos/c2sm/packages/pytorch-fortran-proxy/package.py index 15f0292f88..248389407a 100644 --- a/repos/c2sm/packages/pytorch-fortran-proxy/package.py +++ b/repos/c2sm/packages/pytorch-fortran-proxy/package.py @@ -27,6 +27,8 @@ class PytorchFortranProxy(CMakePackage): depends_on('libtorch') depends_on('py-pybind11') + requires('%nvhpc') + root_cmakelists_dir = 'src/proxy_lib' def cmake_args(self): diff --git a/test/system_test.py b/test/system_test.py index a53502f624..e705518e00 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -141,13 +141,11 @@ def test_install_py_typing_extensions(): def test_install_pytorch_fortran(): - spack_install( - 'pytorch-fortran %nvhpc ^pytorch-fortran-proxy@0.4%gcc ^python@3.10 ^gmake%gcc ^cmake%gcc' - ) + spack_install('pytorch-fortran %nvhpc') def test_install_pytorch_fortran_proxy(): - spack_install('pytorch-fortran-proxy %gcc ^python@3.10') + spack_install('pytorch-fortran-proxy') def test_install_yaxt(): From 65b37b94a5cb36ac5190e8f4b87413716bd106d4 Mon Sep 17 00:00:00 2001 From: github-actions Date: Wed, 27 Nov 2024 08:50:28 +0000 Subject: [PATCH 061/104] GitHub Action: Apply Pep8-formatting --- test/system_test.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index e705518e00..bd60007c05 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -30,12 +30,11 @@ def test_install_flexpart_ifs(): spack_install('flexpart-ifs') -@pytest.mark.parametrize('version', ['2024.01-1', '2.6.6-mch2a', '2.6.6-mch2b']) +@pytest.mark.parametrize('version', + ['2024.01-1', '2.6.6-mch2a', '2.6.6-mch2b']) def test_install_icon(version): # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. - spack_install( - f'icon @{version} %nvhpc ^cray-mpich%nvhpc' - ) + spack_install(f'icon @{version} %nvhpc ^cray-mpich%nvhpc') def test_install_icon_conditional_dependencies(): From 201a1c09b77a1af95f762bad444c066247b2543d Mon Sep 17 00:00:00 2001 From: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> Date: Wed, 4 Dec 2024 22:51:37 +0100 Subject: [PATCH 062/104] pytorch-fortran-proxy: requires gcc --- repos/c2sm/packages/pytorch-fortran-proxy/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/c2sm/packages/pytorch-fortran-proxy/package.py b/repos/c2sm/packages/pytorch-fortran-proxy/package.py index 248389407a..2f896c4b03 100644 --- a/repos/c2sm/packages/pytorch-fortran-proxy/package.py +++ b/repos/c2sm/packages/pytorch-fortran-proxy/package.py @@ -27,7 +27,7 @@ class PytorchFortranProxy(CMakePackage): depends_on('libtorch') depends_on('py-pybind11') - requires('%nvhpc') + requires('%gcc') root_cmakelists_dir = 'src/proxy_lib' From adf2ad97edbbc502cae9e424d1878257d44fbe49 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 08:48:19 +0000 Subject: [PATCH 063/104] GitHub Action: Apply Pep8-formatting --- test/system_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 1761a1e2f8..8a123118c4 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -110,7 +110,6 @@ def test_install_py_gridtools_cpp(): spack_install('py-gridtools-cpp') - @pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9', '1.0.3.10']) def test_install_py_gt4py_for_version(version): spack_install(f'py-gt4py @{version}') From e804ce0c8a93b85646158e59d86f9519d92e9618 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 10:20:11 +0100 Subject: [PATCH 064/104] dirty gt4py fix --- repos/c2sm/packages/py-gt4py/package.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index a5e725230b..35dff30419 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -84,5 +84,10 @@ class PyGt4py(PythonPackage): depends_on('py-pytest-xdist', type=('build', 'run')) def test(self): + python_spec = self.spec['python'] + python_version = python_spec.version.up_to(2) + install_path = join_path(self.prefix, 'lib', f"python{python_version}", 'site-packages') + os.environ['PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" + python('-m', 'pytest', '-v', '-s', '-n', 'auto', '-k', '.run_gtfn]', 'tests/next_tests', 'tests/eve_tests') From 90cbaf5d4dedbf30ad2b034d5838f82c512637ed Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 09:21:23 +0000 Subject: [PATCH 065/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/py-gt4py/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index a90b071fa1..896bcb90f4 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -87,8 +87,10 @@ class PyGt4py(PythonPackage): def test(self): python_spec = self.spec['python'] python_version = python_spec.version.up_to(2) - install_path = join_path(self.prefix, 'lib', f"python{python_version}", 'site-packages') - os.environ['PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" + install_path = join_path(self.prefix, 'lib', f"python{python_version}", + 'site-packages') + os.environ[ + 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" python('-m', 'pytest', '-v', '-s', '-n', 'auto', '-k', '.run_gtfn]', 'tests/next_tests', 'tests/eve_tests') From 8ac19848371616888479d1bbda7190cf03dfef85 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 11:09:59 +0100 Subject: [PATCH 066/104] rm exclaim --- repos/c2sm/packages/icon/package.py | 147 ++++------------------------ 1 file changed, 19 insertions(+), 128 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 730cea2e96..8be64d821e 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -7,14 +7,6 @@ import spack.error as error -def validate_variant_dsl(pkg, name, value): - set_mutual_excl = set(['substitute', 'verify', 'serialize']) - set_input_var = set(value) - if len(set_mutual_excl.intersection(set_input_var)) > 1: - raise error.SpecError( - 'Cannot have more than one of (substitute, verify, serialize) in the same build' - ) - def check_variant_fcgroup(fcgroup): pattern = re.compile(r"^[A-Z]+\..+\..") @@ -46,14 +38,6 @@ class Icon(SpackIcon): version("2024.01-1", tag="icon-2024.01-1", submodules=True) version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) - version('exclaim-master', - branch='master', - git='git@github.com:C2SM/icon-exclaim.git', - submodules=True) - version('exclaim', - branch='icon-dsl', - git='git@github.com:C2SM/icon-exclaim.git', - submodules=True) version('nwp-master', git='git@gitlab.dkrz.de:icon/icon-nwp.git', submodules=True) @@ -137,33 +121,12 @@ class Icon(SpackIcon): variant('silent-rules', default=True, description='Enable silent-rules for build-process') - variant( - 'pytorch', - default=False, - description= - 'Build with pytorch for inference with machine-learning models. Experimental, needs non-standard codebase!' - ) variant( 'eccodes-definitions', default=False, description= 'Enable extension of eccodes with center specific definition files') - # EXCLAIM-GT4Py specific features: - dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') - variant('dsl', - default='none', - validator=validate_variant_dsl, - values=('none', ) + dsl_values, - description='Build with GT4Py dynamical core', - multi=True) - - for x in dsl_values: - depends_on('py-icon4py', when='dsl={0}'.format(x)) - depends_on('py-gridtools-cpp', when='dsl={0}'.format(x)) - depends_on('boost', when='dsl={0}'.format(x)) - conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) - depends_on('cosmo-eccodes-definitions', type='run', when='+eccodes-definitions') @@ -186,7 +149,6 @@ class Icon(SpackIcon): depends_on('netcdf-fortran %nvhpc', when='%nvhpc') depends_on('netcdf-fortran %gcc', when='%gcc') - depends_on('pytorch-fortran', when='+pytorch') depends_on('hdf5 +szip', when='+sct') # patch_libtool is a function from Autotoolspackage. @@ -196,13 +158,6 @@ class Icon(SpackIcon): # also the folder where libtool package itself is installed. patch_libtool = False - def setup_build_environment(self, env): - # help cmake to build dsl-stencils - if 'none' not in self.spec.variants['dsl'].value: - env.set("CUDAARCHS", self.spec.variants['cuda_arch'].value[0]) - env.unset("CUDAHOSTCXX") - env.set("BOOST_ROOT", self.spec['boost'].prefix) - def configure_args(self): args = super().configure_args() super_libs = args.pop() @@ -238,62 +193,12 @@ def configure_args(self): if '+sct' in self.spec: libs += self.spec['hdf5'].libs - if '+pytorch' in self.spec: - libs += self.spec['pytorch-fortran'].libs - fcgroup = self.spec.variants['fcgroup'].value # ('none',) is the values spack assign if fcgroup is not set if fcgroup != ('none', ): args.extend(self.fcgroup_to_config_arg()) flags.update(self.fcgroup_to_config_var()) - # Check for DSL variants and set corresponding Liskov options - dsl = self.spec.variants['dsl'].value - if dsl != ('none', ): - if 'substitute' in dsl: - args.append('--enable-liskov=substitute') - elif 'verify' in dsl: - args.append('--enable-liskov=verify') - elif 'serialize' in dsl: - raise error.UnsupportedPlatformError( - 'serialize mode is not supported yet by icon-liskov') - - if 'lam' in dsl: - args.append('--enable-dsl-local') - if 'nvtx' in dsl: - args.append('--enable-nvtx') - if 'fused' in dsl: - raise error.UnsupportedPlatformError( - 'liskov does not support fusing just yet') - - flags['LOC_GT4PY'].append(self.spec['py-gt4py'].prefix) - flags['LOC_ICON4PY_BIN'].append(self.spec['py-icon4py'].prefix) - - flags['LOC_ICON4PY_ATM_DYN_ICONAM'].append( - self.spec['py-icon4py:atm_dyn_iconam'].headers.directories[0]) - - if self.spec['py-icon4py'].version < Version("0.0.4"): - flags['LOC_ICON4PY_UTILS'].append( - os.path.dirname( - self.spec['py-icon4py:utils'].headers.directories[0])) - else: - flags['LOC_ICON4PY_TOOLS'].append( - self.spec['py-icon4py:tools'].headers.directories[0]) - if self.spec['py-icon4py'].version > Version("0.0.7"): - flags['LOC_ICON4PY_DIFFUSION'].append( - self.spec['py-icon4py:diffusion'].headers. - directories[0]) - flags['LOC_ICON4PY_INTERPOLATION'].append( - self.spec['py-icon4py:interpolation'].headers. - directories[0]) - if self.spec['py-icon4py'].version > Version("0.0.8"): - flags['LOC_ICON4PY_ADVECTION'].append( - self.spec['py-icon4py:advection'].headers. - directories[0]) - flags['LOC_GRIDTOOLS'].append( - self.spec['py-gridtools-cpp:data'].headers.directories[0]) - flags['GT4PYNVCFLAGS'] = flags['NVCFLAGS'] - # add configure arguments not yet available as variant extra_config_args = self.spec.variants['extra-config-args'].value if extra_config_args != ('none', ): @@ -361,36 +266,22 @@ def validate_extra_config_args(self, arg): f'with the existing variant {variant_from_arg}. Set this variant instead.' ) - def check(self): - # By default "check" calls make with targets "check" and "test". - # This testing is beyond the scope of BuildBot test at CSCS. - # Therefore override this function, saves a lot of time too. - pass - - @run_after('install') - @on_package_attributes(run_tests=True) - def checksuite(self): - # script needs cdo to work, but not listed as dep of ICON - test_script = 'scripts/spack/test.py' - if os.path.exists(test_script): - test_py = Executable(test_script) - - # test.py fails if PYTHONHOME has any value, - # even '' or ' ' is failing, therefore delete - # it temporary from env - if 'PYTHONHOME' in os.environ: - PYTHONHOME = os.environ['PYTHONHOME'] - os.environ.pop('PYTHONHOME') - pythonhome_is_set = True - else: - pythonhome_is_set = False - - with open('spec.yaml', mode='w') as f: - f.write(self.spec.to_yaml()) - test_py('--spec', 'spec.yaml', fail_on_error=True) - - # restore PYTHONHOME after test.py - if pythonhome_is_set: - os.environ['PYTHONHOME'] = PYTHONHOME - else: - tty.warn('Cannot find test.py -> skipping tests') + @run_after('configure') + def copy_runscript_related_input_files(self): + with working_dir(self.build_directory): + Rsync = which('rsync', required=True) + icon_dir = self.configure_directory + Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", + "--exclude=.*", "--exclude=standard_*") + Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", + "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", + "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", + "--exclude=*.mod", "--exclude=*.o") + Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") + + Ln = which('ln', required=True) + dirs = glob.glob(f"{icon_dir}/run/standard_*") + for dir in dirs: + Ln("-sf", "-t", "run/", f"{dir}") + Ln("-sf", f"{icon_dir}/data") + Ln("-sf", f"{icon_dir}/vertical_coord_tables") \ No newline at end of file From e9f2c4475a04be50a222286e1d98b1d3451c0d74 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 10:10:25 +0000 Subject: [PATCH 067/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon/package.py | 35 ++++++++++++++--------------- 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 8be64d821e..dfc88293e2 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -7,7 +7,6 @@ import spack.error as error - def check_variant_fcgroup(fcgroup): pattern = re.compile(r"^[A-Z]+\..+\..") # fcgroup is False as default @@ -268,20 +267,20 @@ def validate_extra_config_args(self, arg): @run_after('configure') def copy_runscript_related_input_files(self): - with working_dir(self.build_directory): - Rsync = which('rsync', required=True) - icon_dir = self.configure_directory - Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", - "--exclude=.*", "--exclude=standard_*") - Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", - "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", - "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", - "--exclude=*.mod", "--exclude=*.o") - Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") - - Ln = which('ln', required=True) - dirs = glob.glob(f"{icon_dir}/run/standard_*") - for dir in dirs: - Ln("-sf", "-t", "run/", f"{dir}") - Ln("-sf", f"{icon_dir}/data") - Ln("-sf", f"{icon_dir}/vertical_coord_tables") \ No newline at end of file + with working_dir(self.build_directory): + Rsync = which('rsync', required=True) + icon_dir = self.configure_directory + Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", + "--exclude=.*", "--exclude=standard_*") + Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", + "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", + "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", + "--exclude=*.mod", "--exclude=*.o") + Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") + + Ln = which('ln', required=True) + dirs = glob.glob(f"{icon_dir}/run/standard_*") + for dir in dirs: + Ln("-sf", "-t", "run/", f"{dir}") + Ln("-sf", f"{icon_dir}/data") + Ln("-sf", f"{icon_dir}/vertical_coord_tables") From 639d2b86951e9e829a6f1f5a8548c96dfcbff43b Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 11:50:56 +0100 Subject: [PATCH 068/104] fix test and add icon-exlcaim --- repos/c2sm/packages/icon-exclaim/package.py | 127 ++++++++++++++++++++ test/integration_test.py | 10 +- 2 files changed, 132 insertions(+), 5 deletions(-) create mode 100755 repos/c2sm/packages/icon-exclaim/package.py diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py new file mode 100755 index 0000000000..6aeebf468e --- /dev/null +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -0,0 +1,127 @@ +# from spack.pkg.builtin.icon import Icon as SpackIcon +from spack.pkg.c2sm.icon import Icon as IconC2sm +from spack.pkg.c2sm.icon import check_variant_fcgroup, check_variant_extra_config_args + +import os +import re +from collections import defaultdict +from llnl.util import tty +from spack.util.environment import is_system_path +import spack.error as error + + +def validate_variant_dsl(pkg, name, value): + set_mutual_excl = set(['substitute', 'verify', 'serialize']) + set_input_var = set(value) + if len(set_mutual_excl.intersection(set_input_var)) > 1: + raise error.SpecError( + 'Cannot have more than one of (substitute, verify, serialize) in the same build' + ) + + +class IconExclaim(IconC2sm): + git = 'git@gitlab.dkrz.de:icon/icon.git' + + maintainers('jonasjucker', 'huppd') + + version('exclaim-master', + branch='master', + git='git@github.com:C2SM/icon-exclaim.git', + submodules=True) + version('exclaim', + branch='icon-dsl', + git='git@github.com:C2SM/icon-exclaim.git', + submodules=True) + + # The variants' default follow those of ICON + # as described here + # https://gitlab.dkrz.de/icon/icon/-/blob/icon-2024.01/configure?ref_type=tags#L1492-1638 + + + # EXCLAIM-GT4Py specific features: + dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') + variant('dsl', + default='none', + validator=validate_variant_dsl, + values=('none', ) + dsl_values, + description='Build with GT4Py dynamical core', + multi=True) + + for x in dsl_values: + depends_on('py-icon4py', when='dsl={0}'.format(x)) + depends_on('py-gridtools-cpp', when='dsl={0}'.format(x)) + depends_on('boost', when='dsl={0}'.format(x)) + conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) + + depends_on('cosmo-eccodes-definitions', + type='run', + when='+eccodes-definitions') + + def setup_build_environment(self, env): + super().set_build_envirionment(self,env) + # help cmake to build dsl-stencils + if 'none' not in self.spec.variants['dsl'].value: + env.set("CUDAARCHS", self.spec.variants['cuda_arch'].value[0]) + env.unset("CUDAHOSTCXX") + env.set("BOOST_ROOT", self.spec['boost'].prefix) + + def configure_args(self): + args = super().configure_args() + super_libs = args.pop() + + libs = LibraryList([]) + flags = defaultdict(list) + + # Check for DSL variants and set corresponding Liskov options + dsl = self.spec.variants['dsl'].value + if dsl != ('none', ): + if 'substitute' in dsl: + args.append('--enable-liskov=substitute') + elif 'verify' in dsl: + args.append('--enable-liskov=verify') + elif 'serialize' in dsl: + raise error.UnsupportedPlatformError( + 'serialize mode is not supported yet by icon-liskov') + + if 'lam' in dsl: + args.append('--enable-dsl-local') + if 'nvtx' in dsl: + args.append('--enable-nvtx') + if 'fused' in dsl: + raise error.UnsupportedPlatformError( + 'liskov does not support fusing just yet') + + flags['LOC_GT4PY'].append(self.spec['py-gt4py'].prefix) + flags['LOC_ICON4PY_BIN'].append(self.spec['py-icon4py'].prefix) + + flags['LOC_ICON4PY_ATM_DYN_ICONAM'].append( + self.spec['py-icon4py:atm_dyn_iconam'].headers.directories[0]) + + if self.spec['py-icon4py'].version < Version("0.0.4"): + flags['LOC_ICON4PY_UTILS'].append( + os.path.dirname( + self.spec['py-icon4py:utils'].headers.directories[0])) + else: + flags['LOC_ICON4PY_TOOLS'].append( + self.spec['py-icon4py:tools'].headers.directories[0]) + if self.spec['py-icon4py'].version > Version("0.0.7"): + flags['LOC_ICON4PY_DIFFUSION'].append( + self.spec['py-icon4py:diffusion'].headers. + directories[0]) + flags['LOC_ICON4PY_INTERPOLATION'].append( + self.spec['py-icon4py:interpolation'].headers. + directories[0]) + if self.spec['py-icon4py'].version > Version("0.0.8"): + flags['LOC_ICON4PY_ADVECTION'].append( + self.spec['py-icon4py:advection'].headers. + directories[0]) + flags['LOC_GRIDTOOLS'].append( + self.spec['py-gridtools-cpp:data'].headers.directories[0]) + flags['GT4PYNVCFLAGS'] = flags['NVCFLAGS'] + + args.extend([ + "{0}={1}".format(name, " ".join(value)) + for name, value in flags.items() + ]) + args.append(f"{super_libs} {libs.link_flags}") + return args diff --git a/test/integration_test.py b/test/integration_test.py index abf1f4e3c6..ead9735733 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -20,17 +20,17 @@ def test_spec(package: str): spack_spec(package) -def test_icon_c2sm_serialization(): - spack_spec("icon-c2sm serialization=create") +def test_icon_serialization(): + spack_spec("icon serialization=create") -def test_icon_c2sm_fcgroup(): - spack_spec("icon-c2sm fcgroup=DACE.externals/dace_icon.-O1") +def test_icon_fcgroup(): + spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") def test_icon_extra_config_args(): spack_spec( - "icon-c2sm extra-config-args=--disable-new_feature,--enable-old_config_arg" + "icon extra-config-args=--disable-new_feature,--enable-old_config_arg" ) From c4eab38c92e176f1b269a6df1fc5ad61d2267796 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 10:51:20 +0000 Subject: [PATCH 069/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon-exclaim/package.py | 3 +-- test/integration_test.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 6aeebf468e..05201d439d 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -37,7 +37,6 @@ class IconExclaim(IconC2sm): # as described here # https://gitlab.dkrz.de/icon/icon/-/blob/icon-2024.01/configure?ref_type=tags#L1492-1638 - # EXCLAIM-GT4Py specific features: dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') variant('dsl', @@ -58,7 +57,7 @@ class IconExclaim(IconC2sm): when='+eccodes-definitions') def setup_build_environment(self, env): - super().set_build_envirionment(self,env) + super().set_build_envirionment(self, env) # help cmake to build dsl-stencils if 'none' not in self.spec.variants['dsl'].value: env.set("CUDAARCHS", self.spec.variants['cuda_arch'].value[0]) diff --git a/test/integration_test.py b/test/integration_test.py index ead9735733..ca4e0b6fef 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -30,8 +30,7 @@ def test_icon_fcgroup(): def test_icon_extra_config_args(): spack_spec( - "icon extra-config-args=--disable-new_feature,--enable-old_config_arg" - ) + "icon extra-config-args=--disable-new_feature,--enable-old_config_arg") def test_int2lm_parallel(): From 941db8bb960a71fbc93161ef0387db884fabf41a Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 11:56:54 +0100 Subject: [PATCH 070/104] make out-of-source work --- repos/c2sm/packages/icon/package.py | 33 ++++++++++++++++------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index dfc88293e2..dd8a620ee7 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -1,6 +1,7 @@ from spack.pkg.builtin.icon import Icon as SpackIcon import os import re +import glob from collections import defaultdict from llnl.util import tty from spack.util.environment import is_system_path @@ -268,19 +269,21 @@ def validate_extra_config_args(self, arg): @run_after('configure') def copy_runscript_related_input_files(self): with working_dir(self.build_directory): - Rsync = which('rsync', required=True) icon_dir = self.configure_directory - Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", - "--exclude=.*", "--exclude=standard_*") - Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", - "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", - "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", - "--exclude=*.mod", "--exclude=*.o") - Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") - - Ln = which('ln', required=True) - dirs = glob.glob(f"{icon_dir}/run/standard_*") - for dir in dirs: - Ln("-sf", "-t", "run/", f"{dir}") - Ln("-sf", f"{icon_dir}/data") - Ln("-sf", f"{icon_dir}/vertical_coord_tables") + # only synchronize if out-of-source build + if os.path.abspath(icon_dir) != os.path.abspath(self.build_directory): + Rsync = which('rsync', required=True) + Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", + "--exclude=.*", "--exclude=standard_*") + Rsync("-uavz", f"{icon_dir}/externals", ".", "--exclude=.git", + "--exclude=*.f90", "--exclude=*.F90", "--exclude=*.c", + "--exclude=*.h", "--exclude=*.Po", "--exclude=tests", + "--exclude=*.mod", "--exclude=*.o") + Rsync("-uavz", f"{icon_dir}/make_runscripts", ".") + + Ln = which('ln', required=True) + dirs = glob.glob(f"{icon_dir}/run/standard_*") + for dir in dirs: + Ln("-sf", "-t", "run/", f"{dir}") + Ln("-sf", f"{icon_dir}/data") + Ln("-sf", f"{icon_dir}/vertical_coord_tables") From 82ce7e28a5cbe23bde50819e7e5440283601e714 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 10:57:40 +0000 Subject: [PATCH 071/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon/package.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index dd8a620ee7..1843647045 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -271,7 +271,8 @@ def copy_runscript_related_input_files(self): with working_dir(self.build_directory): icon_dir = self.configure_directory # only synchronize if out-of-source build - if os.path.abspath(icon_dir) != os.path.abspath(self.build_directory): + if os.path.abspath(icon_dir) != os.path.abspath( + self.build_directory): Rsync = which('rsync', required=True) Rsync("-uavz", f"{icon_dir}/run", ".", "--exclude=*.in", "--exclude=.*", "--exclude=standard_*") From 85577711a9dc73d95949ac21e2b5a7db673fc508 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:06:35 +0100 Subject: [PATCH 072/104] fix url --- repos/c2sm/packages/icon/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 1843647045..4e1a45f363 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -30,7 +30,7 @@ def check_variant_extra_config_args(extra_config_arg): class Icon(SpackIcon): - git = 'git@gitlab.dkrz.de:icon/icon.git' + git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' maintainers('jonasjucker', 'huppd') From c30fb38cd679f2629d3d8c8c043bfeb90fb68866 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:07:53 +0100 Subject: [PATCH 073/104] rm file --- repos/c2sm/packages/icon-mch/package.py | 89 ------------------------- 1 file changed, 89 deletions(-) delete mode 100644 repos/c2sm/packages/icon-mch/package.py diff --git a/repos/c2sm/packages/icon-mch/package.py b/repos/c2sm/packages/icon-mch/package.py deleted file mode 100644 index 025d3b194d..0000000000 --- a/repos/c2sm/packages/icon-mch/package.py +++ /dev/null @@ -1,89 +0,0 @@ -from spack.pkg.builtin.icon import Icon as SpackIcon - - -class IconMch(SpackIcon): - git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' - - maintainers("huppd") - - version('master', branch='master', submodules=True) - version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) - version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) - - # Model Features: - variant('dace', - default=False, - description='Enable the DACE modules for data assimilation') - requires("+mpi", when="+dace") - - variant('emvorado', - default=False, - description='Enable the radar forward operator EMVORADO') - requires("+mpi", when="+emvorado") - - # Infrastructural Features: - variant('async-io-rma', - default=True, - description='Enable remote memory access (RMA) for async I/O') - variant('realloc-buf', - default=False, - description='Enable reallocatable communication buffer') - - # Optimization Features: - variant( - 'pgi-inlib', - default=False, - description= - 'Enable PGI/NVIDIA cross-file function inlining via an inline library') - - # MCH specific features: - variant( - 'eccodes-definitions', - default=False, - description= - 'Enable extension of eccodes with center specific definition files') - - depends_on('cosmo-eccodes-definitions', - type='run', - when='+eccodes-definitions') - - with when('+emvorado'): - depends_on('hdf5 +szip +hl +fortran') - depends_on('zlib') - depends_on('eccodes +fortran') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('eccodes %nvhpc', when='%nvhpc') - depends_on('eccodes %gcc', when='%gcc') - - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - for __x in SpackIcon.serialization_values: - with when("serialization={0}".format(__x)): - depends_on('serialbox %nvhpc', when='%nvhpc') - depends_on('serialbox %gcc', when='%gcc') - - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('netcdf-fortran %nvhpc', when='%nvhpc') - depends_on('netcdf-fortran %gcc', when='%gcc') - - def configure_args(self): - args = super().configure_args() - super_libs = args.pop() - - libs = LibraryList([]) - - for x in [ - 'dace', - 'emvorado', - 'async-io-rma', - 'realloc-buf', - 'pgi-inlib', - ]: - args += self.enable_or_disable(x) - - if '+emvorado' in self.spec: - libs += self.spec['eccodes:fortran'].libs - libs += self.spec['hdf5:fortran,hl'].libs - libs += self.spec['zlib'].libs - - args.append(f"{super_libs} {libs.link_flags}") - return args From b46ca728102db6891530f5767f44e6b8b4b09694 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:35:39 +0100 Subject: [PATCH 074/104] update test --- test/system_test.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 8a123118c4..51313df273 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -55,12 +55,6 @@ def test_install_icon_ham(version): spack_install('icon-ham') -@pytest.mark.parametrize('version', ['2.6.6-mch2a', '2.6.6-mch2b']) -def test_install_icon_mch(version): - # WORKAROUND: A build and link dependency should imply that the same compiler is used. ^cray-mpich%nvhpc enforces it. - spack_install(f'icon-mch @{version} %nvhpc ^cray-mpich%nvhpc') - - def test_install_icontools(): spack_install('icontools') From b94b5376dac568d60f23b91c6c3776848505fae5 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:45:42 +0100 Subject: [PATCH 075/104] cleanup --- repos/c2sm/packages/icon-exclaim/package.py | 1 - test/integration_test.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 05201d439d..21415cf03f 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -6,7 +6,6 @@ import re from collections import defaultdict from llnl.util import tty -from spack.util.environment import is_system_path import spack.error as error diff --git a/test/integration_test.py b/test/integration_test.py index ca4e0b6fef..4394f24c5c 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -23,6 +23,8 @@ def test_spec(package: str): def test_icon_serialization(): spack_spec("icon serialization=create") +def test_icon_exclaim_dsl(): + spack_spec("icon-exclaim dsl=substition") def test_icon_fcgroup(): spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") From 7ea4ac710fb04ab7feb9702fe2bebe383cf4f38c Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 12:46:07 +0000 Subject: [PATCH 076/104] GitHub Action: Apply Pep8-formatting --- test/integration_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/integration_test.py b/test/integration_test.py index 4394f24c5c..947bcabce4 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -23,9 +23,11 @@ def test_spec(package: str): def test_icon_serialization(): spack_spec("icon serialization=create") + def test_icon_exclaim_dsl(): spack_spec("icon-exclaim dsl=substition") + def test_icon_fcgroup(): spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") From ca49b5ae5e5c0158ee0c2de7f082ec6290e165d8 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:47:44 +0100 Subject: [PATCH 077/104] fix --- repos/c2sm/packages/icon-exclaim/package.py | 1 - test/integration_test.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 21415cf03f..43effb56ae 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -5,7 +5,6 @@ import os import re from collections import defaultdict -from llnl.util import tty import spack.error as error diff --git a/test/integration_test.py b/test/integration_test.py index 4394f24c5c..43f52676c0 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -24,7 +24,7 @@ def test_icon_serialization(): spack_spec("icon serialization=create") def test_icon_exclaim_dsl(): - spack_spec("icon-exclaim dsl=substition") + spack_spec("icon-exclaim dsl=substitute") def test_icon_fcgroup(): spack_spec("icon fcgroup=DACE.externals/dace_icon.-O1") From 59ecadc09a8cf797694f7257f307d7cd22103695 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 13:56:52 +0100 Subject: [PATCH 078/104] fix version --- repos/c2sm/packages/icon/package.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 4e1a45f363..f5e45a502f 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -31,16 +31,15 @@ def check_variant_extra_config_args(extra_config_arg): class Icon(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' + maintainers('jonasjucker', 'huppd') version('develop', submodules=True) - version("2024.01-1", tag="icon-2024.01-1", submodules=True) + version("2024.01-1", tag="icon-2024.01-1",git='git@gitlab.dkrz.de:icon/icon.git', submodules=True) version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) - version('nwp-master', - git='git@gitlab.dkrz.de:icon/icon-nwp.git', - submodules=True) + version('nwp-master', submodules=True) # The variants' default follow those of ICON # as described here From 4ed055d0b812f4203c0c29c7dd91c161335fd949 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 13:57:31 +0100 Subject: [PATCH 079/104] rm include --- repos/c2sm/packages/icon-exclaim/package.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 43effb56ae..22b60efaa5 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -1,6 +1,4 @@ -# from spack.pkg.builtin.icon import Icon as SpackIcon from spack.pkg.c2sm.icon import Icon as IconC2sm -from spack.pkg.c2sm.icon import check_variant_fcgroup, check_variant_extra_config_args import os import re From 5ad980508809ced917972b5a1a4a80c0c950fc50 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 12:57:34 +0000 Subject: [PATCH 080/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon/package.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index f5e45a502f..4d7c8037e7 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -31,12 +31,14 @@ def check_variant_extra_config_args(extra_config_arg): class Icon(SpackIcon): git = 'git@gitlab.dkrz.de:icon/icon-nwp.git' - maintainers('jonasjucker', 'huppd') version('develop', submodules=True) - version("2024.01-1", tag="icon-2024.01-1",git='git@gitlab.dkrz.de:icon/icon.git', submodules=True) + version("2024.01-1", + tag="icon-2024.01-1", + git='git@gitlab.dkrz.de:icon/icon.git', + submodules=True) version('2.6.6-mch2b', tag='icon-nwp/icon-2.6.6-mch2b', submodules=True) version('2.6.6-mch2a', tag='icon-nwp/icon-2.6.6-mch2a', submodules=True) version('nwp-master', submodules=True) From aefb2a9d178a85d18bbca92fb7a49554fe0fd2af Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 14:03:34 +0100 Subject: [PATCH 081/104] fix icon-ham --- test/system_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 51313df273..54df0461fa 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -51,7 +51,7 @@ def test_install_icon_conditional_dependencies(): ) -def test_install_icon_ham(version): +def test_install_icon_ham(): spack_install('icon-ham') From 81ad9427f380ca55361c94bfafa71375d3534d96 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 14:11:46 +0100 Subject: [PATCH 082/104] add maintainer --- repos/c2sm/packages/icon-ham/package.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repos/c2sm/packages/icon-ham/package.py b/repos/c2sm/packages/icon-ham/package.py index af1dfc495e..8ba3b6222b 100644 --- a/repos/c2sm/packages/icon-ham/package.py +++ b/repos/c2sm/packages/icon-ham/package.py @@ -4,6 +4,8 @@ class IconHam(SpackIcon): + maintainers('stelliom') + @run_before('build') def generate_hammoz_nml(self): with working_dir(self.configure_directory + From 13d4c089d7008e0f883a86994702eef83857bd68 Mon Sep 17 00:00:00 2001 From: juckerj <39263956+jonasjucker@users.noreply.github.com> Date: Thu, 5 Dec 2024 14:19:06 +0100 Subject: [PATCH 083/104] Update repos/c2sm/packages/icon-exclaim/package.py Co-authored-by: Dominic Hofer <6570912+dominichofer@users.noreply.github.com> --- repos/c2sm/packages/icon-exclaim/package.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 22b60efaa5..321c8a33d9 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -48,9 +48,6 @@ class IconExclaim(IconC2sm): depends_on('boost', when='dsl={0}'.format(x)) conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) - depends_on('cosmo-eccodes-definitions', - type='run', - when='+eccodes-definitions') def setup_build_environment(self, env): super().set_build_envirionment(self, env) From 57010f3645e7e49703512a533f5f4d27b46f9ebb Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 13:19:25 +0000 Subject: [PATCH 084/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon-exclaim/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 321c8a33d9..705636a512 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -48,7 +48,6 @@ class IconExclaim(IconC2sm): depends_on('boost', when='dsl={0}'.format(x)) conflicts('^python@:3.9,3.11:', when='dsl={0}'.format(x)) - def setup_build_environment(self, env): super().set_build_envirionment(self, env) # help cmake to build dsl-stencils From cb59d550b8cbd05163055777f877faf9f681f8c0 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 14:22:06 +0100 Subject: [PATCH 085/104] change --- repos/c2sm/packages/icon-exclaim/package.py | 13 ++----------- repos/c2sm/packages/py-gt4py/package.py | 2 +- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 705636a512..14e4fa7c60 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -16,23 +16,14 @@ def validate_variant_dsl(pkg, name, value): class IconExclaim(IconC2sm): - git = 'git@gitlab.dkrz.de:icon/icon.git' + git='git@github.com:C2SM/icon-exclaim.git', maintainers('jonasjucker', 'huppd') - version('exclaim-master', - branch='master', - git='git@github.com:C2SM/icon-exclaim.git', - submodules=True) - version('exclaim', + version('icon-dsl', branch='icon-dsl', - git='git@github.com:C2SM/icon-exclaim.git', submodules=True) - # The variants' default follow those of ICON - # as described here - # https://gitlab.dkrz.de/icon/icon/-/blob/icon-2024.01/configure?ref_type=tags#L1492-1638 - # EXCLAIM-GT4Py specific features: dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') variant('dsl', diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index 896bcb90f4..958515f224 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -2,7 +2,7 @@ # Spack Project Developers. See the top-level COPYRIGHT file for details. # # SPDX-License-Identifier: (Apache-2.0 OR MIT) - +import os from spack import * From 2aef51057fa19eb207ee5e31ef1be4a08a5c3517 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 13:22:31 +0000 Subject: [PATCH 086/104] GitHub Action: Apply Pep8-formatting --- repos/c2sm/packages/icon-exclaim/package.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repos/c2sm/packages/icon-exclaim/package.py b/repos/c2sm/packages/icon-exclaim/package.py index 14e4fa7c60..599e71ac00 100755 --- a/repos/c2sm/packages/icon-exclaim/package.py +++ b/repos/c2sm/packages/icon-exclaim/package.py @@ -16,13 +16,11 @@ def validate_variant_dsl(pkg, name, value): class IconExclaim(IconC2sm): - git='git@github.com:C2SM/icon-exclaim.git', + git = 'git@github.com:C2SM/icon-exclaim.git', maintainers('jonasjucker', 'huppd') - version('icon-dsl', - branch='icon-dsl', - submodules=True) + version('icon-dsl', branch='icon-dsl', submodules=True) # EXCLAIM-GT4Py specific features: dsl_values = ('substitute', 'verify', 'serialize', 'fused', 'nvtx', 'lam') From 2dabff662dff14c1b953013f64f058437099e545 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 14:50:43 +0100 Subject: [PATCH 087/104] disable version --- test/system_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/system_test.py b/test/system_test.py index 54df0461fa..75c37538de 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -104,10 +104,14 @@ def test_install_py_gridtools_cpp(): spack_install('py-gridtools-cpp') -@pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9', '1.0.3.10']) +@pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9']) def test_install_py_gt4py_for_version(version): spack_install(f'py-gt4py @{version}') +# fails due to sql error +def test_build_only_py_gt4py_for_1_0_3_10(): + spack_install('py-gt4py @1.0.3.10', test_root=False) + @pytest.mark.parametrize("version, gt4py_version", [('0.0.13', '1.0.3.9'), ('0.0.14', '1.0.3.10')]) From f66f2b60593a47051e45e7c0d26f15d877d63a30 Mon Sep 17 00:00:00 2001 From: github-actions Date: Thu, 5 Dec 2024 13:51:06 +0000 Subject: [PATCH 088/104] GitHub Action: Apply Pep8-formatting --- test/system_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/system_test.py b/test/system_test.py index 75c37538de..1ba01bed73 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -108,6 +108,7 @@ def test_install_py_gridtools_cpp(): def test_install_py_gt4py_for_version(version): spack_install(f'py-gt4py @{version}') + # fails due to sql error def test_build_only_py_gt4py_for_1_0_3_10(): spack_install('py-gt4py @1.0.3.10', test_root=False) From ae454ecf9059d5e5c65b39a549652414701a8682 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 15:53:32 +0100 Subject: [PATCH 089/104] remove loop-exchange --- repos/c2sm/packages/icon/package.py | 1 - 1 file changed, 1 deletion(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 4d7c8037e7..113b011d9f 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -95,7 +95,6 @@ class Icon(SpackIcon): ) # Optimization Features: - variant('loop-exchange', default=True, description='Enable loop exchange') variant('vectorized-lrtm', default=False, description='Enable the parallelization-invariant version of LRTM') From 42a5efe4342a5fae8b244ae5c42771084cdfb834 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Thu, 5 Dec 2024 16:37:51 +0100 Subject: [PATCH 090/104] fix icon --- test/system_test.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/system_test.py b/test/system_test.py index 1ba01bed73..fd1cf87ed8 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -51,10 +51,6 @@ def test_install_icon_conditional_dependencies(): ) -def test_install_icon_ham(): - spack_install('icon-ham') - - def test_install_icontools(): spack_install('icontools') From c40e511d2e48514cf22af94d0d4d616f401cb3be Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 5 Dec 2024 18:02:03 +0100 Subject: [PATCH 091/104] dirty fix icon4py --- repos/c2sm/packages/py-gt4py/package.py | 1 + repos/c2sm/packages/py-icon4py/package.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/repos/c2sm/packages/py-gt4py/package.py b/repos/c2sm/packages/py-gt4py/package.py index 958515f224..5b2c95cedf 100644 --- a/repos/c2sm/packages/py-gt4py/package.py +++ b/repos/c2sm/packages/py-gt4py/package.py @@ -85,6 +85,7 @@ class PyGt4py(PythonPackage): depends_on('py-pytest-xdist', type=('build', 'run')) def test(self): + # workaround for not finding own python module python_spec = self.spec['python'] python_version = python_spec.version.up_to(2) install_path = join_path(self.prefix, 'lib', f"python{python_version}", diff --git a/repos/c2sm/packages/py-icon4py/package.py b/repos/c2sm/packages/py-icon4py/package.py index 4e5072f92d..c9dea8fb2e 100644 --- a/repos/c2sm/packages/py-icon4py/package.py +++ b/repos/c2sm/packages/py-icon4py/package.py @@ -50,6 +50,13 @@ def setup_build_environment(self, env): env.set("CMAKE_INCLUDE_PATH", self.spec['boost'].prefix.include) def test(self): + # workaround for not finding own python module + python_spec = self.spec['python'] + python_version = python_spec.version.up_to(2) + install_path = join_path(self.prefix, 'lib', f"python{python_version}", + 'site-packages') + os.environ[ + 'PYTHONPATH'] = f"{install_path}:{os.environ.get('PYTHONPATH', '')}" # check if all installed module can be imported super().test() # unit tests From b139e235523eb0fdab09e950ab80219da918d425 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 6 Dec 2024 13:31:09 +0100 Subject: [PATCH 092/104] readd loop-exchange --- repos/c2sm/packages/icon/package.py | 1 + 1 file changed, 1 insertion(+) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 113b011d9f..4d7c8037e7 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -95,6 +95,7 @@ class Icon(SpackIcon): ) # Optimization Features: + variant('loop-exchange', default=True, description='Enable loop exchange') variant('vectorized-lrtm', default=False, description='Enable the parallelization-invariant version of LRTM') From eb6d5b3c1e63db440da8fb30982b7b82cc346be0 Mon Sep 17 00:00:00 2001 From: juckerj <39263956+jonasjucker@users.noreply.github.com> Date: Mon, 9 Dec 2024 10:14:50 +0100 Subject: [PATCH 093/104] Update package.py Set default loop-exchange to False --- repos/c2sm/packages/icon/package.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 4d7c8037e7..1d18c6d5c2 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -95,7 +95,7 @@ class Icon(SpackIcon): ) # Optimization Features: - variant('loop-exchange', default=True, description='Enable loop exchange') + variant('loop-exchange', default=False, description='Enable loop exchange') variant('vectorized-lrtm', default=False, description='Enable the parallelization-invariant version of LRTM') From 8693130b2d3ccadba3bb77d7365269fbeb619c1a Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 9 Dec 2024 11:10:08 +0100 Subject: [PATCH 094/104] fix bug again of serialbox dep, cleanup --- repos/c2sm/packages/icon/package.py | 12 ++-- .../packages/pytorch-fortran-proxy/package.py | 41 ------------- .../c2sm/packages/pytorch-fortran/package.py | 57 ------------------- test/system_test.py | 10 +--- 4 files changed, 5 insertions(+), 115 deletions(-) delete mode 100644 repos/c2sm/packages/pytorch-fortran-proxy/package.py delete mode 100644 repos/c2sm/packages/pytorch-fortran/package.py diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 1d18c6d5c2..3e4f976ef7 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -81,10 +81,6 @@ class Icon(SpackIcon): default=False, description='Enable reallocatable communication buffer') variant('sct', default=False, description='Enable the SCT timer') - variant('testbed', - default=False, - description='Enable ICON Testbed infrastructure') - variant( 'extra-config-args', default='none', @@ -142,9 +138,10 @@ class Icon(SpackIcon): # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. for __x in SpackIcon.serialization_values: - with when("serialization={0}".format(__x)): - depends_on('serialbox %nvhpc', when='%nvhpc') - depends_on('serialbox %gcc', when='%gcc') + depends_on('serialbox+fortran %nvhpc', + when='serialization={0} %nvhpc'.format(__x)) + depends_on('serialbox+fortran %gcc', + when='serialization={0} %gcc'.format(__x)) # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. depends_on('netcdf-fortran %nvhpc', when='%nvhpc') @@ -176,7 +173,6 @@ def configure_args(self): 'realloc-buf', 'parallel-netcdf', 'sct', - 'testbed', 'loop-exchange', 'vectorized-lrtm', 'pgi-inlib', diff --git a/repos/c2sm/packages/pytorch-fortran-proxy/package.py b/repos/c2sm/packages/pytorch-fortran-proxy/package.py deleted file mode 100644 index 2f896c4b03..0000000000 --- a/repos/c2sm/packages/pytorch-fortran-proxy/package.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# -from spack import * - - -class PytorchFortranProxy(CMakePackage): - ''' - Pytorch Fortran bindings - C++ Backend - - The goal of this code is to provide Fortran HPC codes with a simple way to use - Pytorch deep learning framework. We want Fortran developers to take advantage - of rich and optimized Torch ecosystem from within their existing codes. - ''' - - homepage = "https://github.com/alexeedm/pytorch-fortran" - url = "https://github.com/alexeedm/pytorch-fortran.git" - - version('0.4', git=url, tag='v0.4') - - maintainers = ['juckerj'] - - depends_on('cuda') - depends_on('libtorch') - depends_on('py-pybind11') - - requires('%gcc') - - root_cmakelists_dir = 'src/proxy_lib' - - def cmake_args(self): - args = [ - self.define('OPENACC', 1), - self.define('CUDA_TOOLKIT_ROOT_DIR', self.spec['cuda'].prefix), - self.define('TORCH_CUDA_ARCH_LIST', "6.0") - ] - - return args diff --git a/repos/c2sm/packages/pytorch-fortran/package.py b/repos/c2sm/packages/pytorch-fortran/package.py deleted file mode 100644 index 8c19f3ee9d..0000000000 --- a/repos/c2sm/packages/pytorch-fortran/package.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# -from spack import * -import os - - -class PytorchFortran(CMakePackage): - ''' - Pytorch Fortran bindings - Fortran Frontend - - The goal of this code is to provide Fortran HPC codes with a simple way to use - Pytorch deep learning framework. We want Fortran developers to take advantage - of rich and optimized Torch ecosystem from within their existing codes. - ''' - - homepage = "https://github.com/alexeedm/pytorch-fortran" - url = "https://github.com/alexeedm/pytorch-fortran.git" - - version('0.4', git=url, tag='v0.4') - - maintainers = ['juckerj'] - - depends_on('pytorch-fortran-proxy') - - root_cmakelists_dir = 'src/f90_bindings' - - def cmake_args(self): - args = [self.define('OPENACC', 1)] - - return args - - @property - def libs(self): - libraries = ['libpytorch_fort_proxy'] - - libs = find_libraries(libraries, - root=self.prefix, - shared=True, - recursive=True) - - if libs and len(libs) == len(libraries): - return libs - - msg = 'Unable to recursively locate shared {0} libraries in {1}' - raise spack.error.NoLibrariesError( - msg.format(self.spec.name, self.spec.prefix)) - - @run_after('install') - def link_fmod_into_include(self): - mod = 'torch_ftn.mod' - src = os.path.join(self.prefix.include, 'mod_files', mod) - dest = os.path.join(self.prefix.include, mod) - os.symlink(src, dest) diff --git a/test/system_test.py b/test/system_test.py index fd1cf87ed8..b94f3f1d55 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -100,7 +100,7 @@ def test_install_py_gridtools_cpp(): spack_install('py-gridtools-cpp') -@pytest.mark.parametrize("version", ['1.0.3.7', '1.0.3.9']) +@pytest.mark.parametrize("version", ['1.0.3.9']) def test_install_py_gt4py_for_version(version): spack_install(f'py-gt4py @{version}') @@ -136,13 +136,5 @@ def test_install_py_typing_extensions(): spack_install('py-typing-extensions') -def test_install_pytorch_fortran(): - spack_install('pytorch-fortran %nvhpc') - - -def test_install_pytorch_fortran_proxy(): - spack_install('pytorch-fortran-proxy') - - def test_install_yaxt(): spack_install('yaxt') From fe686ce8ae79194af6145e0c61ef193e858d7717 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 9 Dec 2024 13:23:40 +0100 Subject: [PATCH 095/104] reintroduce skip configure --- repos/c2sm/packages/icon/package.py | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index 3e4f976ef7..facf46bf0c 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -263,6 +263,55 @@ def validate_extra_config_args(self, arg): f'with the existing variant {variant_from_arg}. Set this variant instead.' ) + def configure(self, spec, prefix): + if os.path.exists( + os.path.join(self.build_directory, + 'icon.mk')) and self.build_uses_same_spec(): + tty.warn( + 'icon.mk already present -> skip configure stage', + '\t delete "icon.mk" or run "make distclean" to not skip configure' + ) + return + + # Call configure of Autotools + super().configure(spec, prefix) + + def build_uses_same_spec(self): + """ + Ensure that configure is rerun in case spec has changed, + otherwise for the case below + + $ spack dev-build icon @develop ~dace + $ spack dev-build icon @develop +dace + + configure is skipped for the latter. + """ + + is_same_spec = False + + previous_spec = os.path.join(self.build_directory, + '.previous_spec.yaml') + + # not the first build in self.build_directory + if os.path.exists(previous_spec): + with open(previous_spec, mode='r') as f: + if self.spec == Spec.from_yaml(f): + is_same_spec = True + else: + is_same_spec = False + tty.warn( + 'Cannot skip configure phase because spec changed') + + # first build in self.build_directory, no worries + else: + is_same_spec = True + + # dump spec of new build + with open(previous_spec, mode='w') as f: + f.write(self.spec.to_yaml()) + + return is_same_spec + @run_after('configure') def copy_runscript_related_input_files(self): with working_dir(self.build_directory): From d9e66b76d9b353bb58c543794e4417187780d506 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 9 Dec 2024 13:27:48 +0100 Subject: [PATCH 096/104] remove libtorch --- repos/c2sm/packages/libtorch/package.py | 49 ------------------------- test/system_test.py | 8 ---- 2 files changed, 57 deletions(-) delete mode 100644 repos/c2sm/packages/libtorch/package.py diff --git a/repos/c2sm/packages/libtorch/package.py b/repos/c2sm/packages/libtorch/package.py deleted file mode 100644 index 55e11ec2ec..0000000000 --- a/repos/c2sm/packages/libtorch/package.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright 2013-2021 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack import * -import shutil - - -class Libtorch(Package): - ''' - PyTorch is a Python package that provides two high-level features: - - - Tensor computation (like NumPy) with strong GPU acceleration - - Deep neural networks built on a tape-based autograd system - ''' - - homepage = "https://pytorch.org/" - url = "https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.1%2Bcu117.zip" - - maintainers = ['juckerj'] - - version('2.0.1', - url=url, - sha256= - '262f723ee5a2caac977e089bc06e9d840ca33d70706fbd4a2fca04995bb94eb4') - - phases = ['install'] - - def install(self, spec, prefix): - # can't use Spack convenience-function 'install_tree' because it uses - # shutil.copy2 under the hood. For an unknown reason installing from - # the unzipped tarbal only works using shutil.copy. - shutil.copytree('lib', - prefix.lib, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('include', - prefix.include, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('share', - prefix.share, - symlinks=True, - copy_function=shutil.copy) - shutil.copytree('bin', - prefix.bin, - symlinks=True, - copy_function=shutil.copy) diff --git a/test/system_test.py b/test/system_test.py index b94f3f1d55..fb38b1f8ec 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -62,18 +62,10 @@ def test_install_int2lm_3_00_nvhpc(): spack_install('int2lm @int2lm-3.00 %nvhpc', test_root=False) -def test_install_libfyaml(): - spack_install('libfyaml') - - def test_install_libgrib1_nvhpc(): spack_install('libgrib1 %nvhpc') -def test_install_libtorch(): - spack_install('libtorch') - - def test_install_makedepf90(): # Tests are disabled because they fail with: # test1.sh: No such file or directory From 52db158352e354fafc2e17c207c5d1044030f5e4 Mon Sep 17 00:00:00 2001 From: juckerj <39263956+jonasjucker@users.noreply.github.com> Date: Mon, 9 Dec 2024 15:12:56 +0100 Subject: [PATCH 097/104] upgrade docs (#1038) * upgrade docs * add yaml example * update docs --- docs/CodeDevelopment.rst | 48 ++++++++++------------------------------ docs/QuickStart.rst | 48 +++++++++++++++++++--------------------- 2 files changed, 35 insertions(+), 61 deletions(-) diff --git a/docs/CodeDevelopment.rst b/docs/CodeDevelopment.rst index cc3c7caac9..ce73988233 100644 --- a/docs/CodeDevelopment.rst +++ b/docs/CodeDevelopment.rst @@ -3,42 +3,6 @@ Develop packages Spack offers several options for package development. Depending on your workflow, one or the other option is preferred. -Also some packages like ICON or COSMO have their own -development workflow which is maintained by C2SM. - -Plain dev-build ---------------- - -This is the easiest way to build local sources. -Enter the root of your source repository and execute: - -.. code-block:: console - - $ spack dev-build --until build @ - -This will build the package as is. The downside of this approach is that -you need to go through all phases of a package build. - -Dev-build in combination with build-env ---------------------------------------- - -We assume that developers of a package are familiar with its build system. -Therefore, we reccomend to use Spack to set up the environment for the package. -Building and testing should be done with the package's build and test system. - -.. code-block:: console - - # Load Spack! - $ spack dev-build --before build @develop # stops dev-build before executing the phase 'build' - $ spack build-env @develop -- bash # nests a bash shell with the build env vars loaded - # Work on the package! - # Use the build system of the package! (e.g. 'make') - # Use the testing infrastructure of the package! - $ exit # to exit the nested bash - -If you want multiple dev-builds at the same time, label them with separate ``@``. -The identifier ``@develop`` is common in the Spack documentation but you can use any string. - Environments with Spack develop ------------------------------- @@ -95,3 +59,15 @@ To deactivate a Spack environment, type Most of the Spack commands are sensitive to environments, see `Spack docs `__. +Plain dev-build +--------------- + +This is the easiest way to build local sources. +Enter the root of your source repository and execute: + +.. code-block:: console + + $ spack dev-build --until build @ + +This will build the package as is. The downside of this approach is that +you need to go through all phases of a package build. diff --git a/docs/QuickStart.rst b/docs/QuickStart.rst index e1ad39f7aa..519cb241c4 100644 --- a/docs/QuickStart.rst +++ b/docs/QuickStart.rst @@ -11,19 +11,17 @@ To set up a Spack instance, clone the repository using a specific Spack tag (lat $ git clone --depth 1 --recurse-submodules --shallow-submodules -b $SPACK_TAG https://github.com/C2SM/spack-c2sm.git -To load it into your command line, execute +To load it into your command line, execute one of the following commands: .. code-block:: console $ . spack-c2sm/setup-env.sh + $ . spack-c2sm/setup-env.sh /user-environment + $ . spack-c2sm/setup-env.sh /mch-environment/v6 + $ . spack-c2sm/setup-env.sh /mch-environment/v7 + $ . spack-c2sm/setup-env.sh any_other_upstream -This auto-detects your machine and configures your instance for it. -You can force a machine with an argument. The name has to match a folder in sysconfigs. - -.. code-block:: console - - $ . spack-c2sm/setup-env.sh balfrin - +This will make upstream installation from user-environment available in spack-c2sm. Local machines and Containers ----------------------------- @@ -89,16 +87,6 @@ For ICON, they are located in ``config/cscs/spack//__< They work with a special Spack tag, that is provided in the ICON repository at ``config/cscs/SPACK_TAG_*``. So make sure you clone Spack with the specified tag. -.. tip:: - **On Balfrin:** - In case your Spack environment requires Python, a compatability issue - with `openssl` and `git` appears. - - ``/usr/bin/ssh: symbol lookup error: /usr/bin/ssh: undefined symbol: EVP_KDF_CTX_free, version OPENSSL_1_1_1d`` - - To circumvent that simply do - ``spack load git`` prior to activation of the environment. - To activate the Spack environment, type .. code-block:: console @@ -127,15 +115,25 @@ Out-of-source builds are possible as follows: .. code-block:: console - $ mkdir cpu && cd cpu - $ cp -r ../config . - $ spack env activate -d config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc + $ mkdir cpu + $ spack env activate config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc + $ # tell spack to build icon in folder cpu + $ spack develop --build-directory cpu icon@develop $ spack install -.. attention:: - Out-of-source build for AutotoolsPackages is not supported by Spack. - The implementation for ICON relies on some hacks inside package.py and - only works if the build-folder is located inside the Git repo of ICON. +By executing the commands above, spack will add some lines directly into ``spack.yaml``: + +.. code-block:: yaml + + spack: + packages: + icon: + package_attributes: + build_directory: /scratch/mch/juckerj/icon-nwp/cpu + +Any further ``spack install`` command will use the build directory specified in the ``spack.yaml`` file. +In case you want to change the build directory, edit the ``spack.yaml`` file or remove the ``build_directory`` line +and run ``spack concretize -f`` afterwards. COSMO ----- From ca2ba85f99434f8543f6b08452ed3e029a3d7785 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 9 Dec 2024 15:53:31 +0100 Subject: [PATCH 098/104] update docs --- docs/QuickStart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/QuickStart.rst b/docs/QuickStart.rst index 519cb241c4..7b17770d9f 100644 --- a/docs/QuickStart.rst +++ b/docs/QuickStart.rst @@ -118,7 +118,7 @@ Out-of-source builds are possible as follows: $ mkdir cpu $ spack env activate config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc $ # tell spack to build icon in folder cpu - $ spack develop --build-directory cpu icon@develop + $ spack develop --path $(pwd) --build-directory cpu icon@develop $ spack install By executing the commands above, spack will add some lines directly into ``spack.yaml``: From 8150572e17e253cb1165f7fb69e2fae070413ffb Mon Sep 17 00:00:00 2001 From: Xavier Lapillonne <7485594+lxavier@users.noreply.github.com> Date: Wed, 15 Jan 2025 15:19:30 +0100 Subject: [PATCH 099/104] Update cosmo-eccodes-definitions versions to 2.36.0.3 This is used with eccodes 2.36.4 This is required to test v8 env on balfrin --- repos/c2sm/packages/cosmo-eccodes-definitions/package.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py index d854b35e35..1666160a2d 100644 --- a/repos/c2sm/packages/cosmo-eccodes-definitions/package.py +++ b/repos/c2sm/packages/cosmo-eccodes-definitions/package.py @@ -8,17 +8,21 @@ class CosmoEccodesDefinitions(Package): url = "https://github.com/COSMO-ORG/eccodes-cosmo-resources.git" git = 'https://github.com/COSMO-ORG/eccodes-cosmo-resources.git' - maintainers = ['petrabaumann'] + maintainers = ['huppd,lxavier'] + version('2.36.0.3', tag='v2.36.0.3') version('2.25.0.3', tag='v2.25.0.3') version('2.25.0.2', tag='v2.25.0.2') version('2.25.0.1', tag='v2.25.0.1') version('2.18.0.1', tag='v2.18.0.1') depends_on('eccodes') + depends_on('eccodes@2.36.4', + type=('build', 'link', 'run'), + when='@2.36.0.3') depends_on('eccodes@2.25.0', type=('build', 'link', 'run'), - when='@2.25.0.1:') + when='@2.25.0.1:2.25.0.3') depends_on('eccodes@2.18.0', type=('build', 'link', 'run'), when='@2.18.0.1') From 98de61cc1a3b295bde14e73dbc00bc3866e55272 Mon Sep 17 00:00:00 2001 From: Ben Cumming Date: Mon, 20 Jan 2025 13:24:14 +0100 Subject: [PATCH 100/104] use explicit zlib-ng dependency for icon (#1040) --- repos/c2sm/packages/icon/package.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repos/c2sm/packages/icon/package.py b/repos/c2sm/packages/icon/package.py index facf46bf0c..21384b32c4 100755 --- a/repos/c2sm/packages/icon/package.py +++ b/repos/c2sm/packages/icon/package.py @@ -131,7 +131,7 @@ class Icon(SpackIcon): with when('+emvorado'): depends_on('eccodes +fortran') depends_on('hdf5 +szip +hl +fortran') - depends_on('zlib') + depends_on('zlib-ng') # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. depends_on('eccodes %nvhpc', when='%nvhpc') depends_on('eccodes %gcc', when='%gcc') @@ -185,7 +185,7 @@ def configure_args(self): if '+emvorado' in self.spec: libs += self.spec['eccodes:fortran'].libs libs += self.spec['hdf5:fortran,hl'].libs - libs += self.spec['zlib'].libs + libs += self.spec['zlib-ng'].libs if '+sct' in self.spec: libs += self.spec['hdf5'].libs From 08b457a5d901ef93cf0d851f596703befa4b18c1 Mon Sep 17 00:00:00 2001 From: juckerj <39263956+jonasjucker@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:52:36 +0100 Subject: [PATCH 101/104] use rc 4 --- Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index f3865140e4..30dea72ade 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -36,7 +36,7 @@ pipeline { ./uenv/install --yes --destdir=$WORKSPACE source $WORKSPACE/etc/profile.d/uenv.sh uenv repo create - uenv image pull mch/v8:rc2 + uenv image pull mch/v8:rc4 """ } } @@ -63,7 +63,7 @@ pipeline { source $WORKSPACE/etc/profile.d/uenv.sh source ./setup-env.sh /user-environment source .venv/bin/activate - uenv run mch/v8:rc2 -- pytest -v -n auto test/integration_test.py + uenv run mch/v8:rc4 -- pytest -v -n auto test/integration_test.py """ } } @@ -73,7 +73,7 @@ pipeline { source $WORKSPACE/etc/profile.d/uenv.sh source ./setup-env.sh /user-environment source .venv/bin/activate - uenv run mch/v8:rc2 -- pytest -v -n auto test/system_test.py + uenv run mch/v8:rc4 -- pytest -v -n auto test/system_test.py """ } } From 979f9b7a2146a059dfdfe38688a7eeb06879bc9e Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 27 Jan 2025 17:02:37 +0100 Subject: [PATCH 102/104] remove obsolete packages --- repos/c2sm/packages/int2lm/package.py | 185 ------------------ .../int2lm/patches/testsuite/patch.to_python3 | 10 - repos/c2sm/packages/oasis/package.py | 154 --------------- .../patch.PSMILE_add_timer_as_dep_of_string | 13 -- test/system_test.py | 11 -- 5 files changed, 373 deletions(-) delete mode 100644 repos/c2sm/packages/int2lm/package.py delete mode 100644 repos/c2sm/packages/int2lm/patches/testsuite/patch.to_python3 delete mode 100644 repos/c2sm/packages/oasis/package.py delete mode 100644 repos/c2sm/packages/oasis/patches/patch.PSMILE_add_timer_as_dep_of_string diff --git a/repos/c2sm/packages/int2lm/package.py b/repos/c2sm/packages/int2lm/package.py deleted file mode 100644 index 7ea4875c17..0000000000 --- a/repos/c2sm/packages/int2lm/package.py +++ /dev/null @@ -1,185 +0,0 @@ -# Copyright 2013-2020 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -# ---------------------------------------------------------------------------- -from spack import * - -import os -import subprocess - - -class Int2lm(MakefilePackage): - """INT2LM performs the interpolation from coarse grid model data to initial - and/or boundary data for the COSMO-Model.""" - - homepage = "http://www.cosmo-model.org/content/model/" - url = "https://github.com/MeteoSwiss-APN/int2lm/archive/refs/tags/v2.8.4.tar.gz" - git = 'git@github.com:MeteoSwiss-APN/int2lm.git' - c2smgit = 'git@github.com:C2SM-RCM/int2lm.git' - orggit = 'git@github.com:COSMO-ORG/int2lm.git' - - maintainers = ['mjaehn', 'juckerj'] - - # APN tags - version('apn-master', git=git, branch='master') - - # C2SM tags - version('v2.8.4', git=c2smgit, tag='v2.8.4') - version('c2sm-master', git=c2smgit, branch='master') - version('c2sm-features', git=c2smgit, branch='c2sm-features') - - # ORG tags - version('org-master', git=orggit, branch='master') - version('int2lm-3.00', git=orggit, tag='int2lm-3.00') - - depends_on('eccodes +fortran') - # WORKAROUND: A build and link dependency should imply that the same compiler is used. This enforces it. - depends_on('eccodes %nvhpc', when='%nvhpc') - depends_on('eccodes %gcc', when='%gcc') - depends_on('cosmo-eccodes-definitions', type=('build', 'run')) - depends_on('libgrib1 @22-01-2020', type='build') - depends_on('mpi', type=('build', 'link', 'run'), when='+parallel') - depends_on('netcdf-c', type=('build', 'link')) - depends_on('netcdf-fortran', type=('build', 'link')) - depends_on('jasper@1.900.1', type=('build', 'link')) - - variant('debug', default=False, description='Build debug INT2LM') - variant('parallel', default=True, description='Build parallel INT2LM') - variant('pollen', default=True, description='Build with pollen enabled') - variant('slave', default='none', description='Build on slave') - variant('verbose', default=False, description='Build with verbose enabled') - - # from Spack v0.18 we don't load a Python module prior sourcing Spack. - # Therefore #!/usr/bin/env python points to python2. - # Replace with #!/usr/bin/env python3 instead - patch('patches/testsuite/patch.to_python3', - when="@apn-master,c2sm-master,c2sm-features") - - conflicts( - 'pollen=True', - when='@org-master,int2lm-3.00', - msg= - 'int2lm-org is currently broken with pollen, set variant pollen=False') - - conflicts('%cce', msg='cce compiler not supported for int2lm') - - build_directory = 'TESTSUITE' - - def setup_build_environment(self, env): - self.setup_run_environment(env) - - # Eccodes libraries - grib_prefix = self.spec['eccodes'].prefix - env.set( - 'GRIBAPIL', - str(self.spec['eccodes:fortran'].libs.ld_flags) + ' ' + - str(self.spec['jasper'].libs.ld_flags)) - grib_inc_dir_path = os.path.join(grib_prefix, 'include') - if os.path.exists(grib_inc_dir_path): - env.set('GRIBAPII', '-I' + grib_inc_dir_path) - else: - env.set('GRIBAPII', '') - - # Netcdf library - env.set( - 'NETCDFL', - '-L' + self.spec['netcdf-fortran'].prefix + '/lib -lnetcdff -L' + - self.spec['netcdf-c'].prefix + '/lib64 -lnetcdf') - env.set('NETCDFI', - '-I' + self.spec['netcdf-fortran'].prefix + '/include') - - # Grib1 library - if self.compiler.name == 'gcc': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_gnu') - elif self.compiler.name == 'cce': - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_cray') - elif self.compiler.name in ('pgi', 'nvhpc'): - env.set('GRIBDWDL', - '-L' + self.spec['libgrib1'].prefix + '/lib -lgrib1_pgi') - else: - env.set( - 'GRIBDWDL', '-L' + self.spec['libgrib1'].prefix + - '/lib -lgrib1_' + self.compiler.name) - - # MPI library - if self.spec['mpi'].name == 'openmpi': - env.set('MPIL', '-L' + self.spec['mpi'].prefix + ' -lmpi_mpifh') - env.set('MPII', '-I' + self.spec['mpi'].prefix + '/include') - - # Compiler & linker variables - if self.compiler.name == 'pgi': - env.set('F90', 'pgf90 -D__PGI_FORTRAN__') - env.set('LD', 'pgf90 -D__PGI_FORTRAN__') - elif self.compiler.name == 'nvhpc': - env.set('F90', self.spec['mpi'].mpifc + ' -D__PGI_FORTRAN__') - env.set('LD', self.spec['mpi'].mpifc + ' -D__PGI_FORTRAN__') - else: - env.set('F90', self.spec['mpi'].mpifc) - env.set('LD', self.spec['mpi'].mpifc) - - @property - def build_targets(self): - build = [] - if self.spec.variants['verbose'].value: - build.append('VERBOSE=1') - if self.spec.variants['pollen'].value: - build.append('ART=1') - MakeFileTarget = '' - if '+parallel' in self.spec: - MakeFileTarget += 'par' - else: - MakeFileTarget += 'seq' - if '+debug' in self.spec: - MakeFileTarget += 'debug' - else: - MakeFileTarget += 'opt' - build.append(MakeFileTarget) - - return build - - def edit(self, spec, prefix): - print(os.getcwd()) - with working_dir(self.build_directory): - makefile = FileFilter('Makefile') - OptionsFileName = 'Options' - if self.compiler.name == 'gcc': - OptionsFileName += '.gnu' - elif self.compiler.name in ('pgi', 'nvhpc'): - OptionsFileName += '.pgi' - elif self.compiler.name == 'cce': - OptionsFileName += '.cray' - makefile.filter('/Options.*', '/' + OptionsFileName) - - def install(self, spec, prefix): - with working_dir(self.build_directory): - install('int2lm', prefix.bin) - install('int2lm', '../test/testsuite') - - @run_before('edit') - def create_build_directory(self): - os.makedirs(self.build_directory, exist_ok=True) - - # link all files for APN and C2SM version of int2lm - if not os.path.isfile(self.build_directory + '/Options.tsa.pgi'): - os.chdir(self.build_directory) - - for item in os.listdir('../'): - os.symlink('../' + item, item) - - os.chdir('..') - - @run_after('install') - @on_package_attributes(run_tests=True) - def test(self): - with working_dir('test/testsuite'): - try: - subprocess.run( - ['./test_int2lm.py', str(self.spec)], - stderr=subprocess.STDOUT, - check=True) - except: - raise InstallError('Testsuite failed') diff --git a/repos/c2sm/packages/int2lm/patches/testsuite/patch.to_python3 b/repos/c2sm/packages/int2lm/patches/testsuite/patch.to_python3 deleted file mode 100644 index 63efb1c600..0000000000 --- a/repos/c2sm/packages/int2lm/patches/testsuite/patch.to_python3 +++ /dev/null @@ -1,10 +0,0 @@ -diff --git a/test/testsuite/src/testsuite.py b/test/testsuite/src/testsuite.py -index ba56465..3c7b8d0 100755 ---- a/test/testsuite/src/testsuite.py -+++ b/test/testsuite/src/testsuite.py -@@ -1,4 +1,4 @@ --#!/usr/bin/env python -+#!/usr/bin/env python3 - - """ - COSMO TECHNICAL TESTSUITE diff --git a/repos/c2sm/packages/oasis/package.py b/repos/c2sm/packages/oasis/package.py deleted file mode 100644 index e819286f8d..0000000000 --- a/repos/c2sm/packages/oasis/package.py +++ /dev/null @@ -1,154 +0,0 @@ -# Copyright 2013-2018 Lawrence Livermore National Security, LLC and other -# Spack Project Developers. See the top-level COPYRIGHT file for details. -# -# SPDX-License-Identifier: (Apache-2.0 OR MIT) - -from spack.build_systems.makefile import MakefilePackage -from spack.directives import depends_on, version -from llnl.util.filesystem import working_dir, FileFilter, install_tree -import os -import re -from pathlib import Path -from typing import Dict - - -class Oasis(MakefilePackage): - """The OASIS coupler is a software allowing synchronized exchanges - of coupling information between numerical codes representing - different components of the climate system.""" - - homepage = "https://portal.enes.org/oasis" - git = 'https://gitlab.com/cerfacs/oasis3-mct.git' - maintainers = ['leclairm'] - - version('4.0', commit='13a9100e26d6d30a61fe8c3c056768d558adffd9') - - depends_on('mpi', type=('build', 'link', 'run')) - depends_on('netcdf-fortran', type=('build', 'link', 'run')) - - variant( - 'fix_mct_conflict', - default=False, - description= - 'rename mct modules xxx as xxx_oasis to solve conflict with other mct instance at runtime' - ) - - patch('patches/patch.PSMILE_add_timer_as_dep_of_string', when='@4.0') - - build_directory = 'util/make_dir' - - makefile_file = 'TopMakefileOasis3' - - # Relative path where the built libraries are stored (corresponds - # to the absolute path called ARCHDIR in the Makefile) - rel_ARCHDIR = 'spack-build' - - def setup_build_environment(self, env): - - CHAN = 'MPI1' - env.set('CHAN', CHAN) - env.set('F90', self.spec['mpi'].mpifc) - env.set('f90', self.spec['mpi'].mpifc) - env.set('F', self.spec['mpi'].mpifc) - env.set('f', self.spec['mpi'].mpifc) - env.set('MAKE', 'gmake') - - LIBBUILD = os.path.join('../..', self.rel_ARCHDIR, 'build/lib') - INCPSMILE = '-I{LIBBUILD}/psmile.{CHAN} -I{LIBBUILD}/mct -I{LIBBUILD}/scrip'.format( - LIBBUILD=LIBBUILD, CHAN=CHAN) - - CPPDEF = '-Duse_comm_{CHAN} -D__VERBOSE -DTREAT_OVERLAY -D__NO_16BYTE_REALS'.format( - CHAN=CHAN) - env.set('CPPDEF', CPPDEF) - - FFLAGS = '-O2 {INCPSMILE} {CPPDEF}'.format(CPPDEF=CPPDEF, - INCPSMILE=INCPSMILE) - - if self.compiler.name == 'gcc': - FFLAGS += ' -ffree-line-length-512' - - env.set('F90FLAGS', FFLAGS) - env.set('f90FLAGS', FFLAGS) - env.set('FFLAGS', FFLAGS) - env.set('fFLAGS', FFLAGS) - env.set('CCFLAGS', FFLAGS) - - def edit(self, spec, prefix): - - COUPLE = os.getcwd() - ARCHDIR = os.path.join(COUPLE, self.rel_ARCHDIR) - with working_dir(self.build_directory): - makefile = FileFilter(self.makefile_file) - makefile.filter( - 'include make.inc', - 'export COUPLE = {}\nexport ARCHDIR = {}'.format( - COUPLE, ARCHDIR)) - makefile.filter('\$\(modifmakefile\)\s\;\s', '') - - def patch(self): - - # Remove old directives for Fujitsu comilers. Already fixed in MCT [1] but not merged in OASIS yet - # [1] https://github.com/MCSclimate/MCT/commit/dcb4fa4527bbc51729fb67fbc2e0179bfcb4baa2 - with working_dir('lib/mct/mct'): - m_AttrVect = FileFilter('m_AttrVect.F90') - m_AttrVect.filter('\s*\!DIR\$ COLLAPSE', '') - - @run_before('build') - def fix_mct_conflict(self): - """Rename modules mct_xxx as mct_xxx_oasis""" - - # Only do something when the fix_mct_conflict variant is set to True - if not self.spec.variants['fix_mct_conflict'].value: - return - - # Define regexps - re_module = re.compile( - r'^(?P\s*)(?Pmodule|end\s+module)' - r'(?P\s*)(?Pm\w*)(?P.*)$', re.IGNORECASE) - re_use = re.compile( - r'^(?P\s*)use(?P\s*)' - r'(?Pm_\w*)(?P.*)$', re.IGNORECASE) - re_mct_mod = re.compile(r'^(?P.*)mct_mod(?P.*)$', - re.IGNORECASE) - - # File modification function - def mod_file(file: Path, regex_subs: Dict, indent=3) -> None: - - print(' ' * indent + file.name) - data = file.read_text() - data_oasis = [] - for line in data.splitlines(): - for regex, sub_str in regex_subs.items(): - line = regex.sub(sub_str, line) - data_oasis.append(line) - file.write_text('\n'.join(data_oasis) + '\n') - - # Modify mct and mpeu source files - for direc in 'mct', 'mpeu': - print(direc) - for src_file in Path(f'lib/mct/{direc}').glob('*90'): - mod_file( - src_file, { - re_module: - r'\g\g\g\g_oasis\g', - re_use: r'\guse\g\g_oasis\g' - }) - - # Modify psmile source files - print('psmile') - for src_file in Path(f'lib/psmile/src').glob('*90'): - mod_file(src_file, - {re_mct_mod: r'\gmct_mod_oasis\g'}) - - def build(self, spec, prefix): - - with working_dir(self.build_directory): - make('-f', self.makefile_file) - - def install(self, spec, prefix): - - with working_dir(os.path.join(self.rel_ARCHDIR, 'lib')): - os.symlink('libmct.a', 'libmct_oasis.a') - os.symlink('libmpeu.a', 'libmpeu_oasis.a') - - install_tree(self.rel_ARCHDIR, prefix) diff --git a/repos/c2sm/packages/oasis/patches/patch.PSMILE_add_timer_as_dep_of_string b/repos/c2sm/packages/oasis/patches/patch.PSMILE_add_timer_as_dep_of_string deleted file mode 100644 index b9eb171124..0000000000 --- a/repos/c2sm/packages/oasis/patches/patch.PSMILE_add_timer_as_dep_of_string +++ /dev/null @@ -1,13 +0,0 @@ -diff --git a/lib/psmile/src/Makefile b/lib/psmile/src/Makefile -index 50086e0d..94762839 100755 ---- a/lib/psmile/src/Makefile -+++ b/lib/psmile/src/Makefile -@@ -65,7 +65,7 @@ mod_oasis_mpi.o: mod_oasis_kinds.o mod_oasis_data.o mod_oasis_sys.o mod_oasis_ti - mod_oasis_reprosum.o: mod_oasis_kinds.o mod_oasis_data.o mod_oasis_sys.o \ - mod_oasis_timer.o shr_reprosumx86.o mod_oasis_mpi.o - mod_oasis_string.o: mod_oasis_kinds.o mod_oasis_data.o mod_oasis_sys.o \ -- mod_oasis_parameters.o -+ mod_oasis_parameters.o mod_oasis_timer.o - mod_oasis_namcouple.o: mod_oasis_kinds.o mod_oasis_data.o mod_oasis_sys.o \ - mod_oasis_parameters.o mod_oasis_mpi.o mod_oasis_string.o - mod_oasis_part.o: mod_oasis_kinds.o mod_oasis_data.o mod_oasis_sys.o \ diff --git a/test/system_test.py b/test/system_test.py index fb38b1f8ec..659318cb5c 100644 --- a/test/system_test.py +++ b/test/system_test.py @@ -55,13 +55,6 @@ def test_install_icontools(): spack_install('icontools') -def test_install_int2lm_3_00_nvhpc(): - # Tests are disabled because they fail with: - # Error: cmake is a duplicate dependency, with conflicting dependency types - # which stems from the package's 'test_int2lm.py'. - spack_install('int2lm @int2lm-3.00 %nvhpc', test_root=False) - - def test_install_libgrib1_nvhpc(): spack_install('libgrib1 %nvhpc') @@ -72,10 +65,6 @@ def test_install_makedepf90(): spack_install('makedepf90', test_root=False) -def test_install_oasis_nvhpc(): - spack_install('oasis %nvhpc') - - def test_install_py_cytoolz(): spack_install('py-cytoolz') From 7869af50db4bda1f0d3c632fb7b61300193f3f10 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 27 Jan 2025 17:04:18 +0100 Subject: [PATCH 103/104] adjust integration tests --- test/integration_test.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/test/integration_test.py b/test/integration_test.py index 2ed2c2bdd0..aa90c3cace 100644 --- a/test/integration_test.py +++ b/test/integration_test.py @@ -37,14 +37,6 @@ def test_icon_extra_config_args(): "icon extra-config-args=--disable-new_feature,--enable-old_config_arg") -def test_int2lm_parallel(): - spack_spec("int2lm +parallel") - - -def test_int2lm_no_parallel(): - spack_spec("int2lm ~parallel") - - def test_compilers(): run_with_spack("spack compilers", log=log_file("compilers")) From 615da3ecec95b17af36173ce58ee5c49cfaef7e3 Mon Sep 17 00:00:00 2001 From: Jonas Jucker Date: Mon, 27 Jan 2025 18:20:49 +0100 Subject: [PATCH 104/104] upgrade manual to build icon --- docs/QuickStart.rst | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/docs/QuickStart.rst b/docs/QuickStart.rst index 7b17770d9f..6f53efcd19 100644 --- a/docs/QuickStart.rst +++ b/docs/QuickStart.rst @@ -82,8 +82,16 @@ ICON ---- ICON is built using environments. + +For convenience, ICON provides bash scripts to set up the environment and install ICON for in-source +and out-of-source builds. +These scripts are located in ``config/cscs``, e.g. ``config/cscs/alps_mch.cpu.nvidia``. + +For development, sometimes it is necessary to build ICON in a more customized way. +To do so please follow the instructions below. + Environments are located in a folder named after the environment and are defined in a ``spack.yaml`` file. -For ICON, they are located in ``config/cscs/spack//__``. +For ICON, they are located in ``config/cscs/spack/__``. They work with a special Spack tag, that is provided in the ICON repository at ``config/cscs/SPACK_TAG_*``. So make sure you clone Spack with the specified tag. @@ -97,14 +105,16 @@ To install the environment and so ICON, type .. code-block:: console + $ spack develop --path $(pwd) icon@develop $ spack install -Example to build ICON for CPU with NVHPC on Piz Daint: +Example to build ICON for CPU with NVHPC on Balfrin: .. code-block:: console - $ SPACK_TAG=$(cat "config/cscs/SPACK_TAG_DAINT") - $ spack env activate -d config/cscs/spack/$SPACK_TAG/daint_cpu_nvhpc + $ SPACK_TAG=$(cat "config/cscs/SPACK_TAG_MCH") + $ spack env activate -d config/cscs/spack/mch_cpu_double + $ spack develop --path $(pwd) icon@develop $ spack install .. attention:: @@ -116,7 +126,7 @@ Out-of-source builds are possible as follows: .. code-block:: console $ mkdir cpu - $ spack env activate config/cscs/spack/v0.20.1.5/daint_cpu_nvhpc + $ spack env activate -d config/cscs/spack/mch_cpu_double $ # tell spack to build icon in folder cpu $ spack develop --path $(pwd) --build-directory cpu icon@develop $ spack install @@ -134,14 +144,3 @@ By executing the commands above, spack will add some lines directly into ``spack Any further ``spack install`` command will use the build directory specified in the ``spack.yaml`` file. In case you want to change the build directory, edit the ``spack.yaml`` file or remove the ``build_directory`` line and run ``spack concretize -f`` afterwards. - -COSMO ------ - -Building COSMO is not supported anymore starting with spack-c2sm v0.20.1.0 for CSCS. - -For Euler Cluster a CPU-only setup is supported using GCC 8.5.0 - -.. code-block:: console - - $ spack install cosmo%gcc@8.5.0