Update spack submodule from spack/develop as of 2024/01/26 (#977)

* Update submodule pointer for spack
* For spack-ext/lib: env.env_file_config_scope_name() --> env.scope_name
* Change 'spack.config.get(repos, scope=None)' to 'spack.config.get(repos)' in spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py
* Update macOS site packages.yaml and corresponding documentation in doc/source/NewSiteConfigs.rst
* Update openmpi version for macOS CI runs, fix macOS Intel CI by adding gettext with correct (non-sys) path
* Avoid duplicate setting of CMAKE_PREFIX_PATH in configs/common/modules_*.yaml
* Remove variant '+download_boost' for mysql from configs/common/packages.yaml, no longer exists
* Add '--exclude python' to 'spack external find' in documentation and Github CI scripts

.github/workflows/macos-ci-aarch64.yaml

@@ -56,7 +56,9 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system --exclude bison --exclude openssl
+          spack external find --scope system \
+              --exclude bison --exclude openssl \
+              --exclude python
           spack external find --scope system perl
           spack external find --scope system wget
           PATH="/opt/homebrew/opt/curl/bin:$PATH" \
@@ -77,7 +79,7 @@ jobs:
           # *DH
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
@@ -136,7 +138,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
-          module load stack-openmpi/4.1.6
+          module load stack-openmpi/5.0.1
           module load stack-python/3.10.13
           module available
 

.github/workflows/macos-ci-x86_64.yaml

@@ -49,7 +49,11 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system --exclude bison --exclude openssl
+          spack external find --scope system \
+              --exclude bison --exclude openssl  \
+              --exclude python --exclude gettext
+          # Need to find gettext outside of default (presumed to be a system) path for krb5
+          spack external find --path=/usr/local/Cellar/gettext/0.21.1 gettext
           spack external find --scope system perl
           spack external find --scope system wget
           PATH="/usr/local/opt/curl/bin:$PATH" \
@@ -70,7 +74,7 @@ jobs:
           # *DH
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
@@ -127,7 +131,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.0
-          module load stack-openmpi/4.1.6
+          module load stack-openmpi/5.0.1
           module load stack-python/3.10.13
           module available
 

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -49,7 +49,9 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system --exclude bison --exclude openssl
+          spack external find --scope system \
+              --exclude bison --exclude openssl \
+              --exclude python
           spack external find --scope system sed
           spack external find --scope system perl
           spack external find --scope system wget

configs/common/modules_lmod.yaml

@@ -285,5 +285,3 @@ modules:
       - PKG_CONFIG_PATH
       lib64/pkgconfig:
       - PKG_CONFIG_PATH
-      '':
-      - CMAKE_PREFIX_PATH

configs/common/modules_tcl.yaml

@@ -284,5 +284,3 @@ modules:
       - PKG_CONFIG_PATH
       lib64/pkgconfig:
       - PKG_CONFIG_PATH
-      '':
-      - CMAKE_PREFIX_PATH

configs/common/packages.yaml

@@ -40,7 +40,7 @@
     ecbuild:
       version: ['3.7.2']
     eccodes:
-      version: ['2.32.0']
+      version: ['2.33.0']
       variants: +png
     ecflow:
       version: ['5.11.4']
@@ -49,7 +49,7 @@
       version: ['1.24.5']
       variants: linalg=eigen,lapack compression=lz4,bzip2
     ecmwf-atlas:
-      version: ['0.35.1']
+      version: ['0.36.0']
       variants: +fckit +trans +tesselation +fftw
     ectrans:
       version: ['1.2.0']
@@ -113,7 +113,7 @@
       version: ['4.2.15']
       variants: +external-xdr ~fortran ~netcdf
     hdf5:
-      version: ['1.14.0']
+      version: ['1.14.3']
       variants: +hl +fortran +mpi ~threadsafe ~szip
     ip:
       version: ['4.3.0']
@@ -124,8 +124,6 @@
       version: ['2.0.32']
     jedi-cmake:
       version: ['1.4.0']
-    jpeg:
-      version: ['9.1.0']
     landsfcutil:
       version: ['2.4.1']
     libjpeg-turbo:
@@ -148,10 +146,8 @@
       require: "+int64 +real64"
     mpich:
       variants: ~hwloc +two_level_namespace
-    mysql:
-      variants: +download_boost
     nco:
-      version: ['5.0.6']
+      version: ['5.1.6']
       variants: ~doc
     # ncview - when adding information here, also check Orion
     # and Discover site configs
@@ -183,17 +179,17 @@
       version: ['0.3.24']
       variants: +noavx512
     openmpi:
-      variants: +internal-hwloc +two_level_namespace
+      variants: ~internal-hwloc +two_level_namespace
     # Pin openssl to avoid duplicate packages being built
     openssl:
       variants: +shared
     p4est:
       version: ['2.8']
     parallelio:
-      version: ['2.5.10']
+      version: ['2.6.2']
       variants: +pnetcdf
     parallel-netcdf:
-      version: ['1.12.2']
+      version: ['1.12.3']
     pflogger:
       version: ['1.12.0']
       variants: +mpi
@@ -221,6 +217,9 @@
     # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
     py-cython:
       require: "@0.29.36"
+    ## https://github.com/JCSDA/spack-stack/issues/980
+    #py-gitpython:
+    #  require: "@3.1.27"
     py-h5py:
       version: ['3.7.0']
       variants: ~mpi
@@ -237,9 +236,6 @@
       require: ['@1.22.3']
     py-pandas:
       variants: +excel
-    # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer
-    # versions of py-poetry-core when using external/homebrew Python as
-    # we do at the moment in spack-stack.
     # Pin the py-setuptools version to avoid duplicate Python packages
     py-setuptools:
       require: ['@63.4.3']
@@ -263,6 +259,7 @@
       variants: precision=4,d,8
     udunits:
       version: ['2.2.28']
+    # Note - we can remove upp from stack at some point?
     upp:
       version: ['10.0.10']
     w3emc:
@@ -278,7 +275,7 @@
     wrf-io:
       version: ['1.2.0']
     yafyaml:
-      version: ['0.5.1']
+      version: ['1.2.0']
     zlib:
       version: ['1.2.13']
     zstd:

doc/source/NewSiteConfigs.rst

@@ -206,7 +206,9 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
 
 .. code-block:: console
 
-   spack external find --scope system --exclude bison --exclude openssl
+   spack external find --scope system \
+       --exclude bison --exclude openssl \
+       --exclude python
    spack external find --scope system libiconv
    spack external find --scope system perl
    spack external find --scope system wget
@@ -249,7 +251,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
    # Check your clang version then add it to your site compiler config.
    clang --version
    spack config add "packages:all:compiler:[apple-clang@YOUR-VERSION]"
-   spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+   spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
 
 8. If applicable (depends on the environment), edit the main config file for the environment and adjust the compiler matrix to match the compilers for macOS, as above:
 
@@ -449,7 +451,7 @@ It is recommended to increase the stacksize limit by using ``ulimit -S -s unlimi
    spack external find --scope system \
        --exclude bison --exclude cmake \
        --exclude curl --exclude openssl \
-       --exclude openssh
+       --exclude openssh --exclude python
    spack external find --scope system wget
    spack external find --scope system mysql
    spack external find --scope system texlive

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -213,7 +213,7 @@ def write(self):
         # Activate environment
         env = ev.Environment(manifest_dir=env_dir)
         ev.activate(env)
-        env_scope = env.env_file_config_scope_name()
+        env_scope = env.scope_name
 
         # Save original data in spack.yaml because it has higest precedence.
         # spack.config.add will overwrite as it goes.
@@ -271,7 +271,7 @@ def write(self):
             os.makedirs(env_pkgs_path, exist_ok=False)
             with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f:
                 f.write("repo:\n  namespace: envrepo")
-            repo_paths = spack.config.get("repos", scope=spack.config.default_list_scope())
+            repo_paths = spack.config.get("repos")
             repo_paths = [p.replace("$spack/", spack.paths.spack_root + "/") for p in repo_paths]
             for pkg_name in self.modifypkg:
                 pkg_found = False

spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py

@@ -49,7 +49,7 @@ def test_setup_meta_modules():
 
     # Setup env and pretend that a build exists
     # by creating the module directory structure.
-    scope = env.env_file_config_scope_name()
+    scope = env.scope_name
     spack.config.add("packages:all:compiler:[{}]".format(comp), scope=scope)
     spack.config.add("packages:all:providers:mpi:[{}]".format(mpi), scope=scope)
     spack.config.add("packages:openmpi:version:[{}]".format(mpi_ver))