Update spack submodule from spack/develop as of 2024/01/26

.github/workflows/macos-ci-aarch64.yaml

@@ -56,11 +56,7 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system --exclude bison --exclude openssl
-          spack external find --scope system perl
-          spack external find --scope system wget
-          PATH="/opt/homebrew/opt/curl/bin:$PATH" \
-              spack external find --scope system curl
+          spack external find --scope system --exclude bison --exclude openssl --exclude=gettext
           PATH="/opt/homebrew/opt/qt5/bin:$PATH" \
               spack external find --scope system qt
           spack external find --scope system texlive
@@ -77,7 +73,7 @@ jobs:
           # *DH
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
@@ -136,7 +132,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.3
-          module load stack-openmpi/4.1.6
+          module load stack-openmpi/5.0.1
           module load stack-python/3.10.13
           module available
 

.github/workflows/macos-ci-x86_64.yaml

@@ -49,11 +49,7 @@ jobs:
           export SPACK_SYSTEM_CONFIG_PATH="${ENVDIR}/site"
 
           # Find external packages
-          spack external find --scope system --exclude bison --exclude openssl
-          spack external find --scope system perl
-          spack external find --scope system wget
-          PATH="/usr/local/opt/curl/bin:$PATH" \
-              spack external find --scope system curl
+          spack external find --scope system --exclude bison --exclude openssl --exclude=gettext
           PATH="/usr/local/opt/qt5/bin:$PATH" \
               spack external find --scope system qt
           spack external find --scope system texlive
@@ -70,7 +66,7 @@ jobs:
           # *DH
 
           # Set compiler and MPI
-          spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+          spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i '' "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%apple-clang'\]/g" $ENVDIR/spack.yaml
 
@@ -127,7 +123,7 @@ jobs:
 
           module use ${ENVDIR}/install/modulefiles/Core
           module load stack-apple-clang/14.0.0
-          module load stack-openmpi/4.1.6
+          module load stack-openmpi/5.0.1
           module load stack-python/3.10.13
           module available
 

.github/workflows/ubuntu-ci-x86_64.yaml

@@ -113,6 +113,9 @@ jobs:
           spack config add "packages:all:compiler:[[email protected]]"
           sed -i "s/\['\%aocc', '\%apple-clang', '\%gcc', '\%intel'\]/\['\%intel'\]/g" $ENVDIR/spack.yaml
 
+          # Force using external curl to avoid duplicate packages being built
+          spack config add "packages:curl:buildable:false"
+
           # Add additional variants for MET packages, different from config/common/packages.yaml
           spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis"
 

.gitmodules

@@ -1,9 +1,11 @@
 [submodule "spack"]
   path = spack
-  #url = https://github.com/spack/spack
-  #branch = develop
-  url = https://github.com/jcsda/spack
-  branch = jcsda_emc_spack_stack
+  ##url = https://github.com/spack/spack
+  ##branch = develop
+  #url = https://github.com/jcsda/spack
+  #branch = jcsda_emc_spack_stack
+  url = https://github.com/climbfuji/spack
+  branch = feature/merge_spack_dev_20240126
 [submodule "doc/CMakeModules"]
   path = doc/CMakeModules
   url = https://github.com/noaa-emc/cmakemodules

configs/common/packages.yaml

@@ -17,8 +17,8 @@
       version: ['1.27.84']
     bacio:
       version: ['2.4.1']
-    bison:
-      version: ['3.8.2']
+    #bison:
+    #  version: ['3.8.2']
     boost:
       version: ['1.83.0']
       variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden
@@ -31,7 +31,7 @@
       version: ['2.2.0']
       variants: ~openmp
     cmake:
-      version: ['3.23.1']
+      #version: ['3.23.1']
       variants: +ownlibs
     # Attention - when updating also check the various jcsda-emc-bundles env packages
     crtm:
@@ -40,7 +40,7 @@
     ecbuild:
       version: ['3.7.2']
     eccodes:
-      version: ['2.32.0']
+      #version: ['2.33.0']
       variants: +png
     ecflow:
       version: ['5.11.4']
@@ -49,13 +49,13 @@
       version: ['1.24.5']
       variants: linalg=eigen,lapack compression=lz4,bzip2
     ecmwf-atlas:
-      version: ['0.35.1']
+      version: ['0.36.0']
       variants: +fckit +trans +tesselation +fftw
     ectrans:
       version: ['1.2.0']
       variants: ~mkl +fftw
-    eigen:
-      version: ['3.4.0']
+    #eigen:
+    #  version: ['3.4.0']
     # Attention - when updating the version also check the common modules.yaml
     # config and update the projections for lmod/tcl.
     # Also, check the acorn and derecho site configs which have esmf modifications.
@@ -89,8 +89,8 @@
       version: ['1.6.4']
     g2tmpl:
       version: ['1.10.2']
-    gettext:
-      version: ['0.21.1']
+    #gettext:
+    #  version: ['0.21.1']
     gfsio:
       version: ['1.4.1']
     gftl-shared:
@@ -107,13 +107,13 @@
       version: ['1.1.3']
     gsi-ncdiag:
       version: ['1.1.2']
-    gsl-lite:
-      version: ['0.37.0']
+    #gsl-lite:
+    #  version: ['0.37.0']
     hdf:
-      version: ['4.2.15']
+      #version: ['4.2.15']
       variants: +external-xdr ~fortran ~netcdf
     hdf5:
-      version: ['1.14.0']
+      version: ['1.14.3']
       variants: +hl +fortran +mpi ~threadsafe +szip
     ip:
       version: ['4.3.0']
@@ -124,17 +124,17 @@
       version: ['2.0.32']
     jedi-cmake:
       version: ['1.4.0']
-    jpeg:
-      version: ['9.1.0']
+    #jpeg:
+    #  version: ['9.1.0']
     landsfcutil:
       version: ['2.4.1']
-    libjpeg-turbo:
-      version: ['2.1.0']
+    #libjpeg-turbo:
+    #  version: ['2.1.0']
     libpng:
       version: ['1.6.37']
       variants: +pic
-    libyaml:
-      version: ['0.2.5']
+    #libyaml:
+    #  version: ['0.2.5']
     mapl:
       version: ['2.40.3']
       variants: +shared +pflogger ~f2py
@@ -151,7 +151,7 @@
     mysql:
       variants: +download_boost
     nco:
-      version: ['5.0.6']
+      #version: ['5.0.6']
       variants: ~doc
     # ncview - when adding information here, also check Orion
     # and Discover site configs
@@ -172,28 +172,28 @@
     netcdf-fortran:
       version: ['4.6.1']
     # ninja - when adding information here, also check Discover site config
-    nlohmann-json:
-      version: ['3.10.5']
-    nlohmann-json-schema-validator:
-      version: ['2.1.0']
+    #nlohmann-json:
+    #  version: ['3.10.5']
+    #nlohmann-json-schema-validator:
+    #  version: ['2.1.0']
     odc:
       version: ['1.4.6']
       variants: ~fortran
     openblas:
-      version: ['0.3.24']
+      #version: ['0.3.24']
       variants: +noavx512
     openmpi:
-      variants: +internal-hwloc +two_level_namespace
+      variants: ~internal-hwloc +two_level_namespace
     # Pin openssl to avoid duplicate packages being built
     openssl:
       variants: +shared
     p4est:
       version: ['2.8']
     parallelio:
-      version: ['2.5.10']
+      version: ['2.6.2']
       variants: +pnetcdf
     parallel-netcdf:
-      version: ['1.12.2']
+      version: ['1.12.3']
     pflogger:
       version: ['1.12.0']
       variants: +mpi
@@ -205,50 +205,49 @@
     prod-util:
       version: ['2.1.1']
     proj:
-      version: ['8.1.0']
+      #version: ['8.1.0']
       variants: ~tiff
     python:
       require: "@3.10.13"
-    py-attrs:
-      # https://github.com/JCSDA/spack-stack/issues/740
-      version: ['21.4.0']
+    #py-attrs:
+    #  # https://github.com/JCSDA/spack-stack/issues/740
+    #  version: ['21.4.0']
     py-cartopy:
       variants: +plotting
       require: "@0.21.1"
     py-cryptography:
       variants: +rust_bootstrap
+    # This may be a problem for oneAPI?
     # Introduced in https://github.com/JCSDA/spack-stack/pull/894, pin py-cython
     # to avoid duplicate packages being built (cylc dependencies soft-want @3:)
     py-cython:
       require: "@0.29.36"
     py-h5py:
-      version: ['3.7.0']
+    ###   version: ['3.7.0']
       variants: ~mpi
     # Comment out for now until build problems are solved
     # https://github.com/jcsda/spack-stack/issues/522
     # see also ewok-env virtual package and container
     # README.md
     #py-mysql-connector-python:
     #  version: ['8.0.32']
-    py-netcdf4:
-      version: ['1.5.8']
-      variants: ~mpi
+    #py-netcdf4:
+    #  version: ['1.6.2']
+    #  # Does this variant work as expected for 1.6.2?
+    #  variants: ~mpi
     py-numpy:
       require: ['@1.22.3']
     py-pandas:
       variants: +excel
-    # To avoid pip._vendor.pep517.wrappers.BackendInvalid errors with newer
-    # versions of py-poetry-core when using external/homebrew Python as
-    # we do at the moment in spack-stack.
     # Pin the py-setuptools version to avoid duplicate Python packages
     py-setuptools:
       require: ['@63.4.3']
     py-setuptools-rust:
       variants: +rust_bootstrap
     py-shapely:
       require: ['@1.8.0']
-    qt:
-      version: ['5.15.3']
+    #qt:
+    #  version: ['5.15.3']
     scotch:
       version: ['7.0.4']
       variants: +mpi+metis~shared~threads~mpi_thread+noarch
@@ -261,26 +260,27 @@
     sp:
       version: ['2.5.0']
       variants: precision=4,d,8
-    udunits:
-      version: ['2.2.28']
+    #udunits:
+    #  version: ['2.2.28']
+    # Note - we can remove upp from stack at some point?
     upp:
       version: ['10.0.10']
     w3emc:
       version: ['2.10.0']
       variants: precision=4,d,8
     w3nco:
       version: ['2.4.1']
-    wget:
-      version: ['1.21.2']
+    #wget:
+    #  version: ['1.21.2']
     # When changing wgrib2, also check Hercules and Nautilus site configs
     wgrib2:
       version: ['2.0.8']
     wrf-io:
       version: ['1.2.0']
-    yafyaml:
-      version: ['0.5.1']
+    #yafyaml:
+    #  version: ['0.5.1']
     zlib:
       version: ['1.2.13']
     zstd:
-      version: ['1.5.2']
+      version: ['1.5.5']
       variants: +programs

configs/sites/macos.default/packages.yaml

@@ -3,7 +3,5 @@ packages:
     variants: ~png ~svg
   git:
     buildable: false
-  libiconv:
-    buildable: false
   wgrib2:
     variants: ~openmp

doc/source/NewSiteConfigs.rst

@@ -206,18 +206,9 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
 
 .. code-block:: console
 
-   spack external find --scope system --exclude bison --exclude openssl
-   spack external find --scope system libiconv
-   spack external find --scope system perl
-   spack external find --scope system wget
+   spack external find --scope system --exclude bison --exclude openssl --exclude=gettext
    spack external find --scope system mysql
 
-   PATH="$HOMEBREW_ROOT/opt/curl/bin:$PATH" \
-        spack external find --scope system curl
-
-   PATH="$HOMEBREW_ROOT/opt/qt5/bin:$PATH" \
-       spack external find --scope system qt
-
    # Optional, only if planning to build jedi-tools environment with LaTeX support
    # The texlive bin directory must have been added to PATH (see above)
    spack external find --scope system texlive
@@ -249,7 +240,7 @@ Remember to activate the ``lua`` module environment and have MacTeX in your sear
    # Check your clang version then add it to your site compiler config.
    clang --version
    spack config add "packages:all:compiler:[apple-clang@YOUR-VERSION]"
-   spack config add "packages:all:providers:mpi:[openmpi@4.1.6]"
+   spack config add "packages:all:providers:mpi:[openmpi@5.0.1]"
 
 8. If applicable (depends on the environment), edit the main config file for the environment and adjust the compiler matrix to match the compilers for macOS, as above:
 

spack-ext/lib/jcsda-emc/spack-stack/stack/stack_env.py

@@ -213,7 +213,7 @@ def write(self):
         # Activate environment
         env = ev.Environment(manifest_dir=env_dir)
         ev.activate(env)
-        env_scope = env.env_file_config_scope_name()
+        env_scope = env.scope_name
 
         # Save original data in spack.yaml because it has higest precedence.
         # spack.config.add will overwrite as it goes.
@@ -271,7 +271,8 @@ def write(self):
             os.makedirs(env_pkgs_path, exist_ok=False)
             with open(os.path.join(env_repo_path, "repo.yaml"), "w") as f:
                 f.write("repo:\n  namespace: envrepo")
-            repo_paths = spack.config.get("repos", scope=spack.config.default_list_scope())
+            # DH* ???
+            repo_paths = spack.config.get("repos", scope=None) # scope=spack.config.default_list_scope())
             repo_paths = [p.replace("$spack/", spack.paths.spack_root + "/") for p in repo_paths]
             for pkg_name in self.modifypkg:
                 pkg_found = False

spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py

@@ -49,7 +49,7 @@ def test_setup_meta_modules():
 
     # Setup env and pretend that a build exists
     # by creating the module directory structure.
-    scope = env.env_file_config_scope_name()
+    scope = env.scope_name
     spack.config.add("packages:all:compiler:[{}]".format(comp), scope=scope)
     spack.config.add("packages:all:providers:mpi:[{}]".format(mpi), scope=scope)
     spack.config.add("packages:openmpi:version:[{}]".format(mpi_ver))