From 13312198b10994adc84685f3a24d41787c4aa9ae Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 20 Aug 2024 15:20:03 -0600 Subject: [PATCH 01/22] For Intel environments, build boost with %gcc to avoid problems with cereal in ecflow --- configs/common/packages.yaml | 3 ++- configs/common/packages_intel.yaml | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index bbd16064b..a7da104cc 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -17,7 +17,8 @@ packages: require: '@3.4:' # 1.85 incompatible with ecflow@5.11.4 - use latest 'good version' boost: - require: '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' + require: + - '@1.84 ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden' bufr: require: '@12.0.1 +python' cairo: diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 79fa77090..1443184df 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -18,3 +18,6 @@ packages: py-numpy: require: - '^intel-oneapi-mkl' + boost: + require: + - '%gcc' \ No newline at end of file From 032671bf3cfb5b0d903faaa0611f0b43aa511fd5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 20 Aug 2024 16:00:55 -0600 Subject: [PATCH 02/22] Temporarily move workflows out of the way to test ubuntu intel only --- .github/{workflows => tmp}/macos-ci-aarch64.yaml | 0 .github/{workflows => tmp}/ubuntu-ci-x86_64-gnu.yaml | 0 .github/{workflows => tmp}/ubuntu-ci-x86_64-oneapi.yaml | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => tmp}/macos-ci-aarch64.yaml (100%) rename .github/{workflows => tmp}/ubuntu-ci-x86_64-gnu.yaml (100%) rename .github/{workflows => tmp}/ubuntu-ci-x86_64-oneapi.yaml (100%) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/tmp/macos-ci-aarch64.yaml similarity index 100% rename from .github/workflows/macos-ci-aarch64.yaml rename to .github/tmp/macos-ci-aarch64.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-gnu.yaml b/.github/tmp/ubuntu-ci-x86_64-gnu.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-gnu.yaml rename to .github/tmp/ubuntu-ci-x86_64-gnu.yaml diff --git a/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml b/.github/tmp/ubuntu-ci-x86_64-oneapi.yaml similarity index 100% rename from .github/workflows/ubuntu-ci-x86_64-oneapi.yaml rename to .github/tmp/ubuntu-ci-x86_64-oneapi.yaml From eb8b401af5d4e9b331cc94f4b3aba41eb2c75c83 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 20 Aug 2024 16:08:59 -0600 Subject: [PATCH 03/22] Update .github/workflows/ubuntu-ci-x86_64-intel.yaml: remove external ecflow, add external qt@5 --- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index cb44dafaf..b13e66405 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -104,14 +104,13 @@ jobs: #echo " - spec: intel-oneapi-runtime@2023.2.0%oneapi@2024.2.0" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml #echo " prefix: /opt/intel/oneapi" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - # Add external ecflow for Intel + # Add external qt@5 to build ecflow echo "" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " ecflow::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " qt::" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " buildable: False" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml echo " externals:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - spec: ecflow@5.8.4+ui+static_boost" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " modules:" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml - echo " - ecflow/5.11.4" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " - spec: qt@5.15.3" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml + echo " prefix: /usr" >> ${SPACK_SYSTEM_CONFIG_PATH}/packages.yaml export -n SPACK_SYSTEM_CONFIG_PATH @@ -125,8 +124,8 @@ jobs: # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" - # Don't generate ecflow module when using external package - spack config add "modules:default:tcl:exclude:[ecflow]" + # Don't generate qt module when using external package + spack config add "modules:default:tcl:exclude:[qt]" # Concretize and check for duplicates spack concretize 2>&1 | tee log.concretize.intel-2021.10.0-buildcache From d1274887dfa8e8e8eb90cfd9284089d19c7f70bd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Wed, 21 Aug 2024 14:59:57 -0600 Subject: [PATCH 04/22] Add boost to module exclude list for tcl and lmod so that ecflow module for intel still works --- configs/common/modules_lmod.yaml | 1 + configs/common/modules_tcl.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 41bfa24df..3d6da390a 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -23,6 +23,7 @@ modules: - automake - berkeley-db - bison + - boost - bzip2 - diffutils - expat diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index 5ce83e8b4..cf23b05ce 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -25,6 +25,7 @@ modules: - automake - berkeley-db - bison + - boost - bzip2 - diffutils - expat From 40aa2e02df6d262fa61f087bdc38d579c6110abb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 11:47:03 -0600 Subject: [PATCH 05/22] Revert boost module exclusion and require ecflow to be built with GCC for principal compiler Intel --- configs/common/modules_lmod.yaml | 1 - configs/common/modules_tcl.yaml | 1 - configs/common/packages_intel.yaml | 3 +++ 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/configs/common/modules_lmod.yaml b/configs/common/modules_lmod.yaml index 3d6da390a..41bfa24df 100644 --- a/configs/common/modules_lmod.yaml +++ b/configs/common/modules_lmod.yaml @@ -23,7 +23,6 @@ modules: - automake - berkeley-db - bison - - boost - bzip2 - diffutils - expat diff --git a/configs/common/modules_tcl.yaml b/configs/common/modules_tcl.yaml index cf23b05ce..5ce83e8b4 100644 --- a/configs/common/modules_tcl.yaml +++ b/configs/common/modules_tcl.yaml @@ -25,7 +25,6 @@ modules: - automake - berkeley-db - bison - - boost - bzip2 - diffutils - expat diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 1443184df..147630aa1 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -19,5 +19,8 @@ packages: require: - '^intel-oneapi-mkl' boost: + require: + - '%gcc' + ecflow: require: - '%gcc' \ No newline at end of file From f34e6a9baa7084346438e36173de75ea23c43b68 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 11:47:21 -0600 Subject: [PATCH 06/22] [skip ci] From 5d22dd7640440b4b52e592b85ed8625ad8a0cefc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 12:23:21 -0600 Subject: [PATCH 07/22] [skip ci] First attempt to load modules from different compilers in an env using one principal (preferred) compiler --- .../spack-stack/stack/meta_modules.py | 259 +++++++++++------- .../spack-stack/stack/templates/compiler | 7 +- .../spack-stack/stack/templates/compiler.lua | 2 +- .../jcsda-emc/spack-stack/stack/templates/mpi | 2 +- .../spack-stack/stack/templates/mpi.lua | 2 +- 5 files changed, 164 insertions(+), 108 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 305bcaa8c..74438f8d9 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -39,7 +39,7 @@ SUBSTITUTES_TEMPLATE = { "MODULELOADS": "", "MODULEPREREQS": "", - "MODULEPATH": "", + "MODULEPATHS": "", "CC": "", "CXX": "", "F77": "", @@ -85,6 +85,7 @@ def get_matched_dict(root_dir, candidate_list, sub_candidate_list=None): matched_name = None matched_version = None for xdir in dirs: + print("XXX", candidate, xdir) # Partial matches: name/version/ if ("@" in candidate and xdir == candidate.split("@")[0]) or (xdir == candidate): candidate_dir = os.path.join(root_dir, xdir) @@ -198,6 +199,13 @@ def module_prereq_command(module_choice, module): return "" +def modulepath_prepend_command(module_choice, modulepath): + if module_choice == "lmod": + return 'prepend_path("MODULEPATH", "{})\n'.format(modulepath) + else: + return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) + + def substitute_config_vars(config_str): """ Substitute spack-specific and environment variables that may be present @@ -299,15 +307,17 @@ def setup_meta_modules(): "Compiler candidate list is not unique: {}".format(compiler_candidate_list) ) if not len(mpi_candidate_list) == len(set(mpi_candidate_list)): - raise Exception("Compiler candidate list is not unique: {}".format(mpi_candidate_list)) + raise Exception("MPI candidate list is not unique: {}".format(mpi_candidate_list)) # Parse the directory tree under the top-level module directory logging.debug(os.listdir(module_dir)) # First, check for compilers + #raise Exception(compiler_candidate_list) compiler_dict = get_matched_dict(module_dir, compiler_candidate_list) if not compiler_dict: raise Exception("No matching compilers found") logging.info(" ... stack compilers: '{}'".format(compiler_dict)) + print("XXX COMPILER DICT:", compiler_dict) # Then, check for mpi providers - recursively for compilers mpi_dict = get_matched_dict(module_dir, mpi_candidate_list, compiler_candidate_list) if not mpi_dict: @@ -324,6 +334,7 @@ def setup_meta_modules(): compiler_candidate_list, ) compiler_dict = merge_dicts(compiler_dict, compiler_dict_tmp) + print("XXX COMPILER DICT UPDATED:", compiler_dict) # For future use, we need a flattened list of all compilers flattened_compiler_list = [ @@ -349,108 +360,147 @@ def setup_meta_modules(): os.mkdir(meta_module_dir) logging.info(" ... meta module directory : {}".format(meta_module_dir)) + try: + preferred_compilers = spack.config.get("packages")["all"]["prefer"] + except: + raise Exception( + """Unable to detect preferred compiler from environment. + Does the environment have the config entry 'packages:all:prefer?'""" + ) + if len(preferred_compilers)>1: + raise Exception(f"Not supported: more than one preferred compiler: {preferred_compilers}") + preferred_compiler = preferred_compilers[0].replace("%","") + logging.info(" PREFERRED COMPILER: {}".format(preferred_compiler)) + del preferred_compilers + # Custom key function for sorting + def custom_sort_key(entry): + # Return a tuple where the first element is 1 if the entry contains the word, else 0 + # The second element is the entry itself for natural sorting within groups + return (1 if preferred_compiler in entry else 0, entry) + + # Sort the list using the custom key + # add debug logging message here + sorted_flattened_compiler_list = sorted(flattened_compiler_list, key=custom_sort_key) + + # Create compiler modules logging.info("Creating compiler modules ...") compiler_config = spack.config.get("compilers") + # Collecti and save modulepaths for the preferred compiler + MODULEPATHS_SAVE = [] for compiler in compiler_config: # On macOS, since July 2023, spack compiler find creates 'apple-clang@=14.0.0' etc. compiler_spec = compiler["compiler"]["spec"].replace("@=", "@") - if compiler_spec in flattened_compiler_list: + if compiler_spec in sorted_flattened_compiler_list: (compiler_name, compiler_version) = compiler_spec.split("@") - logging.info( - " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) - ) - compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler_name) - compiler_module_file = os.path.join( - compiler_module_dir, compiler_version + MODULE_FILE_EXTENSION[module_choice] - ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler["compiler"]["paths"]["cc"] - substitutes["CXX"] = compiler["compiler"]["paths"]["cxx"] - substitutes["F77"] = compiler["compiler"]["paths"]["f77"] - substitutes["FC"] = compiler["compiler"]["paths"]["fc"] - logging.debug(" ... ... CC : {}".format(substitutes["CC"])) - logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) - logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) - logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) - - # Compiler flags; names are lowercase in spack - for flag_name in compiler["compiler"]["flags"]: - flag_values = compiler["compiler"]["flags"][flag_name] - substitutes["COMPFLAGS"] += setenv_command( - module_choice, flag_name.upper(), flag_values - ) - substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") - logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - - # Existing non-spack modules to load - for module in compiler["compiler"]["modules"]: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - - # Environment variables; case-sensitive in spack - if ( - "environment" in compiler["compiler"].keys() - and compiler["compiler"]["environment"] - ): - # append_path - if "append_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["append_path"]: - env_values = compiler["compiler"]["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["prepend_path"]: - env_values = compiler["compiler"]["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["set"]: - env_values = compiler["compiler"]["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") - logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) - - # Spack compiler module hierarchy - substitutes["MODULEPATH"] = os.path.join(module_dir, compiler_name, compiler_version) - logging.debug(" ... ... MODULEPATH : {}".format(substitutes["MODULEPATH"])) - # If the environment doesn't have compiler-only dependent modules - # then simply create the placeholder directory - if not os.path.isdir(substitutes["MODULEPATH"]): - os.makedirs(substitutes["MODULEPATH"]) + logging.info(" ... bla") + modulepath_save = os.path.join(module_dir, compiler_name, compiler_version) + if not os.path.isdir(modulepath_save): + os.makedirs(modulepath_save) + MODULEPATHS_SAVE.append(modulepath_save) # For tcl modules remove the compiler prefices from the module contents if module_choice == "tcl": logging.info( " ... ... removing compiler prefices from tcl modulefiles in {}".format( - substitutes["MODULEPATH"] + modulepath_save ) ) - for root, ddir, files in os.walk(substitutes["MODULEPATH"]): + for root, ddir, files in os.walk(modulepath_save): for ffile in files: filepath = os.path.join(root, ffile) logging.debug( " ... ... ... removing compiler prefices in {}".format(filepath) ) for pattern in module_replace_patterns: - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, compiler_name, compiler_version, filepath, sed_syntax_fix + for tmp_compiler_spec in sorted_flattened_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Paranoid, but better safe than sorry + del pattern + del tmp_compiler_spec + + if compiler_name in preferred_compiler: + logging.info( + " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) + ) + compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler_name) + compiler_module_file = os.path.join( + compiler_module_dir, compiler_version + MODULE_FILE_EXTENSION[module_choice] + ) + substitutes = SUBSTITUTES_TEMPLATE.copy() + + # Compiler environment variables; names are lowercase in spack + substitutes["CC"] = compiler["compiler"]["paths"]["cc"] + substitutes["CXX"] = compiler["compiler"]["paths"]["cxx"] + substitutes["F77"] = compiler["compiler"]["paths"]["f77"] + substitutes["FC"] = compiler["compiler"]["paths"]["fc"] + logging.debug(" ... ... CC : {}".format(substitutes["CC"])) + logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) + logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) + logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) + + # Compiler flags; names are lowercase in spack + for flag_name in compiler["compiler"]["flags"]: + flag_values = compiler["compiler"]["flags"][flag_name] + substitutes["COMPFLAGS"] += setenv_command( + module_choice, flag_name.upper(), flag_values + ) + substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") + logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) + + # Existing non-spack modules to load + for module in compiler["compiler"]["modules"]: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + + # Environment variables; case-sensitive in spack + if ( + "environment" in compiler["compiler"].keys() + and compiler["compiler"]["environment"] + ): + # append_path + if "append_path" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["append_path"]: + env_values = compiler["compiler"]["environment"]["append_path"][env_name] + substitutes["ENVVARS"] += append_path_command( + module_choice, env_name, env_values + ) + # prepend_path + if "prepend_path" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["prepend_path"]: + env_values = compiler["compiler"]["environment"]["prepend_path"][env_name] + substitutes["ENVVARS"] += prepend_path_command( + module_choice, env_name, env_values + ) + # set + if "set" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["set"]: + env_values = compiler["compiler"]["environment"]["set"][env_name] + substitutes["ENVVARS"] += setenv_command( + module_choice, env_name, env_values ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) + substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") + logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + + # Spack compiler module hierarchy + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + + if not compiler_name in preferred_compiler: + continue # Read compiler template into module_content string with open(COMPILER_TEMPLATES[module_choice]) as f: @@ -466,6 +516,8 @@ def setup_meta_modules(): with open(compiler_module_file, "w") as f: f.write(module_content) logging.info(" ... writing {}".format(compiler_module_file)) + del MODULEPATHS_SAVE + # Create mpi modules for mpi_name in mpi_dict.keys(): @@ -638,24 +690,25 @@ def setup_meta_modules(): substitutes["MPIF77"] = os.path.join("mpif77") substitutes["MPIF90"] = os.path.join("mpif90") - # Spack compiler module hierarchy - substitutes["MODULEPATH"] = os.path.join( + # Spack mpi+compiler module hierarchy + modulepath = os.path.join( module_dir, mpi_name, mpi_version, compiler_name, compiler_version ) - logging.debug(" ... ... MODULEPATH : {}".format(substitutes["MODULEPATH"])) # If the environment doesn't have mpi dependent modules # then simply create the placeholder directory - if not os.path.isdir(substitutes["MODULEPATH"]): - os.makedirs(substitutes["MODULEPATH"]) + if not os.path.isdir(modulepath): + os.makedirs(modulepath) + substitutes["MODULEPATHS"] = modulepath_prepend_command(module_choice, modulepath).rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) # For tcl modules remove the compiler/mpi prefices from the module contents if module_choice == "tcl": logging.info( " ... ... removing compiler/mpi prefices from tcl modulefiles in {}".format( - substitutes["MODULEPATH"] + modulepath ) ) - for root, ddir, files in os.walk(substitutes["MODULEPATH"]): + for root, ddir, files in os.walk(modulepath): for ffile in files: filepath = os.path.join(root, ffile) logging.debug( @@ -666,18 +719,23 @@ def setup_meta_modules(): # Search patterns patterns = ["is-loaded", "module load", "depends-on"] # First, compiler-only dependent modules + # These can depend on other compilers than the + # compiler this MPI was built with, loop over all for pattern in module_replace_patterns: - cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( - pattern, - compiler_name, - compiler_version, - filepath, - sed_syntax_fix, - ) - status = os.system(cmd) - if not status == 0: - raise Exception("Error while calling '{}'".format(cmd)) + for tmp_compiler_spec in sorted_flattened_compiler_list: + (tmp_compiler_name, tmp_compiler_version) = tmp_compiler_spec.split("@") + cmd = "sed -i {4} 's#{0} {1}/{2}/#{0} #g' {3}".format( + pattern, tmp_compiler_name, tmp_compiler_version, filepath, sed_syntax_fix + ) + status = os.system(cmd) + if not status == 0: + raise Exception("Error while calling '{}'".format(cmd)) + # Paranoid, but better safe than sorry + del pattern + del tmp_compiler_spec # Then, compiler+mpi-dependent modules + # By definition, these can only depend on the compiler that + # this MPI was built with - no need to loop over all compilers for pattern in module_replace_patterns: cmd = "sed -i {6} 's#{0} {1}/{2}/{3}/{4}/#{0} #g' {5}".format( pattern, @@ -731,9 +789,6 @@ def setup_meta_modules(): not compiler_name in python_dict.keys() or not compiler_version in python_dict[compiler_name].keys() ): - logging.warn( - f"No Python version found for compiler {compiler_name}@{compiler_version}, skipping Python modules" - ) continue spec = python_dict[compiler_name][compiler_version] python_version = str(spec.version) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler index 24725a82c..78891450f 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler @@ -4,18 +4,19 @@ # # conflicts +conflict stack-aocc conflict stack-apple-clang conflict stack-clang -conflict stack-gnu +conflict stack-gcc conflict stack-intel -conflict stack-intel-oneapi-compilers +conflict stack-oneapi # prerequisite modules @MODULELOADS@ @MODULEPREREQS@ # spack compiler module hierarchy -prepend-path {MODULEPATH} {@MODULEPATH@} +@MODULEPATHS@ # compiler environment variables setenv {F77} {@F77@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 21daea763..594e52436 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -19,7 +19,7 @@ conflict("stack-intel-oneapi-compilers") @MODULEPREREQS@ -- spack compiler module hierarchy -prepend_path("MODULEPATH", "@MODULEPATH@") +@MODULEPATHS@ -- compiler environment variables setenv("F77", "@F77@") diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi index d239fd5d5..188d50946 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi @@ -15,7 +15,7 @@ conflict stack-mpt @MODULEPREREQS@ # spack compiler module hierarchy -prepend-path {MODULEPATH} {@MODULEPATH@} +@MODULEPATHS@ # mpi wrapper environment variables setenv {MPICC} {@MPICC@} diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua index bf805321c..c84007f84 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/mpi.lua @@ -19,7 +19,7 @@ conflict("stack-mpt") @MODULEPREREQS@ -- spack compiler module hierarchy -prepend_path("MODULEPATH", "@MODULEPATH@") +@MODULEPATHS@ -- mpi wrapper environment variables setenv("MPICC", "@MPICC@") From d72d4ebfb10eb5b990bfc898669a3cf4598205dc Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 13:48:11 -0600 Subject: [PATCH 08/22] Add gcc@11.4.0 to list of compilers in .github/workflows/ubuntu-ci-x86_64-intel.yaml --- .github/workflows/ubuntu-ci-x86_64-intel.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ubuntu-ci-x86_64-intel.yaml b/.github/workflows/ubuntu-ci-x86_64-intel.yaml index b13e66405..a15ff1c25 100644 --- a/.github/workflows/ubuntu-ci-x86_64-intel.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-intel.yaml @@ -119,7 +119,7 @@ jobs: # Set compiler and MPI specs spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.10.0]" - spack config add "packages:all:compiler:[intel@2021.10.0]" + spack config add "packages:all:compiler:[intel@2021.10.0, gcc@11.4.0]" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" From 709060d5370a44eafe4841b23cc249f567552491 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 13:54:34 -0600 Subject: [PATCH 09/22] Update Narwhal site config: add correct gcc compiler to intel environment --- configs/sites/tier1/narwhal/compilers.yaml | 4 ++-- configs/sites/tier1/narwhal/packages_intel.yaml | 2 +- doc/source/PreConfiguredSites.rst | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index eb74a5993..a136b20ca 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -36,11 +36,11 @@ compilers:: modules: - PrgEnv-gnu/8.3.3 - gcc/10.3.0 - - cray-libsci/22.11.1.2 + - cray-libsci/23.05.1.4 - libfabric/1.12.1.2.2.1 environment: prepend_path: - LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/22.11.1.2/GNU/9.1/x86_64/lib' + LD_LIBRARY_PATH: '/opt/cray/libfabric/1.12.1.2.2.1/lib64:/opt/cray/pe/libsci/23.05.1.4/GNU/10.3/x86_64/lib' set: CRAYPE_LINK_TYPE: 'dynamic' extra_rpaths: [] diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml index b8effd96d..333e86236 100644 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ b/configs/sites/tier1/narwhal/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0, gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.21] mpi: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index b80cd7164..c6d135a59 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -288,7 +288,7 @@ With GNU, the following is required for building new spack environments and for module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci - module load cray-libsci/22.11.1.2 + module load cray-libsci/23.05.1.4 THIS SECTION IS OUT OF DATE, REFER TO 1.7.0 RELEASE DOCUMENTATION - For ``spack-stack-1.7.0`` with GNU, proceed with loading the following modules: From 0bb27fe78ab9fdf9d53106d173323929869632b2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 13:54:45 -0600 Subject: [PATCH 10/22] [skip ci] From edabc30affb44d1b94027ed49c405bf342e73ec9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 15:24:16 -0600 Subject: [PATCH 11/22] Switch to no cray wrappers on Narwhal for Intel --- configs/sites/tier1/narwhal/compilers.yaml | 13 +++++-------- configs/sites/tier1/narwhal/packages_intel.yaml | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index a136b20ca..f0300cef6 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -2,14 +2,11 @@ compilers:: - compiler: spec: intel@2021.10.0 paths: - cc: cc - cxx: CC - f77: ftn - fc: ftn - flags: - cflags: -diag-disable=10441 - cxxflags: -diag-disable=10441 - fflags: -diag-disable=10448 + cc: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/icc + cxx: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/icpc + f77: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/ifort + fc: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/ifort + flags: {} operating_system: sles15 modules: - PrgEnv-intel/8.3.3 diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml index 333e86236..f090a84d5 100644 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ b/configs/sites/tier1/narwhal/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers + - spec: cray-mpich@8.1.21%intel@2021.10.0 +wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx From 433f3f40b397a768bd72f9db85d0a2896a1b82e3 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Thu, 22 Aug 2024 20:52:37 -0600 Subject: [PATCH 12/22] Update configs/sites/**: for Intel, define gcc backend compiler in compilers.yaml if not already there and list in packages_intel.yaml (or packages.yaml if only an Intel build exists). Still todo: atlantis, gaea-c5, gaea-c6 (added notes) --- configs/sites/tier1/acorn/compilers.yaml | 22 ++++++++++ configs/sites/tier1/acorn/packages_intel.yaml | 2 +- configs/sites/tier1/atlantis/compilers.yaml | 1 + configs/sites/tier1/atlantis/packages.yaml | 2 +- .../tier1/aws-pcluster/packages_intel.yaml | 2 +- .../sites/tier1/derecho/packages_intel.yaml | 2 +- .../sites/tier1/discover-scu16/compilers.yaml | 15 +++++++ .../tier1/discover-scu16/packages_intel.yaml | 2 +- .../sites/tier1/discover-scu17/compilers.yaml | 15 +++++++ .../tier1/discover-scu17/packages_intel.yaml | 2 +- configs/sites/tier1/gaea-c5/compilers.yaml | 1 + configs/sites/tier1/gaea-c5/packages.yaml | 2 +- configs/sites/tier1/gaea-c6/compilers.yaml | 1 + configs/sites/tier1/gaea-c6/packages.yaml | 2 +- configs/sites/tier1/hera/packages_intel.yaml | 2 +- .../sites/tier1/hercules/packages_intel.yaml | 2 +- configs/sites/tier1/jet/packages_intel.yaml | 2 +- .../sites/tier1/narwhal/packages_intel.yaml | 2 +- configs/sites/tier1/nautilus/compilers.yaml | 41 +++++++++++++------ .../sites/tier1/nautilus/packages_intel.yaml | 2 +- .../sites/tier1/nautilus/packages_oneapi.yaml | 2 +- .../sites/tier1/noaa-aws/packages_intel.yaml | 2 +- .../tier1/noaa-azure/packages_intel.yaml | 2 +- .../tier1/noaa-gcloud/packages_intel.yaml | 2 +- configs/sites/tier1/orion/packages_intel.yaml | 2 +- configs/sites/tier1/s4/packages.yaml | 2 +- .../tier2/blackpearl/packages_oneapi.yaml | 2 +- 27 files changed, 103 insertions(+), 33 deletions(-) diff --git a/configs/sites/tier1/acorn/compilers.yaml b/configs/sites/tier1/acorn/compilers.yaml index a85b80871..e118f32c3 100644 --- a/configs/sites/tier1/acorn/compilers.yaml +++ b/configs/sites/tier1/acorn/compilers.yaml @@ -70,3 +70,25 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Needed for Intel 19/2022 backend +- compiler: + spec: gcc@10.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + modules: + - PrgEnv-gnu/8.3.3 + - craype/2.7.13 + - gcc/10.2.0 + - libfabric + environment: + set: + # OpenSUSE on WCOSS2 machines sets CONFIG_SITE so + # Automake-based builds are installed in lib64 + # which confuses some packages. + CONFIG_SITE: '' + extra_rpaths: [] diff --git a/configs/sites/tier1/acorn/packages_intel.yaml b/configs/sites/tier1/acorn/packages_intel.yaml index 13c5246fd..34065c870 100644 --- a/configs/sites/tier1/acorn/packages_intel.yaml +++ b/configs/sites/tier1/acorn/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@19.1.3.304,intel@2022.0.2.262] + compiler:: [intel@19.1.3.304,intel@2022.0.2.262,gcc@10.2.0] providers: mpi:: [cray-mpich@8.1.9] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml index 05b5731e6..c4ad28b31 100644 --- a/configs/sites/tier1/atlantis/compilers.yaml +++ b/configs/sites/tier1/atlantis/compilers.yaml @@ -16,3 +16,4 @@ compilers: prepend_path: LD_LIBRARY_PATH: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin extra_rpaths: [] +# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) \ No newline at end of file diff --git a/configs/sites/tier1/atlantis/packages.yaml b/configs/sites/tier1/atlantis/packages.yaml index 0fb097d39..2e0c5976a 100644 --- a/configs/sites/tier1/atlantis/packages.yaml +++ b/configs/sites/tier1/atlantis/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.6.0] + compiler:: [intel@2021.6.0] # todo: add gcc here providers: mpi:: [intel-oneapi-mpi@2021.6.0] diff --git a/configs/sites/tier1/aws-pcluster/packages_intel.yaml b/configs/sites/tier1/aws-pcluster/packages_intel.yaml index bc5f464d4..1363ff58f 100644 --- a/configs/sites/tier1/aws-pcluster/packages_intel.yaml +++ b/configs/sites/tier1/aws-pcluster/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@11.4.0] providers: mpi:: [intel-oneapi-mpi@2021.10.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/derecho/packages_intel.yaml b/configs/sites/tier1/derecho/packages_intel.yaml index edf995c8f..f37ddcacf 100644 --- a/configs/sites/tier1/derecho/packages_intel.yaml +++ b/configs/sites/tier1/derecho/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@12.2.0] providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/discover-scu16/compilers.yaml b/configs/sites/tier1/discover-scu16/compilers.yaml index 84cfc9b9a..949f2b8b5 100644 --- a/configs/sites/tier1/discover-scu16/compilers.yaml +++ b/configs/sites/tier1/discover-scu16/compilers.yaml @@ -34,3 +34,18 @@ compilers: - comp/gcc/12.1.0 environment: {} extra_rpaths: [] +# Needed for Intel backend +- compiler: + spec: gcc@=11.2.0 + paths: + cc: /usr/local/other/gcc/11.2.0/bin/gcc + cxx: /usr/local/other/gcc/11.2.0/bin/g++ + f77: /usr/local/other/gcc/11.2.0/bin/gfortran + fc: /usr/local/other/gcc/11.2.0/bin/gfortran + flags: {} + operating_system: sles12 + target: x86_64 + modules: + - comp/gcc/11.2.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/discover-scu16/packages_intel.yaml b/configs/sites/tier1/discover-scu16/packages_intel.yaml index 139fd869c..8bd48da65 100644 --- a/configs/sites/tier1/discover-scu16/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu16/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.6.0] + compiler:: [intel@2021.6.0,gcc@11.2.0] providers: mpi:: [intel-oneapi-mpi@2021.6.0] mpi: diff --git a/configs/sites/tier1/discover-scu17/compilers.yaml b/configs/sites/tier1/discover-scu17/compilers.yaml index 4ea0fa10d..989c471c2 100644 --- a/configs/sites/tier1/discover-scu17/compilers.yaml +++ b/configs/sites/tier1/discover-scu17/compilers.yaml @@ -51,3 +51,18 @@ compilers: - comp/gcc/12.3.0 environment: {} extra_rpaths: [] +# Needed for Intel backend +- compiler: + spec: gcc@=11.4.0 + paths: + cc: /usr/local/other/gcc/11.4.0/bin/gcc + cxx: /usr/local/other/gcc/11.4.0/bin/g++ + f77: /usr/local/other/gcc/11.4.0/bin/gfortran + fc: /usr/local/other/gcc/11.4.0/bin/gfortran + flags: {} + operating_system: sles15 + target: x86_64 + modules: + - comp/gcc/11.4.0 + environment: {} + extra_rpaths: [] diff --git a/configs/sites/tier1/discover-scu17/packages_intel.yaml b/configs/sites/tier1/discover-scu17/packages_intel.yaml index 0595c84a7..3512790ab 100644 --- a/configs/sites/tier1/discover-scu17/packages_intel.yaml +++ b/configs/sites/tier1/discover-scu17/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0] + compiler:: [intel@2021.10.0,gcc@11.4.0] providers: mpi:: [intel-oneapi-mpi@2021.10.0] mpi: diff --git a/configs/sites/tier1/gaea-c5/compilers.yaml b/configs/sites/tier1/gaea-c5/compilers.yaml index 8aba0ea69..b041afe99 100644 --- a/configs/sites/tier1/gaea-c5/compilers.yaml +++ b/configs/sites/tier1/gaea-c5/compilers.yaml @@ -23,3 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) \ No newline at end of file diff --git a/configs/sites/tier1/gaea-c5/packages.yaml b/configs/sites/tier1/gaea-c5/packages.yaml index 99b4bae6a..da5173fad 100644 --- a/configs/sites/tier1/gaea-c5/packages.yaml +++ b/configs/sites/tier1/gaea-c5/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2023.1.0] + compiler:: [intel@2023.1.0] # todo: add gcc here providers: mpi:: [cray-mpich@8.1.25] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index f34608197..180fb1803 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -23,3 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] +# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) \ No newline at end of file diff --git a/configs/sites/tier1/gaea-c6/packages.yaml b/configs/sites/tier1/gaea-c6/packages.yaml index 321b33a7b..92f7d0a7c 100644 --- a/configs/sites/tier1/gaea-c6/packages.yaml +++ b/configs/sites/tier1/gaea-c6/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2023.2.0] + compiler:: [intel@2023.2.0] # todo: add gcc here providers: mpi:: [cray-mpich@8.1.29] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/hera/packages_intel.yaml b/configs/sites/tier1/hera/packages_intel.yaml index 983a808db..4f8025c5f 100644 --- a/configs/sites/tier1/hera/packages_intel.yaml +++ b/configs/sites/tier1/hera/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.5.1] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/hercules/packages_intel.yaml b/configs/sites/tier1/hercules/packages_intel.yaml index 1919c5d1d..78339c1c5 100644 --- a/configs/sites/tier1/hercules/packages_intel.yaml +++ b/configs/sites/tier1/hercules/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.9.0] + compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/jet/packages_intel.yaml b/configs/sites/tier1/jet/packages_intel.yaml index a31345813..d9cdd7209 100644 --- a/configs/sites/tier1/jet/packages_intel.yaml +++ b/configs/sites/tier1/jet/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.5.1] mpi: diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml index f090a84d5..e415e07ca 100644 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ b/configs/sites/tier1/narwhal/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.10.0, gcc@10.3.0] + compiler:: [intel@2021.10.0,gcc@10.3.0] providers: mpi:: [cray-mpich@8.1.21] mpi: diff --git a/configs/sites/tier1/nautilus/compilers.yaml b/configs/sites/tier1/nautilus/compilers.yaml index c3d08c859..c4460ac05 100644 --- a/configs/sites/tier1/nautilus/compilers.yaml +++ b/configs/sites/tier1/nautilus/compilers.yaml @@ -35,6 +35,25 @@ compilers: CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/p/app/compilers/intel/oneapi/compiler/2022.0.2/linux/compiler/lib/intel64_lin:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' extra_rpaths: [] +- compiler: + spec: oneapi@2024.1.2 + paths: + cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx + cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx + f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + flags: {} + operating_system: rhel8 + target: x86_64 + modules: [] + environment: + prepend_path: + PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' + CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' + LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' + append_path: + CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + extra_rpaths: [] - compiler: spec: gcc@12.2.1 paths: @@ -49,22 +68,18 @@ compilers: - scl/gcc-toolset-12 environment: {} extra_rpaths: [] +# Required for Intel backend - compiler: - spec: oneapi@2024.1.2 + spec: gcc@11.2.1 paths: - cc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icx - cxx: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/icpx - f77: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort - fc: /p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/bin/ifort + cc: /opt/rh/gcc-toolset-11/root/usr/bin/gcc + cxx: /opt/rh/gcc-toolset-11/root/usr/bin/g++ + f77: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran + fc: /opt/rh/gcc-toolset-11/root/usr/bin/gfortran flags: {} operating_system: rhel8 target: x86_64 - modules: [] - environment: - prepend_path: - PATH: '/opt/rh/gcc-toolset-11/root/usr/bin' - CPATH: '/opt/rh/gcc-toolset-11/root/usr/include' - LD_LIBRARY_PATH: '/opt/scyld/slurm/lib64:/opt/scyld/slurm/lib64/slurm:/opt/rh/gcc-toolset-11/root/usr/lib64:/opt/rh/gcc-toolset-11/root/usr/lib' - append_path: - CPATH: '/p/app/projects/NEPTUNE/spack-stack/oneapi-2024.1.2/compiler/2024.1/opt/compiler/include/intel64' + modules: + - scl/gcc-toolset-11 + environment: {} extra_rpaths: [] diff --git a/configs/sites/tier1/nautilus/packages_intel.yaml b/configs/sites/tier1/nautilus/packages_intel.yaml index 625387414..7516471db 100644 --- a/configs/sites/tier1/nautilus/packages_intel.yaml +++ b/configs/sites/tier1/nautilus/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@11.2.1] providers: mpi:: [openmpi@4.1.6] mpi: diff --git a/configs/sites/tier1/nautilus/packages_oneapi.yaml b/configs/sites/tier1/nautilus/packages_oneapi.yaml index 40d0378e7..ce40a89e7 100644 --- a/configs/sites/tier1/nautilus/packages_oneapi.yaml +++ b/configs/sites/tier1/nautilus/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2] + compiler:: [oneapi@2024.1.2,gcc@11.2.1] providers: mpi:: [intel-oneapi-mpi@2021.12] mpi: diff --git a/configs/sites/tier1/noaa-aws/packages_intel.yaml b/configs/sites/tier1/noaa-aws/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-aws/packages_intel.yaml +++ b/configs/sites/tier1/noaa-aws/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/noaa-azure/packages_intel.yaml b/configs/sites/tier1/noaa-azure/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-azure/packages_intel.yaml +++ b/configs/sites/tier1/noaa-azure/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml index c125f2911..995404d2d 100644 --- a/configs/sites/tier1/noaa-gcloud/packages_intel.yaml +++ b/configs/sites/tier1/noaa-gcloud/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.3.0] + compiler:: [intel@2021.3.0,gcc@9.2.0] providers: mpi:: [intel-oneapi-mpi@2021.3.0] # Remove the next three lines to switch to intel-oneapi-mkl diff --git a/configs/sites/tier1/orion/packages_intel.yaml b/configs/sites/tier1/orion/packages_intel.yaml index 193d5aac0..88131d85c 100644 --- a/configs/sites/tier1/orion/packages_intel.yaml +++ b/configs/sites/tier1/orion/packages_intel.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.9.0] + compiler:: [intel@2021.9.0,gcc@12.2.0] providers: mpi:: [intel-oneapi-mpi@2021.9.0] # https://github.com/JCSDA/spack-stack/issues/1055 diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index a5e542ffd..5e7ac0dc4 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [intel@2021.5.0] + compiler:: [intel@2021.5.0,gcc@9.3.0] providers: mpi:: [intel-oneapi-mpi@2021.5.0] # https://github.com/JCSDA/spack-stack/issues/1055 diff --git a/configs/sites/tier2/blackpearl/packages_oneapi.yaml b/configs/sites/tier2/blackpearl/packages_oneapi.yaml index 8466a209c..f4c243c1b 100644 --- a/configs/sites/tier2/blackpearl/packages_oneapi.yaml +++ b/configs/sites/tier2/blackpearl/packages_oneapi.yaml @@ -1,6 +1,6 @@ packages: all: - compiler:: [oneapi@2024.1.2] + compiler:: [oneapi@2024.1.2,gcc@11.4.1] providers: mpi:: [intel-oneapi-mpi] From 7091de5f870ca4e2685f2a5aa85f48364d79ae92 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 07:12:19 -0600 Subject: [PATCH 13/22] Add missing newlines at end of file in various config files --- configs/common/packages_intel.yaml | 2 +- configs/sites/tier1/atlantis/compilers.yaml | 2 +- configs/sites/tier1/gaea-c5/compilers.yaml | 2 +- configs/sites/tier1/gaea-c6/compilers.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/configs/common/packages_intel.yaml b/configs/common/packages_intel.yaml index 147630aa1..15c38b784 100644 --- a/configs/common/packages_intel.yaml +++ b/configs/common/packages_intel.yaml @@ -23,4 +23,4 @@ packages: - '%gcc' ecflow: require: - - '%gcc' \ No newline at end of file + - '%gcc' diff --git a/configs/sites/tier1/atlantis/compilers.yaml b/configs/sites/tier1/atlantis/compilers.yaml index c4ad28b31..3c9b262ba 100644 --- a/configs/sites/tier1/atlantis/compilers.yaml +++ b/configs/sites/tier1/atlantis/compilers.yaml @@ -16,4 +16,4 @@ compilers: prepend_path: LD_LIBRARY_PATH: /cm/shared/apps/intel/oneapi/compiler/2022.1.0/linux/compiler/lib/intel64_lin extra_rpaths: [] -# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) \ No newline at end of file +# Todo: Define gcc compiler (default for Atlantis) for Intel (ecflow) diff --git a/configs/sites/tier1/gaea-c5/compilers.yaml b/configs/sites/tier1/gaea-c5/compilers.yaml index b041afe99..e60d7fece 100644 --- a/configs/sites/tier1/gaea-c5/compilers.yaml +++ b/configs/sites/tier1/gaea-c5/compilers.yaml @@ -23,4 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) \ No newline at end of file +# Todo: Define gcc compiler 10.3.0 for Intel backend (ecflow) diff --git a/configs/sites/tier1/gaea-c6/compilers.yaml b/configs/sites/tier1/gaea-c6/compilers.yaml index 180fb1803..1bca116f0 100644 --- a/configs/sites/tier1/gaea-c6/compilers.yaml +++ b/configs/sites/tier1/gaea-c6/compilers.yaml @@ -23,4 +23,4 @@ compilers: # which confuses some packages. CONFIG_SITE: '' extra_rpaths: [] -# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) \ No newline at end of file +# Todo: Define gcc compiler 12.x.y for Intel backend (ecflow) From c31c506b8eb4d6531dd5c6a4b52cb71c1bc17aef Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 07:12:33 -0600 Subject: [PATCH 14/22] Half way through cleaning up spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- .../lib/jcsda-emc/spack-stack/stack/meta_modules.py | 11 ++++++----- .../spack-stack/stack/templates/compiler.lua | 5 +++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index 74438f8d9..fe25a1341 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -312,12 +312,11 @@ def setup_meta_modules(): # Parse the directory tree under the top-level module directory logging.debug(os.listdir(module_dir)) # First, check for compilers - #raise Exception(compiler_candidate_list) compiler_dict = get_matched_dict(module_dir, compiler_candidate_list) if not compiler_dict: raise Exception("No matching compilers found") logging.info(" ... stack compilers: '{}'".format(compiler_dict)) - print("XXX COMPILER DICT:", compiler_dict) + # Then, check for mpi providers - recursively for compilers mpi_dict = get_matched_dict(module_dir, mpi_candidate_list, compiler_candidate_list) if not mpi_dict: @@ -334,7 +333,6 @@ def setup_meta_modules(): compiler_candidate_list, ) compiler_dict = merge_dicts(compiler_dict, compiler_dict_tmp) - print("XXX COMPILER DICT UPDATED:", compiler_dict) # For future use, we need a flattened list of all compilers flattened_compiler_list = [ @@ -343,6 +341,7 @@ def setup_meta_modules(): for version in compiler_dict[name] ] + # Determine core compiler(s) and make sure they are not used (usually something ancient) core_compilers = module_config["default"][module_choice]["core_compilers"] logging.info(" ... core compilers: {}".format(core_compilers)) # Check that none of the compilers used for the stack is a core compiler @@ -360,6 +359,10 @@ def setup_meta_modules(): os.mkdir(meta_module_dir) logging.info(" ... meta module directory : {}".format(meta_module_dir)) + # Determine the preferred compiler and sort the flattened list of compilers + # such that the preferred compiler comes last. This is so that all other + # compilers populate the MODULEPATHS_SAVE list before the preferred compiler + # takes it and adds it to the stack-COMPILER metamodule. try: preferred_compilers = spack.config.get("packages")["all"]["prefer"] except: @@ -382,7 +385,6 @@ def custom_sort_key(entry): # add debug logging message here sorted_flattened_compiler_list = sorted(flattened_compiler_list, key=custom_sort_key) - # Create compiler modules logging.info("Creating compiler modules ...") compiler_config = spack.config.get("compilers") @@ -518,7 +520,6 @@ def custom_sort_key(entry): logging.info(" ... writing {}".format(compiler_module_file)) del MODULEPATHS_SAVE - # Create mpi modules for mpi_name in mpi_dict.keys(): for mpi_version in mpi_dict[mpi_name].keys(): diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua index 594e52436..2f741ddbc 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/templates/compiler.lua @@ -8,11 +8,12 @@ local pkgNameVer = myModuleFullName() family("MetaCompiler") -- conflicts +conflict("stack-aocc") conflict("stack-apple-clang") conflict("stack-clang") -conflict("stack-gnu") +conflict("stack-gcc") conflict("stack-intel") -conflict("stack-intel-oneapi-compilers") +conflict("stack-oneapi") -- prerequisite modules @MODULELOADS@ From c01018bf21e6bf9681e5b45867dbc9547e5dc662 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 07:13:53 -0600 Subject: [PATCH 15/22] Move other CI workflows back in place and add gcc to compiler list in .github/workflows/ubuntu-ci-x86_64-oneapi.yaml --- .github/{tmp => workflows}/macos-ci-aarch64.yaml | 0 .github/{tmp => workflows}/ubuntu-ci-x86_64-gnu.yaml | 0 .github/{tmp => workflows}/ubuntu-ci-x86_64-oneapi.yaml | 2 +- 3 files changed, 1 insertion(+), 1 deletion(-) rename .github/{tmp => workflows}/macos-ci-aarch64.yaml (100%) rename .github/{tmp => workflows}/ubuntu-ci-x86_64-gnu.yaml (100%) rename .github/{tmp => workflows}/ubuntu-ci-x86_64-oneapi.yaml (99%) diff --git a/.github/tmp/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml similarity index 100% rename from .github/tmp/macos-ci-aarch64.yaml rename to .github/workflows/macos-ci-aarch64.yaml diff --git a/.github/tmp/ubuntu-ci-x86_64-gnu.yaml b/.github/workflows/ubuntu-ci-x86_64-gnu.yaml similarity index 100% rename from .github/tmp/ubuntu-ci-x86_64-gnu.yaml rename to .github/workflows/ubuntu-ci-x86_64-gnu.yaml diff --git a/.github/tmp/ubuntu-ci-x86_64-oneapi.yaml b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml similarity index 99% rename from .github/tmp/ubuntu-ci-x86_64-oneapi.yaml rename to .github/workflows/ubuntu-ci-x86_64-oneapi.yaml index 44eeec3aa..b36f5a5ce 100644 --- a/.github/tmp/ubuntu-ci-x86_64-oneapi.yaml +++ b/.github/workflows/ubuntu-ci-x86_64-oneapi.yaml @@ -110,7 +110,7 @@ jobs: # Set compiler and MPI specs spack config add "packages:all:providers:mpi:[intel-oneapi-mpi@2021.13]" - spack config add "packages:all:compiler:[oneapi@2024.2.0]" + spack config add "packages:all:compiler:[oneapi@2024.2.0,gcc@11.4.0]" # Add additional variants for MET packages, different from config/common/packages.yaml spack config add "packages:met:variants:+python +grib2 +graphics +lidar2nc +modis" From d13707a8beaa6c34084e4ef701f879f74fec76b0 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 08:38:56 -0600 Subject: [PATCH 16/22] Trigger CI From 99aa58d900e2e14ecbfea4f9235cbd7aa5acd153 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 08:55:26 -0600 Subject: [PATCH 17/22] Clean up changes in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- .../spack-stack/stack/meta_modules.py | 152 +++++++++--------- 1 file changed, 74 insertions(+), 78 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index fe25a1341..e92acf173 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -85,7 +85,6 @@ def get_matched_dict(root_dir, candidate_list, sub_candidate_list=None): matched_name = None matched_version = None for xdir in dirs: - print("XXX", candidate, xdir) # Partial matches: name/version/ if ("@" in candidate and xdir == candidate.split("@")[0]) or (xdir == candidate): candidate_dir = os.path.join(root_dir, xdir) @@ -373,16 +372,13 @@ def setup_meta_modules(): if len(preferred_compilers)>1: raise Exception(f"Not supported: more than one preferred compiler: {preferred_compilers}") preferred_compiler = preferred_compilers[0].replace("%","") - logging.info(" PREFERRED COMPILER: {}".format(preferred_compiler)) + logging.info(" ... preferred compiler: {}".format(preferred_compiler)) del preferred_compilers - # Custom key function for sorting + # Sort the list using a custom key def custom_sort_key(entry): # Return a tuple where the first element is 1 if the entry contains the word, else 0 # The second element is the entry itself for natural sorting within groups return (1 if preferred_compiler in entry else 0, entry) - - # Sort the list using the custom key - # add debug logging message here sorted_flattened_compiler_list = sorted(flattened_compiler_list, key=custom_sort_key) # Create compiler modules @@ -396,10 +392,10 @@ def custom_sort_key(entry): if compiler_spec in sorted_flattened_compiler_list: (compiler_name, compiler_version) = compiler_spec.split("@") - logging.info(" ... bla") modulepath_save = os.path.join(module_dir, compiler_name, compiler_version) if not os.path.isdir(modulepath_save): os.makedirs(modulepath_save) + logging.info(" ... ... appending {} to MODULEPATHS_SAVE".format(modulepath_save)) MODULEPATHS_SAVE.append(modulepath_save) # For tcl modules remove the compiler prefices from the module contents @@ -428,81 +424,81 @@ def custom_sort_key(entry): del pattern del tmp_compiler_spec - if compiler_name in preferred_compiler: - logging.info( - " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) - ) - compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler_name) - compiler_module_file = os.path.join( - compiler_module_dir, compiler_version + MODULE_FILE_EXTENSION[module_choice] - ) - substitutes = SUBSTITUTES_TEMPLATE.copy() - - # Compiler environment variables; names are lowercase in spack - substitutes["CC"] = compiler["compiler"]["paths"]["cc"] - substitutes["CXX"] = compiler["compiler"]["paths"]["cxx"] - substitutes["F77"] = compiler["compiler"]["paths"]["f77"] - substitutes["FC"] = compiler["compiler"]["paths"]["fc"] - logging.debug(" ... ... CC : {}".format(substitutes["CC"])) - logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) - logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) - logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) - - # Compiler flags; names are lowercase in spack - for flag_name in compiler["compiler"]["flags"]: - flag_values = compiler["compiler"]["flags"][flag_name] - substitutes["COMPFLAGS"] += setenv_command( - module_choice, flag_name.upper(), flag_values - ) - substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") - logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) + # The remainder of the loop is only needed for the preferred compiler + if not compiler_name in preferred_compiler: + continue - # Existing non-spack modules to load - for module in compiler["compiler"]["modules"]: - substitutes["MODULELOADS"] += module_load_command(module_choice, module) - substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) - substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") - substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") - logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) - logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) + logging.info( + " ... configuring stack compiler {}@{}".format(compiler_name, compiler_version) + ) + compiler_module_dir = os.path.join(meta_module_dir, "stack-" + compiler_name) + compiler_module_file = os.path.join( + compiler_module_dir, compiler_version + MODULE_FILE_EXTENSION[module_choice] + ) + substitutes = SUBSTITUTES_TEMPLATE.copy() - # Environment variables; case-sensitive in spack - if ( - "environment" in compiler["compiler"].keys() - and compiler["compiler"]["environment"] - ): - # append_path - if "append_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["append_path"]: - env_values = compiler["compiler"]["environment"]["append_path"][env_name] - substitutes["ENVVARS"] += append_path_command( - module_choice, env_name, env_values - ) - # prepend_path - if "prepend_path" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["prepend_path"]: - env_values = compiler["compiler"]["environment"]["prepend_path"][env_name] - substitutes["ENVVARS"] += prepend_path_command( - module_choice, env_name, env_values - ) - # set - if "set" in compiler["compiler"]["environment"].keys(): - for env_name in compiler["compiler"]["environment"]["set"]: - env_values = compiler["compiler"]["environment"]["set"][env_name] - substitutes["ENVVARS"] += setenv_command( - module_choice, env_name, env_values - ) - substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") - logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + # Compiler environment variables; names are lowercase in spack + substitutes["CC"] = compiler["compiler"]["paths"]["cc"] + substitutes["CXX"] = compiler["compiler"]["paths"]["cxx"] + substitutes["F77"] = compiler["compiler"]["paths"]["f77"] + substitutes["FC"] = compiler["compiler"]["paths"]["fc"] + logging.debug(" ... ... CC : {}".format(substitutes["CC"])) + logging.debug(" ... ... CXX : {}".format(substitutes["CXX"])) + logging.debug(" ... ... F77 : {}".format(substitutes["F77"])) + logging.debug(" ... ... FC' : {}".format(substitutes["FC"])) + + # Compiler flags; names are lowercase in spack + for flag_name in compiler["compiler"]["flags"]: + flag_values = compiler["compiler"]["flags"][flag_name] + substitutes["COMPFLAGS"] += setenv_command( + module_choice, flag_name.upper(), flag_values + ) + substitutes["COMPFLAGS"] = substitutes["COMPFLAGS"].rstrip("\n") + logging.debug(" ... ... COMPFLAGS: {}".format(substitutes["COMPFLAGS"])) - # Spack compiler module hierarchy - for modulepath in MODULEPATHS_SAVE: - substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) - substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") - logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) + # Existing non-spack modules to load + for module in compiler["compiler"]["modules"]: + substitutes["MODULELOADS"] += module_load_command(module_choice, module) + substitutes["MODULEPREREQS"] += module_prereq_command(module_choice, module) + substitutes["MODULELOADS"] = substitutes["MODULELOADS"].rstrip("\n") + substitutes["MODULEPREREQS"] = substitutes["MODULEPREREQS"].rstrip("\n") + logging.debug(" ... ... MODULELOADS: {}".format(substitutes["MODULELOADS"])) + logging.debug(" ... ... MODULEPREREQS: {}".format(substitutes["MODULEPREREQS"])) - if not compiler_name in preferred_compiler: - continue + # Environment variables; case-sensitive in spack + if ( + "environment" in compiler["compiler"].keys() + and compiler["compiler"]["environment"] + ): + # append_path + if "append_path" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["append_path"]: + env_values = compiler["compiler"]["environment"]["append_path"][env_name] + substitutes["ENVVARS"] += append_path_command( + module_choice, env_name, env_values + ) + # prepend_path + if "prepend_path" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["prepend_path"]: + env_values = compiler["compiler"]["environment"]["prepend_path"][env_name] + substitutes["ENVVARS"] += prepend_path_command( + module_choice, env_name, env_values + ) + # set + if "set" in compiler["compiler"]["environment"].keys(): + for env_name in compiler["compiler"]["environment"]["set"]: + env_values = compiler["compiler"]["environment"]["set"][env_name] + substitutes["ENVVARS"] += setenv_command( + module_choice, env_name, env_values + ) + substitutes["ENVVARS"] = substitutes["ENVVARS"].rstrip("\n") + logging.debug(" ... ... ENVVARS : {}".format(substitutes["ENVVARS"])) + + # Spack compiler module hierarchy - append all saved modulepaths + for modulepath in MODULEPATHS_SAVE: + substitutes["MODULEPATHS"] += modulepath_prepend_command(module_choice, modulepath) + substitutes["MODULEPATHS"] = substitutes["MODULEPATHS"].rstrip("\n") + logging.debug(" ... ... MODULEPATHS : {}".format(substitutes["MODULEPATHS"])) # Read compiler template into module_content string with open(COMPILER_TEMPLATES[module_choice]) as f: From 871fd3d03f7ffb76a49d97703bc560c3defd4c05 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 09:39:35 -0600 Subject: [PATCH 18/22] Fix compiler sorting/looping logic in spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py --- .../spack-stack/stack/meta_modules.py | 23 +++++++++++++------ 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py index e92acf173..e2d75b651 100755 --- a/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/stack/meta_modules.py @@ -200,7 +200,7 @@ def module_prereq_command(module_choice, module): def modulepath_prepend_command(module_choice, modulepath): if module_choice == "lmod": - return 'prepend_path("MODULEPATH", "{})\n'.format(modulepath) + return 'prepend_path("MODULEPATH", "{}")\n'.format(modulepath) else: return "prepend-path {{MODULEPATH}} {{{}}}\n".format(modulepath) @@ -384,14 +384,23 @@ def custom_sort_key(entry): # Create compiler modules logging.info("Creating compiler modules ...") compiler_config = spack.config.get("compilers") - # Collecti and save modulepaths for the preferred compiler + # Collect and save modulepaths for the preferred compiler MODULEPATHS_SAVE = [] - for compiler in compiler_config: - # On macOS, since July 2023, spack compiler find creates 'apple-clang@=14.0.0' etc. - compiler_spec = compiler["compiler"]["spec"].replace("@=", "@") - if compiler_spec in sorted_flattened_compiler_list: - (compiler_name, compiler_version) = compiler_spec.split("@") + for compiler_identifier in sorted_flattened_compiler_list: + (compiler_name, compiler_version) = compiler_identifier.replace("@=","@").split("@") + # Loop through all configured compilers and find the correct match + compiler = False + for candidate_compiler in compiler_config: + if compiler_identifier.replace("@=","@") == \ + candidate_compiler["compiler"]["spec"].replace("@=", "@"): + compiler = candidate_compiler + break + if not compiler: + raise Exception("No matching compiler for {} found in spack compiler config".format(compiler)) + continue + # This is only to keep the current level of indentation for easier reviewing + if True: modulepath_save = os.path.join(module_dir, compiler_name, compiler_version) if not os.path.isdir(modulepath_save): os.makedirs(modulepath_save) From 5e27a097a29bdcce509e4b90fca6ef21bbb89f8b Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Fri, 23 Aug 2024 12:44:56 -0600 Subject: [PATCH 19/22] Fix unit tests for setup-meta-modules: define compiler --- .../spack-stack/tests/test_setup_meta_modules.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py index 36f08c1aa..3be5afa38 100644 --- a/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py +++ b/spack-ext/lib/jcsda-emc/spack-stack/tests/test_setup_meta_modules.py @@ -50,6 +50,21 @@ def test_setup_meta_modules(): # Setup env and pretend that a build exists # by creating the module directory structure. scope = env.scope_name + # Mock compiler config + compiler_def = { + "compiler": { + "flags": {}, + "modules": [], + "paths": {"cc": "gcc", "cxx": "g++", "f77": "gfortran", "fc": "gfortran"}, + "extra_rpaths": [], + "operating_system": "test", + "target": "test", + "environment": {}, + "spec": "gcc@9.4.0", + } + } + compiler_config = [compiler_def] + spack.config.set("compilers", compiler_config, scope=scope) spack.config.add("packages:all:compiler:[{}]".format(comp), scope=scope) spack.config.add("packages:all:providers:mpi:[{}]".format(mpi), scope=scope) spack.config.add("packages:openmpi:version:[{}]".format(mpi_ver)) From f8b393f2979c030791b9de2caea3ca95234f6201 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Aug 2024 15:43:37 -0600 Subject: [PATCH 20/22] Update configs/sites/tier1/s4 --- configs/sites/tier1/s4/modules.yaml | 2 -- configs/sites/tier1/s4/packages.yaml | 8 +------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/configs/sites/tier1/s4/modules.yaml b/configs/sites/tier1/s4/modules.yaml index 5b7d734eb..e06812257 100644 --- a/configs/sites/tier1/s4/modules.yaml +++ b/configs/sites/tier1/s4/modules.yaml @@ -3,8 +3,6 @@ modules: enable:: - lmod lmod: - exclude: - - ecflow include: - mpich - python diff --git a/configs/sites/tier1/s4/packages.yaml b/configs/sites/tier1/s4/packages.yaml index 5e7ac0dc4..3650e0678 100644 --- a/configs/sites/tier1/s4/packages.yaml +++ b/configs/sites/tier1/s4/packages.yaml @@ -68,13 +68,6 @@ packages: externals: - spec: doxygen@1.8.5+graphviz~mscgen prefix: /usr - ecflow:: - buildable: False - externals: - - spec: ecflow@5.8.4+ui+static_boost - prefix: /data/prod/jedi/spack-stack/ecflow-5.8.4 - modules: - - ecflow/5.8.4 file: externals: - spec: file@5.11 @@ -161,6 +154,7 @@ packages: - spec: pkg-config@0.27.1 prefix: /usr qt: + buildable: False externals: - spec: qt@5.9.7 prefix: /usr From 75f6a1e74064a6f99c1a10bf53fdc1c1ec00741d Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Aug 2024 15:44:08 -0600 Subject: [PATCH 21/22] Bug fix in ./configs/common/packages.yaml. Specify ecflow requirements correctly --- configs/common/packages.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index bc86657d1..4e78711ea 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -38,7 +38,8 @@ packages: eccodes: require: '@2.33.0 +png +tools' ecflow: - require: '@5.11.4 +ui' + require: + - '@5.11.4 +ui' eckit: require: '@1.27.0 linalg=eigen,lapack compression=lz4,bzip2' ecmwf-atlas: From 7b3c9d3eb9f0dc8055a3c51ad85e2eaafc4b586c Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 27 Aug 2024 21:07:52 -0600 Subject: [PATCH 22/22] Update configs/sites/tier1/narwhal and doc/source/PreConfiguredSites.rst for UCX bug fixes --- configs/sites/tier1/narwhal/compilers.yaml | 16 ++++++++-------- configs/sites/tier1/narwhal/packages_gcc.yaml | 2 +- configs/sites/tier1/narwhal/packages_intel.yaml | 2 +- doc/source/PreConfiguredSites.rst | 16 ++++++++++++++-- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/configs/sites/tier1/narwhal/compilers.yaml b/configs/sites/tier1/narwhal/compilers.yaml index f0300cef6..1b7c23dcf 100644 --- a/configs/sites/tier1/narwhal/compilers.yaml +++ b/configs/sites/tier1/narwhal/compilers.yaml @@ -2,10 +2,10 @@ compilers:: - compiler: spec: intel@2021.10.0 paths: - cc: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/icc - cxx: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/icpc - f77: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/ifort - fc: /opt/intel/oneapi_2023.2.0.49397/compiler/2023.2.0/linux/bin/intel64/ifort + cc: cc + cxx: CC + f77: ftn + fc: ftn flags: {} operating_system: sles15 modules: @@ -24,10 +24,10 @@ compilers:: - compiler: spec: gcc@10.3.0 paths: - cc: /opt/cray/pe/gcc/10.3.0/bin/gcc - cxx: /opt/cray/pe/gcc/10.3.0/bin/g++ - f77: /opt/cray/pe/gcc/10.3.0/bin/gfortran - fc: /opt/cray/pe/gcc/10.3.0/bin/gfortran + cc: cc + cxx: CC + f77: ftn + fc: ftn flags: {} operating_system: sles15 modules: diff --git a/configs/sites/tier1/narwhal/packages_gcc.yaml b/configs/sites/tier1/narwhal/packages_gcc.yaml index 0cc712415..3261960c6 100644 --- a/configs/sites/tier1/narwhal/packages_gcc.yaml +++ b/configs/sites/tier1/narwhal/packages_gcc.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%gcc@10.3.0 +wrappers + - spec: cray-mpich@8.1.21%gcc@10.3.0 ~wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx diff --git a/configs/sites/tier1/narwhal/packages_intel.yaml b/configs/sites/tier1/narwhal/packages_intel.yaml index e415e07ca..0d631f58b 100644 --- a/configs/sites/tier1/narwhal/packages_intel.yaml +++ b/configs/sites/tier1/narwhal/packages_intel.yaml @@ -7,7 +7,7 @@ packages: buildable: False cray-mpich: externals: - - spec: cray-mpich@8.1.21%intel@2021.10.0 +wrappers + - spec: cray-mpich@8.1.21%intel@2021.10.0 ~wrappers modules: - cray-mpich-ucx/8.1.21 - craype-network-ucx diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index c6d135a59..319b53b4a 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -255,7 +255,13 @@ With Intel, the following is required for building new spack environments and fo module unload cray-mpich module unload craype-network-ofi module load craype-network-ucx - module load cray-mpich-ucx/8.1.21 + # Warning. Do not load cray-mpich-ucx/8.1.21 + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci module load cray-libsci/23.05.1.4 @@ -285,7 +291,13 @@ With GNU, the following is required for building new spack environments and for module unload cray-mpich module unload craype-network-ofi module load craype-network-ucx - module load cray-mpich-ucx/8.1.21 + # Warning. Do not load cray-mpich-ucx/8.1.21 + # There is a bug in the modulefile that prevents + # spack from setting the environment for its + # build steps when the module is already + # loaded. Instead, let spack load it when the + # package requires it. + #module load cray-mpich-ucx/8.1.21 module load libfabric/1.12.1.2.2.1 module unload cray-libsci module load cray-libsci/23.05.1.4