From bc466112edb64a3e8c1377da920f7f0836138f39 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sat, 9 Sep 2023 05:44:17 -0600 Subject: [PATCH 01/22] Update Orion site config --- configs/sites/orion/packages.yaml | 40 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 707826850..a2fc2772a 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -28,25 +28,25 @@ packages: prefix: /apps/gcc-10.2.0/openmpi-4.0.4/openmpi-4.0.4 modules: - openmpi/4.0.4 - python: - buildable: False - externals: - - spec: python@3.9.7 - prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - modules: - - miniconda/3.9.7 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.4 - prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - modules: - - miniconda/3.9.7 + #python: + # buildable: False + # externals: + # - spec: python@3.9.7 + # prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 + # modules: + # - miniconda/3.9.7 + #py-pip: + # buildable: False + # externals: + # - spec: py-pip@21.2.4 + # prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 + # modules: + # - miniconda/3.9.7 -### Modifications of common packages - # Versions 1.73+ don't compile on orion (bootstrap.sh fails) - boost: - version:: ['1.72.0'] +### ### Modifications of common packages +### # Versions 1.73+ don't compile on orion (bootstrap.sh fails) +### boost: +### version:: ['1.72.0'] ### All other external packages listed alphabetically autoconf: @@ -91,7 +91,7 @@ packages: buildable: False externals: - spec: ecflow@5.8.4+ui+static_boost - prefix: /work/noaa/da/role-da/spack-stack/ecflow-5.8.4 + prefix: /work/noaa/epic/role-epic/spack-stack/orion/ecflow-5.8.4 modules: - ecflow/5.8.4 file: @@ -168,7 +168,7 @@ packages: buildable: False externals: - spec: mysql@8.0.31 - prefix: /work/noaa/da/role-da/spack-stack/mysql-8.0.31 + prefix: /work/noaa/epic/role-epic/spack-stack/orion/mysql-8.0.31 modules: - mysql/8.0.31 ncurses: From 32affe233e21ec156da4537da9eab00c6a7dc4f9 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 00:42:44 +0000 Subject: [PATCH 02/22] TEMPORARILY add envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py and envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test --- .../lib/python3.10/distutils/unixccompiler.py | 325 ++++++++++++++++++ .../distutils/unixccompiler.py.dom.test | 325 ++++++++++++++++++ 2 files changed, 650 insertions(+) create mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py create mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py new file mode 100644 index 000000000..c3c6a0918 --- /dev/null +++ b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py @@ -0,0 +1,325 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils import sysconfig +from distutils.dep_util import newer +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options, gen_lib_options +from distutils.errors import \ + DistutilsExecError, CompileError, LibError, LinkError +from distutils import log + +if sys.platform == 'darwin': + import _osx_support + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["c++"], + 'compiler_so_cxx' : ["c++"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'linker_so_cxx' : ["c++", "-shared"], + 'linker_exe_cxx' : ["c++"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + compiler_so = self.compiler_so + compiler_so_cxx = self.compiler_so_cxx + if sys.platform == 'darwin': + compiler_so = _osx_support.compiler_fixup(compiler_so, + cc_args + extra_postargs) + compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, + cc_args + extra_postargs) + try: + if self.detect_language(src) == 'c++': + self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + + extra_postargs) + else: + self.spawn(compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) + + def create_static_lib(self, objects, output_libname, + output_dir=None, debug=0, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + self.mkpath(os.path.dirname(output_filename)) + self.spawn(self.archiver + + [output_filename] + + objects + self.objects) + + # Not many Unices required ranlib anymore -- SunOS 4.x is, I + # think the only major Unix that does. Maybe we need some + # platform intelligence here to skip ranlib if it's not + # needed -- or maybe Python's configure script took care of + # it for us, hence the check for leading colon. + if self.ranlib: + try: + self.spawn(self.ranlib + [output_filename]) + except DistutilsExecError as msg: + raise LibError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, + output_filename, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=0, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + libraries, library_dirs, runtime_library_dirs = fixed_args + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if not isinstance(output_dir, (str, type(None))): + raise TypeError("'output_dir' must be a string or None") + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + ld_args = (objects + self.objects + + lib_opts + ['-o', output_filename]) + if debug: + ld_args[:0] = ['-g'] + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + self.mkpath(os.path.dirname(output_filename)) + try: + if target_lang == "c++": + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe_cxx[:] + else: + linker = self.linker_so_cxx[:] + else: + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe[:] + else: + linker = self.linker_so[:] + + if sys.platform == 'darwin': + linker = _osx_support.compiler_fixup(linker, ld_args) + + self.spawn(linker + ld_args) + except DistutilsExecError as msg: + raise LinkError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "-L" + dir + + def _is_gcc(self, compiler_name): + # clang uses same syntax for rpath as gcc + return any(name in compiler_name for name in ("gcc", "g++", "clang")) + + def runtime_library_dir_option(self, dir): + # XXX Hackish, at the very least. See Python bug #445902: + # http://sourceforge.net/tracker/index.php + # ?func=detail&aid=445902&group_id=5470&atid=105470 + # Linkers on different platforms need different options to + # specify that directories need to be added to the list of + # directories searched for dependencies when a dynamic library + # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to + # be told to pass the -R option through to the linker, whereas + # other compilers and gcc on other systems just know this. + # Other compilers may need something slightly different. At + # this time, there's no way to determine this information from + # the configuration data stored in the Python installation, so + # we use this hack. + compiler = os.path.basename(sysconfig.get_config_var("CC")) + if sys.platform[:6] == "darwin": + # MacOSX's linker doesn't understand the -R flag at all + return "-L" + dir + elif sys.platform[:7] == "freebsd": + return "-Wl,-rpath=" + dir + elif sys.platform[:5] == "hp-ux": + if self._is_gcc(compiler): + return ["-Wl,+s", "-L" + dir] + return ["+s", "-L" + dir] + else: + if self._is_gcc(compiler): + # gcc on non-GNU systems does not need -Wl, but can + # use it anyway. Since distutils has always passed in + # -Wl whenever gcc was used in the past it is probably + # safest to keep doing so. + if sysconfig.get_config_var("GNULD") == "yes": + # GNU ld needs an extra option to get a RUNPATH + # instead of just an RPATH. + return "-Wl,--enable-new-dtags,-R" + dir + else: + return "-Wl,-R" + dir + else: + # No idea how --enable-new-dtags would be passed on to + # ld if this system was using GNU ld. Don't know if a + # system like this even exists. + return "-Wl,-R" + dir + + def library_option(self, lib): + return "-l" + lib + + def find_library_file(self, dirs, lib, debug=0): + shared_f = self.library_filename(lib, lib_type='shared') + dylib_f = self.library_filename(lib, lib_type='dylib') + xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') + static_f = self.library_filename(lib, lib_type='static') + + if sys.platform == 'darwin': + # On OSX users can specify an alternate SDK using + # '-isysroot', calculate the SDK root if it is specified + # (and use it further on) + # + # Note that, as of Xcode 7, Apple SDKs may contain textual stub + # libraries with .tbd extensions rather than the normal .dylib + # shared libraries installed in /. The Apple compiler tool + # chain handles this transparently but it can cause problems + # for programs that are being built with an SDK and searching + # for specific libraries. Callers of find_library_file need to + # keep in mind that the base filename of the returned SDK library + # file might have a different extension from that of the library + # file installed on the running system, for example: + # /Applications/Xcode.app/Contents/Developer/Platforms/ + # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ + # usr/lib/libedit.tbd + # vs + # /usr/lib/libedit.dylib + cflags = sysconfig.get_config_var('CFLAGS') + m = re.search(r'-isysroot\s*(\S+)', cflags) + if m is None: + sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) + else: + sysroot = m.group(1) + + + + for dir in dirs: + shared = os.path.join(dir, shared_f) + dylib = os.path.join(dir, dylib_f) + static = os.path.join(dir, static_f) + xcode_stub = os.path.join(dir, xcode_stub_f) + + if sys.platform == 'darwin' and ( + dir.startswith('/System/') or ( + dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): + + shared = os.path.join(sysroot, dir[1:], shared_f) + dylib = os.path.join(sysroot, dir[1:], dylib_f) + static = os.path.join(sysroot, dir[1:], static_f) + xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) + + # We're second-guessing the linker here, with not much hard + # data to go on: GCC seems to prefer the shared library, so I'm + # assuming that *all* Unix C compilers do. And of course I'm + # ignoring even GCC's "-static" option. So sue me. + if os.path.exists(dylib): + return dylib + elif os.path.exists(xcode_stub): + return xcode_stub + elif os.path.exists(shared): + return shared + elif os.path.exists(static): + return static + + # Oops, didn't find it in *any* of 'dirs' + return None diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test new file mode 100644 index 000000000..4a3d271fe --- /dev/null +++ b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test @@ -0,0 +1,325 @@ +"""distutils.unixccompiler + +Contains the UnixCCompiler class, a subclass of CCompiler that handles +the "typical" Unix-style command-line C compiler: + * macros defined with -Dname[=value] + * macros undefined with -Uname + * include search directories specified with -Idir + * libraries specified with -lllib + * library search directories specified with -Ldir + * compile handled by 'cc' (or similar) executable with -c option: + compiles .c to .o + * link static library handled by 'ar' command (possibly with 'ranlib') + * link shared library handled by 'cc -shared' +""" + +import os, sys, re + +from distutils import sysconfig +from distutils.dep_util import newer +from distutils.ccompiler import \ + CCompiler, gen_preprocess_options, gen_lib_options +from distutils.errors import \ + DistutilsExecError, CompileError, LibError, LinkError +from distutils import log + +if sys.platform == 'darwin': + import _osx_support + +# XXX Things not currently handled: +# * optimization/debug/warning flags; we just use whatever's in Python's +# Makefile and live with it. Is this adequate? If not, we might +# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, +# SunCCompiler, and I suspect down that road lies madness. +# * even if we don't know a warning flag from an optimization flag, +# we need some way for outsiders to feed preprocessor/compiler/linker +# flags in to us -- eg. a sysadmin might want to mandate certain flags +# via a site config file, or a user might want to set something for +# compiling this module distribution only via the setup.py command +# line, whatever. As long as these options come from something on the +# current system, they can be as system-dependent as they like, and we +# should just happily stuff them into the preprocessor/compiler/linker +# options and carry on. + + +class UnixCCompiler(CCompiler): + + compiler_type = 'unix' + + # These are used by CCompiler in two places: the constructor sets + # instance attributes 'preprocessor', 'compiler', etc. from them, and + # 'set_executable()' allows any of these to be set. The defaults here + # are pretty generic; they will probably have to be set by an outsider + # (eg. using information discovered by the sysconfig about building + # Python extensions). + executables = {'preprocessor' : None, + 'compiler' : ["cc"], + 'compiler_so' : ["cc"], + 'compiler_cxx' : ["c++"], + 'compiler_so_cxx' : ["c++"], + 'linker_so' : ["cc", "-shared"], + 'linker_exe' : ["cc"], + 'linker_so_cxx' : ["c++", "-shared"], + 'linker_exe_cxx' : ["c++"], + 'archiver' : ["ar", "-cr"], + 'ranlib' : None, + } + + if sys.platform[:6] == "darwin": + executables['ranlib'] = ["ranlib"] + + # Needed for the filename generation methods provided by the base + # class, CCompiler. NB. whoever instantiates/uses a particular + # UnixCCompiler instance should set 'shared_lib_ext' -- we set a + # reasonable common default here, but it's not necessarily used on all + # Unices! + + src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] + obj_extension = ".o" + static_lib_extension = ".a" + shared_lib_extension = ".so" + dylib_lib_extension = ".dylib" + xcode_stub_lib_extension = ".tbd" + static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" + xcode_stub_lib_format = dylib_lib_format + if sys.platform == "cygwin": + exe_extension = ".exe" + + def preprocess(self, source, output_file=None, macros=None, + include_dirs=None, extra_preargs=None, extra_postargs=None): + fixed_args = self._fix_compile_args(None, macros, include_dirs) + ignore, macros, include_dirs = fixed_args + pp_opts = gen_preprocess_options(macros, include_dirs) + pp_args = self.preprocessor + pp_opts + if output_file: + pp_args.extend(['-o', output_file]) + if extra_preargs: + pp_args[:0] = extra_preargs + if extra_postargs: + pp_args.extend(extra_postargs) + pp_args.append(source) + + # We need to preprocess: either we're being forced to, or we're + # generating output to stdout, or there's a target output file and + # the source file is newer than the target (or the target doesn't + # exist). + if self.force or output_file is None or newer(source, output_file): + if output_file: + self.mkpath(os.path.dirname(output_file)) + try: + self.spawn(pp_args) + except DistutilsExecError as msg: + raise CompileError(msg) + + def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): + compiler_so = self.compiler_so + compiler_so_cxx = self.compiler_so_cxx + if sys.platform == 'darwin': + compiler_so = _osx_support.compiler_fixup(compiler_so, + cc_args + extra_postargs) + compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, + cc_args + extra_postargs) + try: + if self.detect_language(src) == 'c++': + self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + + extra_postargs) + else: + self.spawn(compiler_so + cc_args + [src, '-o', obj] + + extra_postargs) + except DistutilsExecError as msg: + raise CompileError(msg) + + def create_static_lib(self, objects, output_libname, + output_dir=None, debug=0, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + + output_filename = \ + self.library_filename(output_libname, output_dir=output_dir) + + if self._need_link(objects, output_filename): + self.mkpath(os.path.dirname(output_filename)) + self.spawn(self.archiver + + [output_filename] + + objects + self.objects) + + # Not many Unices required ranlib anymore -- SunOS 4.x is, I + # think the only major Unix that does. Maybe we need some + # platform intelligence here to skip ranlib if it's not + # needed -- or maybe Python's configure script took care of + # it for us, hence the check for leading colon. + if self.ranlib: + try: + self.spawn(self.ranlib + [output_filename]) + except DistutilsExecError as msg: + raise LibError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + def link(self, target_desc, objects, + output_filename, output_dir=None, libraries=None, + library_dirs=None, runtime_library_dirs=None, + export_symbols=None, debug=0, extra_preargs=None, + extra_postargs=None, build_temp=None, target_lang=None): + objects, output_dir = self._fix_object_args(objects, output_dir) + fixed_args = self._fix_lib_args(libraries, library_dirs, + runtime_library_dirs) + libraries, library_dirs, runtime_library_dirs = fixed_args + + lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, + libraries) + if not isinstance(output_dir, (str, type(None))): + raise TypeError("'output_dir' must be a string or None") + if output_dir is not None: + output_filename = os.path.join(output_dir, output_filename) + + if self._need_link(objects, output_filename): + ld_args = (objects + self.objects + + lib_opts + ['-o', output_filename]) + if debug: + ld_args[:0] = ['-g'] + if extra_preargs: + ld_args[:0] = extra_preargs + if extra_postargs: + ld_args.extend(extra_postargs) + self.mkpath(os.path.dirname(output_filename)) + try: + if target_lang == "c++": + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe_cxx[:] + else: + linker = self.linker_so_cxx[:] + else: + if target_desc == CCompiler.EXECUTABLE: + linker = self.linker_exe[:] + else: + linker = self.linker_so[:] + + if sys.platform == 'darwin': + linker = _osx_support.compiler_fixup(linker, ld_args) + + self.spawn(linker + ld_args) + except DistutilsExecError as msg: + raise LinkError(msg) + else: + log.debug("skipping %s (up-to-date)", output_filename) + + # -- Miscellaneous methods ----------------------------------------- + # These are all used by the 'gen_lib_options() function, in + # ccompiler.py. + + def library_dir_option(self, dir): + return "-L" + dir + + def _is_gcc(self, compiler_name): + # clang uses same syntax for rpath as gcc + return any(name in compiler_name for name in ("gcc", "g++", "clang")) + + def runtime_library_dir_option(self, dir): + # XXX Hackish, at the very least. See Python bug #445902: + # http://sourceforge.net/tracker/index.php + # ?func=detail&aid=445902&group_id=5470&atid=105470 + # Linkers on different platforms need different options to + # specify that directories need to be added to the list of + # directories searched for dependencies when a dynamic library + # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to + # be told to pass the -R option through to the linker, whereas + # other compilers and gcc on other systems just know this. + # Other compilers may need something slightly different. At + # this time, there's no way to determine this information from + # the configuration data stored in the Python installation, so + # we use this hack. + compiler = os.path.basename(sysconfig.get_config_var("CC")) + if sys.platform[:6] == "darwin": + # MacOSX's linker doesn't understand the -R flag at all + return "-L" + dir + elif sys.platform[:7] == "freebsd": + return "-Wl,-rpath=" + dir + elif sys.platform[:5] == "hp-ux": + if self._is_gcc(compiler): + return ["-Wl,+s", "-L" + dir] + return ["+s", "-L" + dir] + else: + if self._is_gcc(compiler): + # gcc on non-GNU systems does not need -Wl, but can + # use it anyway. Since distutils has always passed in + # -Wl whenever gcc was used in the past it is probably + # safest to keep doing so. + if sysconfig.get_config_var("GNULD") == "yes": + # GNU ld needs an extra option to get a RUNPATH + # instead of just an RPATH. + return "-Wl,--enable-new-dtags,-R" + dir + else: + return "-Wl,-R" + dir + else: + # No idea how --enable-new-dtags would be passed on to + # ld if this system was using GNU ld. Don't know if a + # system like this even exists. + return "-R" + dir + + def library_option(self, lib): + return "-l" + lib + + def find_library_file(self, dirs, lib, debug=0): + shared_f = self.library_filename(lib, lib_type='shared') + dylib_f = self.library_filename(lib, lib_type='dylib') + xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') + static_f = self.library_filename(lib, lib_type='static') + + if sys.platform == 'darwin': + # On OSX users can specify an alternate SDK using + # '-isysroot', calculate the SDK root if it is specified + # (and use it further on) + # + # Note that, as of Xcode 7, Apple SDKs may contain textual stub + # libraries with .tbd extensions rather than the normal .dylib + # shared libraries installed in /. The Apple compiler tool + # chain handles this transparently but it can cause problems + # for programs that are being built with an SDK and searching + # for specific libraries. Callers of find_library_file need to + # keep in mind that the base filename of the returned SDK library + # file might have a different extension from that of the library + # file installed on the running system, for example: + # /Applications/Xcode.app/Contents/Developer/Platforms/ + # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ + # usr/lib/libedit.tbd + # vs + # /usr/lib/libedit.dylib + cflags = sysconfig.get_config_var('CFLAGS') + m = re.search(r'-isysroot\s*(\S+)', cflags) + if m is None: + sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) + else: + sysroot = m.group(1) + + + + for dir in dirs: + shared = os.path.join(dir, shared_f) + dylib = os.path.join(dir, dylib_f) + static = os.path.join(dir, static_f) + xcode_stub = os.path.join(dir, xcode_stub_f) + + if sys.platform == 'darwin' and ( + dir.startswith('/System/') or ( + dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): + + shared = os.path.join(sysroot, dir[1:], shared_f) + dylib = os.path.join(sysroot, dir[1:], dylib_f) + static = os.path.join(sysroot, dir[1:], static_f) + xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) + + # We're second-guessing the linker here, with not much hard + # data to go on: GCC seems to prefer the shared library, so I'm + # assuming that *all* Unix C compilers do. And of course I'm + # ignoring even GCC's "-static" option. So sue me. + if os.path.exists(dylib): + return dylib + elif os.path.exists(xcode_stub): + return xcode_stub + elif os.path.exists(shared): + return shared + elif os.path.exists(static): + return static + + # Oops, didn't find it in *any* of 'dirs' + return None From 0c554ec85a6d8b243b4727c2035c8e79be293e0e Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 00:43:02 +0000 Subject: [PATCH 03/22] Update Narwhal site config --- configs/sites/narwhal/mirrors.yaml | 4 ++-- configs/sites/narwhal/modules.yaml | 2 ++ configs/sites/narwhal/packages.yaml | 36 ++++++++++++++--------------- 3 files changed, 22 insertions(+), 20 deletions(-) diff --git a/configs/sites/narwhal/mirrors.yaml b/configs/sites/narwhal/mirrors.yaml index 8f845ff68..709f9096c 100644 --- a/configs/sites/narwhal/mirrors.yaml +++ b/configs/sites/narwhal/mirrors.yaml @@ -1,7 +1,7 @@ mirrors: local-source: fetch: - url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null @@ -9,7 +9,7 @@ mirrors: profile: null endpoint_url: null push: - url: file:///p/app/projects/NEPTUNE/spack-stack/spack-stack-v1/source-cache + url: file:///p/app/projects/NEPTUNE/spack-stack/source-cache access_pair: - null - null diff --git a/configs/sites/narwhal/modules.yaml b/configs/sites/narwhal/modules.yaml index 9724c405d..f83e5dc2c 100644 --- a/configs/sites/narwhal/modules.yaml +++ b/configs/sites/narwhal/modules.yaml @@ -5,3 +5,5 @@ modules: tcl: exclude: - ecflow + include: + - python diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 5cf51e0ad..640aae18d 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -27,20 +27,20 @@ packages: # prefix: /opt/intel/oneapi_2021.3.0.3219 # modules: # - intel/2021.3.0 - python: - buildable: False - externals: - - spec: python@3.9.7+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - prefix: /opt/cray/pe/python/3.9.7.1 - modules: - - cray-python/3.9.7.1 - py-pip: - buildable: False - externals: - - spec: py-pip@21.2.3 - prefix: /opt/cray/pe/python/3.9.7.1 - modules: - - cray-python/3.9.7.1 + #python: + # buildable: False + # externals: + # - spec: python@3.9.7+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib + # prefix: /opt/cray/pe/python/3.9.7.1 + # modules: + # - cray-python/3.9.7.1 + #py-pip: + # buildable: False + # externals: + # - spec: py-pip@21.2.3 + # prefix: /opt/cray/pe/python/3.9.7.1 + # modules: + # - cray-python/3.9.7.1 ### All other external packages listed alphabetically autoconf: @@ -55,10 +55,10 @@ packages: externals: - spec: binutils@2.37.20211103 prefix: /usr - bison: - externals: - - spec: bison@3.0.4 - prefix: /usr + #bison: + # externals: + # - spec: bison@3.0.4 + #prefix: /usr curl: externals: - spec: curl@7.66.0+gssapi+ldap+nghttp2 From 073626fad1d01f0740a5267deb3fa8226f438bf4 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Sep 2023 18:59:24 -0600 Subject: [PATCH 04/22] Update .gitmodules and submodule pointer for spack for code review and testing --- .gitmodules | 10 ++++++---- spack | 2 +- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/.gitmodules b/.gitmodules index 31d7a8707..7d8e5c869 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,11 @@ [submodule "spack"] path = spack - #url = https://github.com/spack/spack - #branch = develop - url = https://github.com/jcsda/spack - branch = release/1.5.0 + ##url = https://github.com/spack/spack + ##branch = develop + #url = https://github.com/jcsda/spack + #branch = release/1.5.0 + url = https://github.com/climbfuji/spack + branch = bugfix/release150_python_cray_rpath [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 872c4a113..5b6c8e542 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 872c4a1136c32321895199e074e46be5f1571d8c +Subproject commit 5b6c8e542ec26121e81a02db93c46eea597ea815 From 0f1f028bf6bc9c94ed5f64f2373dc0945d3b963f Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Sep 2023 19:05:10 -0600 Subject: [PATCH 05/22] Turn off variants ~extdata2g and ~pflogger for mapl to avoid compilation errors with intel@2021.7.0 and later and to avoid segmentation faults with ufs-weather-model --- configs/common/packages.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 5e81ed879..13934ca43 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -138,7 +138,7 @@ mapl: # 2.35.2 goes with esmf@8.4.2, 2.40.3 goes with esmf@8.5.0 version: ['2.35.2'] - variants: ~shared + variants: ~shared ~extdata2g ~pflogger # If making changes here, also check the Discover site config and the CI workflows met: version: ['11.1.0'] From 94e4497f9e42bc2f79c4f40d7e03b3f440b27af2 Mon Sep 17 00:00:00 2001 From: Mark A Potts Date: Sun, 10 Sep 2023 20:22:32 -0500 Subject: [PATCH 06/22] Update Hercules site config --- configs/common/packages.yaml | 5 +++++ configs/sites/hercules/packages.yaml | 22 +++++++++++++--------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 13934ca43..3bc8b356f 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -137,6 +137,10 @@ version: ['0.2.5'] mapl: # 2.35.2 goes with esmf@8.4.2, 2.40.3 goes with esmf@8.5.0 + # turn off ~pflogger and extdata2g to avoid compilation + # errors with intel@2021.7.0+, see + # https://github.com/JCSDA/spack-stack/issues/769 + # also: ... extdata2g segfault UFS? version: ['2.35.2'] variants: ~shared ~extdata2g ~pflogger # If making changes here, also check the Discover site config and the CI workflows @@ -260,6 +264,7 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] + # Check Hercules site config when changing this py-pyyaml: version: ['6.0'] py-scipy: diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 26df869b3..793f4328b 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -20,13 +20,17 @@ packages: prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1 modules: - openmpi/4.1.5 - python: - buildable: False - externals: - - spec: python@3.9.14 #+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - prefix: /usr + #python: + #buildable: False + #externals: + #- spec: python@3.9.14 #+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib + #prefix: /usr ### Modifications of common packages + # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version + py-pyyaml: + require: + - "@5.4.1" # Version 2.0.8 doesn't compile on Hercules wgrib2: version:: ['3.1.1'] @@ -44,10 +48,10 @@ packages: externals: - spec: binutils@2.35.2 prefix: /usr - bison: - externals: - - spec: bison@3.7.4 - prefix: /usr + #bison: + #externals: + #- spec: bison@3.7.4 + #prefix: /usr # Do not use! #cmake: # externals: From 862c67788cb41af7224a3aae5f1104429d26aa99 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Sep 2023 19:34:59 -0600 Subject: [PATCH 07/22] Remove temporary files in envs --- .../lib/python3.10/distutils/unixccompiler.py | 325 ------------------ .../distutils/unixccompiler.py.dom.test | 325 ------------------ 2 files changed, 650 deletions(-) delete mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py delete mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py deleted file mode 100644 index c3c6a0918..000000000 --- a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py +++ /dev/null @@ -1,325 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["c++"], - 'compiler_so_cxx' : ["c++"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'linker_so_cxx' : ["c++", "-shared"], - 'linker_exe_cxx' : ["c++"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - compiler_so_cxx = self.compiler_so_cxx - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, - cc_args + extra_postargs) - try: - if self.detect_language(src) == 'c++': - self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + - extra_postargs) - else: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_lang == "c++": - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe_cxx[:] - else: - linker = self.linker_so_cxx[:] - else: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - # clang uses same syntax for rpath as gcc - return any(name in compiler_name for name in ("gcc", "g++", "clang")) - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-Wl,-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s*(\S+)', cflags) - if m is None: - sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test deleted file mode 100644 index 4a3d271fe..000000000 --- a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test +++ /dev/null @@ -1,325 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["c++"], - 'compiler_so_cxx' : ["c++"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'linker_so_cxx' : ["c++", "-shared"], - 'linker_exe_cxx' : ["c++"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - compiler_so_cxx = self.compiler_so_cxx - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, - cc_args + extra_postargs) - try: - if self.detect_language(src) == 'c++': - self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + - extra_postargs) - else: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_lang == "c++": - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe_cxx[:] - else: - linker = self.linker_so_cxx[:] - else: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - # clang uses same syntax for rpath as gcc - return any(name in compiler_name for name in ("gcc", "g++", "clang")) - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s*(\S+)', cflags) - if m is None: - sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None From 2825882145110a2109ca1c70c1629d7e31368ee2 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Sep 2023 19:34:59 -0600 Subject: [PATCH 08/22] Remove temporary files in envs --- .../lib/python3.10/distutils/unixccompiler.py | 325 ------------------ .../distutils/unixccompiler.py.dom.test | 325 ------------------ spack | 2 +- 3 files changed, 1 insertion(+), 651 deletions(-) delete mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py delete mode 100644 envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py deleted file mode 100644 index c3c6a0918..000000000 --- a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py +++ /dev/null @@ -1,325 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["c++"], - 'compiler_so_cxx' : ["c++"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'linker_so_cxx' : ["c++", "-shared"], - 'linker_exe_cxx' : ["c++"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - compiler_so_cxx = self.compiler_so_cxx - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, - cc_args + extra_postargs) - try: - if self.detect_language(src) == 'c++': - self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + - extra_postargs) - else: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_lang == "c++": - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe_cxx[:] - else: - linker = self.linker_so_cxx[:] - else: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - # clang uses same syntax for rpath as gcc - return any(name in compiler_name for name in ("gcc", "g++", "clang")) - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-Wl,-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s*(\S+)', cflags) - if m is None: - sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test b/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test deleted file mode 100644 index 4a3d271fe..000000000 --- a/envs/unified-env-gcc-10.3.0/install/gcc/10.3.0/python-3.10.8-6pvtcn2/lib/python3.10/distutils/unixccompiler.py.dom.test +++ /dev/null @@ -1,325 +0,0 @@ -"""distutils.unixccompiler - -Contains the UnixCCompiler class, a subclass of CCompiler that handles -the "typical" Unix-style command-line C compiler: - * macros defined with -Dname[=value] - * macros undefined with -Uname - * include search directories specified with -Idir - * libraries specified with -lllib - * library search directories specified with -Ldir - * compile handled by 'cc' (or similar) executable with -c option: - compiles .c to .o - * link static library handled by 'ar' command (possibly with 'ranlib') - * link shared library handled by 'cc -shared' -""" - -import os, sys, re - -from distutils import sysconfig -from distutils.dep_util import newer -from distutils.ccompiler import \ - CCompiler, gen_preprocess_options, gen_lib_options -from distutils.errors import \ - DistutilsExecError, CompileError, LibError, LinkError -from distutils import log - -if sys.platform == 'darwin': - import _osx_support - -# XXX Things not currently handled: -# * optimization/debug/warning flags; we just use whatever's in Python's -# Makefile and live with it. Is this adequate? If not, we might -# have to have a bunch of subclasses GNUCCompiler, SGICCompiler, -# SunCCompiler, and I suspect down that road lies madness. -# * even if we don't know a warning flag from an optimization flag, -# we need some way for outsiders to feed preprocessor/compiler/linker -# flags in to us -- eg. a sysadmin might want to mandate certain flags -# via a site config file, or a user might want to set something for -# compiling this module distribution only via the setup.py command -# line, whatever. As long as these options come from something on the -# current system, they can be as system-dependent as they like, and we -# should just happily stuff them into the preprocessor/compiler/linker -# options and carry on. - - -class UnixCCompiler(CCompiler): - - compiler_type = 'unix' - - # These are used by CCompiler in two places: the constructor sets - # instance attributes 'preprocessor', 'compiler', etc. from them, and - # 'set_executable()' allows any of these to be set. The defaults here - # are pretty generic; they will probably have to be set by an outsider - # (eg. using information discovered by the sysconfig about building - # Python extensions). - executables = {'preprocessor' : None, - 'compiler' : ["cc"], - 'compiler_so' : ["cc"], - 'compiler_cxx' : ["c++"], - 'compiler_so_cxx' : ["c++"], - 'linker_so' : ["cc", "-shared"], - 'linker_exe' : ["cc"], - 'linker_so_cxx' : ["c++", "-shared"], - 'linker_exe_cxx' : ["c++"], - 'archiver' : ["ar", "-cr"], - 'ranlib' : None, - } - - if sys.platform[:6] == "darwin": - executables['ranlib'] = ["ranlib"] - - # Needed for the filename generation methods provided by the base - # class, CCompiler. NB. whoever instantiates/uses a particular - # UnixCCompiler instance should set 'shared_lib_ext' -- we set a - # reasonable common default here, but it's not necessarily used on all - # Unices! - - src_extensions = [".c",".C",".cc",".cxx",".cpp",".m"] - obj_extension = ".o" - static_lib_extension = ".a" - shared_lib_extension = ".so" - dylib_lib_extension = ".dylib" - xcode_stub_lib_extension = ".tbd" - static_lib_format = shared_lib_format = dylib_lib_format = "lib%s%s" - xcode_stub_lib_format = dylib_lib_format - if sys.platform == "cygwin": - exe_extension = ".exe" - - def preprocess(self, source, output_file=None, macros=None, - include_dirs=None, extra_preargs=None, extra_postargs=None): - fixed_args = self._fix_compile_args(None, macros, include_dirs) - ignore, macros, include_dirs = fixed_args - pp_opts = gen_preprocess_options(macros, include_dirs) - pp_args = self.preprocessor + pp_opts - if output_file: - pp_args.extend(['-o', output_file]) - if extra_preargs: - pp_args[:0] = extra_preargs - if extra_postargs: - pp_args.extend(extra_postargs) - pp_args.append(source) - - # We need to preprocess: either we're being forced to, or we're - # generating output to stdout, or there's a target output file and - # the source file is newer than the target (or the target doesn't - # exist). - if self.force or output_file is None or newer(source, output_file): - if output_file: - self.mkpath(os.path.dirname(output_file)) - try: - self.spawn(pp_args) - except DistutilsExecError as msg: - raise CompileError(msg) - - def _compile(self, obj, src, ext, cc_args, extra_postargs, pp_opts): - compiler_so = self.compiler_so - compiler_so_cxx = self.compiler_so_cxx - if sys.platform == 'darwin': - compiler_so = _osx_support.compiler_fixup(compiler_so, - cc_args + extra_postargs) - compiler_so_cxx = _osx_support.compiler_fixup(compiler_so_cxx, - cc_args + extra_postargs) - try: - if self.detect_language(src) == 'c++': - self.spawn(compiler_so_cxx + cc_args + [src, '-o', obj] + - extra_postargs) - else: - self.spawn(compiler_so + cc_args + [src, '-o', obj] + - extra_postargs) - except DistutilsExecError as msg: - raise CompileError(msg) - - def create_static_lib(self, objects, output_libname, - output_dir=None, debug=0, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - - output_filename = \ - self.library_filename(output_libname, output_dir=output_dir) - - if self._need_link(objects, output_filename): - self.mkpath(os.path.dirname(output_filename)) - self.spawn(self.archiver + - [output_filename] + - objects + self.objects) - - # Not many Unices required ranlib anymore -- SunOS 4.x is, I - # think the only major Unix that does. Maybe we need some - # platform intelligence here to skip ranlib if it's not - # needed -- or maybe Python's configure script took care of - # it for us, hence the check for leading colon. - if self.ranlib: - try: - self.spawn(self.ranlib + [output_filename]) - except DistutilsExecError as msg: - raise LibError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - def link(self, target_desc, objects, - output_filename, output_dir=None, libraries=None, - library_dirs=None, runtime_library_dirs=None, - export_symbols=None, debug=0, extra_preargs=None, - extra_postargs=None, build_temp=None, target_lang=None): - objects, output_dir = self._fix_object_args(objects, output_dir) - fixed_args = self._fix_lib_args(libraries, library_dirs, - runtime_library_dirs) - libraries, library_dirs, runtime_library_dirs = fixed_args - - lib_opts = gen_lib_options(self, library_dirs, runtime_library_dirs, - libraries) - if not isinstance(output_dir, (str, type(None))): - raise TypeError("'output_dir' must be a string or None") - if output_dir is not None: - output_filename = os.path.join(output_dir, output_filename) - - if self._need_link(objects, output_filename): - ld_args = (objects + self.objects + - lib_opts + ['-o', output_filename]) - if debug: - ld_args[:0] = ['-g'] - if extra_preargs: - ld_args[:0] = extra_preargs - if extra_postargs: - ld_args.extend(extra_postargs) - self.mkpath(os.path.dirname(output_filename)) - try: - if target_lang == "c++": - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe_cxx[:] - else: - linker = self.linker_so_cxx[:] - else: - if target_desc == CCompiler.EXECUTABLE: - linker = self.linker_exe[:] - else: - linker = self.linker_so[:] - - if sys.platform == 'darwin': - linker = _osx_support.compiler_fixup(linker, ld_args) - - self.spawn(linker + ld_args) - except DistutilsExecError as msg: - raise LinkError(msg) - else: - log.debug("skipping %s (up-to-date)", output_filename) - - # -- Miscellaneous methods ----------------------------------------- - # These are all used by the 'gen_lib_options() function, in - # ccompiler.py. - - def library_dir_option(self, dir): - return "-L" + dir - - def _is_gcc(self, compiler_name): - # clang uses same syntax for rpath as gcc - return any(name in compiler_name for name in ("gcc", "g++", "clang")) - - def runtime_library_dir_option(self, dir): - # XXX Hackish, at the very least. See Python bug #445902: - # http://sourceforge.net/tracker/index.php - # ?func=detail&aid=445902&group_id=5470&atid=105470 - # Linkers on different platforms need different options to - # specify that directories need to be added to the list of - # directories searched for dependencies when a dynamic library - # is sought. GCC on GNU systems (Linux, FreeBSD, ...) has to - # be told to pass the -R option through to the linker, whereas - # other compilers and gcc on other systems just know this. - # Other compilers may need something slightly different. At - # this time, there's no way to determine this information from - # the configuration data stored in the Python installation, so - # we use this hack. - compiler = os.path.basename(sysconfig.get_config_var("CC")) - if sys.platform[:6] == "darwin": - # MacOSX's linker doesn't understand the -R flag at all - return "-L" + dir - elif sys.platform[:7] == "freebsd": - return "-Wl,-rpath=" + dir - elif sys.platform[:5] == "hp-ux": - if self._is_gcc(compiler): - return ["-Wl,+s", "-L" + dir] - return ["+s", "-L" + dir] - else: - if self._is_gcc(compiler): - # gcc on non-GNU systems does not need -Wl, but can - # use it anyway. Since distutils has always passed in - # -Wl whenever gcc was used in the past it is probably - # safest to keep doing so. - if sysconfig.get_config_var("GNULD") == "yes": - # GNU ld needs an extra option to get a RUNPATH - # instead of just an RPATH. - return "-Wl,--enable-new-dtags,-R" + dir - else: - return "-Wl,-R" + dir - else: - # No idea how --enable-new-dtags would be passed on to - # ld if this system was using GNU ld. Don't know if a - # system like this even exists. - return "-R" + dir - - def library_option(self, lib): - return "-l" + lib - - def find_library_file(self, dirs, lib, debug=0): - shared_f = self.library_filename(lib, lib_type='shared') - dylib_f = self.library_filename(lib, lib_type='dylib') - xcode_stub_f = self.library_filename(lib, lib_type='xcode_stub') - static_f = self.library_filename(lib, lib_type='static') - - if sys.platform == 'darwin': - # On OSX users can specify an alternate SDK using - # '-isysroot', calculate the SDK root if it is specified - # (and use it further on) - # - # Note that, as of Xcode 7, Apple SDKs may contain textual stub - # libraries with .tbd extensions rather than the normal .dylib - # shared libraries installed in /. The Apple compiler tool - # chain handles this transparently but it can cause problems - # for programs that are being built with an SDK and searching - # for specific libraries. Callers of find_library_file need to - # keep in mind that the base filename of the returned SDK library - # file might have a different extension from that of the library - # file installed on the running system, for example: - # /Applications/Xcode.app/Contents/Developer/Platforms/ - # MacOSX.platform/Developer/SDKs/MacOSX10.11.sdk/ - # usr/lib/libedit.tbd - # vs - # /usr/lib/libedit.dylib - cflags = sysconfig.get_config_var('CFLAGS') - m = re.search(r'-isysroot\s*(\S+)', cflags) - if m is None: - sysroot = _osx_support._default_sysroot(sysconfig.get_config_var('CC')) - else: - sysroot = m.group(1) - - - - for dir in dirs: - shared = os.path.join(dir, shared_f) - dylib = os.path.join(dir, dylib_f) - static = os.path.join(dir, static_f) - xcode_stub = os.path.join(dir, xcode_stub_f) - - if sys.platform == 'darwin' and ( - dir.startswith('/System/') or ( - dir.startswith('/usr/') and not dir.startswith('/usr/local/'))): - - shared = os.path.join(sysroot, dir[1:], shared_f) - dylib = os.path.join(sysroot, dir[1:], dylib_f) - static = os.path.join(sysroot, dir[1:], static_f) - xcode_stub = os.path.join(sysroot, dir[1:], xcode_stub_f) - - # We're second-guessing the linker here, with not much hard - # data to go on: GCC seems to prefer the shared library, so I'm - # assuming that *all* Unix C compilers do. And of course I'm - # ignoring even GCC's "-static" option. So sue me. - if os.path.exists(dylib): - return dylib - elif os.path.exists(xcode_stub): - return xcode_stub - elif os.path.exists(shared): - return shared - elif os.path.exists(static): - return static - - # Oops, didn't find it in *any* of 'dirs' - return None diff --git a/spack b/spack index 5b6c8e542..deeac5981 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 5b6c8e542ec26121e81a02db93c46eea597ea815 +Subproject commit deeac5981fb9d2bfbd2bd73584dc9ea60e07ff5d From b2f86b740a3ad960054746dd081d994e2543e987 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Sun, 10 Sep 2023 20:09:37 -0600 Subject: [PATCH 09/22] Update orion site packages to avoid duplicate packages --- configs/common/packages.yaml | 2 +- configs/sites/orion/packages.yaml | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 3bc8b356f..83b0befdf 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -264,7 +264,7 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] - # Check Hercules site config when changing this + # Check Orion and Hercules site configs when changing this py-pyyaml: version: ['6.0'] py-scipy: diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index a2fc2772a..f1b689dbd 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -47,6 +47,10 @@ packages: ### # Versions 1.73+ don't compile on orion (bootstrap.sh fails) ### boost: ### version:: ['1.72.0'] + # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version + py-pyyaml: + require: + - "@5.4.1" ### All other external packages listed alphabetically autoconf: @@ -179,10 +183,10 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - openssl: - externals: - - spec: openssl@1.0.2k-fips - prefix: /usr + #openssl: + # externals: + # - spec: openssl@1.0.2k-fips + # prefix: /usr perl: externals: - spec: perl@5.16.3+cpanm+shared+threads @@ -207,10 +211,10 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - sqlite: - externals: - - spec: sqlite@3.7.17~fts~functions+rtree - prefix: /usr + #sqlite: + # externals: + # - spec: sqlite@3.7.17~fts~functions+rtree + # prefix: /usr tar: externals: - spec: tar@1.26 From 09ad7b350f8b2355d889a79bd5bfbd763e93ece5 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 07:37:37 -0600 Subject: [PATCH 10/22] First updates of doc/source/MaintainersSection.rst doc/source/PreConfiguredSites.rst --- doc/source/MaintainersSection.rst | 8 ++------ doc/source/PreConfiguredSites.rst | 10 +++++----- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/doc/source/MaintainersSection.rst b/doc/source/MaintainersSection.rst index b3970130b..e62f7b43b 100644 --- a/doc/source/MaintainersSection.rst +++ b/doc/source/MaintainersSection.rst @@ -227,22 +227,18 @@ MSU Orion On Orion, it is necessary to change the default ``umask`` from ``0027`` to ``0022`` so that users not in the group of the role account can still see and use the software stack. This can be done by running ``umask 022`` after logging into the role account. -miniconda - Follow the instructions in :numref:`Section %s ` to create a basic ``miniconda`` installation and associated modulefile for working with spack. Don't forget to log off and back on to forget about the conda environment. - ecflow ``ecFlow`` must be built manually using the GNU compilers and linked against a static ``boost`` library. After installing `miniconda`, and loading the following modules, follow the instructions in :numref:`Section %s `. Note that the default/system ``qt@5`` can be used on Orion. .. code-block:: console module purge - module use /work/noaa/da/jedipara/spack-stack/modulefiles - module load miniconda/3.9.7 + module load python/3.9.2 module load cmake/3.22.1 module load gcc/10.2.0 mysql - ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s ` to install ``mysql`` in ``/work/noaa/da/role-da/spack-stack/mysql-8.0.31``. + ``mysql`` must be installed separately from ``spack`` using a binary tarball provided by the MySQL community. Follow the instructions in :numref:`Section %s `. .. _MaintainersSection_Hercules: diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 8844451ce..ebd51f89b 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -20,7 +20,7 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Hercules^** | GCC^*, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | NASA | Discover | GCC, Intel | ``/gpfsm/dswdev/jcsda/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Dom Heinzeller / ??? | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -88,8 +88,8 @@ The following is required for building new spack environments and for using spac .. code-block:: console module purge - module use /work/noaa/da/role-da/spack-stack/modulefiles - module load miniconda/3.9.7 + module use /work/noaa/epic/role-epic/spack-stack/orion/modulefiles + module load python/3.9.2 module load ecflow/5.8.4 module load mysql/8.0.31 @@ -97,7 +97,7 @@ For ``spack-stack-1.4.1`` with Intel, load the following modules after loading m .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2022.0.2 module load stack-intel-oneapi-mpi/2021.5.1 module load stack-python/3.9.7 @@ -107,7 +107,7 @@ For ``spack-stack-1.4.1`` with GNU, load the following modules after loading min .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/spack-stack-1.4.1/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/10.2.0 module load stack-openmpi/4.0.4 module load stack-python/3.9.7 From a07aba13b00c3f369a0813752b71d902e1eb7cf8 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 08:54:02 -0600 Subject: [PATCH 11/22] Add include statement for spack-stack python modules for Orion and Hercules --- configs/sites/hercules/modules.yaml | 2 ++ configs/sites/orion/modules.yaml | 2 ++ spack | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/configs/sites/hercules/modules.yaml b/configs/sites/hercules/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/hercules/modules.yaml +++ b/configs/sites/hercules/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/configs/sites/orion/modules.yaml b/configs/sites/orion/modules.yaml index 8264ecf47..4c530d42d 100644 --- a/configs/sites/orion/modules.yaml +++ b/configs/sites/orion/modules.yaml @@ -5,3 +5,5 @@ modules: lmod: exclude: - ecflow + include: + - python diff --git a/spack b/spack index deeac5981..f26ce49a8 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit deeac5981fb9d2bfbd2bd73584dc9ea60e07ff5d +Subproject commit f26ce49a8a9dc8ef4e34bdb817aa47169243deca From 416ec1a548fa87cd4062fe37cfe8ca8a4acda427 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 11:09:25 -0600 Subject: [PATCH 12/22] Clean up configs/sites/orion/packages.yaml --- configs/sites/orion/packages.yaml | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index f1b689dbd..f2ffeb78b 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -28,25 +28,8 @@ packages: prefix: /apps/gcc-10.2.0/openmpi-4.0.4/openmpi-4.0.4 modules: - openmpi/4.0.4 - #python: - # buildable: False - # externals: - # - spec: python@3.9.7 - # prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - # modules: - # - miniconda/3.9.7 - #py-pip: - # buildable: False - # externals: - # - spec: py-pip@21.2.4 - # prefix: /work/noaa/da/jedipara/spack-stack/miniconda-3.9.7 - # modules: - # - miniconda/3.9.7 -### ### Modifications of common packages -### # Versions 1.73+ don't compile on orion (bootstrap.sh fails) -### boost: -### version:: ['1.72.0'] +### Modifications of common packages # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version py-pyyaml: require: @@ -183,6 +166,7 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr + # Do not use, leads to duplicate packages #openssl: # externals: # - spec: openssl@1.0.2k-fips @@ -211,6 +195,7 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr + # Do not use, leads to duplicate packages #sqlite: # externals: # - spec: sqlite@3.7.17~fts~functions+rtree From 05c92499a60a99db68855911ce2626792383fa31 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 11:10:46 -0600 Subject: [PATCH 13/22] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index f26ce49a8..dffa9fdf0 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit f26ce49a8a9dc8ef4e34bdb817aa47169243deca +Subproject commit dffa9fdf02137b298c48e098b8d253b2e66eb061 From b17679f0557f8be6d922fa300e7e984e6e758470 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 13:22:37 -0600 Subject: [PATCH 14/22] More documentation updates for Narwhal, Orion --- doc/source/PreConfiguredSites.rst | 33 ++++++++++--------------------- spack | 2 +- 2 files changed, 11 insertions(+), 24 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index ebd51f89b..3f3ad07ab 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -8,10 +8,10 @@ Directory ``configs/sites`` contains site configurations for several HPC systems Pre-configured sites are split into two categories: Tier 1 with officially supported spack-stack installations (see :numref:`Section %s `), and Tier 2 (sites with configuration files that were tested or contributed by others in the past, but that are not officially supported by the spack-stack team; see :numref:`Section %s `). ============================================================= -Officially supported spack-stack 1.4.1 installations (tier 1) +Officially supported spack-stack 1.5.0 installations (tier 1) ============================================================= -Ready-to-use spack-stack 1.4.1 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Note that some platforms have not received the 1.4.1 installations, for these the previous 1.4.0 installations are listed below. Amazon Web Services AMI are available in the US East 1 or 2 regions for the previous 1.4.0 release (1.4.1 is not provided on AWS AMIs). +Ready-to-use spack-stack 1.5.0 installations are available on the following, fully supported platforms. This version supports the JEDI Skylab release 5 of June 2023, and the UFS Weather Model of July 2023. It can also be used for testing spack-stack with other UFS applications (e.g. the UFS Short Range Weather Application, and the EMC Global Workflow). Amazon Web Services AMI are available in the US East 1 or 2 regions. +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | Organization | System | Compilers | Location | Maintainers | @@ -40,11 +40,9 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Jet | GCC, Intel | ``/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.4.1/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.4.0-hdf5-1.14.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | Intel | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal^** | Intel (test) | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0`` | Dom Heinzeller / Sarah King | -| +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | +| | Narwhal | GCC | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0`` | Dom Heinzeller / Sarah King | | U.S. Navy (HPCMP) +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Nautilus | Intel^* | ``/p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.5.0-openmpi-4.1.5`` | Dom Heinzeller / Sarah King | | +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -67,8 +65,6 @@ Ready-to-use spack-stack 1.4.1 installations are available on the following, ful ^* This system uses a different wgrib2 version 3.1.1 than the default 2.0.8. -^** This system uses a different version of spack-stack (mostly newer), which is usually the case when we onboard a new system in between spack-stack releases. - For questions or problems, please consult the known issues in :numref:`Section %s `, the currently open GitHub `issues `_ and `discussions `_ first. .. _Preconfigured_Sites_Tier1: @@ -93,7 +89,7 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.1`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading miniconda and ecflow: .. code-block:: console @@ -103,7 +99,7 @@ For ``spack-stack-1.4.1`` with Intel, load the following modules after loading m module load stack-python/3.9.7 module available -For ``spack-stack-1.4.1`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading miniconda and ecflow: .. code-block:: console @@ -211,20 +207,11 @@ With Intel, the following is required for building new spack environments and fo module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.0`` with Intel, load the following modules after loading the above modules. - -.. code-block:: console - - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-intel-2021.4.0-hdf5-1.14.0/install/modulefiles/Core - module load stack-intel/2021.4.0 - module load stack-cray-mpich/8.1.14 - module load stack-python/3.9.7 - -For a more recent version of spack-stack based on develop as of June 26, 2023, load the following modules after loading the basic modules above. +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-dev-20230628/envs/unified-env-intel-2021.4.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-intel-2021.4.0/install/modulefiles/Core module load stack-intel/2021.4.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 @@ -249,11 +236,11 @@ With GNU, the following is required for building new spack environments and for module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-1.4.0`` with GNU, load the following modules after loading the above modules. +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading the above modules. .. code-block:: console - module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.4.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core + module use /p/app/projects/NEPTUNE/spack-stack/spack-stack-1.5.0/envs/unified-env-gcc-10.3.0/install/modulefiles/Core module load stack-gcc/10.3.0 module load stack-cray-mpich/8.1.14 module load stack-python/3.9.7 diff --git a/spack b/spack index dffa9fdf0..79fd79d79 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit dffa9fdf02137b298c48e098b8d253b2e66eb061 +Subproject commit 79fd79d79c748b2891cfc35fcfb676cb43234fd6 From c0279a72949512a7355b2dc7734d1400ae542b93 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 15:27:57 -0600 Subject: [PATCH 15/22] Update submodule pointer for spack --- spack | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spack b/spack index 79fd79d79..2b150c871 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 79fd79d79c748b2891cfc35fcfb676cb43234fd6 +Subproject commit 2b150c8718c8b012881d8606d571e13c33f67498 From eafbf9c576908f39a324e85a3e2251180badf068 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 15:40:55 -0600 Subject: [PATCH 16/22] Update .github/workflows/macos-ci-aarch64.yaml - write spack caches to different partition to avoid disk space issues --- .github/workflows/macos-ci-aarch64.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 47659d1f2..0fc0e46a1 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -111,6 +111,12 @@ jobs: echo "Packages in combined spack build caches:" spack buildcache list + # Workaround for limited disk space on macOS arm instance + spack config add "config:build_stage:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/build_stage" + spack config add "config:test_stage:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/test_stage" + spack config add "config:source_cache:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/source_cache" + spack config add "config:misc_cache:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/misc_cache" + # Break installation up in pieces and create build caches in between # This allows us to "spin up" builds that altogether take longer than # six hours, and/or fail later in the build process. From d9a2ba2970094b2cc021b7cbd2cfb1c5e966ac99 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 16:03:46 -0600 Subject: [PATCH 17/22] Correct path for diskspace workaround for macOS arm64 CI --- .github/workflows/macos-ci-aarch64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 0fc0e46a1..406105958 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -112,10 +112,10 @@ jobs: spack buildcache list # Workaround for limited disk space on macOS arm instance - spack config add "config:build_stage:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/build_stage" - spack config add "config:test_stage:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/test_stage" - spack config add "config:source_cache:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/source_cache" - spack config add "config:misc_cache:/private/tmp/tmp-mount-wZIUh7/diskspace_workaround_spack_stack/spack-cache/misc_cache" + spack config add "config:build_stage:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/build_stage" + spack config add "config:test_stage:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/test_stage" + spack config add "config:source_cache:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/source_cache" + spack config add "config:misc_cache:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/misc_cache" # Break installation up in pieces and create build caches in between # This allows us to "spin up" builds that altogether take longer than From 09712b248c72fcc6fd1bebab43b7f19b2a8ad391 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 20:07:31 -0600 Subject: [PATCH 18/22] Revert .gitmodules and update submodule pointer for spack --- .gitmodules | 10 ++++------ spack | 2 +- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7d8e5c869..31d7a8707 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,9 @@ [submodule "spack"] path = spack - ##url = https://github.com/spack/spack - ##branch = develop - #url = https://github.com/jcsda/spack - #branch = release/1.5.0 - url = https://github.com/climbfuji/spack - branch = bugfix/release150_python_cray_rpath + #url = https://github.com/spack/spack + #branch = develop + url = https://github.com/jcsda/spack + branch = release/1.5.0 [submodule "doc/CMakeModules"] path = doc/CMakeModules url = https://github.com/noaa-emc/cmakemodules diff --git a/spack b/spack index 2b150c871..a4675f0c9 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit 2b150c8718c8b012881d8606d571e13c33f67498 +Subproject commit a4675f0c9ce2a5035149e013d9ebc21cf490524e From 3b05deea396c11b29cc1dfa372d5d4d3d566c6cb Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 20:14:13 -0600 Subject: [PATCH 19/22] Update Hercules site config --- doc/source/PreConfiguredSites.rst | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/doc/source/PreConfiguredSites.rst b/doc/source/PreConfiguredSites.rst index 3f3ad07ab..35577bb64 100644 --- a/doc/source/PreConfiguredSites.rst +++ b/doc/source/PreConfiguredSites.rst @@ -18,7 +18,7 @@ Ready-to-use spack-stack 1.5.0 installations are available on the following, ful +=====================+==================================+=================+=========================================================================================================+===============================+ | **HPC platforms** | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ -| | Hercules^** | GCC^*, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env`` | Cam Book / Dom Heinzeller | +| | Hercules^* | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | | MSU +----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ | | Orion | GCC, Intel | ``/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.0/envs/unified-env`` | Cam Book / Dom Heinzeller | +---------------------+----------------------------------+-----------------+---------------------------------------------------------------------------------------------------------+-------------------------------+ @@ -122,29 +122,26 @@ The following is required for building new spack environments and for using spac module load ecflow/5.8.4 module load mysql/8.0.31 -For ``spack-stack-dev-20230814`` with Intel, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with Intel, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-intel/2021.9.0 module load stack-intel-oneapi-mpi/2021.9.0 module load stack-python/3.9.14 module available -For ``spack-stack-dev-20230814`` with GNU, load the following modules after loading miniconda and ecflow: +For ``spack-stack-1.5.0`` with GNU, load the following modules after loading mysql and ecflow: .. code-block:: console - module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-dev-20230814/envs/unified-env/install/modulefiles/Core + module use /work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.5.0/envs/unified-env/install/modulefiles/Core module load stack-gcc/11.3.1 module load stack-openmpi/4.1.5 module load stack-python/3.9.14 module available -.. note:: - The recent update to ``spack-stack-dev-20230814`` was required on Hercules due to a bug in the Intel compilers used in ``spack-stack-1.4.1``. - .. _Preconfigured_Sites_Discover: ------------------------------ From ac30b2a266cd92ec354b5ecff99d0fd237f7581a Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Mon, 11 Sep 2023 20:14:29 -0600 Subject: [PATCH 20/22] Try shorter path for spack caches in .github/workflows/macos-ci-aarch64.yaml --- .github/workflows/macos-ci-aarch64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/macos-ci-aarch64.yaml b/.github/workflows/macos-ci-aarch64.yaml index 406105958..27a4c9525 100644 --- a/.github/workflows/macos-ci-aarch64.yaml +++ b/.github/workflows/macos-ci-aarch64.yaml @@ -112,10 +112,10 @@ jobs: spack buildcache list # Workaround for limited disk space on macOS arm instance - spack config add "config:build_stage:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/build_stage" - spack config add "config:test_stage:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/test_stage" - spack config add "config:source_cache:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/source_cache" - spack config add "config:misc_cache:/private/tmp/tmp-mount-MGMMwD/diskspace_workaround_spack_stack/spack-cache/misc_cache" + spack config add "config:build_stage:/Users/ec2-user/spack-stack/spack-cache/build_stage" + spack config add "config:test_stage:/Users/ec2-user/spack-stack/spack-cache/test_stage" + spack config add "config:source_cache:/Users/ec2-user/spack-stack/spack-cache/source_cache" + spack config add "config:misc_cache:/Users/ec2-user/spack-stack/spack-cache/misc_cache" # Break installation up in pieces and create build caches in between # This allows us to "spin up" builds that altogether take longer than From d4fbbdde43544f76c0408d965476d5c544f337fd Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Sep 2023 09:59:05 -0600 Subject: [PATCH 21/22] Final cleanup of Hercules, Orion, Narwhal site configs --- configs/common/packages.yaml | 2 +- configs/sites/hercules/packages.yaml | 37 ---------------------------- configs/sites/narwhal/packages.yaml | 18 -------------- configs/sites/orion/packages.yaml | 16 ------------ spack | 2 +- 5 files changed, 2 insertions(+), 73 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index 83b0befdf..f36955407 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -17,7 +17,7 @@ version: ['2.4.1'] bison: version: ['3.8.2'] - # Attention - when updating also check orion site config + # Attention - when updating also check S4 site config boost: version: ['1.78.0'] variants: ~atomic +chrono +date_time +exception +filesystem ~graph ~iostreams ~locale ~log ~math ~mpi ~numpy +pic +program_options +python ~random +regex +serialization ~signals +system +test +thread +timer ~wave cxxstd=17 visibility=hidden diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 793f4328b..7195dd5bc 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -20,11 +20,6 @@ packages: prefix: /work/noaa/epic/role-epic/spack-stack/hercules/openmpi-4.1.5/gcc-11.3.1 modules: - openmpi/4.1.5 - #python: - #buildable: False - #externals: - #- spec: python@3.9.14 #+bz2+crypt+ctypes+dbm+lzma~nis+pyexpat~pythoncmd+readline+sqlite3+ssl~tix~tkinter+uuid+zlib - #prefix: /usr ### Modifications of common packages # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version @@ -48,15 +43,6 @@ packages: externals: - spec: binutils@2.35.2 prefix: /usr - #bison: - #externals: - #- spec: bison@3.7.4 - #prefix: /usr - # Do not use! - #cmake: - # externals: - # - spec: cmake@3.20.2 - # prefix: /usr coreutils: externals: - spec: coreutils@8.32 @@ -125,17 +111,6 @@ packages: externals: - spec: openssh@8.7p1 prefix: /usr - # Do not use, can lead to duplicate packages being built - #openssl: - # buildable: False - # externals: - # - spec: openssl@3.0.1 - # prefix: /usr - # Do not use, incomplete package (missing FindBin for example) - #perl: - # externals: - # - spec: perl@5.32.1~cpanm+shared+threads - # prefix: /usr pkgconf: externals: - spec: pkgconf@1.7.3 @@ -150,11 +125,6 @@ packages: externals: - spec: subversion@1.14.1 prefix: /usr - # Do not use, problems on compute nodes with Intel - #tar: - # externals: - # - spec: tar@1.34 - # prefix: /usr texinfo: externals: - spec: texinfo@6.7 @@ -163,10 +133,3 @@ packages: externals: - spec: wget@1.21.1 prefix: /usr - # Do not use external zlib; causes issues with tar due to qt/zlib dependencies - # zlib: - # externals: - # - spec: zlib@1.2.13 - # prefix: /apps/spack-managed/gcc-11.3.1/zlib-1.2.13-ltp4c3zzde3zi3gf7x4b7c7nj5ww4i4g - # modules: - # - zlib/1.2.13 diff --git a/configs/sites/narwhal/packages.yaml b/configs/sites/narwhal/packages.yaml index 640aae18d..987b5ad0e 100644 --- a/configs/sites/narwhal/packages.yaml +++ b/configs/sites/narwhal/packages.yaml @@ -27,20 +27,6 @@ packages: # prefix: /opt/intel/oneapi_2021.3.0.3219 # modules: # - intel/2021.3.0 - #python: - # buildable: False - # externals: - # - spec: python@3.9.7+bz2+ctypes+dbm+lzma+nis+pyexpat+pythoncmd+readline+sqlite3+ssl+tix+tkinter+uuid+zlib - # prefix: /opt/cray/pe/python/3.9.7.1 - # modules: - # - cray-python/3.9.7.1 - #py-pip: - # buildable: False - # externals: - # - spec: py-pip@21.2.3 - # prefix: /opt/cray/pe/python/3.9.7.1 - # modules: - # - cray-python/3.9.7.1 ### All other external packages listed alphabetically autoconf: @@ -55,10 +41,6 @@ packages: externals: - spec: binutils@2.37.20211103 prefix: /usr - #bison: - # externals: - # - spec: bison@3.0.4 - #prefix: /usr curl: externals: - spec: curl@7.66.0+gssapi+ldap+nghttp2 diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index f2ffeb78b..0873bdd55 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -52,11 +52,6 @@ packages: externals: - spec: berkeley-db@5.3.21 prefix: /usr - # Do not use, can lead to duplicate versions of nco being installed - #bison: - # externals: - # - spec: bison@3.0.4 - # prefix: /usr bzip2: externals: - spec: bzip2@1.0.6 @@ -65,7 +60,6 @@ packages: externals: - spec: cpio@2.11 prefix: /usr - # Don't use OS curl, doesn't support secure transfers diffutils: externals: - spec: diffutils@3.3 @@ -166,11 +160,6 @@ packages: externals: - spec: openssh@7.4p1 prefix: /usr - # Do not use, leads to duplicate packages - #openssl: - # externals: - # - spec: openssl@1.0.2k-fips - # prefix: /usr perl: externals: - spec: perl@5.16.3+cpanm+shared+threads @@ -195,11 +184,6 @@ packages: externals: - spec: sed@4.2.2 prefix: /usr - # Do not use, leads to duplicate packages - #sqlite: - # externals: - # - spec: sqlite@3.7.17~fts~functions+rtree - # prefix: /usr tar: externals: - spec: tar@1.26 diff --git a/spack b/spack index a4675f0c9..cce83b11d 160000 --- a/spack +++ b/spack @@ -1 +1 @@ -Subproject commit a4675f0c9ce2a5035149e013d9ebc21cf490524e +Subproject commit cce83b11d25f230d44ff27367ffefd22bf7c9e39 From d0fcb261cd688f470addb94fbd9bdd76272d1842 Mon Sep 17 00:00:00 2001 From: Dom Heinzeller Date: Tue, 12 Sep 2023 12:08:01 -0600 Subject: [PATCH 22/22] Pin awscli to 1.27.84 and py-pyyaml to 5.4.1 --- configs/common/packages.yaml | 9 +++++---- configs/sites/hercules/packages.yaml | 4 ---- configs/sites/orion/packages.yaml | 4 ---- 3 files changed, 5 insertions(+), 12 deletions(-) diff --git a/configs/common/packages.yaml b/configs/common/packages.yaml index f36955407..52e1e37d3 100644 --- a/configs/common/packages.yaml +++ b/configs/common/packages.yaml @@ -11,9 +11,10 @@ lapack: [openblas] yacc: [bison] # + # This version of awscli goes with py-pyyaml@5.4.1 awscli: - version: ['1.29.41'] - bacio: + version: ['1.27.84'] + bacio: version: ['2.4.1'] bison: version: ['3.8.2'] @@ -264,9 +265,9 @@ # Versions earlier than 0.11.0 don't compile on macOS with llvm-clang/13.0.0 and Python/3.9, # and 0.11.0 leads to downstream errors in py-scipy with the Intel compilers version: ['0.12.2'] - # Check Orion and Hercules site configs when changing this + # This version of py-pyyaml goes with awscli@1.27.84 py-pyyaml: - version: ['6.0'] + version: ['5.4.1'] py-scipy: version: ['1.9.3'] # Pin the py-setuptools version to avoid duplicate Python packages diff --git a/configs/sites/hercules/packages.yaml b/configs/sites/hercules/packages.yaml index 7195dd5bc..6a328485b 100644 --- a/configs/sites/hercules/packages.yaml +++ b/configs/sites/hercules/packages.yaml @@ -22,10 +22,6 @@ packages: - openmpi/4.1.5 ### Modifications of common packages - # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version - py-pyyaml: - require: - - "@5.4.1" # Version 2.0.8 doesn't compile on Hercules wgrib2: version:: ['3.1.1'] diff --git a/configs/sites/orion/packages.yaml b/configs/sites/orion/packages.yaml index 0873bdd55..47726af43 100644 --- a/configs/sites/orion/packages.yaml +++ b/configs/sites/orion/packages.yaml @@ -30,10 +30,6 @@ packages: - openmpi/4.0.4 ### Modifications of common packages - # "Soft suggestion" in common/packages.yaml doesn't work, need 5.4.1 because of gcc version - py-pyyaml: - require: - - "@5.4.1" ### All other external packages listed alphabetically autoconf: