forked from spack/spack
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Remove the old concretizer (spack#45215)
The old concretizer is still used to bootstrap clingo from source. If we switch to a DAG model
where compilers are treated as nodes, we need to either:
1. fix the old concretizer to support this (which is a lot of work and possibly research), or
2. bootstrap `clingo` without the old concretizer.
This PR takes the second approach and gets rid of the old concretizer code. To bootstrap
`clingo`, we store some concrete spec prototypes as JSON, select one according to the
coarse-grained system architecture, and tweak them according to the current host.
The old concretizer and related dead code are removed. In particular, this removes
`Spec.normalize()` and related methods, which were used in many unit-tests to set
up the test context. The tests have been updated not to use `normalize()`.
- [x] Bootstrap clingo concretization based on a JSON file
- [x] Bootstrap clingo *before* patchelf
- [x] Remove any use of the old concretizer, including:
* Remove only_clingo and only_original fixtures
* Remove _old_concretize and _new_concretize
* Remove _concretize_together_old
* Remove _concretize_together_new
* Remove any use of `SPACK_TEST_SOLVER`
* Simplify CI jobs
- [x] ensure bootstrapping `clingo` works on on Darwin and Windows
- [x] Raise an intelligible error when a compiler is missing
- [x] Ensure bootstrapping works on FreeBSD
- [x] remove normalize and related methods
Signed-off-by: Todd Gamblin <[email protected]>- Loading branch information
Showing
40 changed files
with
286 additions
and
1,938 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,6 +1,5 @@ | ||
| config: | ||
| locks: false | ||
| concretizer: clingo | ||
| build_stage:: | ||
| - '$spack/.staging' | ||
| stage_name: '{name}-{version}-{hash:7}' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,154 @@ | ||
| # Copyright 2013-2024 Lawrence Livermore National Security, LLC and other | ||
| # Spack Project Developers. See the top-level COPYRIGHT file for details. | ||
| # | ||
| # SPDX-License-Identifier: (Apache-2.0 OR MIT) | ||
| """Bootstrap concrete specs for clingo | ||
| Spack uses clingo to concretize specs. When clingo itself needs to be bootstrapped from sources, | ||
| we need to rely on another mechanism to get a concrete spec that fits the current host. | ||
| This module contains the logic to get a concrete spec for clingo, starting from a prototype | ||
| JSON file for a similar platform. | ||
| """ | ||
| import pathlib | ||
| import sys | ||
| from typing import Dict, Optional, Tuple | ||
|
|
||
| import archspec.cpu | ||
|
|
||
| import spack.compiler | ||
| import spack.compilers | ||
| import spack.platforms | ||
| import spack.spec | ||
| import spack.traverse | ||
|
|
||
| from .config import spec_for_current_python | ||
|
|
||
|
|
||
| class ClingoBootstrapConcretizer: | ||
| def __init__(self, configuration): | ||
| self.host_platform = spack.platforms.host() | ||
| self.host_os = self.host_platform.operating_system("frontend") | ||
| self.host_target = archspec.cpu.host().family | ||
| self.host_architecture = spack.spec.ArchSpec.frontend_arch() | ||
| self.host_architecture.target = str(self.host_target) | ||
| self.host_compiler = self._valid_compiler_or_raise() | ||
| self.host_python = self.python_external_spec() | ||
| if str(self.host_platform) == "linux": | ||
| self.host_libc = self.libc_external_spec() | ||
|
|
||
| self.external_cmake, self.external_bison = self._externals_from_yaml(configuration) | ||
|
|
||
| def _valid_compiler_or_raise(self) -> "spack.compiler.Compiler": | ||
| if str(self.host_platform) == "linux": | ||
| compiler_name = "gcc" | ||
| elif str(self.host_platform) == "darwin": | ||
| compiler_name = "apple-clang" | ||
| elif str(self.host_platform) == "windows": | ||
| compiler_name = "msvc" | ||
| elif str(self.host_platform) == "freebsd": | ||
| compiler_name = "clang" | ||
| else: | ||
| raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}") | ||
| candidates = spack.compilers.compilers_for_spec( | ||
| compiler_name, arch_spec=self.host_architecture | ||
| ) | ||
| if not candidates: | ||
| raise RuntimeError( | ||
| f"Cannot find any version of {compiler_name} to bootstrap clingo from sources" | ||
| ) | ||
| candidates.sort(key=lambda x: x.spec.version, reverse=True) | ||
| return candidates[0] | ||
|
|
||
| def _externals_from_yaml( | ||
| self, configuration: "spack.config.Configuration" | ||
| ) -> Tuple[Optional["spack.spec.Spec"], Optional["spack.spec.Spec"]]: | ||
| packages_yaml = configuration.get("packages") | ||
| requirements = {"cmake": "@3.20:", "bison": "@2.5:"} | ||
| selected: Dict[str, Optional["spack.spec.Spec"]] = {"cmake": None, "bison": None} | ||
| for pkg_name in ["cmake", "bison"]: | ||
| if pkg_name not in packages_yaml: | ||
| continue | ||
|
|
||
| candidates = packages_yaml[pkg_name].get("externals", []) | ||
| for candidate in candidates: | ||
| s = spack.spec.Spec(candidate["spec"], external_path=candidate["prefix"]) | ||
| if not s.satisfies(requirements[pkg_name]): | ||
| continue | ||
|
|
||
| if not s.intersects(f"%{self.host_compiler.spec}"): | ||
| continue | ||
|
|
||
| if not s.intersects(f"arch={self.host_architecture}"): | ||
| continue | ||
|
|
||
| selected[pkg_name] = self._external_spec(s) | ||
| break | ||
| return selected["cmake"], selected["bison"] | ||
|
|
||
| def prototype_path(self) -> pathlib.Path: | ||
| """Path to a prototype concrete specfile for clingo""" | ||
| parent_dir = pathlib.Path(__file__).parent | ||
| result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-{self.host_target}.json" | ||
| if str(self.host_platform) == "linux": | ||
| # Using aarch64 as a fallback, since it has gnuconfig (x86_64 doesn't have it) | ||
| if not result.exists(): | ||
| result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-aarch64.json" | ||
|
|
||
| elif str(self.host_platform) == "freebsd": | ||
| result = parent_dir / "prototypes" / f"clingo-{self.host_platform}-amd64.json" | ||
|
|
||
| elif not result.exists(): | ||
| raise RuntimeError(f"Cannot bootstrap clingo from sources on {self.host_platform}") | ||
|
|
||
| return result | ||
|
|
||
| def concretize(self) -> "spack.spec.Spec": | ||
| # Read the prototype and mark it NOT concrete | ||
| s = spack.spec.Spec.from_specfile(str(self.prototype_path())) | ||
| s._mark_concrete(False) | ||
|
|
||
| # Tweak it to conform to the host architecture | ||
| for node in s.traverse(): | ||
| node.architecture.os = str(self.host_os) | ||
| node.compiler = self.host_compiler.spec | ||
| node.architecture = self.host_architecture | ||
|
|
||
| if node.name == "gcc-runtime": | ||
| node.versions = self.host_compiler.spec.versions | ||
|
|
||
| for edge in spack.traverse.traverse_edges([s], cover="edges"): | ||
| if edge.spec.name == "python": | ||
| edge.spec = self.host_python | ||
|
|
||
| if edge.spec.name == "bison" and self.external_bison: | ||
| edge.spec = self.external_bison | ||
|
|
||
| if edge.spec.name == "cmake" and self.external_cmake: | ||
| edge.spec = self.external_cmake | ||
|
|
||
| if "libc" in edge.virtuals: | ||
| edge.spec = self.host_libc | ||
|
|
||
| s._finalize_concretization() | ||
|
|
||
| # Work around the fact that the installer calls Spec.dependents() and | ||
| # we modified edges inconsistently | ||
| return s.copy() | ||
|
|
||
| def python_external_spec(self) -> "spack.spec.Spec": | ||
| """Python external spec corresponding to the current running interpreter""" | ||
| result = spack.spec.Spec(spec_for_current_python(), external_path=sys.exec_prefix) | ||
| return self._external_spec(result) | ||
|
|
||
| def libc_external_spec(self) -> "spack.spec.Spec": | ||
| result = self.host_compiler.default_libc | ||
| return self._external_spec(result) | ||
|
|
||
| def _external_spec(self, initial_spec) -> "spack.spec.Spec": | ||
| initial_spec.namespace = "builtin" | ||
| initial_spec.compiler = self.host_compiler.spec | ||
| initial_spec.architecture = self.host_architecture | ||
| for flag_type in spack.spec.FlagMap.valid_compiler_flags(): | ||
| initial_spec.compiler_flags[flag_type] = [] | ||
| return spack.spec.parse_with_version_concrete(initial_spec) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
1 change: 1 addition & 0 deletions
1
lib/spack/spack/bootstrap/prototypes/clingo-darwin-aarch64.json
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
Large diffs are not rendered by default.
Oops, something went wrong.
1 change: 1 addition & 0 deletions
1
lib/spack/spack/bootstrap/prototypes/clingo-windows-x86_64.json
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1 @@ | ||
| {"spec":{"_meta":{"version":4},"nodes":[{"name":"clingo-bootstrap","version":"spack","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","docs":false,"generator":"ninja","ipo":true,"optimized":false,"patches":["311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a","4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065","c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a"],"python":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["4ccfd173d439ed1e23eff42d5a01a8fbb21341c632d86b5691242dc270dbf065","311bd2ae3f2f5274d1d36a2d65f887dfdf4c309a3c6bb29a53bbafb82b42ba7a","c5c4db292a920ded6eecfbb6749d88ce9c4f179500aee6aee3a417b93c7c5c7a"],"package_hash":"hkhwttazqtgz7nw7e6yzka5nc7o6akrqe23kb3gkdl37dcmwjxkq====","dependencies":[{"name":"cmake","hash":"4ezjoethijaqxue2xt3nal4txe767xns","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ninja","hash":"f5ggzol7zdybfonmhgh4ujmlb2or4ut6","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"python","hash":"6aoldahfjkgxhhqij3254wekxgkw76j7","parameters":{"deptypes":["build","link","run"],"virtuals":[]}},{"name":"python-venv","hash":"x6lyli3psq6zk3644k2wugm576lci33r","parameters":{"deptypes":["build","run"],"virtuals":[]}},{"name":"re2c","hash":"7bxfiqnxbqtqsyb2un5c7gqyeqeovmuk","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"winbison","hash":"xvhc7don5aszzxvlizjiau2pbs4ar2a6","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"hash":"wfpfomrvcbtfjrjxjc2f3fi3nj22cyat"},{"name":"cmake","version":"3.29.6","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","build_type":"Release","doc":false,"ncurses":false,"ownlibs":true,"patches":["d041289e3e9483cbdbbac46705a1fb01a7c40a8fc13e291229fb4fa2b071369b"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["d041289e3e9483cbdbbac46705a1fb01a7c40a8fc13e291229fb4fa2b071369b"],"package_hash":"6eevq4j4p722uva3cej3xbkvh57fv7ahuqg4pheefouecqh5rnxa====","dependencies":[{"name":"curl","hash":"jt3nq766b6qphqfhezutxbsls4r5ecf5","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"ninja","hash":"f5ggzol7zdybfonmhgh4ujmlb2or4ut6","parameters":{"deptypes":["build","link"],"virtuals":[]}},{"name":"zlib","hash":"xfzaydeo4udjmwno4hi5mehuje43mrp5","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"hash":"4ezjoethijaqxue2xt3nal4txe767xns"},{"name":"curl","version":"8.7.1","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"nmake","gssapi":false,"ldap":false,"libidn2":false,"librtmp":false,"libs":"shared","libssh":false,"libssh2":false,"nghttp2":false,"tls":["sspi"],"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"kcgsfmigaqmusztsy67k2gfkizipob2uj5o5yub2i4onsxph454q====","dependencies":[{"name":"perl","hash":"pkljs7xtxfgvkcbczmygiubpdczoqrlb","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"zlib","hash":"xfzaydeo4udjmwno4hi5mehuje43mrp5","parameters":{"deptypes":["build","link"],"virtuals":["zlib-api"]}}],"hash":"jt3nq766b6qphqfhezutxbsls4r5ecf5"},{"name":"perl","version":"5.38.2","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","cpanm":true,"opcode":true,"open":true,"shared":true,"threads":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"xx43demwtox532nxdz7gqwrlx2g5ksgxfv62c7h4zqfpmclnrruq====","hash":"pkljs7xtxfgvkcbczmygiubpdczoqrlb"},{"name":"zlib","version":"1.3.1","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","optimize":true,"pic":true,"shared":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"2jkvm4tfyhgosw533mwogyfqz2z32hvk4h5leguukrofpebi5xgq====","hash":"xfzaydeo4udjmwno4hi5mehuje43mrp5"},{"name":"ninja","version":"1.12.0","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","re2c":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"jcf35uxhgv42a53liynorg4clnpbgbirydp5s3fjeobf7ur2obbq====","dependencies":[{"name":"python","hash":"6aoldahfjkgxhhqij3254wekxgkw76j7","parameters":{"deptypes":["build"],"virtuals":[]}}],"hash":"f5ggzol7zdybfonmhgh4ujmlb2or4ut6"},{"name":"python","version":"3.11.9","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","bz2":true,"ctypes":true,"dbm":true,"debug":false,"libxml2":true,"lzma":true,"nis":false,"optimizations":false,"patches":["01b5df08776d2c3ffeb75da4ccff144cd554b63fcf9962f27c6ecb5fca06a33d","7abb961432aa530349755d639c2902a342f3b5744d11103901e0acf88fae533e","bccfd87e518d2ebc1dafe5d009b9071c046fe8400d52d2f0283bda6904c4dbf1","fc5b6c586b1b654ac2ed00ba6417b1eb526bd24a0dc28074ce14ff56f6057f7c"],"pic":true,"pyexpat":true,"pythoncmd":false,"readline":false,"shared":true,"sqlite3":true,"ssl":true,"tkinter":false,"uuid":true,"zlib":true,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"patches":["01b5df08776d2c3ffeb75da4ccff144cd554b63fcf9962f27c6ecb5fca06a33d","fc5b6c586b1b654ac2ed00ba6417b1eb526bd24a0dc28074ce14ff56f6057f7c","bccfd87e518d2ebc1dafe5d009b9071c046fe8400d52d2f0283bda6904c4dbf1","7abb961432aa530349755d639c2902a342f3b5744d11103901e0acf88fae533e"],"package_hash":"u4pa2ashu6det7izfzs7hl7fhelecogvtit6mb3nzsn6gvzpnnoa====","hash":"6aoldahfjkgxhhqij3254wekxgkw76j7"},{"name":"python-venv","version":"1.0","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"bvjgntlwbvi343x5ctophqqvq6nbx2h4ggbxnjrvnjb3jneitahq====","dependencies":[{"name":"python","hash":"6aoldahfjkgxhhqij3254wekxgkw76j7","parameters":{"deptypes":["build","run"],"virtuals":[]}}],"hash":"x6lyli3psq6zk3644k2wugm576lci33r"},{"name":"re2c","version":"3.0","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"generic","cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"e5avvdpvjibybyeqgefi3xrpxyzr2mejjap4mx7q2lgxmpqzco4q====","dependencies":[{"name":"cmake","hash":"4ezjoethijaqxue2xt3nal4txe767xns","parameters":{"deptypes":["build","link"],"virtuals":[]}}],"hash":"7bxfiqnxbqtqsyb2un5c7gqyeqeovmuk"},{"name":"winbison","version":"2.5.25","arch":{"platform":"windows","platform_os":"windows10.0.20348","target":"x86_64"},"compiler":{"name":"msvc","version":"19.40.33811"},"namespace":"builtin","parameters":{"build_system":"cmake","build_type":"Release","generator":"ninja","ipo":false,"cflags":[],"cppflags":[],"cxxflags":[],"fflags":[],"ldflags":[],"ldlibs":[]},"package_hash":"t3g2slcnnleieqtz66oly6vsfe5ibje6b2wmamxv5chuewwds5la====","dependencies":[{"name":"cmake","hash":"4ezjoethijaqxue2xt3nal4txe767xns","parameters":{"deptypes":["build"],"virtuals":[]}},{"name":"ninja","hash":"f5ggzol7zdybfonmhgh4ujmlb2or4ut6","parameters":{"deptypes":["build"],"virtuals":[]}}],"hash":"xvhc7don5aszzxvlizjiau2pbs4ar2a6"}]}} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.