From 28af00c611164bfbb242b2184bd3168ac1e57d2b Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Thu, 8 Jun 2023 21:22:23 +0000 Subject: [PATCH 1/2] Support configuring the set of codegen backends to build per host triple This allows building the compiler itself with one backend while using another backend at runtime. For example this allows compiling rustc to wasm using LLVM, while using Cranelift at runtime to produce actual code. Cranelift can't compile to wasm, but is perfectly capable of running on wasm. LLVM can compile to wasm, but can't run on wasm. [^1] [^1]: The prototype of this still requires a couple of other patches. --- config.example.toml | 5 +++ src/bootstrap/src/core/build_steps/compile.rs | 16 ++++----- src/bootstrap/src/core/build_steps/dist.rs | 4 +-- src/bootstrap/src/core/build_steps/test.rs | 7 ++-- src/bootstrap/src/core/builder.rs | 5 +-- src/bootstrap/src/core/config/config.rs | 35 ++++++++++++++++--- src/bootstrap/src/core/sanity.rs | 32 +++++++++-------- src/bootstrap/src/lib.rs | 7 ++-- 8 files changed, 74 insertions(+), 37 deletions(-) diff --git a/config.example.toml b/config.example.toml index a5ef4022d39d5..098811195d7c9 100644 --- a/config.example.toml +++ b/config.example.toml @@ -829,6 +829,11 @@ # target triples containing `-none`, `nvptx`, `switch`, or `-uefi`. #no-std = (bool) +# This is an array of the codegen backends that will be compiled a rustc +# compiled for this target, overriding the global rust.codegen-backends option. +# See that option for more info. +#codegen-backends = rust.codegen-backends (array) + # ============================================================================= # Distribution options # diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index e06bc3fb09ac1..b2a9a89fff643 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -1003,7 +1003,7 @@ impl Step for Rustc { pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { cargo .arg("--features") - .arg(builder.rustc_features(builder.kind)) + .arg(builder.rustc_features(builder.kind, target)) .arg("--manifest-path") .arg(builder.src.join("compiler/rustc/Cargo.toml")); @@ -1060,7 +1060,7 @@ pub fn rustc_cargo_env( cargo.env("CFG_OMIT_GIT_HASH", "1"); } - if let Some(backend) = builder.config.default_codegen_backend() { + if let Some(backend) = builder.config.default_codegen_backend(target) { cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); } @@ -1101,7 +1101,7 @@ pub fn rustc_cargo_env( // build. If we are in a check build we still go ahead here presuming we've // detected that LLVM is already built and good to go which helps prevent // busting caches (e.g. like #71152). - if builder.config.llvm_enabled() { + if builder.config.llvm_enabled(target) { let building_is_expensive = crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler @@ -1250,7 +1250,7 @@ pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { let mut needs_codegen_backend_config = true; - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if path .path .to_str() @@ -1287,7 +1287,7 @@ impl Step for CodegenBackend { return; } - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1387,7 +1387,7 @@ fn copy_codegen_backends_to_sysroot( return; } - for backend in builder.config.rust_codegen_backends.iter() { + for backend in builder.config.codegen_backends(target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1694,7 +1694,7 @@ impl Step for Assemble { // to not fail while linking the artifacts. build_compiler.stage = actual_stage; - for &backend in builder.config.rust_codegen_backends.iter() { + for &backend in builder.config.codegen_backends(target_compiler.host) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1779,7 +1779,7 @@ impl Step for Assemble { } } - if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + if builder.config.llvm_enabled(target_compiler.host) { let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target: target_compiler.host }); if !builder.config.dry_run() && builder.config.llvm_tools_enabled { diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index f50026368dabd..6d56492e41b56 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -1278,7 +1278,7 @@ impl Step for CodegenBackend { } fn make_run(run: RunConfig<'_>) { - for &backend in &run.builder.config.rust_codegen_backends { + for &backend in run.builder.config.codegen_backends(run.target) { if backend == "llvm" { continue; // Already built as part of rustc } @@ -1302,7 +1302,7 @@ impl Step for CodegenBackend { return None; } - if !builder.config.rust_codegen_backends.contains(&self.backend) { + if !builder.config.codegen_backends(self.compiler.host).contains(&self.backend) { return None; } diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 04728e2e00dcc..e0e94f194e0a0 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1841,7 +1841,7 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the let mut llvm_components_passed = false; let mut copts_passed = false; - if builder.config.llvm_enabled() { + if builder.config.llvm_enabled(compiler.host) { let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target: builder.config.build }); if !builder.config.dry_run() { @@ -3155,7 +3155,8 @@ impl Step for CodegenCranelift { return; } - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { + if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("cranelift")) + { builder.info("cranelift not in rust.codegen-backends. skipping"); return; } @@ -3277,7 +3278,7 @@ impl Step for CodegenGCC { return; } - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + if !builder.config.codegen_backends(run.target).contains(&INTERNER.intern_str("gcc")) { builder.info("gcc not in rust.codegen-backends. skipping"); return; } diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs index 3770d0687b242..0ececcdcacc1a 100644 --- a/src/bootstrap/src/core/builder.rs +++ b/src/bootstrap/src/core/builder.rs @@ -1230,7 +1230,7 @@ impl<'a> Builder<'a> { /// Note that this returns `None` if LLVM is disabled, or if we're in a /// check build or dry-run, where there's no need to build all of LLVM. fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { + if self.config.llvm_enabled(target) && self.kind != Kind::Check && !self.config.dry_run() { let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); if llvm_config.is_file() { return Some(llvm_config); @@ -2113,7 +2113,8 @@ impl<'a> Builder<'a> { }; if let Some(limit) = limit { - if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" + if stage == 0 + || self.config.default_codegen_backend(target).unwrap_or_default() == "llvm" { rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); } diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index fcdd742e69c22..5969ddfdf9003 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -581,6 +581,7 @@ pub struct Target { pub wasi_root: Option, pub qemu_rootfs: Option, pub no_std: bool, + pub codegen_backends: Option>>, } impl Target { @@ -1139,6 +1140,7 @@ define_config! { wasi_root: Option = "wasi-root", qemu_rootfs: Option = "qemu-rootfs", no_std: Option = "no-std", + codegen_backends: Option> = "codegen-backends", } } @@ -1845,6 +1847,24 @@ impl Config { target.profiler = cfg.profiler; target.rpath = cfg.rpath; + if let Some(ref backends) = cfg.codegen_backends { + let available_backends = vec!["llvm", "cranelift", "gcc"]; + + target.codegen_backends = Some(backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'target.{triple}.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'target.{triple}.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + INTERNER.intern_str(s) + }).collect()); + } + config.target_config.insert(TargetSelection::from_user(&triple), target); } } @@ -2227,8 +2247,8 @@ impl Config { self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) } - pub fn llvm_enabled(&self) -> bool { - self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + pub fn llvm_enabled(&self, target: TargetSelection) -> bool { + self.codegen_backends(target).contains(&INTERNER.intern_str("llvm")) } pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { @@ -2247,8 +2267,15 @@ impl Config { self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) } - pub fn default_codegen_backend(&self) -> Option> { - self.rust_codegen_backends.get(0).cloned() + pub fn codegen_backends(&self, target: TargetSelection) -> &[Interned] { + self.target_config + .get(&target) + .and_then(|cfg| cfg.codegen_backends.as_deref()) + .unwrap_or(&self.rust_codegen_backends) + } + + pub fn default_codegen_backend(&self, target: TargetSelection) -> Option> { + self.codegen_backends(target).get(0).cloned() } pub fn git_config(&self) -> GitConfig<'_> { diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs index 5f1ca5de74afb..1dce8d8ac7184 100644 --- a/src/bootstrap/src/core/sanity.rs +++ b/src/bootstrap/src/core/sanity.rs @@ -16,7 +16,6 @@ use std::path::PathBuf; use std::process::Command; use crate::core::config::Target; -use crate::utils::cache::INTERNER; use crate::utils::helpers::output; use crate::Build; @@ -88,19 +87,19 @@ pub fn check(build: &mut Build) { } // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - && build - .hosts - .iter() - .map(|host| { - build + let building_llvm = build + .hosts + .iter() + .map(|host| { + build.config.llvm_enabled(*host) + && build .config .target_config .get(host) .map(|config| config.llvm_config.is_none()) .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); let need_cmake = building_llvm || build.config.any_sanitizers_to_build(); if need_cmake && cmd_finder.maybe_have("cmake").is_none() { @@ -190,13 +189,16 @@ than building it. if !build.config.dry_run() { cmd_finder.must_have(build.cxx(*host).unwrap()); } - } - if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {filecheck:?} does not exist"); + if build.config.llvm_enabled(*host) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) + && !filecheck.exists() + && build.config.codegen_tests + { + panic!("FileCheck executable {filecheck:?} does not exist"); + } } } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 1726e7aacbcea..6a7bba2463dd1 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -734,12 +734,12 @@ impl Build { } /// Gets the space-separated set of activated features for the compiler. - fn rustc_features(&self, kind: Kind) -> String { + fn rustc_features(&self, kind: Kind, target: TargetSelection) -> String { let mut features = vec![]; if self.config.jemalloc { features.push("jemalloc"); } - if self.config.llvm_enabled() || kind == Kind::Check { + if self.config.llvm_enabled(target) || kind == Kind::Check { features.push("llvm"); } // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` @@ -1560,7 +1560,8 @@ impl Build { || target .map(|t| self.config.profiler_enabled(t)) .unwrap_or_else(|| self.config.any_profiler_enabled())) - && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + && (dep != "rustc_codegen_llvm" + || self.config.hosts.iter().any(|host| self.config.llvm_enabled(*host))) { list.push(*dep); } From f4e279f2db0aa8235c59c1da1dc146b1274a63e4 Mon Sep 17 00:00:00 2001 From: bjorn3 <17426603+bjorn3@users.noreply.github.com> Date: Sat, 27 Jan 2024 11:39:43 +0100 Subject: [PATCH 2/2] Add bootstrap changelog entry --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 327b4674acfdb..3c076f80f5177 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -111,4 +111,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Warning, summary: "A new `optimized-compiler-builtins` option has been introduced. Whether to build llvm's `compiler-rt` from source is no longer implicitly controlled by git state. See the PR for more details.", }, + ChangeInfo { + change_id: 120348, + severity: ChangeSeverity::Info, + summary: "New option `target..codegen-backends` added to config.toml.", + }, ];