diff --git a/config.example.toml b/config.example.toml index ac5e491b4b520..c3d2ad094ceba 100644 --- a/config.example.toml +++ b/config.example.toml @@ -163,6 +163,16 @@ # Custom CMake defines to set when building LLVM. #build-config = {} +# ============================================================================= +# Tweaking how GCC is compiled +# ============================================================================= +[gcc] +# Download GCC from CI instead of building it locally. +# Note that this will attempt to download GCC even if there are local +# modifications to the `src/gcc` submodule. +# Currently, this is only supported for the `x86_64-unknown-linux-gnu` target. +# download-ci-gcc = false + # ============================================================================= # General build configuration options # ============================================================================= diff --git a/src/bootstrap/download-ci-gcc-stamp b/src/bootstrap/download-ci-gcc-stamp new file mode 100644 index 0000000000000..bbe26afc95269 --- /dev/null +++ b/src/bootstrap/download-ci-gcc-stamp @@ -0,0 +1,4 @@ +Change this file to make users of the `download-ci-gcc` configuration download +a new version of GCC from CI, even if the GCC submodule hasn’t changed. + +Last change is for: https://github.com/rust-lang/rust/pull/138051 diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index 70789fbbeeba0..5a4bc9bdbcba9 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -14,13 +14,67 @@ use std::sync::OnceLock; use build_helper::ci::CiEnv; -use crate::Kind; -use crate::core::builder::{Builder, Cargo, RunConfig, ShouldRun, Step}; +use crate::core::builder::{Builder, Cargo, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash}; use crate::utils::exec::command; use crate::utils::helpers::{self, t}; +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +pub struct Gcc { + pub target: TargetSelection, +} + +#[derive(Clone)] +pub struct GccOutput { + pub libgccjit: PathBuf, +} + +impl Step for Gcc { + type Output = GccOutput; + + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/gcc").alias("gcc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Gcc { target: run.target }); + } + + /// Compile GCC (specifically `libgccjit`) for `target`. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let target = self.target; + + // If GCC has already been built, we avoid building it again. + let metadata = match get_gcc_build_status(builder, target) { + GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path }, + GccBuildStatus::ShouldBuild(m) => m, + }; + + let _guard = builder.msg_unstaged(Kind::Build, "GCC", target); + t!(metadata.stamp.remove()); + let _time = helpers::timeit(builder); + + let libgccjit_path = libgccjit_built_path(&metadata.install_dir); + if builder.config.dry_run() { + return GccOutput { libgccjit: libgccjit_path }; + } + + build_gcc(&metadata, builder, target); + + let lib_alias = metadata.install_dir.join("lib/libgccjit.so.0"); + if !lib_alias.exists() { + t!(builder.symlink_file(&libgccjit_path, lib_alias)); + } + + t!(metadata.stamp.write()); + + GccOutput { libgccjit: libgccjit_path } + } +} + pub struct Meta { stamp: BuildStamp, out_dir: PathBuf, @@ -34,17 +88,45 @@ pub enum GccBuildStatus { ShouldBuild(Meta), } -/// This returns whether we've already previously built GCC. +/// Tries to download GCC from CI if it is enabled and GCC artifacts +/// are available for the given target. +/// Returns a path to the libgccjit.so file. +#[cfg(not(test))] +fn try_download_gcc(builder: &Builder<'_>, target: TargetSelection) -> Option { + // Try to download GCC from CI if configured and available + if !matches!(builder.config.gcc_ci_mode, crate::core::config::GccCiMode::DownloadFromCi) { + return None; + } + if target != "x86_64-unknown-linux-gnu" { + eprintln!("GCC CI download is only available for the `x86_64-unknown-linux-gnu` target"); + return None; + } + let sha = + detect_gcc_sha(&builder.config, builder.config.rust_info.is_managed_git_subrepository()); + let root = ci_gcc_root(&builder.config); + let gcc_stamp = BuildStamp::new(&root).with_prefix("gcc").add_stamp(&sha); + if !gcc_stamp.is_up_to_date() && !builder.config.dry_run() { + builder.config.download_ci_gcc(&sha, &root); + t!(gcc_stamp.write()); + } + // FIXME: put libgccjit.so into a lib directory in dist::Gcc + Some(root.join("libgccjit.so")) +} + +#[cfg(test)] +fn try_download_gcc(_builder: &Builder<'_>, _target: TargetSelection) -> Option { + None +} + +/// This returns information about whether GCC should be built or if it's already built. +/// It transparently handles downloading GCC from CI if needed. /// /// It's used to avoid busting caches during x.py check -- if we've already built /// GCC, it's fine for us to not try to avoid doing so. -pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus { - // Initialize the gcc submodule if not initialized already. - builder.config.update_submodule("src/gcc"); - - let root = builder.src.join("src/gcc"); - let out_dir = builder.gcc_out(target).join("build"); - let install_dir = builder.gcc_out(target).join("install"); +pub fn get_gcc_build_status(builder: &Builder<'_>, target: TargetSelection) -> GccBuildStatus { + if let Some(path) = try_download_gcc(builder, target) { + return GccBuildStatus::AlreadyBuilt(path); + } static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { @@ -55,6 +137,13 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc ) }); + // Initialize the gcc submodule if not initialized already. + builder.config.update_submodule("src/gcc"); + + let root = builder.src.join("src/gcc"); + let out_dir = builder.gcc_out(target).join("build"); + let install_dir = builder.gcc_out(target).join("install"); + let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { @@ -87,125 +176,72 @@ fn libgccjit_built_path(install_dir: &Path) -> PathBuf { install_dir.join("lib/libgccjit.so") } -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -pub struct Gcc { - pub target: TargetSelection, -} - -#[derive(Clone)] -pub struct GccOutput { - pub libgccjit: PathBuf, -} - -impl Step for Gcc { - type Output = GccOutput; - - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/gcc").alias("gcc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Gcc { target: run.target }); - } - - /// Compile GCC (specifically `libgccjit`) for `target`. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let target = self.target; - - // If GCC has already been built, we avoid building it again. - let Meta { stamp, out_dir, install_dir, root } = match prebuilt_gcc_config(builder, target) - { - GccBuildStatus::AlreadyBuilt(path) => return GccOutput { libgccjit: path }, - GccBuildStatus::ShouldBuild(m) => m, - }; +fn build_gcc(metadata: &Meta, builder: &Builder<'_>, target: TargetSelection) { + let Meta { stamp: _, out_dir, install_dir, root } = metadata; - let _guard = builder.msg_unstaged(Kind::Build, "GCC", target); - t!(stamp.remove()); - let _time = helpers::timeit(builder); - t!(fs::create_dir_all(&out_dir)); - t!(fs::create_dir_all(&install_dir)); + t!(fs::create_dir_all(out_dir)); + t!(fs::create_dir_all(install_dir)); - let libgccjit_path = libgccjit_built_path(&install_dir); - if builder.config.dry_run() { - return GccOutput { libgccjit: libgccjit_path }; + // GCC creates files (e.g. symlinks to the downloaded dependencies) + // in the source directory, which does not work with our CI setup, where we mount + // source directories as read-only on Linux. + // Therefore, as a part of the build in CI, we first copy the whole source directory + // to the build directory, and perform the build from there. + let src_dir = if CiEnv::is_ci() { + let src_dir = builder.gcc_out(target).join("src"); + if src_dir.exists() { + builder.remove_dir(&src_dir); } + builder.create_dir(&src_dir); + builder.cp_link_r(root, &src_dir); + src_dir + } else { + root.clone() + }; - // GCC creates files (e.g. symlinks to the downloaded dependencies) - // in the source directory, which does not work with our CI setup, where we mount - // source directories as read-only on Linux. - // Therefore, as a part of the build in CI, we first copy the whole source directory - // to the build directory, and perform the build from there. - let src_dir = if CiEnv::is_ci() { - let src_dir = builder.gcc_out(target).join("src"); - if src_dir.exists() { - builder.remove_dir(&src_dir); - } - builder.create_dir(&src_dir); - builder.cp_link_r(&root, &src_dir); - src_dir - } else { - root - }; + command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder); + let mut configure_cmd = command(src_dir.join("configure")); + configure_cmd + .current_dir(out_dir) + // On CI, we compile GCC with Clang. + // The -Wno-everything flag is needed to make GCC compile with Clang 19. + // `-g -O2` are the default flags that are otherwise used by Make. + // FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml. + .env("CXXFLAGS", "-Wno-everything -g -O2") + .env("CFLAGS", "-Wno-everything -g -O2") + .arg("--enable-host-shared") + .arg("--enable-languages=jit") + .arg("--enable-checking=release") + .arg("--disable-bootstrap") + .arg("--disable-multilib") + .arg(format!("--prefix={}", install_dir.display())); + let cc = builder.build.cc(target).display().to_string(); + let cc = builder + .build + .config + .ccache + .as_ref() + .map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}")); + configure_cmd.env("CC", cc); - command(src_dir.join("contrib/download_prerequisites")).current_dir(&src_dir).run(builder); - let mut configure_cmd = command(src_dir.join("configure")); - configure_cmd - .current_dir(&out_dir) - // On CI, we compile GCC with Clang. - // The -Wno-everything flag is needed to make GCC compile with Clang 19. - // `-g -O2` are the default flags that are otherwise used by Make. - // FIXME(kobzol): change the flags once we have [gcc] configuration in config.toml. - .env("CXXFLAGS", "-Wno-everything -g -O2") - .env("CFLAGS", "-Wno-everything -g -O2") - .arg("--enable-host-shared") - .arg("--enable-languages=jit") - .arg("--enable-checking=release") - .arg("--disable-bootstrap") - .arg("--disable-multilib") - .arg(format!("--prefix={}", install_dir.display())); - let cc = builder.build.cc(target).display().to_string(); - let cc = builder + if let Ok(ref cxx) = builder.build.cxx(target) { + let cxx = cxx.display().to_string(); + let cxx = builder .build .config .ccache .as_ref() - .map_or_else(|| cc.clone(), |ccache| format!("{ccache} {cc}")); - configure_cmd.env("CC", cc); - - if let Ok(ref cxx) = builder.build.cxx(target) { - let cxx = cxx.display().to_string(); - let cxx = builder - .build - .config - .ccache - .as_ref() - .map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}")); - configure_cmd.env("CXX", cxx); - } - configure_cmd.run(builder); - - command("make") - .current_dir(&out_dir) - .arg("--silent") - .arg(format!("-j{}", builder.jobs())) - .run_capture_stdout(builder); - command("make") - .current_dir(&out_dir) - .arg("--silent") - .arg("install") - .run_capture_stdout(builder); - - let lib_alias = install_dir.join("lib/libgccjit.so.0"); - if !lib_alias.exists() { - t!(builder.symlink_file(&libgccjit_path, lib_alias)); - } - - t!(stamp.write()); - - GccOutput { libgccjit: libgccjit_path } + .map_or_else(|| cxx.clone(), |ccache| format!("{ccache} {cxx}")); + configure_cmd.env("CXX", cxx); } + configure_cmd.run(builder); + + command("make") + .current_dir(out_dir) + .arg("--silent") + .arg(format!("-j{}", builder.jobs())) + .run_capture_stdout(builder); + command("make").current_dir(out_dir).arg("--silent").arg("install").run_capture_stdout(builder); } /// Configures a Cargo invocation so that it can build the GCC codegen backend. @@ -213,3 +249,38 @@ pub fn add_cg_gcc_cargo_flags(cargo: &mut Cargo, gcc: &GccOutput) { // Add the path to libgccjit.so to the linker search paths. cargo.rustflag(&format!("-L{}", gcc.libgccjit.parent().unwrap().to_str().unwrap())); } + +/// The absolute path to the downloaded GCC artifacts. +#[cfg(not(test))] +fn ci_gcc_root(config: &crate::Config) -> PathBuf { + config.out.join(config.build).join("ci-gcc") +} + +/// This retrieves the GCC sha we *want* to use, according to git history. +#[cfg(not(test))] +fn detect_gcc_sha(config: &crate::Config, is_git: bool) -> String { + use build_helper::git::get_closest_merge_commit; + + let gcc_sha = if is_git { + get_closest_merge_commit( + Some(&config.src), + &config.git_config(), + &[config.src.join("src/gcc"), config.src.join("src/bootstrap/download-ci-gcc-stamp")], + ) + .unwrap() + } else if let Some(info) = crate::utils::channel::read_commit_info_file(&config.src) { + info.sha.trim().to_owned() + } else { + "".to_owned() + }; + + if gcc_sha.is_empty() { + eprintln!("error: could not find commit hash for downloading GCC"); + eprintln!("HELP: maybe your repository history is too shallow?"); + eprintln!("HELP: consider disabling `download-ci-gcc`"); + eprintln!("HELP: or fetch enough history to include one upstream commit"); + panic!(); + } + + gcc_sha +} diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs index ac24da9f86b25..e73659773ab2b 100644 --- a/src/bootstrap/src/core/config/config.rs +++ b/src/bootstrap/src/core/config/config.rs @@ -171,6 +171,17 @@ impl LldMode { } } +/// Determines how will GCC be provided. +#[derive(Default, Clone)] +pub enum GccCiMode { + /// Build GCC from the local `src/gcc` submodule. + #[default] + BuildLocally, + /// Try to download GCC from CI. + /// If it is not available on CI, it will be built locally instead. + DownloadFromCi, +} + /// Global configuration for the entire build and/or bootstrap. /// /// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. @@ -283,6 +294,9 @@ pub struct Config { pub llvm_ldflags: Option, pub llvm_use_libcxx: bool, + // gcc codegen options + pub gcc_ci_mode: GccCiMode, + // rust codegen options pub rust_optimize: RustOptimize, pub rust_codegen_units: Option, @@ -676,6 +690,7 @@ pub(crate) struct TomlConfig { build: Option, install: Option, llvm: Option, + gcc: Option, rust: Option, target: Option>, dist: Option, @@ -710,7 +725,7 @@ trait Merge { impl Merge for TomlConfig { fn merge( &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile, change_id }: Self, + TomlConfig { build, install, llvm, gcc, rust, dist, target, profile, change_id }: Self, replace: ReplaceOpt, ) { fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { @@ -729,6 +744,7 @@ impl Merge for TomlConfig { do_merge(&mut self.build, build, replace); do_merge(&mut self.install, install, replace); do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.gcc, gcc, replace); do_merge(&mut self.rust, rust, replace); do_merge(&mut self.dist, dist, replace); @@ -995,6 +1011,13 @@ define_config! { } } +define_config! { + /// TOML representation of how the GCC build is configured. + struct Gcc { + download_ci_gcc: Option = "download-ci-gcc", + } +} + define_config! { struct Dist { sign_folder: Option = "sign-folder", @@ -2136,6 +2159,16 @@ impl Config { config.llvm_from_ci = config.parse_download_ci_llvm(None, false); } + if let Some(gcc) = toml.gcc { + config.gcc_ci_mode = match gcc.download_ci_gcc { + Some(value) => match value { + true => GccCiMode::DownloadFromCi, + false => GccCiMode::BuildLocally, + }, + None => GccCiMode::default(), + }; + } + if let Some(t) = toml.target { for (triple, cfg) in t { let mut target = Target::from_triple(&triple); diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index c477bdb829a91..95feb41ffd0cc 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -826,6 +826,34 @@ download-rustc = false let llvm_root = self.ci_llvm_root(); self.unpack(&tarball, &llvm_root, "rust-dev"); } + + pub fn download_ci_gcc(&self, gcc_sha: &str, root_dir: &Path) { + let cache_prefix = format!("gcc-{gcc_sha}"); + let cache_dst = + self.bootstrap_cache_path.as_ref().cloned().unwrap_or_else(|| self.out.join("cache")); + + let gcc_cache = cache_dst.join(cache_prefix); + if !gcc_cache.exists() { + t!(fs::create_dir_all(&gcc_cache)); + } + let base = &self.stage0_metadata.config.artifacts_server; + let filename = format!("gcc-nightly-{}.tar.xz", self.build.triple); + let tarball = gcc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "ERROR: failed to download gcc from ci + + HELP: There could be two reasons behind this: + 1) The host triple is not supported for `download-ci-gcc`. + 2) Old builds get deleted after a certain time. + HELP: In either case, disable `download-ci-gcc` in your config.toml: + + [gcc] + download-ci-gcc = false + "; + self.download_file(&format!("{base}/{gcc_sha}/{filename}"), &tarball, help_on_error); + } + self.unpack(&tarball, root_dir, "gcc"); + } } fn path_is_dylib(path: &Path) -> bool { diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 425ffdccad57f..ec27109c117ae 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -370,4 +370,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Info, summary: "The rust.description option has moved to build.description and rust.description is now deprecated.", }, + ChangeInfo { + change_id: 138051, + severity: ChangeSeverity::Info, + summary: "There is now a new `gcc` config section that can be used to download GCC from CI using `gcc.download-ci-gcc = true`", + }, ]; diff --git a/triagebot.toml b/triagebot.toml index e4231b2966baa..91d0edbe76289 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -958,6 +958,9 @@ If appropriate, please update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/src/util [mentions."src/bootstrap/src/core/build_steps/llvm.rs"] message = "This PR changes how LLVM is built. Consider updating src/bootstrap/download-ci-llvm-stamp." +[mentions."src/bootstrap/src/core/build_steps/gcc.rs"] +message = "This PR changes how GCC is built. Consider updating src/bootstrap/download-ci-gcc-stamp." + [mentions."tests/crashes"] message = "This PR changes a file inside `tests/crashes`. If a crash was fixed, please move into the corresponding `ui` subdir and add 'Fixes #' to the PR description to autoclose the issue upon merge."