From d0bcbf4979db2014c0668c0016c353cb71978751 Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 3 Jun 2023 13:09:01 -0500 Subject: [PATCH 01/11] Make `--dry-run` more useful when download-rustc is enabled Previously, it would always treat download-rustc as set to false, which made bootstrap issues with download-rustc hard to debug. --- src/bootstrap/config.rs | 4 ++-- src/bootstrap/download.rs | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 5f5f7ea25fb95..2320661bd030a 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1813,11 +1813,11 @@ impl Config { self.download_rustc_commit().is_some() } - pub(crate) fn download_rustc_commit(&self) -> Option<&'static str> { + pub(crate) fn download_rustc_commit(&self) -> Option<&str> { static DOWNLOAD_RUSTC: OnceCell> = OnceCell::new(); if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { // avoid trying to actually download the commit - return None; + return self.download_rustc_commit.as_deref(); } DOWNLOAD_RUSTC diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index cb40521dda763..fbb97a1ba6de0 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -402,6 +402,10 @@ impl Config { fn ci_component_contents(&self, stamp_file: &str) -> Vec { assert!(self.download_rustc()); + if self.dry_run() { + return vec![]; + } + let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc"); let stamp_file = ci_rustc_dir.join(stamp_file); let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); From 02bacb207cd07f139198a20b4ab87596f82bd19d Mon Sep 17 00:00:00 2001 From: jyn Date: Thu, 25 May 2023 21:35:55 -0500 Subject: [PATCH 02/11] check ci-rustc options --- src/bootstrap/config.rs | 42 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 2320661bd030a..a16f762b2fb03 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -39,6 +39,16 @@ macro_rules! check_ci_llvm { }; } +macro_rules! check_ci_rustc { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-rustc.", + stringify!($name) + ); + }; +} + #[derive(Clone, Default)] pub enum DryRun { /// This isn't a dry run. @@ -1328,6 +1338,25 @@ impl Config { let mut omit_git_hash = None; if let Some(rust) = toml.rust { + set(&mut config.channel, rust.channel); + + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); + // This list is incomplete, please help by expanding it! + if config.download_rustc_commit.is_some() { + // We need the channel used by the downloaded compiler to match the one we set for rustdoc; + // otherwise rustdoc-ui tests break. + let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); + let ci_channel = ci_channel.trim_end(); + if config.channel != ci_channel + && !(config.channel == "dev" && ci_channel == "nightly") + { + panic!( + "setting rust.channel={} is incompatible with download-rustc", + config.channel + ); + } + } + debug = rust.debug; debug_assertions = rust.debug_assertions; debug_assertions_std = rust.debug_assertions_std; @@ -1339,6 +1368,7 @@ impl Config { debuginfo_level_std = rust.debuginfo_level_std; debuginfo_level_tools = rust.debuginfo_level_tools; debuginfo_level_tests = rust.debuginfo_level_tests; + config.rust_split_debuginfo = rust .split_debuginfo .as_deref() @@ -1354,7 +1384,6 @@ impl Config { set(&mut config.jemalloc, rust.jemalloc); set(&mut config.test_compare_mode, rust.test_compare_mode); set(&mut config.backtrace, rust.backtrace); - set(&mut config.channel, rust.channel); config.description = rust.description; set(&mut config.rust_dist_src, rust.dist_src); set(&mut config.verbose_tests, rust.verbose_tests); @@ -1395,8 +1424,6 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); - config.rust_lto = rust .lto .as_deref() @@ -1491,6 +1518,10 @@ impl Config { check_ci_llvm!(llvm.plugins); } + if config.download_rustc_commit.is_some() { + check_ci_rustc!(llvm.assertions); + } + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. if config.llvm_thin_lto && llvm.link_shared.is_none() { // If we're building with ThinLTO on, by default we want to link @@ -1508,6 +1539,11 @@ impl Config { let mut target = Target::from_triple(&triple); if let Some(ref s) = cfg.llvm_config { + if config.download_rustc_commit.is_some() && triple == &*config.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } target.llvm_config = Some(config.src.join(s)); } target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; From 93733f989755c960d21f54428567d5fe66f53a51 Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 3 Jun 2023 14:01:45 -0500 Subject: [PATCH 03/11] fix `x test --stage 0 core` --- src/bootstrap/test.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs index eed7a584b6035..9a50f48519048 100644 --- a/src/bootstrap/test.rs +++ b/src/bootstrap/test.rs @@ -2348,7 +2348,7 @@ impl Step for Crate { // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. // Override it. - if builder.download_rustc() { + if builder.download_rustc() && compiler.stage > 0 { let sysroot = builder .out .join(compiler.host.triple) From 9cc176597f0795572f1c9c0fd39ef4f3fe49f095 Mon Sep 17 00:00:00 2001 From: jyn Date: Sun, 4 Jun 2023 23:22:17 -0500 Subject: [PATCH 04/11] don't try to cross-compile sanitizer tests this was a pre-existing latent bug, we just didn't have any CI builders exercising it. fixes the following errors: ``` ---- [ui] tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs#opt1 stdout ---- error in revision `opt1`: test compilation failed although it shouldn't! status: exit status: 1 command: "/checkout/obj/build/x86_64-unknown-linux-gnu/stage2/bin/rustc" "/checkout/tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs" "-Zthreads=1" "-Zsimulate-remapped-rust-src-base=/rustc/FAKE_PREFIX" "-Ztranslate-remapped-path-to-local-path=no" "--sysroot" "/checkout/obj/build/x86_64-unknown-linux-gnu/stage2" "--target=i686-unknown-linux-gnu" "--cfg" "opt1" "--error-format" "json" "--json" "future-incompat" "-Ccodegen-units=1" "-Zui-testing" "-Zdeduplicate-diagnostics=no" "-Cstrip=debuginfo" "--remap-path-prefix=/checkout/tests/ui=fake-test-src-base" "-o" "/checkout/obj/build/x86_64-unknown-linux-gnu/test/ui/sanitize/new-llvm-pass-manager-thin-lto.opt1/a" "-A" "unused" "-Crpath" "-Cdebuginfo=0" "-Lnative=/checkout/obj/build/i686-unknown-linux-gnu/native/rust-test-helpers" "-Clinker=x86_64-linux-gnu-gcc" "-L" "/checkout/obj/build/x86_64-unknown-linux-gnu/test/ui/sanitize/new-llvm-pass-manager-thin-lto.opt1/auxiliary" "-Zsanitizer=address" "-Clto=thin" "-Copt-level=1" --- stderr ------------------------------- error: linking with `x86_64-linux-gnu-gcc` failed: exit status: 1 = note: x86_64-linux-gnu-gcc: error: /checkout/obj/build/x86_64-unknown-linux-gnu/stage2/lib/rustlib/i686-unknown-linux-gnu/lib/librustc-nightly_rt.asan.a: No such file or directory failures: [ui] tests/ui/sanitize/badfree.rs [ui] tests/ui/sanitize/address.rs [ui] tests/ui/sanitize/use-after-scope.rs [ui] tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs#opt0 [ui] tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs#opt1 ``` --- tests/ui/sanitize/address.rs | 3 ++- tests/ui/sanitize/badfree.rs | 1 + tests/ui/sanitize/issue-72154-lifetime-markers.rs | 1 + tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs | 1 + tests/ui/sanitize/use-after-scope.rs | 1 + 5 files changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/ui/sanitize/address.rs b/tests/ui/sanitize/address.rs index 5b2cea87560fd..1faab1fd2fce1 100644 --- a/tests/ui/sanitize/address.rs +++ b/tests/ui/sanitize/address.rs @@ -1,11 +1,12 @@ // needs-sanitizer-support // needs-sanitizer-address +// ignore-cross-compile // // compile-flags: -Z sanitizer=address -O -g // // run-fail // error-pattern: AddressSanitizer: stack-buffer-overflow -// error-pattern: 'xs' (line 13) <== Memory access at offset +// error-pattern: 'xs' (line 14) <== Memory access at offset use std::hint::black_box; diff --git a/tests/ui/sanitize/badfree.rs b/tests/ui/sanitize/badfree.rs index 095a6f4697b1c..c8d1ce7dff25d 100644 --- a/tests/ui/sanitize/badfree.rs +++ b/tests/ui/sanitize/badfree.rs @@ -1,5 +1,6 @@ // needs-sanitizer-support // needs-sanitizer-address +// ignore-cross-compile // // compile-flags: -Z sanitizer=address -O // diff --git a/tests/ui/sanitize/issue-72154-lifetime-markers.rs b/tests/ui/sanitize/issue-72154-lifetime-markers.rs index b2e182238ce28..3d9c51daa6505 100644 --- a/tests/ui/sanitize/issue-72154-lifetime-markers.rs +++ b/tests/ui/sanitize/issue-72154-lifetime-markers.rs @@ -5,6 +5,7 @@ // // needs-sanitizer-support // needs-sanitizer-address +// ignore-cross-compile // // compile-flags: -Copt-level=0 -Zsanitizer=address // run-pass diff --git a/tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs b/tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs index 33e18e35522ec..052a40598a856 100644 --- a/tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs +++ b/tests/ui/sanitize/new-llvm-pass-manager-thin-lto.rs @@ -4,6 +4,7 @@ // // needs-sanitizer-support // needs-sanitizer-address +// ignore-cross-compile // // no-prefer-dynamic // revisions: opt0 opt1 diff --git a/tests/ui/sanitize/use-after-scope.rs b/tests/ui/sanitize/use-after-scope.rs index 30be2ae6f0906..de63eea194ba9 100644 --- a/tests/ui/sanitize/use-after-scope.rs +++ b/tests/ui/sanitize/use-after-scope.rs @@ -1,5 +1,6 @@ // needs-sanitizer-support // needs-sanitizer-address +// ignore-cross-compile // // compile-flags: -Zsanitizer=address // run-fail From 7f27750c18bae8cdb923c0baa686f5e948b5f21e Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 1 Jul 2023 12:03:16 -0500 Subject: [PATCH 05/11] allow mixing `llvm.assertions` and `download-rustc` by using `rustc-builds-alt` if download-rustc is set this also changes the download code to use a separate build/cache/ directory and .rustc-stamp stamp file depending on whether assertions are enabled. --- src/bootstrap/config.rs | 14 -------------- src/bootstrap/download.rs | 30 ++++++++++++++++++++++-------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index a16f762b2fb03..af29fe0baae24 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -39,16 +39,6 @@ macro_rules! check_ci_llvm { }; } -macro_rules! check_ci_rustc { - ($name:expr) => { - assert!( - $name.is_none(), - "setting {} is incompatible with download-ci-rustc.", - stringify!($name) - ); - }; -} - #[derive(Clone, Default)] pub enum DryRun { /// This isn't a dry run. @@ -1518,10 +1508,6 @@ impl Config { check_ci_llvm!(llvm.plugins); } - if config.download_rustc_commit.is_some() { - check_ci_rustc!(llvm.assertions); - } - // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. if config.llvm_thin_lto && llvm.link_shared.is_none() { // If we're building with ThinLTO on, by default we want to link diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index fbb97a1ba6de0..90578065ba2d2 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -423,7 +423,7 @@ impl Config { self.download_toolchain( &version, "ci-rustc", - commit, + &format!("{commit}-{}", self.llvm_assertions), &extra_components, Self::download_ci_component, ); @@ -499,8 +499,15 @@ impl Config { /// Download a single component of a CI-built toolchain (not necessarily a published nightly). // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident - fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { - Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") + fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { + Self::download_component( + self, + DownloadSource::CI, + filename, + prefix, + commit_with_assertions, + "ci-rustc", + ) } fn download_component( @@ -520,11 +527,18 @@ impl Config { let bin_root = self.out.join(self.build.triple).join(destination); let tarball = cache_dir.join(&filename); let (base_url, url, should_verify) = match mode { - DownloadSource::CI => ( - self.stage0_metadata.config.artifacts_server.clone(), - format!("{key}/{filename}"), - false, - ), + DownloadSource::CI => { + let dist_server = if self.llvm_assertions { + self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() + } else { + self.stage0_metadata.config.artifacts_server.clone() + }; + let url = format!( + "{}/{filename}", + key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() + ); + (dist_server, url, false) + } DownloadSource::Dist => { let dist_server = env::var("RUSTUP_DIST_SERVER") .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); From 96c2e2b6b5a9afbe19149c9efc9b8ca3c076f2d2 Mon Sep 17 00:00:00 2001 From: jyn Date: Wed, 5 Jul 2023 11:38:55 -0500 Subject: [PATCH 06/11] Move `ci_rustc_dir` to Config and use it consistently --- src/bootstrap/compile.rs | 4 ++-- src/bootstrap/config.rs | 7 +++++++ src/bootstrap/download.rs | 2 +- src/bootstrap/lib.rs | 5 ----- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 14c3ef79a78f2..53197212b2e0f 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -645,8 +645,8 @@ fn cp_rustc_component_to_ci_sysroot( contents: Vec, ) { let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + let ci_rustc_dir = builder.config.ci_rustc_dir(); - let ci_rustc_dir = builder.out.join(&*builder.build.build.triple).join("ci-rustc"); for file in contents { let src = ci_rustc_dir.join(&file); let dst = sysroot.join(file); @@ -1381,7 +1381,7 @@ impl Step for Sysroot { // FIXME: this is wrong when compiler.host != build, but we don't support that today OsStr::new(std::env::consts::DLL_EXTENSION), ]; - let ci_rustc_dir = builder.ci_rustc_dir(builder.config.build); + let ci_rustc_dir = builder.config.ci_rustc_dir(); builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { return true; diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index af29fe0baae24..61827b2ca8b0c 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1800,6 +1800,13 @@ impl Config { self.out.join(&*self.build.triple).join("ci-llvm") } + /// Directory where the extracted `rustc-dev` component is stored. + pub(crate) fn ci_rustc_dir(&self) -> PathBuf { + // assert!(self.download_rustc_commit.is_some()); + assert!(self.download_rustc()); + self.out.join(self.build.triple).join("ci-rustc") + } + /// Determine whether llvm should be linked dynamically. /// /// If `false`, llvm should be linked statically. diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs index 90578065ba2d2..a081f6189d77a 100644 --- a/src/bootstrap/download.rs +++ b/src/bootstrap/download.rs @@ -406,7 +406,7 @@ impl Config { return vec![]; } - let ci_rustc_dir = self.out.join(&*self.build.triple).join("ci-rustc"); + let ci_rustc_dir = self.ci_rustc_dir(); let stamp_file = ci_rustc_dir.join(stamp_file); let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); t!(BufReader::new(contents_file).lines().collect()) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 0a7aff62257a5..0b509132043cc 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -822,11 +822,6 @@ impl Build { self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) } - /// Directory where the extracted `rustc-dev` component is stored. - fn ci_rustc_dir(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("ci-rustc") - } - /// Root output directory for LLVM compiled for `target` /// /// Note that if LLVM is configured externally then the directory returned From b80c2f8da8365fe8735159e68fa37b524d6e6395 Mon Sep 17 00:00:00 2001 From: jyn Date: Sun, 4 Jun 2023 21:15:38 -0500 Subject: [PATCH 07/11] Test `x86_64-gnu` in PR CI, not x86_64-gnu-llvm-14 This is a pre-requisite for https://github.com/rust-lang/rust/pull/112143, which wants to start using download-rustc in PRs. download-rustc doesn't allow providing an external LLVM. --- .github/workflows/ci.yml | 2 +- .../docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile | 10 +--------- src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile | 11 ++++++++++- .../{x86_64-gnu-llvm-14 => x86_64-gnu}/script.sh | 0 src/ci/github-actions/ci.yml | 2 +- 5 files changed, 13 insertions(+), 12 deletions(-) rename src/ci/docker/host-x86_64/{x86_64-gnu-llvm-14 => x86_64-gnu}/script.sh (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3766adf794372..6b5d3dabef743 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,7 @@ jobs: - name: mingw-check-tidy os: ubuntu-20.04-16core-64gb env: {} - - name: x86_64-gnu-llvm-14 + - name: x86_64-gnu os: ubuntu-20.04-16core-64gb env: {} - name: x86_64-gnu-tools diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile index 93d18bcf1b18e..c7ac93d7378e0 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/Dockerfile @@ -4,7 +4,6 @@ ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ - gcc-multilib \ make \ ninja-build \ file \ @@ -26,11 +25,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ mingw-w64 \ && rm -rf /var/lib/apt/lists/* -# Install powershell (universal package) so we can test x.ps1 on Linux -RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ - dpkg -i powershell.deb && \ - rm -f powershell.deb - COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -49,6 +43,4 @@ ENV RUST_CONFIGURE_ARGS \ --enable-llvm-link-shared \ --set rust.thin-lto-import-instr-limit=10 -COPY host-x86_64/x86_64-gnu-llvm-14/script.sh /tmp/ - -ENV SCRIPT /tmp/script.sh +ENV SCRIPT ../x.py --stage 2 test diff --git a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile index fbec368c9ee55..8478d16b7f116 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu/Dockerfile @@ -3,6 +3,7 @@ FROM ubuntu:20.04 ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ g++ \ + gcc-multilib \ make \ ninja-build \ file \ @@ -19,6 +20,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ mingw-w64 \ && rm -rf /var/lib/apt/lists/* +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh @@ -27,4 +33,7 @@ ENV RUST_CONFIGURE_ARGS \ --enable-sanitizers \ --enable-profiler \ --enable-compiler-docs -ENV SCRIPT python3 ../x.py --stage 2 test + +COPY host-x86_64/x86_64-gnu/script.sh /tmp/ + +ENV SCRIPT /tmp/script.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu/script.sh similarity index 100% rename from src/ci/docker/host-x86_64/x86_64-gnu-llvm-14/script.sh rename to src/ci/docker/host-x86_64/x86_64-gnu/script.sh diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 24fc72e660a65..f87e2f3745841 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -323,7 +323,7 @@ jobs: - name: mingw-check-tidy <<: *job-linux-16c - - name: x86_64-gnu-llvm-14 + - name: x86_64-gnu <<: *job-linux-16c - name: x86_64-gnu-tools From a6dcffa220035bb2d368db657891cea55898f3f6 Mon Sep 17 00:00:00 2001 From: jyn Date: Wed, 31 May 2023 10:51:22 -0500 Subject: [PATCH 08/11] Use `download-rustc = "if-unchanged"` in PR CI and `x86_64-gnu` This has two main benefits: 1. It tests that download-rustc doesn't regress. This doesn't reduce our test coverage, since we still never use `download-rustc` in a full bors merge, but it should make it a lot less likely that this breaks by accident. 2. It greatly speeds up CI when compiler/ and library/ haven't been modified. Once the changes in https://github.com/rust-lang/compiler-team/issues/619 land, this will also be faster for changes to library/, and only changes to compiler/ will have to rebuild. --- .github/workflows/ci.yml | 4 +++- src/ci/docker/run.sh | 1 + src/ci/github-actions/ci.yml | 3 +++ src/ci/run.sh | 4 ++++ 4 files changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6b5d3dabef743..cd255e237adb1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,7 @@ jobs: pr: name: "PR - ${{ matrix.name }}" env: + DOWNLOAD_RUSTC: 1 PR_CI_JOB: 1 CI_JOB_NAME: "${{ matrix.name }}" CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse @@ -268,8 +269,9 @@ jobs: os: ubuntu-20.04-8core-32gb env: {} - name: x86_64-gnu + env: + DOWNLOAD_RUSTC: 1 os: ubuntu-20.04-4core-16gb - env: {} - name: x86_64-gnu-stable env: IMAGE: x86_64-gnu diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index 4b218d5772769..f67e4fe57508e 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -254,6 +254,7 @@ docker \ --env DEPLOY \ --env DEPLOY_ALT \ --env CI \ + --env DOWNLOAD_RUSTC \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index f87e2f3745841..362a9ae89ed67 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -310,6 +310,7 @@ jobs: <<: *base-ci-job name: PR - ${{ matrix.name }} env: + DOWNLOAD_RUSTC: 1 <<: [*shared-ci-variables, *public-variables] PR_CI_JOB: 1 if: github.event_name == 'pull_request' @@ -433,6 +434,8 @@ jobs: <<: *job-linux-8c - name: x86_64-gnu + env: + DOWNLOAD_RUSTC: 1 <<: *job-linux-4c # This job ensures commits landing on nightly still pass the full diff --git a/src/ci/run.sh b/src/ci/run.sh index 48fb40d6a6dd8..803e13bd9817e 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -121,6 +121,10 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions" fi + if [ "$DOWNLOAD_RUSTC" = 1 ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged" + fi + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" # We enable this for non-dist builders, since those aren't trying to produce From 20f9b0c2abff3e54e5d64119866a4df0ff6ee676 Mon Sep 17 00:00:00 2001 From: jyn Date: Mon, 19 Jun 2023 12:02:29 -0500 Subject: [PATCH 09/11] download-rustc: Don't consider the current commit for download if running in CI Avoids the following error: ``` curl: (22) The requested URL returned error: 404 error: failed to download pre-built rustc from CI note: old builds get deleted after a certain time help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: [rust] download-rustc = false ``` Note that this strategy is more complicated than the one for download-ci-llvm, which doesn't have to deal with rollup commits. Eventually we should probably adopt this for LLVM too, to avoid bugs if someone forgets to mark an LLVM bump as rollup=never. I tested this with `git -c user.email=bors@rust-lang.org -c user.name=bors commit --allow-empty -m 'test bors commit' && GITHUB_ACTIONS=true x check`. --- src/bootstrap/config.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 61827b2ca8b0c..6908ad100b232 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -23,6 +23,7 @@ use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags, Warnings}; use crate::util::{exe, output, t}; +use build_helper::ci::CiEnv; use build_helper::detail_exit_macro; use once_cell::sync::OnceCell; use semver::Version; @@ -1985,11 +1986,15 @@ impl Config { // Look for a version to compare to based on the current commit. // Only commits merged by bors will have CI artifacts. + + // If we are running in CI, the current commit will never have artifacts already built. + let tip_commit = if CiEnv::is_ci() { "HEAD^" } else { "HEAD" }; + // FIXME(#113250): This may not be the best way to find the right commit; it certainly doesn't match what we do for LLVM. let merge_base = output( self.git() .arg("rev-list") .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), + .args(&["-n1", "--first-parent", tip_commit]), ); let commit = merge_base.trim_end(); if commit.is_empty() { From ce127e142961a4b59e1b4a8997d1e0cb536744f3 Mon Sep 17 00:00:00 2001 From: jyn Date: Wed, 5 Jul 2023 14:12:52 -0500 Subject: [PATCH 10/11] give a hard error if setting DOWNLOAD_RUSTC in a dist builder these should compile from source; i have reasonable confidence in download-rustc, but not so much that i want to allow shipping it in dist artifacts. --- src/ci/run.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/ci/run.sh b/src/ci/run.sh index 803e13bd9817e..db75ab80a9de8 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -103,6 +103,11 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" fi + + if [ "$DOWNLOAD_RUSTC" = 1 ]; then + echo "error: DOWNLOAD_RUSTC should not be set in dist builders!" >&2 + exit 1 + fi else # We almost always want debug assertions enabled, but sometimes this takes too # long for too little benefit, so we just turn them off. From 2c3ad2df66bc41fd097e89b85a48acc22300789d Mon Sep 17 00:00:00 2001 From: jyn Date: Sat, 8 Jul 2023 14:50:57 -0500 Subject: [PATCH 11/11] Don't depend on crate names in tests/ui-fulldeps/missing-rustc-driver-error.rs Not only are they a pain to update, but they differ depending on whether `parallel-rustc` is enabled or not --- tests/ui-fulldeps/missing-rustc-driver-error.rs | 2 ++ tests/ui-fulldeps/missing-rustc-driver-error.stderr | 12 +----------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/tests/ui-fulldeps/missing-rustc-driver-error.rs b/tests/ui-fulldeps/missing-rustc-driver-error.rs index 654cd6f6dc9aa..75714208b7e7d 100644 --- a/tests/ui-fulldeps/missing-rustc-driver-error.rs +++ b/tests/ui-fulldeps/missing-rustc-driver-error.rs @@ -3,6 +3,8 @@ // compile-flags: --emit link // The exactly list of required crates depends on the target. as such only test Unix targets. // only-unix +// normalize-stderr-test ".*crate .* required.*\n\n" -> "" +// normalize-stderr-test: "aborting due to [0-9]+" -> "aborting due to NUMBER" #![feature(rustc_private)] diff --git a/tests/ui-fulldeps/missing-rustc-driver-error.stderr b/tests/ui-fulldeps/missing-rustc-driver-error.stderr index 939e888e5cb00..d7bf27d634913 100644 --- a/tests/ui-fulldeps/missing-rustc-driver-error.stderr +++ b/tests/ui-fulldeps/missing-rustc-driver-error.stderr @@ -2,15 +2,5 @@ error: crate `rustc_serialize` required to be available in rlib format, but was | = help: try adding `extern crate rustc_driver;` at the top level of this crate -error: crate `smallvec` required to be available in rlib format, but was not found in this form - -error: crate `thin_vec` required to be available in rlib format, but was not found in this form - -error: crate `indexmap` required to be available in rlib format, but was not found in this form - -error: crate `hashbrown` required to be available in rlib format, but was not found in this form - -error: crate `equivalent` required to be available in rlib format, but was not found in this form - -error: aborting due to 6 previous errors +error: aborting due to NUMBER previous errors