diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6aa9fbcb58d..ce0be2b2288 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,6 +99,22 @@ jobs: NEXTEST_DOUBLE_SPAWN: 0 run: cargo local-nt run --profile ci + - name: Test without rustup wrappers + env: + CARGO_NEXTEST: target/debug/cargo-nextest + RUSTUP_AVAILABLE: 1 + shell: bash + run: ./scripts/nextest-without-rustup.sh run --profile ci + + - name: Upload nextest binary + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: cargo-nextest-${{ matrix.os }}-${{ matrix.rust-version }} + path: | + target/debug/cargo-nextest + target/debug/cargo-nextest.exe + if-no-files-found: error + test-archive-target-runner: name: Test archives with a target runner # gcc-mingw-w64-x86-64-win32 requires Ubuntu 22.04 @@ -128,11 +144,83 @@ jobs: - name: Build cargo-nextest run: cargo build --package cargo-nextest - name: Archive test fixtures + env: + CARGO_NEXTEST: target/debug/cargo-nextest + RUSTUP_AVAILABLE: 1 run: | - cargo local-nt archive --manifest-path fixtures/nextest-tests/Cargo.toml \ + ./scripts/nextest-without-rustup.sh archive --manifest-path fixtures/nextest-tests/Cargo.toml \ --target x86_64-pc-windows-gnu --archive-file target/fixture-archive.tar.zst \ --package cdylib-example --package nextest-derive + + # This ensures that for target binaries, we always use the libdir in the archive, never the + # one installed by rustup. + - name: Remove x86_64-pc-windows-gnu target from rustup + run: rustup target remove x86_64-pc-windows-gnu + - name: Run test fixtures env: CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER: wine run: cargo local-nt run --archive-file target/fixture-archive.tar.zst + + # Completely uninstall rustup -- this is similar to running an archive on a machine without + # cargo. + - name: Uninstall rustup + run: rustup self uninstall -y + + - name: Run test fixtures without rustup wrappers + env: + CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER: wine + CARGO_NEXTEST: target/debug/cargo-nextest + run: ./scripts/nextest-without-rustup.sh run --archive-file target/fixture-archive.tar.zst + + # Upload the archive for use in the next job. + - name: Upload archive + uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3 + with: + name: fixture-archive + path: target/fixture-archive.tar.zst + if-no-files-found: error + + test-archive-runner-dest: + name: Test archives with a runner (destination) + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + runs-on: ${{ matrix.os }} + needs: + - test-archive-target-runner + - build + steps: + - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4 + - name: Download nextest binary + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + name: cargo-nextest-${{ matrix.os }}-stable + path: nextest-bin + - name: Download archive + uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7 + with: + name: fixture-archive + path: fixture-archive + - name: Run test fixtures (host) + if: ${{ matrix.os == 'ubuntu-latest' }} + env: + CARGO_NEXTEST: nextest-bin/cargo-nextest + run: | + chmod +x $CARGO_NEXTEST + ./scripts/nextest-without-rustup.sh run \ + --archive-file fixture-archive/fixture-archive.tar.zst \ + --workspace-remap $GITHUB_WORKSPACE/fixtures/nextest-tests \ + -E 'platform(host)' + - name: Run test fixtures (target) + if: ${{ matrix.os == 'windows-latest' }} + shell: bash + env: + CARGO_NEXTEST: nextest-bin/cargo-nextest.exe + run: | + ./scripts/nextest-without-rustup.sh run \ + --archive-file fixture-archive/fixture-archive.tar.zst \ + --workspace-remap $GITHUB_WORKSPACE/fixtures/nextest-tests \ + -E 'platform(target)' diff --git a/cargo-nextest/src/reuse_build.rs b/cargo-nextest/src/reuse_build.rs index 7fcb65975ab..6ce371fee91 100644 --- a/cargo-nextest/src/reuse_build.rs +++ b/cargo-nextest/src/reuse_build.rs @@ -213,6 +213,7 @@ pub(crate) fn make_path_mapper( info.workspace_remap(), orig_target_dir, info.target_dir_remap(), + info.libdir_mapper.clone(), ) .map_err(|err| { let arg_name = match err.kind() { diff --git a/fixtures/nextest-tests/tests/basic.rs b/fixtures/nextest-tests/tests/basic.rs index a46ff2b6e9c..5a2d814b66d 100644 --- a/fixtures/nextest-tests/tests/basic.rs +++ b/fixtures/nextest-tests/tests/basic.rs @@ -144,8 +144,11 @@ fn test_cargo_env_vars() { Ok("process-per-test"), "NEXTEST_EXECUTION_MODE set to process-per-test" ); + // https://doc.rust-lang.org/cargo/reference/environment-variables.html#environment-variables-cargo-sets-for-crates - assert_env!("CARGO"); + + // Note: we do not test CARGO here because nextest does not set it -- it's set by Cargo when + // invoked as `cargo nextest`. assert_env!( "CARGO_MANIFEST_DIR", "__NEXTEST_ORIGINAL_CARGO_MANIFEST_DIR" diff --git a/integration-tests/tests/integration/snapshots/integration__archive_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_includes.snap index 031c8ec4727..f37403d2b76 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_includes.snap @@ -2,7 +2,7 @@ source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, and 7 extra paths to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 7 extra paths, and 1 standard library to Warning ignoring extra path `/excluded-dir` because it does not exist Warning ignoring extra path `/depth-0-dir` specified with depth 0 since it is a directory Warning ignoring extra path `/file_that_does_not_exist.txt` because it does not exist diff --git a/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap b/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap index 1207aced2c2..1a0a428de33 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_includes_without_uds.snap @@ -2,7 +2,7 @@ source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, and 7 extra paths to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 7 extra paths, and 1 standard library to Warning ignoring extra path `/excluded-dir` because it does not exist Warning ignoring extra path `/depth-0-dir` specified with depth 0 since it is a directory Warning ignoring extra path `/file_that_does_not_exist.txt` because it does not exist diff --git a/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap index 216ad0a2bc3..894731c41b9 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_missing_includes.snap @@ -2,7 +2,7 @@ source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, and 1 extra path to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, 1 extra path, and 1 standard library to error: error creating archive `` Caused by: diff --git a/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap b/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap index 5b7e571f30b..6d262313979 100644 --- a/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap +++ b/integration-tests/tests/integration/snapshots/integration__archive_no_includes.snap @@ -2,7 +2,7 @@ source: integration-tests/tests/integration/main.rs expression: output.stderr_as_str() --- - Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, and 2 linked paths to + Archiving 17 binaries (including 2 non-test binaries), 2 build script output directories, 2 linked paths, and 1 standard library to Warning linked path `/debug/build//does-not-exist` not found, requested by: cdylib-link v0.1.0 (this is a bug in this crate that should be fixed) Archived files to in diff --git a/nextest-metadata/src/test_list.rs b/nextest-metadata/src/test_list.rs index 84c67abe87f..48e41dee3a0 100644 --- a/nextest-metadata/src/test_list.rs +++ b/nextest-metadata/src/test_list.rs @@ -590,6 +590,10 @@ impl PlatformLibdirUnavailable { /// version of nextest. pub const OLD_SUMMARY: Self = Self::new_const("old-summary"); + /// The libdir is unavailable because a build was reused from an archive, and the libdir was not + /// present in the archive + pub const NOT_IN_ARCHIVE: Self = Self::new_const("not-in-archive"); + /// Converts a static string into Self. pub const fn new_const(reason: &'static str) -> Self { Self(Cow::Borrowed(reason)) diff --git a/nextest-runner/src/helpers.rs b/nextest-runner/src/helpers.rs index 0820d8005cc..fcccd0a8c1b 100644 --- a/nextest-runner/src/helpers.rs +++ b/nextest-runner/src/helpers.rs @@ -62,6 +62,14 @@ pub(crate) mod plural { "these crates" } } + + pub(crate) fn libraries_str(count: usize) -> &'static str { + if count == 1 { + "library" + } else { + "libraries" + } + } } /// Write out a test name. diff --git a/nextest-runner/src/list/rust_build_meta.rs b/nextest-runner/src/list/rust_build_meta.rs index 1423dfbc50e..b4d5be74b56 100644 --- a/nextest-runner/src/list/rust_build_meta.rs +++ b/nextest-runner/src/list/rust_build_meta.rs @@ -75,7 +75,7 @@ impl RustBuildMeta { build_script_out_dirs: self.build_script_out_dirs.clone(), linked_paths: self.linked_paths.clone(), state: PhantomData, - build_platforms: self.build_platforms.clone(), + build_platforms: self.build_platforms.map_libdir(path_mapper.libdir_mapper()), } } } @@ -103,6 +103,11 @@ impl RustBuildMeta { /// These paths are prepended to the dynamic library environment variable for the current /// platform (e.g. `LD_LIBRARY_PATH` on non-Apple Unix platforms). pub fn dylib_paths(&self) -> Vec { + // Add rust libdirs to the path if available, so we can run test binaries that depend on + // libstd. + // + // We could be smarter here and only add the host libdir for host binaries and the target + // libdir for target binaries, but it's simpler to just add both for now. let libdirs = self .build_platforms .host @@ -115,11 +120,12 @@ impl RustBuildMeta { .as_ref() .and_then(|target| target.libdir.as_path()), ) - .cloned() + .map(|libdir| libdir.to_path_buf()) .collect::>(); if libdirs.is_empty() { log::warn!("failed to detect the rustc libdir, may fail to list or run tests"); } + // Cargo puts linked paths before base output directories. self.linked_paths .keys() @@ -138,8 +144,6 @@ impl RustBuildMeta { // This is the order paths are added in by Cargo. [with_deps, abs_base] })) - // Add the rustc libdir paths to the search paths to run procudure macro binaries. See - // details in https://github.com/nextest-rs/nextest/issues/1493. .chain(libdirs) .unique() .collect() diff --git a/nextest-runner/src/list/test_list.rs b/nextest-runner/src/list/test_list.rs index 8bed5b48022..289b18dcfbd 100644 --- a/nextest-runner/src/list/test_list.rs +++ b/nextest-runner/src/list/test_list.rs @@ -234,6 +234,10 @@ impl<'g> TestList<'g> { let stream = futures::stream::iter(test_artifacts).map(|test_binary| { async { if filter.should_obtain_test_list_from_binary(&test_binary) { + log::debug!( + "executing test binary to obtain test list: {}", + test_binary.binary_id, + ); // Run the binary to obtain the test list. let (non_ignored, ignored) = test_binary.exec(&lctx, ctx.target_runner).await?; let (bin, info) = Self::process_output( @@ -245,6 +249,10 @@ impl<'g> TestList<'g> { Ok::<_, CreateTestListError>((bin, info)) } else { // Skipped means no tests, so test_count doesn't need to be modified. + log::debug!( + "skipping test binary because of filters: {}", + test_binary.binary_id, + ); Ok(Self::process_skipped(test_binary)) } } diff --git a/nextest-runner/src/platform.rs b/nextest-runner/src/platform.rs index 8d410f862de..e153558bba6 100644 --- a/nextest-runner/src/platform.rs +++ b/nextest-runner/src/platform.rs @@ -6,8 +6,9 @@ use crate::{ cargo_config::{CargoTargetArg, TargetTriple}, errors::{RustBuildMetaParseError, TargetTripleError, UnknownHostPlatform}, + reuse_build::{LibdirMapper, PlatformLibdirMapper}, }; -use camino::Utf8PathBuf; +use camino::{Utf8Path, Utf8PathBuf}; use nextest_metadata::{ BuildPlatformsSummary, HostPlatformSummary, PlatformLibdirSummary, PlatformLibdirUnavailable, TargetPlatformSummary, @@ -40,6 +41,17 @@ impl BuildPlatforms { }) } + /// Maps libdir paths. + pub fn map_libdir(&self, mapper: &LibdirMapper) -> Self { + Self { + host: self.host.map_libdir(&mapper.host), + target: self + .target + .as_ref() + .map(|target| target.map_libdir(&mapper.target)), + } + } + /// Returns the argument to pass into `cargo metadata --filter-platform `. pub fn to_cargo_target_arg(&self) -> Result { match &self.target { @@ -192,6 +204,13 @@ impl HostPlatform { libdir: PlatformLibdir::from_summary(summary.libdir), }) } + + fn map_libdir(&self, mapper: &PlatformLibdirMapper) -> Self { + Self { + platform: self.platform.clone(), + libdir: mapper.map(&self.libdir), + } + } } /// The target platform. @@ -227,6 +246,13 @@ impl TargetPlatform { libdir: PlatformLibdir::from_summary(summary.libdir), }) } + + fn map_libdir(&self, mapper: &PlatformLibdirMapper) -> Self { + Self { + triple: self.triple.clone(), + libdir: mapper.map(&self.libdir), + } + } } /// A platform libdir. @@ -292,7 +318,7 @@ impl PlatformLibdir { } /// Returns self as a path if available. - pub fn as_path(&self) -> Option<&Utf8PathBuf> { + pub fn as_path(&self) -> Option<&Utf8Path> { match self { Self::Available(path) => Some(path), Self::Unavailable(_) => None, diff --git a/nextest-runner/src/reuse_build/archive_reporter.rs b/nextest-runner/src/reuse_build/archive_reporter.rs index 5f7af62663e..80d4f999a47 100644 --- a/nextest-runner/src/reuse_build/archive_reporter.rs +++ b/nextest-runner/src/reuse_build/archive_reporter.rs @@ -46,23 +46,12 @@ impl ArchiveReporter { ) -> io::Result<()> { match event { ArchiveEvent::ArchiveStarted { - test_binary_count, - non_test_binary_count, - build_script_out_dir_count, - linked_path_count, - extra_path_count, + counts, output_file, } => { write!(writer, "{:>12} ", "Archiving".style(self.styles.success))?; - self.report_counts( - test_binary_count, - non_test_binary_count, - build_script_out_dir_count, - linked_path_count, - extra_path_count, - &mut writer, - )?; + self.report_counts(counts, &mut writer)?; writeln!( writer, @@ -72,6 +61,14 @@ impl ArchiveReporter { .style(self.styles.bold) )?; } + ArchiveEvent::StdlibPathError { error } => { + write!(writer, "{:>12} ", "Warning".style(self.styles.bold))?; + writeln!( + writer, + "could not find standard library for host (proc macro tests may not work): {}", + error + )?; + } ArchiveEvent::ExtraPathMissing { path, warn } => { if warn { write!(writer, "{:>12} ", "Warning".style(self.styles.warning))?; @@ -170,13 +167,16 @@ impl ArchiveReporter { write!(writer, "{:>12} ", "Extracting".style(self.styles.success))?; self.report_counts( - test_binary_count, - non_test_binary_count, - build_script_out_dir_count, - linked_path_count, - // TODO: we currently don't store a list of extra paths at manifest creation - // time, so we can't report this count here. - 0, + ArchiveCounts { + test_binary_count, + non_test_binary_count, + build_script_out_dir_count, + linked_path_count, + // TODO: we currently don't store a list of extra paths or standard libs at + // manifest creation time, so we can't report this count here. + extra_path_count: 0, + stdlib_count: 0, + }, &mut writer, )?; @@ -206,15 +206,16 @@ impl ArchiveReporter { Ok(()) } - fn report_counts( - &mut self, - test_binary_count: usize, - non_test_binary_count: usize, - build_script_out_dir_count: usize, - linked_path_count: usize, - extra_path_count: usize, - mut writer: impl Write, - ) -> io::Result<()> { + fn report_counts(&mut self, counts: ArchiveCounts, mut writer: impl Write) -> io::Result<()> { + let ArchiveCounts { + test_binary_count, + non_test_binary_count, + build_script_out_dir_count, + linked_path_count, + extra_path_count, + stdlib_count, + } = counts; + let total_binary_count = test_binary_count + non_test_binary_count; let non_test_text = if non_test_binary_count > 0 { format!( @@ -247,6 +248,13 @@ impl ArchiveReporter { plural::paths_str(extra_path_count), )); } + if stdlib_count > 0 { + more.push(format!( + "{} standard {}", + stdlib_count.style(self.styles.bold), + plural::libraries_str(stdlib_count), + )); + } write!( writer, @@ -297,25 +305,19 @@ impl Styles { pub enum ArchiveEvent<'a> { /// The archive process started. ArchiveStarted { - /// The number of test binaries to archive. - test_binary_count: usize, - - /// The number of non-test binaries to archive. - non_test_binary_count: usize, - - /// The number of build script output directories to archive. - build_script_out_dir_count: usize, - - /// The number of linked paths to archive. - linked_path_count: usize, - - /// The number of extra paths to archive. - extra_path_count: usize, + /// File counts. + counts: ArchiveCounts, /// The archive output file. output_file: &'a Utf8Path, }, + /// An error occurred while obtaining the path to a standard library. + StdlibPathError { + /// The error that occurred. + error: &'a str, + }, + /// A provided extra path did not exist. ExtraPathMissing { /// The path that was missing. @@ -406,3 +408,25 @@ pub enum ArchiveEvent<'a> { elapsed: Duration, }, } + +/// Counts of various types of files in an archive. +#[derive(Clone, Copy, Debug)] +pub struct ArchiveCounts { + /// The number of test binaries. + pub test_binary_count: usize, + + /// The number of non-test binaries. + pub non_test_binary_count: usize, + + /// The number of build script output directories. + pub build_script_out_dir_count: usize, + + /// The number of linked paths. + pub linked_path_count: usize, + + /// The number of extra paths. + pub extra_path_count: usize, + + /// The number of standard libraries. + pub stdlib_count: usize, +} diff --git a/nextest-runner/src/reuse_build/archiver.rs b/nextest-runner/src/reuse_build/archiver.rs index e6d8080c253..80e4a9af805 100644 --- a/nextest-runner/src/reuse_build/archiver.rs +++ b/nextest-runner/src/reuse_build/archiver.rs @@ -1,7 +1,7 @@ // Copyright (c) The nextest Contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ArchiveEvent, BINARIES_METADATA_FILE_NAME, CARGO_METADATA_FILE_NAME}; +use super::{ArchiveCounts, ArchiveEvent, BINARIES_METADATA_FILE_NAME, CARGO_METADATA_FILE_NAME}; use crate::{ config::{ get_num_cpus, ArchiveConfig, ArchiveIncludeOnMissing, FinalConfig, NextestProfile, @@ -11,7 +11,7 @@ use crate::{ helpers::{convert_rel_path_to_forward_slash, rel_path_join}, list::{BinaryList, OutputFormat, SerializableFormat}, redact::Redactor, - reuse_build::PathMapper, + reuse_build::{PathMapper, LIBDIRS_BASE_DIR}, }; use atomicwrites::{AtomicFile, OverwriteBehavior}; use camino::{Utf8Path, Utf8PathBuf}; @@ -74,37 +74,92 @@ where { let config = profile.archive_config(); - let file = AtomicFile::new(output_file, OverwriteBehavior::AllowOverwrite); - let test_binary_count = binary_list.rust_binaries.len(); - let non_test_binary_count = binary_list.rust_build_meta.non_test_binaries.len(); - let build_script_out_dir_count = binary_list.rust_build_meta.build_script_out_dirs.len(); - let linked_path_count = binary_list.rust_build_meta.linked_paths.len(); - let extra_path_count = config.include.len(); let start_time = Instant::now(); + let file = AtomicFile::new(output_file, OverwriteBehavior::AllowOverwrite); let file_count = file .write(|file| { - callback(ArchiveEvent::ArchiveStarted { - test_binary_count, - non_test_binary_count, - build_script_out_dir_count, - linked_path_count, - extra_path_count, - output_file, - }) - .map_err(ArchiveCreateError::ReporterIo)?; - // Write out the archive. + // Tests require the standard library in two cases: + // * proc-macro tests (host) + // * tests compiled with -C prefer-dynamic (target) + // + // We only care about libstd -- empirically, other libraries in the path aren't + // required. + let (host_stdlib, host_stdlib_err) = if let Some(libdir) = binary_list + .rust_build_meta + .build_platforms + .host + .libdir + .as_path() + { + split_result(find_std(libdir)) + } else { + (None, None) + }; + + let (target_stdlib, target_stdlib_err) = + if let Some(target) = &binary_list.rust_build_meta.build_platforms.target { + if let Some(libdir) = target.libdir.as_path() { + split_result(find_std(libdir)) + } else { + (None, None) + } + } else { + (None, None) + }; + + let stdlib_count = host_stdlib.is_some() as usize + target_stdlib.is_some() as usize; + let archiver = Archiver::new( config, binary_list, cargo_metadata, graph, path_mapper, + host_stdlib, + target_stdlib, format, zstd_level, file, redactor, )?; + + let test_binary_count = binary_list.rust_binaries.len(); + let non_test_binary_count = binary_list.rust_build_meta.non_test_binaries.len(); + let build_script_out_dir_count = + binary_list.rust_build_meta.build_script_out_dirs.len(); + let linked_path_count = binary_list.rust_build_meta.linked_paths.len(); + let extra_path_count = config.include.len(); + + let counts = ArchiveCounts { + test_binary_count, + non_test_binary_count, + build_script_out_dir_count, + linked_path_count, + extra_path_count, + stdlib_count, + }; + + callback(ArchiveEvent::ArchiveStarted { + counts, + output_file, + }) + .map_err(ArchiveCreateError::ReporterIo)?; + + // Was there an error finding the standard library? + if let Some(err) = host_stdlib_err { + callback(ArchiveEvent::StdlibPathError { + error: &err.to_string(), + }) + .map_err(ArchiveCreateError::ReporterIo)?; + } + if let Some(err) = target_stdlib_err { + callback(ArchiveEvent::StdlibPathError { + error: &err.to_string(), + }) + .map_err(ArchiveCreateError::ReporterIo)?; + } + let (_, file_count) = archiver.archive(&mut callback)?; Ok(file_count) }) @@ -130,6 +185,8 @@ struct Archiver<'a, W: Write> { cargo_metadata: &'a str, graph: &'a PackageGraph, path_mapper: &'a PathMapper, + host_stdlib: Option, + target_stdlib: Option, builder: tar::Builder>>, unix_timestamp: u64, added_files: HashSet, @@ -145,6 +202,8 @@ impl<'a, W: Write> Archiver<'a, W> { cargo_metadata: &'a str, graph: &'a PackageGraph, path_mapper: &'a PathMapper, + host_stdlib: Option, + target_stdlib: Option, format: ArchiveFormat, compression_level: i32, writer: W, @@ -175,6 +234,8 @@ impl<'a, W: Write> Archiver<'a, W> { cargo_metadata, graph, path_mapper, + host_stdlib, + target_stdlib, builder, unix_timestamp, added_files: HashSet::new(), @@ -405,6 +466,26 @@ impl<'a, W: Write> Archiver<'a, W> { } } + // Add the standard libraries to the archive if available. + if let Some(host_stdlib) = self.host_stdlib.clone() { + let rel_path = Utf8Path::new(LIBDIRS_BASE_DIR) + .join("host") + .join(host_stdlib.file_name().unwrap()); + let rel_path = convert_rel_path_to_forward_slash(&rel_path); + + self.append_file(ArchiveStep::ExtraPaths, &host_stdlib, &rel_path)?; + } + if let Some(target_stdlib) = self.target_stdlib.clone() { + // Use libdir/target/0 as the path to the target standard library, to support multiple + // targets in the future. + let rel_path = Utf8Path::new(LIBDIRS_BASE_DIR) + .join("target/0") + .join(target_stdlib.file_name().unwrap()); + let rel_path = convert_rel_path_to_forward_slash(&rel_path); + + self.append_file(ArchiveStep::ExtraPaths, &target_stdlib, &rel_path)?; + } + // Finish writing the archive. let encoder = self .builder @@ -562,6 +643,37 @@ impl<'a, W: Write> Archiver<'a, W> { } } +fn find_std(libdir: &Utf8Path) -> io::Result { + for path in libdir.read_dir_utf8()? { + let path = path?; + // As of Rust 1.78, std is of the form: + // + // libstd-.so (non-macOS Unix) + // libstd-.dylib (macOS) + // std-.dll (Windows) + let file_name = path.file_name(); + let is_unix = file_name.starts_with("libstd-") + && (file_name.ends_with(".so") || file_name.ends_with(".dylib")); + let is_windows = file_name.starts_with("std-") && file_name.ends_with(".dll"); + + if is_unix || is_windows { + return Ok(path.into_path()); + } + } + + Err(io::Error::new( + io::ErrorKind::Other, + "could not find the Rust standard library in the libdir", + )) +} + +fn split_result(result: Result) -> (Option, Option) { + match result { + Ok(v) => (Some(v), None), + Err(e) => (None, Some(e)), + } +} + /// The part of the archive process that is currently in progress. /// /// This is used for better warnings and errors. @@ -581,6 +693,9 @@ pub enum ArchiveStep { /// Extra paths are being archived. ExtraPaths, + + /// The standard library is being archived. + Stdlib, } impl fmt::Display for ArchiveStep { @@ -591,6 +706,7 @@ impl fmt::Display for ArchiveStep { Self::BuildScriptOutDirs => write!(f, "build script output directories"), Self::LinkedPaths => write!(f, "linked paths"), Self::ExtraPaths => write!(f, "extra paths"), + Self::Stdlib => write!(f, "standard library"), } } } diff --git a/nextest-runner/src/reuse_build/mod.rs b/nextest-runner/src/reuse_build/mod.rs index 1cf98c3deaf..49d72389a39 100644 --- a/nextest-runner/src/reuse_build/mod.rs +++ b/nextest-runner/src/reuse_build/mod.rs @@ -13,11 +13,12 @@ use crate::{ PathMapperConstructKind, }, list::BinaryList, + platform::PlatformLibdir, }; use camino::{Utf8Path, Utf8PathBuf}; use camino_tempfile::Utf8TempDir; use guppy::graph::PackageGraph; -use nextest_metadata::BinaryListSummary; +use nextest_metadata::{BinaryListSummary, PlatformLibdirUnavailable}; use std::{fmt, fs, io, sync::Arc}; mod archive_reporter; @@ -34,6 +35,9 @@ pub const CARGO_METADATA_FILE_NAME: &str = "target/nextest/cargo-metadata.json"; /// The name of the file in which binaries metadata is stored. pub const BINARIES_METADATA_FILE_NAME: &str = "target/nextest/binaries-metadata.json"; +/// The name of the directory in which libdirs are stored. +pub const LIBDIRS_BASE_DIR: &str = "target/nextest/libdirs"; + /// Reuse build information. #[derive(Debug, Default)] pub struct ReuseBuildInfo { @@ -43,6 +47,9 @@ pub struct ReuseBuildInfo { /// Binaries metadata JSON and remapping for the target directory. pub binaries_metadata: Option>, + /// A remapper for libdirs. + pub libdir_mapper: LibdirMapper, + /// Optional temporary directory used for cleanup. _temp_dir: Option, } @@ -52,10 +59,12 @@ impl ReuseBuildInfo { pub fn new( cargo_metadata: Option>, binaries_metadata: Option>, + // TODO: accept libdir_mapper as an argument, as well ) -> Self { Self { cargo_metadata, binaries_metadata, + libdir_mapper: LibdirMapper::default(), _temp_dir: None, } } @@ -81,6 +90,7 @@ impl ReuseBuildInfo { binary_list, cargo_metadata_json, graph, + libdir_mapper, } = unarchiver.extract(dest, callback)?; let cargo_metadata = MetadataWithRemap { @@ -95,6 +105,7 @@ impl ReuseBuildInfo { Ok(Self { cargo_metadata: Some(cargo_metadata), binaries_metadata: Some(binaries_metadata), + libdir_mapper, _temp_dir: temp_dir, }) } @@ -238,11 +249,13 @@ impl MetadataKind for ReusedCargoMetadata { /// A helper for path remapping. /// -/// This is useful when running tests in a different directory, or a different computer, from building them. +/// This is useful when running tests in a different directory, or a different computer, from +/// building them. #[derive(Clone, Debug, Default)] pub struct PathMapper { workspace: Option<(Utf8PathBuf, Utf8PathBuf)>, target_dir: Option<(Utf8PathBuf, Utf8PathBuf)>, + libdir_mapper: LibdirMapper, } impl PathMapper { @@ -252,17 +265,19 @@ impl PathMapper { workspace_remap: Option<&Utf8Path>, orig_target_dir: impl Into, target_dir_remap: Option<&Utf8Path>, + libdir_mapper: LibdirMapper, ) -> Result { let workspace_root = workspace_remap .map(|root| Self::canonicalize_dir(root, PathMapperConstructKind::WorkspaceRoot)) .transpose()?; let target_dir = target_dir_remap - .map(|dir| Self::canonicalize_dir(dir, PathMapperConstructKind::WorkspaceRoot)) + .map(|dir| Self::canonicalize_dir(dir, PathMapperConstructKind::TargetDir)) .transpose()?; Ok(Self { workspace: workspace_root.map(|w| (orig_workspace_root.into(), w)), target_dir: target_dir.map(|d| (orig_target_dir.into(), d)), + libdir_mapper, }) } @@ -271,9 +286,15 @@ impl PathMapper { Self { workspace: None, target_dir: None, + libdir_mapper: LibdirMapper::default(), } } + /// Returns the libdir mapper. + pub fn libdir_mapper(&self) -> &LibdirMapper { + &self.libdir_mapper + } + fn canonicalize_dir( input: &Utf8Path, kind: PathMapperConstructKind, @@ -330,6 +351,48 @@ impl PathMapper { } } +/// A mapper for lib dirs. +/// +/// Archives store parts of lib dirs, which must be remapped to the new target directory. +#[derive(Clone, Debug, Default)] +pub struct LibdirMapper { + /// The host libdir mapper. + pub(crate) host: PlatformLibdirMapper, + + /// The target libdir mapper. + pub(crate) target: PlatformLibdirMapper, +} + +/// A mapper for an individual platform libdir. +/// +/// Part of [`LibdirMapper`]. +#[derive(Clone, Debug, Default)] +pub(crate) enum PlatformLibdirMapper { + Path(Utf8PathBuf), + Unavailable, + #[default] + NotRequested, +} + +impl PlatformLibdirMapper { + pub(crate) fn map(&self, original: &PlatformLibdir) -> PlatformLibdir { + match self { + PlatformLibdirMapper::Path(new) => { + // Just use the new path. (We may have to check original in the future, but it + // doesn't seem necessary for now -- if a libdir has been provided to the remapper, + // that's that.) + PlatformLibdir::Available(new.clone()) + } + PlatformLibdirMapper::Unavailable => { + // In this case, the original value is ignored -- we expected a libdir to be + // present, but it wasn't. + PlatformLibdir::Unavailable(PlatformLibdirUnavailable::NOT_IN_ARCHIVE) + } + PlatformLibdirMapper::NotRequested => original.clone(), + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -372,6 +435,7 @@ mod tests { Some(&rel_workspace_root), &orig_target_dir, Some(&rel_target_dir), + LibdirMapper::default(), ) .expect("remapped paths exist"); diff --git a/nextest-runner/src/reuse_build/unarchiver.rs b/nextest-runner/src/reuse_build/unarchiver.rs index eb87d43cf00..1fbd0ea88c2 100644 --- a/nextest-runner/src/reuse_build/unarchiver.rs +++ b/nextest-runner/src/reuse_build/unarchiver.rs @@ -1,9 +1,13 @@ // Copyright (c) The nextest Contributors // SPDX-License-Identifier: MIT OR Apache-2.0 -use super::{ArchiveEvent, ArchiveFormat, BINARIES_METADATA_FILE_NAME, CARGO_METADATA_FILE_NAME}; +use super::{ + ArchiveEvent, ArchiveFormat, LibdirMapper, PlatformLibdirMapper, BINARIES_METADATA_FILE_NAME, + CARGO_METADATA_FILE_NAME, LIBDIRS_BASE_DIR, +}; use crate::{ errors::{ArchiveExtractError, ArchiveReadError}, + helpers::convert_rel_path_to_main_sep, list::BinaryList, }; use camino::{Utf8Component, Utf8Path, Utf8PathBuf}; @@ -85,9 +89,11 @@ impl<'a> Unarchiver<'a> { let mut archive_reader = ArchiveReader::new(self.file, self.format).map_err(ArchiveExtractError::Read)?; - // Will be filled out by the for loop below\ + // Will be filled out by the for loop below. let mut binary_list = None; let mut graph_data = None; + let mut host_libdir = PlatformLibdirMapper::Unavailable; + let mut target_libdir = PlatformLibdirMapper::Unavailable; let binaries_metadata_path = Utf8Path::new(BINARIES_METADATA_FILE_NAME); let cargo_metadata_path = Utf8Path::new(CARGO_METADATA_FILE_NAME); @@ -165,6 +171,18 @@ impl<'a> Unarchiver<'a> { })?; graph_data = Some((json, package_graph)); continue; + } else if let Ok(suffix) = path.strip_prefix(LIBDIRS_BASE_DIR) { + if suffix.starts_with("host") { + host_libdir = PlatformLibdirMapper::Path(dest_dir.join( + convert_rel_path_to_main_sep(&Utf8Path::new(LIBDIRS_BASE_DIR).join("host")), + )); + } else if suffix.starts_with("target/0") { + // Currently we only support one target, so just check explicitly for target/0. + target_libdir = + PlatformLibdirMapper::Path(dest_dir.join(convert_rel_path_to_main_sep( + &Utf8Path::new(LIBDIRS_BASE_DIR).join("target/0"), + ))); + } } } @@ -201,6 +219,10 @@ impl<'a> Unarchiver<'a> { binary_list, cargo_metadata_json, graph, + libdir_mapper: LibdirMapper { + host: host_libdir, + target: target_libdir, + }, }) } } @@ -221,6 +243,9 @@ pub(crate) struct ExtractInfo { /// The [`PackageGraph`] read from the archive. pub graph: PackageGraph, + + /// A remapper for the Rust libdir. + pub libdir_mapper: LibdirMapper, } struct ArchiveReader<'a> { diff --git a/scripts/nextest-without-rustup.sh b/scripts/nextest-without-rustup.sh new file mode 100755 index 00000000000..d0b49fb3738 --- /dev/null +++ b/scripts/nextest-without-rustup.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Run tests without rustup or cargo wrappers. +# +# Some scenarios involve running tests without rustup or cargo. We can't run `cargo nextest run` +# directly because it will use the `cargo` wrapper installed by rustup. Instead, we try our best +# to not use rustup or cargo. +# +# * CARGO is set to $(rustup which cargo) to bypass the rustup wrapper. +# * Nextest is invoked as "cargo-nextest nextest run" rather than "cargo nextest run". + +set -e -o pipefail + +if [[ $RUSTUP_AVAILABLE -eq 1 ]]; then + CARGO="$(rustup which cargo)" + RUSTC="$(rustup which rustc)" +else + # These paths should hopefully make it clear that cargo and rustc are not available -- if we try + # to run them then they'll fail. + CARGO="cargo-unavailable" + RUSTC="rustc-unavailable" +fi + +export CARGO RUSTC + +export CARGO_NEXTEST=${CARGO_NEXTEST:-"cargo-nextest"} + +echo "[nextest-without-rustup] nextest with CARGO_NEXTEST=$CARGO_NEXTEST" >&2 +$CARGO_NEXTEST nextest "$@" diff --git a/site/src/book/env-vars.md b/site/src/book/env-vars.md index c03b613ed32..3c94a379051 100644 --- a/site/src/book/env-vars.md +++ b/site/src/book/env-vars.md @@ -51,7 +51,7 @@ Nextest delegates to Cargo for the build, which controls the environment variabl Nextest also sets these environment variables at runtime, matching the behavior of `cargo test`: -- `CARGO` — Path to the `cargo` binary performing the build. +- `CARGO` — Path to the `cargo` binary performing the build. (This is set by Cargo, not nextest, so if you invoke nextest with `cargo-nextest nextest run` it will not be set.) - `CARGO_MANIFEST_DIR` — The directory containing the manifest of your package. If [`--workspace-remap`](reusing-builds.md#specifying-a-new-location-for-the-workspace) is passed in, this is set to the remapped manifest directory. You can obtain the non-remapped directory using the value of this variable at compile-time, e.g. `env!("CARGO_MANIFEST_DIR")`. - `CARGO_PKG_VERSION` — The full version of your package. - `CARGO_PKG_VERSION_MAJOR` — The major version of your package.