From 127fdfeb899fa60f6d398304b507363bba4b317d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 26 Apr 2019 13:53:51 -0700 Subject: [PATCH 01/12] Implement the Cargo half of pipelined compilation This commit starts to lay the groundwork for #6660 where Cargo will invoke rustc in a "pipelined" fashion. The goal here is to execute one command to produce both an `*.rmeta` file as well as an `*.rlib` file for candidate compilations. In that case if another rlib depends on that compilation, then it can start as soon as the `*.rmeta` is ready and not have to wait for the `*.rlib` compilation. Initially attempted in #6864 with a pretty invasive refactoring this iteration is much more lightweight and fits much more cleanly into Cargo's backend. The approach taken here is to update the `DependencyQueue` structure to carry a piece of data on each dependency edge. This edge information represents the artifact that one node requires from another, and then we a node has no outgoing edges it's ready to build. A dependency on a metadata file is modeled as just that, a dependency on just the metadata and not the full build itself. Most of cargo's backend doesn't really need to know about this edge information so it's basically just calculated as we insert nodes into the `DependencyQueue`. Once that's all in place it's just a few pieces here and there to identify compilations that *can* be pipelined and then they're wired up to depend on the rmeta file instead of the rlib file. --- .../compiler/build_context/target_info.rs | 4 +- .../compiler/context/compilation_files.rs | 7 +- src/cargo/core/compiler/context/mod.rs | 47 +++- .../compiler/context/unit_dependencies.rs | 88 ++++--- src/cargo/core/compiler/job_queue.rs | 137 +++++++++-- src/cargo/core/compiler/mod.rs | 42 +++- src/cargo/core/manifest.rs | 31 +++ src/cargo/util/dependency_queue.rs | 118 +++++----- tests/testsuite/build.rs | 74 ++++-- tests/testsuite/build_lib.rs | 2 +- tests/testsuite/build_script.rs | 11 +- tests/testsuite/check.rs | 2 +- tests/testsuite/cross_compile.rs | 2 +- tests/testsuite/freshness.rs | 4 +- tests/testsuite/profile_overrides.rs | 14 +- tests/testsuite/profile_targets.rs | 216 +++++++++--------- tests/testsuite/profiles.rs | 12 +- tests/testsuite/run.rs | 8 +- tests/testsuite/rustc.rs | 26 +-- 19 files changed, 542 insertions(+), 303 deletions(-) diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index 8a83643c4cf..a30ec19dc16 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -20,12 +20,12 @@ pub struct TargetInfo { } /// Type of each file generated by a Unit. -#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[derive(Clone, PartialEq, Eq, Debug)] pub enum FileFlavor { /// Not a special file type. Normal, /// Something you can link against (e.g., a library). - Linkable, + Linkable { rmeta: PathBuf }, /// Piece of external debug information (e.g., `.dSYM`/`.pdb` file). DebugInfo, } diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs index 217022f7132..7a618263770 100644 --- a/src/cargo/core/compiler/context/compilation_files.rs +++ b/src/cargo/core/compiler/context/compilation_files.rs @@ -305,10 +305,10 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> { // for both libraries and binaries. let path = out_dir.join(format!("lib{}.rmeta", file_stem)); ret.push(OutputFile { - path, + path: path.clone(), hardlink: None, export_path: None, - flavor: FileFlavor::Linkable, + flavor: FileFlavor::Linkable { rmeta: path }, }); } else { let mut add = |crate_type: &str, flavor: FileFlavor| -> CargoResult<()> { @@ -372,7 +372,8 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> { add( kind.crate_type(), if kind.linkable() { - FileFlavor::Linkable + let rmeta = out_dir.join(format!("lib{}.rmeta", file_stem)); + FileFlavor::Linkable { rmeta } } else { FileFlavor::Normal }, diff --git a/src/cargo/core/compiler/context/mod.rs b/src/cargo/core/compiler/context/mod.rs index 3eb20bb4bde..61b05a0d782 100644 --- a/src/cargo/core/compiler/context/mod.rs +++ b/src/cargo/core/compiler/context/mod.rs @@ -42,6 +42,17 @@ pub struct Context<'a, 'cfg: 'a> { unit_dependencies: HashMap, Vec>>, files: Option>, package_cache: HashMap, + + /// A flag indicating whether pipelining is enabled for this compilation + /// session. Pipelining largely only affects the edges of the dependency + /// graph that we generate at the end, and otherwise it's pretty + /// straightforward. + pipelining: bool, + + /// A set of units which are compiling rlibs and are expected to produce + /// metadata files in addition to the rlib itself. This is only filled in + /// when `pipelining` above is enabled. + rmeta_required: HashSet>, } impl<'a, 'cfg> Context<'a, 'cfg> { @@ -60,6 +71,12 @@ impl<'a, 'cfg> Context<'a, 'cfg> { .chain_err(|| "failed to create jobserver")?, }; + let pipelining = bcx + .config + .get_bool("build.pipelining")? + .map(|t| t.val) + .unwrap_or(false); + Ok(Self { bcx, compilation: Compilation::new(bcx)?, @@ -76,6 +93,8 @@ impl<'a, 'cfg> Context<'a, 'cfg> { unit_dependencies: HashMap::new(), files: None, package_cache: HashMap::new(), + rmeta_required: HashSet::new(), + pipelining, }) } @@ -261,12 +280,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { self.primary_packages .extend(units.iter().map(|u| u.pkg.package_id())); - build_unit_dependencies( - units, - self.bcx, - &mut self.unit_dependencies, - &mut self.package_cache, - )?; + build_unit_dependencies(self, units)?; let files = CompilationFiles::new( units, host_layout, @@ -453,6 +467,27 @@ impl<'a, 'cfg> Context<'a, 'cfg> { } Ok(()) } + + /// Returns whether when `parent` depends on `dep` if it only requires the + /// metadata file from `dep`. + pub fn only_requires_rmeta(&self, parent: &Unit<'a>, dep: &Unit<'a>) -> bool { + // this is only enabled when pipelining is enabled + self.pipelining + // We're only a candidate for requiring an `rmeta` file if we + // ourselves are building an rlib, + && !parent.target.requires_upstream_objects() + && parent.mode == CompileMode::Build + // Our dependency must also be built as an rlib, otherwise the + // object code must be useful in some fashion + && !dep.target.requires_upstream_objects() + && dep.mode == CompileMode::Build + } + + /// Returns whether when `unit` is built whether it should emit metadata as + /// well because some compilations rely on that. + pub fn rmeta_required(&self, unit: &Unit<'a>) -> bool { + self.rmeta_required.contains(unit) + } } #[derive(Default)] diff --git a/src/cargo/core/compiler/context/unit_dependencies.rs b/src/cargo/core/compiler/context/unit_dependencies.rs index 618c925b37a..3a85070937f 100644 --- a/src/cargo/core/compiler/context/unit_dependencies.rs +++ b/src/cargo/core/compiler/context/unit_dependencies.rs @@ -15,41 +15,35 @@ //! (for example, with and without tests), so we actually build a dependency //! graph of `Unit`s, which capture these properties. -use std::cell::RefCell; -use std::collections::{HashMap, HashSet}; - -use log::trace; - -use super::{BuildContext, CompileMode, Kind}; use crate::core::compiler::Unit; +use crate::core::compiler::{BuildContext, CompileMode, Context, Kind}; use crate::core::dependency::Kind as DepKind; use crate::core::package::Downloads; use crate::core::profiles::UnitFor; use crate::core::{Package, PackageId, Target}; use crate::CargoResult; +use log::trace; +use std::collections::{HashMap, HashSet}; struct State<'a: 'tmp, 'cfg: 'a, 'tmp> { - bcx: &'tmp BuildContext<'a, 'cfg>, - deps: &'tmp mut HashMap, Vec>>, - pkgs: RefCell<&'tmp mut HashMap>, + cx: &'tmp mut Context<'a, 'cfg>, waiting_on_download: HashSet, downloads: Downloads<'a, 'cfg>, } pub fn build_unit_dependencies<'a, 'cfg>( + cx: &mut Context<'a, 'cfg>, roots: &[Unit<'a>], - bcx: &BuildContext<'a, 'cfg>, - deps: &mut HashMap, Vec>>, - pkgs: &mut HashMap, ) -> CargoResult<()> { - assert!(deps.is_empty(), "can only build unit deps once"); + assert!( + cx.unit_dependencies.is_empty(), + "can only build unit deps once" + ); let mut state = State { - bcx, - deps, - pkgs: RefCell::new(pkgs), + downloads: cx.bcx.packages.enable_download()?, + cx, waiting_on_download: HashSet::new(), - downloads: bcx.packages.enable_download()?, }; loop { @@ -62,7 +56,7 @@ pub fn build_unit_dependencies<'a, 'cfg>( // cleared, and avoid building the lib thrice (once with `panic`, once // without, once for `--test`). In particular, the lib included for // Doc tests and examples are `Build` mode here. - let unit_for = if unit.mode.is_any_test() || bcx.build_config.test() { + let unit_for = if unit.mode.is_any_test() || state.cx.bcx.build_config.test() { UnitFor::new_test() } else if unit.target.is_custom_build() { // This normally doesn't happen, except `clean` aggressively @@ -79,20 +73,23 @@ pub fn build_unit_dependencies<'a, 'cfg>( if !state.waiting_on_download.is_empty() { state.finish_some_downloads()?; - state.deps.clear(); + state.cx.unit_dependencies.clear(); } else { break; } } - trace!("ALL UNIT DEPENDENCIES {:#?}", state.deps); connect_run_custom_build_deps(&mut state); + trace!("ALL UNIT DEPENDENCIES {:#?}", state.cx.unit_dependencies); + + record_units_requiring_metadata(state.cx); + // Dependencies are used in tons of places throughout the backend, many of // which affect the determinism of the build itself. As a result be sure // that dependency lists are always sorted to ensure we've always got a // deterministic output. - for list in state.deps.values_mut() { + for list in state.cx.unit_dependencies.values_mut() { list.sort(); } @@ -104,16 +101,16 @@ fn deps_of<'a, 'cfg, 'tmp>( state: &mut State<'a, 'cfg, 'tmp>, unit_for: UnitFor, ) -> CargoResult<()> { - // Currently the `deps` map does not include `unit_for`. This should + // Currently the `unit_dependencies` map does not include `unit_for`. This should // be safe for now. `TestDependency` only exists to clear the `panic` // flag, and you'll never ask for a `unit` with `panic` set as a // `TestDependency`. `CustomBuild` should also be fine since if the // requested unit's settings are the same as `Any`, `CustomBuild` can't // affect anything else in the hierarchy. - if !state.deps.contains_key(unit) { + if !state.cx.unit_dependencies.contains_key(unit) { let unit_deps = compute_deps(unit, state, unit_for)?; let to_insert: Vec<_> = unit_deps.iter().map(|&(unit, _)| unit).collect(); - state.deps.insert(*unit, to_insert); + state.cx.unit_dependencies.insert(*unit, to_insert); for (unit, unit_for) in unit_deps { deps_of(&unit, state, unit_for)?; } @@ -131,13 +128,13 @@ fn compute_deps<'a, 'cfg, 'tmp>( unit_for: UnitFor, ) -> CargoResult, UnitFor)>> { if unit.mode.is_run_custom_build() { - return compute_deps_custom_build(unit, state.bcx); + return compute_deps_custom_build(unit, state.cx.bcx); } else if unit.mode.is_doc() && !unit.mode.is_any_test() { // Note: this does not include doc test. return compute_deps_doc(unit, state); } - let bcx = state.bcx; + let bcx = state.cx.bcx; let id = unit.pkg.package_id(); let deps = bcx.resolve.deps(id).filter(|&(_id, deps)| { assert!(!deps.is_empty()); @@ -295,7 +292,7 @@ fn compute_deps_doc<'a, 'cfg, 'tmp>( unit: &Unit<'a>, state: &mut State<'a, 'cfg, 'tmp>, ) -> CargoResult, UnitFor)>> { - let bcx = state.bcx; + let bcx = state.cx.bcx; let deps = bcx .resolve .deps(unit.pkg.package_id()) @@ -448,7 +445,7 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_, '_>) { // have the build script as the key and the library would be in the // value's set. let mut reverse_deps = HashMap::new(); - for (unit, deps) in state.deps.iter() { + for (unit, deps) in state.cx.unit_dependencies.iter() { for dep in deps { if dep.mode == CompileMode::RunCustomBuild { reverse_deps @@ -469,7 +466,8 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_, '_>) { // `dep_build_script` to manufacture an appropriate build script unit to // depend on. for unit in state - .deps + .cx + .unit_dependencies .keys() .filter(|k| k.mode == CompileMode::RunCustomBuild) { @@ -480,13 +478,13 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_, '_>) { let to_add = reverse_deps .iter() - .flat_map(|reverse_dep| state.deps[reverse_dep].iter()) + .flat_map(|reverse_dep| state.cx.unit_dependencies[reverse_dep].iter()) .filter(|other| { other.pkg != unit.pkg && other.target.linkable() && other.pkg.manifest().links().is_some() }) - .filter_map(|other| dep_build_script(other, state.bcx).map(|p| p.0)) + .filter_map(|other| dep_build_script(other, state.cx.bcx).map(|p| p.0)) .collect::>(); if !to_add.is_empty() { @@ -497,21 +495,39 @@ fn connect_run_custom_build_deps(state: &mut State<'_, '_, '_>) { // And finally, add in all the missing dependencies! for (unit, new_deps) in new_deps { - state.deps.get_mut(&unit).unwrap().extend(new_deps); + state + .cx + .unit_dependencies + .get_mut(&unit) + .unwrap() + .extend(new_deps); + } +} + +/// Records the list of units which are required to emit metadata. +/// +/// Units which depend only on the metadata of others requires the others to +/// actually produce metadata, so we'll record that here. +fn record_units_requiring_metadata(cx: &mut Context<'_, '_>) { + for (key, deps) in cx.unit_dependencies.iter() { + for dep in deps { + if cx.only_requires_rmeta(key, dep) { + cx.rmeta_required.insert(*dep); + } + } } } impl<'a, 'cfg, 'tmp> State<'a, 'cfg, 'tmp> { fn get(&mut self, id: PackageId) -> CargoResult> { - let mut pkgs = self.pkgs.borrow_mut(); - if let Some(pkg) = pkgs.get(&id) { + if let Some(pkg) = self.cx.package_cache.get(&id) { return Ok(Some(pkg)); } if !self.waiting_on_download.insert(id) { return Ok(None); } if let Some(pkg) = self.downloads.start(id)? { - pkgs.insert(id, pkg); + self.cx.package_cache.insert(id, pkg); self.waiting_on_download.remove(&id); return Ok(Some(pkg)); } @@ -531,7 +547,7 @@ impl<'a, 'cfg, 'tmp> State<'a, 'cfg, 'tmp> { loop { let pkg = self.downloads.wait()?; self.waiting_on_download.remove(&pkg.package_id()); - self.pkgs.borrow_mut().insert(pkg.package_id(), pkg); + self.cx.package_cache.insert(pkg.package_id(), pkg); // Arbitrarily choose that 5 or more packages concurrently download // is a good enough number to "fill the network pipe". If we have diff --git a/src/cargo/core/compiler/job_queue.rs b/src/cargo/core/compiler/job_queue.rs index 1d387d89c28..4bdda90bb9b 100644 --- a/src/cargo/core/compiler/job_queue.rs +++ b/src/cargo/core/compiler/job_queue.rs @@ -1,4 +1,5 @@ use std::collections::{HashMap, HashSet}; +use std::cell::Cell; use std::io; use std::marker; use std::process::Output; @@ -29,7 +30,7 @@ use crate::util::{Progress, ProgressStyle}; /// actual compilation step of each package. Packages enqueue units of work and /// then later on the entire graph is processed and compiled. pub struct JobQueue<'a, 'cfg> { - queue: DependencyQueue, Job>, + queue: DependencyQueue, Artifact, Job>, tx: Sender, rx: Receiver, active: HashMap>, @@ -42,12 +43,43 @@ pub struct JobQueue<'a, 'cfg> { } pub struct JobState<'a> { + /// Channel back to the main thread to coordinate messages and such. tx: Sender, + + /// The job id that this state is associated with, used when sending + /// messages back to the main thread. + id: u32, + + /// Whether or not we're expected to have a call to `rmeta_produced`. Once + /// that method is called this is dynamically set to `false` to prevent + /// sending a double message later on. + rmeta_required: Cell, + // Historical versions of Cargo made use of the `'a` argument here, so to // leave the door open to future refactorings keep it here. _marker: marker::PhantomData<&'a ()>, } +/// Possible artifacts that can be produced by compilations, used as edge values +/// in the dependency graph. +/// +/// As edge values we can have multiple kinds of edges depending on one node, +/// for example some units may only depend on the metadata for an rlib while +/// others depend on the full rlib. This `Artifact` enum is used to distinguish +/// this case and track the progress of compilations as they proceed. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug)] +enum Artifact { + /// A generic placeholder for "depends on everything run by a step" and + /// means that we can't start the next compilation until the previous has + /// finished entirely. + All, + + /// A node indicating that we only depend on the metadata of a compilation, + /// but the compilation is typically also producing an rlib. We can start + /// our step, however, before the full rlib is available. + Metadata, +} + enum Message { Run(String), BuildPlanMsg(String, ProcessBuilder, Arc>), @@ -55,7 +87,7 @@ enum Message { Stderr(String), FixDiagnostic(diagnostic_server::Message), Token(io::Result), - Finish(u32, CargoResult<()>), + Finish(u32, Artifact, CargoResult<()>), } impl<'a> JobState<'a> { @@ -93,6 +125,19 @@ impl<'a> JobState<'a> { capture_output, ) } + + /// A method used to signal to the coordinator thread that the rmeta file + /// for an rlib has been produced. This is only called for some rmeta + /// builds when required, and can be called at any time before a job ends. + /// This should only be called once because a metadata file can only be + /// produced once! + pub fn rmeta_produced(&self) { + assert!(self.rmeta_required.get()); + self.rmeta_required.set(false); + let _ = self + .tx + .send(Message::Finish(self.id, Artifact::Metadata, Ok(()))); + } } impl<'a, 'cfg> JobQueue<'a, 'cfg> { @@ -128,8 +173,19 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { !unit.target.is_test() || !unit.target.is_bin() }) .cloned() - .collect::>(); - self.queue.queue(unit, job, &dependencies); + .map(|dep| { + // Handle the case here where our `unit -> dep` dependency may + // only require the metadata, not the full compilation to + // finish. Use the tables in `cx` to figure out what kind + // of artifact is associated with this dependency. + let artifact = if cx.only_requires_rmeta(unit, &dep) { + Artifact::Metadata + } else { + Artifact::All + }; + (dep, artifact) + }); + self.queue.queue(*unit, job, dependencies); *self.counts.entry(unit.pkg.package_id()).or_insert(0) += 1; Ok(()) } @@ -139,7 +195,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { /// This function will spawn off `config.jobs()` workers to build all of the /// necessary dependencies, in order. Freshness is propagated as far as /// possible along each dependency chain. - pub fn execute(&mut self, cx: &mut Context<'_, '_>, plan: &mut BuildPlan) -> CargoResult<()> { + pub fn execute(&mut self, cx: &mut Context<'a, '_>, plan: &mut BuildPlan) -> CargoResult<()> { let _p = profile::start("executing the job graph"); self.queue.queue_finished(); @@ -176,7 +232,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { fn drain_the_queue( &mut self, - cx: &mut Context<'_, '_>, + cx: &mut Context<'a, '_>, plan: &mut BuildPlan, scope: &Scope<'a>, jobserver_helper: &HelperThread, @@ -264,16 +320,24 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { Message::FixDiagnostic(msg) => { print.print(&msg)?; } - Message::Finish(id, result) => { - let unit = self.active.remove(&id).unwrap(); - info!("end: {:?}", unit); - - if !self.active.is_empty() { - assert!(!tokens.is_empty()); - drop(tokens.pop()); - } + Message::Finish(id, artifact, result) => { + let unit = match artifact { + // If `id` has completely finished we remove it + // from the `active` map ... + Artifact::All => { + info!("end: {:?}", id); + self.active.remove(&id).unwrap() + } + // ... otherwise if it hasn't finished we leave it + // in there as we'll get another `Finish` later on. + Artifact::Metadata => { + info!("end (meta): {:?}", id); + self.active[&id] + } + }; + info!("end ({:?}): {:?}", unit, result); match result { - Ok(()) => self.finish(&unit, cx)?, + Ok(()) => self.finish(&unit, artifact, cx)?, Err(e) => { let msg = "The following warnings were emitted during compilation:"; self.emit_warnings(Some(msg), &unit, cx)?; @@ -321,7 +385,9 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { let time_elapsed = util::elapsed(cx.bcx.config.creation_time().elapsed()); - if self.queue.is_empty() { + if let Some(e) = error { + Err(e) + } else if self.queue.is_empty() && queue.is_empty() { let message = format!( "{} [{}] target(s) in {}", build_type, opt_type, time_elapsed @@ -330,8 +396,6 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { cx.bcx.config.shell().status("Finished", message)?; } Ok(()) - } else if let Some(e) = error { - Err(e) } else { debug!("queue: {:#?}", self.queue); Err(internal("finished with jobs still left in the queue")) @@ -376,24 +440,47 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { &mut self, unit: &Unit<'a>, job: Job, - cx: &Context<'_, '_>, + cx: &Context<'a, '_>, scope: &Scope<'a>, ) -> CargoResult<()> { - info!("start: {:?}", unit); let id = self.next_id; self.next_id = id.checked_add(1).unwrap(); + + info!("start {}: {:?}", id, unit); + assert!(self.active.insert(id, *unit).is_none()); *self.counts.get_mut(&unit.pkg.package_id()).unwrap() -= 1; let my_tx = self.tx.clone(); let fresh = job.freshness(); + let rmeta_required = cx.rmeta_required(unit); let doit = move || { - let res = job.run(&JobState { + let state = JobState { + id, tx: my_tx.clone(), + rmeta_required: Cell::new(rmeta_required), _marker: marker::PhantomData, - }); - my_tx.send(Message::Finish(id, res)).unwrap(); + }; + let res = job.run(&state); + + // If the `rmeta_required` wasn't consumed but it was set + // previously, then we either have: + // + // 1. The `job` didn't do anything because it was "fresh". + // 2. The `job` returned an error and didn't reach the point where + // it called `rmeta_produced`. + // 3. We forgot to call `rmeta_produced` and there's a bug in Cargo. + // + // Ruling out the third, the other two are pretty common for 2 + // we'll just naturally abort the compilation operation but for 1 + // we need to make sure that the metadata is flagged as produced so + // send a synthetic message here. + if state.rmeta_required.get() && res.is_ok() { + my_tx.send(Message::Finish(id, Artifact::Metadata, Ok(()))).unwrap(); + } + + my_tx.send(Message::Finish(id, Artifact::All, res)).unwrap(); }; if !cx.bcx.build_config.build_plan { @@ -439,11 +526,11 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { Ok(()) } - fn finish(&mut self, unit: &Unit<'a>, cx: &mut Context<'_, '_>) -> CargoResult<()> { + fn finish(&mut self, unit: &Unit<'a>, artifact: Artifact, cx: &mut Context<'_, '_>) -> CargoResult<()> { if unit.mode.is_run_custom_build() && cx.bcx.show_warnings(unit.pkg.package_id()) { self.emit_warnings(None, unit, cx)?; } - self.queue.finish(unit); + self.queue.finish(unit, &artifact); Ok(()) } diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 587662293ab..a2a024e0ffd 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -15,7 +15,7 @@ use std::env; use std::ffi::{OsStr, OsString}; use std::fs; use std::io::{self, Write}; -use std::path::{self, Path, PathBuf}; +use std::path::{Path, PathBuf}; use std::sync::Arc; use failure::Error; @@ -241,6 +241,7 @@ fn rustc<'a, 'cfg>( .unwrap_or_else(|| cx.bcx.config.cwd()) .to_path_buf(); let fingerprint_dir = cx.files().fingerprint_dir(unit); + let rmeta_produced = cx.rmeta_required(unit); return Ok(Work::new(move |state| { // Only at runtime have we discovered what the extra -L and -l @@ -312,6 +313,25 @@ fn rustc<'a, 'cfg>( .chain_err(|| format!("Could not compile `{}`.", name))?; } + // FIXME(rust-lang/rust#58465): this is the whole point of "pipelined + // compilation" in Cargo. We want to, here in this unit, call + // `finish_rmeta` as soon as we can which indicates that the metadata + // file is emitted by rustc and ready to go. This will start dependency + // compilations as soon as possible. + // + // The compiler doesn't currently actually implement the ability to let + // us know, however, when the metadata file is ready to go. It actually + // today produces the file very late in compilation, far later than it + // would otherwise be able to do! + // + // In any case this is all covered by the issue above. This is just a + // marker for "yes we unconditionally do this today but tomorrow we + // should actually read what rustc is doing and execute this at an + // appropriate time, ideally long before rustc finishes completely". + if rmeta_produced { + state.rmeta_produced(); + } + if do_rename && real_name != crate_name { let dst = &outputs[0].path; let src = dst.with_file_name( @@ -789,6 +809,11 @@ fn build_base_args<'a, 'cfg>( if unit.mode.is_check() { cmd.arg("--emit=dep-info,metadata"); + } else if !unit.target.requires_upstream_objects() { + // Always produce metdata files for rlib outputs. Metadata may be used + // in this session for a pipelined compilation, or it may be used in a + // future Cargo session as part of a pipelined compile.c + cmd.arg("--emit=dep-info,metadata,link"); } else { cmd.arg("--emit=dep-info,link"); } @@ -995,16 +1020,19 @@ fn build_deps_args<'a, 'cfg>( ) -> CargoResult<()> { let bcx = cx.bcx; for output in cx.outputs(dep)?.iter() { - if output.flavor != FileFlavor::Linkable { - continue; - } + let rmeta = match &output.flavor { + FileFlavor::Linkable { rmeta } => rmeta, + _ => continue, + }; let mut v = OsString::new(); let name = bcx.extern_crate_name(current, dep)?; v.push(name); v.push("="); - v.push(cx.files().out_dir(dep)); - v.push(&path::MAIN_SEPARATOR.to_string()); - v.push(&output.path.file_name().unwrap()); + if cx.only_requires_rmeta(current, dep) { + v.push(&rmeta); + } else { + v.push(&output.path); + } if current .pkg diff --git a/src/cargo/core/manifest.rs b/src/cargo/core/manifest.rs index 3cb9481680e..0272a200c15 100644 --- a/src/cargo/core/manifest.rs +++ b/src/cargo/core/manifest.rs @@ -120,6 +120,19 @@ impl LibKind { LibKind::Other(..) => false, } } + + pub fn requires_upstream_objects(&self) -> bool { + match *self { + // "lib" == "rlib" and is a compilation that doesn't actually + // require upstream object files to exist, only upstream metadata + // files. As a result, it doesn't require upstream artifacts + LibKind::Lib | LibKind::Rlib => false, + + // Everything else, however, is some form of "linkable output" or + // something that requires upstream object files. + _ => true, + } + } } impl fmt::Debug for LibKind { @@ -795,6 +808,10 @@ impl Target { }) } + /// Returns whether this target produces an artifact which can be linked + /// into a Rust crate. + /// + /// This only returns true for certain kinds of libraries. pub fn linkable(&self) -> bool { match self.kind { TargetKind::Lib(ref kinds) => kinds.iter().any(|k| k.linkable()), @@ -802,6 +819,20 @@ impl Target { } } + /// Returns whether production of this artifact requires the object files + /// from dependencies to be available. + /// + /// This only returns `false` when all we're producing is an rlib, otherwise + /// it will return `true`. + pub fn requires_upstream_objects(&self) -> bool { + match &self.kind { + TargetKind::Lib(kinds) | TargetKind::ExampleLib(kinds) => { + kinds.iter().any(|k| k.requires_upstream_objects()) + } + _ => true, + } + } + pub fn is_bin(&self) -> bool { self.kind == TargetKind::Bin } diff --git a/src/cargo/util/dependency_queue.rs b/src/cargo/util/dependency_queue.rs index 9df373ee4f0..9a218408069 100644 --- a/src/cargo/util/dependency_queue.rs +++ b/src/cargo/util/dependency_queue.rs @@ -3,68 +3,80 @@ //! //! This structure is used to store the dependency graph and dynamically update //! it to figure out when a dependency should be built. +//! +//! Dependencies in this queue are represented as a (node, edge) pair. This is +//! used to model nodes which produce multiple outputs at different times but +//! some nodes may only require one of the outputs and can start before the +//! whole node is finished. use std::collections::{HashMap, HashSet}; use std::hash::Hash; #[derive(Debug)] -pub struct DependencyQueue { +pub struct DependencyQueue { /// A list of all known keys to build. /// /// The value of the hash map is list of dependencies which still need to be /// built before the package can be built. Note that the set is dynamically /// updated as more dependencies are built. - dep_map: HashMap, V)>, + dep_map: HashMap, V)>, /// A reverse mapping of a package to all packages that depend on that /// package. /// /// This map is statically known and does not get updated throughout the /// lifecycle of the DependencyQueue. - reverse_dep_map: HashMap>, - - /// The packages which are currently being built, waiting for a call to - /// `finish`. - pending: HashSet, + /// + /// This is sort of like a `HashMap<(N, E), HashSet>` map, but more + /// easily indexable with just an `N` + reverse_dep_map: HashMap>>, /// Topological depth of each key - depth: HashMap, + depth: HashMap, } -impl Default for DependencyQueue { - fn default() -> DependencyQueue { +impl Default for DependencyQueue { + fn default() -> DependencyQueue { DependencyQueue::new() } } -impl DependencyQueue { +impl DependencyQueue { /// Creates a new dependency queue with 0 packages. - pub fn new() -> DependencyQueue { + pub fn new() -> DependencyQueue { DependencyQueue { dep_map: HashMap::new(), reverse_dep_map: HashMap::new(), - pending: HashSet::new(), depth: HashMap::new(), } } +} - /// Adds a new package to this dependency queue. +impl DependencyQueue { + /// Adds a new ndoe and its dependencies to this queue. + /// + /// The `key` specified is a new node in the dependency graph, and the node + /// depend on all the dependencies iterated by `dependencies`. Each + /// dependency is a node/edge pair, where edges can be thought of as + /// productions from nodes (aka if it's just `()` it's just waiting for the + /// node to finish). /// - /// It is assumed that any dependencies of this package will eventually also - /// be added to the dependency queue. - pub fn queue(&mut self, key: &K, value: V, dependencies: &[K]) { - assert!(!self.dep_map.contains_key(key)); + /// An optional `value` can also be associated with `key` which is reclaimed + /// when the node is ready to go. + pub fn queue(&mut self, key: N, value: V, dependencies: impl IntoIterator) { + assert!(!self.dep_map.contains_key(&key)); let mut my_dependencies = HashSet::new(); - for dep in dependencies { - my_dependencies.insert(dep.clone()); - let rev = self - .reverse_dep_map - .entry(dep.clone()) - .or_insert_with(HashSet::new); - rev.insert(key.clone()); + for (dep, edge) in dependencies { + my_dependencies.insert((dep.clone(), edge.clone())); + self.reverse_dep_map + .entry(dep) + .or_insert_with(HashMap::new) + .entry(edge) + .or_insert_with(HashSet::new) + .insert(key.clone()); } - self.dep_map.insert(key.clone(), (my_dependencies, value)); + self.dep_map.insert(key, (my_dependencies, value)); } /// All nodes have been added, calculate some internal metadata and prepare @@ -74,10 +86,10 @@ impl DependencyQueue { depth(key, &self.reverse_dep_map, &mut self.depth); } - fn depth( - key: &K, - map: &HashMap>, - results: &mut HashMap, + fn depth( + key: &N, + map: &HashMap>>, + results: &mut HashMap, ) -> usize { const IN_PROGRESS: usize = !0; @@ -91,7 +103,8 @@ impl DependencyQueue { let depth = 1 + map .get(key) .into_iter() - .flat_map(|it| it) + .flat_map(|it| it.values()) + .flat_map(|set| set) .map(|dep| depth(dep, map, results)) .max() .unwrap_or(0); @@ -106,7 +119,7 @@ impl DependencyQueue { /// /// A package is ready to be built when it has 0 un-built dependencies. If /// `None` is returned then no packages are ready to be built. - pub fn dequeue(&mut self) -> Option<(K, V)> { + pub fn dequeue(&mut self) -> Option<(N, V)> { // Look at all our crates and find everything that's ready to build (no // deps). After we've got that candidate set select the one which has // the maximum depth in the dependency graph. This way we should @@ -120,7 +133,7 @@ impl DependencyQueue { let next = self .dep_map .iter() - .filter(|&(_, &(ref deps, _))| deps.is_empty()) + .filter(|(_, (deps, _))| deps.is_empty()) .map(|(key, _)| key.clone()) .max_by_key(|k| self.depth[k]); let key = match next { @@ -128,32 +141,33 @@ impl DependencyQueue { None => return None, }; let (_, data) = self.dep_map.remove(&key).unwrap(); - self.pending.insert(key.clone()); Some((key, data)) } /// Returns `true` if there are remaining packages to be built. pub fn is_empty(&self) -> bool { - self.dep_map.is_empty() && self.pending.is_empty() + self.dep_map.is_empty() } /// Returns the number of remaining packages to be built. pub fn len(&self) -> usize { - self.dep_map.len() + self.pending.len() + self.dep_map.len() } - /// Indicate that a package has been built. + /// Indicate that something has finished. /// - /// This function will update the dependency queue with this information, - /// possibly allowing the next invocation of `dequeue` to return a package. - pub fn finish(&mut self, key: &K) { - assert!(self.pending.remove(key)); - let reverse_deps = match self.reverse_dep_map.get(key) { + /// Calling this function indicates that the `node` has produced `edge`. All + /// remaining work items which only depend on this node/edge pair are now + /// candidates to start their job. + pub fn finish(&mut self, node: &N, edge: &E) { + let reverse_deps = self.reverse_dep_map.get(node).and_then(|map| map.get(edge)); + let reverse_deps = match reverse_deps { Some(deps) => deps, None => return, }; + let key = (node.clone(), edge.clone()); for dep in reverse_deps.iter() { - assert!(self.dep_map.get_mut(dep).unwrap().0.remove(key)); + assert!(self.dep_map.get_mut(dep).unwrap().0.remove(&key)); } } } @@ -166,25 +180,25 @@ mod test { fn deep_first() { let mut q = DependencyQueue::new(); - q.queue(&1, (), &[]); - q.queue(&2, (), &[1]); - q.queue(&3, (), &[]); - q.queue(&4, (), &[2, 3]); - q.queue(&5, (), &[4, 3]); + q.queue(1, (), vec![]); + q.queue(2, (), vec![(1, ())]); + q.queue(3, (), vec![]); + q.queue(4, (), vec![(2, ()), (3, ())]); + q.queue(5, (), vec![(4, ()), (3, ())]); q.queue_finished(); assert_eq!(q.dequeue(), Some((1, ()))); assert_eq!(q.dequeue(), Some((3, ()))); assert_eq!(q.dequeue(), None); - q.finish(&3); + q.finish(&3, &()); assert_eq!(q.dequeue(), None); - q.finish(&1); + q.finish(&1, &()); assert_eq!(q.dequeue(), Some((2, ()))); assert_eq!(q.dequeue(), None); - q.finish(&2); + q.finish(&2, &()); assert_eq!(q.dequeue(), Some((4, ()))); assert_eq!(q.dequeue(), None); - q.finish(&4); + q.finish(&4, &()); assert_eq!(q.dequeue(), Some((5, ()))); } } diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index 14e5263b929..d69e1a72cd0 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -1148,14 +1148,14 @@ fn cargo_default_env_metadata_env_var() { "\ [COMPILING] bar v0.0.1 ([CWD]/bar) [RUNNING] `rustc --crate-name bar bar/src/lib.rs --color never --crate-type dylib \ - --emit=dep-info,link \ + --emit=[..]link \ -C prefer-dynamic -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ -C extra-filename=[..] \ --out-dir [..] \ @@ -1176,14 +1176,14 @@ fn cargo_default_env_metadata_env_var() { "\ [COMPILING] bar v0.0.1 ([CWD]/bar) [RUNNING] `rustc --crate-name bar bar/src/lib.rs --color never --crate-type dylib \ - --emit=dep-info,link \ + --emit=[..]link \ -C prefer-dynamic -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ -C extra-filename=[..] \ --out-dir [..] \ @@ -1557,7 +1557,7 @@ fn lto_build() { "\ [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/main.rs --color never --crate-type bin \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=3 \ -C lto \ -C metadata=[..] \ @@ -1577,7 +1577,7 @@ fn verbose_build() { "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` @@ -1595,7 +1595,7 @@ fn verbose_release_build() { "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=3 \ -C metadata=[..] \ --out-dir [..] \ @@ -1645,7 +1645,7 @@ fn verbose_release_build_deps() { [COMPILING] foo v0.0.0 ([CWD]/foo) [RUNNING] `rustc --crate-name foo foo/src/lib.rs --color never \ --crate-type dylib --crate-type rlib \ - --emit=dep-info,link \ + --emit=[..]link \ -C prefer-dynamic \ -C opt-level=3 \ -C metadata=[..] \ @@ -1653,7 +1653,7 @@ fn verbose_release_build_deps() { -L dependency=[CWD]/target/release/deps` [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=3 \ -C metadata=[..] \ --out-dir [..] \ @@ -4184,11 +4184,11 @@ fn build_filter_infer_profile() { p.cargo("build -v") .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) .run(); @@ -4196,15 +4196,15 @@ fn build_filter_infer_profile() { p.cargo("build -v --test=t1") .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 [..]", + --emit=[..]link -C debuginfo=2 [..]", ) .with_stderr_contains( - "[RUNNING] `rustc --crate-name t1 tests/t1.rs --color never --emit=dep-info,link \ + "[RUNNING] `rustc --crate-name t1 tests/t1.rs --color never --emit=[..]link \ -C debuginfo=2 [..]", ) .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link -C debuginfo=2 [..]", + --emit=[..]link -C debuginfo=2 [..]", ) .run(); @@ -4213,16 +4213,16 @@ fn build_filter_infer_profile() { p.cargo("build -v --bench=b1") .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 [..]", + --emit=[..]link -C debuginfo=2 [..]", ) .with_stderr_contains( - "[RUNNING] `rustc --crate-name b1 benches/b1.rs --color never --emit=dep-info,link \ + "[RUNNING] `rustc --crate-name b1 benches/b1.rs --color never --emit=[..]link \ -C debuginfo=2 [..]", ) .with_stderr_does_not_contain("opt-level") .with_stderr_contains( "[RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link -C debuginfo=2 [..]", + --emit=[..]link -C debuginfo=2 [..]", ) .run(); } @@ -4235,18 +4235,18 @@ fn targets_selected_default() { .with_stderr_contains( "\ [RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) // Benchmarks. .with_stderr_does_not_contain( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C opt-level=3 --test [..]", ) // Unit tests. .with_stderr_does_not_contain( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C debuginfo=2 --test [..]", ) .run(); @@ -4260,12 +4260,12 @@ fn targets_selected_all() { .with_stderr_contains( "\ [RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) // Unit tests. .with_stderr_contains( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C debuginfo=2 --test [..]", ) .run(); @@ -4279,12 +4279,12 @@ fn all_targets_no_lib() { .with_stderr_contains( "\ [RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) // Unit tests. .with_stderr_contains( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C debuginfo=2 --test [..]", ) .run(); @@ -4569,3 +4569,29 @@ Caused by: .with_status(101) .run(); } + +#[test] +fn tricky_pipelining() { + let foo = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + [dependencies] + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "extern crate bar;") + .file("bar/Cargo.toml", &basic_lib_manifest("bar")) + .file("bar/src/lib.rs", "") + .build(); + + foo.cargo("build -p bar") + .env("CARGO_BUILD_PIPELINING", "true") + .run(); + foo.cargo("build -p foo") + .env("CARGO_BUILD_PIPELINING", "true") + .run(); +} diff --git a/tests/testsuite/build_lib.rs b/tests/testsuite/build_lib.rs index 00c256b610f..05627fe0552 100644 --- a/tests/testsuite/build_lib.rs +++ b/tests/testsuite/build_lib.rs @@ -12,7 +12,7 @@ fn build_lib_only() { "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index 3303327fea9..a59cb59038f 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -241,7 +241,7 @@ fn custom_build_script_rustc_flags() { -C metadata=[..] \ -C extra-filename=-[..] \ --out-dir [CWD]/target \ - --emit=dep-info,link \ + --emit=[..]link \ -L [CWD]/target \ -L [CWD]/target/deps` ", @@ -1015,19 +1015,19 @@ fn build_cmd_with_a_build_cmd() { [RUNNING] `rustc [..] a/build.rs [..] --extern b=[..]` [RUNNING] `[..]/a-[..]/build-script-build` [RUNNING] `rustc --crate-name a [..]lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..]target/debug/deps \ -L [..]target/debug/deps` [COMPILING] foo v0.5.0 ([CWD]) [RUNNING] `rustc --crate-name build_script_build build.rs --color never --crate-type bin \ - --emit=dep-info,link \ + --emit=[..]link \ -C debuginfo=2 -C metadata=[..] --out-dir [..] \ -L [..]target/debug/deps \ --extern a=[..]liba[..].rlib` [RUNNING] `[..]/foo-[..]/build-script-build` [RUNNING] `rustc --crate-name foo [..]lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L [..]target/debug/deps` @@ -2217,6 +2217,7 @@ fn diamond_passes_args_only_once() { .build(); p.cargo("build -v") + .env("CARGO_BUILD_PIPELINING", "true") .with_stderr( "\ [COMPILING] c v0.5.0 ([..] @@ -2228,7 +2229,7 @@ fn diamond_passes_args_only_once() { [COMPILING] a v0.5.0 ([..] [RUNNING] `rustc [..]` [COMPILING] foo v0.5.0 ([..] -[RUNNING] `[..]rlib -L native=test` +[RUNNING] `[..]rmeta -L native=test` [FINISHED] dev [unoptimized + debuginfo] target(s) in [..] ", ) diff --git a/tests/testsuite/check.rs b/tests/testsuite/check.rs index 628fd807850..24c61b3635c 100644 --- a/tests/testsuite/check.rs +++ b/tests/testsuite/check.rs @@ -204,7 +204,7 @@ fn issue_3418() { .build(); foo.cargo("check -v") - .with_stderr_contains("[..] --emit=dep-info,metadata [..]") + .with_stderr_contains("[..] --emit=[..]metadata [..]") .run(); } diff --git a/tests/testsuite/cross_compile.rs b/tests/testsuite/cross_compile.rs index 6e9623aba21..e98d0b04456 100644 --- a/tests/testsuite/cross_compile.rs +++ b/tests/testsuite/cross_compile.rs @@ -379,7 +379,7 @@ fn linker_and_ar() { "\ [COMPILING] foo v0.5.0 ([CWD]) [RUNNING] `rustc --crate-name foo src/foo.rs --color never --crate-type bin \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [CWD]/target/{target}/debug/deps \ --target {target} \ diff --git a/tests/testsuite/freshness.rs b/tests/testsuite/freshness.rs index 0909313e219..d216430d6aa 100644 --- a/tests/testsuite/freshness.rs +++ b/tests/testsuite/freshness.rs @@ -1439,8 +1439,8 @@ fn reuse_panic_pm() { .with_stderr_unordered( "\ [COMPILING] bar [..] -[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C debuginfo=2 [..] -[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C debuginfo=2 [..] +[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C debuginfo=2 [..] +[RUNNING] `rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C debuginfo=2 [..] [COMPILING] somepm [..] [RUNNING] `rustc --crate-name somepm [..] [COMPILING] foo [..] diff --git a/tests/testsuite/profile_overrides.rs b/tests/testsuite/profile_overrides.rs index 09bfd7dba48..141a709402a 100644 --- a/tests/testsuite/profile_overrides.rs +++ b/tests/testsuite/profile_overrides.rs @@ -321,17 +321,17 @@ fn profile_override_hierarchy() { p.cargo("build -v").masquerade_as_nightly_cargo().with_stderr_unordered("\ [COMPILING] m3 [..] [COMPILING] dep [..] -[RUNNING] `rustc --crate-name m3 m3/src/lib.rs --color never --crate-type lib --emit=dep-info,link -C codegen-units=4 [..] -[RUNNING] `rustc --crate-name dep [..]dep/src/lib.rs --color never --crate-type lib --emit=dep-info,link -C codegen-units=3 [..] -[RUNNING] `rustc --crate-name m3 m3/src/lib.rs --color never --crate-type lib --emit=dep-info,link -C codegen-units=1 [..] -[RUNNING] `rustc --crate-name build_script_build m1/build.rs --color never --crate-type bin --emit=dep-info,link -C codegen-units=4 [..] +[RUNNING] `rustc --crate-name m3 m3/src/lib.rs --color never --crate-type lib --emit=[..]link -C codegen-units=4 [..] +[RUNNING] `rustc --crate-name dep [..]dep/src/lib.rs --color never --crate-type lib --emit=[..]link -C codegen-units=3 [..] +[RUNNING] `rustc --crate-name m3 m3/src/lib.rs --color never --crate-type lib --emit=[..]link -C codegen-units=1 [..] +[RUNNING] `rustc --crate-name build_script_build m1/build.rs --color never --crate-type bin --emit=[..]link -C codegen-units=4 [..] [COMPILING] m2 [..] -[RUNNING] `rustc --crate-name build_script_build m2/build.rs --color never --crate-type bin --emit=dep-info,link -C codegen-units=2 [..] +[RUNNING] `rustc --crate-name build_script_build m2/build.rs --color never --crate-type bin --emit=[..]link -C codegen-units=2 [..] [RUNNING] `[..]/m1-[..]/build-script-build` [RUNNING] `[..]/m2-[..]/build-script-build` -[RUNNING] `rustc --crate-name m2 m2/src/lib.rs --color never --crate-type lib --emit=dep-info,link -C codegen-units=2 [..] +[RUNNING] `rustc --crate-name m2 m2/src/lib.rs --color never --crate-type lib --emit=[..]link -C codegen-units=2 [..] [COMPILING] m1 [..] -[RUNNING] `rustc --crate-name m1 m1/src/lib.rs --color never --crate-type lib --emit=dep-info,link -C codegen-units=1 [..] +[RUNNING] `rustc --crate-name m1 m1/src/lib.rs --color never --crate-type lib --emit=[..]link -C codegen-units=1 [..] [FINISHED] dev [unoptimized + debuginfo] [..] ", ) diff --git a/tests/testsuite/profile_targets.rs b/tests/testsuite/profile_targets.rs index f9de30d1acf..e1f5f8b510f 100644 --- a/tests/testsuite/profile_targets.rs +++ b/tests/testsuite/profile_targets.rs @@ -78,16 +78,16 @@ fn profile_selection_build() { // - build_script_build is built without panic because it thinks `build.rs` is a plugin. p.cargo("build -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]/target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [FINISHED] dev [unoptimized + debuginfo] [..] ").run(); p.cargo("build -vv") @@ -109,16 +109,16 @@ fn profile_selection_build_release() { // `build --release` p.cargo("build --release -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [RUNNING] `[..]/target/release/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=release DEBUG=false OPT_LEVEL=3 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [FINISHED] release [optimized] [..] ").run(); p.cargo("build --release -vv") @@ -165,22 +165,22 @@ fn profile_selection_build_all_targets() { // example dev build p.cargo("build --all-targets -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]/target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..]` -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..]` -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..]` -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..]` -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..]` -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..]` +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..]` +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..]` +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..]` +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..]` [FINISHED] dev [unoptimized + debuginfo] [..] ").run(); p.cargo("build -vv") @@ -230,22 +230,22 @@ fn profile_selection_build_all_targets_release() { // example release build p.cargo("build --all-targets --release -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [RUNNING] `[..]/target/release/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=release DEBUG=false OPT_LEVEL=3 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..]` -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..]` -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..]` -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..]` -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..]` -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..]` +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..]` +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..]` +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..]` +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..]` +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..]` [FINISHED] release [optimized] [..] ").run(); p.cargo("build --all-targets --release -vv") @@ -286,21 +286,21 @@ fn profile_selection_test() { // p.cargo("test -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]/target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,link -C codegen-units=3 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]link -C codegen-units=3 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [FINISHED] dev [unoptimized + debuginfo] [..] [RUNNING] `[..]/deps/foo-[..]` [RUNNING] `[..]/deps/foo-[..]` @@ -351,21 +351,21 @@ fn profile_selection_test_release() { // p.cargo("test --release -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [RUNNING] `[..]/target/release/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=release DEBUG=false OPT_LEVEL=3 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [FINISHED] release [optimized] [..] [RUNNING] `[..]/deps/foo-[..]` [RUNNING] `[..]/deps/foo-[..]` @@ -416,20 +416,20 @@ fn profile_selection_bench() { // p.cargo("bench -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [RUNNING] `[..]target/release/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=release DEBUG=false OPT_LEVEL=3 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,link -C opt-level=3 -C codegen-units=4 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]link -C opt-level=3 -C codegen-units=4 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [FINISHED] release [optimized] [..] [RUNNING] `[..]/deps/foo-[..] --bench` [RUNNING] `[..]/deps/foo-[..] --bench` @@ -480,23 +480,23 @@ fn profile_selection_check_all_targets() { // p.cargo("check --all-targets -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep[..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]metadata -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [FINISHED] dev [unoptimized + debuginfo] [..] ").run(); // Starting with Rust 1.27, rustc emits `rmeta` files for bins, so @@ -525,23 +525,23 @@ fn profile_selection_check_all_targets_release() { // `dev` for all targets. p.cargo("check --all-targets --release -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [COMPILING] bdep[..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C opt-level=3 -C codegen-units=2 [..] [RUNNING] `[..]target/release/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=release DEBUG=false OPT_LEVEL=3 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 --test [..] -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 --test [..] -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,metadata -C opt-level=3 -C codegen-units=2 --test [..] -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=dep-info,metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=dep-info,metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]metadata -C opt-level=3 -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]metadata -C opt-level=3 -C codegen-units=2 --test [..] +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]metadata -C opt-level=3 -C codegen-units=2 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]metadata -C opt-level=3 -C codegen-units=2 --test [..] +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]metadata -C opt-level=3 -C codegen-units=2 --test [..] +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--crate-type bin --emit=[..]metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--crate-type bin --emit=[..]metadata -C opt-level=3 -C panic=abort -C codegen-units=2 [..] [FINISHED] release [optimized] [..] ").run(); @@ -586,20 +586,20 @@ fn profile_selection_check_all_targets_test() { // p.cargo("check --all-targets --profile=test -vv").with_stderr_unordered("\ [COMPILING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep[..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 [..] -[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] -[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--emit=dep-info,metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--crate-type lib --emit=[..]metadata -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name foo src/lib.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name test1 tests/test1.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name foo src/main.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name bench1 benches/bench1.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] +[RUNNING] `[..] rustc --crate-name ex1 examples/ex1.rs [..]--emit=[..]metadata -C codegen-units=1 -C debuginfo=2 --test [..] [FINISHED] dev [unoptimized + debuginfo] [..] ").run(); @@ -632,13 +632,13 @@ fn profile_selection_doc() { p.cargo("doc -vv").with_stderr_unordered("\ [COMPILING] bar [..] [DOCUMENTING] bar [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `rustdoc --crate-name bar bar/src/lib.rs [..] -[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=dep-info,metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bar bar/src/lib.rs [..]--crate-type lib --emit=[..]metadata -C panic=abort -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] bdep [..] -[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name bdep bdep/src/lib.rs [..]--crate-type lib --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [COMPILING] foo [..] -[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=dep-info,link -C codegen-units=1 -C debuginfo=2 [..] +[RUNNING] `[..] rustc --crate-name build_script_build build.rs [..]--crate-type bin --emit=[..]link -C codegen-units=1 -C debuginfo=2 [..] [RUNNING] `[..]target/debug/build/foo-[..]/build-script-build` [foo 0.0.1] foo custom build PROFILE=debug DEBUG=true OPT_LEVEL=0 [DOCUMENTING] foo [..] diff --git a/tests/testsuite/profiles.rs b/tests/testsuite/profiles.rs index dd5608a0573..f2214e58365 100644 --- a/tests/testsuite/profiles.rs +++ b/tests/testsuite/profiles.rs @@ -27,7 +27,7 @@ fn profile_overrides() { "\ [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=1 \ -C debug-assertions=on \ -C metadata=[..] \ @@ -63,7 +63,7 @@ fn opt_level_override_0() { "\ [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ @@ -96,7 +96,7 @@ fn debug_override_1() { "\ [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C debuginfo=1 \ -C metadata=[..] \ --out-dir [..] \ @@ -132,7 +132,7 @@ fn check_opt_level_override(profile_level: &str, rustc_level: &str) { "\ [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level={level} \ -C debuginfo=2 \ -C debug-assertions=on \ @@ -206,7 +206,7 @@ fn top_level_overrides_deps() { [COMPILING] foo v0.0.0 ([CWD]/foo) [RUNNING] `rustc --crate-name foo foo/src/lib.rs --color never \ --crate-type dylib --crate-type rlib \ - --emit=dep-info,link \ + --emit=[..]link \ -C prefer-dynamic \ -C opt-level=1 \ -C debuginfo=2 \ @@ -215,7 +215,7 @@ fn top_level_overrides_deps() { -L dependency=[CWD]/target/release/deps` [COMPILING] test v0.0.0 ([CWD]) [RUNNING] `rustc --crate-name test src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=1 \ -C debuginfo=2 \ -C metadata=[..] \ diff --git a/tests/testsuite/run.rs b/tests/testsuite/run.rs index 25494a51cc4..70e938dd09c 100644 --- a/tests/testsuite/run.rs +++ b/tests/testsuite/run.rs @@ -697,14 +697,14 @@ fn example_with_release_flag() { "\ [COMPILING] bar v0.5.0 ([CWD]/bar) [RUNNING] `rustc --crate-name bar bar/src/bar.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=3 \ -C metadata=[..] \ --out-dir [CWD]/target/release/deps \ -L dependency=[CWD]/target/release/deps` [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name a examples/a.rs --color never --crate-type bin \ - --emit=dep-info,link \ + --emit=[..]link \ -C opt-level=3 \ -C metadata=[..] \ --out-dir [CWD]/target/release/examples \ @@ -726,14 +726,14 @@ fast2", "\ [COMPILING] bar v0.5.0 ([CWD]/bar) [RUNNING] `rustc --crate-name bar bar/src/bar.rs --color never --crate-type lib \ - --emit=dep-info,link \ + --emit=[..]link \ -C debuginfo=2 \ -C metadata=[..] \ --out-dir [CWD]/target/debug/deps \ -L dependency=[CWD]/target/debug/deps` [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name a examples/a.rs --color never --crate-type bin \ - --emit=dep-info,link \ + --emit=[..]link \ -C debuginfo=2 \ -C metadata=[..] \ --out-dir [CWD]/target/debug/examples \ diff --git a/tests/testsuite/rustc.rs b/tests/testsuite/rustc.rs index 8fef310b173..7674afdb62d 100644 --- a/tests/testsuite/rustc.rs +++ b/tests/testsuite/rustc.rs @@ -16,7 +16,7 @@ fn build_lib_for_foo() { "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` @@ -38,7 +38,7 @@ fn lib() { "\ [COMPILING] foo v0.0.1 ([CWD]) [RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C debug-assertions=off \ -C metadata=[..] \ --out-dir [..] \ @@ -61,12 +61,12 @@ fn build_main_and_allow_unstable_options() { "\ [COMPILING] {name} v{version} ([CWD]) [RUNNING] `rustc --crate-name {name} src/lib.rs --color never --crate-type lib \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C metadata=[..] \ --out-dir [..] \ -L dependency=[CWD]/target/debug/deps` [RUNNING] `rustc --crate-name {name} src/main.rs --color never --crate-type bin \ - --emit=dep-info,link -C debuginfo=2 \ + --emit=[..]link -C debuginfo=2 \ -C debug-assertions \ -C metadata=[..] \ --out-dir [..] \ @@ -106,10 +106,10 @@ fn build_with_args_to_one_of_multiple_binaries() { .with_stderr( "\ [COMPILING] foo v0.0.1 ([CWD]) -[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib --emit=dep-info,link \ +[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib --emit=[..]link \ -C debuginfo=2 -C metadata=[..] \ --out-dir [..]` -[RUNNING] `rustc --crate-name bar src/bin/bar.rs --color never --crate-type bin --emit=dep-info,link \ +[RUNNING] `rustc --crate-name bar src/bin/bar.rs --color never --crate-type bin --emit=[..]link \ -C debuginfo=2 -C debug-assertions [..]` [FINISHED] dev [unoptimized + debuginfo] target(s) in [..] ", @@ -144,10 +144,10 @@ fn build_with_args_to_one_of_multiple_tests() { .with_stderr( "\ [COMPILING] foo v0.0.1 ([CWD]) -[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib --emit=dep-info,link \ +[RUNNING] `rustc --crate-name foo src/lib.rs --color never --crate-type lib --emit=[..]link \ -C debuginfo=2 -C metadata=[..] \ --out-dir [..]` -[RUNNING] `rustc --crate-name bar tests/bar.rs --color never --emit=dep-info,link -C debuginfo=2 \ +[RUNNING] `rustc --crate-name bar tests/bar.rs --color never --emit=[..]link -C debuginfo=2 \ -C debug-assertions [..]--test[..]` [FINISHED] dev [unoptimized + debuginfo] target(s) in [..] ", @@ -233,18 +233,18 @@ fn targets_selected_default() { .with_stderr_contains( "\ [RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) // bench .with_stderr_does_not_contain( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C opt-level=3 --test [..]", ) // unit test .with_stderr_does_not_contain( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C debuginfo=2 --test [..]", ) .run(); @@ -258,12 +258,12 @@ fn targets_selected_all() { .with_stderr_contains( "\ [RUNNING] `rustc --crate-name foo src/main.rs --color never --crate-type bin \ - --emit=dep-info,link[..]", + --emit=[..]link[..]", ) // unit test .with_stderr_contains( "\ - [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=dep-info,link \ + [RUNNING] `rustc --crate-name foo src/main.rs --color never --emit=[..]link \ -C debuginfo=2 --test [..]", ) .run(); From 1a6e452407629a24db9273dd9a740efbad16a327 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 30 Apr 2019 10:59:25 -0700 Subject: [PATCH 02/12] Refactor routing output from rustc/rustdoc This commit refactors slightly how we actually spawn rustc/rustdoc processes and how their output is routed. Over time lots has changed around this, but it turns out that we unconditionally capture all output from the compiler/rustdoc today, no exceptions. As a result simplify the various execution functions in the `Executor` trait as well as branches for emitting json messages. Instead throw everything in the same bucket and just always look for lines that start with `{` which indicate a JSON message. This also fixes a few issues where output printed in each thread is now routed through the main coordinator thread to handle updating the progress bar if necessary. --- src/cargo/core/compiler/custom_build.rs | 86 ++++++++-------- src/cargo/core/compiler/job_queue.rs | 25 ++--- src/cargo/core/compiler/mod.rs | 128 +++++++++--------------- src/cargo/util/machine_message.rs | 4 +- tests/testsuite/build_script.rs | 4 +- 5 files changed, 107 insertions(+), 140 deletions(-) diff --git a/src/cargo/core/compiler/custom_build.rs b/src/cargo/core/compiler/custom_build.rs index 8f04085491f..6dfddfa22b7 100644 --- a/src/cargo/core/compiler/custom_build.rs +++ b/src/cargo/core/compiler/custom_build.rs @@ -5,6 +5,7 @@ use std::path::{Path, PathBuf}; use std::str; use std::sync::{Arc, Mutex}; +use crate::core::compiler::job_queue::JobState; use crate::core::PackageId; use crate::util::errors::{CargoResult, CargoResultExt}; use crate::util::machine_message; @@ -96,20 +97,21 @@ pub fn prepare<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRe } } -fn emit_build_output(output: &BuildOutput, package_id: PackageId) { +fn emit_build_output(state: &JobState<'_>, output: &BuildOutput, package_id: PackageId) { let library_paths = output .library_paths .iter() .map(|l| l.display().to_string()) .collect::>(); - machine_message::emit(&machine_message::BuildScript { + let msg = machine_message::emit(&machine_message::BuildScript { package_id, linked_libs: &output.library_links, linked_paths: &library_paths, cfgs: &output.cfgs, env: &output.env, }); + state.stdout(&msg); } fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult { @@ -299,52 +301,58 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes } } - // And now finally, run the build command itself! if build_plan { state.build_plan(invocation_name, cmd.clone(), Arc::new(Vec::new())); - } else { - state.running(&cmd); - let timestamp = paths::set_invocation_time(&script_run_dir)?; - let output = if extra_verbose { - let prefix = format!("[{} {}] ", id.name(), id.version()); - state.capture_output(&cmd, Some(prefix), true) - } else { - cmd.exec_with_output() - }; - let output = output.map_err(|e| { - failure::format_err!( - "failed to run custom build command for `{}`\n{}", - pkg_name, - e - ) - })?; + return Ok(()); + } - // After the build command has finished running, we need to be sure to - // remember all of its output so we can later discover precisely what it - // was, even if we don't run the build command again (due to freshness). - // - // This is also the location where we provide feedback into the build - // state informing what variables were discovered via our script as - // well. - paths::write(&output_file, &output.stdout)?; - filetime::set_file_times(output_file, timestamp, timestamp)?; - paths::write(&err_file, &output.stderr)?; - paths::write(&root_output_file, util::path2bytes(&script_out_dir)?)?; - let parsed_output = - BuildOutput::parse(&output.stdout, &pkg_name, &script_out_dir, &script_out_dir)?; - - if json_messages { - emit_build_output(&parsed_output, id); - } - build_state.insert(id, kind, parsed_output); + // And now finally, run the build command itself! + state.running(&cmd); + let timestamp = paths::set_invocation_time(&script_run_dir)?; + let prefix = format!("[{} {}] ", id.name(), id.version()); + let output = cmd + .exec_with_streaming( + &mut |stdout| { + if extra_verbose { + state.stdout(&format!("{}{}", prefix, stdout)); + } + Ok(()) + }, + &mut |stderr| { + if extra_verbose { + state.stderr(&format!("{}{}", prefix, stderr)); + } + Ok(()) + }, + true, + ) + .chain_err(|| format!("failed to run custom build command for `{}`", pkg_name))?; + + // After the build command has finished running, we need to be sure to + // remember all of its output so we can later discover precisely what it + // was, even if we don't run the build command again (due to freshness). + // + // This is also the location where we provide feedback into the build + // state informing what variables were discovered via our script as + // well. + paths::write(&output_file, &output.stdout)?; + filetime::set_file_times(output_file, timestamp, timestamp)?; + paths::write(&err_file, &output.stderr)?; + paths::write(&root_output_file, util::path2bytes(&script_out_dir)?)?; + let parsed_output = + BuildOutput::parse(&output.stdout, &pkg_name, &script_out_dir, &script_out_dir)?; + + if json_messages { + emit_build_output(state, &parsed_output, id); } + build_state.insert(id, kind, parsed_output); Ok(()) }); // Now that we've prepared our work-to-do, we need to prepare the fresh work // itself to run when we actually end up just discarding what we calculated // above. - let fresh = Work::new(move |_tx| { + let fresh = Work::new(move |state| { let (id, pkg_name, build_state, output_file, script_out_dir) = all; let output = match prev_output { Some(output) => output, @@ -357,7 +365,7 @@ fn build_work<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoRes }; if json_messages { - emit_build_output(&output, id); + emit_build_output(state, &output, id); } build_state.insert(id, kind, output); diff --git a/src/cargo/core/compiler/job_queue.rs b/src/cargo/core/compiler/job_queue.rs index 4bdda90bb9b..f86ad3d58ca 100644 --- a/src/cargo/core/compiler/job_queue.rs +++ b/src/cargo/core/compiler/job_queue.rs @@ -2,7 +2,6 @@ use std::collections::{HashMap, HashSet}; use std::cell::Cell; use std::io; use std::marker; -use std::process::Output; use std::sync::mpsc::{channel, Receiver, Sender}; use std::sync::Arc; @@ -106,24 +105,12 @@ impl<'a> JobState<'a> { .send(Message::BuildPlanMsg(module_name, cmd, filenames)); } - pub fn capture_output( - &self, - cmd: &ProcessBuilder, - prefix: Option, - capture_output: bool, - ) -> CargoResult { - let prefix = prefix.unwrap_or_else(String::new); - cmd.exec_with_streaming( - &mut |out| { - let _ = self.tx.send(Message::Stdout(format!("{}{}", prefix, out))); - Ok(()) - }, - &mut |err| { - let _ = self.tx.send(Message::Stderr(format!("{}{}", prefix, err))); - Ok(()) - }, - capture_output, - ) + pub fn stdout(&self, stdout: &str) { + drop(self.tx.send(Message::Stdout(stdout.to_string()))); + } + + pub fn stderr(&self, stderr: &str) { + drop(self.tx.send(Message::Stderr(stderr.to_string()))); } /// A method used to signal to the coordinator thread that the rmeta file diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index a2a024e0ffd..c6f992a3d2a 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -14,7 +14,6 @@ mod unit; use std::env; use std::ffi::{OsStr, OsString}; use std::fs; -use std::io::{self, Write}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -31,7 +30,7 @@ pub use self::context::Context; pub use self::custom_build::{BuildMap, BuildOutput, BuildScripts}; pub use self::job::Freshness; use self::job::{Job, Work}; -use self::job_queue::JobQueue; +use self::job_queue::{JobQueue, JobState}; pub use self::layout::is_bad_artifact_name; use self::output_depinfo::output_depinfo; pub use crate::core::compiler::unit::{Unit, UnitInterner}; @@ -65,40 +64,14 @@ pub trait Executor: Send + Sync + 'static { /// In case of an `Err`, Cargo will not continue with the build process for /// this package. fn exec( - &self, - cmd: ProcessBuilder, - _id: PackageId, - _target: &Target, - _mode: CompileMode, - ) -> CargoResult<()> { - cmd.exec()?; - Ok(()) - } - - fn exec_and_capture_output( &self, cmd: ProcessBuilder, id: PackageId, target: &Target, mode: CompileMode, - _state: &job_queue::JobState<'_>, - ) -> CargoResult<()> { - // We forward to `exec()` to keep RLS working. - self.exec(cmd, id, target, mode) - } - - fn exec_json( - &self, - cmd: ProcessBuilder, - _id: PackageId, - _target: &Target, - _mode: CompileMode, - handle_stdout: &mut dyn FnMut(&str) -> CargoResult<()>, - handle_stderr: &mut dyn FnMut(&str) -> CargoResult<()>, - ) -> CargoResult<()> { - cmd.exec_with_streaming(handle_stdout, handle_stderr, false)?; - Ok(()) - } + on_stdout_line: &mut dyn FnMut(&str) -> CargoResult<()>, + on_stderr_line: &mut dyn FnMut(&str) -> CargoResult<()>, + ) -> CargoResult<()>; /// Queried when queuing each unit of work. If it returns true, then the /// unit will always be rebuilt, independent of whether it needs to be. @@ -113,15 +86,17 @@ pub trait Executor: Send + Sync + 'static { pub struct DefaultExecutor; impl Executor for DefaultExecutor { - fn exec_and_capture_output( + fn exec( &self, cmd: ProcessBuilder, _id: PackageId, _target: &Target, _mode: CompileMode, - state: &job_queue::JobState<'_>, + on_stdout_line: &mut dyn FnMut(&str) -> CargoResult<()>, + on_stderr_line: &mut dyn FnMut(&str) -> CargoResult<()>, ) -> CargoResult<()> { - state.capture_output(&cmd, None, false).map(drop) + cmd.exec_with_streaming(on_stdout_line, on_stderr_line, false) + .map(drop) } } @@ -226,7 +201,6 @@ fn rustc<'a, 'cfg>( let dep_info_loc = fingerprint::dep_info_loc(cx, unit); rustc.args(cx.bcx.rustflags_args(unit)); - let json_messages = cx.bcx.build_config.json_messages(); let package_id = unit.pkg.package_id(); let target = unit.target.clone(); let mode = unit.mode; @@ -294,23 +268,19 @@ fn rustc<'a, 'cfg>( state.running(&rustc); let timestamp = paths::set_invocation_time(&fingerprint_dir)?; - if json_messages { - exec.exec_json( + if build_plan { + state.build_plan(buildkey, rustc.clone(), outputs.clone()); + } else { + exec.exec( rustc, package_id, &target, mode, - &mut assert_is_empty, - &mut |line| json_stderr(line, package_id, &target), + &mut |line| on_stdout_line(state, line, package_id, &target), + &mut |line| on_stderr_line(state, line, package_id, &target), ) .map_err(internal_if_simple_exit_code) .chain_err(|| format!("Could not compile `{}`.", name))?; - } else if build_plan { - state.build_plan(buildkey, rustc.clone(), outputs.clone()); - } else { - exec.exec_and_capture_output(rustc, package_id, &target, mode, state) - .map_err(internal_if_simple_exit_code) - .chain_err(|| format!("Could not compile `{}`.", name))?; } // FIXME(rust-lang/rust#58465): this is the whole point of "pipelined @@ -447,7 +417,7 @@ fn link_targets<'a, 'cfg>( target.set_src_path(TargetSourcePath::Path(path)); } - Ok(Work::new(move |_| { + Ok(Work::new(move |state| { // If we're a "root crate", e.g., the target of this compilation, then we // hard link our outputs out of the `deps` directory into the directory // above. This means that `cargo build` will produce binaries in @@ -488,7 +458,7 @@ fn link_targets<'a, 'cfg>( test: unit_mode.is_any_test(), }; - machine_message::emit(&machine_message::Artifact { + let msg = machine_message::emit(&machine_message::Artifact { package_id, target: &target, profile: art_profile, @@ -497,6 +467,7 @@ fn link_targets<'a, 'cfg>( executable, fresh, }); + state.stdout(&msg); } Ok(()) })) @@ -658,7 +629,7 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult rustdoc.arg("--cfg").arg(&format!("feature=\"{}\"", feat)); } - add_error_format(bcx, &mut rustdoc); + add_error_format(cx, &mut rustdoc); if let Some(args) = bcx.extra_args_for(unit) { rustdoc.args(args); @@ -671,7 +642,6 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult let name = unit.pkg.name().to_string(); let build_state = cx.build_state.clone(); let key = (unit.pkg.package_id(), unit.kind); - let json_messages = bcx.build_config.json_messages(); let package_id = unit.pkg.package_id(); let target = unit.target.clone(); @@ -686,18 +656,13 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult } state.running(&rustdoc); - let exec_result = if json_messages { - rustdoc - .exec_with_streaming( - &mut assert_is_empty, - &mut |line| json_stderr(line, package_id, &target), - false, - ) - .map(drop) - } else { - state.capture_output(&rustdoc, None, false).map(drop) - }; - exec_result.chain_err(|| format!("Could not document `{}`.", name))?; + rustdoc + .exec_with_streaming( + &mut |line| on_stdout_line(state, line, package_id, &target), + &mut |line| on_stderr_line(state, line, package_id, &target), + false, + ) + .chain_err(|| format!("Could not document `{}`.", name))?; Ok(()) })) } @@ -760,8 +725,8 @@ fn add_color(bcx: &BuildContext<'_, '_>, cmd: &mut ProcessBuilder) { cmd.args(&["--color", color]); } -fn add_error_format(bcx: &BuildContext<'_, '_>, cmd: &mut ProcessBuilder) { - match bcx.build_config.message_format { +fn add_error_format(cx: &Context<'_, '_>, cmd: &mut ProcessBuilder) { + match cx.bcx.build_config.message_format { MessageFormat::Human => (), MessageFormat::Json => { cmd.arg("--error-format").arg("json"); @@ -799,7 +764,7 @@ fn build_base_args<'a, 'cfg>( add_path_args(bcx, unit, cmd); add_color(bcx, cmd); - add_error_format(bcx, cmd); + add_error_format(cx, cmd); if !test { for crate_type in crate_types.iter() { @@ -1074,32 +1039,37 @@ impl Kind { } } -fn assert_is_empty(line: &str) -> CargoResult<()> { - if !line.is_empty() { - Err(internal(&format!( - "compiler stdout is not empty: `{}`", - line - ))) - } else { - Ok(()) - } +fn on_stdout_line( + state: &JobState<'_>, + line: &str, + _package_id: PackageId, + _target: &Target, +) -> CargoResult<()> { + state.stdout(line); + Ok(()) } -fn json_stderr(line: &str, package_id: PackageId, target: &Target) -> CargoResult<()> { - // Stderr from rustc/rustdoc can have a mix of JSON and non-JSON output. +fn on_stderr_line( + state: &JobState<'_>, + line: &str, + package_id: PackageId, + target: &Target, +) -> CargoResult<()> { + // Switch json lines from rustc/rustdoc that appear on stderr to instead. + // We want the stdout of Cargo to always be machine parseable as stderr has + // our colorized human-readable messages. if line.starts_with('{') { - // Handle JSON lines. let compiler_message = serde_json::from_str(line) .map_err(|_| internal(&format!("compiler produced invalid json: `{}`", line)))?; - machine_message::emit(&machine_message::FromCompiler { + let msg = machine_message::emit(&machine_message::FromCompiler { package_id, target, message: compiler_message, }); + state.stdout(&msg); } else { - // Forward non-JSON to stderr. - writeln!(io::stderr(), "{}", line)?; + state.stderr(line); } Ok(()) } diff --git a/src/cargo/util/machine_message.rs b/src/cargo/util/machine_message.rs index a1dad7dd6bc..1495a4034d6 100644 --- a/src/cargo/util/machine_message.rs +++ b/src/cargo/util/machine_message.rs @@ -10,11 +10,11 @@ pub trait Message: ser::Serialize { fn reason(&self) -> &str; } -pub fn emit(t: &T) { +pub fn emit(t: &T) -> String { let json = serde_json::to_string(t).unwrap(); assert!(json.starts_with("{\"")); let reason = json!(t.reason()); - println!("{{\"reason\":{},{}", reason, &json[1..]); + format!("{{\"reason\":{},{}", reason, &json[1..]) } #[derive(Serialize)] diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index a59cb59038f..2846be2645b 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -35,7 +35,9 @@ fn custom_build_script_failed() { [RUNNING] `rustc --crate-name build_script_build build.rs --color never --crate-type bin [..]` [RUNNING] `[..]/build-script-build` [ERROR] failed to run custom build command for `foo v0.5.0 ([CWD])` -process didn't exit successfully: `[..]/build-script-build` (exit code: 101)", + +Caused by: + process didn't exit successfully: `[..]/build-script-build` (exit code: 101)", ) .run(); } From 569e973a6ce002bede0fab94b35e564f3473f387 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 30 Apr 2019 11:30:49 -0700 Subject: [PATCH 03/12] Route rendered JSON diagnostics from rustc through Cargo This commit updates Cargo's behavior when it is printing "human readable" error messages when paired with pipelined compilation. In this mode Cargo needs to switch rustc into JSON mode to get a JSON directive message when metadata is ready, but we still want to produce human readable error messages if they crop up. As a result parse a bit of JSON coming out of rustc, if any, and ensure that everything is configured correctly along the way. --- src/cargo/core/compiler/mod.rs | 98 ++++++++++++++++++++++++++++------ 1 file changed, 82 insertions(+), 16 deletions(-) diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index c6f992a3d2a..1540126ca50 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -17,7 +17,7 @@ use std::fs; use std::path::{Path, PathBuf}; use std::sync::Arc; -use failure::Error; +use failure::{bail, Error}; use log::debug; use same_file::is_same_file; use serde::Serialize; @@ -217,6 +217,43 @@ fn rustc<'a, 'cfg>( let fingerprint_dir = cx.files().fingerprint_dir(unit); let rmeta_produced = cx.rmeta_required(unit); + // If this unit is producing a required rmeta file then we need to know + // when the rmeta file is ready so we can signal to the rest of Cargo that + // it can continue dependant compilations. To do this we are currently + // required to switch the compiler into JSON message mode, but we still + // want to present human readable errors as well. (this rabbit hole just + // goes and goes) + // + // All that means is that if we're not already in JSON mode we need to + // switch to JSON mode, ensure that rustc error messages can be rendered + // prettily, and then when parsing JSON messages from rustc we need to + // internally understand that we should extract the `rendered` field and + // present it if we can. + let extract_rendered_errors = if rmeta_produced { + match cx.bcx.build_config.message_format { + MessageFormat::Json => false, + MessageFormat::Human => { + rustc + .arg("--error-format=json") + .arg("--json-rendered=termcolor") + .arg("-Zunstable-options"); + true + } + + // FIXME(rust-lang/rust#60419): right now we have no way of turning + // on JSON messages from the compiler and also asking the rendered + // field to be in the `short` format. + MessageFormat::Short => { + bail!( + "currently `--message-format short` is incompatible with \ + pipelined compilation" + ); + } + } + } else { + false + }; + return Ok(Work::new(move |state| { // Only at runtime have we discovered what the extra -L and -l // arguments are for native libraries, so we process those here. We @@ -277,7 +314,9 @@ fn rustc<'a, 'cfg>( &target, mode, &mut |line| on_stdout_line(state, line, package_id, &target), - &mut |line| on_stderr_line(state, line, package_id, &target), + &mut |line| { + on_stderr_line(state, line, package_id, &target, extract_rendered_errors) + }, ) .map_err(internal_if_simple_exit_code) .chain_err(|| format!("Could not compile `{}`.", name))?; @@ -659,7 +698,7 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult rustdoc .exec_with_streaming( &mut |line| on_stdout_line(state, line, package_id, &target), - &mut |line| on_stderr_line(state, line, package_id, &target), + &mut |line| on_stderr_line(state, line, package_id, &target, false), false, ) .chain_err(|| format!("Could not document `{}`.", name))?; @@ -1054,22 +1093,49 @@ fn on_stderr_line( line: &str, package_id: PackageId, target: &Target, + extract_rendered_errors: bool, ) -> CargoResult<()> { + // We primarily want to use this function to process JSON messages from + // rustc. The compiler should always print one JSON message per line, and + // otherwise it may have other output intermingled (think RUST_LOG or + // something like that), so skip over everything that doesn't look like a + // JSON message. + if !line.starts_with('{') { + state.stderr(line); + return Ok(()); + } + + let compiler_message: Box = serde_json::from_str(line) + .map_err(|_| internal(&format!("compiler produced invalid json: `{}`", line)))?; + + // In some modes of compilation Cargo switches the compiler to JSON mode but + // the user didn't request that so we still want to print pretty rustc + // colorized errors. In those cases (`extract_rendered_errors`) we take a + // look at the JSON blob we go, see if it's a relevant diagnostics, and if + // so forward just that diagnostic for us to print. + if extract_rendered_errors { + #[derive(serde::Deserialize)] + struct CompilerError { + rendered: String, + } + if let Ok(error) = serde_json::from_str::(compiler_message.get()) { + state.stderr(&error.rendered); + return Ok(()); + } + } + + // And failing all that above we should have a legitimate JSON diagnostic + // from the compiler, so wrap it in an external Cargo JSON message + // indicating which package it came from and then emit it. + let msg = machine_message::emit(&machine_message::FromCompiler { + package_id, + target, + message: compiler_message, + }); + // Switch json lines from rustc/rustdoc that appear on stderr to instead. // We want the stdout of Cargo to always be machine parseable as stderr has // our colorized human-readable messages. - if line.starts_with('{') { - let compiler_message = serde_json::from_str(line) - .map_err(|_| internal(&format!("compiler produced invalid json: `{}`", line)))?; - - let msg = machine_message::emit(&machine_message::FromCompiler { - package_id, - target, - message: compiler_message, - }); - state.stdout(&msg); - } else { - state.stderr(line); - } + state.stdout(&msg); Ok(()) } From 4617f78dbcd4030577880b7e4de6bd9a87de1781 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 30 Apr 2019 11:37:24 -0700 Subject: [PATCH 04/12] Parse rmeta directives coming from rustc This commit updates Cargo to process JSON directives emitted by rustc when we're pipelining compilation. In this mode Cargo will attempt to start subsequent compilations of crates as soon as possible, fully completing the features of pipelined compilations in Cargo! --- src/cargo/core/compiler/mod.rs | 62 +++++++++++++++++++++------------ tests/testsuite/build.rs | 4 +++ tests/testsuite/build_script.rs | 3 +- 3 files changed, 44 insertions(+), 25 deletions(-) diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 1540126ca50..3a5bcd208d3 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -231,12 +231,16 @@ fn rustc<'a, 'cfg>( // present it if we can. let extract_rendered_errors = if rmeta_produced { match cx.bcx.build_config.message_format { - MessageFormat::Json => false, + MessageFormat::Json => { + rustc.arg("-Zemit-directives"); + false + } MessageFormat::Human => { rustc .arg("--error-format=json") .arg("--json-rendered=termcolor") - .arg("-Zunstable-options"); + .arg("-Zunstable-options") + .arg("-Zemit-directives"); true } @@ -315,32 +319,20 @@ fn rustc<'a, 'cfg>( mode, &mut |line| on_stdout_line(state, line, package_id, &target), &mut |line| { - on_stderr_line(state, line, package_id, &target, extract_rendered_errors) + on_stderr_line( + state, + line, + package_id, + &target, + extract_rendered_errors, + rmeta_produced, + ) }, ) .map_err(internal_if_simple_exit_code) .chain_err(|| format!("Could not compile `{}`.", name))?; } - // FIXME(rust-lang/rust#58465): this is the whole point of "pipelined - // compilation" in Cargo. We want to, here in this unit, call - // `finish_rmeta` as soon as we can which indicates that the metadata - // file is emitted by rustc and ready to go. This will start dependency - // compilations as soon as possible. - // - // The compiler doesn't currently actually implement the ability to let - // us know, however, when the metadata file is ready to go. It actually - // today produces the file very late in compilation, far later than it - // would otherwise be able to do! - // - // In any case this is all covered by the issue above. This is just a - // marker for "yes we unconditionally do this today but tomorrow we - // should actually read what rustc is doing and execute this at an - // appropriate time, ideally long before rustc finishes completely". - if rmeta_produced { - state.rmeta_produced(); - } - if do_rename && real_name != crate_name { let dst = &outputs[0].path; let src = dst.with_file_name( @@ -698,7 +690,7 @@ fn rustdoc<'a, 'cfg>(cx: &mut Context<'a, 'cfg>, unit: &Unit<'a>) -> CargoResult rustdoc .exec_with_streaming( &mut |line| on_stdout_line(state, line, package_id, &target), - &mut |line| on_stderr_line(state, line, package_id, &target, false), + &mut |line| on_stderr_line(state, line, package_id, &target, false, false), false, ) .chain_err(|| format!("Could not document `{}`.", name))?; @@ -1094,6 +1086,7 @@ fn on_stderr_line( package_id: PackageId, target: &Target, extract_rendered_errors: bool, + look_for_metadata_directive: bool, ) -> CargoResult<()> { // We primarily want to use this function to process JSON messages from // rustc. The compiler should always print one JSON message per line, and @@ -1124,6 +1117,29 @@ fn on_stderr_line( } } + // In some modes of execution we will execute rustc with `-Z + // emit-directives` to look for metadata files being produced. When this + // happens we may be able to start subsequent compilations more quickly than + // waiting for an entire compile to finish, possibly using more parallelism + // available to complete a compilation session more quickly. + // + // In these cases look for a matching directive and inform Cargo internally + // that a metadata file has been produced. + if look_for_metadata_directive { + #[derive(serde::Deserialize)] + struct CompilerDirective { + directive: String, + } + if let Ok(directive) = serde_json::from_str::(compiler_message.get()) { + log::trace!("found directive from rustc: `{}`", directive.directive); + if directive.directive.starts_with("metadata file written") { + log::debug!("looks like metadata finished early!"); + state.rmeta_produced(); + return Ok(()) + } + } + } + // And failing all that above we should have a legitimate JSON diagnostic // from the compiler, so wrap it in an external Cargo JSON message // indicating which package it came from and then emit it. diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index d69e1a72cd0..99ae673cbaa 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -4572,6 +4572,10 @@ Caused by: #[test] fn tricky_pipelining() { + if !crate::support::is_nightly() { + return; + } + let foo = project() .file( "Cargo.toml", diff --git a/tests/testsuite/build_script.rs b/tests/testsuite/build_script.rs index 2846be2645b..91528d92286 100644 --- a/tests/testsuite/build_script.rs +++ b/tests/testsuite/build_script.rs @@ -2219,7 +2219,6 @@ fn diamond_passes_args_only_once() { .build(); p.cargo("build -v") - .env("CARGO_BUILD_PIPELINING", "true") .with_stderr( "\ [COMPILING] c v0.5.0 ([..] @@ -2231,7 +2230,7 @@ fn diamond_passes_args_only_once() { [COMPILING] a v0.5.0 ([..] [RUNNING] `rustc [..]` [COMPILING] foo v0.5.0 ([..] -[RUNNING] `[..]rmeta -L native=test` +[RUNNING] `[..]rlib -L native=test` [FINISHED] dev [unoptimized + debuginfo] target(s) in [..] ", ) From 205bab90099a002c9d80d299c13591779ae4bd98 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 08:10:11 -0700 Subject: [PATCH 05/12] Remove machine_message::emit It's not actually emitting any more! --- src/cargo/core/compiler/custom_build.rs | 6 ++--- src/cargo/core/compiler/mod.rs | 35 +++++++++++++------------ src/cargo/util/machine_message.rs | 12 ++++----- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/cargo/core/compiler/custom_build.rs b/src/cargo/core/compiler/custom_build.rs index 6dfddfa22b7..15f824ce1ea 100644 --- a/src/cargo/core/compiler/custom_build.rs +++ b/src/cargo/core/compiler/custom_build.rs @@ -8,7 +8,7 @@ use std::sync::{Arc, Mutex}; use crate::core::compiler::job_queue::JobState; use crate::core::PackageId; use crate::util::errors::{CargoResult, CargoResultExt}; -use crate::util::machine_message; +use crate::util::machine_message::{self, Message}; use crate::util::Cfg; use crate::util::{self, internal, paths, profile}; @@ -104,13 +104,13 @@ fn emit_build_output(state: &JobState<'_>, output: &BuildOutput, package_id: Pac .map(|l| l.display().to_string()) .collect::>(); - let msg = machine_message::emit(&machine_message::BuildScript { + let msg = machine_message::BuildScript { package_id, linked_libs: &output.library_links, linked_paths: &library_paths, cfgs: &output.cfgs, env: &output.env, - }); + }.to_json_string(); state.stdout(&msg); } diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 3a5bcd208d3..da0e166265b 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -42,6 +42,7 @@ use crate::util::errors::{CargoResult, CargoResultExt, Internal, ProcessError}; use crate::util::paths; use crate::util::{self, machine_message, process, ProcessBuilder}; use crate::util::{internal, join_paths, profile}; +use crate::util::machine_message::Message; /// Indicates whether an object is for the host architcture or the target architecture. /// @@ -489,7 +490,7 @@ fn link_targets<'a, 'cfg>( test: unit_mode.is_any_test(), }; - let msg = machine_message::emit(&machine_message::Artifact { + let msg = machine_message::Artifact { package_id, target: &target, profile: art_profile, @@ -497,7 +498,7 @@ fn link_targets<'a, 'cfg>( filenames: destinations, executable, fresh, - }); + }.to_json_string(); state.stdout(&msg); } Ok(()) @@ -808,7 +809,7 @@ fn build_base_args<'a, 'cfg>( } else if !unit.target.requires_upstream_objects() { // Always produce metdata files for rlib outputs. Metadata may be used // in this session for a pipelined compilation, or it may be used in a - // future Cargo session as part of a pipelined compile.c + // future Cargo session as part of a pipelined compile. cmd.arg("--emit=dep-info,metadata,link"); } else { cmd.arg("--emit=dep-info,link"); @@ -1085,7 +1086,7 @@ fn on_stderr_line( line: &str, package_id: PackageId, target: &Target, - extract_rendered_errors: bool, + extract_rendered_messages: bool, look_for_metadata_directive: bool, ) -> CargoResult<()> { // We primarily want to use this function to process JSON messages from @@ -1101,17 +1102,17 @@ fn on_stderr_line( let compiler_message: Box = serde_json::from_str(line) .map_err(|_| internal(&format!("compiler produced invalid json: `{}`", line)))?; - // In some modes of compilation Cargo switches the compiler to JSON mode but - // the user didn't request that so we still want to print pretty rustc - // colorized errors. In those cases (`extract_rendered_errors`) we take a - // look at the JSON blob we go, see if it's a relevant diagnostics, and if - // so forward just that diagnostic for us to print. - if extract_rendered_errors { + // In some modes of compilation Cargo switches the compiler to JSON mode + // but the user didn't request that so we still want to print pretty rustc + // colorized diagnostics. In those cases (`extract_rendered_messages`) we + // take a look at the JSON blob we go, see if it's a relevant diagnostics, + // and if so forward just that diagnostic for us to print. + if extract_rendered_messages { #[derive(serde::Deserialize)] - struct CompilerError { + struct CompilerMessage { rendered: String, } - if let Ok(error) = serde_json::from_str::(compiler_message.get()) { + if let Ok(error) = serde_json::from_str::(compiler_message.get()) { state.stderr(&error.rendered); return Ok(()); } @@ -1143,15 +1144,15 @@ fn on_stderr_line( // And failing all that above we should have a legitimate JSON diagnostic // from the compiler, so wrap it in an external Cargo JSON message // indicating which package it came from and then emit it. - let msg = machine_message::emit(&machine_message::FromCompiler { + let msg = machine_message::FromCompiler { package_id, target, message: compiler_message, - }); + }.to_json_string(); - // Switch json lines from rustc/rustdoc that appear on stderr to instead. - // We want the stdout of Cargo to always be machine parseable as stderr has - // our colorized human-readable messages. + // Switch json lines from rustc/rustdoc that appear on stderr to stdout + // instead. We want the stdout of Cargo to always be machine parseable as + // stderr has our colorized human-readable messages. state.stdout(&msg); Ok(()) } diff --git a/src/cargo/util/machine_message.rs b/src/cargo/util/machine_message.rs index 1495a4034d6..b52e3b4302e 100644 --- a/src/cargo/util/machine_message.rs +++ b/src/cargo/util/machine_message.rs @@ -8,13 +8,13 @@ use crate::core::{PackageId, Target}; pub trait Message: ser::Serialize { fn reason(&self) -> &str; -} -pub fn emit(t: &T) -> String { - let json = serde_json::to_string(t).unwrap(); - assert!(json.starts_with("{\"")); - let reason = json!(t.reason()); - format!("{{\"reason\":{},{}", reason, &json[1..]) + fn to_json_string(&self) -> String { + let json = serde_json::to_string(self).unwrap(); + assert!(json.starts_with("{\"")); + let reason = json!(self.reason()); + format!("{{\"reason\":{},{}", reason, &json[1..]) + } } #[derive(Serialize)] From 03a72aacf3d95a36690e83ed726dbcdc7e830ecc Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 08:42:41 -0700 Subject: [PATCH 06/12] Fix progress bar being eaten on last packages The return value of `DependencyQueue::len` changed, so we need to account for that here as well! --- src/cargo/core/compiler/job_queue.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cargo/core/compiler/job_queue.rs b/src/cargo/core/compiler/job_queue.rs index f86ad3d58ca..9df4e1e6da8 100644 --- a/src/cargo/core/compiler/job_queue.rs +++ b/src/cargo/core/compiler/job_queue.rs @@ -390,7 +390,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { } fn show_progress(&mut self, total: usize) { - let count = total - self.queue.len(); + let count = total - self.queue.len() - self.active.len(); let active_names = self .active .values() From 95e240448deb49869d8f044dfc4957aec9e34bf6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 08:43:09 -0700 Subject: [PATCH 07/12] Switch to `-Zemit-artifact-notifications` This is the renamed version of `-Zemit-directives`. --- src/cargo/core/compiler/mod.rs | 18 +++++++++--------- tests/testsuite/build.rs | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index da0e166265b..49aa791d336 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -233,7 +233,7 @@ fn rustc<'a, 'cfg>( let extract_rendered_errors = if rmeta_produced { match cx.bcx.build_config.message_format { MessageFormat::Json => { - rustc.arg("-Zemit-directives"); + rustc.arg("-Zemit-artifact-notifications"); false } MessageFormat::Human => { @@ -241,7 +241,7 @@ fn rustc<'a, 'cfg>( .arg("--error-format=json") .arg("--json-rendered=termcolor") .arg("-Zunstable-options") - .arg("-Zemit-directives"); + .arg("-Zemit-artifact-notifications"); true } @@ -1119,7 +1119,7 @@ fn on_stderr_line( } // In some modes of execution we will execute rustc with `-Z - // emit-directives` to look for metadata files being produced. When this + // emit-artifact-notifications` to look for metadata files being produced. When this // happens we may be able to start subsequent compilations more quickly than // waiting for an entire compile to finish, possibly using more parallelism // available to complete a compilation session more quickly. @@ -1128,16 +1128,16 @@ fn on_stderr_line( // that a metadata file has been produced. if look_for_metadata_directive { #[derive(serde::Deserialize)] - struct CompilerDirective { - directive: String, + struct ArtifactNotification { + artifact: String, } - if let Ok(directive) = serde_json::from_str::(compiler_message.get()) { - log::trace!("found directive from rustc: `{}`", directive.directive); - if directive.directive.starts_with("metadata file written") { + if let Ok(artifact) = serde_json::from_str::(compiler_message.get()) { + log::trace!("found directive from rustc: `{}`", artifact.artifact); + if artifact.artifact.ends_with(".rmeta") { log::debug!("looks like metadata finished early!"); state.rmeta_produced(); - return Ok(()) } + return Ok(()) } } diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index 99ae673cbaa..b2b2a65100a 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -4599,3 +4599,36 @@ fn tricky_pipelining() { .env("CARGO_BUILD_PIPELINING", "true") .run(); } + +#[test] +fn pipelining_works() { + if !crate::support::is_nightly() { + return; + } + + let foo = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + [dependencies] + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "extern crate bar;") + .file("bar/Cargo.toml", &basic_lib_manifest("bar")) + .file("bar/src/lib.rs", "") + .build(); + + foo.cargo("build") + .env("CARGO_BUILD_PIPELINING", "true") + .with_stdout("") + .with_stderr("\ +[COMPILING] [..] +[COMPILING] [..] +[FINISHED] [..] +") + .run(); +} From 8f032b3bd4698fad4eb3206cb9d0df92af723634 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 08:51:40 -0700 Subject: [PATCH 08/12] Handle "invalid JSON" from the compiler Just opportunistically consider lines which start with `{` as valid JSON, otherwise forward everything else to normal stdout/stderr. --- src/cargo/core/compiler/mod.rs | 13 ++++++-- tests/testsuite/build.rs | 58 ++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 2 deletions(-) diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 49aa791d336..6e261d8c21b 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -1099,8 +1099,17 @@ fn on_stderr_line( return Ok(()); } - let compiler_message: Box = serde_json::from_str(line) - .map_err(|_| internal(&format!("compiler produced invalid json: `{}`", line)))?; + let compiler_message: Box = match serde_json::from_str(line) { + Ok(msg) => msg, + + // If the compiler produced a line that started with `{` but it wasn't + // valid JSON, maybe it wasn't JSON in the first place! Forward it along + // to stderr. + Err(_) => { + state.stderr(line); + return Ok(()); + } + }; // In some modes of compilation Cargo switches the compiler to JSON mode // but the user didn't request that so we still want to print pretty rustc diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index b2b2a65100a..21f18d6a0fb 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -4632,3 +4632,61 @@ fn pipelining_works() { ") .run(); } + +#[test] +fn forward_rustc_output() { + let foo = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.1.0" + edition = '2018' + [dependencies] + bar = { path = "bar" } + "#, + ) + .file("src/lib.rs", "bar::foo!();") + .file( + "bar/Cargo.toml", + r#" + [package] + name = "bar" + version = "0.1.0" + [lib] + proc-macro = true + "#, + ) + .file( + "bar/src/lib.rs", + r#" + extern crate proc_macro; + use proc_macro::*; + + #[proc_macro] + pub fn foo(input: TokenStream) -> TokenStream { + println!("a"); + println!("b"); + println!("{{}}"); + eprintln!("c"); + eprintln!("d"); + eprintln!("{{a"); // "malformed json" + input + } + "#, + ) + .build(); + + foo.cargo("build") + .with_stdout("a\nb\n{}") + .with_stderr("\ +[COMPILING] [..] +[COMPILING] [..] +c +d +{a +[FINISHED] [..] +") + .run(); +} From c2152f08058d35324e83126fd6d8e7e0c9850441 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 09:00:27 -0700 Subject: [PATCH 09/12] Delete rmeta files for rlibs during `cargo clean` --- src/cargo/ops/cargo_clean.rs | 5 ++++- tests/testsuite/clean.rs | 23 +++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs index 1a8a15cf0ef..55f01a30b3f 100644 --- a/src/cargo/ops/cargo_clean.rs +++ b/src/cargo/ops/cargo_clean.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; -use crate::core::compiler::UnitInterner; +use crate::core::compiler::{UnitInterner, FileFlavor}; use crate::core::compiler::{BuildConfig, BuildContext, CompileMode, Context, Kind}; use crate::core::profiles::UnitFor; use crate::core::Workspace; @@ -119,6 +119,9 @@ pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> { if let Some(ref dst) = output.hardlink { rm_rf(dst, config)?; } + if let FileFlavor::Linkable { rmeta } = &output.flavor { + rm_rf(rmeta, config)?; + } } } diff --git a/tests/testsuite/clean.rs b/tests/testsuite/clean.rs index a0ec3affcfb..8f746962a8c 100644 --- a/tests/testsuite/clean.rs +++ b/tests/testsuite/clean.rs @@ -294,3 +294,26 @@ fn clean_verbose() { .run(); p.cargo("build").run(); } + +#[test] +fn clean_remove_rlib_rmeta() { + let p = project() + .file( + "Cargo.toml", + r#" + [package] + name = "foo" + version = "0.0.1" + "#, + ) + .file("src/lib.rs", "") + .build(); + + p.cargo("build").run(); + assert!(p.target_debug_dir().join("libfoo.rlib").exists()); + let rmeta = p.glob("target/debug/deps/*.rmeta").next().unwrap().unwrap(); + assert!(rmeta.exists()); + p.cargo("clean -p foo").run(); + assert!(!p.target_debug_dir().join("libfoo.rlib").exists()); + assert!(!rmeta.exists()); +} From 6b28a0c0505f19a574a44bffdeccbc07a2d25980 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 11:28:05 -0700 Subject: [PATCH 10/12] Fix fingerprint handling in pipelining mode This commit fixes an issue when pipelining mode is used in handling recompilations. Previously a sequence of compilations could look like: * Crate A starts to build * Crate A produces metadata * Crate B, which depends on A, starts * Crate B finishes * Crate A finishes In this case the mtime for B is before that of A, which fooled Cargo into thinking that B needed to be recompiled. In this case, however, B doesn't actually need to be recompiled because it only depends on the metadata of A, not the final artifacts. This unfortunately resulted in some duplication in a few places, but not really much moreso than already exists between fingerprinting and compilation. --- .../compiler/build_context/target_info.rs | 2 +- .../compiler/context/compilation_files.rs | 16 +++- src/cargo/core/compiler/fingerprint.rs | 94 ++++++++++++------- src/cargo/core/compiler/mod.rs | 53 +++++++---- src/cargo/ops/cargo_clean.rs | 5 +- tests/testsuite/build.rs | 78 ++------------- tests/testsuite/build_plan.rs | 6 +- tests/testsuite/check.rs | 4 +- tests/testsuite/clean.rs | 1 + tests/testsuite/metabuild.rs | 15 ++- 10 files changed, 134 insertions(+), 140 deletions(-) diff --git a/src/cargo/core/compiler/build_context/target_info.rs b/src/cargo/core/compiler/build_context/target_info.rs index a30ec19dc16..d8a013f3ee5 100644 --- a/src/cargo/core/compiler/build_context/target_info.rs +++ b/src/cargo/core/compiler/build_context/target_info.rs @@ -25,7 +25,7 @@ pub enum FileFlavor { /// Not a special file type. Normal, /// Something you can link against (e.g., a library). - Linkable { rmeta: PathBuf }, + Linkable { rmeta: bool }, /// Piece of external debug information (e.g., `.dSYM`/`.pdb` file). DebugInfo, } diff --git a/src/cargo/core/compiler/context/compilation_files.rs b/src/cargo/core/compiler/context/compilation_files.rs index 7a618263770..b079307cd36 100644 --- a/src/cargo/core/compiler/context/compilation_files.rs +++ b/src/cargo/core/compiler/context/compilation_files.rs @@ -305,10 +305,10 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> { // for both libraries and binaries. let path = out_dir.join(format!("lib{}.rmeta", file_stem)); ret.push(OutputFile { - path: path.clone(), + path, hardlink: None, export_path: None, - flavor: FileFlavor::Linkable { rmeta: path }, + flavor: FileFlavor::Linkable { rmeta: false }, }); } else { let mut add = |crate_type: &str, flavor: FileFlavor| -> CargoResult<()> { @@ -372,13 +372,21 @@ impl<'a, 'cfg: 'a> CompilationFiles<'a, 'cfg> { add( kind.crate_type(), if kind.linkable() { - let rmeta = out_dir.join(format!("lib{}.rmeta", file_stem)); - FileFlavor::Linkable { rmeta } + FileFlavor::Linkable { rmeta: false } } else { FileFlavor::Normal }, )?; } + let path = out_dir.join(format!("lib{}.rmeta", file_stem)); + if !unit.target.requires_upstream_objects() { + ret.push(OutputFile { + path, + hardlink: None, + export_path: None, + flavor: FileFlavor::Linkable { rmeta: true }, + }); + } } } } diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index ebad436df4e..30e8a9a9dbb 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -187,6 +187,7 @@ //! See the `A-rebuild-detection` flag on the issue tracker for more: //! +use std::collections::HashMap; use std::env; use std::fs; use std::hash::{self, Hasher}; @@ -322,6 +323,7 @@ struct DepFingerprint { pkg_id: u64, name: String, public: bool, + only_requires_rmeta: bool, fingerprint: Arc, } @@ -395,17 +397,15 @@ enum FsStatus { /// unit needs to subsequently be recompiled. Stale, - /// This unit is up-to-date, it does not need to be recompiled. If there are - /// any outputs then the `FileTime` listed here is the minimum of all their - /// mtimes. This is then later used to see if a unit is newer than one of - /// its dependants, causing the dependant to be recompiled. - UpToDate(Option), + /// This unit is up-to-date. All outputs and their corresponding mtime are + /// listed in the payload here for other dependencies to compare against. + UpToDate { mtimes: HashMap }, } impl FsStatus { fn up_to_date(&self) -> bool { match self { - FsStatus::UpToDate(_) => true, + FsStatus::UpToDate { .. } => true, FsStatus::Stale => false, } } @@ -442,6 +442,7 @@ impl<'de> Deserialize<'de> for DepFingerprint { pkg_id, name, public, + only_requires_rmeta: false, fingerprint: Arc::new(Fingerprint { memoized_hash: Mutex::new(Some(hash)), ..Fingerprint::new() @@ -753,51 +754,71 @@ impl Fingerprint { ) -> CargoResult<()> { assert!(!self.fs_status.up_to_date()); + let mut mtimes = HashMap::new(); + // Get the `mtime` of all outputs. Optionally update their mtime // afterwards based on the `mtime_on_use` flag. Afterwards we want the // minimum mtime as it's the one we'll be comparing to inputs and // dependencies. - let status = self - .outputs - .iter() - .map(|f| { - let mtime = paths::mtime(f).ok(); - if mtime_on_use { - let t = FileTime::from_system_time(SystemTime::now()); - drop(filetime::set_file_times(f, t, t)); + for output in self.outputs.iter() { + let mtime = match paths::mtime(output) { + Ok(mtime) => mtime, + + // This path failed to report its `mtime`. It probably doesn't + // exists, so leave ourselves as stale and bail out. + Err(e) => { + log::debug!("failed to get mtime of {:?}: {}", output, e); + return Ok(()); } - mtime - }) - .min(); + }; + if mtime_on_use { + let t = FileTime::from_system_time(SystemTime::now()); + filetime::set_file_times(output, t, t)?; + } + assert!(mtimes.insert(output.clone(), mtime).is_none()); + } + + let max_mtime = match mtimes.values().max() { + Some(mtime) => mtime, - let mtime = match status { // We had no output files. This means we're an overridden build // script and we're just always up to date because we aren't // watching the filesystem. None => { - self.fs_status = FsStatus::UpToDate(None); + self.fs_status = FsStatus::UpToDate { mtimes }; return Ok(()); } - - // At least one path failed to report its `mtime`. It probably - // doesn't exists, so leave ourselves as stale and bail out. - Some(None) => return Ok(()), - - // All files successfully reported an `mtime`, and we've got the - // minimum one, so let's keep going with that. - Some(Some(mtime)) => mtime, }; for dep in self.deps.iter() { - let dep_mtime = match dep.fingerprint.fs_status { + let dep_mtimes = match &dep.fingerprint.fs_status { + FsStatus::UpToDate { mtimes } => mtimes, // If our dependency is stale, so are we, so bail out. FsStatus::Stale => return Ok(()), + }; - // If our dependencies is up to date and has no filesystem - // interactions, then we can move on to the next dependency. - FsStatus::UpToDate(None) => continue, - - FsStatus::UpToDate(Some(mtime)) => mtime, + // If our dependency edge only requires the rmeta fiel to be present + // then we only need to look at that one output file, otherwise we + // need to consider all output files to see if we're out of date. + let dep_mtime = if dep.only_requires_rmeta { + dep_mtimes + .iter() + .filter_map(|(path, mtime)| { + if path.extension().and_then(|s| s.to_str()) == Some("rmeta") { + Some(mtime) + } else { + None + } + }) + .next() + .expect("failed to find rmeta") + } else { + match dep_mtimes.values().max() { + Some(mtime) => mtime, + // If our dependencies is up to date and has no filesystem + // interactions, then we can move on to the next dependency. + None => continue, + } }; // If the dependency is newer than our own output then it was @@ -807,7 +828,8 @@ impl Fingerprint { // Note that this comparison should probably be `>=`, not `>`, but // for a discussion of why it's `>` see the discussion about #5918 // below in `find_stale`. - if dep_mtime > mtime { + if dep_mtime > max_mtime { + log::info!("dependency on `{}` is newer than we are", dep.name); return Ok(()); } } @@ -824,7 +846,7 @@ impl Fingerprint { } // Everything was up to date! Record such. - self.fs_status = FsStatus::UpToDate(Some(mtime)); + self.fs_status = FsStatus::UpToDate { mtimes }; Ok(()) } @@ -856,6 +878,7 @@ impl hash::Hash for Fingerprint { name, public, fingerprint, + only_requires_rmeta: _, } in deps { pkg_id.hash(h); @@ -929,6 +952,7 @@ impl DepFingerprint { name, public, fingerprint, + only_requires_rmeta: cx.only_requires_rmeta(parent, dep), }) } } diff --git a/src/cargo/core/compiler/mod.rs b/src/cargo/core/compiler/mod.rs index 6e261d8c21b..9bc5d707dea 100644 --- a/src/cargo/core/compiler/mod.rs +++ b/src/cargo/core/compiler/mod.rs @@ -39,10 +39,10 @@ use crate::core::profiles::{Lto, PanicStrategy, Profile}; use crate::core::Feature; use crate::core::{PackageId, Target}; use crate::util::errors::{CargoResult, CargoResultExt, Internal, ProcessError}; +use crate::util::machine_message::Message; use crate::util::paths; use crate::util::{self, machine_message, process, ProcessBuilder}; use crate::util::{internal, join_paths, profile}; -use crate::util::machine_message::Message; /// Indicates whether an object is for the host architcture or the target architecture. /// @@ -498,7 +498,8 @@ fn link_targets<'a, 'cfg>( filenames: destinations, executable, fresh, - }.to_json_string(); + } + .to_json_string(); state.stdout(&msg); } Ok(()) @@ -1016,20 +1017,14 @@ fn build_deps_args<'a, 'cfg>( need_unstable_opts: &mut bool, ) -> CargoResult<()> { let bcx = cx.bcx; - for output in cx.outputs(dep)?.iter() { - let rmeta = match &output.flavor { - FileFlavor::Linkable { rmeta } => rmeta, - _ => continue, - }; - let mut v = OsString::new(); - let name = bcx.extern_crate_name(current, dep)?; - v.push(name); - v.push("="); - if cx.only_requires_rmeta(current, dep) { - v.push(&rmeta); - } else { - v.push(&output.path); - } + + let mut value = OsString::new(); + value.push(bcx.extern_crate_name(current, dep)?); + value.push("="); + + let mut pass = |file| { + let mut value = value.clone(); + value.push(file); if current .pkg @@ -1045,7 +1040,26 @@ fn build_deps_args<'a, 'cfg>( cmd.arg("--extern"); } - cmd.arg(&v); + cmd.arg(&value); + }; + + let outputs = cx.outputs(dep)?; + let mut outputs = outputs.iter().filter_map(|output| match output.flavor { + FileFlavor::Linkable { rmeta } => Some((output, rmeta)), + _ => None, + }); + + if cx.only_requires_rmeta(current, dep) { + let (output, _rmeta) = outputs + .find(|(_output, rmeta)| *rmeta) + .expect("failed to find rlib dep for pipelined dep"); + pass(&output.path); + } else { + for (output, rmeta) in outputs { + if !rmeta { + pass(&output.path); + } + } } Ok(()) } @@ -1146,7 +1160,7 @@ fn on_stderr_line( log::debug!("looks like metadata finished early!"); state.rmeta_produced(); } - return Ok(()) + return Ok(()); } } @@ -1157,7 +1171,8 @@ fn on_stderr_line( package_id, target, message: compiler_message, - }.to_json_string(); + } + .to_json_string(); // Switch json lines from rustc/rustdoc that appear on stderr to stdout // instead. We want the stdout of Cargo to always be machine parseable as diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs index 55f01a30b3f..1a8a15cf0ef 100644 --- a/src/cargo/ops/cargo_clean.rs +++ b/src/cargo/ops/cargo_clean.rs @@ -2,7 +2,7 @@ use std::collections::HashMap; use std::fs; use std::path::Path; -use crate::core::compiler::{UnitInterner, FileFlavor}; +use crate::core::compiler::UnitInterner; use crate::core::compiler::{BuildConfig, BuildContext, CompileMode, Context, Kind}; use crate::core::profiles::UnitFor; use crate::core::Workspace; @@ -119,9 +119,6 @@ pub fn clean(ws: &Workspace<'_>, opts: &CleanOptions<'_>) -> CargoResult<()> { if let Some(ref dst) = output.hardlink { rm_rf(dst, config)?; } - if let FileFlavor::Linkable { rmeta } = &output.flavor { - rm_rf(rmeta, config)?; - } } } diff --git a/tests/testsuite/build.rs b/tests/testsuite/build.rs index 21f18d6a0fb..e849895c0bc 100644 --- a/tests/testsuite/build.rs +++ b/tests/testsuite/build.rs @@ -3101,7 +3101,10 @@ fn compiler_json_error_format() { "name":"bar", "src_path":"[..]lib.rs" }, - "filenames":["[..].rlib"], + "filenames":[ + "[..].rlib", + "[..].rmeta" + ], "fresh": false } @@ -3200,7 +3203,10 @@ fn compiler_json_error_format() { "name":"bar", "src_path":"[..]lib.rs" }, - "filenames":["[..].rlib"], + "filenames":[ + "[..].rlib", + "[..].rmeta" + ], "fresh": true } @@ -4502,74 +4508,6 @@ Caused by: .run(); } -#[test] -fn json_parse_fail() { - // Ensure when JSON parsing fails, and rustc exits with non-zero exit - // code, a useful error message is displayed. - let foo = project() - .file( - "Cargo.toml", - r#" - [package] - name = "foo" - version = "0.1.0" - [dependencies] - pm = { path = "pm" } - "#, - ) - .file( - "src/lib.rs", - r#" - #[macro_use] - extern crate pm; - - #[derive(Foo)] - pub struct S; - "#, - ) - .file( - "pm/Cargo.toml", - r#" - [package] - name = "pm" - version = "0.1.0" - [lib] - proc-macro = true - "#, - ) - .file( - "pm/src/lib.rs", - r#" - extern crate proc_macro; - use proc_macro::TokenStream; - - #[proc_macro_derive(Foo)] - pub fn derive(_input: TokenStream) -> TokenStream { - eprintln!("{{evil proc macro}}"); - panic!("something went wrong"); - } - "#, - ) - .build(); - - foo.cargo("build --message-format=json") - .with_stderr( - "\ -[COMPILING] pm [..] -[COMPILING] foo [..] -[ERROR] Could not compile `foo`. - -Caused by: - compiler produced invalid json: `{evil proc macro}` - -Caused by: - failed to parse process output: `rustc [..] -", - ) - .with_status(101) - .run(); -} - #[test] fn tricky_pipelining() { if !crate::support::is_nightly() { diff --git a/tests/testsuite/build_plan.rs b/tests/testsuite/build_plan.rs index 4fa915a8491..4b53af0440d 100644 --- a/tests/testsuite/build_plan.rs +++ b/tests/testsuite/build_plan.rs @@ -85,7 +85,8 @@ fn cargo_build_plan_single_dep() { "kind": "Host", "links": "{...}", "outputs": [ - "[..]/foo/target/debug/deps/libbar-[..].rlib" + "[..]/foo/target/debug/deps/libbar-[..].rlib", + "[..]/foo/target/debug/deps/libbar-[..].rmeta" ], "package_name": "bar", "package_version": "0.0.1", @@ -101,7 +102,8 @@ fn cargo_build_plan_single_dep() { "kind": "Host", "links": "{...}", "outputs": [ - "[..]/foo/target/debug/deps/libfoo-[..].rlib" + "[..]/foo/target/debug/deps/libfoo-[..].rlib", + "[..]/foo/target/debug/deps/libfoo-[..].rmeta" ], "package_name": "foo", "package_version": "0.5.0", diff --git a/tests/testsuite/check.rs b/tests/testsuite/check.rs index 24c61b3635c..b64ac2b8c20 100644 --- a/tests/testsuite/check.rs +++ b/tests/testsuite/check.rs @@ -190,8 +190,8 @@ fn build_check() { .file("src/lib.rs", "pub fn baz() {}") .build(); - foo.cargo("build").run(); - foo.cargo("check").run(); + foo.cargo("build -v").run(); + foo.cargo("check -v").run(); } // Checks that where a project has both a lib and a bin, the lib is only checked diff --git a/tests/testsuite/clean.rs b/tests/testsuite/clean.rs index 8f746962a8c..01a4d2826b9 100644 --- a/tests/testsuite/clean.rs +++ b/tests/testsuite/clean.rs @@ -289,6 +289,7 @@ fn clean_verbose() { "\ [REMOVING] [..] [REMOVING] [..] +[REMOVING] [..] ", ) .run(); diff --git a/tests/testsuite/metabuild.rs b/tests/testsuite/metabuild.rs index 6d3d741d361..7bc2d696859 100644 --- a/tests/testsuite/metabuild.rs +++ b/tests/testsuite/metabuild.rs @@ -461,7 +461,10 @@ fn metabuild_build_plan() { "compile_mode": "build", "kind": "Host", "deps": [], - "outputs": ["[..]/target/debug/deps/libmb-[..].rlib"], + "outputs": [ + "[..]/target/debug/deps/libmb-[..].rlib", + "[..]/target/debug/deps/libmb-[..].rmeta" + ], "links": {}, "program": "rustc", "args": "{...}", @@ -475,7 +478,10 @@ fn metabuild_build_plan() { "compile_mode": "build", "kind": "Host", "deps": [], - "outputs": ["[..]/target/debug/deps/libmb_other-[..].rlib"], + "outputs": [ + "[..]/target/debug/deps/libmb_other-[..].rlib", + "[..]/target/debug/deps/libmb_other-[..].rmeta" + ], "links": {}, "program": "rustc", "args": "{...}", @@ -517,7 +523,10 @@ fn metabuild_build_plan() { "compile_mode": "build", "kind": "Host", "deps": [3], - "outputs": ["[..]/foo/target/debug/deps/libfoo-[..].rlib"], + "outputs": [ + "[..]/foo/target/debug/deps/libfoo-[..].rlib", + "[..]/foo/target/debug/deps/libfoo-[..].rmeta" + ], "links": "{...}", "program": "rustc", "args": "{...}", From 7892472588a5572252d4e01325ce6d727f6725d6 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 8 May 2019 11:49:50 -0700 Subject: [PATCH 11/12] Handle transitive `All` dependency edges We need to synthesize some dependency edges in the dependency graph to get everything ordered correctly! (more details in the commit itself) --- src/cargo/core/compiler/fingerprint.rs | 24 +++++++--- src/cargo/core/compiler/job_queue.rs | 64 ++++++++++++++++++++++---- 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/src/cargo/core/compiler/fingerprint.rs b/src/cargo/core/compiler/fingerprint.rs index 30e8a9a9dbb..9a625887aba 100644 --- a/src/cargo/core/compiler/fingerprint.rs +++ b/src/cargo/core/compiler/fingerprint.rs @@ -313,17 +313,24 @@ pub fn prepare_target<'a, 'cfg>( Ok(Job::new(write_fingerprint, Dirty)) } -/// A compilation unit dependency has a fingerprint that is comprised of: -/// * its package ID -/// * its extern crate name -/// * its public/private status -/// * its calculated fingerprint for the dependency +/// Dependency edge information for fingerprints. This is generated for each +/// unit in `dep_targets` and is stored in a `Fingerprint` below. #[derive(Clone)] struct DepFingerprint { + /// The hash of the package id that this dependency points to pkg_id: u64, + /// The crate name we're using for this dependency, which if we change we'll + /// need to recompile! name: String, + /// Whether or not this dependency is flagged as a public dependency or not. public: bool, + /// Whether or not this dependency is an rmeta dependency or a "full" + /// dependency. In the case of an rmeta dependency our dependency edge only + /// actually requires the rmeta from what we depend on, so when checking + /// mtime information all files other than the rmeta can be ignored. only_requires_rmeta: bool, + /// The dependency's fingerprint we recursively point to, containing all the + /// other hash information we'd otherwise need. fingerprint: Arc, } @@ -442,11 +449,14 @@ impl<'de> Deserialize<'de> for DepFingerprint { pkg_id, name, public, - only_requires_rmeta: false, fingerprint: Arc::new(Fingerprint { memoized_hash: Mutex::new(Some(hash)), ..Fingerprint::new() }), + // This field is never read since it's only used in + // `check_filesystem` which isn't used by fingerprints loaded from + // disk. + only_requires_rmeta: false, }) } } @@ -878,7 +888,7 @@ impl hash::Hash for Fingerprint { name, public, fingerprint, - only_requires_rmeta: _, + only_requires_rmeta: _, // static property, no need to hash } in deps { pkg_id.hash(h); diff --git a/src/cargo/core/compiler/job_queue.rs b/src/cargo/core/compiler/job_queue.rs index 9df4e1e6da8..bff2d5b6aa5 100644 --- a/src/cargo/core/compiler/job_queue.rs +++ b/src/cargo/core/compiler/job_queue.rs @@ -1,5 +1,5 @@ -use std::collections::{HashMap, HashSet}; use std::cell::Cell; +use std::collections::{HashMap, HashSet}; use std::io; use std::marker; use std::sync::mpsc::{channel, Receiver, Sender}; @@ -152,7 +152,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { job: Job, ) -> CargoResult<()> { let dependencies = cx.dep_targets(unit); - let dependencies = dependencies + let mut queue_deps = dependencies .iter() .filter(|unit| { // Binaries aren't actually needed to *compile* tests, just to run @@ -171,10 +171,52 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { Artifact::All }; (dep, artifact) - }); - self.queue.queue(*unit, job, dependencies); + }) + .collect::>(); + + // This is somewhat tricky, but we may need to synthesize some + // dependencies for this target if it requires full upstream + // compilations to have completed. If we're in pipelining mode then some + // dependency edges may be `Metadata` due to the above clause (as + // opposed to everything being `All`). For example consider: + // + // a (binary) + // └ b (lib) + // └ c (lib) + // + // Here the dependency edge from B to C will be `Metadata`, and the + // dependency edge from A to B will be `All`. For A to be compiled, + // however, it currently actually needs the full rlib of C. This means + // that we need to synthesize a dependency edge for the dependency graph + // from A to C. That's done here. + // + // This will walk all dependencies of the current target, and if any of + // *their* dependencies are `Metadata` then we depend on the `All` of + // the target as well. This should ensure that edges changed to + // `Metadata` propagate upwards `All` dependencies to anything that + // transitively contains the `Metadata` edge. + if unit.target.requires_upstream_objects() { + for dep in dependencies.iter() { + depend_on_deps_of_deps(cx, &mut queue_deps, dep); + } + + fn depend_on_deps_of_deps<'a>( + cx: &Context<'a, '_>, + deps: &mut Vec<(Unit<'a>, Artifact)>, + unit: &Unit<'a>, + ) { + for dep in cx.dep_targets(unit) { + if cx.only_requires_rmeta(unit, &dep) { + deps.push((dep, Artifact::All)); + depend_on_deps_of_deps(cx, deps, &dep); + } + } + } + } + + self.queue.queue(*unit, job, queue_deps); *self.counts.entry(unit.pkg.package_id()).or_insert(0) += 1; - Ok(()) + return Ok(()); } /// Executes all jobs necessary to build the dependency graph. @@ -430,7 +472,6 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { cx: &Context<'a, '_>, scope: &Scope<'a>, ) -> CargoResult<()> { - let id = self.next_id; self.next_id = id.checked_add(1).unwrap(); @@ -464,7 +505,9 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { // we need to make sure that the metadata is flagged as produced so // send a synthetic message here. if state.rmeta_required.get() && res.is_ok() { - my_tx.send(Message::Finish(id, Artifact::Metadata, Ok(()))).unwrap(); + my_tx + .send(Message::Finish(id, Artifact::Metadata, Ok(()))) + .unwrap(); } my_tx.send(Message::Finish(id, Artifact::All, res)).unwrap(); @@ -513,7 +556,12 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { Ok(()) } - fn finish(&mut self, unit: &Unit<'a>, artifact: Artifact, cx: &mut Context<'_, '_>) -> CargoResult<()> { + fn finish( + &mut self, + unit: &Unit<'a>, + artifact: Artifact, + cx: &mut Context<'_, '_>, + ) -> CargoResult<()> { if unit.mode.is_run_custom_build() && cx.bcx.show_warnings(unit.pkg.package_id()) { self.emit_warnings(None, unit, cx)?; } From dcd7c4898510a15bf52654e0241908b1593b28ad Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 10 May 2019 06:48:49 -0700 Subject: [PATCH 12/12] Keep better track of finished crates Instead of juggling a few counters let's just keep an explicit counter of finished packages. --- src/cargo/core/compiler/job_queue.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cargo/core/compiler/job_queue.rs b/src/cargo/core/compiler/job_queue.rs index bff2d5b6aa5..693dc01b603 100644 --- a/src/cargo/core/compiler/job_queue.rs +++ b/src/cargo/core/compiler/job_queue.rs @@ -283,6 +283,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { // and then immediately return. let mut error = None; let total = self.queue.len(); + let mut finished = 0; loop { // Dequeue as much work as we can, learning about everything // possible that can run. Note that this is also the point where we @@ -320,7 +321,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { // unnecessarily. let events: Vec<_> = self.rx.try_iter().collect(); let events = if events.is_empty() { - self.show_progress(total); + self.show_progress(finished, total); vec![self.rx.recv().unwrap()] } else { events @@ -355,6 +356,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { // from the `active` map ... Artifact::All => { info!("end: {:?}", id); + finished += 1; self.active.remove(&id).unwrap() } // ... otherwise if it hasn't finished we leave it @@ -431,8 +433,7 @@ impl<'a, 'cfg> JobQueue<'a, 'cfg> { } } - fn show_progress(&mut self, total: usize) { - let count = total - self.queue.len() - self.active.len(); + fn show_progress(&mut self, count: usize, total: usize) { let active_names = self .active .values()