Auto merge of #6289 - ehuss:no-buffer-output, r=alexcrichton

Avoid retaining all rustc output in memory.

There are still a few cases where all output is buffered. They are:
- Running discovery commands like `rustc -vV`, `rustc --print xxx`, etc.
- Build script output.
- Testsuite's debug `.stream()` function.

This should cover the concern of the issue, though, which is normal compilation.

Closes #6197

Author: bors

src/cargo/core/compiler/job_queue.rs

@@ -113,7 +113,7 @@ impl<'a> JobState<'a> {
         &self,
         cmd: &ProcessBuilder,
         prefix: Option<String>,
-        print_output: bool,
+        capture_output: bool,
     ) -> CargoResult<Output> {
         let prefix = prefix.unwrap_or_else(|| String::new());
         cmd.exec_with_streaming(
@@ -125,7 +125,7 @@ impl<'a> JobState<'a> {
                 let _ = self.tx.send(Message::Stderr(format!("{}{}", prefix, err)));
                 Ok(())
             },
-            print_output,
+            capture_output,
         )
     }
 }

src/cargo/util/process_builder.rs

@@ -202,7 +202,7 @@ impl ProcessBuilder {
         &self,
         on_stdout_line: &mut FnMut(&str) -> CargoResult<()>,
         on_stderr_line: &mut FnMut(&str) -> CargoResult<()>,
-        print_output: bool,
+        capture_output: bool,
     ) -> CargoResult<Output> {
         let mut stdout = Vec::new();
         let mut stderr = Vec::new();
@@ -226,23 +226,33 @@ impl ProcessBuilder {
                         None => return,
                     }
                 };
-                let data = data.drain(..idx);
-                let dst = if is_out { &mut stdout } else { &mut stderr };
-                let start = dst.len();
-                dst.extend(data);
-                for line in String::from_utf8_lossy(&dst[start..]).lines() {
-                    if callback_error.is_some() {
-                        break;
-                    }
-                    let callback_result = if is_out {
-                        on_stdout_line(line)
+                { // scope for new_lines
+                    let new_lines = if capture_output {
+                        let dst = if is_out { &mut stdout } else { &mut stderr };
+                        let start = dst.len();
+                        let data = data.drain(..idx);
+                        dst.extend(data);
+                        &dst[start..]
                     } else {
-                        on_stderr_line(line)
+                        &data[..idx]
                     };
-                    if let Err(e) = callback_result {
-                        callback_error = Some(e);
+                    for line in String::from_utf8_lossy(new_lines).lines() {
+                        if callback_error.is_some() {
+                            break;
+                        }
+                        let callback_result = if is_out {
+                            on_stdout_line(line)
+                        } else {
+                            on_stderr_line(line)
+                        };
+                        if let Err(e) = callback_result {
+                            callback_error = Some(e);
+                        }
                     }
                 }
+                if !capture_output {
+                    data.drain(..idx);
+                }
             })?;
             child.wait()
         })()
@@ -260,7 +270,7 @@ impl ProcessBuilder {
         };
 
         {
-            let to_print = if print_output { Some(&output) } else { None };
+            let to_print = if capture_output { Some(&output) } else { None };
             if !output.status.success() {
                 return Err(process_error(
                     &format!("process didn't exit successfully: {}", self),

tests/testsuite/support/mod.rs

@@ -770,7 +770,7 @@ impl Execs {
             process.exec_with_streaming(
                 &mut |out| Ok(println!("{}", out)),
                 &mut |err| Ok(eprintln!("{}", err)),
-                false,
+                true,
             )
         } else {
             process.exec_with_output()