Auto merge of #45191 - petrochenkov:yesar, r=Mark-Simulacrum

rustbuild: Support specifying archiver and linker explicitly

With this patch `x.py test` passes without toolchain being in `PATH` if `cc`, `cxx`, `ar`, `linker` and `gdb` are specified in `config.toml` (except for a few `run-make` tests using `nm`).

Fixes #41821
r? @Mark-Simulacrum

Author: bors

config.toml.example

@@ -300,7 +300,7 @@
 # =============================================================================
 [target.x86_64-unknown-linux-gnu]
 
-# C compiler to be used to compiler C code and link Rust code. Note that the
+# C compiler to be used to compiler C code. Note that the
 # default value is platform specific, and if not specified it may also depend on
 # what platform is crossing to what platform.
 #cc = "cc"
@@ -309,6 +309,15 @@
 # This is only used for host targets.
 #cxx = "c++"
 
+# Archiver to be used to assemble static libraries compiled from C/C++ code.
+# Note: an absolute path should be used, otherwise LLVM build will break.
+#ar = "ar"
+
+# Linker to be used to link Rust code. Note that the
+# default value is platform specific, and if not specified it may also depend on
+# what platform is crossing to what platform.
+#linker = "cc"
+
 # Path to the `llvm-config` binary of the installation of a custom LLVM to link
 # against. Note that if this is specifed we don't compile LLVM at all for this
 # target.

src/Cargo.lock

@@ -120,10 +120,9 @@ fn main() {
             cmd.arg("-L").arg(&root);
         }
 
-        // Pass down extra flags, commonly used to configure `-Clinker` when
-        // cross compiling.
-        if let Ok(s) = env::var("RUSTC_FLAGS") {
-            cmd.args(&s.split(" ").filter(|s| !s.is_empty()).collect::<Vec<_>>());
+        // Override linker if necessary.
+        if let Ok(target_linker) = env::var("RUSTC_TARGET_LINKER") {
+            cmd.arg(format!("-Clinker={}", target_linker));
         }
 
         // Pass down incremental directory, if any.
@@ -252,6 +251,11 @@ fn main() {
         if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
             cmd.arg("-Z").arg("force-unstable-if-unmarked");
         }
+    } else {
+        // Override linker if necessary.
+        if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") {
+            cmd.arg(format!("-Clinker={}", host_linker));
+        }
     }
 
     let color = match env::var("RUSTC_COLOR") {

src/bootstrap/bin/rustc.rs

@@ -47,6 +47,9 @@ fn main() {
     if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() {
         cmd.arg("-Z").arg("force-unstable-if-unmarked");
     }
+    if let Some(linker) = env::var_os("RUSTC_TARGET_LINKER") {
+        cmd.arg("--linker").arg(linker).arg("-Z").arg("unstable-options");
+    }
 
     // Bootstrap's Cargo-command builder sets this variable to the current Rust version; let's pick
     // it up so we can make rustdoc print this into the docs

src/bootstrap/bin/rustdoc.rs

@@ -413,13 +413,15 @@ impl<'a> Builder<'a> {
     pub fn rustdoc_cmd(&self, host: Interned<String>) -> Command {
         let mut cmd = Command::new(&self.out.join("bootstrap/debug/rustdoc"));
         let compiler = self.compiler(self.top_stage, host);
-        cmd
-            .env("RUSTC_STAGE", compiler.stage.to_string())
-            .env("RUSTC_SYSROOT", self.sysroot(compiler))
-            .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
-            .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
-            .env("RUSTDOC_REAL", self.rustdoc(host))
-            .env("RUSTDOC_CRATE_VERSION", self.build.rust_version());
+        cmd.env("RUSTC_STAGE", compiler.stage.to_string())
+           .env("RUSTC_SYSROOT", self.sysroot(compiler))
+           .env("RUSTC_LIBDIR", self.sysroot_libdir(compiler, self.build.build))
+           .env("CFG_RELEASE_CHANNEL", &self.build.config.channel)
+           .env("RUSTDOC_REAL", self.rustdoc(host))
+           .env("RUSTDOC_CRATE_VERSION", self.build.rust_version());
+        if let Some(linker) = self.build.linker(host) {
+            cmd.env("RUSTC_TARGET_LINKER", linker);
+        }
         cmd
     }
 
@@ -482,8 +484,14 @@ impl<'a> Builder<'a> {
              } else {
                  PathBuf::from("/path/to/nowhere/rustdoc/not/required")
              })
-             .env("TEST_MIRI", self.config.test_miri.to_string())
-             .env("RUSTC_FLAGS", self.rustc_flags(target).join(" "));
+             .env("TEST_MIRI", self.config.test_miri.to_string());
+
+        if let Some(host_linker) = self.build.linker(compiler.host) {
+            cargo.env("RUSTC_HOST_LINKER", host_linker);
+        }
+        if let Some(target_linker) = self.build.linker(target) {
+            cargo.env("RUSTC_TARGET_LINKER", target_linker);
+        }
 
         if mode != Mode::Tool {
             // Tools don't get debuginfo right now, e.g. cargo and rls don't
@@ -557,17 +565,35 @@ impl<'a> Builder<'a> {
 
         cargo.env("RUSTC_VERBOSE", format!("{}", self.verbosity));
 
-        // Specify some various options for build scripts used throughout
-        // the build.
+        // Throughout the build Cargo can execute a number of build scripts
+        // compiling C/C++ code and we need to pass compilers, archivers, flags, etc
+        // obtained previously to those build scripts.
+        // Build scripts use either the `cc` crate or `configure/make` so we pass
+        // the options through environment variables that are fetched and understood by both.
         //
         // FIXME: the guard against msvc shouldn't need to be here
         if !target.contains("msvc") {
-            cargo.env(format!("CC_{}", target), self.cc(target))
-                 .env(format!("AR_{}", target), self.ar(target).unwrap()) // only msvc is None
-                 .env(format!("CFLAGS_{}", target), self.cflags(target).join(" "));
+            let cc = self.cc(target);
+            cargo.env(format!("CC_{}", target), cc)
+                 .env("CC", cc);
+
+            let cflags = self.cflags(target).join(" ");
+            cargo.env(format!("CFLAGS_{}", target), cflags.clone())
+                 .env("CFLAGS", cflags.clone());
+
+            if let Some(ar) = self.ar(target) {
+                let ranlib = format!("{} s", ar.display());
+                cargo.env(format!("AR_{}", target), ar)
+                     .env("AR", ar)
+                     .env(format!("RANLIB_{}", target), ranlib.clone())
+                     .env("RANLIB", ranlib);
+            }
 
             if let Ok(cxx) = self.cxx(target) {
-                 cargo.env(format!("CXX_{}", target), cxx);
+                cargo.env(format!("CXX_{}", target), cxx)
+                     .env("CXX", cxx)
+                     .env(format!("CXXFLAGS_{}", target), cflags.clone())
+                     .env("CXXFLAGS", cflags);
             }
         }
 

src/bootstrap/builder.rs

@@ -31,20 +31,51 @@
 //! ever be probed for. Instead the compilers found here will be used for
 //! everything.
 
+use std::collections::HashSet;
+use std::{env, iter};
+use std::path::{Path, PathBuf};
 use std::process::Command;
-use std::iter;
 
-use build_helper::{cc2ar, output};
+use build_helper::output;
 use cc;
 
 use Build;
 use config::Target;
 use cache::Interned;
 
+// The `cc` crate doesn't provide a way to obtain a path to the detected archiver,
+// so use some simplified logic here. First we respect the environment variable `AR`, then
+// try to infer the archiver path from the C compiler path.
+// In the future this logic should be replaced by calling into the `cc` crate.
+fn cc2ar(cc: &Path, target: &str) -> Option<PathBuf> {
+    if let Some(ar) = env::var_os("AR") {
+        Some(PathBuf::from(ar))
+    } else if target.contains("msvc") {
+        None
+    } else if target.contains("musl") {
+        Some(PathBuf::from("ar"))
+    } else if target.contains("openbsd") {
+        Some(PathBuf::from("ar"))
+    } else {
+        let parent = cc.parent().unwrap();
+        let file = cc.file_name().unwrap().to_str().unwrap();
+        for suffix in &["gcc", "cc", "clang"] {
+            if let Some(idx) = file.rfind(suffix) {
+                let mut file = file[..idx].to_owned();
+                file.push_str("ar");
+                return Some(parent.join(&file));
+            }
+        }
+        Some(parent.join(file))
+    }
+}
+
 pub fn find(build: &mut Build) {
     // For all targets we're going to need a C compiler for building some shims
     // and such as well as for being a linker for Rust code.
-    for target in build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build)) {
+    let targets = build.targets.iter().chain(&build.hosts).cloned().chain(iter::once(build.build))
+                               .collect::<HashSet<_>>();
+    for target in targets.into_iter() {
         let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false)
            .target(&target).host(&build.build);
@@ -57,16 +88,23 @@ pub fn find(build: &mut Build) {
         }
 
         let compiler = cfg.get_compiler();
-        let ar = cc2ar(compiler.path(), &target);
+        let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) {
+            ar
+        } else {
+            cc2ar(compiler.path(), &target)
+        };
+
         build.verbose(&format!("CC_{} = {:?}", &target, compiler.path()));
-        if let Some(ref ar) = ar {
+        build.cc.insert(target, compiler);
+        if let Some(ar) = ar {
             build.verbose(&format!("AR_{} = {:?}", &target, ar));
+            build.ar.insert(target, ar);
         }
-        build.cc.insert(target, (compiler, ar));
     }
 
     // For all host triples we need to find a C++ compiler as well
-    for host in build.hosts.iter().cloned().chain(iter::once(build.build)) {
+    let hosts = build.hosts.iter().cloned().chain(iter::once(build.build)).collect::<HashSet<_>>();
+    for host in hosts.into_iter() {
         let mut cfg = cc::Build::new();
         cfg.cargo_metadata(false).opt_level(0).warnings(false).debug(false).cpp(true)
            .target(&host).host(&build.build);

src/bootstrap/cc_detect.rs

@@ -747,12 +747,14 @@ impl Step for Compiletest {
             flags.push("-g".to_string());
         }
 
-        let mut hostflags = build.rustc_flags(compiler.host);
-        hostflags.extend(flags.clone());
+        if let Some(linker) = build.linker(target) {
+            cmd.arg("--linker").arg(linker);
+        }
+
+        let hostflags = flags.clone();
         cmd.arg("--host-rustcflags").arg(hostflags.join(" "));
 
-        let mut targetflags = build.rustc_flags(target);
-        targetflags.extend(flags);
+        let mut targetflags = flags.clone();
         targetflags.push(format!("-Lnative={}",
                                  build.test_helpers_out(target).display()));
         cmd.arg("--target-rustcflags").arg(targetflags.join(" "));
@@ -806,6 +808,9 @@ impl Step for Compiletest {
                 .arg("--cflags").arg(build.cflags(target).join(" "))
                 .arg("--llvm-components").arg(llvm_components.trim())
                 .arg("--llvm-cxxflags").arg(llvm_cxxflags.trim());
+                if let Some(ar) = build.ar(target) {
+                    cmd.arg("--ar").arg(ar);
+                }
             }
         }
         if suite == "run-make" && !build.config.llvm_enabled {
@@ -831,7 +836,7 @@ impl Step for Compiletest {
         // Note that if we encounter `PATH` we make sure to append to our own `PATH`
         // rather than stomp over it.
         if target.contains("msvc") {
-            for &(ref k, ref v) in build.cc[&target].0.env() {
+            for &(ref k, ref v) in build.cc[&target].env() {
                 if k != "PATH" {
                     cmd.env(k, v);
                 }

src/bootstrap/check.rs

@@ -143,6 +143,8 @@ pub struct Target {
     pub jemalloc: Option<PathBuf>,
     pub cc: Option<PathBuf>,
     pub cxx: Option<PathBuf>,
+    pub ar: Option<PathBuf>,
+    pub linker: Option<PathBuf>,
     pub ndk: Option<PathBuf>,
     pub crt_static: Option<bool>,
     pub musl_root: Option<PathBuf>,
@@ -282,6 +284,8 @@ struct TomlTarget {
     jemalloc: Option<String>,
     cc: Option<String>,
     cxx: Option<String>,
+    ar: Option<String>,
+    linker: Option<String>,
     android_ndk: Option<String>,
     crt_static: Option<bool>,
     musl_root: Option<String>,
@@ -484,8 +488,10 @@ impl Config {
                 if let Some(ref s) = cfg.android_ndk {
                     target.ndk = Some(env::current_dir().unwrap().join(s));
                 }
-                target.cxx = cfg.cxx.clone().map(PathBuf::from);
                 target.cc = cfg.cc.clone().map(PathBuf::from);
+                target.cxx = cfg.cxx.clone().map(PathBuf::from);
+                target.ar = cfg.ar.clone().map(PathBuf::from);
+                target.linker = cfg.linker.clone().map(PathBuf::from);
                 target.crt_static = cfg.crt_static.clone();
                 target.musl_root = cfg.musl_root.clone().map(PathBuf::from);
                 target.qemu_rootfs = cfg.qemu_rootfs.clone().map(PathBuf::from);

src/bootstrap/config.rs

@@ -240,10 +240,11 @@ pub struct Build {
     lldb_python_dir: Option<String>,
 
     // Runtime state filled in later on
-    // target -> (cc, ar)
-    cc: HashMap<Interned<String>, (cc::Tool, Option<PathBuf>)>,
-    // host -> (cc, ar)
+    // C/C++ compilers and archiver for all targets
+    cc: HashMap<Interned<String>, cc::Tool>,
     cxx: HashMap<Interned<String>, cc::Tool>,
+    ar: HashMap<Interned<String>, PathBuf>,
+    // Misc
     crates: HashMap<Interned<String>, Crate>,
     is_sudo: bool,
     ci_env: CiEnv,
@@ -324,6 +325,7 @@ impl Build {
             rls_info,
             cc: HashMap::new(),
             cxx: HashMap::new(),
+            ar: HashMap::new(),
             crates: HashMap::new(),
             lldb_version: None,
             lldb_python_dir: None,
@@ -612,15 +614,15 @@ impl Build {
 
     /// Returns the path to the C compiler for the target specified.
     fn cc(&self, target: Interned<String>) -> &Path {
-        self.cc[&target].0.path()
+        self.cc[&target].path()
     }
 
     /// Returns a list of flags to pass to the C compiler for the target
     /// specified.
     fn cflags(&self, target: Interned<String>) -> Vec<String> {
         // Filter out -O and /O (the optimization flags) that we picked up from
         // cc-rs because the build scripts will determine that for themselves.
-        let mut base = self.cc[&target].0.args().iter()
+        let mut base = self.cc[&target].args().iter()
                            .map(|s| s.to_string_lossy().into_owned())
                            .filter(|s| !s.starts_with("-O") && !s.starts_with("/O"))
                            .collect::<Vec<_>>();
@@ -644,7 +646,7 @@ impl Build {
 
     /// Returns the path to the `ar` archive utility for the target specified.
     fn ar(&self, target: Interned<String>) -> Option<&Path> {
-        self.cc[&target].1.as_ref().map(|p| &**p)
+        self.ar.get(&target).map(|p| &**p)
     }
 
     /// Returns the path to the C++ compiler for the target specified.
@@ -657,21 +659,17 @@ impl Build {
         }
     }
 
-    /// Returns flags to pass to the compiler to generate code for `target`.
-    fn rustc_flags(&self, target: Interned<String>) -> Vec<String> {
-        // New flags should be added here with great caution!
-        //
-        // It's quite unfortunate to **require** flags to generate code for a
-        // target, so it should only be passed here if absolutely necessary!
-        // Most default configuration should be done through target specs rather
-        // than an entry here.
-
-        let mut base = Vec::new();
-        if target != self.config.build && !target.contains("msvc") &&
-            !target.contains("emscripten") {
-            base.push(format!("-Clinker={}", self.cc(target).display()));
+    /// Returns the path to the linker for the given target if it needs to be overriden.
+    fn linker(&self, target: Interned<String>) -> Option<&Path> {
+        if let Some(linker) = self.config.target_config.get(&target)
+                                                       .and_then(|c| c.linker.as_ref()) {
+            Some(linker)
+        } else if target != self.config.build &&
+                  !target.contains("msvc") && !target.contains("emscripten") {
+            Some(self.cc(target))
+        } else {
+            None
         }
-        base
     }
 
     /// Returns if this target should statically link the C runtime, if specified

src/bootstrap/lib.rs

@@ -227,6 +227,13 @@ impl Step for Llvm {
             cfg.build_arg("-j").build_arg(build.jobs().to_string());
             cfg.define("CMAKE_C_FLAGS", build.cflags(target).join(" "));
             cfg.define("CMAKE_CXX_FLAGS", build.cflags(target).join(" "));
+            if let Some(ar) = build.ar(target) {
+                if ar.is_absolute() {
+                    // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it
+                    // tries to resolve this path in the LLVM build directory.
+                    cfg.define("CMAKE_AR", sanitize_cc(ar));
+                }
+            }
         };
 
         configure_compilers(&mut cfg);