Persist ThinLTO import data in incr. comp. session directory.

Author: michaelwoerister

src/librustc_codegen_llvm/back/lto.rs

@@ -20,16 +20,23 @@ use rustc::hir::def_id::LOCAL_CRATE;
 use rustc::middle::exported_symbols::SymbolExportLevel;
 use rustc::session::config::{self, Lto};
 use rustc::util::common::time_ext;
+use rustc_data_structures::fx::FxHashMap;
 use time_graph::Timeline;
 use {ModuleCodegen, ModuleLlvm, ModuleKind, ModuleSource};
 
 use libc;
 
-use std::ffi::CString;
+use std::ffi::{CStr, CString};
+use std::fs::File;
+use std::io;
+use std::mem;
+use std::path::Path;
 use std::ptr;
 use std::slice;
 use std::sync::Arc;
 
+pub const THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME: &str = "thin-lto-imports.bin";
+
 pub fn crate_type_allows_lto(crate_type: config::CrateType) -> bool {
     match crate_type {
         config::CrateType::Executable |
@@ -199,7 +206,7 @@ pub(crate) fn run(cgcx: &CodegenContext,
                 unreachable!("We should never reach this case if the LTO step \
                               is deferred to the linker");
             }
-            thin_lto(&diag_handler, modules, upstream_modules, &arr, timeline)
+            thin_lto(cgcx, &diag_handler, modules, upstream_modules, &arr, timeline)
         }
         Lto::No => unreachable!(),
     }
@@ -362,7 +369,8 @@ impl Drop for Linker<'a> {
 /// calculating the *index* for ThinLTO. This index will then be shared amongst
 /// all of the `LtoModuleCodegen` units returned below and destroyed once
 /// they all go out of scope.
-fn thin_lto(diag_handler: &Handler,
+fn thin_lto(cgcx: &CodegenContext,
+            diag_handler: &Handler,
             modules: Vec<ModuleCodegen>,
             serialized_modules: Vec<(SerializedModule, CString)>,
             symbol_white_list: &[*const libc::c_char],
@@ -439,6 +447,17 @@ fn thin_lto(diag_handler: &Handler,
             write::llvm_err(&diag_handler, "failed to prepare thin LTO context".to_string())
         })?;
 
+        // Save the ThinLTO import information for incremental compilation.
+        if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
+            let path = incr_comp_session_dir.join(THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME);
+            let imports = ThinLTOImports::from_thin_lto_data(data);
+            if let Err(err) = imports.save_to_file(&path) {
+                let msg = format!("Error while writing ThinLTO import data: {}",
+                                  err);
+                return Err(write::llvm_err(&diag_handler, msg));
+            }
+        }
+
         let data = ThinData(data);
         info!("thin LTO data created");
         timeline.record("data");
@@ -776,3 +795,97 @@ impl ThinModule {
         Ok(module)
     }
 }
+
+#[derive(Debug)]
+pub struct ThinLTOImports {
+    // key = llvm name of importing module, value = list of modules it imports from
+    imports: FxHashMap<String, Vec<String>>,
+}
+
+impl ThinLTOImports {
+    pub fn new() -> ThinLTOImports {
+        ThinLTOImports {
+            imports: FxHashMap(),
+        }
+    }
+
+    /// Load the ThinLTO import map from ThinLTOData.
+    unsafe fn from_thin_lto_data(data: *const llvm::ThinLTOData) -> ThinLTOImports {
+        fn module_name_to_str(c_str: &CStr) -> &str {
+            match c_str.to_str() {
+                Ok(s) => s,
+                Err(e) => {
+                    bug!("Encountered non-utf8 LLVM module name `{}`: {}",
+                        c_str.to_string_lossy(),
+                        e)
+                }
+            }
+        }
+        unsafe extern "C" fn imported_module_callback(payload: *mut libc::c_void,
+                                                      importing_module_name: *const libc::c_char,
+                                                      imported_module_name: *const libc::c_char) {
+            let map = &mut* (payload as *mut ThinLTOImports);
+            let importing_module_name = CStr::from_ptr(importing_module_name);
+            let importing_module_name = module_name_to_str(&importing_module_name);
+            let imported_module_name = CStr::from_ptr(imported_module_name);
+            let imported_module_name = module_name_to_str(&imported_module_name);
+            if !map.imports.contains_key(importing_module_name) {
+                map.imports.insert(importing_module_name.to_owned(), vec![]);
+            }
+            map.imports
+               .get_mut(importing_module_name)
+               .unwrap()
+               .push(imported_module_name.to_owned());
+        }
+        let mut map = ThinLTOImports {
+            imports: FxHashMap(),
+        };
+        llvm::LLVMRustGetThinLTOModuleImports(data,
+                                              imported_module_callback,
+                                              &mut map as *mut _ as *mut libc::c_void);
+        map
+    }
+
+    pub fn save_to_file(&self, path: &Path) -> io::Result<()> {
+        use std::io::Write;
+        let file = File::create(path)?;
+        let mut writer = io::BufWriter::new(file);
+        for (importing_module_name, imported_modules) in &self.imports {
+            writeln!(writer, "{}", importing_module_name)?;
+            for imported_module in imported_modules {
+                writeln!(writer, "  {}", imported_module)?;
+            }
+            writeln!(writer)?;
+        }
+        Ok(())
+    }
+
+    pub fn load_from_file(path: &Path) -> io::Result<ThinLTOImports> {
+        use std::io::BufRead;
+        let mut imports = FxHashMap();
+        let mut current_module = None;
+        let mut current_imports = vec![];
+        let file = File::open(path)?;
+        for line in io::BufReader::new(file).lines() {
+            let line = line?;
+            if line.is_empty() {
+                let importing_module = current_module
+                    .take()
+                    .expect("Importing module not set");
+                imports.insert(importing_module,
+                               mem::replace(&mut current_imports, vec![]));
+            } else if line.starts_with(" ") {
+                // This is an imported module
+                assert_ne!(current_module, None);
+                current_imports.push(line.trim().to_string());
+            } else {
+                // This is the beginning of a new module
+                assert_eq!(current_module, None);
+                current_module = Some(line.trim().to_string());
+            }
+        }
+        Ok(ThinLTOImports {
+            imports
+        })
+    }
+}

src/librustc_codegen_llvm/base.rs

@@ -29,6 +29,7 @@ use super::ModuleCodegen;
 use super::ModuleKind;
 
 use abi;
+use back::lto;
 use back::write::{self, OngoingCodegen};
 use llvm::{self, TypeKind, get_param};
 use metadata;
@@ -1314,6 +1315,25 @@ pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility {
     }
 }
 
+#[allow(unused)]
+fn load_thin_lto_imports(sess: &Session) -> lto::ThinLTOImports {
+    let path = rustc_incremental::in_incr_comp_dir_sess(
+        sess,
+        lto::THIN_LTO_IMPORTS_INCR_COMP_FILE_NAME
+    );
+    if !path.exists() {
+        return lto::ThinLTOImports::new();
+    }
+    match lto::ThinLTOImports::load_from_file(&path) {
+        Ok(imports) => imports,
+        Err(e) => {
+            let msg = format!("Error while trying to load ThinLTO import data \
+                               for incremental compilation: {}", e);
+            sess.fatal(&msg)
+        }
+    }
+}
+
 // FIXME(mw): Anything that is produced via DepGraph::with_task() must implement
 //            the HashStable trait. Normally DepGraph::with_task() calls are
 //            hidden behind queries, but CGU creation is a special case in two

src/librustc_codegen_llvm/lib.rs

@@ -100,7 +100,7 @@ mod back {
     mod command;
     pub mod linker;
     pub mod link;
-    mod lto;
+    pub mod lto;
     pub mod symbol_export;
     pub mod write;
     mod rpath;

src/librustc_incremental/lib.rs

@@ -44,6 +44,7 @@ pub use persist::copy_cgu_workproducts_to_incr_comp_cache_dir;
 pub use persist::save_dep_graph;
 pub use persist::save_work_product_index;
 pub use persist::in_incr_comp_dir;
+pub use persist::in_incr_comp_dir_sess;
 pub use persist::prepare_session_directory;
 pub use persist::finalize_session_directory;
 pub use persist::delete_workproduct_files;

src/librustc_incremental/persist/mod.rs

@@ -23,6 +23,7 @@ mod file_format;
 pub use self::fs::finalize_session_directory;
 pub use self::fs::garbage_collect_session_directories;
 pub use self::fs::in_incr_comp_dir;
+pub use self::fs::in_incr_comp_dir_sess;
 pub use self::fs::prepare_session_directory;
 pub use self::load::dep_graph_tcx_init;
 pub use self::load::load_dep_graph;