[BrdigingHeader] Auto bridging header chaining

Add ability to automatically chaining the bridging headers discovered from all
dependencies module when doing swift caching build. This will eliminate all
implicit bridging header imports from the build and make the bridging header
importing behavior much more reliable, while keep the compatibility at maximum.

For example, if the current module A depends on module B and C, and both B and
C are binary modules that uses bridging header, when building module A,
dependency scanner will construct a new header that chains three bridging
headers together with the option to build a PCH from it. This will make all
importing errors more obvious while improving the performance.

Author: cachemeifyoucan

include/swift/AST/ModuleDependencies.h

@@ -30,6 +30,7 @@
 #include "llvm/ADT/DenseSet.h"
 #include "llvm/ADT/IntrusiveRefCntPtr.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/CAS/ActionCache.h"
 #include "llvm/CAS/CASProvidingFileSystem.h"
 #include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/CachingOnDiskFileSystem.h"
@@ -786,11 +787,11 @@ class ModuleDependencyInfo {
   setLinkLibraries(const ArrayRef<LinkLibrary> linkLibraries) {
     storage->linkLibraries.assign(linkLibraries.begin(), linkLibraries.end());
   }
-  
+
   const ArrayRef<std::string> getAuxiliaryFiles() const {
     return storage->auxiliaryFiles;
   }
-  
+
   void
   setAuxiliaryFiles(const ArrayRef<std::string> auxiliaryFiles) {
     storage->auxiliaryFiles.assign(auxiliaryFiles.begin(), auxiliaryFiles.end());
@@ -978,7 +979,7 @@ class ModuleDependencyInfo {
   void addSourceFile(StringRef sourceFile);
 
   /// Add source files that the header input depends on.
-  void addHeaderSourceFile(StringRef bridgingSourceFile);
+  void setHeaderSourceFiles(const std::vector<std::string> &sourceFiles);
 
   /// Add bridging header include tree.
   void addBridgingHeaderIncludeTree(StringRef ID);
@@ -1042,8 +1043,9 @@ class SwiftDependencyScanningService {
   /// If use clang include tree.
   bool UseClangIncludeTree = false;
 
-  /// CAS ObjectStore Instance.
+  /// CAS Instance.
   std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
 
   /// File prefix mapper.
   std::unique_ptr<llvm::PrefixMapper> Mapper;
@@ -1196,11 +1198,11 @@ class ModuleDependenciesCache {
   /// Query all dependencies
   ModuleDependencyIDSetVector
   getAllDependencies(const ModuleDependencyID &moduleID) const;
-  
+
   /// Query all Clang module dependencies.
   ModuleDependencyIDSetVector
   getClangDependencies(const ModuleDependencyID &moduleID) const;
-  
+
   /// Query all directly-imported Swift dependencies
   llvm::ArrayRef<ModuleDependencyID>
   getImportedSwiftDependencies(const ModuleDependencyID &moduleID) const;

include/swift/AST/SearchPathOptions.h

@@ -580,6 +580,9 @@ class SearchPathOptions {
   /// "in-package", must not require package-only module dependencies.
   bool ResolveInPackageModuleDependencies = false;
 
+  /// Enable auto bridging header chaining.
+  bool BridgingHeaderChaining = false;
+
   /// Return all module search paths that (non-recursively) contain a file whose
   /// name is in \p Filenames.
   SmallVector<const ModuleSearchPath *, 4>

include/swift/Basic/LangOptions.h

@@ -201,7 +201,7 @@ namespace swift {
     /// Maximum number of typo corrections we are allowed to perform.
     /// This is disabled by default until we can get typo-correction working within acceptable performance bounds.
     unsigned TypoCorrectionLimit = 0;
-    
+
     /// Should access control be respected?
     bool EnableAccessControl = true;
 
@@ -260,7 +260,7 @@ namespace swift {
 
     /// Emit a remark when indexing a system module.
     bool EnableIndexingSystemModuleRemarks = false;
-    
+
     /// Emit a remark on early exit in explicit interface build
     bool EnableSkipExplicitInterfaceModuleBuildRemarks = false;
 
@@ -691,7 +691,7 @@ namespace swift {
     void clearAllPlatformConditionValues() {
       PlatformConditionValues.clear();
     }
-    
+
     /// Returns the value for the given platform condition or an empty string.
     StringRef getPlatformConditionValue(PlatformConditionKind Kind) const;
 
@@ -871,7 +871,7 @@ namespace swift {
     /// 4.2 GHz Intel Core i7.
     /// (It's arbitrary, but will keep the compiler from taking too much time.)
     unsigned SwitchCheckingInvocationThreshold = 200000;
-    
+
     /// If true, the time it takes to type-check each function will be dumped
     /// to llvm::errs().
     bool DebugTimeFunctionBodies = false;
@@ -915,7 +915,7 @@ namespace swift {
 
     /// Enable experimental operator designated types feature.
     bool EnableOperatorDesignatedTypes = false;
-    
+
     /// Disable constraint system performance hacks.
     bool DisableConstraintSolverPerformanceHacks = false;
 
@@ -963,6 +963,9 @@ namespace swift {
     /// The bridging header or PCH that will be imported.
     std::string BridgingHeader;
 
+    /// The bridging header PCH file.
+    std::string BridgingHeaderPCH;
+
     /// When automatically generating a precompiled header from the bridging
     /// header, place it in this directory.
     std::string PrecompiledHeaderOutputDir;
@@ -1084,6 +1087,9 @@ namespace swift {
     /// compilation source targets.
     std::vector<std::string>
     getReducedExtraArgsForSwiftModuleDependency() const;
+
+    /// Get PCH input path. Return empty string if there is no PCH input.
+    std::string getPCHInputPath() const;
   };
 
 } // end namespace swift

include/swift/ClangImporter/ClangImporter.h

@@ -506,14 +506,16 @@ class ClangImporter final : public ClangModuleLoader {
   /// information will be augmented with information about the given
   /// textual header inputs.
   ///
+  /// \param headerPath the path to the header to be scanned.
+  ///
   /// \param clangScanningTool The clang dependency scanner.
   ///
   /// \param cache The module dependencies cache to update, with information
   /// about new Clang modules discovered along the way.
   ///
   /// \returns \c true if an error occurred, \c false otherwise
   bool getHeaderDependencies(
-      ModuleDependencyID moduleID,
+      ModuleDependencyID moduleID, StringRef headerPath,
       clang::tooling::dependencies::DependencyScanningTool &clangScanningTool,
       ModuleDependenciesCache &cache,
       ModuleDependencyIDSetVector &headerClangModuleDependencies,

include/swift/DependencyScan/ModuleDependencyScanner.h

@@ -16,6 +16,7 @@
 #include "swift/AST/ModuleDependencies.h"
 #include "swift/Frontend/ModuleInterfaceLoader.h"
 #include "swift/Serialization/SerializedModuleLoader.h"
+#include "llvm/CAS/CASReference.h"
 #include "llvm/Support/ThreadPool.h"
 
 namespace swift {
@@ -48,6 +49,11 @@ class ModuleDependencyScanningWorker {
                                          llvm::PrefixMapper *prefixMapper,
                                          bool isTestableImport = false);
 
+  /// Store cache entry for include tree.
+  llvm::Error
+  createCacheKeyForEmbeddedHeader(std::string embeddedHeaderIncludeTree,
+                                  std::string chainedHeaderIncludeTree);
+
   // Worker-specific instance of CompilerInvocation
   std::unique_ptr<CompilerInvocation> workerCompilerInvocation;
   // Worker-specific instance of ASTContext
@@ -59,6 +65,9 @@ class ModuleDependencyScanningWorker {
   // Swift and Clang module loaders acting as scanners.
   std::unique_ptr<ModuleInterfaceLoader> swiftScannerModuleLoader;
   std::unique_ptr<ClangImporter> clangScannerModuleLoader;
+  // CAS instance.
+  std::shared_ptr<llvm::cas::ObjectStore> CAS;
+  std::shared_ptr<llvm::cas::ActionCache> ActionCache;
   // Restrict access to the parent scanner class.
   friend class ModuleDependencyScanner;
 };
@@ -116,10 +125,10 @@ class ModuleDependencyScanner {
   resolveAllClangModuleDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                     ModuleDependenciesCache &cache,
                                     ModuleDependencyIDSetVector &discoveredClangModules);
-  void
-  resolveHeaderDependencies(ArrayRef<ModuleDependencyID> swiftModules,
-                            ModuleDependenciesCache &cache,
-                            ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
+  void resolveHeaderDependencies(
+      const ModuleDependencyID &rootModuleID,
+      ArrayRef<ModuleDependencyID> swiftModules, ModuleDependenciesCache &cache,
+      ModuleDependencyIDSetVector &discoveredHeaderDependencyClangModules);
   void
   resolveSwiftOverlayDependencies(ArrayRef<ModuleDependencyID> swiftModules,
                                   ModuleDependenciesCache &cache,
@@ -151,13 +160,22 @@ class ModuleDependencyScanner {
       ModuleDependenciesCache &cache,
       llvm::function_ref<void(ModuleDependencyID)> action);
 
+  /// Performance BridgingHeader Chaining.
+  llvm::Error
+  performBridgingHeaderChaining(const ModuleDependencyID &rootModuleID,
+                                ModuleDependenciesCache &cache,
+                                ModuleDependencyIDSetVector &allModules);
+
   /// Perform an operation utilizing one of the Scanning workers
   /// available to this scanner.
   template <typename Function, typename... Args>
   auto withDependencyScanningWorker(Function &&F, Args &&...ArgList);
 
   Identifier getModuleImportIdentifier(StringRef moduleName);
 
+  /// Get scanner output file names.
+  std::string getOutputFilename(StringRef suffix);
+
 private:
   const CompilerInvocation &ScanCompilerInvocation;
   ASTContext &ScanASTContext;

include/swift/Frontend/CompileJobCacheKey.h

@@ -21,6 +21,7 @@
 #include "swift/AST/DiagnosticEngine.h"
 #include "swift/Basic/FileTypes.h"
 #include "llvm/ADT/ArrayRef.h"
+#include "llvm/CAS/CASReference.h"
 #include "llvm/CAS/ObjectStore.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/raw_ostream.h"
@@ -49,6 +50,11 @@ createCompileJobCacheKeyForOutput(llvm::cas::ObjectStore &CAS,
 llvm::Error printCompileJobCacheKey(llvm::cas::ObjectStore &CAS,
                                     llvm::cas::ObjectRef Key,
                                     llvm::raw_ostream &os);
+
+/// Create cache key for embedded bridging header.
+llvm::Expected<llvm::cas::ObjectRef> createEmbeddedBridgingHeaderCacheKey(
+    llvm::cas::ObjectStore &CAS, llvm::cas::ObjectRef ChainedPCHIncludeTree);
+
 } // namespace swift
 
 #endif

include/swift/Frontend/FrontendOptions.h

@@ -54,6 +54,9 @@ class FrontendOptions {
   /// An Objective-C header to import and make implicitly visible.
   std::string ImplicitObjCHeaderPath;
 
+  /// An Objective-C pch to import and make implicitly visible.
+  std::string ImplicitObjCPCHPath;
+
   /// The map of aliases and real names of imported or referenced modules.
   llvm::StringMap<StringRef> ModuleAliasMap;
 
@@ -369,6 +372,9 @@ class FrontendOptions {
   /// Emit remarks indicating use of the serialized module dependency scanning cache.
   bool EmitDependencyScannerCacheRemarks = false;
 
+  /// The path at which the dependency scanner can write generated files.
+  std::string ScannerOutputDir;
+
   /// Whether the dependency scanner invocation should resolve imports
   /// to filesystem modules in parallel.
   bool ParallelDependencyScan = true;

include/swift/Option/FrontendOptions.td

@@ -161,12 +161,12 @@ def verify_generic_signatures : Separate<["-"], "verify-generic-signatures">,
 
 def show_diagnostics_after_fatal : Flag<["-"], "show-diagnostics-after-fatal">,
   HelpText<"Keep emitting subsequent diagnostics after a fatal error">;
-  
+
 def enable_cross_import_overlays : Flag<["-"], "enable-cross-import-overlays">,
   HelpText<"Automatically import declared cross-import overlays.">;
 def disable_cross_import_overlays : Flag<["-"], "disable-cross-import-overlays">,
   HelpText<"Do not automatically import declared cross-import overlays.">;
-  
+
 def enable_testable_attr_requires_testable_module :
   Flag<["-"], "enable-testable-attr-requires-testable-module">,
   HelpText<"Enable checking of @testable">;
@@ -186,7 +186,7 @@ def enable_target_os_checking :
 def disable_target_os_checking :
   Flag<["-"], "disable-target-os-checking">,
   HelpText<"Disable checking the target OS of serialized modules">;
-  
+
 def crosscheck_unqualified_lookup : Flag<["-"], "crosscheck-unqualified-lookup">,
   HelpText<"Compare legacy DeclContext- to ASTScope-based unqualified name lookup (for debugging)">;
 
@@ -378,9 +378,9 @@ def public_autolink_library :
 // HIDDEN FLAGS
 let Flags = [FrontendOption, NoDriverOption, HelpHidden] in {
 
-def enable_experimental_swift_based_closure_specialization : 
+def enable_experimental_swift_based_closure_specialization :
   Flag<["-"], "experimental-swift-based-closure-specialization">,
-  HelpText<"Use the experimental Swift based closure-specialization optimization pass instead of the existing C++ one">; 
+  HelpText<"Use the experimental Swift based closure-specialization optimization pass instead of the existing C++ one">;
 
 def checked_async_objc_bridging : Joined<["-"], "checked-async-objc-bridging=">,
   HelpText<"Control whether checked continuations are used when bridging "
@@ -900,7 +900,7 @@ def disable_concrete_type_metadata_mangled_name_accessors : Flag<["-"], "disable
 def disable_standard_substitutions_in_reflection_mangling : Flag<["-"], "disable-standard-substitutions-in-reflection-mangling">,
   HelpText<"Disable referencing stdlib symbols via mangled names in reflection mangling">,
   Flags<[FrontendOption, HelpHidden]>;
-  
+
 def playground : Flag<["-"], "playground">,
   HelpText<"Apply the playground semantics and transformation">;
 
@@ -940,13 +940,13 @@ def sil_unroll_threshold : Separate<["-"], "sil-unroll-threshold">,
 
 def sil_verify_all : Flag<["-"], "sil-verify-all">,
   HelpText<"Verify SIL after each transform">;
-  
+
 def sil_verify_none : Flag<["-"], "sil-verify-none">,
   HelpText<"Completely disable SIL verification">;
 
 def sil_ownership_verify_all : Flag<["-"], "sil-ownership-verify-all">,
   HelpText<"Verify ownership after each transform">;
-  
+
 def verify_all_substitution_maps : Flag<["-"], "verify-all-substitution-maps">,
   HelpText<"Verify all SubstitutionMaps on construction">;
 
@@ -1165,7 +1165,7 @@ def disable_interface_lockfile : Flag<["-"], "disable-interface-lock">,
 def bridging_header_directory_for_print: Separate<["-"], "bridging-header-directory-for-print">, MetaVarName<"<path>">,
     HelpText<"Directory for bridging header to be printed in compatibility header">;
 
-def entry_point_function_name: Separate<["-"], "entry-point-function-name">, MetaVarName<"<string>">, 
+def entry_point_function_name: Separate<["-"], "entry-point-function-name">, MetaVarName<"<string>">,
     HelpText<"Name of the entry point function">;
 
 def target_sdk_version : Separate<["-"], "target-sdk-version">,

include/swift/Option/Options.td

@@ -173,12 +173,12 @@ def verify_incremental_dependencies :
   Flag<["-"], "verify-incremental-dependencies">,
   Flags<[FrontendOption, HelpHidden]>,
   HelpText<"Enable the dependency verifier for each frontend job">;
-  
+
 def strict_implicit_module_context :
   Flag<["-"], "strict-implicit-module-context">,
   Flags<[FrontendOption, HelpHidden]>,
   HelpText<"Enable the strict forwarding of compilation context to downstream implicit module dependencies">;
-  
+
 def no_strict_implicit_module_context :
   Flag<["-"], "no-strict-implicit-module-context">,
   Flags<[FrontendOption, HelpHidden]>,
@@ -315,7 +315,7 @@ def tools_directory : Separate<["-"], "tools-directory">,
 
 def D : JoinedOrSeparate<["-"], "D">, Flags<[FrontendOption]>,
   HelpText<"Marks a conditional compilation flag as true">;
-  
+
 def e : Separate<["-"], "e">, Flags<[NewDriverOnlyOption]>,
   HelpText<"Executes a line of code provided on the command line">;
 
@@ -353,11 +353,22 @@ def import_objc_header : Separate<["-"], "import-objc-header">,
 def import_bridging_header : Separate<["-"], "import-bridging-header">,
   Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
   Alias<import_objc_header>;
+def import_pch : Separate<["-"], "import-pch">,
+  Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
+  HelpText<"Import bridging header PCH file">;
 
 def pch_output_dir: Separate<["-"], "pch-output-dir">,
   Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
   HelpText<"Directory to persist automatically created precompiled bridging headers">;
 
+def scanner_output_dir: Separate<["-"], "scanner-output-dir">,
+  Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
+  HelpText<"Directory to write generated files from swift dependency scanner">;
+
+def experimental_bridging_header_chaining: Flag<["-"], "experimental-bridging-header-chaining">,
+  Flags<[FrontendOption, HelpHidden, ArgumentIsPath]>,
+  HelpText<"Automatically chaining all the bridging headers">;
+
 // FIXME: Unhide this once it doesn't depend on an output file map.
 def incremental : Flag<["-"], "incremental">,
   Flags<[NoInteractiveOption, HelpHidden, DoesNotAffectIncrementalBuild]>,
@@ -2123,7 +2134,7 @@ def external_plugin_path : Separate<["-"], "external-plugin-path">, Group<plugin
   Flags<[FrontendOption, ArgumentIsPath, SwiftSymbolGraphExtractOption, SwiftAPIDigesterOption]>,
   HelpText<"Add directory to the plugin search path with a plugin server executable">,
   MetaVarName<"<path>#<plugin-server-path>">;
-  
+
 def cas_backend: Flag<["-"], "cas-backend">,
   Flags<[FrontendOption, NoDriverOption]>,
   HelpText<"Enable using CASBackend for object file output">;

include/swift/Serialization/SerializationOptions.h

@@ -45,6 +45,8 @@ class SerializationOptions {
 
   StringRef GroupInfoPath;
   StringRef ImportedHeader;
+  StringRef ImportedPCHPath;
+  bool SerializeBridgingHeader = false;
   StringRef ModuleLinkName;
   StringRef ModuleInterface;
   std::vector<std::string> ExtraClangOptions;