Merge pull request swiftlang#78224 from benlangmuir/cherry-pick-140286819-61

[CAS] Cache symbol graph outputs

Author: benlangmuir

include/swift/Basic/FileTypes.def

@@ -98,6 +98,7 @@ TYPE("swiftoverlay",        SwiftOverlayFile,          "swiftoverlay",    "")
 
 // Misc types
 TYPE("pcm",                 ClangModuleFile,           "pcm",             "")
+TYPE("symbol-graph",        SymbolGraphFile,           "symbols.json",    "")
 TYPE("pch",                 PCH,                       "pch",             "")
 TYPE("none",                Nothing,                   "",                "")
 

include/swift/Frontend/CASOutputBackends.h

@@ -46,6 +46,7 @@ class SwiftCASOutputBackend : public llvm::vfs::OutputBackend {
                         llvm::cas::ActionCache &Cache,
                         llvm::cas::ObjectRef BaseKey,
                         const FrontendInputsAndOutputs &InputsAndOutputs,
+                        const FrontendOptions &Opts,
                         FrontendOptions::ActionType Action);
   ~SwiftCASOutputBackend();
 

include/swift/Frontend/CachingUtils.h

@@ -36,7 +36,7 @@ createSwiftCachingOutputBackend(
     llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
     llvm::cas::ObjectRef BaseKey,
     const FrontendInputsAndOutputs &InputsAndOutputs,
-    FrontendOptions::ActionType Action);
+    const FrontendOptions &Opts, FrontendOptions::ActionType Action);
 
 /// Replay the output of the compilation from cache.
 /// Return true if outputs are replayed, false otherwise.

lib/Basic/FileTypes.cpp

@@ -116,6 +116,7 @@ bool file_types::isTextual(ID Id) {
   case file_types::TY_SwiftABIDescriptor:
   case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
+  case file_types::TY_SymbolGraphFile:
     return true;
   case file_types::TY_Image:
   case file_types::TY_Object:
@@ -202,6 +203,7 @@ bool file_types::isAfterLLVM(ID Id) {
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
   case file_types::TY_CachedDiagnostics:
+  case file_types::TY_SymbolGraphFile:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -263,6 +265,7 @@ bool file_types::isPartOfSwiftCompilation(ID Id) {
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
   case file_types::TY_CachedDiagnostics:
+  case file_types::TY_SymbolGraphFile:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");
@@ -324,6 +327,7 @@ bool file_types::isProducedFromDiagnostics(ID Id) {
   case file_types::TY_SwiftAPIDescriptor:
   case file_types::TY_ConstValues:
   case file_types::TY_ModuleSemanticInfo:
+  case file_types::TY_SymbolGraphFile:
     return false;
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID.");

lib/Driver/Driver.cpp

@@ -1708,6 +1708,7 @@ void Driver::buildActions(SmallVectorImpl<const Action *> &TopLevelActions,
       case file_types::TY_SwiftFixIt:
       case file_types::TY_ModuleSemanticInfo:
       case file_types::TY_CachedDiagnostics:
+      case file_types::TY_SymbolGraphFile:
         // We could in theory handle assembly or LLVM input, but let's not.
         // FIXME: What about LTO?
         Diags.diagnose(SourceLoc(), diag::error_unexpected_input_file,

lib/Driver/ToolChains.cpp

@@ -779,6 +779,7 @@ const char *ToolChain::JobContext::computeFrontendModeForCompile() const {
   case file_types::TY_SwiftFixIt:
   case file_types::TY_ModuleSemanticInfo:
   case file_types::TY_CachedDiagnostics:
+  case file_types::TY_SymbolGraphFile:
     llvm_unreachable("Output type can never be primary output.");
   case file_types::TY_INVALID:
     llvm_unreachable("Invalid type ID");
@@ -1054,6 +1055,7 @@ ToolChain::constructInvocation(const BackendJobAction &job,
     case file_types::TY_SwiftFixIt:
     case file_types::TY_ModuleSemanticInfo:
     case file_types::TY_CachedDiagnostics:
+    case file_types::TY_SymbolGraphFile:
       llvm_unreachable("Output type can never be primary output.");
     case file_types::TY_INVALID:
       llvm_unreachable("Invalid type ID");

lib/Frontend/CASOutputBackends.cpp

@@ -57,16 +57,39 @@ class SwiftCASOutputBackend::Implementation {
 public:
   Implementation(ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
                  const FrontendInputsAndOutputs &InputsAndOutputs,
+                 const FrontendOptions &Opts,
                  FrontendOptions::ActionType Action)
       : CAS(CAS), Cache(Cache), BaseKey(BaseKey),
-        InputsAndOutputs(InputsAndOutputs), Action(Action) {
+        InputsAndOutputs(InputsAndOutputs), Opts(Opts), Action(Action) {
     initBackend(InputsAndOutputs);
   }
 
   llvm::Expected<std::unique_ptr<llvm::vfs::OutputFileImpl>>
   createFileImpl(llvm::StringRef ResolvedPath,
                  std::optional<llvm::vfs::OutputConfig> Config) {
     auto ProducingInput = OutputToInputMap.find(ResolvedPath);
+    if (ProducingInput == OutputToInputMap.end() &&
+        ResolvedPath.starts_with(Opts.SymbolGraphOutputDir)) {
+      return std::make_unique<SwiftCASOutputFile>(
+          ResolvedPath, [this](StringRef Path, StringRef Bytes) -> Error {
+            bool Shortened = Path.consume_front(Opts.SymbolGraphOutputDir);
+            assert(Shortened && "symbol graph path outside output dir");
+            (void)Shortened;
+            std::optional<ObjectRef> PathRef;
+            if (Error E =
+                    CAS.storeFromString(std::nullopt, Path).moveInto(PathRef))
+              return E;
+            std::optional<ObjectRef> BytesRef;
+            if (Error E =
+                    CAS.storeFromString(std::nullopt, Bytes).moveInto(BytesRef))
+              return E;
+            auto Ref = CAS.store({*PathRef, *BytesRef}, {});
+            if (!Ref)
+              return Ref.takeError();
+            SymbolGraphOutputRefs.push_back(*Ref);
+            return Error::success();
+          });
+    }
     assert(ProducingInput != OutputToInputMap.end() && "Unknown output file");
 
     unsigned InputIndex = ProducingInput->second.first;
@@ -97,21 +120,24 @@ class SwiftCASOutputBackend::Implementation {
   ActionCache &Cache;
   ObjectRef BaseKey;
   const FrontendInputsAndOutputs &InputsAndOutputs;
+  const FrontendOptions &Opts;
   FrontendOptions::ActionType Action;
 
   // Map from output path to the input index and output kind.
   StringMap<std::pair<unsigned, file_types::ID>> OutputToInputMap;
 
   // A vector of output refs where the index is the input index.
   SmallVector<DenseMap<file_types::ID, ObjectRef>> OutputRefs;
+
+  SmallVector<ObjectRef> SymbolGraphOutputRefs;
 };
 
 SwiftCASOutputBackend::SwiftCASOutputBackend(
     ObjectStore &CAS, ActionCache &Cache, ObjectRef BaseKey,
     const FrontendInputsAndOutputs &InputsAndOutputs,
-    FrontendOptions::ActionType Action)
+    const FrontendOptions &Opts, FrontendOptions::ActionType Action)
     : Impl(*new SwiftCASOutputBackend::Implementation(
-          CAS, Cache, BaseKey, InputsAndOutputs, Action)) {}
+          CAS, Cache, BaseKey, InputsAndOutputs, Opts, Action)) {}
 
 SwiftCASOutputBackend::~SwiftCASOutputBackend() { delete &Impl; }
 
@@ -122,7 +148,8 @@ bool SwiftCASOutputBackend::isStoredDirectly(file_types::ID Kind) {
 
 IntrusiveRefCntPtr<OutputBackend> SwiftCASOutputBackend::cloneImpl() const {
   return makeIntrusiveRefCnt<SwiftCASOutputBackend>(
-      Impl.CAS, Impl.Cache, Impl.BaseKey, Impl.InputsAndOutputs, Impl.Action);
+      Impl.CAS, Impl.Cache, Impl.BaseKey, Impl.InputsAndOutputs, Impl.Opts,
+      Impl.Action);
 }
 
 Expected<std::unique_ptr<OutputFileImpl>>
@@ -243,6 +270,13 @@ Error SwiftCASOutputBackend::Implementation::finalizeCacheKeysFor(
   llvm::for_each(ProducedOutputs, [&OutputsForInput](auto E) {
     OutputsForInput.emplace_back(E.first, E.second);
   });
+  if (InputIndex == InputsAndOutputs.getIndexOfFirstOutputProducingInput() &&
+      !SymbolGraphOutputRefs.empty()) {
+    auto SGRef = CAS.store(SymbolGraphOutputRefs, {});
+    if (!SGRef)
+      return SGRef.takeError();
+    OutputsForInput.emplace_back(file_types::ID::TY_SymbolGraphFile, *SGRef);
+  }
   // Sort to a stable ordering for deterministic output cache object.
   llvm::sort(OutputsForInput,
              [](auto &LHS, auto &RHS) { return LHS.first < RHS.first; });

lib/Frontend/CachingUtils.cpp

@@ -54,14 +54,13 @@ using namespace llvm::vfs;
 
 namespace swift {
 
-llvm::IntrusiveRefCntPtr<SwiftCASOutputBackend>
-createSwiftCachingOutputBackend(
+llvm::IntrusiveRefCntPtr<SwiftCASOutputBackend> createSwiftCachingOutputBackend(
     llvm::cas::ObjectStore &CAS, llvm::cas::ActionCache &Cache,
     llvm::cas::ObjectRef BaseKey,
     const FrontendInputsAndOutputs &InputsAndOutputs,
-    FrontendOptions::ActionType Action) {
-  return makeIntrusiveRefCnt<SwiftCASOutputBackend>(CAS, Cache, BaseKey,
-                                                    InputsAndOutputs, Action);
+    const FrontendOptions &Opts, FrontendOptions::ActionType Action) {
+  return makeIntrusiveRefCnt<SwiftCASOutputBackend>(
+      CAS, Cache, BaseKey, InputsAndOutputs, Opts, Action);
 }
 
 Error cas::CachedResultLoader::replay(CallbackTy Callback) {
@@ -197,6 +196,31 @@ bool replayCachedCompilerOutputs(
             assert(!DiagnosticsOutput && "more than 1 diagnotics found");
             DiagnosticsOutput.emplace(
                 OutputEntry{OutputPath->second, OutID, Kind, Input, *Proxy});
+          } else if (Kind == file_types::ID::TY_SymbolGraphFile &&
+                     !Opts.SymbolGraphOutputDir.empty()) {
+            auto Err = Proxy->forEachReference([&](ObjectRef Ref) -> Error {
+              auto Proxy = CAS.getProxy(Ref);
+              if (!Proxy)
+                return Proxy.takeError();
+              auto PathRef = Proxy->getReference(0);
+              auto ContentRef = Proxy->getReference(1);
+              auto Path = CAS.getProxy(PathRef);
+              auto Content = CAS.getProxy(ContentRef);
+              if (!Path)
+                return Path.takeError();
+              if (!Content)
+                return Content.takeError();
+
+              SmallString<128> OutputPath(Opts.SymbolGraphOutputDir);
+              llvm::sys::path::append(OutputPath, Path->getData());
+
+              OutputProxies.emplace_back(OutputEntry{
+                  std::string(OutputPath), OutID, Kind, Input, *Content});
+
+              return Error::success();
+            });
+            if (Err)
+              return Err;
           } else
             OutputProxies.emplace_back(
                 OutputEntry{OutputPath->second, OutID, Kind, Input, *Proxy});
@@ -234,6 +258,9 @@ bool replayCachedCompilerOutputs(
     Outputs.try_emplace(file_types::ID::TY_CachedDiagnostics,
                         "<cached-diagnostics>");
 
+    // Add symbol graph entry for lookup. Output path doesn't matter here.
+    Outputs.try_emplace(file_types::ID::TY_SymbolGraphFile, "<symbol-graph>");
+
     return replayOutputsForInputFile(Input, InputPath, InputIndex, Outputs);
   };
 

lib/Frontend/Frontend.cpp

@@ -486,6 +486,7 @@ void CompilerInstance::setupOutputBackend() {
     auto &InAndOuts = Invocation.getFrontendOptions().InputsAndOutputs;
     CASOutputBackend = createSwiftCachingOutputBackend(
         *CAS, *ResultCache, *CompileJobBaseKey, InAndOuts,
+        Invocation.getFrontendOptions(),
         Invocation.getFrontendOptions().RequestedAction);
 
     if (Invocation.getIRGenOptions().UseCASBackend) {

test/CAS/symbol-graph.swift

@@ -0,0 +1,46 @@
+// RUN: %empty-directory(%t)
+// RUN: split-file %s %t
+
+// RUN: %target-swift-frontend -scan-dependencies -module-name Test -module-cache-path %t/clang-module-cache -O \
+// RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/deps.json -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
+// RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph1
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json A > %t/A.cmd
+// RUN: %swift_frontend_plain @%t/A.cmd
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/Test.cmd
+// RUN: %target-swift-frontend -module-name Test -module-cache-path %t/clang-module-cache -O \
+// RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/Test.swiftmodule -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
+// RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph1 \
+// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=CACHE-MISS
+
+// CACHE-MISS: remark: cache miss for input
+
+// RUN: %{python} %S/Inputs/BuildCommandExtractor.py %t/deps.json Test > %t/Test.cmd
+// RUN: %target-swift-frontend -module-name Test -module-cache-path %t/clang-module-cache -O \
+// RUN:   -parse-stdlib -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import \
+// RUN:   %t/main.swift -o %t/Test.swiftmodule -swift-version 5 -cache-compile-job -cas-path %t/cas -I %t/include \
+// RUN:   -emit-symbol-graph -emit-symbol-graph-dir %t/symbol-graph2 \
+// RUN:   -emit-module @%t/Test.cmd -Rcache-compile-job 2>&1 | %FileCheck %s --check-prefix=CACHE-MISS
+
+// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}symbol-graph2{{/|\\}}Test.symbols.json': key 'llvmcas://{{.*}}'
+// CACHE-HIT: remark: replay output file '{{.*}}{{/|\\}}symbol-graph2{{/|\\}}[email protected]': key 'llvmcas://{{.*}}'
+
+// RUN: diff -r -u %t/symbol-graph1 %t/symbol-graph2
+
+//--- include/A.swiftinterface
+// swift-interface-format-version: 1.0
+// swift-module-flags: -module-name A -O -disable-implicit-string-processing-module-import -disable-implicit-concurrency-module-import -parse-stdlib -user-module-version 1.0
+
+public struct A {}
+
+//--- main.swift
+import A
+
+public struct Foo {}
+
+extension A {
+  public func bar() {}
+}