Merge pull request #5 from polac24/cleanup-readme

Cleanup readme

CMakeLists.txt

@@ -9,13 +9,13 @@ find_package(Curses REQUIRED)
 function(add_dia_executable execname)
   add_executable("${execname}" "${execname}.cpp")
   target_include_directories("${execname}" PRIVATE ${LLVM_DIR}/include)
-  target_compile_features("${execname}" PRIVATE cxx_std_14)
+  target_compile_features("${execname}" PUBLIC cxx_std_14)
   set_target_properties("${execname}" PROPERTIES CXX_EXTENSIONS OFF)
-  target_compile_options("${execname}" PRIVATE -Wall -Wextra -fblocks)
+  target_compile_options("${execname}" PRIVATE -Wall -Wextra -fblocks -Wno-unused-parameter)
   if(NOT LLVM_ENABLE_RTTI)
     target_compile_options("${execname}" PRIVATE -fno-rtti)
   endif()
-  target_link_libraries("${execname}" PUBLIC ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMBitWriter.a ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMBitWriter.a ${LLVM_DIR}/lib/libclangDirectoryWatcher.a ${LLVM_DIR}/lib/libclangIndex.a ${LLVM_DIR}/lib/libclangFormat.a ${LLVM_DIR}/lib/libclangToolingInclusions.a ${LLVM_DIR}/lib/libclangFrontend.a ${LLVM_DIR}/lib/libclangDriver.a ${LLVM_DIR}/lib/libclangParse.a ${LLVM_DIR}/lib/libLLVMOption.a ${LLVM_DIR}/lib/libclangSerialization.a ${LLVM_DIR}/lib/libclangSema.a ${LLVM_DIR}/lib/libclangAPINotes.a ${LLVM_DIR}/lib/libclangEdit.a ${LLVM_DIR}/lib/libclangAnalysis.a ${LLVM_DIR}/lib/libclangASTMatchers.a ${LLVM_DIR}/lib/libclangAST.a ${LLVM_DIR}/lib/libLLVMFrontendOpenMP.a ${LLVM_DIR}/lib/libLLVMTransformUtils.a ${LLVM_DIR}/lib/libclangToolingCore.a ${LLVM_DIR}/lib/libclangRewrite.a ${LLVM_DIR}/lib/libclangLex.a ${LLVM_DIR}/lib/libclangBasic.a ${LLVM_DIR}/lib/libLLVMAnalysis.a ${LLVM_DIR}/lib/libLLVMProfileData.a ${LLVM_DIR}/lib/libLLVMObject.a ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMCore.a ${LLVM_DIR}/lib/libLLVMRemarks.a ${LLVM_DIR}/lib/libLLVMBitstreamReader.a ${LLVM_DIR}/lib/libLLVMMCParser.a ${LLVM_DIR}/lib/libLLVMMC.a ${LLVM_DIR}/lib/libLLVMDebugInfoCodeView.a ${LLVM_DIR}/lib/libLLVMDebugInfoMSF.a ${LLVM_DIR}/lib/libLLVMTextAPI.a ${LLVM_DIR}/lib/libLLVMBinaryFormat.a ${LLVM_DIR}/lib/libLLVMSupport.a  ${LLVM_DIR}/lib/libLLVMDemangle.a  ${CURSES_LIBRARIES}
+  target_link_libraries("${execname}" PUBLIC ${LLVM_DIR}/lib/libclangFrontend.a ${LLVM_DIR}/lib/libclangEdit.a ${LLVM_DIR}/lib/libclangLex.a ${LLVM_DIR}/lib/libclangBasic.a ${LLVM_DIR}/lib/libLLVMBitstreamReader.a ${LLVM_DIR}/lib/libLLVMSupport.a ${CURSES_LIBRARIES}
 )
   target_link_options("${execname}" PRIVATE -dead_strip)
 endfunction()

LICENSE

@@ -0,0 +1,13 @@
+# Copyright 2022 Bartosz Polaczyk
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+# 
+#     http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

README.md

@@ -2,11 +2,36 @@
 
 A tool to combine multiple .dia (llvm diagnostics file) files into a single .dia file.
 
+
+## Supported features
+
+* Merging several `.dia` files into a single file
+* Remaps included sourcecode paths using regex
+* Reading `.dia` file from a FIFO file
+
 ## Overview
 
 `.dia` files are generated by swift and clang compilers to let the IDE know about warnings and fix-its. By default, each compilation file generates a single .dia file with an absolute path. When a build system needs (e.g. Bazel) needs to merge all individual .dia files into an aggregation, this tool combines them into a "fat" dia.
 
-This tool supports also remapping. That might be useful in builds distribution where machines do not share the same absolute paths `dia-merge` supports `-remap` flag(s) which `<replacement_fix>=<substitute>`.
+### Remapping
+
+This tool supports also remapping. That might be useful in builds distribution where machines do not share the same absolute paths `dia-merge` supports `-remap` flag(s) which `<replacement_fix>=<substitute>`. You can provide several `-remap` arguments.
+
+### Reading from a FIFO file
+
+If you have a stream file to which compilers write their diagnostics, you can make `dia-merge` a consumer of that file. That may be useful if you don't want to pass unique paths to each compiler invocation and instead pass always the same path. Because a `dia-merge` is a reader of the fifo, it has to be running in the background a compilation happens, otherwise compiler will hang until some process reads written bytes. When `dia-merge` is interrupted with a signal `SIGINT`, it dumps all read diagnostics to the final output file.
+
+### Example flow
+
+1. Start dia-merge in the background with `dia-merge -s /path/fifo.file -o /path/merged.dia &`
+2. Run a build and pass these parameters
+* clang: `--serialize-diagnostics /path/fifo.file`
+* Swift: `-Xfrontend -serialize-diagnostics-path -Xfrontend /path/fifo.file`
+3. Interrupt the `dia-merge` with: `kill -2 %1`
+
+As a result, `/path/merged.dia` will contain all diagnostics published to the stream `path/fifo.file`.
+
+_Note: There is a limit of the `.dia` file that this mode guarantees consistency: 65536 bytes (limited by a macOS pipe buffer). Usually `.dia` are very small files (hundreds of bytes) so 65KB should be big enough. If a file will be bigger, it is possible that a kernel will not write dia bytes to a FIFO file atomically and if more than one compilation process writes to a file, a reader will my parse truncated `.dia` file content._
 
 ### DIA file
 
@@ -25,7 +50,7 @@ On a consumer side, where a source root is `/other_source_root/`, call:
 
 
 ```
-dia-merge -r `^./=/other_source_root/` -output build_system/ExpectedLocation.dia  downloaded_package/Example.dia
+dia-merge -r `^\./=/other_source_root/` -output build_system/ExpectedLocation.dia  downloaded_package/Example.dia
 ```
 
 ## Build Instructions
@@ -46,5 +71,37 @@ Or, if you prefer Xcode for building and debugging, you can replace the last 2 l
 
 ```
 cmake -G Xcode -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLVM_DIR=#{path_to_downloaded_clang} ..
-open index-import.xcodeproj
+open dia-merge.xcodeproj
 ```
+
+#### Tip: full script to build an Xcode project
+
+_Note: this snippet uses clang 13.0.0._
+
+```
+mkdir build
+cd build
+curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz" --output clang.zip
+mkdir -p clang-downloaded
+tar -xvf ./clang.zip -C clang-downloaded --strip 1
+cmake -G Xcode -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLVM_DIR="$PWD/clang-downloaded" ..
+open dia-merge.xcodeproj
+```
+
+## License
+
+```
+Copyright 2022 Bartosz Polaczyk
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+```

dia-merge.cpp

@@ -1,3 +1,23 @@
+// Copyright (c) 2022 Bartosz Polaczyk
+//
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
 #include "clang/Frontend/DiagnosticRenderer.h"
 #include "clang/Frontend/SerializedDiagnosticReader.h"
 #include "clang/Lex/Lexer.h"
@@ -26,6 +46,8 @@ static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps));
 static cl::opt<bool> Stream("stream", cl::desc("Read the dia from the fifo file. Requires exactly one input path"), cl::init(false));
 static cl::alias StreamAlias("s", cl::aliasopt(Stream));
 
+// Remapper inspired by https://github.com/MobileNativeFoundation/index-import
+
 struct Remapper {
 public:
   std::string remap(const llvm::StringRef input) const {
@@ -52,8 +74,8 @@ struct Remapper {
   std::vector<std::pair<std::regex, std::string>> _remaps;
 };
 
-
-
+// Based on LLVM's BitcodeWriter.h, SerializedDiagnosticPrinter.cpp
+// and other files in https://github.com/llvm/llvm-project/
 
 typedef SmallVector<uint64_t, 64> RecordData;
 typedef SmallVectorImpl<uint64_t> RecordDataImpl;
@@ -79,8 +101,6 @@ class AbbreviationMap {
   }
 };
 
-/* Code inspired by the LLVM codebase */
-
 class SDiagsWriter : public DiagnosticConsumer {
   friend class SDiagsRenderer;
   friend class SDiagsMerger;
@@ -392,24 +412,24 @@ void SDiagsWriter::AddCharSourceRangeToRecord(CharSourceRange Range,
 unsigned SDiagsWriter::getEmitFile(const char *FileName){
   if (!FileName)
     return 0;
-
-    StringRef Name(FileName);
-    auto remapped = PathsRemappper->remap(Name);
-    StringRef RemappedName(remapped);
 
-    // TODO: Reuse file abbrevations
-    // This was added as Densemap returns values for not valid entries
-    State->Files.clear();
+  StringRef Name(FileName);
+  auto remapped = PathsRemappper->remap(Name);
+  StringRef RemappedName(remapped);
+
+  // TODO(polac24): Reuse file abbrevations - right now it always emits filepath
+  // This workaround was added as Densemap returns values for not valid entries.
+  State->Files.clear();
   unsigned &entry = State->Files[RemappedName.str().c_str()];
     if (entry) {
         return entry;
     }
 
   // Lazily generate the record for the file.
-    entry = State->Files.size();
-    RecordData::value_type Record[] = {clang::serialized_diags::RECORD_FILENAME, entry, 0 /* For legacy */,
+  entry = State->Files.size();
+  RecordData::value_type Record[] = {clang::serialized_diags::RECORD_FILENAME, entry, 0 /* For legacy */,
                                      0 /* For legacy */, RemappedName.size()};
-    State->Stream.EmitRecordWithBlob(State->Abbrevs.get(clang::serialized_diags::RECORD_FILENAME), Record,
+  State->Stream.EmitRecordWithBlob(State->Abbrevs.get(clang::serialized_diags::RECORD_FILENAME), Record,
                                      RemappedName);
 
   return entry;
@@ -418,9 +438,9 @@ unsigned SDiagsWriter::getEmitFile(const char *FileName){
 void SDiagsWriter::EmitCharSourceRange(CharSourceRange R,
                                        const SourceManager &SM) {
   State->Record.clear();
-    State->Record.push_back(clang::serialized_diags::RECORD_SOURCE_RANGE);
+  State->Record.push_back(clang::serialized_diags::RECORD_SOURCE_RANGE);
   AddCharSourceRangeToRecord(R, State->Record, SM);
-    State->Stream.EmitRecordWithAbbrev(State->Abbrevs.get(clang::serialized_diags::RECORD_SOURCE_RANGE),
+  State->Stream.EmitRecordWithAbbrev(State->Abbrevs.get(clang::serialized_diags::RECORD_SOURCE_RANGE),
                                      State->Record);
 }
 
@@ -600,9 +620,7 @@ void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel,
   if (IsFinishing) {
     SmallString<256> diagnostic;
     Info.FormatDiagnostic(diagnostic);
-//    getMetaDiags()->Report(
-//        diag::warn_fe_serialized_diag_failure_during_finalisation)
-//        << diagnostic;
+    // TODO(polac24): implement error handling
     return;
   }
 
@@ -792,7 +810,7 @@ void SDiagsWriter::RemoveOldDiagnostics() {
   if (!llvm::sys::fs::remove(State->OutputFile))
     return;
 
-//  getMetaDiags()->Report(diag::warn_fe_serialized_diag_merge_failure);
+  // TODO(polac24): implement error handling
   // Disable merging child records, as whatever is in this file may be
   // misleading.
   MergeChildRecords = false;
@@ -818,16 +836,15 @@ void SDiagsWriter::finish() {
 
     if (llvm::sys::fs::exists(State->OutputFile))
         if (SDiagsMerger(*this).mergeRecordsFromFile(State->OutputFile.c_str())){
-//            getMetaDiags()->Report(diag::warn_fe_serialized_diag_merge_failure);
+          // TODO(polac24): implement error handling
         }
   }
 
   std::error_code EC;
   auto OS = std::make_unique<llvm::raw_fd_ostream>(State->OutputFile.c_str(),
                                                     EC, llvm::sys::fs::OF_None);
   if (EC) {
-//    getMetaDiags()->Report(diag::warn_fe_serialized_diag_failure)
-//        << State->OutputFile << EC.message();
+    // TODO(polac24): implement error handling - write to the stderr
     OS->clear_error();
     return;
   }
@@ -838,7 +855,7 @@ void SDiagsWriter::finish() {
 
   assert(!OS->has_error());
   if (OS->has_error()) {
-//    getMetaDiags()->Report(diag:clang::serialized_diags::OS->error().message();
+    // TODO(polac24): implement error handling - write to the stderr
     OS->clear_error();
   }
 }