From 8cafc3262b77085f0c4c80402ab51063d9b0d8ec Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 22 May 2022 13:00:43 +0200 Subject: [PATCH 1/5] Get rid of redundant libs --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 93ac0f9..0ae214e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,7 @@ function(add_dia_executable execname) if(NOT LLVM_ENABLE_RTTI) target_compile_options("${execname}" PRIVATE -fno-rtti) endif() - target_link_libraries("${execname}" PUBLIC ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMBitWriter.a ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMBitWriter.a ${LLVM_DIR}/lib/libclangDirectoryWatcher.a ${LLVM_DIR}/lib/libclangIndex.a ${LLVM_DIR}/lib/libclangFormat.a ${LLVM_DIR}/lib/libclangToolingInclusions.a ${LLVM_DIR}/lib/libclangFrontend.a ${LLVM_DIR}/lib/libclangDriver.a ${LLVM_DIR}/lib/libclangParse.a ${LLVM_DIR}/lib/libLLVMOption.a ${LLVM_DIR}/lib/libclangSerialization.a ${LLVM_DIR}/lib/libclangSema.a ${LLVM_DIR}/lib/libclangAPINotes.a ${LLVM_DIR}/lib/libclangEdit.a ${LLVM_DIR}/lib/libclangAnalysis.a ${LLVM_DIR}/lib/libclangASTMatchers.a ${LLVM_DIR}/lib/libclangAST.a ${LLVM_DIR}/lib/libLLVMFrontendOpenMP.a ${LLVM_DIR}/lib/libLLVMTransformUtils.a ${LLVM_DIR}/lib/libclangToolingCore.a ${LLVM_DIR}/lib/libclangRewrite.a ${LLVM_DIR}/lib/libclangLex.a ${LLVM_DIR}/lib/libclangBasic.a ${LLVM_DIR}/lib/libLLVMAnalysis.a ${LLVM_DIR}/lib/libLLVMProfileData.a ${LLVM_DIR}/lib/libLLVMObject.a ${LLVM_DIR}/lib/libLLVMBitReader.a ${LLVM_DIR}/lib/libLLVMCore.a ${LLVM_DIR}/lib/libLLVMRemarks.a ${LLVM_DIR}/lib/libLLVMBitstreamReader.a ${LLVM_DIR}/lib/libLLVMMCParser.a ${LLVM_DIR}/lib/libLLVMMC.a ${LLVM_DIR}/lib/libLLVMDebugInfoCodeView.a ${LLVM_DIR}/lib/libLLVMDebugInfoMSF.a ${LLVM_DIR}/lib/libLLVMTextAPI.a ${LLVM_DIR}/lib/libLLVMBinaryFormat.a ${LLVM_DIR}/lib/libLLVMSupport.a ${LLVM_DIR}/lib/libLLVMDemangle.a ${CURSES_LIBRARIES} + target_link_libraries("${execname}" PUBLIC ${LLVM_DIR}/lib/libclangFrontend.a ${LLVM_DIR}/lib/libclangEdit.a ${LLVM_DIR}/lib/libclangLex.a ${LLVM_DIR}/lib/libclangBasic.a ${LLVM_DIR}/lib/libLLVMBitstreamReader.a ${LLVM_DIR}/lib/libLLVMSupport.a ${CURSES_LIBRARIES} ) target_link_options("${execname}" PRIVATE -dead_strip) endfunction() From 5c27bd87ab21964a0e790df7c0404fbe11487754 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 22 May 2022 13:01:05 +0200 Subject: [PATCH 2/5] Update Readme --- README.md | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 851759c..0935b39 100644 --- a/README.md +++ b/README.md @@ -2,11 +2,36 @@ A tool to combine multiple .dia (llvm diagnostics file) files into a single .dia file. + +## Supported features + +* Merging several `.dia` files into a single file +* Remaps included sourcecode paths using regex +* Reading `.dia` file from a FIFO file + ## Overview `.dia` files are generated by swift and clang compilers to let the IDE know about warnings and fix-its. By default, each compilation file generates a single .dia file with an absolute path. When a build system needs (e.g. Bazel) needs to merge all individual .dia files into an aggregation, this tool combines them into a "fat" dia. -This tool supports also remapping. That might be useful in builds distribution where machines do not share the same absolute paths `dia-merge` supports `-remap` flag(s) which `=`. +### Remapping + +This tool supports also remapping. That might be useful in builds distribution where machines do not share the same absolute paths `dia-merge` supports `-remap` flag(s) which `=`. You can provide several `-remap` arguments. + +### Reading from a FIFO file + +If you have a stream file to which compilers write their diagnostics, you can make `dia-merge` a consumer of that file. That may be useful if you don't want to pass unique paths to each compiler invocation and instead pass always the same path. Because a `dia-merge` is a reader of the fifo, it has to be running in the background a compilation happens, otherwise compiler will hang until some process reads written bytes. When `dia-merge` is interrupted with a signal `SIGINT`, it dumps all read diagnostics to the final output file. + +### Example flow + +1. Start dia-merge in the background with `dia-merge -s /path/fifo.file -o /path/merged.dia &` +2. Run a build and pass these parameters +* clang: `--serialize-diagnostics /path/fifo.file` +* Swift: `-Xfrontend -serialize-diagnostics-path -Xfrontend /path/fifo.file` +3. Interrupt the `dia-merge` with: `kill -2 %1` + +As a result, `/path/merged.dia` will contain all diagnostics published to the stream `path/fifo.file`. + +_Note: There is a limit of the `.dia` file that this mode guarantees consistency: 65536 bytes (limited by a macOS pipe buffer). Usually `.dia` are very small files (hundreds of bytes) so 65KB should be big enough. If a file will be bigger, it is possible that a kernel will not write dia bytes to a FIFO file atomically and if more than one compilation process writes to a file, a reader will my parse truncated `.dia` file content._ ### DIA file @@ -25,7 +50,7 @@ On a consumer side, where a source root is `/other_source_root/`, call: ``` -dia-merge -r `^./=/other_source_root/` -output build_system/ExpectedLocation.dia downloaded_package/Example.dia +dia-merge -r `^\./=/other_source_root/` -output build_system/ExpectedLocation.dia downloaded_package/Example.dia ``` ## Build Instructions @@ -46,5 +71,19 @@ Or, if you prefer Xcode for building and debugging, you can replace the last 2 l ``` cmake -G Xcode -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLVM_DIR=#{path_to_downloaded_clang} .. -open index-import.xcodeproj +open dia-merge.xcodeproj +``` + +#### Tip: full script to build an Xcode project + +_Note: this snippet uses clang 13.0.0._ + ``` +mkdir build +cd build +curl -L "https://github.com/llvm/llvm-project/releases/download/llvmorg-13.0.0/clang+llvm-13.0.0-x86_64-apple-darwin.tar.xz" --output clang.zip +mkdir -p clang-downloaded +tar -xvf ./clang.zip -C clang-downloaded --strip 1 +cmake -G Xcode -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLVM_DIR="$PWD/clang-downloaded" .. +open dia-merge.xcodeproj +``` \ No newline at end of file From 18491d4408ff63920f6941af1359812715aefe29 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 22 May 2022 13:10:03 +0200 Subject: [PATCH 3/5] Code cleanup --- dia-merge.cpp | 65 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 41 insertions(+), 24 deletions(-) diff --git a/dia-merge.cpp b/dia-merge.cpp index 4f93446..140a4fc 100644 --- a/dia-merge.cpp +++ b/dia-merge.cpp @@ -1,3 +1,23 @@ +// Copyright (c) 2022 Bartosz Polaczyk +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + #include "clang/Frontend/DiagnosticRenderer.h" #include "clang/Frontend/SerializedDiagnosticReader.h" #include "clang/Lex/Lexer.h" @@ -26,6 +46,8 @@ static cl::alias PathRemapsAlias("r", cl::aliasopt(PathRemaps)); static cl::opt Stream("stream", cl::desc("Read the dia from the fifo file. Requires exactly one input path"), cl::init(false)); static cl::alias StreamAlias("s", cl::aliasopt(Stream)); +// Remapper inspired by https://github.com/MobileNativeFoundation/index-import + struct Remapper { public: std::string remap(const llvm::StringRef input) const { @@ -52,8 +74,8 @@ struct Remapper { std::vector> _remaps; }; - - +// Based on LLVM's BitcodeWriter.h, SerializedDiagnosticPrinter.cpp +// and other files in https://github.com/llvm/llvm-project/ typedef SmallVector RecordData; typedef SmallVectorImpl RecordDataImpl; @@ -79,8 +101,6 @@ class AbbreviationMap { } }; -/* Code inspired by the LLVM codebase */ - class SDiagsWriter : public DiagnosticConsumer { friend class SDiagsRenderer; friend class SDiagsMerger; @@ -392,24 +412,24 @@ void SDiagsWriter::AddCharSourceRangeToRecord(CharSourceRange Range, unsigned SDiagsWriter::getEmitFile(const char *FileName){ if (!FileName) return 0; - - StringRef Name(FileName); - auto remapped = PathsRemappper->remap(Name); - StringRef RemappedName(remapped); - // TODO: Reuse file abbrevations - // This was added as Densemap returns values for not valid entries - State->Files.clear(); + StringRef Name(FileName); + auto remapped = PathsRemappper->remap(Name); + StringRef RemappedName(remapped); + + // TODO(polac24): Reuse file abbrevations - right now it always emits filepath + // This workaround was added as Densemap returns values for not valid entries. + State->Files.clear(); unsigned &entry = State->Files[RemappedName.str().c_str()]; if (entry) { return entry; } // Lazily generate the record for the file. - entry = State->Files.size(); - RecordData::value_type Record[] = {clang::serialized_diags::RECORD_FILENAME, entry, 0 /* For legacy */, + entry = State->Files.size(); + RecordData::value_type Record[] = {clang::serialized_diags::RECORD_FILENAME, entry, 0 /* For legacy */, 0 /* For legacy */, RemappedName.size()}; - State->Stream.EmitRecordWithBlob(State->Abbrevs.get(clang::serialized_diags::RECORD_FILENAME), Record, + State->Stream.EmitRecordWithBlob(State->Abbrevs.get(clang::serialized_diags::RECORD_FILENAME), Record, RemappedName); return entry; @@ -418,9 +438,9 @@ unsigned SDiagsWriter::getEmitFile(const char *FileName){ void SDiagsWriter::EmitCharSourceRange(CharSourceRange R, const SourceManager &SM) { State->Record.clear(); - State->Record.push_back(clang::serialized_diags::RECORD_SOURCE_RANGE); + State->Record.push_back(clang::serialized_diags::RECORD_SOURCE_RANGE); AddCharSourceRangeToRecord(R, State->Record, SM); - State->Stream.EmitRecordWithAbbrev(State->Abbrevs.get(clang::serialized_diags::RECORD_SOURCE_RANGE), + State->Stream.EmitRecordWithAbbrev(State->Abbrevs.get(clang::serialized_diags::RECORD_SOURCE_RANGE), State->Record); } @@ -600,9 +620,7 @@ void SDiagsWriter::HandleDiagnostic(DiagnosticsEngine::Level DiagLevel, if (IsFinishing) { SmallString<256> diagnostic; Info.FormatDiagnostic(diagnostic); -// getMetaDiags()->Report( -// diag::warn_fe_serialized_diag_failure_during_finalisation) -// << diagnostic; + // TODO(polac24): implement error handling return; } @@ -792,7 +810,7 @@ void SDiagsWriter::RemoveOldDiagnostics() { if (!llvm::sys::fs::remove(State->OutputFile)) return; -// getMetaDiags()->Report(diag::warn_fe_serialized_diag_merge_failure); + // TODO(polac24): implement error handling // Disable merging child records, as whatever is in this file may be // misleading. MergeChildRecords = false; @@ -818,7 +836,7 @@ void SDiagsWriter::finish() { if (llvm::sys::fs::exists(State->OutputFile)) if (SDiagsMerger(*this).mergeRecordsFromFile(State->OutputFile.c_str())){ -// getMetaDiags()->Report(diag::warn_fe_serialized_diag_merge_failure); + // TODO(polac24): implement error handling } } @@ -826,8 +844,7 @@ void SDiagsWriter::finish() { auto OS = std::make_unique(State->OutputFile.c_str(), EC, llvm::sys::fs::OF_None); if (EC) { -// getMetaDiags()->Report(diag::warn_fe_serialized_diag_failure) -// << State->OutputFile << EC.message(); + // TODO(polac24): implement error handling - write to the stderr OS->clear_error(); return; } @@ -838,7 +855,7 @@ void SDiagsWriter::finish() { assert(!OS->has_error()); if (OS->has_error()) { -// getMetaDiags()->Report(diag:clang::serialized_diags::OS->error().message(); + // TODO(polac24): implement error handling - write to the stderr OS->clear_error(); } } From 41d6bd67169f0f92c88f1995b1c8892a4425e2f3 Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 22 May 2022 13:26:36 +0200 Subject: [PATCH 4/5] Eliminate unused parameters warnings --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ae214e..f7a4490 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,9 +9,9 @@ find_package(Curses REQUIRED) function(add_dia_executable execname) add_executable("${execname}" "${execname}.cpp") target_include_directories("${execname}" PRIVATE ${LLVM_DIR}/include) - target_compile_features("${execname}" PRIVATE cxx_std_14) + target_compile_features("${execname}" PUBLIC cxx_std_14) set_target_properties("${execname}" PROPERTIES CXX_EXTENSIONS OFF) - target_compile_options("${execname}" PRIVATE -Wall -Wextra -fblocks) + target_compile_options("${execname}" PRIVATE -Wall -Wextra -fblocks -Wno-unused-parameter) if(NOT LLVM_ENABLE_RTTI) target_compile_options("${execname}" PRIVATE -fno-rtti) endif() From f18f8316bf9a67381309b283f59f541757ff655e Mon Sep 17 00:00:00 2001 From: Bartosz Polaczyk Date: Sun, 22 May 2022 13:31:18 +0200 Subject: [PATCH 5/5] Add license --- LICENSE | 13 +++++++++++++ README.md | 18 ++++++++++++++++++ 2 files changed, 31 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5bf326d --- /dev/null +++ b/LICENSE @@ -0,0 +1,13 @@ +# Copyright 2022 Bartosz Polaczyk +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/README.md b/README.md index 0935b39..9b06cf4 100644 --- a/README.md +++ b/README.md @@ -86,4 +86,22 @@ mkdir -p clang-downloaded tar -xvf ./clang.zip -C clang-downloaded --strip 1 cmake -G Xcode -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 -DLLVM_DIR="$PWD/clang-downloaded" .. open dia-merge.xcodeproj +``` + +## License + +``` +Copyright 2022 Bartosz Polaczyk + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. ``` \ No newline at end of file