From a9f7a06c7793fa0064232aed4482067e5b2c177d Mon Sep 17 00:00:00 2001
From: Justin W Smith <103147162+justsmth@users.noreply.github.com>
Date: Mon, 20 Nov 2023 12:51:17 -0500
Subject: [PATCH] Improve Codecov reporting (#1313)

* Improve Codecov reporting

* Fix AL23 lcov errors

* Refactor; Improve data accuracy

* LCOV v1 has different errors to ignore

* per PR comments
---
 .github/workflows/codecov-ci.yml |  5 +--
 CMakeLists.txt                   |  4 +-
 util/codecov-ci.sh               | 68 +++++++++++++++++++++++++-------
 3 files changed, 58 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/codecov-ci.yml b/.github/workflows/codecov-ci.yml
index 6546b040bd..7ed53504da 100644
--- a/.github/workflows/codecov-ci.yml
+++ b/.github/workflows/codecov-ci.yml
@@ -18,7 +18,6 @@ jobs:
         run: ./util/codecov-ci.sh ${{ runner.temp }}/build
       - name: Upload code coverage report to Codecov
         uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
-          file: ${{ runner.temp }}/build/coverage.info
\ No newline at end of file
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: "${{ runner.temp }}/build/coverage-default.info,${{ runner.temp }}/build/coverage-no-asm.info"
diff --git a/CMakeLists.txt b/CMakeLists.txt
index a2dacf558c..92d3f6ebcc 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -630,8 +630,8 @@ if(UBSAN)
 endif()
 
 if(GCOV)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage -O0")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -O0")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
 endif()
 
diff --git a/util/codecov-ci.sh b/util/codecov-ci.sh
index 77efbc0a4e..547821bf81 100755
--- a/util/codecov-ci.sh
+++ b/util/codecov-ci.sh
@@ -7,6 +7,7 @@
 set -xe
 
 SRC=$(pwd)
+SRC=$(readlink -f "$SRC")
 
 # Sanity check
 DIRNAME=$(basename -- "${SRC}")
@@ -17,26 +18,65 @@ fi
 
 BUILD="$1"
 if [ -n "$BUILD" ]; then
-  if [ ! -e "$BUILD" ]; then
-    mkdir -p "$BUILD"
-  fi
+  mkdir -p "${BUILD}"
   BUILD=$(readlink -f "$BUILD")
+  BUILD_HTML=$(mkdir -vp "$BUILD/html")
 else
   echo "Must specify a build directory."
   exit 1
 fi
 
-cmake -DGCOV=1 -DDISABLE_PERL=1 -DBUILD_TESTING=1 -DBUILD_LIBSSL=1 -DCMAKE_BUILD_TYPE=Debug -S "${SRC}" -B "${BUILD}"
-cmake --build "${BUILD}" --target all_tests
-cmake --build "${BUILD}" --target run_tests
+LCOV_PARAMS=()
+LCOV_PARAMS+=(--exclude '*/third_party/*')
+LCOV_PARAMS+=(--exclude '*/tool/*')
+LCOV_PARAMS+=(--exclude '*_test.*')
+LCOV_PARAMS+=(--exclude '*/test_*')
+LCOV_PARAMS+=(--exclude '*_test_*')
+LCOV_PARAMS+=(--exclude '*/gtest_*')
+LCOV_PARAMS+=(--exclude '*/wycheproof_*')
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  LCOV_PARAMS+=(--exclude '/Applications/*')
+  LCOV_IGNORE_ERRORS="inconsistent,inconsistent,gcov,gcov"
+  GENHTML_IGNORE_ERRORS="inconsistent,unmapped"
+else
+  LCOV_PARAMS+=(--exclude '/usr/*')
+  LCOV_PARAMS+=(--exclude '/lib/*')
+  if lcov --version | grep --silent 'LCOV version 1.'; then
+    LCOV_IGNORE_ERRORS="gcov,source,graph"
+  else
+    LCOV_IGNORE_ERRORS="negative,mismatch,unused"
+  fi
+  GENHTML_IGNORE_ERRORS="inconsistent,unmapped"
+fi
+LCOV_PARAMS+=(--ignore-errors ${LCOV_IGNORE_ERRORS})
+
+CMAKE_SETUP_PARAMS=(-DGCOV=1 -DDISABLE_PERL=1 -DBUILD_TESTING=1 -DBUILD_LIBSSL=1 -DCMAKE_BUILD_TYPE=Debug -S "${SRC}")
+
+function generate_coverage() {
+  mkdir -p "${BUILD}/${1}"
+  BUILD_DIR="${BUILD}/${1}"
+
+  # Build
+  cmake ${2} ${CMAKE_SETUP_PARAMS} -B "${BUILD_DIR}"
+  cmake --build "${BUILD_DIR}" --target all_tests
+
+  # Collect initial coverage data
+  lcov --capture "${LCOV_PARAMS[@]}" --initial --directory "${BUILD_DIR}" --output-file "${BUILD}/initial-${1}.info"
+
+  # Run tests
+  cmake --build "${BUILD_DIR}" --target run_tests
+
+  # Collect coverage data and combine it with initial data
+  lcov --capture "${LCOV_PARAMS[@]}" --directory "${BUILD_DIR}" --output-file "${BUILD}/test-${1}.info"
+  lcov "${LCOV_PARAMS[@]}" --add-tracefile "${BUILD}/initial-${1}.info" --add-tracefile "${BUILD}/test-${1}.info" --output-file "${BUILD}/coverage-${1}.info"
+}
+
+# Default x86-64 build/test
+generate_coverage "default" ""
 
-#TODO: Use callgrind
-#mkdir "$BUILD/callgrind/"
-#go run "$SRC/util/all_tests.go" -build-dir "$BUILD" -callgrind -num-workers 16
+# No Assembly x86-64 build/test
+generate_coverage "no-asm" "-DOPENSSL_NO_ASM=1"
 
-shopt -s globstar
+#genhtml --ignore-errors ${GENHTML_IGNORE_ERRORS} --output-directory "${BUILD_HTML}" "${BUILD}"/coverage-*.info
+#open "${BUILD_HTML}"/index.html
 
-pushd "${BUILD}"
-gcov --source-prefix "${SRC}" **/*.gcda
-lcov --capture --exclude "/Applications/*" --exclude "/usr/*" --exclude "/lib/*" --directory . --output-file coverage.info
-popd