Improve Codecov reporting (#1313)

* Improve Codecov reporting

* Fix AL23 lcov errors

* Refactor; Improve data accuracy

* LCOV v1 has different errors to ignore

* per PR comments

.github/workflows/codecov-ci.yml

@@ -18,7 +18,6 @@ jobs:
         run: ./util/codecov-ci.sh ${{ runner.temp }}/build
       - name: Upload code coverage report to Codecov
         uses: codecov/codecov-action@v3
-        env:
-          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
-          file: ${{ runner.temp }}/build/coverage.info
+          token: ${{ secrets.CODECOV_TOKEN }}
+          files: "${{ runner.temp }}/build/coverage-default.info,${{ runner.temp }}/build/coverage-no-asm.info"

CMakeLists.txt

@@ -630,8 +630,8 @@ if(UBSAN)
 endif()
 
 if(GCOV)
-  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage")
-  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage")
+  set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --coverage -O0")
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --coverage -O0")
   set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --coverage")
 endif()
 

util/codecov-ci.sh

@@ -7,6 +7,7 @@
 set -xe
 
 SRC=$(pwd)
+SRC=$(readlink -f "$SRC")
 
 # Sanity check
 DIRNAME=$(basename -- "${SRC}")
@@ -17,26 +18,65 @@ fi
 
 BUILD="$1"
 if [ -n "$BUILD" ]; then
-  if [ ! -e "$BUILD" ]; then
-    mkdir -p "$BUILD"
-  fi
+  mkdir -p "${BUILD}"
   BUILD=$(readlink -f "$BUILD")
+  BUILD_HTML=$(mkdir -vp "$BUILD/html")
 else
   echo "Must specify a build directory."
   exit 1
 fi
 
-cmake -DGCOV=1 -DDISABLE_PERL=1 -DBUILD_TESTING=1 -DBUILD_LIBSSL=1 -DCMAKE_BUILD_TYPE=Debug -S "${SRC}" -B "${BUILD}"
-cmake --build "${BUILD}" --target all_tests
-cmake --build "${BUILD}" --target run_tests
+LCOV_PARAMS=()
+LCOV_PARAMS+=(--exclude '*/third_party/*')
+LCOV_PARAMS+=(--exclude '*/tool/*')
+LCOV_PARAMS+=(--exclude '*_test.*')
+LCOV_PARAMS+=(--exclude '*/test_*')
+LCOV_PARAMS+=(--exclude '*_test_*')
+LCOV_PARAMS+=(--exclude '*/gtest_*')
+LCOV_PARAMS+=(--exclude '*/wycheproof_*')
+if [[ "$(uname -s)" == "Darwin" ]]; then
+  LCOV_PARAMS+=(--exclude '/Applications/*')
+  LCOV_IGNORE_ERRORS="inconsistent,inconsistent,gcov,gcov"
+  GENHTML_IGNORE_ERRORS="inconsistent,unmapped"
+else
+  LCOV_PARAMS+=(--exclude '/usr/*')
+  LCOV_PARAMS+=(--exclude '/lib/*')
+  if lcov --version | grep --silent 'LCOV version 1.'; then
+    LCOV_IGNORE_ERRORS="gcov,source,graph"
+  else
+    LCOV_IGNORE_ERRORS="negative,mismatch,unused"
+  fi
+  GENHTML_IGNORE_ERRORS="inconsistent,unmapped"
+fi
+LCOV_PARAMS+=(--ignore-errors ${LCOV_IGNORE_ERRORS})
+
+CMAKE_SETUP_PARAMS=(-DGCOV=1 -DDISABLE_PERL=1 -DBUILD_TESTING=1 -DBUILD_LIBSSL=1 -DCMAKE_BUILD_TYPE=Debug -S "${SRC}")
+
+function generate_coverage() {
+  mkdir -p "${BUILD}/${1}"
+  BUILD_DIR="${BUILD}/${1}"
+
+  # Build
+  cmake ${2} ${CMAKE_SETUP_PARAMS} -B "${BUILD_DIR}"
+  cmake --build "${BUILD_DIR}" --target all_tests
+
+  # Collect initial coverage data
+  lcov --capture "${LCOV_PARAMS[@]}" --initial --directory "${BUILD_DIR}" --output-file "${BUILD}/initial-${1}.info"
+
+  # Run tests
+  cmake --build "${BUILD_DIR}" --target run_tests
+
+  # Collect coverage data and combine it with initial data
+  lcov --capture "${LCOV_PARAMS[@]}" --directory "${BUILD_DIR}" --output-file "${BUILD}/test-${1}.info"
+  lcov "${LCOV_PARAMS[@]}" --add-tracefile "${BUILD}/initial-${1}.info" --add-tracefile "${BUILD}/test-${1}.info" --output-file "${BUILD}/coverage-${1}.info"
+}
+
+# Default x86-64 build/test
+generate_coverage "default" ""
 
-#TODO: Use callgrind
-#mkdir "$BUILD/callgrind/"
-#go run "$SRC/util/all_tests.go" -build-dir "$BUILD" -callgrind -num-workers 16
+# No Assembly x86-64 build/test
+generate_coverage "no-asm" "-DOPENSSL_NO_ASM=1"
 
-shopt -s globstar
+#genhtml --ignore-errors ${GENHTML_IGNORE_ERRORS} --output-directory "${BUILD_HTML}" "${BUILD}"/coverage-*.info
+#open "${BUILD_HTML}"/index.html
 
-pushd "${BUILD}"
-gcov --source-prefix "${SRC}" **/*.gcda
-lcov --capture --exclude "/Applications/*" --exclude "/usr/*" --exclude "/lib/*" --directory . --output-file coverage.info
-popd