From 67a206890cb3f865b2de199e1b9881c43d290e3c Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 26 Nov 2024 14:38:36 -0500 Subject: [PATCH 01/11] ci: enable `coverage-text` --- .github/workflows/ci.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2f327dae..197baf00 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,19 +430,19 @@ jobs: done iguana_src/.github/make-benchmark-table.rb benchmark_{single_threaded,memoize}.txt | xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY ### coverage - # - name: coverage - # if: ${{ matrix.id == 'coverage' }} - # run: | - # ninja -C iguana_build coverage-html - # ninja -C iguana_build coverage-text - # mv iguana_build/meson-logs/coveragereport coverage-report - # echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY - # echo '```' >> $GITHUB_STEP_SUMMARY - # cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY - # echo '```' >> $GITHUB_STEP_SUMMARY - # echo '' >> $GITHUB_STEP_SUMMARY - # echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY - # echo '- to compare to the report from the `main` branch, see ' >> $GITHUB_STEP_SUMMARY + - name: coverage + if: ${{ matrix.id == 'coverage' }} + run: | + # ninja -C iguana_build coverage-html + ninja -C iguana_build coverage-text + # mv iguana_build/meson-logs/coveragereport coverage-report + echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo '' >> $GITHUB_STEP_SUMMARY + # echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY + # echo '- to compare to the report from the `main` branch, see ' >> $GITHUB_STEP_SUMMARY ### test relocatability - name: test relocatability if: ${{ matrix.id == 'cpp' }} # don't bother re-running santizers, etc. From 8247127fe8c9582b67f1b56dfa0493ef0b016f77 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 26 Nov 2024 15:41:29 -0500 Subject: [PATCH 02/11] test: `lcov` instead of `gcovr` --- .github/install-dependency-packages.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/install-dependency-packages.sh b/.github/install-dependency-packages.sh index d3838662..c69df8d8 100755 --- a/.github/install-dependency-packages.sh +++ b/.github/install-dependency-packages.sh @@ -22,9 +22,9 @@ GENERAL_PACKAGE_LIST_LINUX=( meson llvm # for `llvm-symbolizer`, for human-readable sanitizer results ### coverage - python-colorlog - python-pygments - gcovr + # python-colorlog + # python-pygments + lcov ### ROOT dependencies binutils libx11 From 21caa218d7eb600b87e10747ac342ef56baee921 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 26 Nov 2024 15:55:01 -0500 Subject: [PATCH 03/11] fix: only `coverage-html` target is available for `lcov` --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 197baf00..e6e73f91 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -433,14 +433,14 @@ jobs: - name: coverage if: ${{ matrix.id == 'coverage' }} run: | - # ninja -C iguana_build coverage-html - ninja -C iguana_build coverage-text + ninja -C iguana_build coverage-html + # ninja -C iguana_build coverage-text # mv iguana_build/meson-logs/coveragereport coverage-report - echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - echo '' >> $GITHUB_STEP_SUMMARY + # echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY + # echo '```' >> $GITHUB_STEP_SUMMARY + # cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY + # echo '```' >> $GITHUB_STEP_SUMMARY + # echo '' >> $GITHUB_STEP_SUMMARY # echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY # echo '- to compare to the report from the `main` branch, see ' >> $GITHUB_STEP_SUMMARY ### test relocatability From 6de401093affd2a74a545fd8c08e74bbbf02f903 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 3 Dec 2024 14:37:54 -0500 Subject: [PATCH 04/11] ci: revert to `gcovr` --- .github/install-dependency-packages.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/install-dependency-packages.sh b/.github/install-dependency-packages.sh index c69df8d8..2498f3ce 100755 --- a/.github/install-dependency-packages.sh +++ b/.github/install-dependency-packages.sh @@ -22,9 +22,10 @@ GENERAL_PACKAGE_LIST_LINUX=( meson llvm # for `llvm-symbolizer`, for human-readable sanitizer results ### coverage - # python-colorlog - # python-pygments - lcov + python-colorlog + python-pygments + gcovr + # lcov ### ROOT dependencies binutils libx11 From 794b2ca12cfc328a224fea92a5e1eb16d43a8188 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Tue, 3 Dec 2024 14:39:24 -0500 Subject: [PATCH 05/11] ci: run benchmarks on non-coverage workflow --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e6e73f91..d6b3d075 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -423,7 +423,7 @@ jobs: fi ### run benchmarks - name: benchmark algorithms - if: ${{ matrix.id == 'coverage' }} # use the coverage job's GITHUB_STEP_SUMMARY + if: ${{ matrix.id == 'cpp' && inputs.id == 'linux-latest' }} run: | for suite in single_threaded memoize; do meson test --benchmark --suite $suite -C iguana_build | tee benchmark_$suite.txt @@ -433,8 +433,8 @@ jobs: - name: coverage if: ${{ matrix.id == 'coverage' }} run: | - ninja -C iguana_build coverage-html - # ninja -C iguana_build coverage-text + # ninja -C iguana_build coverage-html + ninja -C iguana_build coverage-text # mv iguana_build/meson-logs/coveragereport coverage-report # echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY # echo '```' >> $GITHUB_STEP_SUMMARY From 4355e4923e86ffd1274ada731402111c49719c7c Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 11 Dec 2024 06:44:03 -0500 Subject: [PATCH 06/11] test: exclude multithreading from coverage test --- src/iguana/tests/meson.build | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/iguana/tests/meson.build b/src/iguana/tests/meson.build index 9461415d..4575523a 100644 --- a/src/iguana/tests/meson.build +++ b/src/iguana/tests/meson.build @@ -64,15 +64,15 @@ foreach algo : algo_dict '-m', concurrency_model, '-V', # vary run number ] - test( - '-'.join(['multithreading', concurrency_model, test_name_algo]), - test_exe, - suite: [ 'algorithm', 'multithreading', concurrency_model ], - args: multithreading_args + [ '-n', get_option('test_num_events') ] + test_args, - is_parallel: false, - env: project_test_env, - timeout: 0, - ) + # test( + # '-'.join(['multithreading', concurrency_model, test_name_algo]), + # test_exe, + # suite: [ 'algorithm', 'multithreading', concurrency_model ], + # args: multithreading_args + [ '-n', get_option('test_num_events') ] + test_args, + # is_parallel: false, + # env: project_test_env, + # timeout: 0, + # ) benchmark( '-'.join(['benchmark', f'multithreading_@concurrency_model@', test_name_algo]), test_exe, From 39177749fbfb6b2e682f5f8a05052b0bd1fa0740 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Wed, 11 Dec 2024 06:53:07 -0500 Subject: [PATCH 07/11] fix: don't run `meson test` if not necessary --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d6b3d075..c8e546fc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -413,6 +413,7 @@ jobs: - run: tree iguana ### run tests - name: meson test + if: ${{ matrix.id != 'coverage' && matrix.id != 'documentation' }} working-directory: iguana_build run: | if [ "${{ env.verbose_test }}" = "true" ]; then From 70e27ae7db8708029013f8fe22e0f39f4e1d8093 Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Thu, 12 Dec 2024 03:49:15 -0500 Subject: [PATCH 08/11] ci: re-enable coverage dump to summary --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8e546fc..3669945b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -437,13 +437,13 @@ jobs: # ninja -C iguana_build coverage-html ninja -C iguana_build coverage-text # mv iguana_build/meson-logs/coveragereport coverage-report - # echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY - # echo '```' >> $GITHUB_STEP_SUMMARY - # cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY - # echo '```' >> $GITHUB_STEP_SUMMARY - # echo '' >> $GITHUB_STEP_SUMMARY - # echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY - # echo '- to compare to the report from the `main` branch, see ' >> $GITHUB_STEP_SUMMARY + echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo '' >> $GITHUB_STEP_SUMMARY + echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY + echo '- to compare to the report from the `main` branch, see ' >> $GITHUB_STEP_SUMMARY ### test relocatability - name: test relocatability if: ${{ matrix.id == 'cpp' }} # don't bother re-running santizers, etc. From 9aaf6f1395a886bbfa99f7798ec5839cb4811a0a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Thu, 12 Dec 2024 04:10:30 -0500 Subject: [PATCH 09/11] build!: rename option `test_multithreading` -> `test_num_threads`, and ... ... new option `z_test_multithreading` to optionally disable multithreading tests, which fail the coverage test --- .github/workflows/ci.yml | 2 +- .github/workflows/linux.yml | 2 +- meson.options | 3 ++- src/iguana/tests/meson.build | 52 +++++++++++++++++++----------------- 4 files changed, 31 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3669945b..ade35d6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -389,7 +389,7 @@ jobs: -Dtest_data_file=$(pwd)/test_data.hipo \ -Dtest_num_events=${{ env.num_events }} \ -Dtest_output_dir=$(pwd)/validation_results \ - -Dtest_multithreading=${{ env.num_threads }} \ + -Dtest_num_threads=${{ env.num_threads }} \ ${{ matrix.opts }} - name: dump all build options run: meson configure iguana_build --no-pager diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9e6ace13..7d214298 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -36,7 +36,7 @@ jobs: ], "include": [ { "id": "cpp", "CC": "gcc", "CXX": "g++", "opts": "-Dbuildtype=release -Dz_require_root=true -Dtest_validator_all_stats=true" }, - { "id": "coverage", "CC": "gcc", "CXX": "g++", "opts": "-Dbuildtype=release -Dz_require_root=true -Db_coverage=true" }, + { "id": "coverage", "CC": "gcc", "CXX": "g++", "opts": "-Dbuildtype=release -Dz_require_root=true -Db_coverage=true -Dz_test_multithreading=false" }, { "id": "documentation", "CC": "gcc", "CXX": "g++", "opts": "-Dbuildtype=release -Dz_require_root=true -Dinstall_documentation=true" }, { "id": "address-sanitizer", "CC": "clang", "CXX": "clang++", "opts": "-Dbuildtype=debug -Dz_require_root=true -Db_sanitize=address -Db_lundef=false -Db_pie=true" }, { "id": "thread-sanitizer", "CC": "clang", "CXX": "clang++", "opts": "-Dbuildtype=debug -Dz_require_root=true -Db_sanitize=thread -Db_lundef=false -Db_pie=true" }, diff --git a/meson.options b/meson.options index e916cc99..78cb63eb 100644 --- a/meson.options +++ b/meson.options @@ -11,9 +11,10 @@ option('test_data_file', type: 'string', value: '', description: ' option('test_num_events', type: 'string', value: '10', description: 'Number of events from `test_data_file` to test') option('test_output_dir', type: 'string', value: '', description: 'Output directory for tests. Must be an absolute path. If unspecified, tests will still run, but will not produce output files.') option('test_validator_all_stats', type: 'boolean', value: false, description: 'If true, use all statistics for validators, rather than `test_num_events`') -option('test_multithreading', type: 'integer', value: 4, min: 0, description: 'run multithreading tests with this many threads (use 0 for hardware max)') +option('test_num_threads', type: 'integer', value: 4, min: 0, description: 'run multithreading tests with this many threads (use 0 for hardware max)') # expert options: the defaults should be reasonable for a local installation; different values may be preferred for installation in common areas option('z_install_envfile', type: 'boolean', value: true, description: 'Install a sourceable environment variable file') option('z_require_root', type: 'boolean', value: false, description: 'Fail if ROOT is not found') option('z_require_rcdb', type: 'boolean', value: false, description: 'Fail if RCDB is not found') +option('z_test_multithreading', type: 'boolean', value: true, description: 'Enable multithreading tests') diff --git a/src/iguana/tests/meson.build b/src/iguana/tests/meson.build index 4575523a..cb7eeca8 100644 --- a/src/iguana/tests/meson.build +++ b/src/iguana/tests/meson.build @@ -57,31 +57,33 @@ foreach algo : algo_dict ) # multithreaded tests - foreach concurrency_model : [ 'memoize' ] # note: we used to have more, so leave this array in case we want more - multithreading_args = [ - 'multithreading', - '-j', get_option('test_multithreading').to_string(), - '-m', concurrency_model, - '-V', # vary run number - ] - # test( - # '-'.join(['multithreading', concurrency_model, test_name_algo]), - # test_exe, - # suite: [ 'algorithm', 'multithreading', concurrency_model ], - # args: multithreading_args + [ '-n', get_option('test_num_events') ] + test_args, - # is_parallel: false, - # env: project_test_env, - # timeout: 0, - # ) - benchmark( - '-'.join(['benchmark', f'multithreading_@concurrency_model@', test_name_algo]), - test_exe, - suite: [ 'algorithm', 'multithreading', concurrency_model ], - args: multithreading_args + [ '-n', '0' ] + test_args, # benchmark all the events - env: project_test_env, - timeout: 0, - ) - endforeach + if get_option('z_test_multithreading') + foreach concurrency_model : [ 'memoize' ] # note: we used to have more, so leave this array in case we want more + multithreading_args = [ + 'multithreading', + '-j', get_option('test_num_threads').to_string(), + '-m', concurrency_model, + '-V', # vary run number + ] + test( + '-'.join(['multithreading', concurrency_model, test_name_algo]), + test_exe, + suite: [ 'algorithm', 'multithreading', concurrency_model ], + args: multithreading_args + [ '-n', get_option('test_num_events') ] + test_args, + is_parallel: false, + env: project_test_env, + timeout: 0, + ) + benchmark( + '-'.join(['benchmark', f'multithreading_@concurrency_model@', test_name_algo]), + test_exe, + suite: [ 'algorithm', 'multithreading', concurrency_model ], + args: multithreading_args + [ '-n', '0' ] + test_args, # benchmark all the events + env: project_test_env, + timeout: 0, + ) + endforeach + endif endif From 04b93f11f0c5edf6ab6ae32981abc0b74e2be35f Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Thu, 12 Dec 2024 04:13:34 -0500 Subject: [PATCH 10/11] ci: re-enable HTML coverage report --- .github/install-dependency-packages.sh | 1 - .github/workflows/ci.yml | 30 +++++++++++++------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/.github/install-dependency-packages.sh b/.github/install-dependency-packages.sh index 2498f3ce..d3838662 100755 --- a/.github/install-dependency-packages.sh +++ b/.github/install-dependency-packages.sh @@ -25,7 +25,6 @@ GENERAL_PACKAGE_LIST_LINUX=( python-colorlog python-pygments gcovr - # lcov ### ROOT dependencies binutils libx11 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ade35d6b..46196093 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -434,9 +434,9 @@ jobs: - name: coverage if: ${{ matrix.id == 'coverage' }} run: | - # ninja -C iguana_build coverage-html + ninja -C iguana_build coverage-html ninja -C iguana_build coverage-text - # mv iguana_build/meson-logs/coveragereport coverage-report + mv iguana_build/meson-logs/coveragereport coverage-report echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY @@ -512,13 +512,13 @@ jobs: name: meson_logs_${{ matrix.id }} retention-days: 5 path: iguana_build/meson-logs - # - name: upload coverage artifacts - # uses: actions/upload-artifact@v4 - # if: ${{ matrix.id == 'coverage' }} - # with: - # name: coverage-report - # retention-days: 5 - # path: coverage-report + - name: upload coverage artifacts + uses: actions/upload-artifact@v4 + if: ${{ matrix.id == 'coverage' }} + with: + name: coverage-report + retention-days: 5 + path: coverage-report - name: upload validator artifacts uses: actions/upload-artifact@v4 if: ${{ matrix.id == 'cpp' }} @@ -552,18 +552,18 @@ jobs: with: name: doc_doxygen path: doxygen - # - name: download coverage report - # uses: actions/download-artifact@v4 - # with: - # name: coverage-report - # path: coverage-report + - name: download coverage report + uses: actions/download-artifact@v4 + with: + name: coverage-report + path: coverage-report - run: tree - name: collect run: | mkdir pages cp iguana_src/.github/pages-index.html pages/index.html mv doxygen pages/ - # - run: mv coverage-report pages/ + - run: mv coverage-report pages/ - run: tree - uses: actions/upload-pages-artifact@v3 with: From 3e1588066a766c7695ec660d96e5d7f08f61121a Mon Sep 17 00:00:00 2001 From: Christopher Dilks Date: Thu, 12 Dec 2024 05:08:17 -0500 Subject: [PATCH 11/11] test: need `meson test` first? --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46196093..6edff808 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -413,7 +413,7 @@ jobs: - run: tree iguana ### run tests - name: meson test - if: ${{ matrix.id != 'coverage' && matrix.id != 'documentation' }} + if: ${{ matrix.id != 'documentation' }} working-directory: iguana_build run: | if [ "${{ env.verbose_test }}" = "true" ]; then