ci: use xrootd validation files #5
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | ||
| on: | ||
| workflow_call: | ||
| inputs: | ||
| id: | ||
| description: 'ID' | ||
| required: true | ||
| type: string | ||
| runner: | ||
| description: 'GitHub runner' | ||
| required: true | ||
| type: string | ||
| container: | ||
| description: 'Docker container' | ||
| required: false | ||
| type: string | ||
| default: '' | ||
| verset: | ||
| description: 'Dependency version set' | ||
| required: false | ||
| type: string | ||
| default: 'latest' | ||
| matrix: | ||
| description: 'Iguana job matrix' | ||
| required: true | ||
| type: string | ||
| concurrency: | ||
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | ||
| cancel-in-progress: true | ||
| defaults: | ||
| run: | ||
| shell: bash | ||
| env: | ||
| hipo_fork: c-dilks/hipo | ||
| hipo_ref: 3f615258df28295d8dc3c5f02369dbf89dadb9e5 # for libHipoDataFrame in pkg-config | ||
| # test options | ||
| num_events: 1000 | ||
| verbose_test: 'false' # only set this to 'true' if `meson test` fails and you need more output to investigate | ||
| jobs: | ||
| # download test data | ||
| ######################################################### | ||
| download_test_data: | ||
| name: Download test data | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/cache@v4 | ||
| id: cache | ||
| with: | ||
| key: test_data | ||
| path: test_data.hipo | ||
| lookup-only: true | ||
| - name: install xrootd-client | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| sudo apt -y update | ||
| sudo apt -y upgrade | ||
| sudo apt -y install xrootd-client | ||
| - name: download | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: xrdcp xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/recon/dst/validation_files.tar.zst ./ | ||
| - name: rename | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| tar xvf validation_files.tar.zst | ||
| mv -v $(find validation_files -type f -name "*.hipo" | head -n1) test_data.hipo | ||
| # dependencies | ||
| ######################################################### | ||
| build_root: | ||
| name: Build ROOT | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| outputs: | ||
| key: ${{ steps.key.outputs.key }} | ||
| steps: | ||
| - name: checkout iguana | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: key | ||
| id: key | ||
| run: | | ||
| root_version=$(iguana_src/meson/minimum-version.sh root src | sed 's;.*/;;' | sed 's;\.tar\.gz;;') | ||
| echo key=root---${{ inputs.id }}---${root_version}---$(date +%Y-week%U) >> $GITHUB_OUTPUT | ||
| - uses: actions/cache/restore@v4 | ||
| id: cache | ||
| with: | ||
| key: ${{ steps.key.outputs.key }} | ||
| path: root.tar.zst | ||
| lookup-only: true | ||
| - name: install dependencies | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| - name: download ROOT source code | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| srcURL=$(iguana_src/meson/minimum-version.sh root src) | ||
| echo "[+] DOWNLOADING ROOT SOURCE CODE FROM: $srcURL" | ||
| wget -nv --no-check-certificate $srcURL | ||
| tar xf $(ls -t *.gz | head -n1) | ||
| ls -t | ||
| mv -v $(ls -td root-* | head -n1) root_src | ||
| - name: build ROOT | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| cmake -S root_src -B build -G Ninja --install-prefix $(pwd)/root -DCMAKE_CXX_STANDARD=17 | ||
| cmake --build build | ||
| cmake --install build | ||
| tar caf root{.tar.zst,} | ||
| - uses: actions/cache/save@v4 | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| id: cache_save | ||
| with: | ||
| key: ${{ steps.key.outputs.key }} | ||
| path: root.tar.zst | ||
| build_hipo: | ||
| name: Build HIPO | ||
| needs: | ||
| - build_root # for dataframe support | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| strategy: | ||
| fail-fast: true | ||
| matrix: | ||
| root_dep: | ||
| - noROOT # exclude dataframe lib, which depends on ROOT | ||
| - withROOT | ||
| outputs: | ||
| key: ${{ steps.key.outputs.key }} | ||
| steps: | ||
| - name: get ROOT build | ||
| if: ${{ matrix.root_dep == 'withROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_root.outputs.key }} | ||
| path: root.tar.zst | ||
| - name: untar ROOT build | ||
| if: ${{ matrix.root_dep == 'withROOT' }} | ||
| run: ls *.tar.zst | xargs -I{} tar xvf {} | ||
| - name: key | ||
| id: key | ||
| run: echo key=hipo---${{ inputs.id }}---${{ env.hipo_ref }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT | ||
| - uses: actions/cache/restore@v4 | ||
| id: cache | ||
| with: | ||
| key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }} | ||
| path: hipo.tar.zst | ||
| lookup-only: true | ||
| - name: checkout iguana for dependency installation script | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: checkout hipo | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| repository: ${{ env.hipo_fork }} | ||
| ref: ${{ env.hipo_ref }} | ||
| path: hipo_src | ||
| - name: build | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| run: | | ||
| iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| [ "${{ matrix.root_dep }}" = "withROOT" ] && source root/bin/thisroot.sh | ||
| cmake -S hipo_src -B build -G Ninja --install-prefix $(pwd)/hipo -DCMAKE_POSITION_INDEPENDENT_CODE=ON # using PIE build, for sanitizer readibility | ||
| cmake --build build | ||
| cmake --install build | ||
| tar cavf hipo{.tar.zst,} | ||
| - uses: actions/cache/save@v4 | ||
| if: ${{ steps.cache.outputs.cache-hit != 'true' }} | ||
| id: cache_save | ||
| with: | ||
| key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }} | ||
| path: hipo.tar.zst | ||
| # build and test Iguana | ||
| ######################################################### | ||
| iguana: | ||
| name: Iguana | ||
| needs: | ||
| - build_hipo | ||
| - build_root | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: ${{ fromJson(inputs.matrix) }} | ||
| env: | ||
| CC: ${{ matrix.CC }} | ||
| CXX: ${{ matrix.CXX }} | ||
| steps: | ||
| ### setup | ||
| - uses: actions/checkout@v4 | ||
| with: # settings needed for version number detection | ||
| clean: false | ||
| fetch-tags: true | ||
| fetch-depth: 0 | ||
| path: iguana_src | ||
| ### get test data | ||
| - name: get test data | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: test_data | ||
| path: test_data.hipo | ||
| ### dependencies | ||
| ###### system | ||
| - name: install dependency packages | ||
| run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }} | ||
| ###### python bindings | ||
| - name: install python bindings dependencies | ||
| if: ${{ matrix.id == 'python' }} | ||
| run: | | ||
| python -m venv .venv | ||
| source .venv/bin/activate | ||
| echo PATH=$PATH >> $GITHUB_ENV | ||
| python -m pip install -r iguana_src/bind/python/requirements.txt | ||
| ###### hipo | ||
| - name: get `hipo-withROOT` build | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_hipo.outputs.key }}---withROOT | ||
| path: hipo.tar.zst | ||
| - name: get `hipo-noROOT` build | ||
| if: ${{ matrix.id == 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_hipo.outputs.key }}---noROOT | ||
| path: hipo.tar.zst | ||
| - name: untar hipo build | ||
| run: tar xf hipo.tar.zst | ||
| - run: tree hipo | ||
| ###### ROOT | ||
| - name: get `ROOT` build | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| uses: actions/cache/restore@v4 | ||
| with: | ||
| key: ${{ needs.build_root.outputs.key }} | ||
| path: root.tar.zst | ||
| - name: untar ROOT build | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| run: tar xf root.tar.zst | ||
| - name: set ROOT environment | ||
| if: ${{ matrix.id != 'noROOT' }} | ||
| run: iguana_src/.github/source-ROOT.sh root | ||
| ###### summarize dependencies | ||
| - name: summarize dependencies | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| echo '### Dependencies' >> $GITHUB_STEP_SUMMARY | ||
| echo '| Dependency | Version |' >> $GITHUB_STEP_SUMMARY | ||
| echo '| --- | --- |' >> $GITHUB_STEP_SUMMARY | ||
| echo "| \`hipo\` | ${{ env.hipo_ref }} |" >> $GITHUB_STEP_SUMMARY | ||
| cat pkg_summary.md >> $GITHUB_STEP_SUMMARY | ||
| ### build | ||
| - name: meson setup | ||
| run: | | ||
| meson setup iguana_build iguana_src \ | ||
| --prefix=$(pwd)/iguana \ | ||
| --cmake-prefix-path=$(pwd)/root \ | ||
| --pkg-config-path=$(pwd)/hipo/lib/pkgconfig \ | ||
| -Dexamples=true \ | ||
| -Dtest_data_file=$(pwd)/test_data.hipo \ | ||
| -Dtest_num_events=${{ env.num_events }} \ | ||
| -Dtest_output_dir=$(pwd)/validation_results \ | ||
| ${{ matrix.opts }} | ||
| - name: dump build options | ||
| run: meson configure iguana_build --no-pager | ||
| - run: meson compile | ||
| working-directory: iguana_build | ||
| - run: meson install | ||
| working-directory: iguana_build | ||
| ### dump info about this build | ||
| - name: dump build log | ||
| if: always() | ||
| run: cat iguana_build/meson-logs/meson-log.txt | ||
| - name: cat pkg-config pc files | ||
| run: | | ||
| pcfiles=$(find iguana -type f -name "*.pc") | ||
| for pcfile in $pcfiles; do | ||
| echo "[+++] cat $pcfile" | ||
| cat $pcfile | ||
| done | ||
| - run: tree iguana | ||
| ### run tests | ||
| - name: meson test | ||
| working-directory: iguana_build | ||
| run: | | ||
| if [ "${{ env.verbose_test }}" = "true" ]; then | ||
| [ ${{ inputs.runner }} = "macos-latest" ] && stdbuf_cmd=gstdbuf || stdbuf_cmd=stdbuf | ||
| $stdbuf_cmd -o0 meson test --print-errorlogs --verbose --no-stdsplit | ||
| else | ||
| meson test --print-errorlogs | ||
| fi | ||
| ### coverage | ||
| - name: coverage | ||
| if: ${{ matrix.id == 'coverage' }} | ||
| run: | | ||
| ninja -C iguana_build coverage-html | ||
| ninja -C iguana_build coverage-text | ||
| mv iguana_build/meson-logs/coveragereport coverage-report | ||
| echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| echo '' >> $GITHUB_STEP_SUMMARY | ||
| echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY | ||
| echo '- to compare to the report from the `main` branch, see <https://jeffersonlab.github.io/iguana/coverage-report>' >> $GITHUB_STEP_SUMMARY | ||
| ### set iguana environment, since the next steps will check the iguana installation | ||
| - name: set iguana environment | ||
| run: source iguana/bin/this_iguana.sh --verbose --githubCI | ||
| ### test installed examples | ||
| ###### cpp | ||
| - run: iguana-example-00-basic test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-01-bank-rows test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-02-dataframes test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| - run: iguana-example-03-config-files iguana/etc/iguana/examples | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| ###### python | ||
| - run: iguana-example-00-basic.py test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'python' }} | ||
| - run: iguana-example-01-bank-rows.py test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'python' }} | ||
| ###### fortran | ||
| - run: iguana-example-fortran test_data.hipo ${{ env.num_events }} | ||
| if: ${{ matrix.id == 'fortran' }} | ||
| ### test consumers | ||
| - name: consumer test make | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| make -C iguana_src/examples/build_with_make | ||
| echo "========================================= TEST RUN =========================================" | ||
| iguana_src/examples/build_with_make/bin/iguana-example-00-basic test_data.hipo 10 | ||
| - name: consumer test meson | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| meson setup build_consumer_meson iguana_src/examples/build_with_meson --prefix=$(pwd)/install_consumer_meson | ||
| meson install -C build_consumer_meson | ||
| echo "========================================= TEST RUN =========================================" | ||
| install_consumer_meson/bin/iguana-example-00-basic test_data.hipo 10 | ||
| - name: consumer test cmake | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| run: | | ||
| cmake -B build_consumer_cmake -S iguana_src/examples/build_with_cmake -DCMAKE_PREFIX_PATH=$(pwd)/hipo -G Ninja --install-prefix=$(pwd)/install_consumer_cmake | ||
| cmake --build build_consumer_cmake | ||
| cmake --install build_consumer_cmake | ||
| echo "========================================= TEST RUN =========================================" | ||
| install_consumer_cmake/bin/iguana-example-00-basic test_data.hipo 10 | ||
| ### upload artifacts | ||
| - uses: actions/upload-artifact@v4 | ||
| if: always() | ||
| with: | ||
| name: logs_iguana_build_${{ matrix.id }} | ||
| retention-days: 5 | ||
| path: iguana_build/meson-logs | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ matrix.id == 'coverage' }} | ||
| with: | ||
| name: coverage-report | ||
| retention-days: 5 | ||
| path: coverage-report | ||
| - uses: actions/upload-artifact@v4 | ||
| if: ${{ matrix.id == 'cpp' }} | ||
| with: | ||
| name: _validation_results | ||
| retention-days: 5 | ||
| path: validation_results | ||
| # documentation | ||
| ######################################################### | ||
| doc_generate: | ||
| if: ${{ inputs.id == 'linux-latest' }} | ||
| name: Generate documentation | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ${{ inputs.container }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| - name: install dependencies | ||
| run: | | ||
| pacman -Syu --noconfirm | ||
| pacman -S --noconfirm doxygen graphviz | ||
| - name: doxygen | ||
| run: doxygen doc/gen/Doxyfile | ||
| - uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: doc_doxygen | ||
| retention-days: 5 | ||
| path: doc/api/ | ||
| # deployment | ||
| ######################################################### | ||
| collect_webpages: | ||
| if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }} | ||
| name: Collect webpages | ||
| needs: | ||
| - doc_generate | ||
| - iguana | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - uses: actions/checkout@v4 | ||
| with: | ||
| path: iguana_src | ||
| - name: download doxygen documentation | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: doc_doxygen | ||
| path: doxygen | ||
| - name: download coverage report | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: coverage-report | ||
| path: coverage-report | ||
| - run: tree | ||
| - name: collect | ||
| run: | | ||
| mkdir pages | ||
| cp iguana_src/.github/pages-index.html pages/index.html | ||
| mv doxygen pages/ | ||
| mv coverage-report pages/ | ||
| - run: tree | ||
| - uses: actions/upload-pages-artifact@v3 | ||
| with: | ||
| retention-days: 5 | ||
| path: pages/ | ||
| deploy_webpages: | ||
| if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }} | ||
| name: Deploy webpages | ||
| needs: collect_webpages | ||
| permissions: | ||
| pages: write | ||
| id-token: write | ||
| environment: | ||
| name: github-pages | ||
| url: ${{ steps.deployment.outputs.page_url }} | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - name: deployment | ||
| id: deployment | ||
| uses: actions/deploy-pages@v4 | ||
| # finalize | ||
| ######################################################### | ||
| final: | ||
| name: Final | ||
| needs: | ||
| - iguana | ||
| runs-on: ${{ inputs.runner }} | ||
| steps: | ||
| - run: exit 0 | ||
