ci.yml

name: CI

on:
  workflow_call:
    inputs:
      id:
        description: 'ID'
        required: true
        type: string
      runner:
        description: 'GitHub runner'
        required: true
        type: string
      container:
        description: 'Docker container'
        required: false
        type: string
        default: ''
      verset:
        description: 'Dependency version set'
        required: false
        type: string
        default: 'latest'
      matrix:
        description: 'Iguana job matrix'
        required: true
        type: string

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  hipo_fork: gavalian/hipo
  hipo_ref: 4.2.0
  # test options
  num_events: 1000
  num_threads: 0 # use 0 for "all" available cores
  verbose_test: 'false' # only set this to 'true' if `meson test` fails and you need more output to investigate

jobs:

  # download test data
  #########################################################

  download_test_data:
    name: Download test data
    runs-on: ubuntu-latest # no need to run this job on other runners
    outputs:
      key: ${{ steps.key.outputs.key }}
    steps:
      - name: key # unique for ${{ inputs.id }}, to avoid cache collisions
        id: key
        run: echo key=test_data---${{ inputs.id }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
      - uses: actions/cache@v4
        id: cache
        with:
          key: ${{ steps.key.outputs.key }}
          path: test_data.hipo
          lookup-only: true
      - name: install xrootd-client
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          sudo apt -y update
          sudo apt -y upgrade
          sudo apt -y install xrootd-client
      - name: download
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: xrdcp xroot://sci-xrootd.jlab.org//osgpool/hallb/clas12/validation/recon/dst/validation_files.tar.zst ./
      - name: rename
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          tar xvf validation_files.tar.zst
          mv -v $(find validation_files -type f -name "*.hipo" | head -n1) test_data.hipo

  # dependencies
  #########################################################

  build_root:
    name: Build ROOT
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.container }}
    outputs:
      key: ${{ steps.key.outputs.key }}
    steps:
      ### define key
      - name: checkout iguana
        uses: actions/checkout@v4
        with:
          path: iguana_src
      - name: key
        id: key
        # FIXME: this builds the source code URL; using the GitHub REST API
        # (via `curl --request`) is preferred to get the source code URL, but
        # needs a PAT to mitigate API rate limits; for now just assume the URL
        # - see https://github.com/JeffersonLab/iguana/pull/276
        run: |
          [ "${{ inputs.verset }}" = "minver" ] && \
            root_version=$(iguana_src/meson/minimum-version.sh root tag) || \
            root_version=$(iguana_src/meson/minimum-version.sh root tag_latest)
          echo root_url=https://api.github.com/repos/root-project/root/tarball/refs/tags/${root_version} >> $GITHUB_OUTPUT
          echo key=root---${{ inputs.id }}---${root_version}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
      ### cache restore
      - uses: actions/cache/restore@v4
        id: cache
        with:
          key: ${{ steps.key.outputs.key }}
          path: root.tar.zst
          lookup-only: true
      ### download and build
      - name: install dependencies
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
      - name: download ROOT source code
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          wget -nv --no-check-certificate --output-document root.tar.gz ${{ steps.key.outputs.root_url }}
          tar xf root.tar.gz
          ls -t
          mv -v $(ls -td root-* | head -n1) root_src
      - name: build ROOT
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          cmake -S root_src -B build -G Ninja --install-prefix $GITHUB_WORKSPACE/root -DCMAKE_CXX_STANDARD=17
          cmake --build build
          cmake --install build
          tar caf root{.tar.zst,}
      ### cache save
      - uses: actions/cache/save@v4
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        id: cache_save
        with:
          key: ${{ steps.key.outputs.key }}
          path: root.tar.zst

  build_hipo:
    name: Build HIPO
    needs:
      - build_root # for dataframe support
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.container }}
    strategy:
      fail-fast: true
      matrix:
        root_dep:
          - noROOT # exclude dataframe lib, which depends on ROOT
          - withROOT
    outputs:
      key: ${{ steps.key.outputs.key }}
    steps:
      ### get ROOT
      - name: get ROOT build
        if: ${{ matrix.root_dep == 'withROOT' }}
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build_root.outputs.key }}
          path: root.tar.zst
      - name: untar ROOT build
        if: ${{ matrix.root_dep == 'withROOT' }}
        run: ls *.tar.zst | xargs -I{} tar xvf {}
      ### define key
      - name: key
        id: key
        run: echo key=hipo---${{ inputs.id }}---${{ env.hipo_ref }}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
      ### cache restore
      - uses: actions/cache/restore@v4
        id: cache
        with:
          key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }}
          path: hipo.tar.zst
          lookup-only: true
      ### build
      - name: checkout iguana for dependency installation script
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/checkout@v4
        with:
          path: iguana_src
      - name: checkout hipo
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        uses: actions/checkout@v4
        with:
          repository: ${{ env.hipo_fork }}
          ref: ${{ env.hipo_ref }}
          path: hipo_src
      - name: build
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        run: |
          iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
          [ "${{ matrix.root_dep }}" = "withROOT" ] && source root/bin/thisroot.sh
          meson setup build hipo_src --prefix=$GITHUB_WORKSPACE/hipo -Dbuildtype=debug -Db_pie=true # using PIE build, for sanitizer readibility
          meson install -C build
          tar cavf hipo{.tar.zst,}
      ### cache save
      - uses: actions/cache/save@v4
        if: ${{ steps.cache.outputs.cache-hit != 'true' }}
        id: cache_save
        with:
          key: ${{ steps.key.outputs.key }}---${{ matrix.root_dep }}
          path: hipo.tar.zst

  build_ruby_minver:
    name: Build Minver Ruby
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.container }}
    outputs:
      key: ${{ steps.key.outputs.key }}
    steps:
      - name: checkout iguana
        if: ${{ inputs.verset == 'minver' }}
        uses: actions/checkout@v4
        with:
          path: iguana_src
      - name: key
        if: ${{ inputs.verset == 'minver' }}
        id: key
        run: |
          ver=$(iguana_src/meson/minimum-version.sh ruby)
          echo ver=$ver >> $GITHUB_OUTPUT
          echo key=ruby---${{ inputs.id }}---${ver}---$(date +%Y-week%U) >> $GITHUB_OUTPUT
      - uses: actions/cache/restore@v4
        if: ${{ inputs.verset == 'minver' }}
        id: cache
        with:
          key: ${{ steps.key.outputs.key }}
          path: ruby.tar.zst
          lookup-only: true
      - name: build ruby
        if: ${{ steps.cache.outputs.cache-hit != 'true' && inputs.verset == 'minver' }}
        run: |
          iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
          export RBENV_ROOT=$GITHUB_WORKSPACE/.rbenv
          git clone https://github.com/rbenv/rbenv.git $RBENV_ROOT
          eval "$($RBENV_ROOT/bin/rbenv init - bash)"
          git clone https://github.com/rbenv/ruby-build.git $(rbenv root)/plugins/ruby-build
          rbenv install ${{ steps.key.outputs.ver }}
          rbenv global ${{ steps.key.outputs.ver }}
          tar cavf ruby.tar.zst .rbenv
      - uses: actions/cache/save@v4
        if: ${{ steps.cache.outputs.cache-hit != 'true' && inputs.verset == 'minver' }}
        id: cache_save
        with:
          key: ${{ steps.key.outputs.key }}
          path: ruby.tar.zst

  # build and test Iguana
  #########################################################

  iguana:
    name: Iguana
    needs:
      - download_test_data
      - build_hipo
      - build_root
      - build_ruby_minver
    runs-on: ${{ inputs.runner }}
    container:
      image: ${{ inputs.container }}
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(inputs.matrix) }}
    env:
      CC: ${{ matrix.CC }}
      CXX: ${{ matrix.CXX }}
    steps:
      ### setup
      - uses: actions/checkout@v4
        with: # settings needed for version number detection
          clean: false
          fetch-tags: true
          fetch-depth: 0
          path: iguana_src
      ### get test data
      - name: get test data
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.download_test_data.outputs.key }}
          path: test_data.hipo
      ### dependencies
      ###### system
      - name: install dependency packages
        run: iguana_src/.github/install-dependency-packages.sh ${{ inputs.runner }} ${{ inputs.verset }}
      ###### ruby
      - name: install ruby from linux package manager
        if: ${{ inputs.id != 'macOS' && inputs.verset != 'minver' }}
        run: pacman -S --noconfirm ruby
      - name: install ruby from cached minver build
        if: ${{ inputs.id != 'macOS' && inputs.verset == 'minver' }}
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build_ruby_minver.outputs.key }}
          path: ruby.tar.zst
      - name: source ruby from cached minver build
        if: ${{ inputs.id != 'macOS' && inputs.verset == 'minver' }}
        run: |
          tar xf ruby.tar.zst
          export RBENV_ROOT=$GITHUB_WORKSPACE/.rbenv
          echo RBENV_ROOT=$RBENV_ROOT >> $GITHUB_ENV
          eval "$($RBENV_ROOT/bin/rbenv init - bash)"
          echo "ruby --version:"
          ruby --version
          echo PATH=$PATH >> $GITHUB_ENV
      ###### python bindings
      - name: install python bindings dependencies
        if: ${{ matrix.id == 'python' }}
        run: |
          python -m venv .venv
          source .venv/bin/activate
          echo PATH=$PATH >> $GITHUB_ENV
          python -m pip install -r iguana_src/bind/python/requirements.txt
      ###### install doxygen
      - name: install doxygen
        if: ${{ matrix.id == 'documentation' }}
        run: |
          pacman -S --noconfirm doxygen graphviz
          echo "INSTALLED VERSIONS:"
          echo "doxygen: $(doxygen --version)"
          echo "$(dot --version)"
      ###### hipo
      - name: get `hipo-withROOT` build
        if: ${{ matrix.id != 'noROOT' }}
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build_hipo.outputs.key }}---withROOT
          path: hipo.tar.zst
      - name: get `hipo-noROOT` build
        if: ${{ matrix.id == 'noROOT' }}
        uses: actions/cache/restore@v4
        with:
          key: ${{  needs.build_hipo.outputs.key }}---noROOT
          path: hipo.tar.zst
      - name: untar hipo build
        run: tar xf hipo.tar.zst
      - run: tree hipo
      ###### ROOT
      - name: get `ROOT` build
        if: ${{ matrix.id != 'noROOT' }}
        uses: actions/cache/restore@v4
        with:
          key: ${{ needs.build_root.outputs.key }}
          path: root.tar.zst
      - name: untar ROOT build
        if: ${{ matrix.id != 'noROOT' }}
        run: tar xf root.tar.zst
      - name: set ROOT environment
        if: ${{ matrix.id != 'noROOT' }}
        run: iguana_src/.github/source-ROOT.sh root
      ###### RCDB
      - name: get RCDB
        uses: actions/checkout@v4
        with:
          repository: JeffersonLab/rcdb
          ref: main
          path: rcdb_src
      - name: set RCDB CI vars
        id: rcdb
        working-directory: rcdb_src
        run: |
          echo prefix=$(pwd) >> $GITHUB_OUTPUT
          echo ver=$(git rev-parse HEAD) >> $GITHUB_OUTPUT
      ###### summarize dependencies
      - name: summarize dependencies
        if: ${{ matrix.id == 'cpp' }}
        run: |
          echo '### Dependencies' >> $GITHUB_STEP_SUMMARY
          echo '| Dependency | Version |' >> $GITHUB_STEP_SUMMARY
          echo '| --- | --- |' >> $GITHUB_STEP_SUMMARY
          echo "| \`hipo\` | \`${{ env.hipo_ref }}\` |" >> $GITHUB_STEP_SUMMARY
          echo "| \`rcdb\` | \`${{ steps.rcdb.outputs.ver }}\` |" >> $GITHUB_STEP_SUMMARY
          echo "| \`root\` | $(root --version 2>&1 | head -n1) |" >> $GITHUB_STEP_SUMMARY
          echo "| \`ruby\` | $(ruby --version) |" >> $GITHUB_STEP_SUMMARY
          cat pkg_summary.md >> $GITHUB_STEP_SUMMARY
      ### build
      - name: meson setup
        run: |
          meson setup iguana_build iguana_src                      \
            --prefix=$GITHUB_WORKSPACE/iguana                      \
            --pkg-config-path=$GITHUB_WORKSPACE/hipo/lib/pkgconfig \
            -Drcdb:home=${{ steps.rcdb.outputs.prefix }}           \
            -Dz_require_rcdb=true                                  \
            -Dwerror=true                                          \
            -Dinstall_examples=true                                \
            -Dtest_data_file=$GITHUB_WORKSPACE/test_data.hipo      \
            -Dtest_num_events=${{ env.num_events }}                \
            -Dtest_output_dir=$GITHUB_WORKSPACE/validation_results \
            -Dtest_num_threads=${{ env.num_threads }}              \
            ${{ matrix.opts }}
      - name: dump all build options
        run: meson configure iguana_build --no-pager
      - name: dump project build options
        run: iguana_src/meson/dump-build-options.sh iguana_build
      - run: meson compile
        working-directory: iguana_build
      - run: meson install
        working-directory: iguana_build
      ### dump info about this build
      - name: dump build log
        if: always()
        run: cat iguana_build/meson-logs/meson-log.txt
      - name: cat pkg-config pc files
        run: |
          pcfiles=$(find iguana -type f -name "*.pc")
          for pcfile in $pcfiles; do
            echo "[+++] cat $pcfile"
            cat $pcfile
          done
      - run: tree iguana
      ### run tests
      - name: meson test
        if: ${{ matrix.id != 'documentation' }}
        working-directory: iguana_build
        run: |
          if [ "${{ env.verbose_test }}" = "true" ]; then
            [ ${{ inputs.runner }} = "macos-latest" ] && stdbuf_cmd=gstdbuf || stdbuf_cmd=stdbuf
            $stdbuf_cmd -o0 meson test --print-errorlogs --verbose --no-stdsplit
          else
            meson test --print-errorlogs
          fi
      ### run benchmarks
      - name: benchmark algorithms
        if: ${{ matrix.id == 'cpp' && inputs.id == 'linux-latest' }}
        run: |
          for suite in single_threaded memoize; do
            meson test --benchmark --suite $suite -C iguana_build | tee benchmark_$suite.txt
          done
          iguana_src/.github/make-benchmark-table.rb benchmark_{single_threaded,memoize}.txt | xargs -0 -I{} echo {} >> $GITHUB_STEP_SUMMARY
      ### reproducible build test
      - name: reproducible build test
        if: ${{ matrix.id == 'cpp' }}
        working-directory: iguana_src
        run: |
          meson reprotest -- \
            --pkg-config-path=$GITHUB_WORKSPACE/hipo/lib/pkgconfig \
            -Drcdb:home=${{ steps.rcdb.outputs.prefix }}
      ### coverage
      - name: coverage
        if: ${{ matrix.id == 'coverage' }}
        run: |
          ninja -C iguana_build coverage-html
          ninja -C iguana_build coverage-text
          mv iguana_build/meson-logs/coveragereport coverage-report
          echo '### Coverage Report' >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          cat iguana_build/meson-logs/coverage.txt >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo '' >> $GITHUB_STEP_SUMMARY
          echo '- for details, see the `coverage-report` artifact' >> $GITHUB_STEP_SUMMARY
          echo '- to compare to the report from the `main` branch, see <https://jeffersonlab.github.io/iguana/coverage-report>' >> $GITHUB_STEP_SUMMARY
      ### test relocatability
      - name: test relocatability
        if: ${{ matrix.id == 'cpp' }} # don't bother re-running santizers, etc.
        run: |
          mv iguana relocated
          source relocated/bin/this_iguana.sh --verbose # do not use --githubCI option, since we want this environment to be for only this step
          relocated/bin/iguana_ex_cpp_00_run_functions test_data.hipo ${{ env.num_events }}
          mv relocated iguana
      ### set iguana environment, since the next steps will check the iguana installation
      - name: set iguana environment
        run: source iguana/bin/this_iguana.sh --verbose --githubCI
      ### test installed examples
      ###### cpp
      - run: iguana_ex_cpp_00_run_functions test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'cpp' }}
      - run: iguana_ex_cpp_01_action_functions test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'cpp' }}
      - run: iguana_ex_cpp_dataframes test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'cpp' }}
      - run: iguana_ex_cpp_config_files iguana/etc/iguana/examples
        if: ${{ matrix.id == 'cpp' }}
      ###### python
      - run: iguana_ex_python_00_run_functions.py test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'python' }}
      - run: iguana_ex_python_01_action_functions.py test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'python' }}
      - run: iguana_ex_python_hipopy.py test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'python' }}
      ###### fortran
      - run: iguana_ex_fortran_01_action_functions test_data.hipo ${{ env.num_events }}
        if: ${{ matrix.id == 'fortran' }}
      ### test ROOT macro
      - name: test iguana_ex_cpp_ROOT_macro.C interpreted
        run: root -b -q iguana_src/examples/iguana_ex_cpp_ROOT_macro.C
        if: ${{ matrix.id == 'cpp' }}
      - name: test iguana_ex_cpp_ROOT_macro.C compiled
        run: root -b -q iguana_src/examples/iguana_ex_cpp_ROOT_macro.C+
        if: ${{ matrix.id == 'cpp' }}
      ### test consumers
      - name: consumer test make
        if: ${{ matrix.id == 'cpp' }}
        run: |
          make -C iguana_src/examples/build_with_make
          echo "========================================= TEST RUN ========================================="
          iguana_src/examples/build_with_make/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
      - name: consumer test meson
        if: ${{ matrix.id == 'cpp' }}
        run: |
          meson setup build_consumer_meson iguana_src/examples/build_with_meson --prefix=$GITHUB_WORKSPACE/install_consumer_meson
          meson install -C build_consumer_meson
          echo "========================================= TEST RUN ========================================="
          install_consumer_meson/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
      - name: consumer test cmake
        if: ${{ matrix.id == 'cpp' }}
        run: |
          cmake -B build_consumer_cmake -S iguana_src/examples/build_with_cmake -DCMAKE_PREFIX_PATH=$GITHUB_WORKSPACE/hipo -G Ninja --install-prefix=$GITHUB_WORKSPACE/install_consumer_cmake
          cmake --build build_consumer_cmake
          cmake --install build_consumer_cmake
          echo "========================================= TEST RUN ========================================="
          install_consumer_cmake/bin/iguana_ex_cpp_00_run_functions test_data.hipo 10
      ### upload artifacts
      - name: upload build log artifacts
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: meson_logs_${{ matrix.id }}
          retention-days: 5
          path: iguana_build/meson-logs
      - name: upload coverage artifacts
        uses: actions/upload-artifact@v4
        if: ${{ matrix.id == 'coverage' }}
        with:
          name: coverage-report
          retention-days: 5
          path: coverage-report
      - name: upload validator artifacts
        uses: actions/upload-artifact@v4
        if: ${{ matrix.id == 'cpp' }}
        with:
          name: _validation_results
          retention-days: 5
          path: validation_results
      - name: upload documentation artifacts
        uses: actions/upload-artifact@v4
        if: ${{ matrix.id == 'documentation' }}
        with:
          name: doc_doxygen
          retention-days: 5
          path: iguana/share/doc/iguana/html/

  # deployment
  #########################################################

  collect_webpages:
    if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }}
    name: Collect webpages
    needs:
      - iguana
    runs-on: ${{ inputs.runner }}
    steps:
      - uses: actions/checkout@v4
        with:
          path: iguana_src
      - name: download doxygen documentation
        uses: actions/download-artifact@v4
        with:
          name: doc_doxygen
          path: doxygen
      - name: download coverage report
        uses: actions/download-artifact@v4
        with:
          name: coverage-report
          path: coverage-report
      - run: tree
      - name: collect
        run: |
          mkdir pages
          cp iguana_src/.github/pages-index.html pages/index.html
          mv doxygen pages/
      - run: mv coverage-report pages/
      - run: tree
      - uses: actions/upload-pages-artifact@v3
        with:
          retention-days: 5
          path: pages/

  deploy_webpages:
    if: ${{ github.ref == 'refs/heads/main' && inputs.id == 'linux-latest' }}
    name: Deploy webpages
    needs: collect_webpages
    permissions:
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ${{ inputs.runner }}
    steps:
      - name: deployment
        id: deployment
        uses: actions/deploy-pages@v4

  # finalize
  #########################################################

  final:
    name: Final
    needs:
      - iguana
    runs-on: ${{ inputs.runner }}
    steps:
      - run: exit 0