sql-benchmarks.yml

name: "SQL-related benchmarks"

on:
  workflow_call:
    inputs:
      mode:
        required: true
        type: string

jobs:
  bench:
    strategy:
      fail-fast: false
      matrix:
        # Regarding "include:":
        # https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/running-variations-of-jobs-in-a-workflow#example-adding-configurations
        include:
          - id: tpch-nvme
            binary_name: tpch
            name: TPC-H on NVME
          - id: clickbench-nvme
            binary_name: clickbench
            name: Clickbench on NVME
          - id: tpch-s3
            binary_name: tpch
            name: TPC-H on S3
            remote_storage: s3://vortex-bench-dev-eu/tpch-sf1/
    runs-on:
      - runs-on=${{ github.run_id }}
      - family=c7i.8xlarge
      - image=ubuntu24-full-x64
      - disk=large
      - spot=false
      - tag=${{ matrix.include.id }}

    steps:
      - uses: runs-on/action@v1
      - uses: actions/checkout@v4
      # rustup is pre-installed on the ubuntu24-full-x64 image.

      - name: Run ${{ matrix.name }} benchmark
        if: matrix.remote_storage == null
        shell: bash
        env:
          RUSTFLAGS: '-C target-cpu=native'
          OTEL_SERVICE_NAME: 'vortex-bench'
          OTEL_EXPORTER_OTLP_PROTOCOL: 'http/protobuf'
          OTEL_EXPORTER_OTLP_ENDPOINT: '${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }}'
          OTEL_EXPORTER_OTLP_HEADERS: '${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}'
          OTEL_RESOURCE_ATTRIBUTES: 'bench-name=${{ matrix.id }}'
        run: |
          cargo run \
              --bin ${{ matrix.binary_name }} \
              --release \
              -- \
              -d gh-json \
              --export-spans \
            | tee results.json

      - name: Run ${{ matrix.name }} benchmark
        if: matrix.remote_storage != null
        shell: bash
        env:
          RUSTFLAGS: '-C target-cpu=native'
          AWS_REGION: 'eu-west-1'
          OTEL_SERVICE_NAME: 'vortex-bench'
          OTEL_EXPORTER_OTLP_PROTOCOL: 'http/protobuf'
          OTEL_EXPORTER_OTLP_ENDPOINT: '${{ secrets.OTEL_EXPORTER_OTLP_ENDPOINT }}'
          OTEL_EXPORTER_OTLP_HEADERS: '${{ secrets.OTEL_EXPORTER_OTLP_HEADERS }}'
          OTEL_RESOURCE_ATTRIBUTES: 'bench-name=${{ matrix.id }}'
        run: |
          cargo run \
              --bin ${{ matrix.binary_name }} \
              --release \
              -- \
              --use-remote-data-dir ${{ matrix.remote_storage }} \
              --formats 'parquet,vortex' \
              --export-spans \
              -d gh-json \
            | tee results.json

      - name: Setup AWS CLI
        uses: aws-actions/configure-aws-credentials@v4
        with:
          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
          aws-region: us-east-1

      - name: Install uv
        if: inputs.mode == 'pr'
        uses: astral-sh/setup-uv@v5
      - name: Compare results

        if: inputs.mode == 'pr'
        shell: bash
        run: |
          set -Eeu -o pipefail -x

          base_commit_sha=${{ github.event.pull_request.base.sha }}

          aws s3 cp s3://vortex-benchmark-results-database/data.json - \
            | grep $base_commit_sha \
            > base.json

          echo '# Benchmarks: ${{ matrix.name }}' > comment.md
          echo '<details>' >> comment.md
          echo '<summary>Table of Results</summary>' >> comment.md
          echo '' >> comment.md
          uv run --no-project scripts/compare-benchmark-jsons.py base.json results.json \
            >> comment.md
          echo '</details>' >> comment.md

      - name: Comment PR
        if: inputs.mode == 'pr'
        uses: thollander/actions-comment-pull-request@v3
        with:
          file-path: comment.md
          # There is exactly one comment per comment-tag. If a comment with this tag already exists,
          # this action will *update* the comment instead of posting a new comment. Therefore, each
          # unique benchmark configuration must have a unique comment-tag.
          comment-tag: bench-pr-comment-${{ matrix.id }}

      - name: Upload Benchmark Results
        if: inputs.mode == 'develop'
        shell: bash
        run: |
          bash scripts/cat-s3.sh vortex-benchmark-results-database data.json results.json