Merge remote-tracking branch 'origin/develop' into dk/continuous-s3-benchmarking

Author: danking

.github/workflows/bench-pr.yml

@@ -23,41 +23,47 @@ jobs:
           labels: benchmark
 
   codspeed_bench:
-    name: Run Criterion benchmarks with Codspeed
+    name: Benchmark with Codspeed
     needs: label_trigger
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-large
     steps:
       - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
       - uses: ./.github/actions/setup-rust
 
       - name: Install Codspeed
         shell: bash
         run: cargo install --force cargo-codspeed --locked
 
-      - name: Build benchmark targets
+      - name: Build benchmarks
         env:
           RUSTFLAGS: "-C target-cpu=native"
         # We want to run micro-benchmarks with release profile.
         # We run with all features since we feature gate bench utils.
-        run: cargo codspeed build --features test-harness --exclude bench-vortex --workspace --profile release
-
-      - name: Run the benchmarks
+        run: |
+          cargo codspeed build --features test-harness \
+            --exclude bench-vortex \
+            --exclude vortex-datafusion \
+            --exclude vortex-tui \
+            --exclude vortex-fuzz \
+            --exclude pyvortex \
+            --exclude xtask \
+            --workspace \
+            --profile release
+
+      - name: Run benchmarks
         uses: CodSpeedHQ/action@v3
         with:
           run: cargo codspeed run
           token: ${{ secrets.CODSPEED_TOKEN }}
 
-  bench:
+  bench-criterion:
     needs: label_trigger
     strategy:
       matrix:
         benchmark:
-          - id: random_access
-            name: Random Access
           - id: compress
             name: Vortex Compression
-    runs-on: [self-hosted, gcp]
+    runs-on: [ self-hosted, gcp ]
     if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
@@ -125,9 +131,15 @@ jobs:
         with:
           file-path: comment.md
           comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
-  tpch-s3:
+  bench:
     needs: label_trigger
-    runs-on: [self-hosted, aws]
+    runs-on: [ self-hosted, gcp ]
+    strategy:
+      matrix:
+        benchmark:
+          - id: random_access
+            name: Random Access
+    if: ${{ contains(github.event.head_commit.message, '[benchmark]') || github.event.label.name == 'benchmark' && github.event_name == 'pull_request' }}
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/cleanup
@@ -144,75 +156,14 @@ jobs:
         run: |
           echo "TMPDIR=/work" >> $GITHUB_ENV
 
-      - name: Run TPC-H benchmark
+      - name: Run benchmark
         shell: bash
         env:
           BENCH_VORTEX_RATIOS: '.*'
           RUSTFLAGS: '-C target-cpu=native'
         run: |
-          cargo run --bin tpch_benchmark --release -- \
-              --use-remote-data-dir s3://vortex-bench-dev/tpch-sf1/ \
-              --exclude-queries 15 \
-              --formats 'parquet,vortex' \
-              -d gh-json \
-              -t 1 \
-            | tee tpch.json
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-      - name: Compare results
-        shell: bash
-        run: |
-          set -Eeu -o pipefail -x
-
-          base_commit_sha=${{ github.event.pull_request.base.sha }}
-
-          aws s3 cp s3://vortex-benchmark-results-database/data.json - \
-            | grep $base_commit_sha \
-            > base.json
-
-          echo '# Benchmarks: TPC-H on S3' > comment.md
-          echo '<details>' >> comment.md
-          echo '<summary>Table of Results</summary>' >> comment.md
-          echo '' >> comment.md
-          uv run python3 scripts/compare-benchmark-jsons.py base.json tpch.json \
-            >> comment.md
-          echo '</details>' >> comment.md
-      - name: Comment PR
-        uses: thollander/actions-comment-pull-request@v3
-        with:
-          file-path: comment.md
-          comment-tag: bench-pr-comment-tpch-s3
-  tpch-nvme:
-    needs: label_trigger
-    runs-on: [self-hosted, gcp]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-      - uses: ./.github/actions/setup-rust
-      # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
-      - name: Install DuckDB
-        uses: opt-nc/[email protected]
-        if: runner.environment != 'self-hosted'
-        with:
-          version: v1.0.0
-
-      - name: Set tempdir
-        if: runner.environment == 'self-hosted'
-        run: |
-          echo "TMPDIR=/work" >> $GITHUB_ENV
+          cargo run --bin ${{ matrix.benchmark.id }} --release -- -d gh-json -t 1 | tee ${{ matrix.benchmark.id }}.json
 
-      - name: Run TPC-H benchmark
-        shell: bash
-        env:
-          BENCH_VORTEX_RATIOS: '.*'
-          RUSTFLAGS: '-C target-cpu=native'
-        run: |
-          cargo run --bin tpch_benchmark --release -- -d gh-json -t 1 | tee tpch.json
       - name: Setup AWS CLI
         uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -231,73 +182,20 @@ jobs:
             | grep $base_commit_sha \
             > base.json
 
-          echo '# Benchmarks: TPC-H on NVME' > comment.md
+          echo '# Benchmarks: ${{ matrix.benchmark.id }}' > comment.md
           echo '<details>' >> comment.md
           echo '<summary>Table of Results</summary>' >> comment.md
           echo '' >> comment.md
-          uv run python3 scripts/compare-benchmark-jsons.py base.json tpch.json \
+          uv run scripts/compare-benchmark-jsons.py base.json ${{ matrix.benchmark.id }}.json \
             >> comment.md
           echo '</details>' >> comment.md
       - name: Comment PR
         uses: thollander/actions-comment-pull-request@v3
         with:
           file-path: comment.md
-          comment-tag: bench-pr-comment-tpch-nvme
-  clickbench-nvme:
+          comment-tag: bench-pr-comment-${{ matrix.benchmark.id }}
+  sql:
     needs: label_trigger
-    runs-on: [self-hosted, gcp]
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/cleanup
-      - uses: ./.github/actions/setup-rust
-
-      # The compression benchmarks rely on DuckDB being installed to convert CSV to Parquet
-      - name: Install DuckDB
-        uses: opt-nc/[email protected]
-        if: runner.environment != 'self-hosted'
-        with:
-          version: v1.0.0
-
-      - name: Set tempdir
-        if: runner.environment == 'self-hosted'
-        run: |
-          echo "TMPDIR=/work" >> $GITHUB_ENV
-
-      - name: Run Clickbench benchmark
-        shell: bash
-        env:
-          BENCH_VORTEX_RATIOS: '.*'
-          RUSTFLAGS: '-C target-cpu=native'
-          HOME: /home/ci-runner
-        run: |
-          cargo run --bin clickbench --release -- -d gh-json | tee clickbench.json
-      - name: Setup AWS CLI
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: arn:aws:iam::375504701696:role/GitHubBenchmarkRole
-          aws-region: us-east-1
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-      - name: Compare results
-        shell: bash
-        run: |
-          set -Eeu -o pipefail -x
-
-          base_commit_sha=${{ github.event.pull_request.base.sha }}
-
-          aws s3 cp s3://vortex-benchmark-results-database/data.json - \
-            | grep $base_commit_sha \
-            > base.json
-
-          echo '# Benchmarks: Clickbench on NVME' > comment.md
-          echo '<details>' >> comment.md
-          echo '<summary>Table of Results</summary>' >> comment.md
-          echo '' >> comment.md
-          uv run scripts/compare-benchmark-jsons.py base.json clickbench.json \
-            >> comment.md
-          echo '</details>' >> comment.md
-      - name: Comment PR
-        uses: thollander/actions-comment-pull-request@v3
-        with:
-          file-path: comment.md
-          comment-tag: bench-pr-comment-clickbench-nvme
+    uses: ./.github/workflows/sql-benchmarks.yml
+    with:
+      mode: 'pr'