Benchmark PR benches with Bencher

diesel-rs · Nov 14, 2023 · 16150d3 · 16150d3
1 parent ee8617f
commit 16150d3
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 58 deletions.
diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml
@@ -1,18 +1,19 @@
 on:
-  pull_request:
-    types:
-      - labeled
+  pull_request_target:
 
 name: Benchmarks
 
 jobs:
   benchmarks:
-    if: github.event.label.name == 'run-benchmarks'
+    if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks')
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         backend: ["postgres", "sqlite", "mysql"]
+    env:
+      BENCHER_PROJECT: diesel
+      BENCHER_ADAPTER: rust_criterion
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -47,42 +48,30 @@ jobs:
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Install critcmp
-        run: cargo +stable install critcmp
-
-      - name: Benchmark changes
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline changes
-
-      - name: Checkout master
-        run: |
-          git fetch origin
-          git reset --hard origin/master
+      - name: Install Bencher CLI
+        uses: bencherdev/bencher@main
 
       - name: Benchmark master
-        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline master
+        env:
+          BENCHER_BRANCH: master
+          BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
+          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+        run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"'
 
-      - name: Critcmp
-        run: |
-          cd diesel_bench
-          critcmp master changes
-          echo "# ${{matrix.backend}}" >> $GITHUB_STEP_SUMMARY
-          echo "" >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-          critcmp master changes >> $GITHUB_STEP_SUMMARY
-          echo '```' >> $GITHUB_STEP_SUMMARY
-
-      # This does not work due to github not allowing to post comments from forked repos
-      # - name: Post the output as comment
-      #   uses: actions/github-script@v3
-      #   with:
-      #     github-token: ${{secrets.GITHUB_TOKEN}}
-      #     script: |
-      #       const fs = require('fs');
-      #       const data = fs.readFileSync('diesel_bench/output.txt', 'utf8');
+      - name: Checkout PR sources
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
 
-      #       github.issues.createComment({
-      #         issue_number: context.issue.number,
-      #         owner: context.repo.owner,
-      #         repo: context.repo.repo,
-      #         body: data
-      #       })
+      - name: Benchmark PR ${{ github.event.pull_request.number }}
+        env:
+          BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }}
+          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
+        run: |
+          bencher run \\
+          --if-branch "${{ github.event.pull_request.head.ref }}" \\
+          --else-if-branch "${{ github.event.pull_request.base.ref }}" \\
+          --else-if-branch master \\
+          --err \\
+          --github-actions "${{ github.token }}" \\
+          'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"'
diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml
@@ -19,10 +19,6 @@ jobs:
       fail-fast: false
       matrix:
         backend: ["postgres", "sqlite", "mysql"]
-    env:
-      BENCHER_PROJECT: diesel
-      BENCHER_BRANCH: master
-      BENCHER_ADAPTER: rust_criterion
     steps:
       - name: Checkout sources
         uses: actions/checkout@v3
@@ -67,29 +63,17 @@ jobs:
       - name: Install rust toolchain
         uses: dtolnay/rust-toolchain@stable
 
-      - name: Install Bencher CLI
-        uses: bencherdev/bencher@main
-
       - name: Run Benchmarks (Postgres)
         if: matrix.backend == 'postgres'
-        env:
-          BENCHER_TESTBED: ubuntu-latest-postgres
-          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
-        run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres"'
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres"
 
       - name: Run Benchmarks (Sqlite)
         if: matrix.backend == 'sqlite'
-        env:
-          BENCHER_TESTBED: ubuntu-latest-sqlite
-          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
-        run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio"'
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio"
 
       - name: Run Benchmarks (Mysql)
         if: matrix.backend == 'mysql'
-        env:
-          BENCHER_TESTBED: ubuntu-latest-mysql
-          BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }}
-        run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql"'
+        run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql"
 
       - name: Push metrics
         env: