diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 313d17a985c9..a371ca1c313b 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -1,18 +1,19 @@ on: - pull_request: - types: - - labeled + pull_request_target: name: Benchmarks jobs: benchmarks: - if: github.event.label.name == 'run-benchmarks' + if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest strategy: fail-fast: false matrix: backend: ["postgres", "sqlite", "mysql"] + env: + BENCHER_PROJECT: diesel + BENCHER_ADAPTER: rust_criterion steps: - name: Checkout sources uses: actions/checkout@v3 @@ -47,42 +48,34 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Install critcmp - run: cargo +stable install critcmp - - - name: Benchmark changes - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline changes - - - name: Checkout master - run: | - git fetch origin - git reset --hard origin/master + - name: Install Bencher CLI + uses: bencherdev/bencher@main - name: Benchmark master - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline master - - - name: Critcmp + env: + BENCHER_BRANCH: master + BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} run: | - cd diesel_bench - critcmp master changes - echo "# ${{matrix.backend}}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - critcmp master changes >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY + bencher run \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' - # This does not work due to github not allowing to post comments from forked repos - # - name: Post the output as comment - # uses: actions/github-script@v3 - # with: - # github-token: ${{secrets.GITHUB_TOKEN}} - # script: | - # const fs = require('fs'); - # const data = fs.readFileSync('diesel_bench/output.txt', 'utf8'); + - name: Checkout PR sources + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false - # github.issues.createComment({ - # issue_number: context.issue.number, - # owner: context.repo.owner, - # repo: context.repo.repo, - # body: data - # }) + - name: Benchmark PR ${{ github.event.pull_request.number }} + env: + BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} + run: | + bencher run \ + --if-branch "${{ github.event.pull_request.head.ref }}" \ + --else-if-branch "${{ github.event.pull_request.base.ref }}" \ + --else-if-branch master \ + --err \ + --github-actions "${{ secrets.GITHUB_TOKEN }}" \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index b099c054ddf9..95adb4cb4c44 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -19,10 +19,6 @@ jobs: fail-fast: false matrix: backend: ["postgres", "sqlite", "mysql"] - env: - BENCHER_PROJECT: diesel - BENCHER_BRANCH: master - BENCHER_ADAPTER: rust_criterion steps: - name: Checkout sources uses: actions/checkout@v3 @@ -67,29 +63,17 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Install Bencher CLI - uses: bencherdev/bencher@main - - name: Run Benchmarks (Postgres) if: matrix.backend == 'postgres' - env: - BENCHER_TESTBED: ubuntu-latest-postgres - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres"' + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres" - name: Run Benchmarks (Sqlite) if: matrix.backend == 'sqlite' - env: - BENCHER_TESTBED: ubuntu-latest-sqlite - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio"' + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio" - name: Run Benchmarks (Mysql) if: matrix.backend == 'mysql' - env: - BENCHER_TESTBED: ubuntu-latest-mysql - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql"' + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql" - name: Push metrics env: