From 7a9835a177cba21ab36a34b52ac8a765a72d5e4c Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 10 Nov 2023 09:47:59 -0500 Subject: [PATCH 1/9] Track master metrics with Bencher --- .github/workflows/metrics.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index e184e415a921..394376a30210 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -19,6 +19,10 @@ jobs: fail-fast: false matrix: backend: ["postgres", "sqlite", "mysql"] + env: + BENCHER_PROJECT: diesel + BENCHER_BRANCH: master + BENCHER_ADAPTER: rust_criterion steps: - name: Checkout sources uses: actions/checkout@v4 @@ -59,17 +63,29 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install Bencher CLI + uses: bencherdev/bencher@main + - name: Run Benchmarks (Postgres) if: matrix.backend == 'postgres' - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres wtx" + env: + BENCHER_TESTBED: ubuntu-latest-postgres + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres wtx"' - name: Run Benchmarks (Sqlite) if: matrix.backend == 'sqlite' - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio" + env: + BENCHER_TESTBED: ubuntu-latest-sqlite + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio"' - name: Run Benchmarks (Mysql) if: matrix.backend == 'mysql' - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql" + env: + BENCHER_TESTBED: ubuntu-latest-mysql + BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} + run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql"' - name: Push metrics env: From 2fa050f37a05ef89ab1700bdd4b155805bdd5d36 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Tue, 14 Nov 2023 07:41:35 -0500 Subject: [PATCH 2/9] Benchmark PR benches with Bencher --- .github/workflows/benches.yml | 69 ++++++++++++++++------------------- .github/workflows/metrics.yml | 16 +------- 2 files changed, 33 insertions(+), 52 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 6946e1d0c84e..ec3cce5cca06 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -1,18 +1,19 @@ on: - pull_request: - types: - - labeled + pull_request_target: name: Benchmarks jobs: benchmarks: - if: github.event.label.name == 'run-benchmarks' + if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest strategy: fail-fast: false matrix: backend: ["postgres", "sqlite", "mysql"] + env: + BENCHER_PROJECT: diesel + BENCHER_ADAPTER: rust_criterion steps: - name: Checkout sources uses: actions/checkout@v4 @@ -47,42 +48,34 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Install critcmp - run: cargo +stable install critcmp - - - name: Benchmark changes - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline changes - - - name: Checkout master - run: | - git fetch origin - git reset --hard origin/master + - name: Install Bencher CLI + uses: bencherdev/bencher@main - name: Benchmark master - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}" -- --save-baseline master - - - name: Critcmp + env: + BENCHER_BRANCH: master + BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} run: | - cd diesel_bench - critcmp master changes - echo "# ${{matrix.backend}}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY - critcmp master changes >> $GITHUB_STEP_SUMMARY - echo '```' >> $GITHUB_STEP_SUMMARY + bencher run \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' - # This does not work due to github not allowing to post comments from forked repos - # - name: Post the output as comment - # uses: actions/github-script@v3 - # with: - # github-token: ${{secrets.GITHUB_TOKEN}} - # script: | - # const fs = require('fs'); - # const data = fs.readFileSync('diesel_bench/output.txt', 'utf8'); + - name: Checkout PR sources + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + persist-credentials: false - # github.issues.createComment({ - # issue_number: context.issue.number, - # owner: context.repo.owner, - # repo: context.repo.repo, - # body: data - # }) + - name: Benchmark PR ${{ github.event.pull_request.number }} + env: + BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} + run: | + bencher run \ + --if-branch "${{ github.event.pull_request.head.ref }}" \ + --else-if-branch "${{ github.event.pull_request.base.ref }}" \ + --else-if-branch master \ + --err \ + --github-actions "${{ secrets.GITHUB_TOKEN }}" \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' diff --git a/.github/workflows/metrics.yml b/.github/workflows/metrics.yml index 394376a30210..8ae05ce42eb9 100644 --- a/.github/workflows/metrics.yml +++ b/.github/workflows/metrics.yml @@ -63,29 +63,17 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Install Bencher CLI - uses: bencherdev/bencher@main - - name: Run Benchmarks (Postgres) if: matrix.backend == 'postgres' - env: - BENCHER_TESTBED: ubuntu-latest-postgres - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "postgres sqlx-bench sqlx/postgres rust_postgres futures sea-orm sea-orm/sqlx-postgres criterion/async_tokio quaint quaint/postgresql quaint/serde-support serde diesel-async diesel-async/postgres wtx"' - name: Run Benchmarks (Sqlite) if: matrix.backend == 'sqlite' - env: - BENCHER_TESTBED: ubuntu-latest-sqlite - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio"' + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "sqlite sqlx-bench sqlx/sqlite tokio rusqlite futures sea-orm sea-orm/sqlx-sqlite criterion/async_tokio" - name: Run Benchmarks (Mysql) if: matrix.backend == 'mysql' - env: - BENCHER_TESTBED: ubuntu-latest-mysql - BENCHER_API_TOKEN: ${{ secrets.BENCHER_API_TOKEN }} - run: bencher run 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql"' + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "mysql sqlx-bench sqlx/mysql tokio rustorm rustorm/with-mysql rustorm_dao rust_mysql futures sea-orm sea-orm/sqlx-mysql criterion/async_tokio quaint quaint/mysql quaint/serde-support serde diesel-async diesel-async/mysql" - name: Push metrics env: From 7da1508681073b2481501680e3cd2df646d04247 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sat, 2 Dec 2023 06:06:55 -0500 Subject: [PATCH 3/9] Only benchmark master on push --- .github/workflows/benches.yml | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index ec3cce5cca06..768f3229d9b8 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -1,11 +1,13 @@ on: + push: + branches: master pull_request_target: name: Benchmarks jobs: benchmarks: - if: contains(github.event.pull_request.labels.*.name, 'run-benchmarks') + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) runs-on: ubuntu-latest strategy: fail-fast: false @@ -52,15 +54,17 @@ jobs: uses: bencherdev/bencher@main - name: Benchmark master + if: github.event_name == 'push' && github.ref == 'refs/heads/master' env: BENCHER_BRANCH: master BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} run: | bencher run \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ - 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"' - name: Checkout PR sources + if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') uses: actions/checkout@v3 with: ref: ${{ github.event.pull_request.head.sha }} @@ -68,14 +72,15 @@ jobs: persist-credentials: false - name: Benchmark PR ${{ github.event.pull_request.number }} + if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') env: BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} run: | bencher run \ - --if-branch "${{ github.event.pull_request.head.ref }}" \ - --else-if-branch "${{ github.event.pull_request.base.ref }}" \ + --if-branch '${{ github.event.pull_request.head.ref }}' \ + --else-if-branch '${{ github.event.pull_request.base.ref }}' \ --else-if-branch master \ --err \ - --github-actions "${{ secrets.GITHUB_TOKEN }}" \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ - 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{matrix.backend}}"' + --github-actions '${{ secrets.GITHUB_TOKEN }}' \ + --token '${{ secrets.BENCHER_API_TOKEN }}' \ + 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"' From 5192cb6722ce361088e6fe9587374bf1560b38f9 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Wed, 13 Dec 2023 11:26:32 -0500 Subject: [PATCH 4/9] Split out run and upload --- .github/workflows/benches.yml | 86 --------------------------- .github/workflows/run_benches.yml | 64 ++++++++++++++++++++ .github/workflows/track_benches.yml | 90 +++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 86 deletions(-) delete mode 100644 .github/workflows/benches.yml create mode 100644 .github/workflows/run_benches.yml create mode 100644 .github/workflows/track_benches.yml diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml deleted file mode 100644 index 768f3229d9b8..000000000000 --- a/.github/workflows/benches.yml +++ /dev/null @@ -1,86 +0,0 @@ -on: - push: - branches: master - pull_request_target: - -name: Benchmarks - -jobs: - benchmarks: - if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - backend: ["postgres", "sqlite", "mysql"] - env: - BENCHER_PROJECT: diesel - BENCHER_ADAPTER: rust_criterion - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - - name: Install postgres (Linux) - if: matrix.backend == 'postgres' - run: | - sudo apt-get update - sudo apt-get install -y libpq-dev postgresql - echo "host all all 127.0.0.1/32 md5" > sudo tee -a /etc/postgresql/10/main/pg_hba.conf - sudo service postgresql restart && sleep 3 - sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" - sudo service postgresql restart && sleep 3 - echo 'DATABASE_URL=postgres://postgres:postgres@localhost/' >> $GITHUB_ENV - - - name: Install sqlite (Linux) - if: matrix.backend == 'sqlite' - run: | - sudo apt-get update - sudo apt-get install -y libsqlite3-dev - echo 'DATABASE_URL=/tmp/test.db' >> $GITHUB_ENV - - - name: Install mysql (Linux) - if: matrix.backend == 'mysql' - run: | - sudo systemctl start mysql.service - sudo apt-get update - sudo apt-get -y install libmysqlclient-dev - mysql -e "create database diesel_test; create database diesel_unit_test; grant all on \`diesel_%\`.* to 'root'@'localhost';" -uroot -proot - echo 'DATABASE_URL=mysql://root:root@localhost/diesel_test' >> $GITHUB_ENV - - - name: Install rust toolchain - uses: dtolnay/rust-toolchain@stable - - - name: Install Bencher CLI - uses: bencherdev/bencher@main - - - name: Benchmark master - if: github.event_name == 'push' && github.ref == 'refs/heads/master' - env: - BENCHER_BRANCH: master - BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} - run: | - bencher run \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"' - - - name: Checkout PR sources - if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') - uses: actions/checkout@v3 - with: - ref: ${{ github.event.pull_request.head.sha }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - persist-credentials: false - - - name: Benchmark PR ${{ github.event.pull_request.number }} - if: github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') - env: - BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} - run: | - bencher run \ - --if-branch '${{ github.event.pull_request.head.ref }}' \ - --else-if-branch '${{ github.event.pull_request.base.ref }}' \ - --else-if-branch master \ - --err \ - --github-actions '${{ secrets.GITHUB_TOKEN }}' \ - --token '${{ secrets.BENCHER_API_TOKEN }}' \ - 'cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}"' diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml new file mode 100644 index 000000000000..e2a4f80392f2 --- /dev/null +++ b/.github/workflows/run_benches.yml @@ -0,0 +1,64 @@ +on: + push: + branches: master + pull_request: + types: [labeled, opened, reopened, synchronize] + +name: Run and Cache Benchmarks + +jobs: + run_benchmarks: + if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + backend: ["postgres", "sqlite", "mysql"] + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Install postgres (Linux) + if: matrix.backend == 'postgres' + run: | + sudo apt-get update + sudo apt-get install -y libpq-dev postgresql + echo "host all all 127.0.0.1/32 md5" > sudo tee -a /etc/postgresql/10/main/pg_hba.conf + sudo service postgresql restart && sleep 3 + sudo -u postgres psql -c "ALTER USER postgres PASSWORD 'postgres';" + sudo service postgresql restart && sleep 3 + echo 'DATABASE_URL=postgres://postgres:postgres@localhost/' >> $GITHUB_ENV + + - name: Install sqlite (Linux) + if: matrix.backend == 'sqlite' + run: | + sudo apt-get update + sudo apt-get install -y libsqlite3-dev + echo 'DATABASE_URL=/tmp/test.db' >> $GITHUB_ENV + + - name: Install mysql (Linux) + if: matrix.backend == 'mysql' + run: | + sudo systemctl start mysql.service + sudo apt-get update + sudo apt-get -y install libmysqlclient-dev + mysql -e "create database diesel_test; create database diesel_unit_test; grant all on \`diesel_%\`.* to 'root'@'localhost';" -uroot -proot + echo 'DATABASE_URL=mysql://root:root@localhost/diesel_test' >> $GITHUB_ENV + + - name: Install rust toolchain + uses: dtolnay/rust-toolchain@stable + + - name: Benchmark ${{ matrix.backend }} + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > ${{ matrix.backend }}.txt + + - name: Upload ${{ matrix.backend }} Benchmark Results + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.backend }}.txt + path: ./${{ matrix.backend }}.txt + + - name: Upload GitHub Event + uses: actions/upload-artifact@v3 + with: + name: event.json + path: ${{ github.event_path }} diff --git a/.github/workflows/track_benches.yml b/.github/workflows/track_benches.yml new file mode 100644 index 000000000000..38db2958dddc --- /dev/null +++ b/.github/workflows/track_benches.yml @@ -0,0 +1,90 @@ +on: + workflow_run: + workflows: [Run and Cache Benchmarks] + types: + - completed + +name: Track Benchmarks + +jobs: + track_benchmarks: + if: github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + backend: ["postgres", "sqlite", "mysql"] + env: + BENCHER_PROJECT: diesel + BENCHER_ADAPTER: rust_criterion + BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} + BENCHMARK_RESULTS: ${{ matrix.backend }}.txt + GITHUB_EVENT: event.json + steps: + - name: Download Benchmark Results + uses: actions/github-script@v6 + with: + script: | + async function downloadArtifact(artifactName) { + let allArtifacts = await github.rest.actions.listWorkflowRunArtifacts({ + owner: context.repo.owner, + repo: context.repo.repo, + run_id: context.payload.workflow_run.id, + }); + let matchArtifact = allArtifacts.data.artifacts.filter((artifact) => { + return artifact.name == artifactName + })[0]; + if (!matchArtifact) { + core.setFailed(`Failed to find artifact: ${artifactName}`); + } + let download = await github.rest.actions.downloadArtifact({ + owner: context.repo.owner, + repo: context.repo.repo, + artifact_id: matchArtifact.id, + archive_format: 'zip', + }); + let fs = require('fs'); + fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); + } + await downloadArtifact(process.env.BENCHMARK_RESULTS); + await downloadArtifact(process.env.GITHUB_EVENT); + - name: Unzip Benchmark Results + run: | + unzip $BENCHMARK_RESULTS.zip + unzip $GITHUB_EVENT.zip + - name: Export GitHub Event Data + uses: actions/github-script@v6 + with: + script: | + let fs = require('fs'); + let githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT, {encoding: 'utf8'})); + console.log(githubEvent); + if (githubEvent.ref) { + core.exportVariable("EVENT_NAME", "push"); + core.exportVariable("PUSH_REF", githubEvent.ref); + } else { + core.exportVariable("EVENT_NAME", "pull_request"); + core.exportVariable("PR_HEAD", githubEvent.pull_request.head.ref); + core.exportVariable("PR_BASE", githubEvent.pull_request.base.ref); + core.exportVariable("PR_DEFAULT", githubEvent.pull_request.base.repo.default_branch); + core.exportVariable("PR_NUMBER", githubEvent.number); + } + - uses: bencherdev/bencher@main + - name: Track Benchmarks + run: | + (${{ env.EVENT_NAME == 'push' && env.PUSH_REF == 'refs/heads/master' }} && \ + bencher run \ + --branch "master" \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --err \ + --file "$BENCHMARK_RESULTS") || \ + (${{ env.EVENT_NAME == 'pull_request'}} && \ + bencher run \ + --if-branch '${{ env.PR_HEAD }}' \ + --else-if-branch '${{ env.PR_BASE }}' \ + --else-if-branch '${{ env.PR_DEFAULT }}' \ + --ci-number '${{ env.PR_NUMBER }}' \ + --github-actions "${{ secrets.GITHUB_TOKEN }}" \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --err \ + --file "$BENCHMARK_RESULTS") From e9407235e83cf6450b4331cacc64a968f2488e62 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Fri, 15 Dec 2023 09:45:46 -0500 Subject: [PATCH 5/9] Relative Benchmarking --- .github/workflows/run_benches.yml | 29 +++++++++++----- .github/workflows/track_benches.yml | 53 ++++++++++++++++------------- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml index e2a4f80392f2..3b7e9640763b 100644 --- a/.github/workflows/run_benches.yml +++ b/.github/workflows/run_benches.yml @@ -1,6 +1,4 @@ on: - push: - branches: master pull_request: types: [labeled, opened, reopened, synchronize] @@ -8,7 +6,7 @@ name: Run and Cache Benchmarks jobs: run_benchmarks: - if: (github.event_name == 'push' && github.ref == 'refs/heads/master') || (github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks')) + if: github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'run-benchmarks') runs-on: ubuntu-latest strategy: fail-fast: false @@ -48,14 +46,29 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable - - name: Benchmark ${{ matrix.backend }} - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > ${{ matrix.backend }}.txt + - name: Benchmark PR ${{ matrix.backend }} + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > pr_${{ matrix.backend }}.txt - - name: Upload ${{ matrix.backend }} Benchmark Results + - name: Upload PR ${{ matrix.backend }} Benchmark Results uses: actions/upload-artifact@v3 with: - name: ${{ matrix.backend }}.txt - path: ./${{ matrix.backend }}.txt + name: pr_${{ matrix.backend }}.txt + path: ./pr_${{ matrix.backend }}.txt + + - name: Checkout base branch + uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.base.sha }} + repository: ${{ github.event.pull_request.base.repo.full_name }} + + - name: Benchmark base ${{ matrix.backend }} + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > base_${{ matrix.backend }}.txt + + - name: Upload base ${{ matrix.backend }} Benchmark Results + uses: actions/upload-artifact@v3 + with: + name: base_${{ matrix.backend }}.txt + path: ./base_${{ matrix.backend }}.txt - name: Upload GitHub Event uses: actions/upload-artifact@v3 diff --git a/.github/workflows/track_benches.yml b/.github/workflows/track_benches.yml index 38db2958dddc..048e4379ec68 100644 --- a/.github/workflows/track_benches.yml +++ b/.github/workflows/track_benches.yml @@ -18,7 +18,8 @@ jobs: BENCHER_PROJECT: diesel BENCHER_ADAPTER: rust_criterion BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} - BENCHMARK_RESULTS: ${{ matrix.backend }}.txt + PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt + BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt GITHUB_EVENT: event.json steps: - name: Download Benchmark Results @@ -46,11 +47,13 @@ jobs: let fs = require('fs'); fs.writeFileSync(`${process.env.GITHUB_WORKSPACE}/${artifactName}.zip`, Buffer.from(download.data)); } - await downloadArtifact(process.env.BENCHMARK_RESULTS); + await downloadArtifact(process.env.PR_BENCHMARK_RESULTS); + await downloadArtifact(process.env.BASE_BENCHMARK_RESULTS); await downloadArtifact(process.env.GITHUB_EVENT); - name: Unzip Benchmark Results run: | - unzip $BENCHMARK_RESULTS.zip + unzip $PR_BENCHMARK_RESULTS.zip + unzip $BASE_BENCHMARK_RESULTS.zip unzip $GITHUB_EVENT.zip - name: Export GitHub Event Data uses: actions/github-script@v6 @@ -59,32 +62,34 @@ jobs: let fs = require('fs'); let githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT, {encoding: 'utf8'})); console.log(githubEvent); - if (githubEvent.ref) { - core.exportVariable("EVENT_NAME", "push"); - core.exportVariable("PUSH_REF", githubEvent.ref); - } else { - core.exportVariable("EVENT_NAME", "pull_request"); - core.exportVariable("PR_HEAD", githubEvent.pull_request.head.ref); - core.exportVariable("PR_BASE", githubEvent.pull_request.base.ref); - core.exportVariable("PR_DEFAULT", githubEvent.pull_request.base.repo.default_branch); - core.exportVariable("PR_NUMBER", githubEvent.number); - } + core.exportVariable("PR_HEAD", `${githubEvent.pull_request.head.ref}-${githubEvent.pull_request.head.sha.slice(0, 8)}`); + core.exportVariable("PR_ID", `${githubEvent.pull_request.head.ref}/${process.env.BENCHER_TESTBED}/${process.env.BENCHER_ADAPTER}`); + core.exportVariable("PR_NUMBER", githubEvent.number); - uses: bencherdev/bencher@main - - name: Track Benchmarks + - name: Track base Benchmarks run: | - (${{ env.EVENT_NAME == 'push' && env.PUSH_REF == 'refs/heads/master' }} && \ bencher run \ - --branch "master" \ + --if-branch '${{ env.PR_HEAD }}' \ + --else-branch \ --token "${{ secrets.BENCHER_API_TOKEN }}" \ - --err \ - --file "$BENCHMARK_RESULTS") || \ - (${{ env.EVENT_NAME == 'pull_request'}} && \ + --file "$BASE_BENCHMARK_RESULTS" + - name: Create PR threshold + run: | + bencher threshold create \ + --project "$BENCHER_PROJECT" \ + --branch '${{ env.PR_HEAD }}' \ + --testbed "$BENCHER_TESTBED" \ + --measure latency \ + --test t \ + --upper-boundary 0.98 \ + --token "${{ secrets.BENCHER_API_TOKEN }}" + - name: Track PR Benchmarks + run: | bencher run \ - --if-branch '${{ env.PR_HEAD }}' \ - --else-if-branch '${{ env.PR_BASE }}' \ - --else-if-branch '${{ env.PR_DEFAULT }}' \ + --branch '${{ env.PR_HEAD }}' \ + --token "${{ secrets.BENCHER_API_TOKEN }}" \ + --ci-id '${{ env.PR_ID }}' \ --ci-number '${{ env.PR_NUMBER }}' \ --github-actions "${{ secrets.GITHUB_TOKEN }}" \ - --token "${{ secrets.BENCHER_API_TOKEN }}" \ --err \ - --file "$BENCHMARK_RESULTS") + --file "$PR_BENCHMARK_RESULTS" From 424f7b47be9ae595b6454f37f5b859f2d6912a0d Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sun, 17 Dec 2023 08:56:53 -0500 Subject: [PATCH 6/9] Bump GHA Upload --- .github/workflows/run_benches.yml | 8 ++++---- .github/workflows/track_benches.yml | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml index 3b7e9640763b..29fdf2056f05 100644 --- a/.github/workflows/run_benches.yml +++ b/.github/workflows/run_benches.yml @@ -50,7 +50,7 @@ jobs: run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > pr_${{ matrix.backend }}.txt - name: Upload PR ${{ matrix.backend }} Benchmark Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: pr_${{ matrix.backend }}.txt path: ./pr_${{ matrix.backend }}.txt @@ -65,13 +65,13 @@ jobs: run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > base_${{ matrix.backend }}.txt - name: Upload base ${{ matrix.backend }} Benchmark Results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: base_${{ matrix.backend }}.txt path: ./base_${{ matrix.backend }}.txt - name: Upload GitHub Event - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: event.json + name: event_${{ matrix.backend }}.json path: ${{ github.event_path }} diff --git a/.github/workflows/track_benches.yml b/.github/workflows/track_benches.yml index 048e4379ec68..77965d353dc3 100644 --- a/.github/workflows/track_benches.yml +++ b/.github/workflows/track_benches.yml @@ -20,7 +20,7 @@ jobs: BENCHER_TESTBED: ubuntu-latest-${{ matrix.backend }} PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt - GITHUB_EVENT: event.json + GITHUB_EVENT: event_${{ matrix.backend }}.json steps: - name: Download Benchmark Results uses: actions/github-script@v6 @@ -60,7 +60,7 @@ jobs: with: script: | let fs = require('fs'); - let githubEvent = JSON.parse(fs.readFileSync(process.env.GITHUB_EVENT, {encoding: 'utf8'})); + let githubEvent = JSON.parse(fs.readFileSync("event.json", {encoding: 'utf8'})); console.log(githubEvent); core.exportVariable("PR_HEAD", `${githubEvent.pull_request.head.ref}-${githubEvent.pull_request.head.sha.slice(0, 8)}`); core.exportVariable("PR_ID", `${githubEvent.pull_request.head.ref}/${process.env.BENCHER_TESTBED}/${process.env.BENCHER_ADAPTER}`); From 0899f653ab12fc897e1c6cf9aca77a976d897754 Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Mon, 29 Jan 2024 09:43:52 -0500 Subject: [PATCH 7/9] upper_boundary --- .github/workflows/track_benches.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/track_benches.yml b/.github/workflows/track_benches.yml index 77965d353dc3..c2a343e868d5 100644 --- a/.github/workflows/track_benches.yml +++ b/.github/workflows/track_benches.yml @@ -21,6 +21,11 @@ jobs: PR_BENCHMARK_RESULTS: pr_${{ matrix.backend }}.txt BASE_BENCHMARK_RESULTS: base_${{ matrix.backend }}.txt GITHUB_EVENT: event_${{ matrix.backend }}.json + # This is the confidence interval for the t-test Threshold + # Adjust this value to lower to make the test more sensitive to changes + # Adjust this value to higher to make the test less sensitive to changes + # https://bencher.dev/docs/explanation/thresholds/#t-test-threshold-upper-boundary + UPPER_BOUNDARY: 0.98 steps: - name: Download Benchmark Results uses: actions/github-script@v6 @@ -81,7 +86,7 @@ jobs: --testbed "$BENCHER_TESTBED" \ --measure latency \ --test t \ - --upper-boundary 0.98 \ + --upper-boundary ${{ env.UPPER_BOUNDARY }} \ --token "${{ secrets.BENCHER_API_TOKEN }}" - name: Track PR Benchmarks run: | From 12f649d9a03d6cf91495c0ea2e8e65f4a554d33f Mon Sep 17 00:00:00 2001 From: Everett Pompeii Date: Sat, 3 Feb 2024 07:37:50 -0500 Subject: [PATCH 8/9] Keep Critcmp --- .github/workflows/run_benches.yml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml index 29fdf2056f05..af94b464dac5 100644 --- a/.github/workflows/run_benches.yml +++ b/.github/workflows/run_benches.yml @@ -46,8 +46,11 @@ jobs: - name: Install rust toolchain uses: dtolnay/rust-toolchain@stable + - name: Install critcmp + run: cargo +stable install critcmp + - name: Benchmark PR ${{ matrix.backend }} - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > pr_${{ matrix.backend }}.txt + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline changes > pr_${{ matrix.backend }}.txt - name: Upload PR ${{ matrix.backend }} Benchmark Results uses: actions/upload-artifact@v4 @@ -62,7 +65,7 @@ jobs: repository: ${{ github.event.pull_request.base.repo.full_name }} - name: Benchmark base ${{ matrix.backend }} - run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" > base_${{ matrix.backend }}.txt + run: cargo +stable bench --manifest-path diesel_bench/Cargo.toml --no-default-features --features "${{ matrix.backend }}" -- --save-baseline master > base_${{ matrix.backend }}.txt - name: Upload base ${{ matrix.backend }} Benchmark Results uses: actions/upload-artifact@v4 @@ -75,3 +78,13 @@ jobs: with: name: event_${{ matrix.backend }}.json path: ${{ github.event_path }} + + - name: Critcmp + run: | + cd diesel_bench + critcmp master changes + echo "# ${{matrix.backend}}" >> $GITHUB_STEP_SUMMARY + echo "" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + critcmp master changes >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY From 9dd67d5d5b8cd9fd15889931bf02686b567797f2 Mon Sep 17 00:00:00 2001 From: Georg Semmler Date: Thu, 22 Feb 2024 08:16:55 +0100 Subject: [PATCH 9/9] Use checkout@v4 --- .github/workflows/run_benches.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run_benches.yml b/.github/workflows/run_benches.yml index af94b464dac5..8917fc6158ce 100644 --- a/.github/workflows/run_benches.yml +++ b/.github/workflows/run_benches.yml @@ -14,7 +14,7 @@ jobs: backend: ["postgres", "sqlite", "mysql"] steps: - name: Checkout sources - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install postgres (Linux) if: matrix.backend == 'postgres' @@ -59,7 +59,7 @@ jobs: path: ./pr_${{ matrix.backend }}.txt - name: Checkout base branch - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ github.event.pull_request.base.sha }} repository: ${{ github.event.pull_request.base.repo.full_name }}