diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 57ad77d722..176a80deac 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -15,6 +15,8 @@ jobs: bench: name: Benchmark runs-on: self-hosted + permissions: + pull-requests: write defaults: run: shell: bash @@ -40,7 +42,7 @@ jobs: cargo +$TOOLCHAIN bench --features bench --no-run cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server - - name: Download criterion results + - name: Download cached main-branch results id: criterion-cache uses: actions/cache/restore@v4 with: @@ -48,14 +50,6 @@ jobs: key: criterion-${{ runner.os }}-${{ hashFiles('./target/criterion/**.json') }} restore-keys: criterion-${{ runner.os }}- - # - name: Download github-action-benchmark results - # id: benchmark-cache - # uses: actions/cache/restore@v4 - # with: - # path: ./cache - # key: action-benchmark-${{ runner.os}}-${{ hashFiles('./cache/**.TODO') }} - # restore-keys: action-benchmark-${{ runner.os }}- - # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine run: sudo /root/bin/prep.sh @@ -64,15 +58,22 @@ jobs: - name: Run cargo bench run: | taskset -c 0 nice -n -20 \ - cargo +$TOOLCHAIN bench --features bench | tee output.txt + cargo +$TOOLCHAIN bench --features bench | tee results.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - name: Profile cargo bench transfer run: | + # This re-runs part of the previous step, and would hence overwrite part of the criterion results. + # Avoid that by shuffling the directories around so this run uses its own results directory. + mv target/criterion target/criterion-bench + mv target/criterion-transfer-profile target/criterion || true taskset -c 0 nice -n -20 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ --bench --exact "Run multiple transfers with varying seeds" + # And now restore the directories. + mv target/criterion target/criterion-transfer-profile + mv target/criterion-bench target/criterion - name: Profile client/server transfer run: | @@ -98,19 +99,6 @@ jobs: run: sudo /root/bin/unprep.sh if: success() || failure() || cancelled() - # TODO: Wait for this action to be allowlisted. And then figure out how to only upload - # benchmark data when the main branch is being updated (e.g., if: ${{ github.ref == "refs/heads/main" }}) - # - name: Store current benchmark results - # uses: benchmark-action/github-action-benchmark@v1 - # with: - # tool: 'cargo' - # output-file-path: output.txt - # external-data-json-path: ./cache/benchmark-data.json - # fail-on-alert: true - # github-token: ${{ secrets.GITHUB_TOKEN }} - # comment-on-alert: true - # summary-always: true - - name: Convert for profiler.firefox.com run: | perf script -i perf.data -F +pid > transfer.perf & @@ -122,27 +110,42 @@ jobs: mv server/flamegraph.svg server.svg rm neqo.svg - - name: Upload criterion results + - name: Cache main-branch results if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: path: ./target/criterion key: ${{ steps.criterion-cache.outputs.cache-primary-key }} - # - name: Upload github-action-benchmark results - # if: github.ref == 'refs/heads/main' - # uses: actions/cache/save@v4 - # with: - # path: ./cache - # key: ${{ steps.benchmark-cache.outputs.cache-primary-key }} - - - name: Archive perf data + - name: Export perf data + id: export uses: actions/upload-artifact@v4 with: name: ${{ github.event.repository.name }}-${{ github.sha }} path: | *.svg *.perf - output.txt - target/criterion + results.* + target/criterion* compression-level: 9 + + - name: Format results as Markdown + id: results + run: | + grep -Ev 'ignored|running \d+ tests|%\)' results.txt |\ + sed -E -e 's/(Performance has regressed.)/:broken_heart: **\1**/gi' \ + -e 's/(Performance has improved.)/:green_heart: **\1**/gi' \ + -e 's/^ +/ * /gi' \ + -e 's/^([a-z0-9].*)$/**\1**/gi' \ + -e 's/(change:[^%]*%)([^%]*%)(.*)/\1**\2**\3/gi' \ + > results.md + echo '' >> results.md + echo '[:arrow_down: Download full results](${{ steps.export.outputs.artifact-url }})' >> results.md + + - name: "Post results to PR" + uses: thollander/actions-comment-pull-request@v2 + with: + filePath: results.md + pr_number: ${{ github.event.pull_request.number }} + comment_tag: bench-results + diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 2a110e22b1..e96466e2e1 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -156,4 +156,6 @@ jobs: bench: name: "Benchmark" needs: [check] + permissions: + pull-requests: write uses: ./.github/workflows/bench.yml