ci: Post benchmark results to PR as comment

.github/workflows/bench.yml

@@ -15,6 +15,8 @@ jobs:
   bench:
     name: Benchmark
     runs-on: self-hosted
+    permissions:
+      pull-requests: write
     defaults:
       run:
         shell: bash
@@ -40,22 +42,14 @@ jobs:
           cargo +$TOOLCHAIN bench --features bench --no-run
           cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server
 
-      - name: Download criterion results
+      - name: Download cached main-branch results
         id: criterion-cache
         uses: actions/cache/restore@v4
         with:
           path: ./target/criterion
           key: criterion-${{ runner.os }}-${{ hashFiles('./target/criterion/**.json') }}
           restore-keys: criterion-${{ runner.os }}-
 
-      # - name: Download github-action-benchmark results
-      #   id: benchmark-cache
-      #   uses: actions/cache/restore@v4
-      #   with:
-      #     path: ./cache
-      #     key: action-benchmark-${{ runner.os}}-${{ hashFiles('./cache/**.TODO') }}
-      #     restore-keys: action-benchmark-${{ runner.os }}-
-
       # Disable turboboost, hyperthreading and use performance governor.
       - name: Prepare machine
         run: sudo /root/bin/prep.sh
@@ -64,15 +58,22 @@ jobs:
       - name: Run cargo bench
         run: |
           taskset -c 0 nice -n -20 \
-            cargo +$TOOLCHAIN bench --features bench | tee output.txt
+            cargo +$TOOLCHAIN bench --features bench | tee results.txt
 
       # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
       # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
       - name: Profile cargo bench transfer
         run: |
+          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
+          # Avoid that by shuffling the directories around so this run uses its own results directory.
+          mv target/criterion target/criterion-bench
+          mv target/criterion-transfer-profile target/criterion || true
           taskset -c 0 nice -n -20 \
             cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
               --bench --exact "Run multiple transfers with varying seeds"
+          # And now restore the directories.
+          mv target/criterion target/criterion-transfer-profile
+          mv target/criterion-bench target/criterion
 
       - name: Profile client/server transfer
         run: |
@@ -98,19 +99,6 @@ jobs:
         run: sudo /root/bin/unprep.sh
         if: success() || failure() || cancelled()
 
-      # TODO: Wait for this action to be allowlisted. And then figure out how to only upload
-      # benchmark data when the main branch is being updated (e.g., if: ${{ github.ref == "refs/heads/main" }})
-      # - name: Store current benchmark results
-      #   uses: benchmark-action/github-action-benchmark@v1
-      #   with:
-      #     tool: 'cargo'
-      #     output-file-path: output.txt
-      #     external-data-json-path: ./cache/benchmark-data.json
-      #     fail-on-alert: true
-      #     github-token: ${{ secrets.GITHUB_TOKEN }}
-      #     comment-on-alert: true
-      #     summary-always: true
-
       - name: Convert for profiler.firefox.com
         run: |
           perf script -i perf.data -F +pid > transfer.perf &
@@ -122,27 +110,42 @@ jobs:
           mv server/flamegraph.svg server.svg
           rm neqo.svg
 
-      - name: Upload criterion results
+      - name: Cache main-branch results
         if: github.ref == 'refs/heads/main'
         uses: actions/cache/save@v4
         with:
           path: ./target/criterion
           key: ${{ steps.criterion-cache.outputs.cache-primary-key }}
 
-      # - name: Upload github-action-benchmark results
-      #   if: github.ref == 'refs/heads/main'
-      #   uses: actions/cache/save@v4
-      #   with:
-      #     path: ./cache
-      #     key: ${{ steps.benchmark-cache.outputs.cache-primary-key }}
-
-      - name: Archive perf data
+      - name: Export perf data
+        id: export
         uses: actions/upload-artifact@v4
         with:
           name: ${{ github.event.repository.name }}-${{ github.sha }}
           path: |
             *.svg
             *.perf
-            output.txt
-            target/criterion
+            results.*
+            target/criterion*
           compression-level: 9
+
+      - name: Format results as Markdown
+        id: results
+        run: |
+          grep -Ev 'ignored|running \d+ tests|%\)' results.txt |\
+            sed -E  -e 's/(Performance has regressed.)/:broken_heart: **\1**/gi' \
+                    -e 's/(Performance has improved.)/:green_heart: **\1**/gi' \
+                    -e 's/^ +/  * /gi' \
+                    -e 's/^([a-z0-9].*)$/**\1**/gi' \
+                    -e 's/(change:[^%]*%)([^%]*%)(.*)/\1**\2**\3/gi' \
+              > results.md
+          echo '' >> results.md
+          echo '[:arrow_down: Download full results](${{ steps.export.outputs.artifact-url }})' >> results.md
+
+      - name: "Post results to PR"
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          filePath: results.md
+          pr_number: ${{ github.event.pull_request.number }}
+          comment_tag: bench-results
+

.github/workflows/check.yml

@@ -156,4 +156,6 @@ jobs:
   bench:
     name: "Benchmark"
     needs: [check]
+    permissions:
+      pull-requests: write
     uses: ./.github/workflows/bench.yml