bench.yml

name: Bench
on:
  workflow_run:
    workflows: ["CI"]
    types:
      - completed
  workflow_dispatch:
env:
  CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
  CARGO_PROFILE_RELEASE_DEBUG: true
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  TOOLCHAIN: nightly
  RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
  PERF_CMD: record -o perf.data -F997 --call-graph fp -g

jobs:
  bench:
    name: Benchmark
    runs-on: self-hosted
    permissions:
      pull-requests: write
    defaults:
      run:
        shell: bash

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set PATH
        run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"

      - name: Install Rust
        uses: ./.github/actions/rust
        with:
          version: $TOOLCHAIN
          components: rustfmt

      - name: Fetch and build NSS and NSPR
        uses: ./.github/actions/nss

      - name: Build
        run: |
          cargo "+$TOOLCHAIN" bench --features bench --no-run
          cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server

      - name: Download cached main-branch results
        id: criterion-cache
        uses: actions/cache/restore@v4
        with:
          path: ./target/criterion
          key: criterion-${{ runner.os }}-${{ hashFiles('./target/criterion/**.json') }}
          restore-keys: criterion-${{ runner.os }}-

      # Disable turboboost, hyperthreading and use performance governor.
      - name: Prepare machine
        run: sudo /root/bin/prep.sh

      # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
      - name: Run cargo bench
        run: |
          taskset -c 0 nice -n -20 \
            cargo "+$TOOLCHAIN" bench --features bench | tee results.txt

      # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
      # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
      - name: Profile cargo bench transfer
        run: |
          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
          # Avoid that by shuffling the directories around so this run uses its own results directory.
          mv target/criterion target/criterion-bench
          mv target/criterion-transfer-profile target/criterion || true
          taskset -c 0 nice -n -20 \
            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
              --bench --exact "Run multiple transfers with varying seeds"
          # And now restore the directories.
          mv target/criterion target/criterion-transfer-profile
          mv target/criterion-bench target/criterion

      - name: Profile client/server transfer
        run: |
          { mkdir server; \
            cd server; \
            taskset -c 0 nice -n -20 \
            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
              --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
          mkdir client; \
            cd client; \
            time taskset -c 1 nice -n -20 \
            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
              --bin neqo-client -- --output-dir . "https://$HOST:4433/$SIZE"
          killall -INT neqo-server
          cd ${{ github.workspace }}
          [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
        env:
          HOST: localhost
          SIZE: 1073741824 # 1 GB

      # Re-enable turboboost, hyperthreading and use powersave governor.
      - name: Restore machine
        run: sudo /root/bin/unprep.sh
        if: success() || failure() || cancelled()

      - name: Convert for profiler.firefox.com
        run: |
          perf script -i perf.data -F +pid > transfer.perf &
          perf script -i client/perf.data -F +pid > client.perf &
          perf script -i server/perf.data -F +pid > server.perf &
          wait
          mv flamegraph.svg transfer.svg
          mv client/flamegraph.svg client.svg
          mv server/flamegraph.svg server.svg
          rm neqo.svg

      - name: Generate perf reports
        run: |
          perf report -i perf.data --no-children --stdio > transfer.perf.txt &
          perf report -i client/perf.data --no-children --stdio > client.perf.txt &
          perf report -i server/perf.data --no-children --stdio > server.perf.txt &
          wait

      - name: Cache main-branch results
        if: github.ref == 'refs/heads/main'
        uses: actions/cache/save@v4
        with:
          path: ./target/criterion
          key: ${{ steps.criterion-cache.outputs.cache-primary-key }}

      - name: Export perf data
        id: export
        uses: actions/upload-artifact@v4
        with:
          name: ${{ github.event.repository.name }}-${{ github.sha }}
          path: |
            *.svg
            *.perf
            *.txt
            results.*
            target/criterion*
          compression-level: 9

      - name: Format results as Markdown
        id: results
        run: |
          grep -Ev 'ignored|running \d+ tests|%\)' results.txt |\
            sed -E  -e 's/(Performance has regressed.)/:broken_heart: **\1**/gi' \
                    -e 's/(Performance has improved.)/:green_heart: **\1**/gi' \
                    -e 's/^ +/  * /gi' \
                    -e 's/^([a-z0-9].*)$/**\1**/gi' \
                    -e 's/(change:[^%]*%)([^%]*%)(.*)/\1**\2**\3/gi' \
              > results.md
          echo '' >> results.md

      - name: Export PR comment data
        uses: ./.github/actions/pr-comment-data-export
        with:
          name: bench
          contents: results.md
          log-url: ${{ steps.export.outputs.artifact-url }}