ci: Benchmark without a qlog

.github/workflows/bench.yml

@@ -3,6 +3,7 @@ on:
   workflow_call:
 env:
   CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
+  CARGO_PROFILE_RELEASE_DEBUG: true
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
@@ -22,53 +23,93 @@ jobs:
           toolchain: $TOOLCHAIN
           components: rustfmt
 
-      - name: Configure Rust
-        run: echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment" >> "$GITHUB_ENV"
-
       - name: Install sccache
         uses: mozilla-actions/[email protected]
 
-      - name: Enable sccache
+      - name: Configure Rust
         run: |
+          echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment" >> "$GITHUB_ENV"
           echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
           echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
+          cargo install flamegraph
 
       - name: Checkout
         uses: actions/checkout@v4
 
+      - name: Fetch NSS and NSPR
+        run: |
+          hg clone https://hg.mozilla.org/projects/nspr "$NSPR_DIR"
+          hg clone https://hg.mozilla.org/projects/nss "$NSS_DIR"
+          echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV"
+          echo "NSPR_DIR=$NSPR_DIR" >> "$GITHUB_ENV"
+        env:
+          NSS_DIR: ${{ github.workspace }}/nss
+          NSPR_DIR: ${{ github.workspace }}/nspr
+
       - name: Build
-        run: cargo +$TOOLCHAIN bench --features ci,bench --no-run
+        run: |
+          cargo +$TOOLCHAIN bench --features bench --no-run
+          cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server
+          echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Debug/lib" >> "$GITHUB_ENV"
+
+      - name: Download criterion results
+        uses: actions/cache/restore@v4
+        with:
+          path: ./target/criterion
+          key: neqo-${{ runner.os }}-main-criterion
+
+      - name: Download github-action-benchmark results
+        uses: actions/cache/restore@v4
+        with:
+          path: ./cache
+          key: neqo-${{ runner.os }}-main-github-action-benchmark
 
       # Disable turboboost, hyperthreading and use performance governor.
       - name: Prepare machine
-        run: sudo /root/bin/prep.sh
+        run: |
+          sudo /root/bin/prep.sh
+          echo "PERF_CMD=record -o perf.data -F997 --call-graph dwarf,16384 -g" >> "$GITHUB_ENV"
 
       # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
-      - name: Benchmark
+      - name: Run cargo bench
         run: |
-          nice -n -20 taskset -c 0 \
-            cargo +$TOOLCHAIN bench --features ci,bench | tee output.txt
+          taskset -c 0 nice -n -20 \
+            cargo +$TOOLCHAIN bench --features bench | tee output.txt
 
       # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
-      - name: Perf transfer benchmark
+      # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
+      - name: Profile cargo bench transfer
+        run: |
+          taskset -c 0 nice -n -20 \
+            cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
+              --bench --exact "Run multiple transfers with varying seeds"
+
+      - name: Profile client/server transfer
         run: |
-          nice -n -20 taskset -c 0 \
-            perf record -F997 --call-graph=lbr -o perf.data \
-            cargo +$TOOLCHAIN bench --features ci,bench --bench transfer
+          { mkdir server; \
+            cd server; \
+            taskset -c 0 nice -n -20 \
+            cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
+              --bin neqo-server -- --db ../test-fixture/db $HOST:4433 || true; } &
+          mkdir client; \
+            cd client; \
+            time taskset -c 1 nice -n -20 \
+            cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \
+              --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE
+          killall -INT neqo-server
+          cd ${{ github.workspace }}
+          [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+        env:
+          HOST: localhost
+          SIZE: 1073741824 # 1 GB
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine
         run: sudo /root/bin/unprep.sh
-        if: success() || failure()
-
-      - name: Download previous benchmark results
-        uses: actions/cache@v4
-        with:
-          path: ./cache
-          key: ${{ runner.os }}-benchmark
+        if: success() || failure() || cancelled()
 
       # TODO: Wait for this action to be allowlisted. And then figure out how to only upload
-      # benchmark data when the main branch is being updated.
+      # benchmark data when the main branch is being updated (e.g., if: ${{ github.ref == "refs/heads/main" }})
       # - name: Store current benchmark results
       #   uses: benchmark-action/github-action-benchmark@v1
       #   with:
@@ -80,14 +121,38 @@ jobs:
       #     comment-on-alert: true
       #     summary-always: true
 
-      - name: Convert perf data
+      - name: Convert for profiler.firefox.com
         run: |
-          perf script -i perf.data -F +pid | zstd > perf.ff.data.zst
-          zstd perf.data
+          perf script -i perf.data -F +pid > transfer.perf &
+          perf script -i client/perf.data -F +pid > client.perf &
+          perf script -i server/perf.data -F +pid > server.perf &
+          wait
+          mv flamegraph.svg transfer.svg
+          mv client/flamegraph.svg client.svg
+          mv server/flamegraph.svg server.svg
+          rm neqo.svg
+
+      - name: Upload criterion results
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@v4
+        with:
+          path: ./target/criterion
+          key: neqo-${{ runner.os }}-main-criterion
+
+      - name: Upload github-action-benchmark results
+        if: github.ref == 'refs/heads/main'
+        uses: actions/cache/save@v4
+        with:
+          path: ./cache
+          key: neqo-${{ runner.os }}-main-github-action-benchmark
 
       - name: Archive perf data
         uses: actions/upload-artifact@v4
         with:
           name: ${{ github.head_ref || github.ref_name }}-perf
-          path: "*.zst"
-          compression-level: 0
+          path: |
+            *.svg
+            *.perf
+            output.txt
+            target/criterion
+          compression-level: 9

.github/workflows/check.yml

@@ -135,14 +135,16 @@ jobs:
           LIB_DIR: ${{ matrix.type == 'release' && 'Release' || 'Debug' }}
 
       - name: Run tests and determine coverage
-        run: cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci,bench --no-fail-fast --lcov --output-path lcov.info
+        run: |
+          cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info
+          cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run
 
       - name: Run client/server transfer
         run: |
-          cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --features ci,bench --bin neqo-client --bin neqo-server
-          cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE  --features ci,bench --bin neqo-server -- $HOST:4433 &
+          cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server
+          cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --bin neqo-server -- $HOST:4433 &
           PID=$!
-          cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --features ci,bench --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE
+          cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE
           kill $PID
           [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
         env:

test-fixture/Cargo.toml

@@ -18,3 +18,5 @@ neqo-qpack = { path = "../neqo-qpack" }
 neqo-transport = { path = "../neqo-transport" }
 qlog = "0.12"
 
+[features]
+bench = []

test-fixture/src/lib.rs

@@ -393,10 +393,15 @@ impl Display for SharedVec {
 /// Panics if the log cannot be created.
 #[must_use]
 pub fn new_neqo_qlog() -> (NeqoQlog, SharedVec) {
+    let buf = SharedVec::default();
+
+    if cfg!(feature = "bench") {
+        return (NeqoQlog::disabled(), buf);
+    }
+
     let mut trace = new_trace(Role::Client);
     // Set reference time to 0.0 for testing.
     trace.common_fields.as_mut().unwrap().reference_time = Some(0.0);
-    let buf = SharedVec::default();
     let contents = buf.clone();
     let streamer = QlogStreamer::new(
         qlog::QLOG_VERSION.to_string(),