From 9ae301626d29c1aed470844d36927b0c12c70630 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Wed, 13 Mar 2024 09:27:57 +0200
Subject: [PATCH 1/2] test: Make `criterion` calculate transfer benchmark
 throughputs

Like:
```
transfer/Run multiple transfers with varying seeds
                        time:   [213.61 ms 224.95 ms 235.29 ms]
                        thrpt:  [17.000 MiB/s 17.782 MiB/s 18.726 MiB/s]
                 change:
                        time:   [-17.204% -6.9690% +1.8324%] (p = 0.21 > 0.05)
                        thrpt:  [-1.7994% +7.4910% +20.778%]
                        No change in performance detected.
Found 5 outliers among 100 measurements (5.00%)
  5 (5.00%) low mild

transfer/Run multiple transfers with the same seed
                        time:   [175.41 ms 188.40 ms 201.32 ms]
                        thrpt:  [19.869 MiB/s 21.231 MiB/s 22.803 MiB/s]
                 change:
                        time:   [-5.9393% +5.5417% +18.399%] (p = 0.35 > 0.05)
                        thrpt:  [-15.540% -5.2507% +6.3144%]
                        No change in performance detected.
Found 14 outliers among 100 measurements (14.00%)
  8 (8.00%) low mild
  5 (5.00%) high mild
  1 (1.00%) high severe
  ```
---
 neqo-transport/benches/transfer.rs | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/neqo-transport/benches/transfer.rs b/neqo-transport/benches/transfer.rs
index 444f738f9c..b13075a4ff 100644
--- a/neqo-transport/benches/transfer.rs
+++ b/neqo-transport/benches/transfer.rs
@@ -6,7 +6,7 @@
 
 use std::time::Duration;
 
-use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion};
+use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion, Throughput};
 use test_fixture::{
     boxed,
     sim::{
@@ -21,7 +21,9 @@ const JITTER: Duration = Duration::from_millis(10);
 const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte
 
 fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option<impl AsRef<str>>) {
-    c.bench_function(label, |b| {
+    let mut group = c.benchmark_group("transfer");
+    group.throughput(Throughput::Bytes(u64::try_from(TRANSFER_AMOUNT).unwrap()));
+    group.bench_function(label, |b| {
         b.iter_batched(
             || {
                 let nodes = boxed![
@@ -44,6 +46,7 @@ fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option<impl AsRef<st
             SmallInput,
         )
     });
+    group.finish();
 }
 
 fn benchmark_transfer_variable(c: &mut Criterion) {

From 9f6d5e527906aa082d5f2cea2682ba55da8f04b0 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Fri, 15 Mar 2024 08:11:11 +1000
Subject: [PATCH 2/2] This doesn't actually run `cargo bench`, it runs `cargo
 flamegraph`

---
 .github/workflows/bench.yml | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 72b835f843..81ef297a9e 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -62,16 +62,9 @@ jobs:
       # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
       - name: Profile cargo bench transfer
         run: |
-          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
-          # Avoid that by shuffling the directories around so this run uses its own results directory.
-          mv target/criterion target/criterion-bench
-          mv target/criterion-transfer-profile target/criterion || true
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
               --bench --exact "Run multiple transfers with varying seeds" --noplot
-          # And now restore the directories.
-          mv target/criterion target/criterion-transfer-profile
-          mv target/criterion-bench target/criterion
 
       - name: Profile client/server transfer
         run: |