From 89c2e0b39ef5590b7b2456cefa3ec1ed05c2c208 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 08:52:27 +1000
Subject: [PATCH 01/30] ci: Compare performance to msquic

In progress
---
 .github/workflows/bench.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 72b835f843..cb9b0c2672 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -20,9 +20,17 @@ jobs:
         shell: bash
 
     steps:
-      - name: Checkout
+      - name: Checkout neqo
         uses: actions/checkout@v4
 
+      - name: Checkout msquic
+        uses: actions/checkout@v4
+        with:
+          repository: microsoft/msquic
+          ref: main
+          path: msquic
+          submodules: true
+
       - name: Set PATH
         run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"
 
@@ -35,11 +43,18 @@ jobs:
       - name: Fetch and build NSS and NSPR
         uses: ./.github/actions/nss
 
-      - name: Build
+      - name: Build neqo
         run: |
           cargo "+$TOOLCHAIN" bench --features bench --no-run
           cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
 
+      - name: Build msquic
+        run: |
+          mkdir -p msquic/build
+          cd msquic/build
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 ..
+          cmake --build .
+
       - name: Download cached main-branch results
         id: criterion-cache
         uses: actions/cache/restore@v4

From 4d3cb2672e1af9856cce76d4e56511372cfa5841 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 12:43:32 +1000
Subject: [PATCH 02/30] Make it run

---
 .github/actions/rust/action.yml |  2 +-
 .github/workflows/bench.yml     | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml
index bfb09d332d..4b03b37b8d 100644
--- a/.github/actions/rust/action.yml
+++ b/.github/actions/rust/action.yml
@@ -30,7 +30,7 @@ runs:
 
     - name: Install Rust tools
       shell: bash
-      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants
+      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants hyperfine
 
     # sccache slows CI down, so we leave it disabled.
     # Leaving the steps below commented out, so we can re-evaluate enabling it later.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index cb9b0c2672..17630f382a 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -107,6 +107,33 @@ jobs:
           HOST: localhost
           SIZE: 1073741824 # 1 GB
 
+      - name: Compare neqo and msquic
+        env:
+          HOST: 127.0.0.1
+          PORT: 4433
+          SIZE: 67108864 # 64 MB
+        run: |
+          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN"
+          truncate -s "$SIZE" "$SIZE"
+          declare -A client_cmd=(
+            ["neqo"]="neqo/target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+          )
+          declare -A server_cmd=(
+            ["neqo"]="neqo/target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+          )
+          for client in neqo msquic; do
+            for server in neqo msquic; do
+              ${server_cmd["$server"]} &
+              PID=$!
+              echo "Running ${client} against ${server}" | tee -a comparison.txt
+              hyperfine "${client_cmd["$client"]}" | tee -a comparison.txt
+              echo >> comparison.txt
+              kill $PID
+            done
+          done
+
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine
         run: sudo /root/bin/unprep.sh

From a1c63f2700b0dce6b5100eab1529e9a4f91aa673 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:15:44 +1000
Subject: [PATCH 03/30] Fixes

---
 .github/workflows/bench.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 17630f382a..c7254fe9a2 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -113,14 +113,14 @@ jobs:
           PORT: 4433
           SIZE: 67108864 # 64 MB
         run: |
-          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN"
+          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="neqo/target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["neqo"]="target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
-            ["neqo"]="neqo/target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
             ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
           )
           for client in neqo msquic; do

From 054f04d3135b10cee3207ea9c26d4eb85b1a7d86 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:40:15 +1000
Subject: [PATCH 04/30] Fix path

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index c7254fe9a2..3f22f9b089 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -116,7 +116,7 @@ jobs:
           openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(

From 84c3e57ec8512083109c26a3d7a195c0ad08479e Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:52:23 +1000
Subject: [PATCH 05/30] Use msquic `secnetperf`

---
 .github/workflows/bench.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 3f22f9b089..66317adc4c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 ..
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1..
           cmake --build .
 
       - name: Download cached main-branch results
@@ -117,22 +117,22 @@ jobs:
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/secnetperf -target:$HOST -port:$PORT -exec:maxtput -down:$SIZE"
+            # ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+            ["msquic"]="msquic/build/bin/Release/secnetperf -bind:$HOST -port:$PORT -exec:maxtput"
+            # ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
           )
-          for client in neqo msquic; do
-            for server in neqo msquic; do
-              ${server_cmd["$server"]} &
-              PID=$!
-              echo "Running ${client} against ${server}" | tee -a comparison.txt
-              hyperfine "${client_cmd["$client"]}" | tee -a comparison.txt
-              echo >> comparison.txt
-              kill $PID
-            done
-          done
+          for kind in neqo msquic; do
+            ${server_cmd["$kind"]} &
+            PID=$!
+            echo "Running ${kind} against ${kind}" | tee -a comparison.txt
+            hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
+            echo >> comparison.txt
+            kill $PID
+          # done
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From ec22b7819e6235e732a3986ae6951812c4267038 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 14:22:15 +1000
Subject: [PATCH 06/30] Fix actionlint

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 66317adc4c..2b0206b92f 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -132,7 +132,7 @@ jobs:
             hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
             echo >> comparison.txt
             kill $PID
-          # done
+          done
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From d1a7e54a2106977f9f7b7103445cf81dced10c6a Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 14:47:02 +1000
Subject: [PATCH 07/30] Sigh

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2b0206b92f..0a2b1b056d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1..
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
           cmake --build .
 
       - name: Download cached main-branch results

From b77c306268bf08532129772320f5d4c16be876bb Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:14:14 +1000
Subject: [PATCH 08/30] Retry

---
 .github/workflows/bench.yml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0a2b1b056d..eb2bbdc3b7 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -113,26 +113,27 @@ jobs:
           PORT: 4433
           SIZE: 67108864 # 64 MB
         run: |
-          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
-          truncate -s "$SIZE" "$SIZE"
+          TMPDIR=$(mktemp -d)
+          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
+          truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/secnetperf -target:$HOST -port:$PORT -exec:maxtput -down:$SIZE"
-            # ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/secnetperf -bind:$HOST -port:$PORT -exec:maxtput"
-            # ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
           )
-          for kind in neqo msquic; do
-            ${server_cmd["$kind"]} &
+          for server in neqo msquic; do
+            ${server_cmd["$server"]} &
             PID=$!
-            echo "Running ${kind} against ${kind}" | tee -a comparison.txt
-            hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
+            echo "Running teste for ${server} server" | tee -a comparison.txt
+            hyperfine -n "$server" -L client neqo,msquic--export-markdown comparison.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
+            cat comparison.md >> "$GITHUB_STEP_SUMMARY"
           done
+          rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From 82ff1900a676b082f67f48f54dc06662d6f3fd40 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:18:08 +1000
Subject: [PATCH 09/30] Better reporting

---
 .github/workflows/bench.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index eb2bbdc3b7..0ffb0f0dbb 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -127,11 +127,11 @@ jobs:
           for server in neqo msquic; do
             ${server_cmd["$server"]} &
             PID=$!
-            echo "Running teste for ${server} server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic--export-markdown comparison.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
+            echo "Running benchmarks for $server server" | tee -a comparison.txt
+            hyperfine -n "$server" -L client neqo,msquic--export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
-            cat comparison.md >> "$GITHUB_STEP_SUMMARY"
+            cat step.md >> comparison.md
           done
           rm -r "$TMP"
 
@@ -179,6 +179,11 @@ jobs:
                     -e 's/^([a-z0-9].*)$/* **\1**/gi' \
                     -e 's/(change:[^%]*% )([^%]*%)(.*)/\1**\2**\3/gi' \
               >> results.md
+          {
+            echo "### Client/server transfer results"
+            cat comparison.md
+          } >> results.md
+          cat results.md > "$GITHUB_STEP_SUMMARY"
 
       - name: Remember main-branch push URL
         if: github.ref == 'refs/heads/main'

From 4ef69a6e041f4cb033f2114a9c6521ac949034f4 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:45:26 +1000
Subject: [PATCH 10/30] Debug

---
 .github/workflows/bench.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0ffb0f0dbb..983763a67c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -108,6 +108,7 @@ jobs:
           SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
+        shell: /usr/bin/bash -x --noprofile --norc -e -o pipefail {0}
         env:
           HOST: 127.0.0.1
           PORT: 4433

From 5937c01cc9a3830ae10ff5195b2d219543e66232 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 18:10:18 +1000
Subject: [PATCH 11/30] Retry

---
 .github/workflows/bench.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 983763a67c..e163940164 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -108,13 +108,12 @@ jobs:
           SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
-        shell: /usr/bin/bash -x --noprofile --norc -e -o pipefail {0}
         env:
           HOST: 127.0.0.1
           PORT: 4433
-          SIZE: 67108864 # 64 MB
+          SIZE: 134217728 # 128 MB
         run: |
-          TMPDIR=$(mktemp -d)
+          TMP=$(mktemp -d)
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(

From b711ad94d99ebbb5e103bfbc44ebd11ad86311c0 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 19:03:11 +1000
Subject: [PATCH 12/30] Sigh^2

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e163940164..e1f77e82f0 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -128,7 +128,7 @@ jobs:
             ${server_cmd["$server"]} &
             PID=$!
             echo "Running benchmarks for $server server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic--export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
+            hyperfine -n "$server" -L client neqo,msquic --export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
             cat step.md >> comparison.md

From fe49c7fa959f837d29a627df8f9247f40e6c39af Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 19:48:59 +1000
Subject: [PATCH 13/30] Fix

---
 .github/workflows/bench.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e1f77e82f0..7aa04efcdd 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -125,13 +125,15 @@ jobs:
             ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
           )
           for server in neqo msquic; do
-            ${server_cmd["$server"]} &
-            PID=$!
-            echo "Running benchmarks for $server server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic --export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
-            echo >> comparison.txt
-            kill $PID
-            cat step.md >> comparison.md
+            for client in neqo msquic; do
+              ${server_cmd["$server"]} &
+              PID=$!
+              echo "Running benchmarks for $server server" | tee -a comparison.txt
+              hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              echo >> comparison.txt
+              kill $PID
+              cat step.md >> comparison.md
+            done
           done
           rm -r "$TMP"
 

From dead9dd1c837e94f7196c6569c9ad35ad278c84a Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 20:45:00 +1000
Subject: [PATCH 14/30] Fixes

---
 .github/workflows/bench.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 7aa04efcdd..8912e9915c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -90,6 +90,7 @@ jobs:
 
       - name: Profile client/server transfer
         run: |
+          TMP=$(mktemp -d)
           { mkdir server; \
             cd server; \
             taskset -c 0 nice -n -20 \
@@ -97,12 +98,13 @@ jobs:
               --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
           mkdir client; \
             cd client; \
-            time taskset -c 1 nice -n -20 \
+            taskset -c 1 nice -n -20 time \
             cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir . "https://$HOST:4433/$SIZE"
+              --bin neqo-client -- --output-dir "$TMP" "https://$HOST:4433/$SIZE"
           killall -INT neqo-server
           cd ${{ github.workspace }}
-          [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+          [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+          rm -r "$TMP"
         env:
           HOST: localhost
           SIZE: 1073741824 # 1 GB
@@ -117,7 +119,7 @@ jobs:
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
@@ -132,7 +134,10 @@ jobs:
               hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
-              cat step.md >> comparison.md
+              {
+                echo
+                cat step.md
+              } >> comparison.md
             done
           done
           rm -r "$TMP"

From b6f0f74061b109082d5ce6ca11bed6e4c46488fa Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 21:31:03 +1000
Subject: [PATCH 15/30] Fixes

---
 .github/workflows/bench.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 8912e9915c..490ecdba4b 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Build neqo
         run: |
           cargo "+$TOOLCHAIN" bench --features bench --no-run
-          cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
+          cargo "+$TOOLCHAIN" build --release
 
       - name: Build msquic
         run: |
@@ -120,7 +120,7 @@ jobs:
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
@@ -131,7 +131,7 @@ jobs:
               ${server_cmd["$server"]} &
               PID=$!
               echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 56b8818154bac672456fb6dd459f32a2087e6cfd Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 22:06:18 +0200
Subject: [PATCH 16/30] Update bench.yml

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 490ecdba4b..5c77968811 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -131,7 +131,7 @@ jobs:
               ${server_cmd["$server"]} &
               PID=$!
               echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              hyperfine -w 1 -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 4c307e4cdee152b37c8f8575eafd997324155cc8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 08:33:48 +1000
Subject: [PATCH 17/30] Fix

---
 .github/workflows/bench.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 490ecdba4b..7f6a1e0daf 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -120,7 +120,7 @@ jobs:
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
@@ -130,8 +130,8 @@ jobs:
             for client in neqo msquic; do
               ${server_cmd["$server"]} &
               PID=$!
-              echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              echo "Running benchmarks for client $client ← $server server" | tee -a comparison.txt
+              hyperfine -w -n "$client ← $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 7f46abb47db73e2e6771e607cd32a0315dc3e172 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 09:23:56 +1000
Subject: [PATCH 18/30] Squashed commit of the following:

commit 9ca5ecc434bcd2609845b3f27d72960bd5c7cb0a
Merge: f50f4148 f4083215
Author: Lars Eggert <lars@eggert.org>
Date:   Fri Mar 15 00:12:29 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit f50f4148bdb04c21e1e0165d72a8181cbbb39274
Merge: 8e5290b2 bc262a53
Author: Lars Eggert <lars@eggert.org>
Date:   Wed Mar 13 07:59:01 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 8e5290b213ae4ea9459cb1b71a73a7118d4bdcc4
Merge: f0cd19ec 2ff9742e
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 22:42:54 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit f0cd19ecb6196ab861658b675146b57cf20a1f9a
Merge: b2bb855b 17c4175b
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 21:54:08 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit b2bb855b25349d6c51d5877b18ce631b1c800250
Merge: d072504e 4ea2c566
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:25:13 2024 +0200

    Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc

commit d072504e5e80e2bb541f948c6c7733c78e93cb4a
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:24:52 2024 +0200

    Reorder things so `results.ms` is included in the exported artifact

commit 4ea2c5669cfc6bbb9d4b7eddb66ad2f5ab4d6aac
Merge: c82ff3af 5c728907
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:18:37 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit c82ff3af7dd92aa4b2218992f85faaeff0b7cecb
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:41:59 2024 +0200

    `killall` -> `pkill`

commit d37e7068e65ed407e0dc38b15e67707dfc628fd5
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:37:50 2024 +0200

    Go back to `killall`

commit 11320d0095e4a8d637c40ed7eb57886582ce4a21
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:11:38 2024 +0200

    No -INT

commit 407bd4ff8cc34767063b234af1737dfc2ea06339
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 14:33:52 2024 +0200

    kill -> killall

    Also reduce test transfer size.

commit 9d3a8b792e8fed127503fd93ff0406c0b462768d
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 13:57:51 2024 +0200

    Use temp dir, and fix path error

commit 84e22060c89c88ca699c185dc21d86d0902c3be9
Merge: 925cc120 b0d816a8
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 11:10:41 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 925cc120fb1c1330a9b12c9cf3ed6e5107251158
Merge: 3241f931 58890383
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 11:05:42 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 3241f9312951f205c827397a7fc0c59e5ef0b1ee
Merge: 02620a7c d48fbed7
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:59:24 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit 02620a7c48bdfcad703a47b25f324554edb5de7c
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:57:33 2024 +0200

    Try to kill via `$!`

commit b32ce9ef7b93c0960c071594b91cddc9fa674292
Merge: 9ea3a991 db1dbb29
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:15:18 2024 +0200

    Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc

commit 9ea3a99119e9ad19c0e7157b8218d0a631d1de95
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:15:05 2024 +0200

    Address comments from @martinthomson

commit db1dbb29af26054d14a12c8d5b7f074bdc9c0c2b
Merge: 681bbb7c 869afeaa
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 19:33:53 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 681bbb7c6787094ff085aa6a65edff1dd0e2dd38
Merge: bd742af1 532dcc5f
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 18:21:06 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit bd742af1b5b56422294a01a43ed50388d1bc31f5
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 17:00:34 2024 +0200

    mkdir -p

commit bc7b99fe0b156294590dda561318283693b88df6
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:29:14 2024 +0200

    Fix

commit e7bf50959d1c24b430414f38f242dd086b483a83
Merge: de64b3e1 cbd44418
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:27:56 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit de64b3e1a0108a35af58cb8d6d433a576eaa13c4
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:00:19 2024 +0200

    Wait for output before continuing

commit 12386a32c2179727a6b4779d0ed96373571cfafd
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 15:25:40 2024 +0200

    ci: Benchmark NewReno and Cubic
---
 .github/workflows/bench.yml | 117 +++++++++++++++++++++++-------------
 1 file changed, 75 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2725010ded..a779f76c98 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_CMD: record -o perf.data -F997 --call-graph fp -g
+  PERF_OPT: (record -F997 --call-graph fp -g)
 
 jobs:
   bench:
@@ -73,41 +73,71 @@ jobs:
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
-      # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
-      # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
-      - name: Profile cargo bench transfer
-        run: |
-          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
-          # Avoid that by shuffling the directories around so this run uses its own results directory.
-          mv target/criterion target/criterion-bench
-          mv target/criterion-transfer-profile target/criterion || true
-          taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
-              --bench --exact "Run multiple transfers with varying seeds" --noplot
-          # And now restore the directories.
-          mv target/criterion target/criterion-transfer-profile
-          mv target/criterion-bench target/criterion
-
-      - name: Profile client/server transfer
+      - name: Compare neqo and msquic
+        env:
+          HOST: 127.0.0.1
+          PORT: 4433
+          SIZE: 134217728 # 128 MB
         run: |
           TMP=$(mktemp -d)
-          { mkdir server; \
-            cd server; \
-            taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
-          mkdir client; \
-            cd client; \
-            taskset -c 1 nice -n -20 time \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir "$TMP" "https://$HOST:4433/$SIZE"
-          killall -INT neqo-server
-          cd ${{ github.workspace }}
-          [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
+          truncate -s "$SIZE" "/tmp/$SIZE"
+          declare -A client_cmd=(
+            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
+          )
+          declare -A server_cmd=(
+            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
+          )
+
+          function transmogrify {
+            CMD=$1
+            local cc=$2
+            local pacing=$3
+            if [ "$cc" != "" ]; then
+              CMD=${CMD//_cc/--cc $cc}
+              EXT="-$cc"
+            fi
+            if [ "$pacing" == "pacing" ]; then
+              CMD=${CMD//_pacing/--pacing}
+              EXT="$EXT-$pacing"
+            fi
+          }
+
+          for server in neqo msquic; do
+            for client in neqo msquic; do
+              if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
+                cc_opt=("")
+                pacing_opt=("")
+              else
+                cc_opt=("reno" "cubic")
+                pacing_opt=("pacing" "")
+              fi
+              for cc in "${cc_opt[@]}"; do
+                for pacing in "${pacing_opt[@]}"; do
+                  TAG="client $client ← $server server $cc $pacing"
+                  echo "Running benchmarks for $TAG" | tee -a comparison.txt
+                  transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.server.perf" CMD &
+                  PID=$!
+                  transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.client.perf" \
+                    hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
+                    tee -a comparison.txt || true
+                  echo >>comparison.txt
+                  kill $PID
+                  {
+                    echo
+                    cat step.md
+                  } >>comparison.md
+                done
+              done
+            done
+          done
           rm -r "$TMP"
-        env:
-          HOST: localhost
-          SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
         env:
@@ -149,20 +179,23 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i perf.data -F +pid > transfer.perf &
-          perf script -i client/perf.data -F +pid > client.perf &
-          perf script -i server/perf.data -F +pid > server.perf &
+          perf script -i transfer.perf -F +pid > transfer.fx.perf &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
+            done
+          done
           wait
-          mv flamegraph.svg transfer.svg
-          mv client/flamegraph.svg client.svg
-          mv server/flamegraph.svg server.svg
           rm neqo.svg
 
       - name: Generate perf reports
         run: |
-          perf report -i perf.data --no-children --stdio > transfer.perf.txt &
-          perf report -i client/perf.data --no-children --stdio > client.perf.txt &
-          perf report -i server/perf.data --no-children --stdio > server.perf.txt &
+          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &
+             done
+          done
           wait
 
       - name: Format results as Markdown

From 33d243b4cbef8fe299ac92e77c1c0adb7e2540e5 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 10:55:45 +1000
Subject: [PATCH 19/30] Fixes

---
 .github/workflows/bench.yml | 61 ++++++++-----------------------------
 1 file changed, 13 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a779f76c98..81b923d6c8 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_OPT: (record -F997 --call-graph fp -g)
+  PERF_OPT: record -F997 --call-graph fp -g
 
 jobs:
   bench:
@@ -120,13 +120,17 @@ jobs:
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   echo "$CMD"
-                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.server.perf" CMD &
+                  # shellcheck disable=SC2086
+                  taskset -c 0 nice -n -20 \
+                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   echo "$CMD"
-                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.client.perf" \
-                    hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
-                    tee -a comparison.txt || true
+                  # shellcheck disable=SC2086
+                  taskset -c 0 nice -n -20 \
+                    perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
+                      hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
+                      tee -a comparison.txt || true
                   echo >>comparison.txt
                   kill $PID
                   {
@@ -139,39 +143,6 @@ jobs:
           done
           rm -r "$TMP"
 
-      - name: Compare neqo and msquic
-        env:
-          HOST: 127.0.0.1
-          PORT: 4433
-          SIZE: 134217728 # 128 MB
-        run: |
-          TMP=$(mktemp -d)
-          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
-          truncate -s "$SIZE" "/tmp/$SIZE"
-          declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
-          )
-          declare -A server_cmd=(
-            ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
-          )
-          for server in neqo msquic; do
-            for client in neqo msquic; do
-              ${server_cmd["$server"]} &
-              PID=$!
-              echo "Running benchmarks for client $client ← $server server" | tee -a comparison.txt
-              hyperfine -w 1 -n "$client ← $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
-              echo >> comparison.txt
-              kill $PID
-              {
-                echo
-                cat step.md
-              } >> comparison.md
-            done
-          done
-          rm -r "$TMP"
-
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine
         run: sudo /root/bin/unprep.sh
@@ -179,22 +150,16 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i transfer.perf -F +pid > transfer.fx.perf &
-          for CC in newreno cubic; do
-            for PEER in client server; do
-              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
-            done
+          for f in *.perf; do
+            perf script -i "$f" -F +pid > "$(basename -s .perf $f).fx.perf" &
           done
           wait
           rm neqo.svg
 
       - name: Generate perf reports
         run: |
-          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
-          for CC in newreno cubic; do
-            for PEER in client server; do
-              perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &
-             done
+          for f in *.perf; do
+            perf report -i "$f" --no-children --stdio > "$f.txt" &
           done
           wait
 

From ba3f750a9c3f3408ecc19b0482023e350a702e3e Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 10:58:02 +1000
Subject: [PATCH 20/30] Fix

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 81b923d6c8..a167852782 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -151,7 +151,7 @@ jobs:
       - name: Convert for profiler.firefox.com
         run: |
           for f in *.perf; do
-            perf script -i "$f" -F +pid > "$(basename -s .perf $f).fx.perf" &
+            perf script -i "$f" -F +pid > "$(basename -s .perf "$f").fx.perf" &
           done
           wait
           rm neqo.svg

From 72f8203995a8c79d2d78dbd721a4ddbafef5bc36 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 11:33:33 +1000
Subject: [PATCH 21/30] Fixes

---
 .github/workflows/bench.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a167852782..ece487f6c2 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -119,13 +119,11 @@ jobs:
                   TAG="client $client ← $server server $cc $pacing"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
-                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" CMD &
+                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \

From 5cb812a71cb165ec9e557abed9b2113b698ab9cc Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 14:27:57 +1000
Subject: [PATCH 22/30] Tweaks

---
 .github/workflows/bench.yml | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index ece487f6c2..5c1c35dfc4 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
+          cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
           cmake --build .
 
       - name: Download cached main-branch results
@@ -116,7 +116,11 @@ jobs:
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
-                  TAG="client $client ← $server server $cc $pacing"
+                  TAG="client $client ← $server server ($cc"
+                  if [ "$pacing" == "pacing" ]; then
+                    TAG="$TAG, pacing"
+                  fi
+                  TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
@@ -125,7 +129,7 @@ jobs:
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
-                  taskset -c 0 nice -n -20 \
+                  taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                       hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
                       tee -a comparison.txt || true
@@ -146,20 +150,18 @@ jobs:
         run: sudo /root/bin/unprep.sh
         if: success() || failure() || cancelled()
 
-      - name: Convert for profiler.firefox.com
-        run: |
-          for f in *.perf; do
-            perf script -i "$f" -F +pid > "$(basename -s .perf "$f").fx.perf" &
-          done
-          wait
-          rm neqo.svg
-
-      - name: Generate perf reports
+      - name: Post-process perf data
         run: |
           for f in *.perf; do
+            # Convert for profiler.firefox.com
+            perf script -i "$f" -F +pid > "$f.fx" &
+            # Generate perf reports
             perf report -i "$f" --no-children --stdio > "$f.txt" &
+            # Generate flamegraphs
+            flamegraph --perfdata "$f" -o "${f//.perf/.svg}" &
           done
           wait
+          rm neqo.svg
 
       - name: Format results as Markdown
         id: results

From 5f5b793c654f87f93e730e52b11feb66fc8856e8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 15:22:17 +1000
Subject: [PATCH 23/30] Debug

---
 .github/workflows/bench.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5c1c35dfc4..86230de79d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -124,25 +124,26 @@ jobs:
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
+                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
+                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
+                      hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
-                      tee -a comparison.txt || true
-                  echo >>comparison.txt
+                      hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      tee -a comparison.txt
+                  echo >> comparison.txt
                   kill $PID
-                  {
-                    echo
-                    cat step.md
-                  } >>comparison.md
+                  cat step.md >> steps.md
                 done
               done
             done
           done
+          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | sed '/^\|:-/{x;/^$/!d;g;}' > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From e78b1be2d67faf1c3d9bda606ebc2c30fd0ce06b Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 06:37:23 +1000
Subject: [PATCH 24/30] actionlint

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 86230de79d..d7b062e053 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -123,15 +123,15 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  # shellcheck disable=SC2086
                   echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
+                  # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  # shellcheck disable=SC2086
                   echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
                       hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
+                  # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                       hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |

From f36a1b7b8eee4194ee4ec40aa082c3ed8da76ee8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 08:21:20 +1000
Subject: [PATCH 25/30] Fix sed

---
 .github/workflows/bench.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index d7b062e053..0bd50d6167 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -102,6 +102,8 @@ jobs:
             if [ "$pacing" == "pacing" ]; then
               CMD=${CMD//_pacing/--pacing}
               EXT="$EXT-$pacing"
+            else
+              CMD=${CMD//_pacing/}
             fi
           }
 

From 89e8b16bf97b993ae484e68bf81d65a8741c8c96 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 08:28:35 +1000
Subject: [PATCH 26/30] Pacing changed

---
 .github/workflows/bench.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0bd50d6167..22e7da101d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -99,11 +99,11 @@ jobs:
               CMD=${CMD//_cc/--cc $cc}
               EXT="-$cc"
             fi
-            if [ "$pacing" == "pacing" ]; then
-              CMD=${CMD//_pacing/--pacing}
+            if [ "$pacing" == "on" ]; then
+              CMD=${CMD//_pacing/}
               EXT="$EXT-$pacing"
             else
-              CMD=${CMD//_pacing/}
+              CMD=${CMD//_pacing/--no-pacing}
             fi
           }
 
@@ -114,12 +114,12 @@ jobs:
                 pacing_opt=("")
               else
                 cc_opt=("reno" "cubic")
-                pacing_opt=("pacing" "")
+                pacing_opt=("on" "")
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
                   TAG="client $client ← $server server ($cc"
-                  if [ "$pacing" == "pacing" ]; then
+                  if [ "$pacing" == "on" ]; then
                     TAG="$TAG, pacing"
                   fi
                   TAG="$TAG)"

From 51a1ecf3b99d0e7082e59825d067f9a12424c200 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 09:07:35 +1000
Subject: [PATCH 27/30] Fixes

---
 .github/workflows/bench.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 22e7da101d..41d98936ba 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -101,9 +101,10 @@ jobs:
             fi
             if [ "$pacing" == "on" ]; then
               CMD=${CMD//_pacing/}
-              EXT="$EXT-$pacing"
+              EXT="$EXT-pacing"
             else
               CMD=${CMD//_pacing/--no-pacing}
+              EXT="$EXT-nopacing"
             fi
           }
 
@@ -118,25 +119,22 @@ jobs:
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
-                  TAG="client $client ← $server server ($cc"
+                  TAG="client $client ← server $server ($cc"
                   if [ "$pacing" == "on" ]; then
                     TAG="$TAG, pacing"
                   fi
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
-                      hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine --show-output -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID
@@ -145,7 +143,9 @@ jobs:
               done
             done
           done
-          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | sed '/^\|:-/{x;/^$/!d;g;}' > comparison.md
+          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | \
+            sed '/^\|:-/{x;/^$/!d;g;}' | \
+            sed 's/\`//g' > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From 20e9ca97c099218abc60ede3eb8d74b0e91a9953 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 10:01:42 +1000
Subject: [PATCH 28/30] msquic server exits with non-zero

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 41d98936ba..068057ae5f 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -88,7 +88,7 @@ jobs:
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit || true"
           )
 
           function transmogrify {
@@ -134,7 +134,7 @@ jobs:
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine --show-output -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID

From ec5d6d60ff2eea6387b6ac2e4a51fb597ed09048 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 10:37:23 +1000
Subject: [PATCH 29/30] echo

---
 .github/workflows/bench.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 068057ae5f..3185abe91a 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -126,11 +126,13 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf hyperfine -n $TAG -u millisecond --export-markdown step.md $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \

From 6f5bd76d51f477ccbfb7480ecf51cd10702b4f0e Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 12:02:52 +1000
Subject: [PATCH 30/30] ci: Use `fleet`

---
 .github/actions/rust/action.yml |  2 +-
 .github/workflows/check.yml     | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml
index 4b03b37b8d..a567cb422b 100644
--- a/.github/actions/rust/action.yml
+++ b/.github/actions/rust/action.yml
@@ -30,7 +30,7 @@ runs:
 
     - name: Install Rust tools
       shell: bash
-      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants hyperfine
+      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants hyperfine fleet-rs sccache
 
     # sccache slows CI down, so we leave it disabled.
     # Leaving the steps below commented out, so we can re-evaluate enabling it later.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 10085ffda6..a54f76489e 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -108,18 +108,18 @@ jobs:
       - name: Build
         run: |
           # shellcheck disable=SC2086
-          cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci
+          fleet +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci
 
       - name: Run tests and determine coverage
         run: |
           # shellcheck disable=SC2086
-          cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info
-          cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run
+          fleet +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info
+          fleet +${{ matrix.rust-toolchain }} bench --features bench --no-run
 
       - name: Run client/server transfer
         run: |
           # shellcheck disable=SC2086
-          cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server
+          fleet +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server
           "target/$BUILD_DIR/neqo-server" "$HOST:4433" &
           PID=$!
           "target/$BUILD_DIR/neqo-client" --output-dir . "https://$HOST:4433/$SIZE"
@@ -137,7 +137,7 @@ jobs:
             CONFIG_PATH="--config-path=$(mktemp)"
           fi
           # shellcheck disable=SC2086
-          cargo +${{ matrix.rust-toolchain }} fmt --all -- --check $CONFIG_PATH
+          fleet +${{ matrix.rust-toolchain }} fmt --all -- --check $CONFIG_PATH
         if: success() || failure()
 
       - name: Clippy
@@ -146,11 +146,11 @@ jobs:
           # respective default features only. Can reveal warnings otherwise
           # hidden given that a plain cargo clippy combines all features of the
           # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695.
-          cargo +${{ matrix.rust-toolchain }} hack clippy --all-targets -- -D warnings || ${{ matrix.rust-toolchain == 'nightly' }}
+          fleet +${{ matrix.rust-toolchain }} hack clippy --all-targets -- -D warnings || ${{ matrix.rust-toolchain == 'nightly' }}
         if: success() || failure()
 
       - name: Check rustdoc links
-        run: cargo +${{ matrix.rust-toolchain }} doc --workspace --no-deps --document-private-items
+        run: fleet +${{ matrix.rust-toolchain }} doc --workspace --no-deps --document-private-items
         env:
           RUSTDOCFLAGS: "--deny rustdoc::broken_intra_doc_links --deny warnings"
         if: success() || failure()