From 89c2e0b39ef5590b7b2456cefa3ec1ed05c2c208 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 08:52:27 +1000
Subject: [PATCH 01/46] ci: Compare performance to msquic

In progress
---
 .github/workflows/bench.yml | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 72b835f843..cb9b0c2672 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -20,9 +20,17 @@ jobs:
         shell: bash
 
     steps:
-      - name: Checkout
+      - name: Checkout neqo
         uses: actions/checkout@v4
 
+      - name: Checkout msquic
+        uses: actions/checkout@v4
+        with:
+          repository: microsoft/msquic
+          ref: main
+          path: msquic
+          submodules: true
+
       - name: Set PATH
         run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}"
 
@@ -35,11 +43,18 @@ jobs:
       - name: Fetch and build NSS and NSPR
         uses: ./.github/actions/nss
 
-      - name: Build
+      - name: Build neqo
         run: |
           cargo "+$TOOLCHAIN" bench --features bench --no-run
           cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
 
+      - name: Build msquic
+        run: |
+          mkdir -p msquic/build
+          cd msquic/build
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 ..
+          cmake --build .
+
       - name: Download cached main-branch results
         id: criterion-cache
         uses: actions/cache/restore@v4

From 4d3cb2672e1af9856cce76d4e56511372cfa5841 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 12:43:32 +1000
Subject: [PATCH 02/46] Make it run

---
 .github/actions/rust/action.yml |  2 +-
 .github/workflows/bench.yml     | 27 +++++++++++++++++++++++++++
 2 files changed, 28 insertions(+), 1 deletion(-)

diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml
index bfb09d332d..4b03b37b8d 100644
--- a/.github/actions/rust/action.yml
+++ b/.github/actions/rust/action.yml
@@ -30,7 +30,7 @@ runs:
 
     - name: Install Rust tools
       shell: bash
-      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants
+      run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants hyperfine
 
     # sccache slows CI down, so we leave it disabled.
     # Leaving the steps below commented out, so we can re-evaluate enabling it later.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index cb9b0c2672..17630f382a 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -107,6 +107,33 @@ jobs:
           HOST: localhost
           SIZE: 1073741824 # 1 GB
 
+      - name: Compare neqo and msquic
+        env:
+          HOST: 127.0.0.1
+          PORT: 4433
+          SIZE: 67108864 # 64 MB
+        run: |
+          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN"
+          truncate -s "$SIZE" "$SIZE"
+          declare -A client_cmd=(
+            ["neqo"]="neqo/target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+          )
+          declare -A server_cmd=(
+            ["neqo"]="neqo/target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+          )
+          for client in neqo msquic; do
+            for server in neqo msquic; do
+              ${server_cmd["$server"]} &
+              PID=$!
+              echo "Running ${client} against ${server}" | tee -a comparison.txt
+              hyperfine "${client_cmd["$client"]}" | tee -a comparison.txt
+              echo >> comparison.txt
+              kill $PID
+            done
+          done
+
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine
         run: sudo /root/bin/unprep.sh

From a1c63f2700b0dce6b5100eab1529e9a4f91aa673 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:15:44 +1000
Subject: [PATCH 03/46] Fixes

---
 .github/workflows/bench.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 17630f382a..c7254fe9a2 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -113,14 +113,14 @@ jobs:
           PORT: 4433
           SIZE: 67108864 # 64 MB
         run: |
-          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN"
+          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="neqo/target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["neqo"]="target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
-            ["neqo"]="neqo/target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
             ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
           )
           for client in neqo msquic; do

From 054f04d3135b10cee3207ea9c26d4eb85b1a7d86 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:40:15 +1000
Subject: [PATCH 04/46] Fix path

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index c7254fe9a2..3f22f9b089 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -116,7 +116,7 @@ jobs:
           openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/debug/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(

From 84c3e57ec8512083109c26a3d7a195c0ad08479e Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 13:52:23 +1000
Subject: [PATCH 05/46] Use msquic `secnetperf`

---
 .github/workflows/bench.yml | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 3f22f9b089..66317adc4c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 ..
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1..
           cmake --build .
 
       - name: Download cached main-branch results
@@ -117,22 +117,22 @@ jobs:
           truncate -s "$SIZE" "$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/secnetperf -target:$HOST -port:$PORT -exec:maxtput -down:$SIZE"
+            # ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+            ["msquic"]="msquic/build/bin/Release/secnetperf -bind:$HOST -port:$PORT -exec:maxtput"
+            # ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
           )
-          for client in neqo msquic; do
-            for server in neqo msquic; do
-              ${server_cmd["$server"]} &
-              PID=$!
-              echo "Running ${client} against ${server}" | tee -a comparison.txt
-              hyperfine "${client_cmd["$client"]}" | tee -a comparison.txt
-              echo >> comparison.txt
-              kill $PID
-            done
-          done
+          for kind in neqo msquic; do
+            ${server_cmd["$kind"]} &
+            PID=$!
+            echo "Running ${kind} against ${kind}" | tee -a comparison.txt
+            hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
+            echo >> comparison.txt
+            kill $PID
+          # done
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From ec22b7819e6235e732a3986ae6951812c4267038 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 14:22:15 +1000
Subject: [PATCH 06/46] Fix actionlint

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 66317adc4c..2b0206b92f 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -132,7 +132,7 @@ jobs:
             hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
             echo >> comparison.txt
             kill $PID
-          # done
+          done
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From d1a7e54a2106977f9f7b7103445cf81dced10c6a Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 14:47:02 +1000
Subject: [PATCH 07/46] Sigh

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2b0206b92f..0a2b1b056d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1..
+          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
           cmake --build .
 
       - name: Download cached main-branch results

From b77c306268bf08532129772320f5d4c16be876bb Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:14:14 +1000
Subject: [PATCH 08/46] Retry

---
 .github/workflows/bench.yml | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0a2b1b056d..eb2bbdc3b7 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -113,26 +113,27 @@ jobs:
           PORT: 4433
           SIZE: 67108864 # 64 MB
         run: |
-          openssl req -nodes -new -x509 -keyout key -out cert -subj "/CN=DOMAIN" 2> /dev/null
-          truncate -s "$SIZE" "$SIZE"
+          TMPDIR=$(mktemp -d)
+          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
+          truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/secnetperf -target:$HOST -port:$PORT -exec:maxtput -down:$SIZE"
-            # ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/secnetperf -bind:$HOST -port:$PORT -exec:maxtput"
-            # ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:. -listen:$HOST -port:$PORT -file:cert -key:key -noexit"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
           )
-          for kind in neqo msquic; do
-            ${server_cmd["$kind"]} &
+          for server in neqo msquic; do
+            ${server_cmd["$server"]} &
             PID=$!
-            echo "Running ${kind} against ${kind}" | tee -a comparison.txt
-            hyperfine "${client_cmd["$kind"]}" | tee -a comparison.txt
+            echo "Running teste for ${server} server" | tee -a comparison.txt
+            hyperfine -n "$server" -L client neqo,msquic--export-markdown comparison.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
+            cat comparison.md >> "$GITHUB_STEP_SUMMARY"
           done
+          rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine

From 82ff1900a676b082f67f48f54dc06662d6f3fd40 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:18:08 +1000
Subject: [PATCH 09/46] Better reporting

---
 .github/workflows/bench.yml | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index eb2bbdc3b7..0ffb0f0dbb 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -127,11 +127,11 @@ jobs:
           for server in neqo msquic; do
             ${server_cmd["$server"]} &
             PID=$!
-            echo "Running teste for ${server} server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic--export-markdown comparison.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
+            echo "Running benchmarks for $server server" | tee -a comparison.txt
+            hyperfine -n "$server" -L client neqo,msquic--export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
-            cat comparison.md >> "$GITHUB_STEP_SUMMARY"
+            cat step.md >> comparison.md
           done
           rm -r "$TMP"
 
@@ -179,6 +179,11 @@ jobs:
                     -e 's/^([a-z0-9].*)$/* **\1**/gi' \
                     -e 's/(change:[^%]*% )([^%]*%)(.*)/\1**\2**\3/gi' \
               >> results.md
+          {
+            echo "### Client/server transfer results"
+            cat comparison.md
+          } >> results.md
+          cat results.md > "$GITHUB_STEP_SUMMARY"
 
       - name: Remember main-branch push URL
         if: github.ref == 'refs/heads/main'

From 4ef69a6e041f4cb033f2114a9c6521ac949034f4 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 17:45:26 +1000
Subject: [PATCH 10/46] Debug

---
 .github/workflows/bench.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0ffb0f0dbb..983763a67c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -108,6 +108,7 @@ jobs:
           SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
+        shell: /usr/bin/bash -x --noprofile --norc -e -o pipefail {0}
         env:
           HOST: 127.0.0.1
           PORT: 4433

From 5937c01cc9a3830ae10ff5195b2d219543e66232 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 18:10:18 +1000
Subject: [PATCH 11/46] Retry

---
 .github/workflows/bench.yml | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 983763a67c..e163940164 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -108,13 +108,12 @@ jobs:
           SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
-        shell: /usr/bin/bash -x --noprofile --norc -e -o pipefail {0}
         env:
           HOST: 127.0.0.1
           PORT: 4433
-          SIZE: 67108864 # 64 MB
+          SIZE: 134217728 # 128 MB
         run: |
-          TMPDIR=$(mktemp -d)
+          TMP=$(mktemp -d)
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(

From b711ad94d99ebbb5e103bfbc44ebd11ad86311c0 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 19:03:11 +1000
Subject: [PATCH 12/46] Sigh^2

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e163940164..e1f77e82f0 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -128,7 +128,7 @@ jobs:
             ${server_cmd["$server"]} &
             PID=$!
             echo "Running benchmarks for $server server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic--export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
+            hyperfine -n "$server" -L client neqo,msquic --export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
             echo >> comparison.txt
             kill $PID
             cat step.md >> comparison.md

From fe49c7fa959f837d29a627df8f9247f40e6c39af Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 19:48:59 +1000
Subject: [PATCH 13/46] Fix

---
 .github/workflows/bench.yml | 16 +++++++++-------
 1 file changed, 9 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e1f77e82f0..7aa04efcdd 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -125,13 +125,15 @@ jobs:
             ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
           )
           for server in neqo msquic; do
-            ${server_cmd["$server"]} &
-            PID=$!
-            echo "Running benchmarks for $server server" | tee -a comparison.txt
-            hyperfine -n "$server" -L client neqo,msquic --export-markdown step.md "${client_cmd["{client}"]}" | tee -a comparison.txt || true
-            echo >> comparison.txt
-            kill $PID
-            cat step.md >> comparison.md
+            for client in neqo msquic; do
+              ${server_cmd["$server"]} &
+              PID=$!
+              echo "Running benchmarks for $server server" | tee -a comparison.txt
+              hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              echo >> comparison.txt
+              kill $PID
+              cat step.md >> comparison.md
+            done
           done
           rm -r "$TMP"
 

From dead9dd1c837e94f7196c6569c9ad35ad278c84a Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 20:45:00 +1000
Subject: [PATCH 14/46] Fixes

---
 .github/workflows/bench.yml | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 7aa04efcdd..8912e9915c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -90,6 +90,7 @@ jobs:
 
       - name: Profile client/server transfer
         run: |
+          TMP=$(mktemp -d)
           { mkdir server; \
             cd server; \
             taskset -c 0 nice -n -20 \
@@ -97,12 +98,13 @@ jobs:
               --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
           mkdir client; \
             cd client; \
-            time taskset -c 1 nice -n -20 \
+            taskset -c 1 nice -n -20 time \
             cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir . "https://$HOST:4433/$SIZE"
+              --bin neqo-client -- --output-dir "$TMP" "https://$HOST:4433/$SIZE"
           killall -INT neqo-server
           cd ${{ github.workspace }}
-          [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1
+          [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+          rm -r "$TMP"
         env:
           HOST: localhost
           SIZE: 1073741824 # 1 GB
@@ -117,7 +119,7 @@ jobs:
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://localhost:4433/$SIZE"
+            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
           )
           declare -A server_cmd=(
@@ -132,7 +134,10 @@ jobs:
               hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
-              cat step.md >> comparison.md
+              {
+                echo
+                cat step.md
+              } >> comparison.md
             done
           done
           rm -r "$TMP"

From b6f0f74061b109082d5ce6ca11bed6e4c46488fa Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 21:31:03 +1000
Subject: [PATCH 15/46] Fixes

---
 .github/workflows/bench.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 8912e9915c..490ecdba4b 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Build neqo
         run: |
           cargo "+$TOOLCHAIN" bench --features bench --no-run
-          cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
+          cargo "+$TOOLCHAIN" build --release
 
       - name: Build msquic
         run: |
@@ -120,7 +120,7 @@ jobs:
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE/"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
@@ -131,7 +131,7 @@ jobs:
               ${server_cmd["$server"]} &
               PID=$!
               echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -n "$client -> $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 56b8818154bac672456fb6dd459f32a2087e6cfd Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sat, 16 Mar 2024 22:06:18 +0200
Subject: [PATCH 16/46] Update bench.yml

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 490ecdba4b..5c77968811 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -131,7 +131,7 @@ jobs:
               ${server_cmd["$server"]} &
               PID=$!
               echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              hyperfine -w 1 -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 4c307e4cdee152b37c8f8575eafd997324155cc8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 08:33:48 +1000
Subject: [PATCH 17/46] Fix

---
 .github/workflows/bench.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 490ecdba4b..7f6a1e0daf 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -120,7 +120,7 @@ jobs:
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE/"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
@@ -130,8 +130,8 @@ jobs:
             for client in neqo msquic; do
               ${server_cmd["$server"]} &
               PID=$!
-              echo "Running benchmarks for $server server" | tee -a comparison.txt
-              hyperfine -w -n "$client <- $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
+              echo "Running benchmarks for client $client ← $server server" | tee -a comparison.txt
+              hyperfine -w -n "$client ← $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
               echo >> comparison.txt
               kill $PID
               {

From 7f46abb47db73e2e6771e607cd32a0315dc3e172 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 09:23:56 +1000
Subject: [PATCH 18/46] Squashed commit of the following:

commit 9ca5ecc434bcd2609845b3f27d72960bd5c7cb0a
Merge: f50f4148 f4083215
Author: Lars Eggert <lars@eggert.org>
Date:   Fri Mar 15 00:12:29 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit f50f4148bdb04c21e1e0165d72a8181cbbb39274
Merge: 8e5290b2 bc262a53
Author: Lars Eggert <lars@eggert.org>
Date:   Wed Mar 13 07:59:01 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 8e5290b213ae4ea9459cb1b71a73a7118d4bdcc4
Merge: f0cd19ec 2ff9742e
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 22:42:54 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit f0cd19ecb6196ab861658b675146b57cf20a1f9a
Merge: b2bb855b 17c4175b
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 21:54:08 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit b2bb855b25349d6c51d5877b18ce631b1c800250
Merge: d072504e 4ea2c566
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:25:13 2024 +0200

    Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc

commit d072504e5e80e2bb541f948c6c7733c78e93cb4a
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:24:52 2024 +0200

    Reorder things so `results.ms` is included in the exported artifact

commit 4ea2c5669cfc6bbb9d4b7eddb66ad2f5ab4d6aac
Merge: c82ff3af 5c728907
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 17:18:37 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit c82ff3af7dd92aa4b2218992f85faaeff0b7cecb
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:41:59 2024 +0200

    `killall` -> `pkill`

commit d37e7068e65ed407e0dc38b15e67707dfc628fd5
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:37:50 2024 +0200

    Go back to `killall`

commit 11320d0095e4a8d637c40ed7eb57886582ce4a21
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 16:11:38 2024 +0200

    No -INT

commit 407bd4ff8cc34767063b234af1737dfc2ea06339
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 14:33:52 2024 +0200

    kill -> killall

    Also reduce test transfer size.

commit 9d3a8b792e8fed127503fd93ff0406c0b462768d
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 13:57:51 2024 +0200

    Use temp dir, and fix path error

commit 84e22060c89c88ca699c185dc21d86d0902c3be9
Merge: 925cc120 b0d816a8
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 11:10:41 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 925cc120fb1c1330a9b12c9cf3ed6e5107251158
Merge: 3241f931 58890383
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 11:05:42 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 3241f9312951f205c827397a7fc0c59e5ef0b1ee
Merge: 02620a7c d48fbed7
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:59:24 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit 02620a7c48bdfcad703a47b25f324554edb5de7c
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:57:33 2024 +0200

    Try to kill via `$!`

commit b32ce9ef7b93c0960c071594b91cddc9fa674292
Merge: 9ea3a991 db1dbb29
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:15:18 2024 +0200

    Merge branch 'ci-bench-cc' of github.com:larseggert/neqo into ci-bench-cc

commit 9ea3a99119e9ad19c0e7157b8218d0a631d1de95
Author: Lars Eggert <lars@eggert.org>
Date:   Tue Mar 12 09:15:05 2024 +0200

    Address comments from @martinthomson

commit db1dbb29af26054d14a12c8d5b7f074bdc9c0c2b
Merge: 681bbb7c 869afeaa
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 19:33:53 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit 681bbb7c6787094ff085aa6a65edff1dd0e2dd38
Merge: bd742af1 532dcc5f
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 18:21:06 2024 +0200

    Merge branch 'main' into ci-bench-cc

    Signed-off-by: Lars Eggert <lars@eggert.org>

commit bd742af1b5b56422294a01a43ed50388d1bc31f5
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 17:00:34 2024 +0200

    mkdir -p

commit bc7b99fe0b156294590dda561318283693b88df6
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:29:14 2024 +0200

    Fix

commit e7bf50959d1c24b430414f38f242dd086b483a83
Merge: de64b3e1 cbd44418
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:27:56 2024 +0200

    Merge branch 'main' into ci-bench-cc

commit de64b3e1a0108a35af58cb8d6d433a576eaa13c4
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 16:00:19 2024 +0200

    Wait for output before continuing

commit 12386a32c2179727a6b4779d0ed96373571cfafd
Author: Lars Eggert <lars@eggert.org>
Date:   Mon Mar 11 15:25:40 2024 +0200

    ci: Benchmark NewReno and Cubic
---
 .github/workflows/bench.yml | 117 +++++++++++++++++++++++-------------
 1 file changed, 75 insertions(+), 42 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2725010ded..a779f76c98 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_CMD: record -o perf.data -F997 --call-graph fp -g
+  PERF_OPT: (record -F997 --call-graph fp -g)
 
 jobs:
   bench:
@@ -73,41 +73,71 @@ jobs:
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
-      # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf.
-      # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments.
-      - name: Profile cargo bench transfer
-        run: |
-          # This re-runs part of the previous step, and would hence overwrite part of the criterion results.
-          # Avoid that by shuffling the directories around so this run uses its own results directory.
-          mv target/criterion target/criterion-bench
-          mv target/criterion-transfer-profile target/criterion || true
-          taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \
-              --bench --exact "Run multiple transfers with varying seeds" --noplot
-          # And now restore the directories.
-          mv target/criterion target/criterion-transfer-profile
-          mv target/criterion-bench target/criterion
-
-      - name: Profile client/server transfer
+      - name: Compare neqo and msquic
+        env:
+          HOST: 127.0.0.1
+          PORT: 4433
+          SIZE: 134217728 # 128 MB
         run: |
           TMP=$(mktemp -d)
-          { mkdir server; \
-            cd server; \
-            taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } &
-          mkdir client; \
-            cd client; \
-            taskset -c 1 nice -n -20 time \
-            cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \
-              --bin neqo-client -- --output-dir "$TMP" "https://$HOST:4433/$SIZE"
-          killall -INT neqo-server
-          cd ${{ github.workspace }}
-          [ "$(wc -c < "$TMP/$SIZE")" -eq "$SIZE" ] || exit 1
+          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
+          truncate -s "$SIZE" "/tmp/$SIZE"
+          declare -A client_cmd=(
+            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
+          )
+          declare -A server_cmd=(
+            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
+          )
+
+          function transmogrify {
+            CMD=$1
+            local cc=$2
+            local pacing=$3
+            if [ "$cc" != "" ]; then
+              CMD=${CMD//_cc/--cc $cc}
+              EXT="-$cc"
+            fi
+            if [ "$pacing" == "pacing" ]; then
+              CMD=${CMD//_pacing/--pacing}
+              EXT="$EXT-$pacing"
+            fi
+          }
+
+          for server in neqo msquic; do
+            for client in neqo msquic; do
+              if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
+                cc_opt=("")
+                pacing_opt=("")
+              else
+                cc_opt=("reno" "cubic")
+                pacing_opt=("pacing" "")
+              fi
+              for cc in "${cc_opt[@]}"; do
+                for pacing in "${pacing_opt[@]}"; do
+                  TAG="client $client ← $server server $cc $pacing"
+                  echo "Running benchmarks for $TAG" | tee -a comparison.txt
+                  transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.server.perf" CMD &
+                  PID=$!
+                  transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                  echo "$CMD"
+                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.client.perf" \
+                    hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
+                    tee -a comparison.txt || true
+                  echo >>comparison.txt
+                  kill $PID
+                  {
+                    echo
+                    cat step.md
+                  } >>comparison.md
+                done
+              done
+            done
+          done
           rm -r "$TMP"
-        env:
-          HOST: localhost
-          SIZE: 1073741824 # 1 GB
 
       - name: Compare neqo and msquic
         env:
@@ -149,20 +179,23 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i perf.data -F +pid > transfer.perf &
-          perf script -i client/perf.data -F +pid > client.perf &
-          perf script -i server/perf.data -F +pid > server.perf &
+          perf script -i transfer.perf -F +pid > transfer.fx.perf &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
+            done
+          done
           wait
-          mv flamegraph.svg transfer.svg
-          mv client/flamegraph.svg client.svg
-          mv server/flamegraph.svg server.svg
           rm neqo.svg
 
       - name: Generate perf reports
         run: |
-          perf report -i perf.data --no-children --stdio > transfer.perf.txt &
-          perf report -i client/perf.data --no-children --stdio > client.perf.txt &
-          perf report -i server/perf.data --no-children --stdio > server.perf.txt &
+          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
+          for CC in newreno cubic; do
+            for PEER in client server; do
+              perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &
+             done
+          done
           wait
 
       - name: Format results as Markdown

From 33d243b4cbef8fe299ac92e77c1c0adb7e2540e5 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 10:55:45 +1000
Subject: [PATCH 19/46] Fixes

---
 .github/workflows/bench.yml | 61 ++++++++-----------------------------
 1 file changed, 13 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a779f76c98..81b923d6c8 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -9,7 +9,7 @@ env:
   RUST_BACKTRACE: 1
   TOOLCHAIN: nightly
   RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
-  PERF_OPT: (record -F997 --call-graph fp -g)
+  PERF_OPT: record -F997 --call-graph fp -g
 
 jobs:
   bench:
@@ -120,13 +120,17 @@ jobs:
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   echo "$CMD"
-                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.server.perf" CMD &
+                  # shellcheck disable=SC2086
+                  taskset -c 0 nice -n -20 \
+                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   echo "$CMD"
-                  perf "${PERF_OPT[@]}" -o "$client-$server$EXT.client.perf" \
-                    hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
-                    tee -a comparison.txt || true
+                  # shellcheck disable=SC2086
+                  taskset -c 0 nice -n -20 \
+                    perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
+                      hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
+                      tee -a comparison.txt || true
                   echo >>comparison.txt
                   kill $PID
                   {
@@ -139,39 +143,6 @@ jobs:
           done
           rm -r "$TMP"
 
-      - name: Compare neqo and msquic
-        env:
-          HOST: 127.0.0.1
-          PORT: 4433
-          SIZE: 134217728 # 128 MB
-        run: |
-          TMP=$(mktemp -d)
-          openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2> /dev/null
-          truncate -s "$SIZE" "/tmp/$SIZE"
-          declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
-          )
-          declare -A server_cmd=(
-            ["neqo"]="target/release/neqo-server -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
-          )
-          for server in neqo msquic; do
-            for client in neqo msquic; do
-              ${server_cmd["$server"]} &
-              PID=$!
-              echo "Running benchmarks for client $client ← $server server" | tee -a comparison.txt
-              hyperfine -w 1 -n "$client ← $server" --export-markdown step.md "${client_cmd[$client]}" | tee -a comparison.txt || true
-              echo >> comparison.txt
-              kill $PID
-              {
-                echo
-                cat step.md
-              } >> comparison.md
-            done
-          done
-          rm -r "$TMP"
-
       # Re-enable turboboost, hyperthreading and use powersave governor.
       - name: Restore machine
         run: sudo /root/bin/unprep.sh
@@ -179,22 +150,16 @@ jobs:
 
       - name: Convert for profiler.firefox.com
         run: |
-          perf script -i transfer.perf -F +pid > transfer.fx.perf &
-          for CC in newreno cubic; do
-            for PEER in client server; do
-              perf script -i "$PEER-$CC.perf" -F +pid > "$PEER-$CC.fx.perf" &
-            done
+          for f in *.perf; do
+            perf script -i "$f" -F +pid > "$(basename -s .perf $f).fx.perf" &
           done
           wait
           rm neqo.svg
 
       - name: Generate perf reports
         run: |
-          perf report -i transfer.perf --no-children --stdio > transfer.perf.txt &
-          for CC in newreno cubic; do
-            for PEER in client server; do
-              perf report -i "$PEER-$CC.perf" --no-children --stdio > "$PEER-$CC.txt" &
-             done
+          for f in *.perf; do
+            perf report -i "$f" --no-children --stdio > "$f.txt" &
           done
           wait
 

From ba3f750a9c3f3408ecc19b0482023e350a702e3e Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 10:58:02 +1000
Subject: [PATCH 20/46] Fix

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 81b923d6c8..a167852782 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -151,7 +151,7 @@ jobs:
       - name: Convert for profiler.firefox.com
         run: |
           for f in *.perf; do
-            perf script -i "$f" -F +pid > "$(basename -s .perf $f).fx.perf" &
+            perf script -i "$f" -F +pid > "$(basename -s .perf "$f").fx.perf" &
           done
           wait
           rm neqo.svg

From 72f8203995a8c79d2d78dbd721a4ddbafef5bc36 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 11:33:33 +1000
Subject: [PATCH 21/46] Fixes

---
 .github/workflows/bench.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a167852782..ece487f6c2 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -119,13 +119,11 @@ jobs:
                   TAG="client $client ← $server server $cc $pacing"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
-                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" CMD &
+                    perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \

From 5cb812a71cb165ec9e557abed9b2113b698ab9cc Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 14:27:57 +1000
Subject: [PATCH 22/46] Tweaks

---
 .github/workflows/bench.yml | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index ece487f6c2..5c1c35dfc4 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -52,7 +52,7 @@ jobs:
         run: |
           mkdir -p msquic/build
           cd msquic/build
-          cmake -GNinja -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
+          cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 ..
           cmake --build .
 
       - name: Download cached main-branch results
@@ -116,7 +116,11 @@ jobs:
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
-                  TAG="client $client ← $server server $cc $pacing"
+                  TAG="client $client ← $server server ($cc"
+                  if [ "$pacing" == "pacing" ]; then
+                    TAG="$TAG, pacing"
+                  fi
+                  TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
@@ -125,7 +129,7 @@ jobs:
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
-                  taskset -c 0 nice -n -20 \
+                  taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                       hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
                       tee -a comparison.txt || true
@@ -146,20 +150,18 @@ jobs:
         run: sudo /root/bin/unprep.sh
         if: success() || failure() || cancelled()
 
-      - name: Convert for profiler.firefox.com
-        run: |
-          for f in *.perf; do
-            perf script -i "$f" -F +pid > "$(basename -s .perf "$f").fx.perf" &
-          done
-          wait
-          rm neqo.svg
-
-      - name: Generate perf reports
+      - name: Post-process perf data
         run: |
           for f in *.perf; do
+            # Convert for profiler.firefox.com
+            perf script -i "$f" -F +pid > "$f.fx" &
+            # Generate perf reports
             perf report -i "$f" --no-children --stdio > "$f.txt" &
+            # Generate flamegraphs
+            flamegraph --perfdata "$f" -o "${f//.perf/.svg}" &
           done
           wait
+          rm neqo.svg
 
       - name: Format results as Markdown
         id: results

From 5f5b793c654f87f93e730e52b11feb66fc8856e8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Sun, 17 Mar 2024 15:22:17 +1000
Subject: [PATCH 23/46] Debug

---
 .github/workflows/bench.yml | 15 ++++++++-------
 1 file changed, 8 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5c1c35dfc4..86230de79d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -124,25 +124,26 @@ jobs:
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
+                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
+                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
+                      hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -w 1 -n "$TAG" --export-markdown step.md "$CMD" |
-                      tee -a comparison.txt || true
-                  echo >>comparison.txt
+                      hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      tee -a comparison.txt
+                  echo >> comparison.txt
                   kill $PID
-                  {
-                    echo
-                    cat step.md
-                  } >>comparison.md
+                  cat step.md >> steps.md
                 done
               done
             done
           done
+          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | sed '/^\|:-/{x;/^$/!d;g;}' > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From e78b1be2d67faf1c3d9bda606ebc2c30fd0ce06b Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 06:37:23 +1000
Subject: [PATCH 24/46] actionlint

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 86230de79d..d7b062e053 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -123,15 +123,15 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  # shellcheck disable=SC2086
                   echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
+                  # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  # shellcheck disable=SC2086
                   echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
                       hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
+                  # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                       hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |

From f36a1b7b8eee4194ee4ec40aa082c3ed8da76ee8 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 08:21:20 +1000
Subject: [PATCH 25/46] Fix sed

---
 .github/workflows/bench.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index d7b062e053..0bd50d6167 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -102,6 +102,8 @@ jobs:
             if [ "$pacing" == "pacing" ]; then
               CMD=${CMD//_pacing/--pacing}
               EXT="$EXT-$pacing"
+            else
+              CMD=${CMD//_pacing/}
             fi
           }
 

From 89e8b16bf97b993ae484e68bf81d65a8741c8c96 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 08:28:35 +1000
Subject: [PATCH 26/46] Pacing changed

---
 .github/workflows/bench.yml | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0bd50d6167..22e7da101d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -99,11 +99,11 @@ jobs:
               CMD=${CMD//_cc/--cc $cc}
               EXT="-$cc"
             fi
-            if [ "$pacing" == "pacing" ]; then
-              CMD=${CMD//_pacing/--pacing}
+            if [ "$pacing" == "on" ]; then
+              CMD=${CMD//_pacing/}
               EXT="$EXT-$pacing"
             else
-              CMD=${CMD//_pacing/}
+              CMD=${CMD//_pacing/--no-pacing}
             fi
           }
 
@@ -114,12 +114,12 @@ jobs:
                 pacing_opt=("")
               else
                 cc_opt=("reno" "cubic")
-                pacing_opt=("pacing" "")
+                pacing_opt=("on" "")
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
                   TAG="client $client ← $server server ($cc"
-                  if [ "$pacing" == "pacing" ]; then
+                  if [ "$pacing" == "on" ]; then
                     TAG="$TAG, pacing"
                   fi
                   TAG="$TAG)"

From 51a1ecf3b99d0e7082e59825d067f9a12424c200 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 09:07:35 +1000
Subject: [PATCH 27/46] Fixes

---
 .github/workflows/bench.yml | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 22e7da101d..41d98936ba 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -101,9 +101,10 @@ jobs:
             fi
             if [ "$pacing" == "on" ]; then
               CMD=${CMD//_pacing/}
-              EXT="$EXT-$pacing"
+              EXT="$EXT-pacing"
             else
               CMD=${CMD//_pacing/--no-pacing}
+              EXT="$EXT-nopacing"
             fi
           }
 
@@ -118,25 +119,22 @@ jobs:
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
-                  TAG="client $client ← $server server ($cc"
+                  TAG="client $client ← server $server ($cc"
                   if [ "$pacing" == "on" ]; then
                     TAG="$TAG, pacing"
                   fi
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf \
-                      hyperfine -w 1 -n $TAG -u millisecond --export-markdown step.md $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -w 1 -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine --show-output -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID
@@ -145,7 +143,9 @@ jobs:
               done
             done
           done
-          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | sed '/^\|:-/{x;/^$/!d;g;}' > comparison.md
+          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | \
+            sed '/^\|:-/{x;/^$/!d;g;}' | \
+            sed 's/\`//g' > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From 20e9ca97c099218abc60ede3eb8d74b0e91a9953 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 10:01:42 +1000
Subject: [PATCH 28/46] msquic server exits with non-zero

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 41d98936ba..068057ae5f 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -88,7 +88,7 @@ jobs:
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit || true"
           )
 
           function transmogrify {
@@ -134,7 +134,7 @@ jobs:
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine --show-output -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID

From ec5d6d60ff2eea6387b6ac2e4a51fb597ed09048 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 10:37:23 +1000
Subject: [PATCH 29/46] echo

---
 .github/workflows/bench.yml | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 068057ae5f..3185abe91a 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -126,11 +126,13 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
+                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
+                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf hyperfine -n $TAG -u millisecond --export-markdown step.md $CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \

From cd6c224de5fdc9234f179eaf46777f4c19f0b0dc Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 12:31:08 +1000
Subject: [PATCH 30/46] Again

---
 .github/workflows/bench.yml | 27 ++++++++++++++++++---------
 .github/workflows/check.yml |  8 ++++----
 2 files changed, 22 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 3185abe91a..0200dc2d48 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -2,6 +2,13 @@ name: Bench
 on:
   workflow_call:
   workflow_dispatch:
+  push:
+    branches: ["main"]
+    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+  pull_request:
+    branches: ["main"]
+    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+  merge_group:
 env:
   CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
   CARGO_PROFILE_RELEASE_DEBUG: true
@@ -67,11 +74,11 @@ jobs:
       - name: Prepare machine
         run: sudo /root/bin/prep.sh
 
-      # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
-      - name: Run cargo bench
-        run: |
-          taskset -c 0 nice -n -20 \
-            cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
+      # # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
+      # - name: Run cargo bench
+      #   run: |
+      #     taskset -c 0 nice -n -20 \
+      #       cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
       - name: Compare neqo and msquic
         env:
@@ -108,8 +115,8 @@ jobs:
             fi
           }
 
-          for server in neqo msquic; do
-            for client in neqo msquic; do
+          for server in msquic neqo; do
+            for client in msquic neqo; do
               if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
                 cc_opt=("")
                 pacing_opt=("")
@@ -126,19 +133,21 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD"
+                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "perf $PERF_OPT -o $client-$server$EXT.client.perf hyperfine -n $TAG -u millisecond --export-markdown step.md $CMD"
+                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
                       hyperfine -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
+                  ps -ef
+                  echo "Killing $PID"
                   kill $PID
                   cat step.md >> steps.md
                 done
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index 10085ffda6..d04971b370 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -163,7 +163,7 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
         if: matrix.type == 'debug' && matrix.rust-toolchain == 'stable'
 
-  bench:
-    name: "Benchmark"
-    needs: [check]
-    uses: ./.github/workflows/bench.yml
+  # bench:
+  #   name: "Benchmark"
+  #   needs: [check]
+  #   uses: ./.github/workflows/bench.yml

From 32a43d8201bc68238cd5b0c277620bc9f2e42a86 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 12:42:02 +1000
Subject: [PATCH 31/46] Again

---
 .github/workflows/bench.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 0200dc2d48..fd9e0a23c3 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -143,11 +143,9 @@ jobs:
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
-                  ps -ef
-                  echo "Killing $PID"
                   kill $PID
                   cat step.md >> steps.md
                 done

From b96b392f410706e25032459c663636445baa9f30 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 13:02:48 +1000
Subject: [PATCH 32/46] A new hope

---
 .github/workflows/bench.yml | 17 +++++------------
 .github/workflows/check.yml |  8 ++++----
 2 files changed, 9 insertions(+), 16 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index fd9e0a23c3..bb5067b9c8 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -2,13 +2,6 @@ name: Bench
 on:
   workflow_call:
   workflow_dispatch:
-  push:
-    branches: ["main"]
-    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
-  pull_request:
-    branches: ["main"]
-    paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
-  merge_group:
 env:
   CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true
   CARGO_PROFILE_RELEASE_DEBUG: true
@@ -74,11 +67,11 @@ jobs:
       - name: Prepare machine
         run: sudo /root/bin/prep.sh
 
-      # # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
-      # - name: Run cargo bench
-      #   run: |
-      #     taskset -c 0 nice -n -20 \
-      #       cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
+      # Pin the benchmark run to core 0 and run all benchmarks at elevated priority.
+      - name: Run cargo bench
+        run: |
+          taskset -c 0 nice -n -20 \
+            cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
       - name: Compare neqo and msquic
         env:
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index d04971b370..10085ffda6 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -163,7 +163,7 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
         if: matrix.type == 'debug' && matrix.rust-toolchain == 'stable'
 
-  # bench:
-  #   name: "Benchmark"
-  #   needs: [check]
-  #   uses: ./.github/workflows/bench.yml
+  bench:
+    name: "Benchmark"
+    needs: [check]
+    uses: ./.github/workflows/bench.yml

From f69c48a2930541f4ee957ea7d1ad0d6f09a50ef5 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 14:00:53 +1000
Subject: [PATCH 33/46] Finalize

---
 .github/workflows/bench.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index bb5067b9c8..581226a035 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -83,12 +83,12 @@ jobs:
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
+            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE 2> /dev/null"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE > /dev/null"
           )
           declare -A server_cmd=(
-            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT"
-            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit || true"
+            ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
+            ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
           )
 
           function transmogrify {
@@ -145,9 +145,9 @@ jobs:
               done
             done
           done
-          sed '/^\| Command/{x;/^$/!d;g;}' steps.md | \
-            sed '/^\|:-/{x;/^$/!d;g;}' | \
-            sed 's/\`//g' > comparison.md
+          sed 's/`//g' steps.md |\
+            awk '!/^\| Command/ || !c++' |\
+            awk '!/^\|:/ || !c++' > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
@@ -163,7 +163,7 @@ jobs:
             # Generate perf reports
             perf report -i "$f" --no-children --stdio > "$f.txt" &
             # Generate flamegraphs
-            flamegraph --perfdata "$f" -o "${f//.perf/.svg}" &
+            flamegraph --perfdata "$f" --palette rust -o "${f//.perf/.svg}" &
           done
           wait
           rm neqo.svg

From 02ff40b4fbf555ceb5cd00743b40b3f981419199 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 14:54:48 +1000
Subject: [PATCH 34/46] Fixes

---
 .github/workflows/bench.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 581226a035..2a034d9908 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -83,7 +83,7 @@ jobs:
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
           truncate -s "$SIZE" "/tmp/$SIZE"
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE 2> /dev/null"
+            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE 2> /dev/null"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE > /dev/null"
           )
           declare -A server_cmd=(
@@ -141,6 +141,7 @@ jobs:
                   echo >> comparison.txt
                   kill $PID
                   cat step.md >> steps.md
+                  [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
                 done
               done
             done

From 2e3393f8ab2c6f41335676993b4e77fa1cf0c835 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 15:37:55 +1000
Subject: [PATCH 35/46] ls

---
 .github/workflows/bench.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 2a034d9908..23a8c7a542 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -141,6 +141,7 @@ jobs:
                   echo >> comparison.txt
                   kill $PID
                   cat step.md >> steps.md
+                  ls -l
                   [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
                 done
               done

From 41be4cf9512aea14e328363105ab250791a97a10 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 15:46:20 +1000
Subject: [PATCH 36/46] Add comments

---
 .github/workflows/bench.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 23a8c7a542..75b44aa08d 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -73,6 +73,8 @@ jobs:
           taskset -c 0 nice -n -20 \
             cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt
 
+      # Compare various configurations of neqo against msquic, and gather perf data
+      # during the hyperfine runs.
       - name: Compare neqo and msquic
         env:
           HOST: 127.0.0.1
@@ -80,8 +82,11 @@ jobs:
           SIZE: 134217728 # 128 MB
         run: |
           TMP=$(mktemp -d)
+          # Make a cert and key for msquic.
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
+          # Make a test file for msquic to serve.
           truncate -s "$SIZE" "/tmp/$SIZE"
+          # Define the commands to run for each client and server.
           declare -A client_cmd=(
             ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE 2> /dev/null"
             ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE > /dev/null"
@@ -91,6 +96,8 @@ jobs:
             ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit > /dev/null || true"
           )
 
+          # Replace various placeholders in the commands with the actual values.
+          # Also generate an extension to append to the test tag.
           function transmogrify {
             CMD=$1
             local cc=$2
@@ -110,6 +117,7 @@ jobs:
 
           for server in msquic neqo; do
             for client in msquic neqo; do
+              # msquic doesn't let us configure the congestion control or pacing.
               if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then
                 cc_opt=("")
                 pacing_opt=("")
@@ -119,6 +127,7 @@ jobs:
               fi
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
+                  # Make a tag string for this test, for the results.
                   TAG="client $client ← server $server ($cc"
                   if [ "$pacing" == "on" ]; then
                     TAG="$TAG, pacing"
@@ -142,11 +151,13 @@ jobs:
                   kill $PID
                   cat step.md >> steps.md
                   ls -l
+                  # Sanity check the size of the last retrieved file.
                   [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
                 done
               done
             done
           done
+          # Merge the results tables generated by hyperfine into a single table.
           sed 's/`//g' steps.md |\
             awk '!/^\| Command/ || !c++' |\
             awk '!/^\|:/ || !c++' > comparison.md

From 1632971f6738fd5e6259b3ae431a4c5e60851004 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 15:53:29 +1000
Subject: [PATCH 37/46] Report transfer size

---
 .github/workflows/bench.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 75b44aa08d..82ea819034 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -158,9 +158,10 @@ jobs:
             done
           done
           # Merge the results tables generated by hyperfine into a single table.
+          echo "Transfer of $SIZE bytes over loopback." > comparison.md
           sed 's/`//g' steps.md |\
             awk '!/^\| Command/ || !c++' |\
-            awk '!/^\|:/ || !c++' > comparison.md
+            awk '!/^\|:/ || !c++' >> comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From d8ed22312c3a34d1be9d3a703d7d5806aaed9b57 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 16:46:12 +1000
Subject: [PATCH 38/46] Again

---
 .github/workflows/bench.yml | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 82ea819034..a2ebaaaff1 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -85,11 +85,11 @@ jobs:
           # Make a cert and key for msquic.
           openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null
           # Make a test file for msquic to serve.
-          truncate -s "$SIZE" "/tmp/$SIZE"
+          truncate -s "$SIZE" "$TMP/$SIZE"
           # Define the commands to run for each client and server.
           declare -A client_cmd=(
-            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE 2> /dev/null"
-            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE > /dev/null"
+            ["neqo"]="target/release/neqo-client _cc _pacing --output-dir . -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE"
+            ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE"
           )
           declare -A server_cmd=(
             ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT 2> /dev/null"
@@ -145,7 +145,7 @@ jobs:
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
-                      hyperfine -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
+                      hyperfine -N --output null -w 1 -s "sleep 1" -n "$TAG" -u millisecond --export-markdown step.md "$CMD" |
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID

From f0144f254cf754bcac3e19bb43278d483e4c9bdd Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 17:51:37 +1000
Subject: [PATCH 39/46] Quiet the server down

---
 neqo-bin/src/bin/server/old_https.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/neqo-bin/src/bin/server/old_https.rs b/neqo-bin/src/bin/server/old_https.rs
index f36c99c484..a159029007 100644
--- a/neqo-bin/src/bin/server/old_https.rs
+++ b/neqo-bin/src/bin/server/old_https.rs
@@ -202,7 +202,6 @@ impl HttpServer for Http09Server {
                     None => break,
                     Some(e) => e,
                 };
-                eprintln!("Event {event:?}");
                 match event {
                     ConnectionEvent::NewStream { stream_id } => {
                         self.write_state
@@ -221,6 +220,7 @@ impl HttpServer for Http09Server {
                             .unwrap();
                     }
                     ConnectionEvent::StateChange(_)
+                    | ConnectionEvent::SendStreamCreatable { .. }
                     | ConnectionEvent::SendStreamComplete { .. } => (),
                     e => eprintln!("unhandled event {e:?}"),
                 }

From 02a284f98409ac4d3c893d32d6f92cae437042e4 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Mon, 18 Mar 2024 17:52:27 +1000
Subject: [PATCH 40/46] Remove debug output

---
 .github/workflows/bench.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index a2ebaaaff1..e7b6ff1bcb 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -135,13 +135,11 @@ jobs:
                   TAG="$TAG)"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 0 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD &
                   PID=$!
                   transmogrify "${client_cmd[$client]}" "$cc" "$pacing"
-                  echo "$CMD"
                   # shellcheck disable=SC2086
                   taskset -c 1 nice -n -20 \
                     perf $PERF_OPT -o "$client-$server$EXT.client.perf" \
@@ -150,7 +148,6 @@ jobs:
                   echo >> comparison.txt
                   kill $PID
                   cat step.md >> steps.md
-                  ls -l
                   # Sanity check the size of the last retrieved file.
                   [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1
                 done

From babb5e85671fccac87bb96d372ddd026c9208fa4 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 08:25:59 +1000
Subject: [PATCH 41/46] Reformat table

---
 .github/workflows/bench.yml | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index e7b6ff1bcb..14b96920c4 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -128,11 +128,7 @@ jobs:
               for cc in "${cc_opt[@]}"; do
                 for pacing in "${pacing_opt[@]}"; do
                   # Make a tag string for this test, for the results.
-                  TAG="client $client ← server $server ($cc"
-                  if [ "$pacing" == "on" ]; then
-                    TAG="$TAG, pacing"
-                  fi
-                  TAG="$TAG)"
+                  TAG="$client,$server,$cc,$pacing"
                   echo "Running benchmarks for $TAG" | tee -a comparison.txt
                   transmogrify "${server_cmd[$server]}" "$cc" "$pacing"
                   # shellcheck disable=SC2086
@@ -157,8 +153,10 @@ jobs:
           # Merge the results tables generated by hyperfine into a single table.
           echo "Transfer of $SIZE bytes over loopback." > comparison.md
           sed 's/`//g' steps.md |\
+            sed -e 's/,/ \| /g' |\
             awk '!/^\| Command/ || !c++' |\
             awk '!/^\|:/ || !c++' >> comparison.md
+            sed -e 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' |\
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
@@ -231,6 +229,7 @@ jobs:
           path: |
             *.svg
             *.perf
+            *.perf.fx
             *.txt
             results.*
             target/criterion*

From 83a81f8697ff895c4b11282e7c34d269e99d5185 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 08:39:15 +1000
Subject: [PATCH 42/46] Fix

---
 .github/workflows/bench.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 14b96920c4..1c977f2520 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -155,8 +155,8 @@ jobs:
           sed 's/`//g' steps.md |\
             sed -e 's/,/ \| /g' |\
             awk '!/^\| Command/ || !c++' |\
-            awk '!/^\|:/ || !c++' >> comparison.md
-            sed -e 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' |\
+            awk '!/^\|:/ || !c++' |\
+            sed -e 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From b374e175c367103a07716f8a6211cd25c2c67557 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 09:43:18 +1000
Subject: [PATCH 43/46] Fix table

---
 .github/workflows/bench.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 1c977f2520..15e211e10b 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -153,9 +153,10 @@ jobs:
           # Merge the results tables generated by hyperfine into a single table.
           echo "Transfer of $SIZE bytes over loopback." > comparison.md
           sed 's/`//g' steps.md |\
-            sed -e 's/,/ \| /g' |\
             awk '!/^\| Command/ || !c++' |\
             awk '!/^\|:/ || !c++' |\
+            sed -e 's/^\|:/\|:---\|:---\|:---\|:/g' |\
+            sed -e 's/,/ \| /g' |\
             sed -e 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
           rm -r "$TMP"
 

From d2af3e54994c43aaddfb4cc33ab8c302e3758912 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 10:55:41 +1000
Subject: [PATCH 44/46] Mac sed != GNU sed

---
 .github/workflows/bench.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 15e211e10b..5e05763cbb 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -155,9 +155,9 @@ jobs:
           sed 's/`//g' steps.md |\
             awk '!/^\| Command/ || !c++' |\
             awk '!/^\|:/ || !c++' |\
-            sed -e 's/^\|:/\|:---\|:---\|:---\|:/g' |\
-            sed -e 's/,/ \| /g' |\
-            sed -e 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
+            sed -E 's/^\|:/\|:---\|:---\|:---\|:/g' |\
+            sed -E 's/,/ \| /g' |\
+            sed -E 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From 318eea654e0d502a613b12244765823bc1386348 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 13:35:04 +1000
Subject: [PATCH 45/46] Avoid piping commands into themselves

---
 .github/workflows/bench.yml | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 5e05763cbb..7f2829f03c 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -152,12 +152,8 @@ jobs:
           done
           # Merge the results tables generated by hyperfine into a single table.
           echo "Transfer of $SIZE bytes over loopback." > comparison.md
-          sed 's/`//g' steps.md |\
-            awk '!/^\| Command/ || !c++' |\
-            awk '!/^\|:/ || !c++' |\
-            sed -E 's/^\|:/\|:---\|:---\|:---\|:/g' |\
-            sed -E 's/,/ \| /g' |\
-            sed -E 's/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
+          awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
+            sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|:/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pacing/g' >> comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.

From aa064002f8d60199835da10887b01c96655f2827 Mon Sep 17 00:00:00 2001
From: Lars Eggert <lars@eggert.org>
Date: Tue, 19 Mar 2024 13:36:36 +1000
Subject: [PATCH 46/46] Fix comment

---
 .github/workflows/bench.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
index 7f2829f03c..70aebaeb08 100644
--- a/.github/workflows/bench.yml
+++ b/.github/workflows/bench.yml
@@ -97,7 +97,7 @@ jobs:
           )
 
           # Replace various placeholders in the commands with the actual values.
-          # Also generate an extension to append to the test tag.
+          # Also generate an extension to append to the file name.
           function transmogrify {
             CMD=$1
             local cc=$2