Delta

.github/workflows/bench.yml

@@ -41,6 +41,7 @@ jobs:
           persist-credentials: false
 
       - name: Checkout google/quiche
+        if: 0 # FIXME
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           repository: google/quiche
@@ -219,25 +220,38 @@ jobs:
                       tee -a comparison.txt
                   echo >> comparison.txt
                   kill $PID
-                  cat "hyperfine/$FILENAME.md" >> steps.md
-
-                  ls -l "hyperfine/$FILENAME.json"
-                  cat "hyperfine/$FILENAME.json"
-                  ls -l "hyperfine-main/$FILENAME.json"
-                  cat "hyperfine-main/$FILENAME.json"
 
                   # Sanity check the size of the last retrieved file.
                   # google/quiche outputs the HTTP header, too, so we can't just check for -eq.
                   [ "$(wc -c <"$SIZE")" -ge "$SIZE" ] || exit 1
+
+                  # https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae
+                  # Covariance asssumed to be 0, i.e., variables are assumed to be independent.
+                  BASELINE="hyperfine-main/$FILENAME.json"
+                  RESULT="hyperfine/$FILENAME.json"
+                  BASELINE_MEAN=$(jq -r '.results[0].mean' "$BASELINE")
+                  MEAN=$(jq -r '.results[0].mean' "$RESULT")
+                  RATIO=$(echo "$MEAN / $BASELINE_MEAN" | bc)
+                  BASELINE_STDDEV=$(jq -r '.results[0].stddev' "$BASELINE")
+                  STDDEV=$(jq -r '.results[0].stddev' "$RESULT")
+                  RATIO_STDDEV=$(echo "$RATIO * sqrt(($STDDEV / $MEAN)^2 + ($BASELINE_STDDEV / $BASELINE_MEAN)^2)" | bc)
+                  {
+                    cut -f1-8 -d\| < "hyperfine/$FILENAME.md"
+                    printf "| %f.2 ± %f.2 |\n" "$RATIO" "$RATIO_STDDEV"
+                  } >> steps.md
                 done
               done
             done
           done
-          # Merge the results tables generated by hyperfine into a single table.
-          echo "Transfer of $SIZE bytes over loopback, $RUNS runs." > comparison.md
-          awk '(!/^\| Command/ || !c++) && (!/^\|:/ || !d++)' < steps.md |\
-            sed -E 's/`//g; s/^\|:/\|:---\|:---\|:---\|/g; s/,/ \| /g; s/^\| Command/\| Client \| Server \| CC \| Pace/g' |\
-            cut -f1-8 -d\| | sed -e 's/$/|/' >> comparison.md
+
+          # Make a single results table.
+          {
+            echo "Transfer of $SIZE bytes over loopback, $RUNS runs."
+            echo
+            echo '| Client | Server | CC | Pacing | Mean [ms] | Min [ms] | Max [ms] | Delta |'
+            echo '|:---|:---|:---|---|---:|---:|---:|---:|'
+            cat steps.md
+            } > comparison.md
           rm -r "$TMP"
 
       # Re-enable turboboost, hyperthreading and use powersave governor.
@@ -273,7 +287,7 @@ jobs:
             {
               echo "Performance differences relative to $SHA."
               echo
-            } >> results.md
+            } | tee sha.md >> results.md
           fi
           sed -E -e 's/^                 //gi' \
                  -e 's/((change|time|thrpt):[^%]*% )([^%]*%)(.*)/\1<b>\3<\/b>\4/gi' results.txt |\
@@ -285,6 +299,10 @@ jobs:
           {
             echo
             echo "### Client/server transfer results"
+            SHA=$(cat target/criterion/baseline-sha.txt || true)
+            if [ -n "$SHA" ]; then
+              cat sha.md >> results.md
+            fi
             cat comparison.md
           } >> results.md
           cat results.md > "$GITHUB_STEP_SUMMARY"