diff --git a/.github/actions/rust/action.yml b/.github/actions/rust/action.yml index bfb09d332d..a567cb422b 100644 --- a/.github/actions/rust/action.yml +++ b/.github/actions/rust/action.yml @@ -30,7 +30,7 @@ runs: - name: Install Rust tools shell: bash - run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants + run: cargo +${{ inputs.version }} binstall --no-confirm cargo-llvm-cov cargo-nextest flamegraph cargo-hack cargo-mutants hyperfine fleet-rs sccache # sccache slows CI down, so we leave it disabled. # Leaving the steps below commented out, so we can re-evaluate enabling it later. diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 81ef297a9e..3185abe91a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -9,7 +9,7 @@ env: RUST_BACKTRACE: 1 TOOLCHAIN: nightly RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes - PERF_CMD: record -o perf.data -F997 --call-graph fp -g + PERF_OPT: record -F997 --call-graph fp -g jobs: bench: @@ -20,9 +20,17 @@ jobs: shell: bash steps: - - name: Checkout + - name: Checkout neqo uses: actions/checkout@v4 + - name: Checkout msquic + uses: actions/checkout@v4 + with: + repository: microsoft/msquic + ref: main + path: msquic + submodules: true + - name: Set PATH run: echo "/home/bench/.cargo/bin" >> "${GITHUB_PATH}" @@ -35,10 +43,17 @@ jobs: - name: Fetch and build NSS and NSPR uses: ./.github/actions/nss - - name: Build + - name: Build neqo run: | cargo "+$TOOLCHAIN" bench --features bench --no-run - cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server + cargo "+$TOOLCHAIN" build --release + + - name: Build msquic + run: | + mkdir -p msquic/build + cd msquic/build + cmake -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DQUIC_BUILD_TOOLS=1 -DQUIC_BUILD_PERF=1 .. + cmake --build . - name: Download cached main-branch results id: criterion-cache @@ -58,56 +73,101 @@ jobs: taskset -c 0 nice -n -20 \ cargo "+$TOOLCHAIN" bench --features bench -- --noplot | tee results.txt - # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. - # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - - name: Profile cargo bench transfer - run: | - taskset -c 0 nice -n -20 \ - cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ - --bench --exact "Run multiple transfers with varying seeds" --noplot - - - name: Profile client/server transfer - run: | - { mkdir server; \ - cd server; \ - taskset -c 0 nice -n -20 \ - cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \ - --bin neqo-server -- --db ../test-fixture/db "$HOST:4433" || true; } & - mkdir client; \ - cd client; \ - time taskset -c 1 nice -n -20 \ - cargo "+$TOOLCHAIN" flamegraph -v -c "$PERF_CMD" \ - --bin neqo-client -- --output-dir . "https://$HOST:4433/$SIZE" - killall -INT neqo-server - cd ${{ github.workspace }} - [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 + - name: Compare neqo and msquic env: - HOST: localhost - SIZE: 1073741824 # 1 GB + HOST: 127.0.0.1 + PORT: 4433 + SIZE: 134217728 # 128 MB + run: | + TMP=$(mktemp -d) + openssl req -nodes -new -x509 -keyout "$TMP/key" -out "$TMP/cert" -subj "/CN=DOMAIN" 2>/dev/null + truncate -s "$SIZE" "/tmp/$SIZE" + declare -A client_cmd=( + ["neqo"]="target/release/neqo-client _cc _pacing -o -a hq-interop -Q 1 https://$HOST:$PORT/$SIZE" + ["msquic"]="msquic/build/bin/Release/quicinterop -test:D -timeout:99999999 -custom:$HOST -port:$PORT -urls:https://$HOST:$PORT/$SIZE" + ) + declare -A server_cmd=( + ["neqo"]="target/release/neqo-server _cc _pacing -o -a hq-interop -Q 1 $HOST:$PORT" + ["msquic"]="msquic/build/bin/Release/quicinteropserver -root:$TMP -listen:$HOST -port:$PORT -file:$TMP/cert -key:$TMP/key -noexit || true" + ) + + function transmogrify { + CMD=$1 + local cc=$2 + local pacing=$3 + if [ "$cc" != "" ]; then + CMD=${CMD//_cc/--cc $cc} + EXT="-$cc" + fi + if [ "$pacing" == "on" ]; then + CMD=${CMD//_pacing/} + EXT="$EXT-pacing" + else + CMD=${CMD//_pacing/--no-pacing} + EXT="$EXT-nopacing" + fi + } + + for server in neqo msquic; do + for client in neqo msquic; do + if [ "$client" == "msquic" ] && [ "$server" == "msquic" ]; then + cc_opt=("") + pacing_opt=("") + else + cc_opt=("reno" "cubic") + pacing_opt=("on" "") + fi + for cc in "${cc_opt[@]}"; do + for pacing in "${pacing_opt[@]}"; do + TAG="client $client ← server $server ($cc" + if [ "$pacing" == "on" ]; then + TAG="$TAG, pacing" + fi + TAG="$TAG)" + echo "Running benchmarks for $TAG" | tee -a comparison.txt + transmogrify "${server_cmd[$server]}" "$cc" "$pacing" + echo "perf $PERF_OPT -o $client-$server$EXT.server.perf $CMD" + # shellcheck disable=SC2086 + taskset -c 0 nice -n -20 \ + perf $PERF_OPT -o "$client-$server$EXT.server.perf" $CMD & + PID=$! + transmogrify "${client_cmd[$client]}" "$cc" "$pacing" + echo "perf $PERF_OPT -o $client-$server$EXT.client.perf hyperfine -n $TAG -u millisecond --export-markdown step.md $CMD" + # shellcheck disable=SC2086 + taskset -c 1 nice -n -20 \ + perf $PERF_OPT -o "$client-$server$EXT.client.perf" \ + hyperfine -n "$TAG" -u millisecond --export-markdown step.md "$CMD" | + tee -a comparison.txt + echo >> comparison.txt + kill $PID + cat step.md >> steps.md + done + done + done + done + sed '/^\| Command/{x;/^$/!d;g;}' steps.md | \ + sed '/^\|:-/{x;/^$/!d;g;}' | \ + sed 's/\`//g' > comparison.md + rm -r "$TMP" # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine run: sudo /root/bin/unprep.sh if: success() || failure() || cancelled() - - name: Convert for profiler.firefox.com + - name: Post-process perf data run: | - perf script -i perf.data -F +pid > transfer.perf & - perf script -i client/perf.data -F +pid > client.perf & - perf script -i server/perf.data -F +pid > server.perf & + for f in *.perf; do + # Convert for profiler.firefox.com + perf script -i "$f" -F +pid > "$f.fx" & + # Generate perf reports + perf report -i "$f" --no-children --stdio > "$f.txt" & + # Generate flamegraphs + flamegraph --perfdata "$f" -o "${f//.perf/.svg}" & + done wait - mv flamegraph.svg transfer.svg - mv client/flamegraph.svg client.svg - mv server/flamegraph.svg server.svg rm neqo.svg - - name: Generate perf reports - run: | - perf report -i perf.data --no-children --stdio > transfer.perf.txt & - perf report -i client/perf.data --no-children --stdio > client.perf.txt & - perf report -i server/perf.data --no-children --stdio > server.perf.txt & - wait - - name: Format results as Markdown id: results run: | @@ -129,6 +189,11 @@ jobs: -e 's/^([a-z0-9].*)$/* **\1**/gi' \ -e 's/(change:[^%]*% )([^%]*%)(.*)/\1**\2**\3/gi' \ >> results.md + { + echo "### Client/server transfer results" + cat comparison.md + } >> results.md + cat results.md > "$GITHUB_STEP_SUMMARY" - name: Remember main-branch push URL if: github.ref == 'refs/heads/main' diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 10085ffda6..a54f76489e 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -108,18 +108,18 @@ jobs: - name: Build run: | # shellcheck disable=SC2086 - cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci + fleet +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci - name: Run tests and determine coverage run: | # shellcheck disable=SC2086 - cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info - cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run + fleet +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info + fleet +${{ matrix.rust-toolchain }} bench --features bench --no-run - name: Run client/server transfer run: | # shellcheck disable=SC2086 - cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server + fleet +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server "target/$BUILD_DIR/neqo-server" "$HOST:4433" & PID=$! "target/$BUILD_DIR/neqo-client" --output-dir . "https://$HOST:4433/$SIZE" @@ -137,7 +137,7 @@ jobs: CONFIG_PATH="--config-path=$(mktemp)" fi # shellcheck disable=SC2086 - cargo +${{ matrix.rust-toolchain }} fmt --all -- --check $CONFIG_PATH + fleet +${{ matrix.rust-toolchain }} fmt --all -- --check $CONFIG_PATH if: success() || failure() - name: Clippy @@ -146,11 +146,11 @@ jobs: # respective default features only. Can reveal warnings otherwise # hidden given that a plain cargo clippy combines all features of the # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695. - cargo +${{ matrix.rust-toolchain }} hack clippy --all-targets -- -D warnings || ${{ matrix.rust-toolchain == 'nightly' }} + fleet +${{ matrix.rust-toolchain }} hack clippy --all-targets -- -D warnings || ${{ matrix.rust-toolchain == 'nightly' }} if: success() || failure() - name: Check rustdoc links - run: cargo +${{ matrix.rust-toolchain }} doc --workspace --no-deps --document-private-items + run: fleet +${{ matrix.rust-toolchain }} doc --workspace --no-deps --document-private-items env: RUSTDOCFLAGS: "--deny rustdoc::broken_intra_doc_links --deny warnings" if: success() || failure()