From bf0657e12f43d6222b9964b2e642106856c04aeb Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Mon, 12 Feb 2024 19:43:38 +0200 Subject: [PATCH 01/26] ci: Benchmark without a qlog To remove those overheads from the profile. --- test-fixture/src/lib.rs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test-fixture/src/lib.rs b/test-fixture/src/lib.rs index 923aa76c15..ec993c81e6 100644 --- a/test-fixture/src/lib.rs +++ b/test-fixture/src/lib.rs @@ -392,12 +392,16 @@ impl ToString for SharedVec { /// Panics if the log cannot be created. #[must_use] pub fn new_neqo_qlog() -> (NeqoQlog, SharedVec) { - let mut trace = new_trace(Role::Client); - // Set reference time to 0.0 for testing. - trace.common_fields.as_mut().unwrap().reference_time = Some(0.0); let buf = SharedVec { buf: Arc::default(), }; + + #[cfg(feature = "bench")] + return (NeqoQlog::disabled(), buf); + + let mut trace = new_trace(Role::Client); + // Set reference time to 0.0 for testing. + trace.common_fields.as_mut().unwrap().reference_time = Some(0.0); let contents = buf.clone(); let streamer = QlogStreamer::new( qlog::QLOG_VERSION.to_string(), From a3a3c31d7f0073406f7480c93fc3029b1f206ee9 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 09:03:09 +0200 Subject: [PATCH 02/26] Tweaks to workflow --- .github/workflows/bench.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index a29abd39a4..4ddae1bceb 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -37,7 +37,7 @@ jobs: uses: actions/checkout@v4 - name: Build - run: cargo +$TOOLCHAIN bench --features ci,bench --no-run + run: cargo +$TOOLCHAIN bench --features bench --no-run # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine @@ -46,15 +46,16 @@ jobs: # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - name: Benchmark run: | - nice -n -20 taskset -c 0 \ - cargo +$TOOLCHAIN bench --features ci,bench | tee output.txt + /usr/bin/nice -n -20 taskset -c 0 \ + cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. - name: Perf transfer benchmark run: | - nice -n -20 taskset -c 0 \ + /usr/bin/nice -n -20 taskset -c 0 \ perf record -F997 --call-graph=lbr -o perf.data \ - cargo +$TOOLCHAIN bench --features ci,bench --bench transfer + cargo +$TOOLCHAIN bench --features bench --bench transfer -- \ + --exact "Run multiple transfers with varying seeds" # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine @@ -68,7 +69,7 @@ jobs: key: ${{ runner.os }}-benchmark # TODO: Wait for this action to be allowlisted. And then figure out how to only upload - # benchmark data when the main branch is being updated. + # benchmark data when the main branch is being updated (e.g., if: ${{ github.ref == "refs/heads/main" }}) # - name: Store current benchmark results # uses: benchmark-action/github-action-benchmark@v1 # with: From cc17e6a905ac30096b8b102b1f73990dd021d9ba Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 09:23:33 +0200 Subject: [PATCH 03/26] Try and use cargo flamegraph --- .github/workflows/bench.yml | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 4ddae1bceb..cc007e6a71 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,5 +1,7 @@ name: Bench on: + push: + pull_request: workflow_call: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true @@ -22,20 +24,29 @@ jobs: toolchain: $TOOLCHAIN components: rustfmt - - name: Configure Rust - run: echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment" >> "$GITHUB_ENV" - - name: Install sccache uses: mozilla-actions/sccache-action@v0.0.4 - - name: Enable sccache + - name: Configure Rust run: | + echo "RUSTFLAGS=-C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment" >> "$GITHUB_ENV" echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV" echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV" + cargo install flamegraph - name: Checkout uses: actions/checkout@v4 + - name: Fetch NSS and NSPR + run: | + hg clone https://hg.mozilla.org/projects/nspr "$NSPR_DIR" + hg clone https://hg.mozilla.org/projects/nss "$NSS_DIR" + echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV" + echo "NSPR_DIR=$NSPR_DIR" >> "$GITHUB_ENV" + env: + NSS_DIR: ${{ github.workspace }}/nss + NSPR_DIR: ${{ github.workspace }}/nspr + - name: Build run: cargo +$TOOLCHAIN bench --features bench --no-run @@ -53,9 +64,8 @@ jobs: - name: Perf transfer benchmark run: | /usr/bin/nice -n -20 taskset -c 0 \ - perf record -F997 --call-graph=lbr -o perf.data \ - cargo +$TOOLCHAIN bench --features bench --bench transfer -- \ - --exact "Run multiple transfers with varying seeds" + cargo +$TOOLCHAIN flamegraph -v -F 997 --features bench --bench transfer -- \ + --bench --exact "Run multiple transfers with varying seeds" # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine From 0fe33683637c7a876970222bf9b7914b07a53103 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 09:35:52 +0200 Subject: [PATCH 04/26] LD_LIBRARY_PATH --- .github/workflows/bench.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index cc007e6a71..8c9a8277be 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -48,7 +48,9 @@ jobs: NSPR_DIR: ${{ github.workspace }}/nspr - name: Build - run: cargo +$TOOLCHAIN bench --features bench --no-run + run: | + cargo +$TOOLCHAIN bench --features bench --no-run + echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Debug/lib" >> "$GITHUB_ENV" # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine @@ -57,13 +59,13 @@ jobs: # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - name: Benchmark run: | - /usr/bin/nice -n -20 taskset -c 0 \ + nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. - name: Perf transfer benchmark run: | - /usr/bin/nice -n -20 taskset -c 0 \ + nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN flamegraph -v -F 997 --features bench --bench transfer -- \ --bench --exact "Run multiple transfers with varying seeds" From 847345444ea1873b8dabab382febec1fc2962cd5 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 10:07:57 +0200 Subject: [PATCH 05/26] Use explicit perf command --- .github/workflows/bench.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8c9a8277be..207578ddb7 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -66,7 +66,7 @@ jobs: - name: Perf transfer benchmark run: | nice -n -20 taskset -c 0 \ - cargo +$TOOLCHAIN flamegraph -v -F 997 --features bench --bench transfer -- \ + cargo +$TOOLCHAIN flamegraph -v -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" --features bench --bench transfer -- \ --bench --exact "Run multiple transfers with varying seeds" # Re-enable turboboost, hyperthreading and use powersave governor. From 5c6af8dfe9dab96fa10efb51f59135866de5fde3 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 10:52:30 +0200 Subject: [PATCH 06/26] Export more perf data --- .github/workflows/bench.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 207578ddb7..0585d79e45 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -63,6 +63,7 @@ jobs: cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. + # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - name: Perf transfer benchmark run: | nice -n -20 taskset -c 0 \ @@ -93,14 +94,16 @@ jobs: # comment-on-alert: true # summary-always: true - - name: Convert perf data - run: | - perf script -i perf.data -F +pid | zstd > perf.ff.data.zst - zstd perf.data + - name: Convert for profiler.firefox.com + run: perf script -i perf.data -F +pid > perf.firefox.data - name: Archive perf data uses: actions/upload-artifact@v4 with: name: ${{ github.head_ref || github.ref_name }}-perf - path: "*.zst" - compression-level: 0 + path: | + flamegraph.svg + perf*.data + output.txt + target/criterion + compression-level: 9 From 25fc2897aa90d6c8e965de78302a045a8ad57022 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 11:37:29 +0200 Subject: [PATCH 07/26] Hack --- neqo-common/src/qlog.rs | 4 ++++ test-fixture/src/lib.rs | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/neqo-common/src/qlog.rs b/neqo-common/src/qlog.rs index 3da8350990..be34c541a4 100644 --- a/neqo-common/src/qlog.rs +++ b/neqo-common/src/qlog.rs @@ -93,6 +93,10 @@ impl NeqoQlog { F: FnOnce(&mut QlogStreamer) -> Result<(), qlog::Error>, { if let Some(inner) = self.inner.borrow_mut().as_mut() { + // TODO: Hack. + if inner.qlog_path == Path::new("") { + return; + } if let Err(e) = f(&mut inner.streamer) { crate::do_log!( ::log::Level::Error, diff --git a/test-fixture/src/lib.rs b/test-fixture/src/lib.rs index ec993c81e6..fe1351c715 100644 --- a/test-fixture/src/lib.rs +++ b/test-fixture/src/lib.rs @@ -396,8 +396,9 @@ pub fn new_neqo_qlog() -> (NeqoQlog, SharedVec) { buf: Arc::default(), }; - #[cfg(feature = "bench")] - return (NeqoQlog::disabled(), buf); + if cfg!(feature = "bench") { + return (NeqoQlog::disabled(), buf); + } let mut trace = new_trace(Role::Client); // Set reference time to 0.0 for testing. From 63547d2422147d0f4ad2ebfea60eaccbc276a7e7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 11:59:33 +0200 Subject: [PATCH 08/26] Finalize --- .github/workflows/bench.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 0585d79e45..37e31e545c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,7 +1,5 @@ name: Bench on: - push: - pull_request: workflow_call: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true From d32bbf5101578b6d8a27154baaa54e55bd8ef968 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 12:30:04 +0200 Subject: [PATCH 09/26] Remove hack, define `bench` feature in `Cargo.toml` --- neqo-common/src/qlog.rs | 4 ---- test-fixture/Cargo.toml | 1 + 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/neqo-common/src/qlog.rs b/neqo-common/src/qlog.rs index be34c541a4..3da8350990 100644 --- a/neqo-common/src/qlog.rs +++ b/neqo-common/src/qlog.rs @@ -93,10 +93,6 @@ impl NeqoQlog { F: FnOnce(&mut QlogStreamer) -> Result<(), qlog::Error>, { if let Some(inner) = self.inner.borrow_mut().as_mut() { - // TODO: Hack. - if inner.qlog_path == Path::new("") { - return; - } if let Err(e) = f(&mut inner.streamer) { crate::do_log!( ::log::Level::Error, diff --git a/test-fixture/Cargo.toml b/test-fixture/Cargo.toml index 18bdb114be..34727febc7 100644 --- a/test-fixture/Cargo.toml +++ b/test-fixture/Cargo.toml @@ -20,3 +20,4 @@ qlog = "0.12" [features] deny-warnings = [] +bench = [] From 74b512cc7c896d282f72b694c3f25d944826fb58 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 15:18:21 +0200 Subject: [PATCH 10/26] Don't run tests with feature `bench` --- .github/workflows/check.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index afab22ef48..67c134c7ea 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -134,14 +134,16 @@ jobs: LIB_DIR: ${{ matrix.type == 'release' && 'Release' || 'Debug' }} - name: Run tests and determine coverage - run: cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci,bench --no-fail-fast --lcov --output-path lcov.info + run: | + cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --all-targets --features ci --no-fail-fast --lcov --output-path lcov.info + cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run - name: Run client/server transfer run: | - cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --features ci,bench --bin neqo-client --bin neqo-server - cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --features ci,bench --bin neqo-server -- $HOST:4433 & + cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --bin neqo-client --bin neqo-server + cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --bin neqo-server -- $HOST:4433 & PID=$! - cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --features ci,bench --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE + cargo +${{ matrix.rust-toolchain }} run $BUILD_TYPE --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE kill $PID [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1 env: From 1f9463ea1b03ea68fccefccbe993e19eb8d7a1d2 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 15:43:45 +0200 Subject: [PATCH 11/26] See if a separate artifact can be loaded into profiler.firefox.com --- .github/workflows/bench.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 37e31e545c..435de07a01 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,6 +1,8 @@ name: Bench on: workflow_call: + push: + pull_request: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true CARGO_TERM_COLOR: always @@ -95,7 +97,7 @@ jobs: - name: Convert for profiler.firefox.com run: perf script -i perf.data -F +pid > perf.firefox.data - - name: Archive perf data + - name: Archive all benchmark results uses: actions/upload-artifact@v4 with: name: ${{ github.head_ref || github.ref_name }}-perf @@ -105,3 +107,10 @@ jobs: output.txt target/criterion compression-level: 9 + + - name: Archive for profiler.firefox.com + uses: actions/upload-artifact@v4 + with: + name: profiler.firefox.com + path: perf.firefox.data + compression-level: 9 From a50b09a4c3329e980a947df503638d7de9ca8777 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 15:58:34 +0200 Subject: [PATCH 12/26] Done --- .github/workflows/bench.yml | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 435de07a01..37e31e545c 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,8 +1,6 @@ name: Bench on: workflow_call: - push: - pull_request: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true CARGO_TERM_COLOR: always @@ -97,7 +95,7 @@ jobs: - name: Convert for profiler.firefox.com run: perf script -i perf.data -F +pid > perf.firefox.data - - name: Archive all benchmark results + - name: Archive perf data uses: actions/upload-artifact@v4 with: name: ${{ github.head_ref || github.ref_name }}-perf @@ -107,10 +105,3 @@ jobs: output.txt target/criterion compression-level: 9 - - - name: Archive for profiler.firefox.com - uses: actions/upload-artifact@v4 - with: - name: profiler.firefox.com - path: perf.firefox.data - compression-level: 9 From c0eb020a788b0cd486b42714588746eeb2ef4917 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 17:19:01 +0200 Subject: [PATCH 13/26] Benchmark a proper transfer --- .github/workflows/bench.yml | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 37e31e545c..66dfef9d58 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,5 +1,7 @@ name: Bench on: + push: + pull_request: workflow_call: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true @@ -52,7 +54,9 @@ jobs: # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine - run: sudo /root/bin/prep.sh + run: | + sudo /root/bin/prep.sh + echo 'PERF_CMD="record -o perf.data -F997 --call-graph dwarf,16384 -g"' >> "$GITHUB_ENV" # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - name: Benchmark @@ -65,9 +69,25 @@ jobs: - name: Perf transfer benchmark run: | nice -n -20 taskset -c 0 \ - cargo +$TOOLCHAIN flamegraph -v -c "record -o perf.data -F997 --call-graph dwarf,16384 -g" --features bench --bench transfer -- \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ --bench --exact "Run multiple transfers with varying seeds" + - name: Benchmark a client/server transfer + run: | + cargo +$TOOLCHAIN build --profile bench --features bench --bin neqo-client --bin neqo-server + mkdir server && \ + cd server && \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --bin neqo-server -- $HOST:4433 & + PID=$! + mkdir client && \ + cd client && \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE + kill $PID + [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1 + env: + HOST: localhost + SIZE: 7654321 + # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine run: sudo /root/bin/unprep.sh From 08796f33f80314b545868bb19b92512be84b8df7 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 17:31:34 +0200 Subject: [PATCH 14/26] Do more --- .github/workflows/bench.yml | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 66dfef9d58..6506973bef 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -56,7 +56,7 @@ jobs: - name: Prepare machine run: | sudo /root/bin/prep.sh - echo 'PERF_CMD="record -o perf.data -F997 --call-graph dwarf,16384 -g"' >> "$GITHUB_ENV" + echo "PERF_CMD=record -o perf.data -F997 --call-graph dwarf,16384 -g" >> "$GITHUB_ENV" # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - name: Benchmark @@ -77,13 +77,17 @@ jobs: cargo +$TOOLCHAIN build --profile bench --features bench --bin neqo-client --bin neqo-server mkdir server && \ cd server && \ - cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --bin neqo-server -- $HOST:4433 & + nice -n -20 taskset -c 0 \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ + --bin neqo-server -- $HOST:4433 & PID=$! mkdir client && \ cd client && \ - cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE + nice -n -20 taskset -c 1 \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ + --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE kill $PID - [ "$(wc -c <"$SIZE")" -eq "$SIZE" ] || exit 1 + [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost SIZE: 7654321 @@ -113,15 +117,23 @@ jobs: # summary-always: true - name: Convert for profiler.firefox.com - run: perf script -i perf.data -F +pid > perf.firefox.data + run: | + mv flamegraph.svg transfer.svg + mv client/flamegraph.svg client.svg + mv server/flamegraph.svg server.svg + mv perf.data transfer.perf && perf script -i transfer.perf -F +pid > transfer.firefox.perf & + mv client/perf.data client.perf && perf script -i client.perf -F +pid > client.firefox.perf & + mv server/perf.data server.perf && perf script -i server.perf -F +pid > server.firefox.perf & + wait + rm neqo.svg - name: Archive perf data uses: actions/upload-artifact@v4 with: name: ${{ github.head_ref || github.ref_name }}-perf path: | - flamegraph.svg - perf*.data + *.svg + *.perf output.txt target/criterion compression-level: 9 From d198855110fd2fd419bdb30231f9cfebfd75c908 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 17:43:59 +0200 Subject: [PATCH 15/26] Build before prep --- .github/workflows/bench.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6506973bef..6ac02f2c17 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -50,7 +50,8 @@ jobs: - name: Build run: | cargo +$TOOLCHAIN bench --features bench --no-run - echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Debug/lib" >> "$GITHUB_ENV" + cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server + echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Release/lib" >> "$GITHUB_ENV" # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine @@ -74,7 +75,6 @@ jobs: - name: Benchmark a client/server transfer run: | - cargo +$TOOLCHAIN build --profile bench --features bench --bin neqo-client --bin neqo-server mkdir server && \ cd server && \ nice -n -20 taskset -c 0 \ @@ -118,13 +118,13 @@ jobs: - name: Convert for profiler.firefox.com run: | + perf script -i perf.data -F +pid > transfer.perf & + perf script -i client/perf.data -F +pid > client.perf & + perf script -i server/perf.data -F +pid > server.perf & + wait mv flamegraph.svg transfer.svg mv client/flamegraph.svg client.svg mv server/flamegraph.svg server.svg - mv perf.data transfer.perf && perf script -i transfer.perf -F +pid > transfer.firefox.perf & - mv client/perf.data client.perf && perf script -i client.perf -F +pid > client.firefox.perf & - mv server/perf.data server.perf && perf script -i server.perf -F +pid > server.firefox.perf & - wait rm neqo.svg - name: Archive perf data From f5edf819877766bb3397373cc456257b0f0dd430 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 18:03:40 +0200 Subject: [PATCH 16/26] Retry --- .github/workflows/bench.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 6ac02f2c17..945f7dcc41 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -5,6 +5,7 @@ on: workflow_call: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true + CARGO_PROFILE_RELEASE_DEBUG: true CARGO_TERM_COLOR: always RUST_BACKTRACE: 1 TOOLCHAIN: nightly @@ -51,7 +52,7 @@ jobs: run: | cargo +$TOOLCHAIN bench --features bench --no-run cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server - echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Release/lib" >> "$GITHUB_ENV" + echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Debug/lib" >> "$GITHUB_ENV" # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine From 86fecaf18565b24a11b7b101079eee2bd0f1cb17 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 19:48:42 +0200 Subject: [PATCH 17/26] Set server db. Unprep on workflow cancel. --- .github/workflows/bench.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 945f7dcc41..acc212143e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -80,7 +80,7 @@ jobs: cd server && \ nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ - --bin neqo-server -- $HOST:4433 & + --bin neqo-server -- --db ../test-fixture/db $HOST:4433 & PID=$! mkdir client && \ cd client && \ @@ -96,7 +96,7 @@ jobs: # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine run: sudo /root/bin/unprep.sh - if: success() || failure() + if: success() || failure() || cancelled() - name: Download previous benchmark results uses: actions/cache@v4 From 3ded06c9517992fe6c319c46fa6de664b067a38b Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 19:59:39 +0200 Subject: [PATCH 18/26] Again --- .github/workflows/bench.yml | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index acc212143e..336953d0b7 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -61,18 +61,18 @@ jobs: echo "PERF_CMD=record -o perf.data -F997 --call-graph dwarf,16384 -g" >> "$GITHUB_ENV" # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - - name: Benchmark - run: | - nice -n -20 taskset -c 0 \ - cargo +$TOOLCHAIN bench --features bench | tee output.txt + # - name: Benchmark + # run: | + # nice -n -20 taskset -c 0 \ + # cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - - name: Perf transfer benchmark - run: | - nice -n -20 taskset -c 0 \ - cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ - --bench --exact "Run multiple transfers with varying seeds" + # - name: Perf transfer benchmark + # run: | + # nice -n -20 taskset -c 0 \ + # cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ + # --bench --exact "Run multiple transfers with varying seeds" - name: Benchmark a client/server transfer run: | @@ -88,6 +88,7 @@ jobs: cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE kill $PID + cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost From 32351b9a4595e5bece4e7d54795a2dc69acd4ff9 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 20:04:34 +0200 Subject: [PATCH 19/26] Again --- .github/workflows/bench.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 336953d0b7..794835cf2d 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -84,15 +84,15 @@ jobs: PID=$! mkdir client && \ cd client && \ - nice -n -20 taskset -c 1 \ + time nice -n -20 taskset -c 1 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE - kill $PID + kill -INT $PID cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost - SIZE: 7654321 + SIZE: 87654321 # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine @@ -120,11 +120,11 @@ jobs: - name: Convert for profiler.firefox.com run: | - perf script -i perf.data -F +pid > transfer.perf & + # perf script -i perf.data -F +pid > transfer.perf & perf script -i client/perf.data -F +pid > client.perf & perf script -i server/perf.data -F +pid > server.perf & wait - mv flamegraph.svg transfer.svg + # mv flamegraph.svg transfer.svg mv client/flamegraph.svg client.svg mv server/flamegraph.svg server.svg rm neqo.svg From ef30a0ddd6214580b8233cf5b24140007016517d Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 20:15:26 +0200 Subject: [PATCH 20/26] pid --- .github/workflows/bench.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 794835cf2d..8d2fe9cd67 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -82,17 +82,21 @@ jobs: cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-server -- --db ../test-fixture/db $HOST:4433 & PID=$! + ps -ef + echo $PID mkdir client && \ cd client && \ time nice -n -20 taskset -c 1 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE + ps -ef kill -INT $PID + ps -ef cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost - SIZE: 87654321 + SIZE: 256000000 # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine From f451967960b08cfcff4f82a5f2eed5f1627b58ab Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Tue, 13 Feb 2024 20:22:47 +0200 Subject: [PATCH 21/26] bg --- .github/workflows/bench.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8d2fe9cd67..8caf41c0db 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -76,11 +76,11 @@ jobs: - name: Benchmark a client/server transfer run: | - mkdir server && \ + { mkdir server && \ cd server && \ nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ - --bin neqo-server -- --db ../test-fixture/db $HOST:4433 & + --bin neqo-server -- --db ../test-fixture/db $HOST:4433 ; } & PID=$! ps -ef echo $PID @@ -92,6 +92,7 @@ jobs: ps -ef kill -INT $PID ps -ef + fg $PID cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: From ac264448b167e5fdf884d150cade54892b8a1dd0 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 14 Feb 2024 09:54:50 +0200 Subject: [PATCH 22/26] true --- .github/workflows/bench.yml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 8caf41c0db..7c471bcfd6 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -76,23 +76,18 @@ jobs: - name: Benchmark a client/server transfer run: | - { mkdir server && \ - cd server && \ + { mkdir server; \ + cd server; \ nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ - --bin neqo-server -- --db ../test-fixture/db $HOST:4433 ; } & + --bin neqo-server -- --db ../test-fixture/db $HOST:4433 || true; } & PID=$! - ps -ef - echo $PID - mkdir client && \ - cd client && \ + mkdir client; \ + cd client; \ time nice -n -20 taskset -c 1 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE - ps -ef kill -INT $PID - ps -ef - fg $PID cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: From 69c381e9142e0d38aec6849ffc946ef044cb1f37 Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 14 Feb 2024 10:04:43 +0200 Subject: [PATCH 23/26] killall --- .github/workflows/bench.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 7c471bcfd6..151e9e0525 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -81,13 +81,12 @@ jobs: nice -n -20 taskset -c 0 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-server -- --db ../test-fixture/db $HOST:4433 || true; } & - PID=$! mkdir client; \ cd client; \ time nice -n -20 taskset -c 1 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE - kill -INT $PID + killall -INT neqo-server cd ${{ github.workspace }} [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: From 8b6aece7a48a3703cbc998b0906887428b8f1ecb Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 14 Feb 2024 10:12:42 +0200 Subject: [PATCH 24/26] Re-enable all --- .github/workflows/bench.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 151e9e0525..00a909d59e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -61,29 +61,29 @@ jobs: echo "PERF_CMD=record -o perf.data -F997 --call-graph dwarf,16384 -g" >> "$GITHUB_ENV" # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - # - name: Benchmark - # run: | - # nice -n -20 taskset -c 0 \ - # cargo +$TOOLCHAIN bench --features bench | tee output.txt + - name: Benchmark + run: | + taskset -c 0 nice -n -20 \ + cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - # - name: Perf transfer benchmark - # run: | - # nice -n -20 taskset -c 0 \ - # cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ - # --bench --exact "Run multiple transfers with varying seeds" + - name: Perf transfer benchmark + run: | + taskset -c 0 nice -n -20 \ + cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ + --bench --exact "Run multiple transfers with varying seeds" - name: Benchmark a client/server transfer run: | { mkdir server; \ cd server; \ - nice -n -20 taskset -c 0 \ + taskset -c 0 nice -n -20 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-server -- --db ../test-fixture/db $HOST:4433 || true; } & mkdir client; \ cd client; \ - time nice -n -20 taskset -c 1 \ + time taskset -c 1 nice -n -20 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" \ --bin neqo-client -- --output-dir . https://$HOST:4433/$SIZE killall -INT neqo-server @@ -91,7 +91,7 @@ jobs: [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost - SIZE: 256000000 + SIZE: 536870912 # 512 MB # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine @@ -119,11 +119,11 @@ jobs: - name: Convert for profiler.firefox.com run: | - # perf script -i perf.data -F +pid > transfer.perf & + perf script -i perf.data -F +pid > transfer.perf & perf script -i client/perf.data -F +pid > client.perf & perf script -i server/perf.data -F +pid > server.perf & wait - # mv flamegraph.svg transfer.svg + mv flamegraph.svg transfer.svg mv client/flamegraph.svg client.svg mv server/flamegraph.svg server.svg rm neqo.svg From 71f702b7cede5256a36513d2962a95b2e5b9ac6e Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 14 Feb 2024 10:36:10 +0200 Subject: [PATCH 25/26] Properly cache results --- .github/workflows/bench.yml | 33 +++++++++++++++++++++++++++------ 1 file changed, 27 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 00a909d59e..fe103595f9 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -54,6 +54,18 @@ jobs: cargo +$TOOLCHAIN build --release --bin neqo-client --bin neqo-server echo "LD_LIBRARY_PATH=${{ github.workspace }}/dist/Debug/lib" >> "$GITHUB_ENV" + - name: Download criterion results + uses: actions/cache/restore@v4 + with: + path: ./target/criterion + key: neqo-${{ runner.os }}-main-criterion + + - name: Download github-action-benchmark results + uses: actions/cache/restore@v4 + with: + path: ./cache + key: neqo-${{ runner.os }}-main-github-action-benchmark + # Disable turboboost, hyperthreading and use performance governor. - name: Prepare machine run: | @@ -98,12 +110,6 @@ jobs: run: sudo /root/bin/unprep.sh if: success() || failure() || cancelled() - - name: Download previous benchmark results - uses: actions/cache@v4 - with: - path: ./cache - key: ${{ runner.os }}-benchmark - # TODO: Wait for this action to be allowlisted. And then figure out how to only upload # benchmark data when the main branch is being updated (e.g., if: ${{ github.ref == "refs/heads/main" }}) # - name: Store current benchmark results @@ -128,6 +134,21 @@ jobs: mv server/flamegraph.svg server.svg rm neqo.svg + - name: Upload criterion results + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: ./target/criterion + key: neqo-${{ runner.os }}-main-criterion + + - name: Upload github-action-benchmark results + if: github.ref == 'refs/heads/main' + uses: actions/cache/save@v4 + with: + path: ./cache + key: neqo-${{ runner.os }}-main-github-action-benchmark + + - name: Archive perf data uses: actions/upload-artifact@v4 with: From 3c4b92624c5a7e92625943762fa79442a5642b5f Mon Sep 17 00:00:00 2001 From: Lars Eggert Date: Wed, 14 Feb 2024 10:48:35 +0200 Subject: [PATCH 26/26] Finalize --- .github/workflows/bench.yml | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index fe103595f9..a1ea61cf1e 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -1,7 +1,5 @@ name: Bench on: - push: - pull_request: workflow_call: env: CARGO_PROFILE_BENCH_BUILD_OVERRIDE_DEBUG: true @@ -73,20 +71,20 @@ jobs: echo "PERF_CMD=record -o perf.data -F997 --call-graph dwarf,16384 -g" >> "$GITHUB_ENV" # Pin the benchmark run to core 0 and run all benchmarks at elevated priority. - - name: Benchmark + - name: Run cargo bench run: | taskset -c 0 nice -n -20 \ cargo +$TOOLCHAIN bench --features bench | tee output.txt # Pin the transfer benchmark to core 0 and run it at elevated priority inside perf. # Work around https://github.com/flamegraph-rs/flamegraph/issues/248 by passing explicit perf arguments. - - name: Perf transfer benchmark + - name: Profile cargo bench transfer run: | taskset -c 0 nice -n -20 \ cargo +$TOOLCHAIN flamegraph -v -c "$PERF_CMD" --features bench --bench transfer -- \ --bench --exact "Run multiple transfers with varying seeds" - - name: Benchmark a client/server transfer + - name: Profile client/server transfer run: | { mkdir server; \ cd server; \ @@ -103,7 +101,7 @@ jobs: [ "$(wc -c < client/"$SIZE")" -eq "$SIZE" ] || exit 1 env: HOST: localhost - SIZE: 536870912 # 512 MB + SIZE: 1073741824 # 1 GB # Re-enable turboboost, hyperthreading and use powersave governor. - name: Restore machine @@ -148,7 +146,6 @@ jobs: path: ./cache key: neqo-${{ runner.os }}-main-github-action-benchmark - - name: Archive perf data uses: actions/upload-artifact@v4 with: