Skip to content

Commit

Permalink
feat: Use the simulator as a benchmark (#1616)
Browse files Browse the repository at this point in the history 
* Moving simulator to the test fixture

* Split the simulator into two pieces in preparation for bench

* Remove println statements

* Fix server fmt

* chore: add a file that I missed

* Add a fixed seed for the test

And increase the time available for benching.

---------

Signed-off-by: Lars Eggert <[email protected]>
Co-authored-by: Lars Eggert <[email protected]>
  • Loading branch information
@martinthomson @larseggert
martinthomson and larseggert authored Feb 5, 2024
1 parent cb2d623 commit 9260ba3
Show file tree
Hide file tree
Showing 12 changed files with 309 additions and 104 deletions.
7 changes: 6 additions & 1 deletion neqo-transport/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ qlog = { git = "https://github.com/cloudflare/quiche", rev = "09ea4b244096a01307
smallvec = "1.11.1"

[dev-dependencies]
criterion = "0.5.1"
criterion = { version = "0.5.1", features = ["html_reports"] }
enum-map = "2.7"
test-fixture = { path = "../test-fixture" }

Expand All @@ -25,6 +25,11 @@ bench = []
deny-warnings = []
fuzzing = ["neqo-crypto/fuzzing"]

[[bench]]
name = "transfer"
harness = false
required-features = ["bench"]

[[bench]]
name = "rx_stream_orderer"
harness = false
Expand Down
64 changes: 64 additions & 0 deletions neqo-transport/benches/transfer.rs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
use std::time::Duration;

use criterion::{criterion_group, criterion_main, BatchSize::SmallInput, Criterion};
use test_fixture::{
boxed,
sim::{
connection::{ConnectionNode, ReceiveData, SendData},
network::{Delay, TailDrop},
Simulator,
},
};

const ZERO: Duration = Duration::from_millis(0);
const JITTER: Duration = Duration::from_millis(10);
const TRANSFER_AMOUNT: usize = 1 << 22; // 4Mbyte

fn benchmark_transfer(c: &mut Criterion, label: &str, seed: Option<impl AsRef<str>>) {
c.bench_function(label, |b| {
b.iter_batched(
|| {
let nodes = boxed![
ConnectionNode::default_client(boxed![SendData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_uplink(),
Delay::new(ZERO..JITTER),
ConnectionNode::default_server(boxed![ReceiveData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_downlink(),
Delay::new(ZERO..JITTER),
];
let mut sim = Simulator::new(label, nodes);
if let Some(seed) = &seed {
sim.seed_str(seed);
}
sim.setup()
},
|sim| {
sim.run();
},
SmallInput,
)
});
}

fn benchmark_transfer_variable(c: &mut Criterion) {
benchmark_transfer(
c,
"Run multiple transfers with varying seeds",
std::env::var("SIMULATION_SEED").ok(),
);
}

fn benchmark_transfer_fixed(c: &mut Criterion) {
benchmark_transfer(
c,
"Run multiple transfers with the same seed",
Some("62df6933ba1f543cece01db8f27fb2025529b27f93df39e19f006e1db3b8c843"),
);
}

criterion_group! {
name = transfer;
config = Criterion::default().warm_up_time(Duration::from_secs(5)).measurement_time(Duration::from_secs(15));
targets = benchmark_transfer_variable, benchmark_transfer_fixed
}
criterion_main!(transfer);
114 changes: 69 additions & 45 deletions neqo-transport/tests/network.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,17 @@
#![cfg_attr(feature = "deny-warnings", deny(warnings))]
#![warn(clippy::pedantic)]

mod sim;

use std::{ops::Range, time::Duration};

use neqo_transport::{ConnectionError, ConnectionParameters, Error, State};
use sim::{
connection::{ConnectionNode, ReachState, ReceiveData, SendData},
network::{Delay, Drop, TailDrop},
Simulator,
use test_fixture::{
boxed,
sim::{
connection::{ConnectionNode, ReachState, ReceiveData, SendData},
network::{Delay, Drop, TailDrop},
Simulator,
},
simulate,
};

/// The amount of transfer. Much more than this takes a surprising amount of time.
Expand All @@ -32,26 +34,28 @@ const fn weeks(m: u32) -> Duration {
simulate!(
connect_direct,
[
ConnectionNode::default_client(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::default_server(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::new_client(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
ConnectionNode::new_server(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
]
);

simulate!(
idle_timeout,
[
ConnectionNode::default_client(boxed![
ReachState::new(State::Confirmed),
ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))
]),
ConnectionNode::default_server(boxed![
ReachState::new(State::Confirmed),
ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))
]),
ConnectionNode::default_client(boxed![ReachState::new(State::Closed(
ConnectionError::Transport(Error::IdleTimeout)
))]),
ConnectionNode::default_server(boxed![ReachState::new(State::Closed(
ConnectionError::Transport(Error::IdleTimeout)
))]),
]
);

Expand All @@ -60,23 +64,19 @@ simulate!(
[
ConnectionNode::new_client(
ConnectionParameters::default().idle_timeout(weeks(1000)),
boxed![
ReachState::new(State::Confirmed),
ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))
]
boxed![ReachState::new(State::Confirmed),],
boxed![ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))]
),
Delay::new(weeks(6)..weeks(6)),
Drop::percentage(10),
ConnectionNode::new_server(
ConnectionParameters::default().idle_timeout(weeks(1000)),
boxed![
ReachState::new(State::Confirmed),
ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))
]
boxed![ReachState::new(State::Confirmed),],
boxed![ReachState::new(State::Closed(ConnectionError::Transport(
Error::IdleTimeout
)))]
),
Delay::new(weeks(8)..weeks(8)),
Drop::percentage(10),
Expand All @@ -94,32 +94,56 @@ simulate!(
simulate!(
connect_fixed_rtt,
[
ConnectionNode::default_client(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::new_client(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
Delay::new(DELAY..DELAY),
ConnectionNode::default_server(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::new_server(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
Delay::new(DELAY..DELAY),
],
);

simulate!(
connect_taildrop_jitter,
[
ConnectionNode::default_client(boxed![ReachState::new(State::Confirmed)]),
TailDrop::dsl_uplink(),
Delay::new(ZERO..JITTER),
ConnectionNode::default_server(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::new_client(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
TailDrop::dsl_downlink(),
Delay::new(ZERO..JITTER),
ConnectionNode::new_server(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
TailDrop::dsl_uplink(),
Delay::new(ZERO..JITTER),
],
);

simulate!(
connect_taildrop,
[
ConnectionNode::default_client(boxed![ReachState::new(State::Confirmed)]),
TailDrop::dsl_uplink(),
ConnectionNode::default_server(boxed![ReachState::new(State::Confirmed)]),
ConnectionNode::new_client(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
TailDrop::dsl_downlink(),
ConnectionNode::new_server(
ConnectionParameters::default(),
[],
boxed![ReachState::new(State::Confirmed)]
),
TailDrop::dsl_uplink(),
],
);

Expand All @@ -139,20 +163,20 @@ simulate!(
transfer_taildrop,
[
ConnectionNode::default_client(boxed![SendData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_uplink(),
ConnectionNode::default_server(boxed![ReceiveData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_downlink(),
ConnectionNode::default_server(boxed![ReceiveData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_uplink(),
],
);

simulate!(
transfer_taildrop_jitter,
[
ConnectionNode::default_client(boxed![SendData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_uplink(),
TailDrop::dsl_downlink(),
Delay::new(ZERO..JITTER),
ConnectionNode::default_server(boxed![ReceiveData::new(TRANSFER_AMOUNT)]),
TailDrop::dsl_downlink(),
TailDrop::dsl_uplink(),
Delay::new(ZERO..JITTER),
],
);
Expand Down
2 changes: 1 addition & 1 deletion test-fixture/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,4 +17,4 @@ neqo-transport = { path = "../neqo-transport" }
qlog = { git = "https://github.com/cloudflare/quiche", rev = "09ea4b244096a013071cfe2175bbf2945fb7f8d1" }

[features]
deny-warnings = []
deny-warnings = []
1 change: 1 addition & 0 deletions test-fixture/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ use neqo_transport::{
use qlog::{events::EventImportance, streamer::QlogStreamer};

pub mod assertions;
pub mod sim;

/// The path for the database used in tests.
pub const NSS_DB_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/db");
Expand Down
Loading

0 comments on commit 9260ba3

Please sign in to comment.