From 2760fcc7d97154b564a8654f470bc8e657d325dd Mon Sep 17 00:00:00 2001 From: xx01cyx Date: Mon, 29 Apr 2024 21:10:59 +0000 Subject: [PATCH] enable cache1 benchmark --- Cargo.lock | 374 +++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 1 + src/benchmark.rs | 34 +++-- src/utils.rs | 14 +- traces/trace1.csv | 2 +- 5 files changed, 383 insertions(+), 42 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3a2ef41..e316b83 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -198,7 +198,7 @@ dependencies = [ "arrow-data", "arrow-schema", "arrow-select", - "base64", + "base64 0.21.7", "chrono", "half", "lexical-core", @@ -336,6 +336,16 @@ dependencies = [ "regex-syntax 0.8.3", ] +[[package]] +name = "assert-json-diff" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47e4f2b81832e72834d7518d8487a0396a28cc408186a2e8854c0f98011faf12" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "async-stream" version = "0.3.5" @@ -422,6 +432,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "binascii" version = "0.1.4" @@ -550,8 +566,9 @@ dependencies = [ "env_logger", "istziio-client", "log", + "parpulse-client", "parquet", - "reqwest", + "reqwest 0.11.27", "rocket", "thiserror", "tokio", @@ -563,6 +580,16 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" +[[package]] +name = "colored" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" +dependencies = [ + "lazy_static", + "windows-sys 0.48.0", +] + [[package]] name = "const-random" version = "0.1.18" @@ -703,6 +730,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum-as-inner" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ffccbb6966c05b32ef8fbac435df276c4ae4d3dc55a8cd0eb9745e6c12f546a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "env_filter" version = "0.1.0" @@ -820,6 +859,7 @@ checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -842,6 +882,17 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.30" @@ -936,7 +987,26 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http", + "http 0.2.12", + "indexmap 2.2.6", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http 1.1.0", "indexmap 2.2.6", "slab", "tokio", @@ -967,6 +1037,12 @@ version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -993,6 +1069,17 @@ dependencies = [ "itoa", ] +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http-body" version = "0.4.6" @@ -1000,7 +1087,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http", + "http 0.2.12", + "pin-project-lite", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http 1.1.0", +] + +[[package]] +name = "http-body-util" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" +dependencies = [ + "bytes", + "futures-core", + "http 1.1.0", + "http-body 1.0.0", "pin-project-lite", ] @@ -1032,9 +1142,9 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", - "http", - "http-body", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", "httparse", "httpdate", "itoa", @@ -1046,6 +1156,26 @@ dependencies = [ "want", ] +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "httparse", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", + "want", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -1053,10 +1183,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" dependencies = [ "bytes", - "hyper", + "hyper 0.14.28", + "native-tls", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper 1.3.1", + "hyper-util", "native-tls", "tokio", "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "http 1.1.0", + "http-body 1.0.0", + "hyper 1.3.1", + "pin-project-lite", + "socket2", + "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -1155,7 +1321,7 @@ dependencies = [ "env_logger", "log", "parquet", - "reqwest", + "reqwest 0.11.27", "rocket", "thiserror", "tokio", @@ -1354,6 +1520,25 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "mockito" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2f6e023aa5bdf392aa06c78e4a4e6d498baab5138d0c993503350ebbc37bf1e" +dependencies = [ + "assert-json-diff", + "colored", + "futures-core", + "hyper 0.14.28", + "log", + "rand", + "regex", + "serde_json", + "serde_urlencoded", + "similar", + "tokio", +] + [[package]] name = "multer" version = "2.1.0" @@ -1363,7 +1548,7 @@ dependencies = [ "bytes", "encoding_rs", "futures-util", - "http", + "http 0.2.12", "httparse", "log", "memchr", @@ -1597,6 +1782,30 @@ dependencies = [ "windows-targets 0.48.5", ] +[[package]] +name = "parpulse-client" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "552f5301fee99a1a525b349a7e703022896709854bf7218377d7f20218a18070" +dependencies = [ + "anyhow", + "arrow", + "async-trait", + "enum-as-inner", + "env_logger", + "futures", + "hyper 1.3.1", + "istziio-client", + "lazy_static", + "log", + "mockito", + "parquet", + "reqwest 0.12.4", + "serde", + "tempfile", + "tokio", +] + [[package]] name = "parquet" version = "50.0.0" @@ -1611,11 +1820,12 @@ dependencies = [ "arrow-ipc", "arrow-schema", "arrow-select", - "base64", + "base64 0.21.7", "brotli", "bytes", "chrono", "flate2", + "futures", "half", "hashbrown 0.14.3", "lz4_flex", @@ -1625,6 +1835,7 @@ dependencies = [ "seq-macro", "snap", "thrift", + "tokio", "twox-hash", "zstd", ] @@ -1664,6 +1875,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -1834,16 +2065,60 @@ version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ - "base64", + "base64 0.21.7", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2 0.3.26", + "http 0.2.12", + "http-body 0.4.6", + "hyper 0.14.28", + "hyper-tls 0.5.0", + "ipnet", + "js-sys", + "log", + "mime", + "native-tls", + "once_cell", + "percent-encoding", + "pin-project-lite", + "rustls-pemfile 1.0.4", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "system-configuration", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", + "winreg 0.50.0", +] + +[[package]] +name = "reqwest" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" +dependencies = [ + "base64 0.22.0", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2", - "http", - "http-body", - "hyper", - "hyper-tls", + "h2 0.4.4", + "http 1.1.0", + "http-body 1.0.0", + "http-body-util", + "hyper 1.3.1", + "hyper-tls 0.6.0", + "hyper-util", "ipnet", "js-sys", "log", @@ -1852,7 +2127,7 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "serde", "serde_json", "serde_urlencoded", @@ -1867,7 +2142,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams", "web-sys", - "winreg", + "winreg 0.52.0", ] [[package]] @@ -1934,8 +2209,8 @@ dependencies = [ "cookie", "either", "futures", - "http", - "hyper", + "http 0.2.12", + "hyper 0.14.28", "indexmap 2.2.6", "log", "memchr", @@ -1986,9 +2261,25 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" dependencies = [ - "base64", + "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "beb461507cee2c2ff151784c52762cf4d9ff6a61f3e80968600ed24fa837fa54" + [[package]] name = "rustversion" version = "1.0.15" @@ -2127,6 +2418,12 @@ dependencies = [ "libc", ] +[[package]] +name = "similar" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" + [[package]] name = "slab" version = "0.4.9" @@ -2454,6 +2751,28 @@ dependencies = [ "winnow", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "pin-project", + "pin-project-lite", + "tokio", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c20c8dbed6283a09604c3e69b4b7eeb54e298b8a600d4d5ecb5ad39de609f1d0" + [[package]] name = "tower-service" version = "0.3.2" @@ -2466,6 +2785,7 @@ version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -2929,6 +3249,16 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "winreg" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a277a57398d4bfa075df44f501a17cfdf8542d224f0d36095a2adc7aee4ef0a5" +dependencies = [ + "cfg-if", + "windows-sys 0.48.0", +] + [[package]] name = "yansi" version = "1.0.1" diff --git a/Cargo.toml b/Cargo.toml index f0c623d..9bb0722 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,3 +19,4 @@ rocket = { version = "0.5.0", features = ["json"] } reqwest = { version = "0.11", features = ["stream", "json"] } csv = "1.3.0" istziio-client = "0.1.6" +parpulse-client = "0.1" diff --git a/src/benchmark.rs b/src/benchmark.rs index 565c957..799b5cd 100644 --- a/src/benchmark.rs +++ b/src/benchmark.rs @@ -1,27 +1,25 @@ -use istziio_client::client_api::{StorageRequest, StorageClient}; +use csv; +use istziio_client::client_api::{StorageClient, StorageRequest}; +use std::collections::VecDeque; +use std::error::Error; use std::path::PathBuf; +use std::thread::sleep; use std::time::Instant; -use tokio::sync::mpsc; use std::time::{Duration, SystemTime}; -use std::thread::sleep; -use std::collections::VecDeque; -use std::error::Error; -use csv; +use tokio::sync::mpsc; // This scans the bench_files dir to figure out which test files are present, // then builds a map of TableId -> filename to init storage client(only when catalog is not available) // and also generates workload based on table ids. Finally it runs the workload - - pub struct TraceEntry { pub timestamp: u64, - pub request: StorageRequest + pub request: StorageRequest, } pub enum ClientType { Client1(), - Client2() + Client2(), } pub fn parse_trace(trace_path: PathBuf) -> Result, Box> { @@ -31,17 +29,25 @@ pub fn parse_trace(trace_path: PathBuf) -> Result, Box, so we check the // error here. let record = result?; - trace.push_back(TraceEntry{timestamp: record.get(0).unwrap().parse().unwrap(), request: StorageRequest::Table(record.get(1).unwrap().parse().unwrap())}); + trace.push_back(TraceEntry { + timestamp: record.get(0).unwrap().parse().unwrap(), + request: StorageRequest::Table(record.get(1).unwrap().parse().unwrap()), + }); } Ok(trace) } -pub async fn run_trace(mut trace: VecDeque, client_builder: &dyn Fn() -> Box) { +pub async fn run_trace( + mut trace: VecDeque, + client_builder: &dyn Fn() -> Box, +) { let start_time = SystemTime::now(); let request_num = trace.len(); let (tx, mut rx) = mpsc::channel(32); while !trace.is_empty() { let next_entry = trace.pop_front().unwrap(); - if let Some(diff) = Duration::from_millis(next_entry.timestamp).checked_sub(start_time.elapsed().unwrap()) { + if let Some(diff) = + Duration::from_millis(next_entry.timestamp).checked_sub(start_time.elapsed().unwrap()) + { sleep(diff); } println!("next trace: {}", next_entry.timestamp); @@ -56,7 +62,7 @@ pub async fn run_trace(mut trace: VecDeque, client_builder: &dyn Fn( let client_start = Instant::now(); let req = next_entry.request; - let res = client.request_data_sync(req.clone()).await; + let res = client.request_data(req.clone()).await; if let Err(e) = res { println!("Error: {}", e); } diff --git a/src/utils.rs b/src/utils.rs index cdbaddd..a6a497a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,19 +1,23 @@ use istziio_client::client_api::{StorageClient, TableId}; -use istziio_client::storage_client::StorageClientImpl; +use istziio_client::storage_client::StorageClientImpl as IstziioStorageClientImpl; +use parpulse_client::client::StorageClientImpl as ParpulseStorageClientImpl; use std::collections::HashMap; pub fn setup_client_1() -> Box { - let server_url = std::env::var("SERVER_URL").unwrap_or(String::from("http://127.0.0.1:26379")); + let server_url = std::env::var("SERVER_URL").unwrap_or(String::from("http://127.0.0.1:3030")); println!("server url: {}", &server_url); - let map = create_table_file_map().unwrap(); - Box::new(StorageClientImpl::new_for_test(1, map.clone(), &server_url)) + Box::new(ParpulseStorageClientImpl::new(&server_url, "http://127.0.0.1:3031").unwrap()) } pub fn setup_client_2() -> Box { let server_url = std::env::var("SERVER_URL").unwrap_or(String::from("http://127.0.0.1:26379")); println!("server url: {}", &server_url); let map = create_table_file_map().unwrap(); - Box::new(StorageClientImpl::new_for_test(1, map.clone(), &server_url)) + Box::new(IstziioStorageClientImpl::new_for_test( + 1, + map.clone(), + &server_url, + )) } pub fn create_table_file_map() -> Result, std::io::Error> { diff --git a/traces/trace1.csv b/traces/trace1.csv index e63d403..70e48f9 100644 --- a/traces/trace1.csv +++ b/traces/trace1.csv @@ -1,5 +1,5 @@ timestamp,file -300,0 +300,1 1000,1 1000,1 3000,2