update

Author: XiangpengHao

benchmarks/Cargo.toml

@@ -36,6 +36,9 @@ snmalloc = ["snmalloc-rs"]
 
 [dependencies]
 arrow = { workspace = true }
+arrow-flight = { workspace = true }
+arrow-schema = { workspace = true }
+dashmap = { workspace = true }
 datafusion = { workspace = true, default-features = true }
 datafusion-common = { workspace = true, default-features = true }
 env_logger = { workspace = true }
@@ -45,14 +48,18 @@ mimalloc = { version = "0.1", optional = true, default-features = false }
 num_cpus = { workspace = true }
 object_store = { workspace = true, features = ["aws"] }
 parquet = { workspace = true, default-features = true }
+pprof = { version = "0.13", features = ["flamegraph"] }
+prost = { workspace = true }
 serde = { version = "1.0.136", features = ["derive"] }
 serde_json = { workspace = true }
 snmalloc-rs = { version = "0.3", optional = true }
 structopt = { version = "0.3", default-features = false }
+sysinfo = "0.32.0"
 test-utils = { path = "../test-utils/", version = "0.1.0" }
 tokio = { workspace = true, features = ["rt-multi-thread", "parking_lot"] }
+tonic = "0.12.1"
 url.workspace = true
-pprof = { version = "0.13", features = ["flamegraph"] }
+uuid = "1.7"
 
 [dev-dependencies]
 datafusion-proto = { workspace = true }

benchmarks/src/clickbench.rs

@@ -23,7 +23,6 @@ use std::sync::Arc;
 
 use arrow::ipc::reader::FileReader;
 use arrow::util::pretty;
-use datafusion::datasource::physical_plan::parquet::Parquet7FileReaderFactory;
 use datafusion::execution::cache::cache_unit::Cache37;
 use datafusion::execution::object_store::ObjectStoreUrl;
 use datafusion::physical_plan::collect;
@@ -324,6 +323,7 @@ impl RunOpt {
         };
 
         let mut options: ParquetReadOptions<'_> = Default::default();
+        use datafusion::datasource::physical_plan::parquet::Parquet7FileReaderFactory;
         options.reader = Some(Arc::new(Parquet7FileReaderFactory::new(object_store)));
 
         ctx.register_parquet("hits", &path, options)

benchmarks/src/util/run.rs

@@ -163,6 +163,10 @@ impl BenchmarkRun {
 
     /// Stringify data into formatted json
     pub fn to_json(&self) -> String {
+        let pid = sysinfo::get_current_pid().unwrap();
+        let sys = sysinfo::System::new_all();
+        let process = sys.process(pid).unwrap();
+
         let mut output = HashMap::<&str, Value>::new();
         output.insert("context", serde_json::to_value(&self.context).unwrap());
         output.insert("queries", serde_json::to_value(&self.queries).unwrap());
@@ -185,6 +189,11 @@ impl BenchmarkRun {
             )
             .unwrap(),
         );
+        output.insert("memory", serde_json::to_value(&process.memory()).unwrap());
+        output.insert(
+            "virtual_memory",
+            serde_json::to_value(&process.virtual_memory()).unwrap(),
+        );
         serde_json::to_string_pretty(&output).unwrap()
     }