From 2fe844ac26afadbdee1b35f3fb7cd8e091929385 Mon Sep 17 00:00:00 2001 From: Gaius Date: Mon, 26 Aug 2024 21:22:22 +0800 Subject: [PATCH] feat: set prefix_extractor for column families (#693) Signed-off-by: Gaius --- Cargo.lock | 16 ++++++++-------- Cargo.toml | 16 ++++++++-------- dragonfly-client-storage/src/lib.rs | 3 --- .../src/storage_engine/rocksdb.rs | 17 +++++++++++++++-- 4 files changed, 31 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b8d72fa8..f7eb4a50 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -861,7 +861,7 @@ dependencies = [ [[package]] name = "dragonfly-client" -version = "0.1.102" +version = "0.1.103" dependencies = [ "anyhow", "blake3", @@ -931,7 +931,7 @@ dependencies = [ [[package]] name = "dragonfly-client-backend" -version = "0.1.102" +version = "0.1.103" dependencies = [ "dragonfly-api", "dragonfly-client-core", @@ -953,7 +953,7 @@ dependencies = [ [[package]] name = "dragonfly-client-config" -version = "0.1.102" +version = "0.1.103" dependencies = [ "bytesize", "bytesize-serde", @@ -974,7 +974,7 @@ dependencies = [ [[package]] name = "dragonfly-client-core" -version = "0.1.102" +version = "0.1.103" dependencies = [ "hyper 1.4.1", "hyper-util", @@ -989,7 +989,7 @@ dependencies = [ [[package]] name = "dragonfly-client-init" -version = "0.1.102" +version = "0.1.103" dependencies = [ "anyhow", "clap", @@ -1005,7 +1005,7 @@ dependencies = [ [[package]] name = "dragonfly-client-storage" -version = "0.1.102" +version = "0.1.103" dependencies = [ "base16ct", "bincode", @@ -1030,7 +1030,7 @@ dependencies = [ [[package]] name = "dragonfly-client-util" -version = "0.1.102" +version = "0.1.103" dependencies = [ "base16ct", "blake3", @@ -1384,7 +1384,7 @@ dependencies = [ [[package]] name = "hdfs" -version = "0.1.102" +version = "0.1.103" dependencies = [ "dragonfly-client-backend", "dragonfly-client-core", diff --git a/Cargo.toml b/Cargo.toml index 32ce219f..fcbe4de2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -12,7 +12,7 @@ members = [ ] [workspace.package] -version = "0.1.102" +version = "0.1.103" authors = ["The Dragonfly Developers"] homepage = "https://d7y.io/" repository = "https://github.com/dragonflyoss/client.git" @@ -22,13 +22,13 @@ readme = "README.md" edition = "2021" [workspace.dependencies] -dragonfly-client = { path = "dragonfly-client", version = "0.1.102" } -dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.102" } -dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.102" } -dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.102" } -dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.102" } -dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.102" } -dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.102" } +dragonfly-client = { path = "dragonfly-client", version = "0.1.103" } +dragonfly-client-core = { path = "dragonfly-client-core", version = "0.1.103" } +dragonfly-client-config = { path = "dragonfly-client-config", version = "0.1.103" } +dragonfly-client-storage = { path = "dragonfly-client-storage", version = "0.1.103" } +dragonfly-client-backend = { path = "dragonfly-client-backend", version = "0.1.103" } +dragonfly-client-util = { path = "dragonfly-client-util", version = "0.1.103" } +dragonfly-client-init = { path = "dragonfly-client-init", version = "0.1.103" } thiserror = "1.0" dragonfly-api = "=2.0.148" reqwest = { version = "0.12.4", features = ["stream", "native-tls", "default-tls", "rustls-tls"] } diff --git a/dragonfly-client-storage/src/lib.rs b/dragonfly-client-storage/src/lib.rs index 38b611f3..5cb8f244 100644 --- a/dragonfly-client-storage/src/lib.rs +++ b/dragonfly-client-storage/src/lib.rs @@ -126,17 +126,14 @@ impl Storage { // delete_task deletes the task metadatas, task content and piece metadatas. pub async fn delete_task(&self, id: &str) { - info!("start to delete task: {}", id); self.metadata .delete_task(id) .unwrap_or_else(|err| error!("delete task metadata failed: {}", err)); - info!("start to delete pieces of task: {}", id); self.metadata.delete_pieces(id).unwrap_or_else(|err| { error!("delete piece metadatas failed: {}", err); }); - info!("start to delete content of task: {}", id); self.content.delete_task(id).await.unwrap_or_else(|err| { error!("delete task content failed: {}", err); }); diff --git a/dragonfly-client-storage/src/storage_engine/rocksdb.rs b/dragonfly-client-storage/src/storage_engine/rocksdb.rs index 29709052..0d3a6edf 100644 --- a/dragonfly-client-storage/src/storage_engine/rocksdb.rs +++ b/dragonfly-client-storage/src/storage_engine/rocksdb.rs @@ -73,7 +73,6 @@ impl RocksdbStorageEngine { let mut options = rocksdb::Options::default(); options.create_if_missing(true); options.create_missing_column_families(true); - options.optimize_level_style_compaction(Self::DEFAULT_MEMTABLE_MEMORY_BUDGET); options.increase_parallelism(num_cpus::get() as i32); options.set_compression_type(rocksdb::DBCompressionType::Lz4); options.set_max_background_jobs(std::cmp::max( @@ -93,6 +92,19 @@ impl RocksdbStorageEngine { block_options.set_block_size(Self::DEFAULT_BLOCK_SIZE); options.set_block_based_table_factory(&block_options); + // Initialize column family options. + let mut cf_options = rocksdb::Options::default(); + cf_options.set_prefix_extractor(rocksdb::SliceTransform::create_fixed_prefix(64)); + cf_options.set_memtable_prefix_bloom_ratio(0.2); + cf_options.optimize_level_style_compaction(Self::DEFAULT_MEMTABLE_MEMORY_BUDGET); + + // Initialize column families. + let cfs = cf_names + .iter() + .map(|name| (name.to_string(), cf_options.clone())) + .collect::>(); + + // Initialize rocksdb directory. let dir = dir.join(Self::DEFAULT_DIR_NAME); // If the storage is not kept, remove the db. @@ -103,7 +115,8 @@ impl RocksdbStorageEngine { } // Open rocksdb. - let db = rocksdb::DB::open_cf(&options, &dir, cf_names).or_err(ErrorType::StorageError)?; + let db = + rocksdb::DB::open_cf_with_opts(&options, &dir, cfs).or_err(ErrorType::StorageError)?; info!("metadata initialized directory: {:?}", dir); Ok(Self { inner: db })