diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index 65a25bcf195..7d6e71e98a7 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -52,6 +52,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[ ]; const AZBLOB_BATCH_LIMIT: usize = 256; + /// Azure Storage Blob services support. #[derive(Default, Deserialize, Clone)] pub struct AzblobConfig { @@ -124,7 +125,7 @@ pub struct AzblobBuilder { impl Debug for AzblobBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzblobBuilder"); ds.field("config", &self.config); diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index e7243ab45aa..7e128524cee 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -26,13 +26,13 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; +use serde::Deserialize; use super::core::AzdlsCore; use super::error::parse_error; use super::lister::AzdlsLister; -use super::writer::AzdlsWriter; +use super::writer::{AzdlsWriter, AzdlsWriters}; use crate::raw::*; -use crate::services::azdls::writer::AzdlsWriters; use crate::*; /// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax. @@ -46,20 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[ ]; /// Azure Data Lake Storage Gen2 Support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct AzdlsBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct AzdlsConfig { root: Option, filesystem: String, endpoint: Option, account_name: Option, account_key: Option, - http_client: Option, } -impl Debug for AzdlsBuilder { +impl Debug for AzdlsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzdlsConfig"); ds.field("root", &self.root); ds.field("filesystem", &self.filesystem); @@ -76,13 +74,31 @@ impl Debug for AzdlsBuilder { } } +/// Azure Data Lake Storage Gen2 Support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct AzdlsBuilder { + config: AzdlsConfig, + http_client: Option, +} + +impl Debug for AzdlsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzdlsBuilder"); + + ds.field("config", &self.config); + + ds.finish() + } +} + impl AzdlsBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -90,7 +106,7 @@ impl AzdlsBuilder { /// Set filesystem name of this backend. pub fn filesystem(&mut self, filesystem: &str) -> &mut Self { - self.filesystem = filesystem.to_string(); + self.config.filesystem = filesystem.to_string(); self } @@ -104,7 +120,7 @@ impl AzdlsBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -116,7 +132,7 @@ impl AzdlsBuilder { /// - If not, we will try to load it from environment. pub fn account_name(&mut self, account_name: &str) -> &mut Self { if !account_name.is_empty() { - self.account_name = Some(account_name.to_string()); + self.config.account_name = Some(account_name.to_string()); } self @@ -128,7 +144,7 @@ impl AzdlsBuilder { /// - If not, we will try to load it from environment. pub fn account_key(&mut self, account_key: &str) -> &mut Self { if !account_key.is_empty() { - self.account_key = Some(account_key.to_string()); + self.config.account_key = Some(account_key.to_string()); } self @@ -153,19 +169,19 @@ impl Builder for AzdlsBuilder { fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); // Handle endpoint, region and container name. - let filesystem = match self.filesystem.is_empty() { - false => Ok(&self.filesystem), + let filesystem = match self.config.filesystem.is_empty() { + false => Ok(&self.config.filesystem), true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty") .with_operation("Builder::build") .with_context("service", Scheme::Azdls)), }?; debug!("backend use filesystem {}", &filesystem); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -184,10 +200,11 @@ impl Builder for AzdlsBuilder { let config_loader = AzureStorageConfig { account_name: self + .config .account_name .clone() .or_else(|| infer_storage_name_from_endpoint(endpoint.as_str())), - account_key: self.account_key.clone(), + account_key: self.config.account_key.clone(), sas_token: None, ..Default::default() }; @@ -198,7 +215,7 @@ impl Builder for AzdlsBuilder { debug!("backend build finished: {:?}", &self); Ok(AzdlsBackend { core: Arc::new(AzdlsCore { - filesystem: self.filesystem.clone(), + filesystem: self.config.filesystem.clone(), root, endpoint, client, @@ -209,15 +226,13 @@ impl Builder for AzdlsBuilder { } fn from_map(map: HashMap) -> Self { - let mut builder = AzdlsBuilder::default(); + let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - map.get("root").map(|v| builder.root(v)); - map.get("filesystem").map(|v| builder.filesystem(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("account_name").map(|v| builder.account_name(v)); - map.get("account_key").map(|v| builder.account_key(v)); - - builder + AzdlsBuilder { + config, + http_client: None, + } } } @@ -467,7 +482,7 @@ mod tests { assert_eq!(azdls.core.filesystem, "filesystem".to_string()); assert_eq!( - azdls_builder.account_key.unwrap(), + azdls_builder.config.account_key.unwrap(), "account-key".to_string() ); } @@ -488,6 +503,6 @@ mod tests { assert_eq!(azdls.core.filesystem, "filesystem".to_string()); - assert_eq!(azdls_builder.account_key, None); + assert_eq!(azdls_builder.config.account_key, None); } } diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index 84f02c88bf2..d11d7a700b8 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -26,6 +26,7 @@ use log::debug; use reqsign::AzureStorageConfig; use reqsign::AzureStorageLoader; use reqsign::AzureStorageSigner; +use serde::Deserialize; use super::core::AzfileCore; use super::error::parse_error; @@ -39,31 +40,51 @@ use crate::*; const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net"; /// Azure File services support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct AzfileBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct AzfileConfig { root: Option, endpoint: Option, - account_name: Option, share_name: String, + account_name: Option, account_key: Option, sas_token: Option, - http_client: Option, } -impl Debug for AzfileBuilder { +impl Debug for AzfileConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("AzfileConfig"); ds.field("root", &self.root); - ds.field("endpoint", &self.endpoint); ds.field("share_name", &self.share_name); + ds.field("endpoint", &self.endpoint); + if self.account_name.is_some() { ds.field("account_name", &""); } if self.account_key.is_some() { ds.field("account_key", &""); } + if self.sas_token.is_some() { + ds.field("sas_token", &""); + } + + ds.finish() + } +} + +/// Azure File services support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct AzfileBuilder { + config: AzfileConfig, + http_client: Option, +} + +impl Debug for AzfileBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("AzfileBuilder"); + + ds.field("config", &self.config); ds.finish() } @@ -75,7 +96,7 @@ impl AzfileBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -85,7 +106,7 @@ impl AzfileBuilder { pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { // Trim trailing `/` so that we can accept `http://127.0.0.1:9000/` - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -97,7 +118,7 @@ impl AzfileBuilder { /// - If not, we will try to load it from environment. pub fn account_name(&mut self, account_name: &str) -> &mut Self { if !account_name.is_empty() { - self.account_name = Some(account_name.to_string()); + self.config.account_name = Some(account_name.to_string()); } self @@ -109,7 +130,7 @@ impl AzfileBuilder { /// - If not, we will try to load it from environment. pub fn account_key(&mut self, account_key: &str) -> &mut Self { if !account_key.is_empty() { - self.account_key = Some(account_key.to_string()); + self.config.account_key = Some(account_key.to_string()); } self @@ -121,7 +142,7 @@ impl AzfileBuilder { /// You can find more about from: pub fn share_name(&mut self, share_name: &str) -> &mut Self { if !share_name.is_empty() { - self.share_name = share_name.to_string(); + self.config.share_name = share_name.to_string(); } self @@ -144,24 +165,22 @@ impl Builder for AzfileBuilder { type Accessor = AzfileBackend; fn from_map(map: HashMap) -> Self { - let mut builder = AzfileBuilder::default(); - - map.get("root").map(|v| builder.root(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("account_name").map(|v| builder.account_name(v)); - map.get("account_key").map(|v| builder.account_key(v)); - map.get("share_name").map(|v| builder.share_name(v)); + let config = AzfileConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + AzfileBuilder { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -179,6 +198,7 @@ impl Builder for AzfileBuilder { }; let account_name_option = self + .config .account_name .clone() .or_else(|| infer_account_name_from_endpoint(endpoint.as_str())); @@ -194,8 +214,8 @@ impl Builder for AzfileBuilder { let config_loader = AzureStorageConfig { account_name: Some(account_name), - account_key: self.account_key.clone(), - sas_token: self.sas_token.clone(), + account_key: self.config.account_key.clone(), + sas_token: self.config.sas_token.clone(), ..Default::default() }; @@ -211,7 +231,7 @@ impl Builder for AzfileBuilder { loader: cred_loader, client, signer, - share_name: self.share_name.clone(), + share_name: self.config.share_name.clone(), }), }) } @@ -435,7 +455,7 @@ mod tests { ); assert_eq!( - azfile_builder.account_key.unwrap(), + azfile_builder.config.account_key.unwrap(), "account-key".to_string() ); } diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index 4fc47f15e94..153a7d853b9 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -22,26 +22,34 @@ use std::str; use async_trait::async_trait; use cacache; +use serde::Deserialize; use crate::raw::adapters::kv; +use crate::raw::ConfigDeserializer; use crate::Builder; use crate::Error; use crate::ErrorKind; use crate::Scheme; use crate::*; +/// cacache service support. +#[derive(Default, Deserialize, Clone)] +pub struct CacacheConfig { + /// That path to the cacache data directory. + datadir: Option, +} + /// cacache service support. #[doc = include_str!("docs.md")] #[derive(Default)] pub struct CacacheBuilder { - /// That path to the cacache data directory. - datadir: Option, + config: CacacheConfig, } impl CacacheBuilder { /// Set the path to the cacache data directory. Will create if not exists. pub fn datadir(&mut self, path: &str) -> &mut Self { - self.datadir = Some(path.into()); + self.config.datadir = Some(path.into()); self } } @@ -51,15 +59,14 @@ impl Builder for CacacheBuilder { type Accessor = CacacheBackend; fn from_map(map: HashMap) -> Self { - let mut builder = CacacheBuilder::default(); - - map.get("datadir").map(|v| builder.datadir(v)); + let config = CacacheConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } fn build(&mut self) -> Result { - let datadir_path = self.datadir.take().ok_or_else(|| { + let datadir_path = self.config.datadir.take().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Cacache) })?; diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index e05bb3931e2..3050582c842 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -31,9 +31,9 @@ use crate::raw::*; use crate::ErrorKind; use crate::*; -#[doc = include_str!("docs.md")] -#[derive(Default)] -pub struct CloudflareKvBuilder { +/// Cloudflare Kv Service Support. +#[derive(Default, Deserialize, Clone)] +pub struct CloudflareKvConfig { /// The token used to authenticate with CloudFlare. token: Option, /// The account ID used to authenticate with CloudFlare. Used as URI path parameter. @@ -41,18 +41,39 @@ pub struct CloudflareKvBuilder { /// The namespace ID. Used as URI path parameter. namespace_id: Option, - /// The HTTP client used to communicate with CloudFlare. - http_client: Option, /// Root within this backend. root: Option, } +impl Debug for CloudflareKvConfig { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("CloudflareKvConfig"); + + ds.field("root", &self.root); + ds.field("account_id", &self.account_id); + ds.field("namespace_id", &self.namespace_id); + + if self.token.is_some() { + ds.field("token", &""); + } + + ds.finish() + } +} + +#[doc = include_str!("docs.md")] +#[derive(Default)] +pub struct CloudflareKvBuilder { + config: CloudflareKvConfig, + + /// The HTTP client used to communicate with CloudFlare. + http_client: Option, +} + impl Debug for CloudflareKvBuilder { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { f.debug_struct("CloudFlareKvBuilder") - .field("account_id", &self.account_id) - .field("namespace_id", &self.namespace_id) - .field("root", &self.root) + .field("config", &self.config) .finish() } } @@ -61,7 +82,7 @@ impl CloudflareKvBuilder { /// Set the token used to authenticate with CloudFlare. pub fn token(&mut self, token: &str) -> &mut Self { if !token.is_empty() { - self.token = Some(token.to_string()) + self.config.token = Some(token.to_string()) } self } @@ -69,7 +90,7 @@ impl CloudflareKvBuilder { /// Set the account ID used to authenticate with CloudFlare. pub fn account_id(&mut self, account_id: &str) -> &mut Self { if !account_id.is_empty() { - self.account_id = Some(account_id.to_string()) + self.config.account_id = Some(account_id.to_string()) } self } @@ -77,7 +98,7 @@ impl CloudflareKvBuilder { /// Set the namespace ID. pub fn namespace_id(&mut self, namespace_id: &str) -> &mut Self { if !namespace_id.is_empty() { - self.namespace_id = Some(namespace_id.to_string()) + self.config.namespace_id = Some(namespace_id.to_string()) } self } @@ -85,7 +106,7 @@ impl CloudflareKvBuilder { /// Set the root within this backend. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self } @@ -97,28 +118,29 @@ impl Builder for CloudflareKvBuilder { type Accessor = CloudflareKvBackend; fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - map.get("token").map(|v| builder.token(v)); - map.get("account_id").map(|v| builder.account_id(v)); - map.get("namespace_id").map(|v| builder.namespace_id(v)); - map.get("root").map(|v| builder.root(v)); - builder + let config = CloudflareKvConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); + + Self { + config, + http_client: None, + } } fn build(&mut self) -> Result { - let authorization = match &self.token { + let authorization = match &self.config.token { Some(token) => format_authorization_by_bearer(token)?, None => return Err(Error::new(ErrorKind::ConfigInvalid, "token is required")), }; - let Some(account_id) = self.account_id.clone() else { + let Some(account_id) = self.config.account_id.clone() else { return Err(Error::new( ErrorKind::ConfigInvalid, "account_id is required", )); }; - let Some(namespace_id) = self.namespace_id.clone() else { + let Some(namespace_id) = self.config.namespace_id.clone() else { return Err(Error::new( ErrorKind::ConfigInvalid, "namespace_id is required", @@ -135,7 +157,8 @@ impl Builder for CloudflareKvBuilder { }; let root = normalize_root( - self.root + self.config + .root .clone() .unwrap_or_else(|| "/".to_string()) .as_str(), @@ -271,7 +294,7 @@ impl kv::Adapter for Adapter { let body = resp.into_body().bytes().await?; let response: CfKvScanResponse = serde_json::from_slice(&body).map_err(|e| { Error::new( - crate::ErrorKind::Unexpected, + ErrorKind::Unexpected, &format!("failed to parse error response: {}", e), ) })?; diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index decb37bd1da..48f0ca444d9 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -26,6 +26,7 @@ use log::debug; use reqsign::TencentCosConfig; use reqsign::TencentCosCredentialLoader; use reqsign::TencentCosSigner; +use serde::Deserialize; use super::core::*; use super::error::parse_error; @@ -36,27 +37,41 @@ use crate::services::cos::writer::CosWriters; use crate::*; /// Tencent-Cloud COS services support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct CosBuilder { +#[derive(Default, Deserialize, Clone)] +#[serde(default)] +pub struct CosConfig { root: Option, endpoint: Option, secret_id: Option, secret_key: Option, bucket: Option, - http_client: Option, - disable_config_load: bool, } -impl Debug for CosBuilder { +impl Debug for CosConfig { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("Builder") + f.debug_struct("CosConfig") .field("root", &self.root) .field("endpoint", &self.endpoint) .field("secret_id", &"") .field("secret_key", &"") .field("bucket", &self.bucket) + .finish_non_exhaustive() + } +} + +/// Tencent-Cloud COS services support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct CosBuilder { + config: CosConfig, + http_client: Option, +} + +impl Debug for CosBuilder { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CosBuilder") + .field("config", &self.config) .finish() } } @@ -67,7 +82,7 @@ impl CosBuilder { /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -82,7 +97,7 @@ impl CosBuilder { /// - `https://cos.ap-singapore.myqcloud.com` pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { if !endpoint.is_empty() { - self.endpoint = Some(endpoint.trim_end_matches('/').to_string()); + self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string()); } self @@ -93,7 +108,7 @@ impl CosBuilder { /// - If not, we will try to load it from environment. pub fn secret_id(&mut self, secret_id: &str) -> &mut Self { if !secret_id.is_empty() { - self.secret_id = Some(secret_id.to_string()); + self.config.secret_id = Some(secret_id.to_string()); } self @@ -104,7 +119,7 @@ impl CosBuilder { /// - If not, we will try to load it from environment. pub fn secret_key(&mut self, secret_key: &str) -> &mut Self { if !secret_key.is_empty() { - self.secret_key = Some(secret_key.to_string()); + self.config.secret_key = Some(secret_key.to_string()); } self @@ -114,7 +129,7 @@ impl CosBuilder { /// The param is required. pub fn bucket(&mut self, bucket: &str) -> &mut Self { if !bucket.is_empty() { - self.bucket = Some(bucket.to_string()); + self.config.bucket = Some(bucket.to_string()); } self @@ -127,7 +142,7 @@ impl CosBuilder { /// /// - envs like `TENCENTCLOUD_SECRET_ID` pub fn disable_config_load(&mut self) -> &mut Self { - self.disable_config_load = true; + self.config.disable_config_load = true; self } @@ -148,24 +163,22 @@ impl Builder for CosBuilder { type Accessor = CosBackend; fn from_map(map: HashMap) -> Self { - let mut builder = CosBuilder::default(); + let config = CosConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - map.get("root").map(|v| builder.root(v)); - map.get("bucket").map(|v| builder.bucket(v)); - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("secret_id").map(|v| builder.secret_id(v)); - map.get("secret_key").map(|v| builder.secret_key(v)); - - builder + Self { + config, + http_client: None, + } } fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let bucket = match &self.bucket { + let bucket = match &self.config.bucket { Some(bucket) => Ok(bucket.to_string()), None => Err( Error::new(ErrorKind::ConfigInvalid, "The bucket is misconfigured") @@ -174,7 +187,7 @@ impl Builder for CosBuilder { }?; debug!("backend use bucket {}", &bucket); - let uri = match &self.endpoint { + let uri = match &self.config.endpoint { Some(endpoint) => endpoint.parse::().map_err(|err| { Error::new(ErrorKind::ConfigInvalid, "endpoint is invalid") .with_context("service", Scheme::Cos) @@ -204,14 +217,14 @@ impl Builder for CosBuilder { }; let mut cfg = TencentCosConfig::default(); - if !self.disable_config_load { + if !self.config.disable_config_load { cfg = cfg.from_env(); } - if let Some(v) = self.secret_id.take() { + if let Some(v) = self.config.secret_id.take() { cfg.secret_id = Some(v); } - if let Some(v) = self.secret_key.take() { + if let Some(v) = self.config.secret_key.take() { cfg.secret_key = Some(v); } diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index f626bcae28f..ae6a22952bb 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -21,21 +21,29 @@ use std::fmt::Formatter; use async_trait::async_trait; use dashmap::DashMap; +use serde::Deserialize; use crate::raw::adapters::typed_kv; +use crate::raw::ConfigDeserializer; use crate::*; +/// [dashmap](https://github.com/xacrimon/dashmap) backend support. +#[derive(Default, Deserialize, Clone, Debug)] +pub struct DashmapConfig { + root: Option, +} + /// [dashmap](https://github.com/xacrimon/dashmap) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] pub struct DashmapBuilder { - root: Option, + config: DashmapConfig, } impl DashmapBuilder { /// Set the root for dashmap. pub fn root(&mut self, path: &str) -> &mut Self { - self.root = Some(path.into()); + self.config.root = Some(path.into()); self } } @@ -45,18 +53,17 @@ impl Builder for DashmapBuilder { type Accessor = DashmapBackend; fn from_map(map: HashMap) -> Self { - let mut builder = Self::default(); - - map.get("root").map(|v| builder.root(v)); + let config = DashmapConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } fn build(&mut self) -> Result { Ok(DashmapBackend::new(Adapter { inner: DashMap::default(), }) - .with_root(self.root.as_deref().unwrap_or_default())) + .with_root(self.config.root.as_deref().unwrap_or_default())) } } diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index 260a5abec36..1028c8b9dea 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -34,17 +34,16 @@ use crate::raw::*; use crate::*; /// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. -#[doc = include_str!("docs.md")] -#[derive(Default, Clone)] -pub struct DbfsBuilder { +#[derive(Default, Deserialize, Clone)] +pub struct DbfsConfig { root: Option, endpoint: Option, token: Option, } -impl Debug for DbfsBuilder { +impl Debug for DbfsConfig { fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut ds = f.debug_struct("Builder"); + let mut ds = f.debug_struct("DbfsConfig"); ds.field("root", &self.root); ds.field("endpoint", &self.endpoint); @@ -57,13 +56,30 @@ impl Debug for DbfsBuilder { } } +/// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. +#[doc = include_str!("docs.md")] +#[derive(Default, Clone)] +pub struct DbfsBuilder { + config: DbfsConfig, +} + +impl Debug for DbfsBuilder { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut ds = f.debug_struct("DbfsBuilder"); + + ds.field("config", &self.config); + + ds.finish() + } +} + impl DbfsBuilder { /// Set root of this backend. /// /// All operations will happen under this root. pub fn root(&mut self, root: &str) -> &mut Self { if !root.is_empty() { - self.root = Some(root.to_string()) + self.config.root = Some(root.to_string()) } self @@ -76,7 +92,7 @@ impl DbfsBuilder { /// - Azure: `https://adb-1234567890123456.78.azuredatabricks.net` /// - Aws: `https://dbc-123a5678-90bc.cloud.databricks.com` pub fn endpoint(&mut self, endpoint: &str) -> &mut Self { - self.endpoint = if endpoint.is_empty() { + self.config.endpoint = if endpoint.is_empty() { None } else { Some(endpoint.trim_end_matches('/').to_string()) @@ -87,7 +103,7 @@ impl DbfsBuilder { /// Set the token of this backend. pub fn token(&mut self, token: &str) -> &mut Self { if !token.is_empty() { - self.token = Some(token.to_string()); + self.config.token = Some(token.to_string()); } self } @@ -98,22 +114,20 @@ impl Builder for DbfsBuilder { type Accessor = DbfsBackend; fn from_map(map: HashMap) -> Self { - let mut builder = DbfsBuilder::default(); - - map.get("endpoint").map(|v| builder.endpoint(v)); - map.get("token").map(|v| builder.token(v)); + let config = DbfsConfig::deserialize(ConfigDeserializer::new(map)) + .expect("config deserialize must succeed"); - builder + Self { config } } /// Build a DbfsBackend. fn build(&mut self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.take().unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match &self.endpoint { + let endpoint = match &self.config.endpoint { Some(endpoint) => Ok(endpoint.clone()), None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_operation("Builder::build") @@ -121,7 +135,7 @@ impl Builder for DbfsBuilder { }?; debug!("backend use endpoint: {}", &endpoint); - let token = match self.token.take() { + let token = match self.config.token.take() { Some(token) => token, None => { return Err(Error::new(