diff --git a/core/src/docs/internals/accessor.rs b/core/src/docs/internals/accessor.rs index 3158cfdbfa4..8ceb71e3352 100644 --- a/core/src/docs/internals/accessor.rs +++ b/core/src/docs/internals/accessor.rs @@ -256,10 +256,10 @@ //! type Config = DuckConfig; //! //! fn from_config(config: Self::Config) -> Self { -//! DuckBuilder { config } +//! DuckBuilder { config: self } //! } //! -//! fn build(&mut self) -> Result { +//! fn build(self) -> Result { //! debug!("backend build started: {:?}", &self); //! //! let root = normalize_root(&self.config.root.clone().unwrap_or_default()); diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs index d0b13d2718b..7cbb88ca8b8 100644 --- a/core/src/layers/immutable_index.rs +++ b/core/src/layers/immutable_index.rs @@ -251,7 +251,7 @@ mod tests { use super::*; use crate::layers::LoggingLayer; - use crate::services::Http; + use crate::services::HttpConfig; use crate::EntryMode; use crate::Operator; @@ -264,12 +264,12 @@ mod tests { iil.insert(i.to_string()) } - let op = Http::from_map({ + let op = HttpConfig::from_iter({ let mut map = HashMap::new(); map.insert("endpoint".to_string(), "https://xuanwo.io".to_string()); map }) - .and_then(Operator::new)? + .and_then(Operator::from_config)? .layer(LoggingLayer::default()) .layer(iil) .finish(); @@ -302,12 +302,12 @@ mod tests { iil.insert(i.to_string()) } - let op = Http::from_map({ + let op = HttpConfig::from_iter({ let mut map = HashMap::new(); map.insert("endpoint".to_string(), "https://xuanwo.io".to_string()); map }) - .and_then(Operator::new)? + .and_then(Operator::from_config)? .layer(LoggingLayer::default()) .layer(iil) .finish(); @@ -346,12 +346,12 @@ mod tests { iil.insert(i.to_string()) } - let op = Http::from_map({ + let op = HttpConfig::from_iter({ let mut map = HashMap::new(); map.insert("endpoint".to_string(), "https://xuanwo.io".to_string()); map }) - .and_then(Operator::new)? + .and_then(Operator::from_config)? .layer(LoggingLayer::default()) .layer(iil) .finish(); @@ -404,12 +404,12 @@ mod tests { iil.insert(i.to_string()) } - let op = Http::from_map({ + let op = HttpConfig::from_iter({ let mut map = HashMap::new(); map.insert("endpoint".to_string(), "https://xuanwo.io".to_string()); map }) - .and_then(Operator::new)? + .and_then(Operator::from_config)? .layer(LoggingLayer::default()) .layer(iil) .finish(); diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index 4ccc616b61f..be10e0c87eb 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -773,15 +773,9 @@ mod tests { impl Builder for MockBuilder { const SCHEME: Scheme = Scheme::Custom("mock"); - type Accessor = MockService; - type Config = (); - fn from_config(_: Self::Config) -> Self { - Self::default() - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { Ok(MockService { attempt: self.attempt.clone(), }) diff --git a/core/src/services/aliyun_drive/backend.rs b/core/src/services/aliyun_drive/backend.rs index fe7e2b2bf08..227a9c17fa4 100644 --- a/core/src/services/aliyun_drive/backend.rs +++ b/core/src/services/aliyun_drive/backend.rs @@ -35,6 +35,7 @@ use super::lister::AliyunDriveLister; use super::lister::AliyunDriveParent; use super::writer::AliyunDriveWriter; use crate::raw::*; + use crate::*; /// Aliyun Drive services support. @@ -87,6 +88,15 @@ impl Debug for AliyunDriveConfig { } } +impl Configurator for AliyunDriveConfig { + fn into_builder(self) -> impl Builder { + AliyunDriveBuilder { + config: self, + http_client: None, + } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct AliyunDriveBuilder { @@ -167,25 +177,15 @@ impl AliyunDriveBuilder { impl Builder for AliyunDriveBuilder { const SCHEME: Scheme = Scheme::AliyunDrive; - - type Accessor = AliyunDriveBackend; - type Config = AliyunDriveConfig; - fn from_config(config: Self::Config) -> Self { - AliyunDriveBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", &root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -449,7 +449,6 @@ impl Access for AliyunDriveBackend { serde_json::from_reader(res.reader()).map_err(new_json_serialize_error)?; Some(AliyunDriveParent { file_id: file.file_id, - name: file.name, path: path.to_string(), updated_at: file.updated_at, }) diff --git a/core/src/services/aliyun_drive/core.rs b/core/src/services/aliyun_drive/core.rs index 79e2d7a4ff9..c682fc19137 100644 --- a/core/src/services/aliyun_drive/core.rs +++ b/core/src/services/aliyun_drive/core.rs @@ -382,19 +382,6 @@ impl AliyunDriveCore { self.send(req, token.as_deref()).await } - pub async fn get(&self, file_id: &str) -> Result { - let (token, drive_id) = self.get_token_and_drive().await?; - let body = serde_json::to_vec(&FileRequest { - drive_id: &drive_id, - file_id, - }) - .map_err(new_json_serialize_error)?; - let req = Request::post(format!("{}/adrive/v1.0/openFile/get", self.endpoint)) - .body(Buffer::from(body)) - .map_err(new_request_build_error)?; - self.send(req, token.as_deref()).await - } - pub async fn upload(&self, upload_url: &str, body: Buffer) -> Result { let req = Request::put(upload_url) .body(body) diff --git a/core/src/services/aliyun_drive/lister.rs b/core/src/services/aliyun_drive/lister.rs index 536da838a02..a94db21bcd6 100644 --- a/core/src/services/aliyun_drive/lister.rs +++ b/core/src/services/aliyun_drive/lister.rs @@ -39,7 +39,6 @@ pub struct AliyunDriveLister { pub struct AliyunDriveParent { pub file_id: String, - pub name: String, pub path: String, pub updated_at: String, } diff --git a/core/src/services/alluxio/backend.rs b/core/src/services/alluxio/backend.rs index 8471d84a5be..5f83fae48e7 100644 --- a/core/src/services/alluxio/backend.rs +++ b/core/src/services/alluxio/backend.rs @@ -59,6 +59,15 @@ impl Debug for AlluxioConfig { } } +impl Configurator for AlluxioConfig { + fn into_builder(self) -> impl Builder { + AlluxioBuilder { + config: self, + http_client: None, + } + } +} + /// [Alluxio](https://www.alluxio.io/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -117,18 +126,10 @@ impl AlluxioBuilder { impl Builder for AlluxioBuilder { const SCHEME: Scheme = Scheme::Alluxio; - type Accessor = AlluxioBackend; type Config = AlluxioConfig; - fn from_config(config: Self::Config) -> Self { - AlluxioBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of AlluxioBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -142,7 +143,7 @@ impl Builder for AlluxioBuilder { }?; debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -259,13 +260,10 @@ mod test { map.insert("root".to_string(), "/".to_string()); map.insert("endpoint".to_string(), "http://127.0.0.1:39999".to_string()); - let builder = AlluxioBuilder::from_map(map).unwrap(); + let builder = AlluxioConfig::from_iter(map).unwrap(); - assert_eq!(builder.config.root, Some("/".to_string())); - assert_eq!( - builder.config.endpoint, - Some("http://127.0.0.1:39999".to_string()) - ); + assert_eq!(builder.root, Some("/".to_string())); + assert_eq!(builder.endpoint, Some("http://127.0.0.1:39999".to_string())); } #[test] diff --git a/core/src/services/atomicserver/backend.rs b/core/src/services/atomicserver/backend.rs index 463056fe8a2..d9d4dc1305a 100644 --- a/core/src/services/atomicserver/backend.rs +++ b/core/src/services/atomicserver/backend.rs @@ -62,6 +62,12 @@ impl Debug for AtomicserverConfig { } } +impl Configurator for AtomicserverConfig { + fn into_builder(self) -> impl Builder { + AtomicserverBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct AtomicserverBuilder { @@ -113,14 +119,9 @@ impl AtomicserverBuilder { impl Builder for AtomicserverBuilder { const SCHEME: Scheme = Scheme::Atomicserver; - type Accessor = AtomicserverBackend; type Config = AtomicserverConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let root = normalize_root( self.config .root diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs index a9146f319da..850aaf82408 100644 --- a/core/src/services/azblob/backend.rs +++ b/core/src/services/azblob/backend.rs @@ -40,6 +40,7 @@ use super::writer::AzblobWriter; use crate::raw::*; use crate::services::azblob::core::AzblobCore; use crate::services::azblob::writer::AzblobWriters; + use crate::*; /// Known endpoint suffix Azure Storage Blob services resource URI syntax. @@ -117,6 +118,15 @@ impl Debug for AzblobConfig { } } +impl Configurator for AzblobConfig { + fn into_builder(self) -> impl Builder { + AzblobBuilder { + config: self, + http_client: None, + } + } +} + #[doc = include_str!("docs.md")] #[derive(Default, Clone)] pub struct AzblobBuilder { @@ -395,20 +405,12 @@ impl AzblobBuilder { impl Builder for AzblobBuilder { const SCHEME: Scheme = Scheme::Azblob; - type Accessor = AzblobBackend; type Config = AzblobConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); // Handle endpoint, region and container name. @@ -428,7 +430,7 @@ impl Builder for AzblobBuilder { }?; debug!("backend use endpoint {}", &container); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -496,7 +498,6 @@ impl Builder for AzblobBuilder { .batch_max_operations .unwrap_or(AZBLOB_BATCH_LIMIT); - debug!("backend build finished: {:?}", &self); Ok(AzblobBackend { core: Arc::new(AzblobCore { root, @@ -770,9 +771,8 @@ impl Access for AzblobBackend { #[cfg(test)] mod tests { + use super::infer_storage_name_from_endpoint; use super::AzblobBuilder; - use crate::services::azblob::backend::infer_storage_name_from_endpoint; - use crate::Builder; #[test] fn test_infer_storage_name_from_endpoint() { @@ -788,75 +788,6 @@ mod tests { assert_eq!(storage_name, Some("account".to_string())); } - #[test] - fn test_builder_from_endpoint_and_key_infer_account_name() { - let mut azblob_builder = AzblobBuilder::default(); - azblob_builder.endpoint("https://storagesample.blob.core.chinacloudapi.cn"); - azblob_builder.container("container"); - azblob_builder.account_key("account-key"); - let azblob = azblob_builder - .build() - .expect("build azblob should be succeeded."); - - assert_eq!( - azblob.core.endpoint, - "https://storagesample.blob.core.chinacloudapi.cn" - ); - - assert_eq!(azblob.core.container, "container".to_string()); - - assert_eq!( - azblob_builder.config.account_key.unwrap(), - "account-key".to_string() - ); - } - - #[test] - fn test_no_key_wont_infer_account_name() { - let mut azblob_builder = AzblobBuilder::default(); - azblob_builder.endpoint("https://storagesample.blob.core.windows.net"); - azblob_builder.container("container"); - let azblob = azblob_builder - .build() - .expect("build azblob should be succeeded."); - - assert_eq!( - azblob.core.endpoint, - "https://storagesample.blob.core.windows.net" - ); - - assert_eq!(azblob.core.container, "container".to_string()); - - assert_eq!(azblob_builder.config.account_key, None); - } - - #[test] - fn test_builder_from_endpoint_and_sas() { - let mut azblob_builder = AzblobBuilder::default(); - azblob_builder.endpoint("https://storagesample.blob.core.usgovcloudapi.net"); - azblob_builder.container("container"); - azblob_builder.account_name("storagesample"); - azblob_builder.account_key("account-key"); - azblob_builder.sas_token("sas"); - let azblob = azblob_builder - .build() - .expect("build azblob should be succeeded."); - - assert_eq!( - azblob.core.endpoint, - "https://storagesample.blob.core.usgovcloudapi.net" - ); - - assert_eq!(azblob.core.container, "container".to_string()); - - assert_eq!( - azblob_builder.config.account_key.unwrap(), - "account-key".to_string() - ); - - assert_eq!(azblob_builder.config.sas_token.unwrap(), "sas".to_string()); - } - #[test] fn test_builder_from_connection_string() { let builder = AzblobBuilder::from_connection_string( diff --git a/core/src/services/azdls/backend.rs b/core/src/services/azdls/backend.rs index 438ca3b8ec1..aa38d3c55a5 100644 --- a/core/src/services/azdls/backend.rs +++ b/core/src/services/azdls/backend.rs @@ -33,6 +33,7 @@ use super::lister::AzdlsLister; use super::writer::AzdlsWriter; use super::writer::AzdlsWriters; use crate::raw::*; + use crate::*; /// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax. @@ -79,6 +80,15 @@ impl Debug for AzdlsConfig { } } +impl Configurator for AzdlsConfig { + fn into_builder(self) -> impl Builder { + AzdlsBuilder { + config: self, + http_client: None, + } + } +} + /// Azure Data Lake Storage Gen2 Support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -169,20 +179,12 @@ impl AzdlsBuilder { impl Builder for AzdlsBuilder { const SCHEME: Scheme = Scheme::Azdls; - type Accessor = AzdlsBackend; type Config = AzdlsConfig; - fn from_config(config: Self::Config) -> Self { - AzdlsBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); // Handle endpoint, region and container name. @@ -202,7 +204,7 @@ impl Builder for AzdlsBuilder { }?; debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -224,8 +226,6 @@ impl Builder for AzdlsBuilder { let cred_loader = AzureStorageLoader::new(config_loader); let signer = AzureStorageSigner::new(); - - debug!("backend build finished: {:?}", &self); Ok(AzdlsBackend { core: Arc::new(AzdlsCore { filesystem: self.config.filesystem.clone(), @@ -430,9 +430,7 @@ fn infer_storage_name_from_endpoint(endpoint: &str) -> Option { #[cfg(test)] mod tests { - use super::AzdlsBuilder; - use crate::services::azdls::backend::infer_storage_name_from_endpoint; - use crate::Builder; + use super::infer_storage_name_from_endpoint; #[test] fn test_infer_storage_name_from_endpoint() { @@ -447,46 +445,4 @@ mod tests { let storage_name = infer_storage_name_from_endpoint(endpoint); assert_eq!(storage_name, Some("account".to_string())); } - - #[test] - fn test_builder_from_endpoint_and_key_infer_account_name() { - let mut azdls_builder = AzdlsBuilder::default(); - azdls_builder.endpoint("https://storagesample.dfs.core.chinacloudapi.cn"); - azdls_builder.account_key("account-key"); - azdls_builder.filesystem("filesystem"); - let azdls = azdls_builder - .build() - .expect("build Azdls should be succeeded."); - - assert_eq!( - azdls.core.endpoint, - "https://storagesample.dfs.core.chinacloudapi.cn" - ); - - assert_eq!(azdls.core.filesystem, "filesystem".to_string()); - - assert_eq!( - azdls_builder.config.account_key.unwrap(), - "account-key".to_string() - ); - } - - #[test] - fn test_no_key_wont_infer_account_name() { - let mut azdls_builder = AzdlsBuilder::default(); - azdls_builder.endpoint("https://storagesample.dfs.core.windows.net"); - azdls_builder.filesystem("filesystem"); - let azdls = azdls_builder - .build() - .expect("build Azdls should be succeeded."); - - assert_eq!( - azdls.core.endpoint, - "https://storagesample.dfs.core.windows.net" - ); - - assert_eq!(azdls.core.filesystem, "filesystem".to_string()); - - assert_eq!(azdls_builder.config.account_key, None); - } } diff --git a/core/src/services/azfile/backend.rs b/core/src/services/azfile/backend.rs index 7f2fb15bcea..7381e96b8b8 100644 --- a/core/src/services/azfile/backend.rs +++ b/core/src/services/azfile/backend.rs @@ -77,6 +77,15 @@ impl Debug for AzfileConfig { } } +impl Configurator for AzfileConfig { + fn into_builder(self) -> impl Builder { + AzfileBuilder { + config: self, + http_client: None, + } + } +} + /// Azure File services support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -167,20 +176,12 @@ impl AzfileBuilder { impl Builder for AzfileBuilder { const SCHEME: Scheme = Scheme::Azfile; - type Accessor = AzfileBackend; type Config = AzfileConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let endpoint = match &self.config.endpoint { @@ -191,7 +192,7 @@ impl Builder for AzfileBuilder { }?; debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -223,10 +224,7 @@ impl Builder for AzfileBuilder { }; let cred_loader = AzureStorageLoader::new(config_loader); - let signer = AzureStorageSigner::new(); - - debug!("backend build finished: {:?}", &self); Ok(AzfileBackend { core: Arc::new(AzfileCore { root, @@ -401,7 +399,6 @@ impl Access for AzfileBackend { #[cfg(test)] mod tests { use super::*; - use crate::Builder; #[test] fn test_infer_storage_name_from_endpoint() { @@ -422,24 +419,4 @@ mod tests { assert_eq!(account_name, Some(expected.to_string()), "{}", desc); } } - - #[test] - fn test_builder_from_endpoint_and_key_infer_account_name() { - let mut azfile_builder = AzfileBuilder::default(); - azfile_builder.endpoint("https://account.file.core.windows.net/"); - azfile_builder.account_key("account-key"); - let azfile = azfile_builder - .build() - .expect("build Azdls should be succeeded."); - - assert_eq!( - azfile.core.endpoint, - "https://account.file.core.windows.net" - ); - - assert_eq!( - azfile_builder.config.account_key.unwrap(), - "account-key".to_string() - ); - } } diff --git a/core/src/services/b2/backend.rs b/core/src/services/b2/backend.rs index 8f8c7d2c9d0..a0a09261bc5 100644 --- a/core/src/services/b2/backend.rs +++ b/core/src/services/b2/backend.rs @@ -81,6 +81,15 @@ impl Debug for B2Config { } } +impl Configurator for B2Config { + fn into_builder(self) -> impl Builder { + B2Builder { + config: self, + http_client: None, + } + } +} + /// [b2](https://www.backblaze.com/cloud-storage) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -165,18 +174,10 @@ impl B2Builder { impl Builder for B2Builder { const SCHEME: Scheme = Scheme::B2; - type Accessor = B2Backend; type Config = B2Config; - fn from_config(config: Self::Config) -> Self { - B2Builder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of B2Backend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -218,7 +219,7 @@ impl Builder for B2Builder { ), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/cacache/backend.rs b/core/src/services/cacache/backend.rs index a76772a4b7a..6003b263c72 100644 --- a/core/src/services/cacache/backend.rs +++ b/core/src/services/cacache/backend.rs @@ -23,6 +23,7 @@ use cacache; use serde::{Deserialize, Serialize}; use crate::raw::adapters::kv; +use crate::raw::Access; use crate::Builder; use crate::Error; use crate::ErrorKind; @@ -36,6 +37,12 @@ pub struct CacacheConfig { pub datadir: Option, } +impl Configurator for CacacheConfig { + fn into_builder(self) -> impl Builder { + CacacheBuilder { config: self } + } +} + /// cacache service support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -53,15 +60,10 @@ impl CacacheBuilder { impl Builder for CacacheBuilder { const SCHEME: Scheme = Scheme::Cacache; - type Accessor = CacacheBackend; type Config = CacacheConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { - let datadir_path = self.config.datadir.take().ok_or_else(|| { + fn build(self) -> Result { + let datadir_path = self.config.datadir.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Cacache) })?; diff --git a/core/src/services/chainsafe/backend.rs b/core/src/services/chainsafe/backend.rs index 6702f25a53f..1b38084a9fe 100644 --- a/core/src/services/chainsafe/backend.rs +++ b/core/src/services/chainsafe/backend.rs @@ -35,7 +35,7 @@ use super::writer::ChainsafeWriters; use crate::raw::*; use crate::*; -/// Config for backblaze Chainsafe services support. +/// Config for Chainsafe services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] @@ -63,6 +63,15 @@ impl Debug for ChainsafeConfig { } } +impl Configurator for ChainsafeConfig { + fn into_builder(self) -> impl Builder { + ChainsafeBuilder { + config: self, + http_client: None, + } + } +} + /// [chainsafe](https://storage.chainsafe.io/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -129,18 +138,10 @@ impl ChainsafeBuilder { impl Builder for ChainsafeBuilder { const SCHEME: Scheme = Scheme::Chainsafe; - type Accessor = ChainsafeBackend; type Config = ChainsafeConfig; - fn from_config(config: Self::Config) -> Self { - ChainsafeBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of ChainsafeBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -162,7 +163,7 @@ impl Builder for ChainsafeBuilder { .with_context("service", Scheme::Chainsafe)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/chainsafe/core.rs b/core/src/services/chainsafe/core.rs index c65617abd75..9fb56d7fa16 100644 --- a/core/src/services/chainsafe/core.rs +++ b/core/src/services/chainsafe/core.rs @@ -114,33 +114,6 @@ impl ChainsafeCore { self.send(req).await } - pub async fn move_object(&self, from: &str, to: &str) -> Result> { - let from = build_abs_path(&self.root, from); - let to = build_abs_path(&self.root, to); - - let url = format!( - "https://api.chainsafe.io/api/v1/bucket/{}/mv", - self.bucket_id - ); - - let req_body = &json!({ - "path": from, - "new_path": to, - }); - let body = Buffer::from(Bytes::from(req_body.to_string())); - - let req = Request::post(url) - .header( - header::AUTHORIZATION, - format_authorization_by_bearer(&self.api_key)?, - ) - .header(header::CONTENT_TYPE, "application/json") - .body(body) - .map_err(new_request_build_error)?; - - self.send(req).await - } - pub async fn delete_object(&self, path: &str) -> Result> { let path = build_abs_path(&self.root, path); diff --git a/core/src/services/cloudflare_kv/backend.rs b/core/src/services/cloudflare_kv/backend.rs index 43b18efb234..cda381f7939 100644 --- a/core/src/services/cloudflare_kv/backend.rs +++ b/core/src/services/cloudflare_kv/backend.rs @@ -27,6 +27,7 @@ use serde::{Deserialize, Serialize}; use super::error::parse_error; use crate::raw::adapters::kv; use crate::raw::*; + use crate::ErrorKind; use crate::*; @@ -60,6 +61,15 @@ impl Debug for CloudflareKvConfig { } } +impl Configurator for CloudflareKvConfig { + fn into_builder(self) -> impl Builder { + CloudflareKvBuilder { + config: self, + http_client: None, + } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct CloudflareKvBuilder { @@ -113,18 +123,9 @@ impl CloudflareKvBuilder { impl Builder for CloudflareKvBuilder { const SCHEME: Scheme = Scheme::CloudflareKv; - - type Accessor = CloudflareKvBackend; type Config = CloudflareKvConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let authorization = match &self.config.token { Some(token) => format_authorization_by_bearer(token)?, None => return Err(Error::new(ErrorKind::ConfigInvalid, "token is required")), @@ -144,7 +145,7 @@ impl Builder for CloudflareKvBuilder { )); }; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -166,7 +167,7 @@ impl Builder for CloudflareKvBuilder { account_id, namespace_id ); - Ok(kv::Backend::new(Adapter { + Ok(CloudflareKvBackend::new(Adapter { authorization, account_id, namespace_id, diff --git a/core/src/services/compfs/backend.rs b/core/src/services/compfs/backend.rs index 2396f3d01a3..ce72fa2ebe2 100644 --- a/core/src/services/compfs/backend.rs +++ b/core/src/services/compfs/backend.rs @@ -20,11 +20,10 @@ use std::{io::Cursor, sync::Arc}; use compio::{dispatcher::Dispatcher, fs::OpenOptions}; use serde::{Deserialize, Serialize}; +use super::{core::CompfsCore, lister::CompfsLister, reader::CompfsReader, writer::CompfsWriter}; use crate::raw::*; use crate::*; -use super::{core::CompfsCore, lister::CompfsLister, reader::CompfsReader, writer::CompfsWriter}; - /// compio-based file system support. #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] pub struct CompfsConfig { @@ -34,6 +33,12 @@ pub struct CompfsConfig { pub root: Option, } +impl Configurator for CompfsConfig { + fn into_builder(self) -> impl Builder { + CompfsBuilder { config: self } + } +} + /// [`compio`]-based file system support. #[derive(Debug, Clone, Default)] pub struct CompfsBuilder { @@ -55,15 +60,10 @@ impl CompfsBuilder { impl Builder for CompfsBuilder { const SCHEME: Scheme = Scheme::Compfs; - type Accessor = CompfsBackend; type Config = CompfsConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { - let root = match self.config.root.take() { + fn build(self) -> Result { + let root = match self.config.root { Some(root) => Ok(root), None => Err(Error::new( ErrorKind::ConfigInvalid, diff --git a/core/src/services/cos/backend.rs b/core/src/services/cos/backend.rs index 5c6c19d6be3..1be95f87fbe 100644 --- a/core/src/services/cos/backend.rs +++ b/core/src/services/cos/backend.rs @@ -33,6 +33,7 @@ use super::lister::CosLister; use super::writer::CosWriter; use crate::raw::*; use crate::services::cos::writer::CosWriters; + use crate::*; /// Tencent-Cloud COS services support. @@ -65,6 +66,15 @@ impl Debug for CosConfig { } } +impl Configurator for CosConfig { + fn into_builder(self) -> impl Builder { + CosBuilder { + config: self, + http_client: None, + } + } +} + /// Tencent-Cloud COS services support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -165,20 +175,12 @@ impl CosBuilder { impl Builder for CosBuilder { const SCHEME: Scheme = Scheme::Cos; - type Accessor = CosBackend; type Config = CosConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let bucket = match &self.config.bucket { @@ -210,7 +212,7 @@ impl Builder for CosBuilder { let endpoint = uri.host().unwrap().replace(&format!("//{bucket}."), "//"); debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -224,10 +226,10 @@ impl Builder for CosBuilder { cfg = cfg.from_env(); } - if let Some(v) = self.config.secret_id.take() { + if let Some(v) = self.config.secret_id { cfg.secret_id = Some(v); } - if let Some(v) = self.config.secret_key.take() { + if let Some(v) = self.config.secret_key { cfg.secret_key = Some(v); } @@ -235,7 +237,6 @@ impl Builder for CosBuilder { let signer = TencentCosSigner::new(); - debug!("backend build finished"); Ok(CosBackend { core: Arc::new(CosCore { bucket: bucket.clone(), diff --git a/core/src/services/d1/backend.rs b/core/src/services/d1/backend.rs index 768bf92aed1..3c5a781c121 100644 --- a/core/src/services/d1/backend.rs +++ b/core/src/services/d1/backend.rs @@ -29,6 +29,7 @@ use super::error::parse_error; use super::model::D1Response; use crate::raw::adapters::kv; use crate::raw::*; + use crate::ErrorKind; use crate::*; @@ -65,6 +66,15 @@ impl Debug for D1Config { } } +impl Configurator for D1Config { + fn into_builder(self) -> impl Builder { + D1Builder { + config: self, + http_client: None, + } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct D1Builder { @@ -157,24 +167,17 @@ impl D1Builder { impl Builder for D1Builder { const SCHEME: Scheme = Scheme::D1; - type Accessor = D1Backend; type Config = D1Config; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let mut authorization = None; - let config = &self.config; - if let Some(token) = &config.token { - authorization = Some(format_authorization_by_bearer(token)?) + let config = self.config; + + if let Some(token) = config.token { + authorization = Some(format_authorization_by_bearer(&token)?) } - let Some(account_id) = config.account_id.clone() else { + let Some(account_id) = config.account_id else { return Err(Error::new( ErrorKind::ConfigInvalid, "account_id is required", @@ -188,7 +191,7 @@ impl Builder for D1Builder { )); }; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/dashmap/backend.rs b/core/src/services/dashmap/backend.rs index 54cf44b19d2..39e0f5bdbba 100644 --- a/core/src/services/dashmap/backend.rs +++ b/core/src/services/dashmap/backend.rs @@ -22,6 +22,8 @@ use dashmap::DashMap; use serde::{Deserialize, Serialize}; use crate::raw::adapters::typed_kv; + +use crate::raw::Access; use crate::*; /// [dashmap](https://github.com/xacrimon/dashmap) backend support. @@ -31,6 +33,12 @@ pub struct DashmapConfig { pub root: Option, } +impl Configurator for DashmapConfig { + fn into_builder(self) -> impl Builder { + DashmapBuilder { config: self } + } +} + /// [dashmap](https://github.com/xacrimon/dashmap) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -48,14 +56,9 @@ impl DashmapBuilder { impl Builder for DashmapBuilder { const SCHEME: Scheme = Scheme::Dashmap; - type Accessor = DashmapBackend; type Config = DashmapConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { Ok(DashmapBackend::new(Adapter { inner: DashMap::default(), }) @@ -141,7 +144,6 @@ impl typed_kv::Adapter for Adapter { #[cfg(test)] mod tests { use super::*; - use crate::raw::*; #[test] fn test_accessor_metadata_name() { diff --git a/core/src/services/dbfs/backend.rs b/core/src/services/dbfs/backend.rs index fe42708b03d..235f4b58aae 100644 --- a/core/src/services/dbfs/backend.rs +++ b/core/src/services/dbfs/backend.rs @@ -57,6 +57,12 @@ impl Debug for DbfsConfig { } } +impl Configurator for DbfsConfig { + fn into_builder(self) -> impl Builder { + DbfsBuilder { config: self } + } +} + /// [Dbfs](https://docs.databricks.com/api/azure/workspace/dbfs)'s REST API support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -112,18 +118,13 @@ impl DbfsBuilder { impl Builder for DbfsBuilder { const SCHEME: Scheme = Scheme::Dbfs; - type Accessor = DbfsBackend; type Config = DbfsConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - /// Build a DbfsBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let endpoint = match &self.config.endpoint { @@ -134,7 +135,7 @@ impl Builder for DbfsBuilder { }?; debug!("backend use endpoint: {}", &endpoint); - let token = match self.config.token.take() { + let token = match self.config.token { Some(token) => token, None => { return Err(Error::new( @@ -145,8 +146,6 @@ impl Builder for DbfsBuilder { }; let client = HttpClient::new()?; - - debug!("backend build finished: {:?}", &self); Ok(DbfsBackend { core: Arc::new(DbfsCore { root, diff --git a/core/src/services/dbfs/core.rs b/core/src/services/dbfs/core.rs index d1d5c97dd26..ce4117b3dae 100644 --- a/core/src/services/dbfs/core.rs +++ b/core/src/services/dbfs/core.rs @@ -155,47 +155,6 @@ impl DbfsCore { req.body(body).map_err(new_request_build_error) } - pub async fn dbfs_read( - &self, - path: &str, - offset: u64, - length: u64, - ) -> Result> { - let p = build_rooted_abs_path(&self.root, path) - .trim_end_matches('/') - .to_string(); - - let mut url = format!( - "{}/api/2.0/dbfs/read?path={}", - self.endpoint, - percent_encode_path(&p) - ); - - if offset > 0 { - url.push_str(&format!("&offset={}", offset)); - } - - if length > 0 { - url.push_str(&format!("&length={}", length)); - } - - let mut req = Request::get(&url); - - let auth_header_content = format!("Bearer {}", self.token); - req = req.header(header::AUTHORIZATION, auth_header_content); - - let req = req.body(Buffer::new()).map_err(new_request_build_error)?; - - let resp = self.client.send(req).await?; - - let status = resp.status(); - - match status { - StatusCode::OK => Ok(resp), - _ => Err(parse_error(resp).await?), - } - } - pub async fn dbfs_get_status(&self, path: &str) -> Result> { let p = build_rooted_abs_path(&self.root, path) .trim_end_matches('/') diff --git a/core/src/services/dropbox/builder.rs b/core/src/services/dropbox/builder.rs index 521d41310ab..c964b995573 100644 --- a/core/src/services/dropbox/builder.rs +++ b/core/src/services/dropbox/builder.rs @@ -55,6 +55,15 @@ impl Debug for DropboxConfig { } } +impl Configurator for DropboxConfig { + fn into_builder(self) -> impl Builder { + DropboxBuilder { + config: self, + http_client: None, + } + } +} + /// [Dropbox](https://www.dropbox.com/) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -132,19 +141,11 @@ impl DropboxBuilder { impl Builder for DropboxBuilder { const SCHEME: Scheme = Scheme::Dropbox; - type Accessor = DropboxBackend; type Config = DropboxConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); - let client = if let Some(client) = self.http_client.take() { + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -153,10 +154,7 @@ impl Builder for DropboxBuilder { })? }; - let signer = match ( - self.config.access_token.take(), - self.config.refresh_token.take(), - ) { + let signer = match (self.config.access_token, self.config.refresh_token) { (Some(access_token), None) => DropboxSigner { access_token, // We will never expire user specified token. @@ -164,14 +162,14 @@ impl Builder for DropboxBuilder { ..Default::default() }, (None, Some(refresh_token)) => { - let client_id = self.config.client_id.take().ok_or_else(|| { + let client_id = self.config.client_id.ok_or_else(|| { Error::new( ErrorKind::ConfigInvalid, "client_id must be set when refresh_token is set", ) .with_context("service", Scheme::Dropbox) })?; - let client_secret = self.config.client_secret.take().ok_or_else(|| { + let client_secret = self.config.client_secret.ok_or_else(|| { Error::new( ErrorKind::ConfigInvalid, "client_secret must be set when refresh_token is set", diff --git a/core/src/services/etcd/backend.rs b/core/src/services/etcd/backend.rs index 45bc2514b16..9a89adcc13f 100644 --- a/core/src/services/etcd/backend.rs +++ b/core/src/services/etcd/backend.rs @@ -32,6 +32,7 @@ use tokio::sync::OnceCell; use crate::raw::adapters::kv; use crate::raw::*; + use crate::*; const DEFAULT_ETCD_ENDPOINTS: &str = "http://127.0.0.1:2379"; @@ -100,6 +101,12 @@ impl Debug for EtcdConfig { } } +impl Configurator for EtcdConfig { + fn into_builder(self) -> impl Builder { + EtcdBuilder { config: self } + } +} + /// [Etcd](https://etcd.io/) services support. #[doc = include_str!("docs.md")] #[derive(Clone, Default)] @@ -190,14 +197,9 @@ impl EtcdBuilder { impl Builder for EtcdBuilder { const SCHEME: Scheme = Scheme::Etcd; - type Accessor = EtcdBackend; type Config = EtcdConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let endpoints = self .config .endpoints diff --git a/core/src/services/foundationdb/backend.rs b/core/src/services/foundationdb/backend.rs index 70b2ed8a8bd..fb8e80101ff 100644 --- a/core/src/services/foundationdb/backend.rs +++ b/core/src/services/foundationdb/backend.rs @@ -54,6 +54,12 @@ impl Debug for FoundationConfig { } } +impl Configurator for FoundationConfig { + fn into_builder(self) -> impl Builder { + FoundationdbBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct FoundationdbBuilder { @@ -76,14 +82,9 @@ impl FoundationdbBuilder { impl Builder for FoundationdbBuilder { const SCHEME: Scheme = Scheme::Foundationdb; - type Accessor = FoundationdbBackend; type Config = FoundationConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let _network = Arc::new(unsafe { foundationdb::boot() }); let db; if let Some(cfg_path) = &self.config.config_path { diff --git a/core/src/services/fs/backend.rs b/core/src/services/fs/backend.rs index e03c55c9a22..66d215a867a 100644 --- a/core/src/services/fs/backend.rs +++ b/core/src/services/fs/backend.rs @@ -24,14 +24,13 @@ use chrono::DateTime; use log::debug; use serde::{Deserialize, Serialize}; -use crate::raw::*; -use crate::*; - use super::core::*; use super::lister::FsLister; use super::reader::FsReader; use super::writer::FsWriter; use super::writer::FsWriters; +use crate::raw::*; +use crate::*; /// config for file system #[derive(Default, Debug, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -45,6 +44,12 @@ pub struct FsConfig { pub atomic_write_dir: Option, } +impl Configurator for FsConfig { + fn into_builder(self) -> impl Builder { + FsBuilder { config: self } + } +} + /// POSIX file system support. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -79,17 +84,12 @@ impl FsBuilder { impl Builder for FsBuilder { const SCHEME: Scheme = Scheme::Fs; - type Accessor = FsBackend; type Config = FsConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = match self.config.root.take().map(PathBuf::from) { + let root = match self.config.root.map(PathBuf::from) { Some(root) => Ok(root), None => Err(Error::new( ErrorKind::ConfigInvalid, @@ -110,7 +110,7 @@ impl Builder for FsBuilder { } } - let atomic_write_dir = self.config.atomic_write_dir.take().map(PathBuf::from); + let atomic_write_dir = self.config.atomic_write_dir.map(PathBuf::from); // If atomic write dir is not exist, we must create it. if let Some(d) = &atomic_write_dir { @@ -154,7 +154,6 @@ impl Builder for FsBuilder { }) .unwrap_or(Ok(None))?; - debug!("backend build finished: {:?}", &self); Ok(FsBackend { core: Arc::new(FsCore { root, diff --git a/core/src/services/ftp/backend.rs b/core/src/services/ftp/backend.rs index 8fb34753990..82f6448ae06 100644 --- a/core/src/services/ftp/backend.rs +++ b/core/src/services/ftp/backend.rs @@ -43,6 +43,7 @@ use super::lister::FtpLister; use super::reader::FtpReader; use super::writer::FtpWriter; use crate::raw::*; + use crate::*; /// Config for Ftpservices support. @@ -69,6 +70,12 @@ impl Debug for FtpConfig { } } +impl Configurator for FtpConfig { + fn into_builder(self) -> impl Builder { + FtpBuilder { config: self } + } +} + /// FTP and FTPS services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -132,14 +139,9 @@ impl FtpBuilder { impl Builder for FtpBuilder { const SCHEME: Scheme = Scheme::Ftp; - type Accessor = FtpBackend; type Config = FtpConfig; - fn from_config(config: Self::Config) -> Self { - FtpBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("ftp backend build started: {:?}", &self); let endpoint = match &self.config.endpoint { None => return Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")), @@ -175,7 +177,7 @@ impl Builder for FtpBuilder { } }; - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); let user = match &self.config.user { None => "".to_string(), @@ -187,8 +189,6 @@ impl Builder for FtpBuilder { Some(v) => v.clone(), }; - debug!("ftp backend finished: {:?}", &self); - Ok(FtpBackend { endpoint, root, diff --git a/core/src/services/gcs/backend.rs b/core/src/services/gcs/backend.rs index ca327bdb561..dce729b652c 100644 --- a/core/src/services/gcs/backend.rs +++ b/core/src/services/gcs/backend.rs @@ -36,6 +36,7 @@ use super::lister::GcsLister; use super::writer::GcsWriter; use super::writer::GcsWriters; use crate::raw::*; + use crate::*; const DEFAULT_GCS_ENDPOINT: &str = "https://storage.googleapis.com"; @@ -78,6 +79,16 @@ impl Debug for GcsConfig { } } +impl Configurator for GcsConfig { + fn into_builder(self) -> impl Builder { + GcsBuilder { + config: self, + http_client: None, + customized_token_loader: None, + } + } +} + /// [Google Cloud Storage](https://cloud.google.com/storage) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -227,21 +238,12 @@ impl GcsBuilder { impl Builder for GcsBuilder { const SCHEME: Scheme = Scheme::Gcs; - type Accessor = GcsBackend; type Config = GcsConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - customized_token_loader: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); // Handle endpoint and bucket name @@ -256,7 +258,7 @@ impl Builder for GcsBuilder { // TODO: server side encryption - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -298,7 +300,7 @@ impl Builder for GcsBuilder { if let Ok(Some(cred)) = cred_loader.load() { token_loader = token_loader.with_credentials(cred) } - if let Some(loader) = self.customized_token_loader.take() { + if let Some(loader) = self.customized_token_loader { token_loader = token_loader.with_customized_token_loader(loader) } diff --git a/core/src/services/gcs/core.rs b/core/src/services/gcs/core.rs index 47da3904f76..74fa2e50741 100644 --- a/core/src/services/gcs/core.rs +++ b/core/src/services/gcs/core.rs @@ -25,7 +25,6 @@ use backon::ExponentialBuilder; use backon::Retryable; use bytes::Bytes; use http::header::CONTENT_LENGTH; -use http::header::CONTENT_RANGE; use http::header::CONTENT_TYPE; use http::header::HOST; use http::header::IF_MATCH; @@ -503,22 +502,6 @@ impl GcsCore { self.send(req).await } - pub async fn gcs_initiate_resumable_upload(&self, path: &str) -> Result> { - let p = build_abs_path(&self.root, path); - let url = format!( - "{}/upload/storage/v1/b/{}/o?uploadType=resumable&name={}", - self.endpoint, self.bucket, p - ); - - let mut req = Request::post(&url) - .header(CONTENT_LENGTH, 0) - .body(Buffer::new()) - .map_err(new_request_build_error)?; - - self.sign(&mut req).await?; - self.send(req).await - } - pub async fn gcs_upload_part( &self, path: &str, @@ -602,64 +585,6 @@ impl GcsCore { self.sign(&mut req).await?; self.send(req).await } - - pub fn gcs_upload_in_resumable_upload( - &self, - location: &str, - size: u64, - written: u64, - body: Buffer, - ) -> Result> { - let mut req = Request::put(location); - - let range_header = format!("bytes {}-{}/*", written, written + size - 1); - - req = req - .header(CONTENT_LENGTH, size) - .header(CONTENT_RANGE, range_header); - - // Set body - let req = req.body(body).map_err(new_request_build_error)?; - - Ok(req) - } - - pub async fn gcs_complete_resumable_upload( - &self, - location: &str, - written: u64, - size: u64, - body: Buffer, - ) -> Result> { - let mut req = Request::post(location) - .header(CONTENT_LENGTH, size) - .header( - CONTENT_RANGE, - format!( - "bytes {}-{}/{}", - written, - written + size - 1, - written + size - ), - ) - .body(body) - .map_err(new_request_build_error)?; - - self.sign(&mut req).await?; - - self.send(req).await - } - - pub async fn gcs_abort_resumable_upload(&self, location: &str) -> Result> { - let mut req = Request::delete(location) - .header(CONTENT_LENGTH, 0) - .body(Buffer::new()) - .map_err(new_request_build_error)?; - - self.sign(&mut req).await?; - - self.send(req).await - } } /// Response JSON from GCS list objects API. diff --git a/core/src/services/gdrive/builder.rs b/core/src/services/gdrive/builder.rs index 014d158e449..1194d68b2e0 100644 --- a/core/src/services/gdrive/builder.rs +++ b/core/src/services/gdrive/builder.rs @@ -26,12 +26,13 @@ use serde::{Deserialize, Serialize}; use tokio::sync::Mutex; use super::backend::GdriveBackend; -use crate::raw::normalize_root; use crate::raw::HttpClient; use crate::raw::PathCacher; +use crate::raw::{normalize_root, Access}; use crate::services::gdrive::core::GdriveCore; use crate::services::gdrive::core::GdrivePathQuery; use crate::services::gdrive::core::GdriveSigner; + use crate::Scheme; use crate::*; @@ -60,6 +61,15 @@ impl Debug for GdriveConfig { } } +impl Configurator for GdriveConfig { + fn into_builder(self) -> impl Builder { + GdriveBuilder { + config: self, + http_client: None, + } + } +} + /// [GoogleDrive](https://drive.google.com/) backend support. #[derive(Default)] #[doc = include_str!("docs.md")] @@ -139,21 +149,13 @@ impl GdriveBuilder { impl Builder for GdriveBuilder { const SCHEME: Scheme = Scheme::Gdrive; - type Accessor = GdriveBackend; type Config = GdriveConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -163,24 +165,21 @@ impl Builder for GdriveBuilder { }; let mut signer = GdriveSigner::new(client.clone()); - match ( - self.config.access_token.take(), - self.config.refresh_token.take(), - ) { + match (self.config.access_token, self.config.refresh_token) { (Some(access_token), None) => { signer.access_token = access_token; // We will never expire user specified access token. signer.expires_in = DateTime::::MAX_UTC; } (None, Some(refresh_token)) => { - let client_id = self.config.client_id.take().ok_or_else(|| { + let client_id = self.config.client_id.ok_or_else(|| { Error::new( ErrorKind::ConfigInvalid, "client_id must be set when refresh_token is set", ) .with_context("service", Scheme::Gdrive) })?; - let client_secret = self.config.client_secret.take().ok_or_else(|| { + let client_secret = self.config.client_secret.ok_or_else(|| { Error::new( ErrorKind::ConfigInvalid, "client_secret must be set when refresh_token is set", diff --git a/core/src/services/ghac/backend.rs b/core/src/services/ghac/backend.rs index 6cd382c5896..5f4c4e7ad8c 100644 --- a/core/src/services/ghac/backend.rs +++ b/core/src/services/ghac/backend.rs @@ -37,6 +37,7 @@ use serde::Serialize; use super::error::parse_error; use super::writer::GhacWriter; use crate::raw::*; + use crate::*; /// The base url for cache url. @@ -96,6 +97,15 @@ pub struct GhacConfig { pub runtime_token: Option, } +impl Configurator for GhacConfig { + fn into_builder(self) -> impl Builder { + GhacBuilder { + config: self, + http_client: None, + } + } +} + /// GitHub Action Cache Services support. #[doc = include_str!("docs.md")] #[derive(Debug, Default)] @@ -167,23 +177,15 @@ impl GhacBuilder { impl Builder for GhacBuilder { const SCHEME: Scheme = Scheme::Ghac; - type Accessor = GhacBackend; type Config = GhacConfig; - fn from_config(config: Self::Config) -> Self { - GhacBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -195,13 +197,9 @@ impl Builder for GhacBuilder { let backend = GhacBackend { root, - cache_url: value_or_env( - self.config.endpoint.take(), - ACTIONS_CACHE_URL, - "Builder::build", - )?, + cache_url: value_or_env(self.config.endpoint, ACTIONS_CACHE_URL, "Builder::build")?, catch_token: value_or_env( - self.config.runtime_token.take(), + self.config.runtime_token, ACTIONS_RUNTIME_TOKEN, "Builder::build", )?, diff --git a/core/src/services/github/backend.rs b/core/src/services/github/backend.rs index 3cb3c962396..cf0a6abee53 100644 --- a/core/src/services/github/backend.rs +++ b/core/src/services/github/backend.rs @@ -34,7 +34,7 @@ use super::writer::GithubWriters; use crate::raw::*; use crate::*; -/// Config for backblaze GitHub services support. +/// Config for GitHub services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] @@ -71,6 +71,15 @@ impl Debug for GithubConfig { } } +impl Configurator for GithubConfig { + fn into_builder(self) -> impl Builder { + GithubBuilder { + config: self, + http_client: None, + } + } +} + /// [github contents](https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#create-or-update-file-contents) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -139,18 +148,10 @@ impl GithubBuilder { impl Builder for GithubBuilder { const SCHEME: Scheme = Scheme::Github; - type Accessor = GithubBackend; type Config = GithubConfig; - fn from_config(config: Self::Config) -> Self { - GithubBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of GithubBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -174,7 +175,7 @@ impl Builder for GithubBuilder { debug!("backend use repo {}", &self.config.repo); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/github/core.rs b/core/src/services/github/core.rs index 49a0cc50987..d8850915d85 100644 --- a/core/src/services/github/core.rs +++ b/core/src/services/github/core.rs @@ -325,36 +325,15 @@ pub struct Tree { #[derive(Default, Debug, Clone, Deserialize)] pub struct ListResponse { - pub size: u64, - pub sha: String, - #[serde(rename = "type")] - pub type_field: String, pub git_url: String, pub entries: Vec, } #[derive(Default, Debug, Clone, Deserialize)] pub struct Entry { - pub name: String, pub path: String, pub sha: String, pub size: u64, - pub url: String, - pub html_url: String, - pub git_url: String, - pub download_url: Option, #[serde(rename = "type")] pub type_field: String, - pub content: Option, - pub encoding: Option, - #[serde(rename = "_links")] - pub links: Links, -} - -#[derive(Default, Debug, Clone, Deserialize)] -pub struct Links { - #[serde(rename = "self")] - pub self_field: String, - pub git: String, - pub html: String, } diff --git a/core/src/services/gridfs/backend.rs b/core/src/services/gridfs/backend.rs index 26772ca870a..815c4a9dc6c 100644 --- a/core/src/services/gridfs/backend.rs +++ b/core/src/services/gridfs/backend.rs @@ -29,7 +29,9 @@ use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; -use crate::raw::new_std_io_error; +use crate::raw::{new_std_io_error, Access}; + +use crate::services::MongodbConfig; use crate::*; /// Config for Grid file system support. @@ -60,6 +62,12 @@ impl Debug for GridFsConfig { } } +impl Configurator for GridFsConfig { + fn into_builder(self) -> impl Builder { + GridFsBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct GridFsBuilder { @@ -143,14 +151,9 @@ impl GridFsBuilder { impl Builder for GridFsBuilder { const SCHEME: Scheme = Scheme::Mongodb; - type Accessor = GridFsBackend; - type Config = GridFsConfig; - - fn from_config(config: Self::Config) -> Self { - GridFsBuilder { config } - } + type Config = MongodbConfig; - fn build(&mut self) -> Result { + fn build(self) -> Result { let conn = match &self.config.connection_string.clone() { Some(v) => v.clone(), None => { diff --git a/core/src/services/hdfs/backend.rs b/core/src/services/hdfs/backend.rs index cd67a3a8944..1da994343c4 100644 --- a/core/src/services/hdfs/backend.rs +++ b/core/src/services/hdfs/backend.rs @@ -31,6 +31,7 @@ use super::lister::HdfsLister; use super::writer::HdfsWriter; use crate::raw::*; use crate::services::hdfs::reader::HdfsReader; + use crate::*; /// [Hadoop Distributed File System (HDFSâ„¢)](https://hadoop.apache.org/) support. @@ -70,6 +71,12 @@ impl Debug for HdfsConfig { } } +impl Configurator for HdfsConfig { + fn into_builder(self) -> impl Builder { + HdfsBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct HdfsBuilder { @@ -157,14 +164,9 @@ impl HdfsBuilder { impl Builder for HdfsBuilder { const SCHEME: Scheme = Scheme::Hdfs; - type Accessor = HdfsBackend; type Config = HdfsConfig; - fn from_config(config: Self::Config) -> Self { - HdfsBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let name_node = match &self.config.name_node { @@ -175,7 +177,7 @@ impl Builder for HdfsBuilder { } }; - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let mut builder = hdrs::ClientBuilder::new(name_node); @@ -197,7 +199,7 @@ impl Builder for HdfsBuilder { } } - let atomic_write_dir = self.config.atomic_write_dir.take(); + let atomic_write_dir = self.config.atomic_write_dir; // If atomic write dir is not exist, we must create it. if let Some(d) = &atomic_write_dir { @@ -208,7 +210,6 @@ impl Builder for HdfsBuilder { } } - debug!("backend build finished: {:?}", &self); Ok(HdfsBackend { root, atomic_write_dir, diff --git a/core/src/services/hdfs_native/backend.rs b/core/src/services/hdfs_native/backend.rs index 93507db5d3e..02cac864f8b 100644 --- a/core/src/services/hdfs_native/backend.rs +++ b/core/src/services/hdfs_native/backend.rs @@ -57,6 +57,12 @@ impl Debug for HdfsNativeConfig { } } +impl Configurator for HdfsNativeConfig { + fn into_builder(self) -> impl Builder { + HdfsNativeBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct HdfsNativeBuilder { @@ -111,14 +117,9 @@ impl HdfsNativeBuilder { impl Builder for HdfsNativeBuilder { const SCHEME: Scheme = Scheme::HdfsNative; - type Accessor = HdfsNativeBackend; type Config = HdfsNativeConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let url = match &self.config.url { @@ -129,14 +130,13 @@ impl Builder for HdfsNativeBuilder { } }; - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let client = hdfs_native::Client::new(url).map_err(parse_hdfs_error)?; // need to check if root dir exists, create if not - debug!("backend build finished: {:?}", &self); Ok(HdfsNativeBackend { root, client: Arc::new(client), diff --git a/core/src/services/http/backend.rs b/core/src/services/http/backend.rs index c4aa157dcec..5744df53922 100644 --- a/core/src/services/http/backend.rs +++ b/core/src/services/http/backend.rs @@ -30,6 +30,7 @@ use serde::{Deserialize, Serialize}; use super::error::parse_error; use crate::raw::*; + use crate::*; /// Config for Http service support. @@ -59,6 +60,15 @@ impl Debug for HttpConfig { } } +impl Configurator for HttpConfig { + fn into_builder(self) -> impl Builder { + HttpBuilder { + config: self, + http_client: None, + } + } +} + /// HTTP Read-only service support like [Nginx](https://www.nginx.com/) and [Caddy](https://caddyserver.com/). #[doc = include_str!("docs.md")] #[derive(Default)] @@ -144,17 +154,9 @@ impl HttpBuilder { impl Builder for HttpBuilder { const SCHEME: Scheme = Scheme::Http; - type Accessor = HttpBackend; type Config = HttpConfig; - fn from_config(config: Self::Config) -> Self { - HttpBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let endpoint = match &self.config.endpoint { @@ -165,10 +167,10 @@ impl Builder for HttpBuilder { } }; - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -188,7 +190,6 @@ impl Builder for HttpBuilder { auth = Some(format_authorization_by_bearer(token)?) } - debug!("backend build finished: {:?}", &self); Ok(HttpBackend { endpoint: endpoint.to_string(), authorization: auth, diff --git a/core/src/services/huggingface/backend.rs b/core/src/services/huggingface/backend.rs index 3ed4ff1a92b..f2a5c2307b4 100644 --- a/core/src/services/huggingface/backend.rs +++ b/core/src/services/huggingface/backend.rs @@ -30,6 +30,7 @@ use super::core::HuggingfaceStatus; use super::error::parse_error; use super::lister::HuggingfaceLister; use crate::raw::*; + use crate::*; /// Configuration for Huggingface service support. @@ -85,6 +86,12 @@ impl Debug for HuggingfaceConfig { } } +impl Configurator for HuggingfaceConfig { + fn into_builder(self) -> impl Builder { + HuggingfaceBuilder { config: self } + } +} + /// [Huggingface](https://huggingface.co/docs/huggingface_hub/package_reference/hf_api)'s API support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -170,15 +177,10 @@ impl HuggingfaceBuilder { impl Builder for HuggingfaceBuilder { const SCHEME: Scheme = Scheme::Huggingface; - type Accessor = HuggingfaceBackend; type Config = HuggingfaceConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - /// Build a HuggingfaceBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let repo_type = match self.config.repo_type.as_deref() { @@ -212,14 +214,13 @@ impl Builder for HuggingfaceBuilder { }; debug!("backend use revision: {}", &revision); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root: {}", &root); let token = self.config.token.as_ref().cloned(); let client = HttpClient::new()?; - debug!("backend build finished: {:?}", &self); Ok(HuggingfaceBackend { core: Arc::new(HuggingfaceCore { repo_type, diff --git a/core/src/services/icloud/backend.rs b/core/src/services/icloud/backend.rs index 2592b60577f..b66b5bd6087 100644 --- a/core/src/services/icloud/backend.rs +++ b/core/src/services/icloud/backend.rs @@ -26,6 +26,7 @@ use tokio::sync::Mutex; use super::core::*; use crate::raw::*; + use crate::*; /// Config for icloud services support. @@ -70,6 +71,15 @@ impl Debug for IcloudConfig { } } +impl Configurator for IcloudConfig { + fn into_builder(self) -> impl Builder { + IcloudBuilder { + config: self, + http_client: None, + } + } +} + /// [IcloudDrive](https://www.icloud.com/iclouddrive/) service support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -183,18 +193,10 @@ impl IcloudBuilder { impl Builder for IcloudBuilder { const SCHEME: Scheme = Scheme::Icloud; - type Accessor = IcloudBackend; type Config = IcloudConfig; - fn from_config(config: Self::Config) -> Self { - IcloudBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); let apple_id = match &self.config.apple_id { Some(apple_id) => Ok(apple_id.clone()), @@ -226,7 +228,7 @@ impl Builder for IcloudBuilder { .with_context("service", Scheme::Icloud)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/ipfs/backend.rs b/core/src/services/ipfs/backend.rs index ca6b400e867..ca509086f6a 100644 --- a/core/src/services/ipfs/backend.rs +++ b/core/src/services/ipfs/backend.rs @@ -22,6 +22,7 @@ use std::sync::Arc; use super::error::parse_error; use super::ipld::PBNode; use crate::raw::*; + use crate::*; use http::Request; use http::Response; @@ -41,6 +42,15 @@ pub struct IpfsConfig { pub root: Option, } +impl Configurator for IpfsConfig { + fn into_builder(self) -> impl Builder { + IpfsBuilder { + config: self, + http_client: None, + } + } +} + /// IPFS file system support based on [IPFS HTTP Gateway](https://docs.ipfs.tech/concepts/ipfs-gateway/). #[doc = include_str!("docs.md")] #[derive(Default, Clone, Debug)] @@ -99,20 +109,12 @@ impl IpfsBuilder { impl Builder for IpfsBuilder { const SCHEME: Scheme = Scheme::Ipfs; - type Accessor = IpfsBackend; type Config = IpfsConfig; - fn from_config(config: Self::Config) -> Self { - IpfsBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); if !root.starts_with("/ipfs/") && !root.starts_with("/ipns/") { return Err(Error::new( ErrorKind::ConfigInvalid, @@ -131,7 +133,7 @@ impl Builder for IpfsBuilder { }?; debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -140,7 +142,6 @@ impl Builder for IpfsBuilder { })? }; - debug!("backend build finished: {:?}", &self); Ok(IpfsBackend { root, endpoint, diff --git a/core/src/services/ipmfs/builder.rs b/core/src/services/ipmfs/builder.rs index 924b18b06ab..4c638a1a4a3 100644 --- a/core/src/services/ipmfs/builder.rs +++ b/core/src/services/ipmfs/builder.rs @@ -17,6 +17,7 @@ use super::backend::IpmfsBackend; use crate::raw::*; + use crate::*; use log::debug; use serde::{Deserialize, Serialize}; @@ -32,6 +33,15 @@ pub struct IpmfsConfig { pub endpoint: Option, } +impl Configurator for IpmfsConfig { + fn into_builder(self) -> impl Builder { + IpmfsBuilder { + config: self, + http_client: None, + } + } +} + /// IPFS file system support based on [IPFS MFS](https://docs.ipfs.tech/concepts/file-systems/) API. /// /// # Capabilities @@ -117,18 +127,10 @@ impl IpmfsBuilder { impl Builder for IpmfsBuilder { const SCHEME: Scheme = Scheme::Ipmfs; - type Accessor = IpmfsBackend; type Config = IpmfsConfig; - fn from_config(config: Self::Config) -> Self { - IpmfsBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let endpoint = self @@ -137,7 +139,7 @@ impl Builder for IpmfsBuilder { .clone() .unwrap_or_else(|| "http://localhost:5001".to_string()); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -146,7 +148,6 @@ impl Builder for IpmfsBuilder { })? }; - debug!("backend build finished: {:?}", &self); Ok(IpmfsBackend::new(root, client, endpoint)) } } diff --git a/core/src/services/koofr/backend.rs b/core/src/services/koofr/backend.rs index b68eaa169db..7815b988a3b 100644 --- a/core/src/services/koofr/backend.rs +++ b/core/src/services/koofr/backend.rs @@ -35,6 +35,7 @@ use super::lister::KoofrLister; use super::writer::KoofrWriter; use super::writer::KoofrWriters; use crate::raw::*; + use crate::*; /// Config for backblaze Koofr services support. @@ -65,6 +66,15 @@ impl Debug for KoofrConfig { } } +impl Configurator for KoofrConfig { + fn into_builder(self) -> impl Builder { + KoofrBuilder { + config: self, + http_client: None, + } + } +} + /// [Koofr](https://app.koofr.net/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -149,18 +159,10 @@ impl KoofrBuilder { impl Builder for KoofrBuilder { const SCHEME: Scheme = Scheme::Koofr; - type Accessor = KoofrBackend; type Config = KoofrConfig; - fn from_config(config: Self::Config) -> Self { - KoofrBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of KoofrBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -189,7 +191,7 @@ impl Builder for KoofrBuilder { .with_context("service", Scheme::Koofr)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/libsql/backend.rs b/core/src/services/libsql/backend.rs index 52f0ebcef92..2cec2319e61 100644 --- a/core/src/services/libsql/backend.rs +++ b/core/src/services/libsql/backend.rs @@ -79,6 +79,12 @@ impl Debug for LibsqlConfig { } } +impl Configurator for LibsqlConfig { + fn into_builder(self) -> impl Builder { + LibsqlBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct LibsqlBuilder { @@ -166,14 +172,9 @@ impl LibsqlBuilder { impl Builder for LibsqlBuilder { const SCHEME: Scheme = Scheme::Libsql; - type Accessor = LibsqlBackend; type Config = LibsqlConfig; - fn from_config(config: Self::Config) -> Self { - LibsqlBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let conn = self.get_connection_string()?; let table = match self.config.table.clone() { diff --git a/core/src/services/memcached/backend.rs b/core/src/services/memcached/backend.rs index cdb46a488cd..c1dde13e79a 100644 --- a/core/src/services/memcached/backend.rs +++ b/core/src/services/memcached/backend.rs @@ -25,6 +25,7 @@ use tokio::sync::OnceCell; use super::binary; use crate::raw::adapters::kv; use crate::raw::*; + use crate::*; /// Config for MemCached services support @@ -48,6 +49,12 @@ pub struct MemcachedConfig { pub default_ttl: Option, } +impl Configurator for MemcachedConfig { + fn into_builder(self) -> impl Builder { + MemcachedBuilder { config: self } + } +} + /// [Memcached](https://memcached.org/) service support. #[doc = include_str!("docs.md")] #[derive(Clone, Default)] @@ -97,14 +104,9 @@ impl MemcachedBuilder { impl Builder for MemcachedBuilder { const SCHEME: Scheme = Scheme::Memcached; - type Accessor = MemcachedBackend; type Config = MemcachedConfig; - fn from_config(config: Self::Config) -> Self { - MemcachedBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let endpoint = self.config.endpoint.clone().ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "endpoint is empty") .with_context("service", Scheme::Memcached) diff --git a/core/src/services/memory/backend.rs b/core/src/services/memory/backend.rs index 49cbfb25e2e..09b06cae171 100644 --- a/core/src/services/memory/backend.rs +++ b/core/src/services/memory/backend.rs @@ -23,6 +23,7 @@ use std::sync::Mutex; use serde::{Deserialize, Serialize}; use crate::raw::adapters::typed_kv; +use crate::raw::Access; use crate::*; ///Config for memory. @@ -34,6 +35,12 @@ pub struct MemoryConfig { pub root: Option, } +impl Configurator for MemoryConfig { + fn into_builder(self) -> impl Builder { + MemoryBuilder { config: self } + } +} + /// In memory service support. (BTreeMap Based) #[doc = include_str!("docs.md")] #[derive(Default)] @@ -51,14 +58,9 @@ impl MemoryBuilder { impl Builder for MemoryBuilder { const SCHEME: Scheme = Scheme::Memory; - type Accessor = MemoryBackend; type Config = MemoryConfig; - fn from_config(config: Self::Config) -> Self { - MemoryBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let adapter = Adapter { inner: Arc::new(Mutex::new(BTreeMap::default())), }; @@ -154,7 +156,6 @@ impl typed_kv::Adapter for Adapter { #[cfg(test)] mod tests { use super::*; - use crate::raw::*; #[test] fn test_accessor_metadata_name() { diff --git a/core/src/services/mini_moka/backend.rs b/core/src/services/mini_moka/backend.rs index 71374cb8dd5..73817c0fe3b 100644 --- a/core/src/services/mini_moka/backend.rs +++ b/core/src/services/mini_moka/backend.rs @@ -19,6 +19,7 @@ use std::fmt::Debug; use std::time::Duration; use crate::raw::adapters::typed_kv; +use crate::raw::Access; use crate::*; use log::debug; use mini_moka::sync::Cache; @@ -44,6 +45,12 @@ pub struct MiniMokaConfig { pub time_to_idle: Option, } +impl Configurator for MiniMokaConfig { + fn into_builder(self) -> impl Builder { + MiniMokaBuilder { config: self } + } +} + /// [mini-moka](https://github.com/moka-rs/mini-moka) backend support. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -85,14 +92,9 @@ impl MiniMokaBuilder { impl Builder for MiniMokaBuilder { const SCHEME: Scheme = Scheme::MiniMoka; - type Accessor = MiniMokaBackend; type Config = MiniMokaConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let mut builder: CacheBuilder = Cache::builder(); diff --git a/core/src/services/moka/backend.rs b/core/src/services/moka/backend.rs index 8ef470be5ea..fcad03513b4 100644 --- a/core/src/services/moka/backend.rs +++ b/core/src/services/moka/backend.rs @@ -25,6 +25,7 @@ use moka::sync::SegmentedCache; use serde::{Deserialize, Serialize}; use crate::raw::adapters::typed_kv; +use crate::raw::*; use crate::*; /// Config for Mokaservices support. @@ -64,6 +65,12 @@ impl Debug for MokaConfig { } } +impl Configurator for MokaConfig { + fn into_builder(self) -> impl Builder { + MokaBuilder { config: self } + } +} + /// [moka](https://github.com/moka-rs/moka) backend support. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -122,14 +129,9 @@ impl MokaBuilder { impl Builder for MokaBuilder { const SCHEME: Scheme = Scheme::Moka; - type Accessor = MokaBackend; type Config = MokaConfig; - fn from_config(config: Self::Config) -> Self { - MokaBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let mut builder: CacheBuilder = diff --git a/core/src/services/mongodb/backend.rs b/core/src/services/mongodb/backend.rs index 75cdd2dc224..a0a9b740457 100644 --- a/core/src/services/mongodb/backend.rs +++ b/core/src/services/mongodb/backend.rs @@ -27,6 +27,7 @@ use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; use crate::raw::adapters::kv; +use crate::raw::Access; use crate::*; /// Config for Mongodb service support. @@ -61,6 +62,12 @@ impl Debug for MongodbConfig { } } +impl Configurator for MongodbConfig { + fn into_builder(self) -> impl Builder { + MongodbBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct MongodbBuilder { @@ -151,14 +158,9 @@ impl MongodbBuilder { impl Builder for MongodbBuilder { const SCHEME: Scheme = Scheme::Mongodb; - type Accessor = MongodbBackend; type Config = MongodbConfig; - fn from_config(config: Self::Config) -> Self { - MongodbBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let conn = match &self.config.connection_string.clone() { Some(v) => v.clone(), None => { diff --git a/core/src/services/monoiofs/backend.rs b/core/src/services/monoiofs/backend.rs index c4bb92dfa97..2ef3c060903 100644 --- a/core/src/services/monoiofs/backend.rs +++ b/core/src/services/monoiofs/backend.rs @@ -39,6 +39,12 @@ pub struct MonoiofsConfig { pub root: Option, } +impl Configurator for MonoiofsConfig { + fn into_builder(self) -> impl Builder { + MonoiofsBuilder { config: self } + } +} + /// File system support via [`monoio`]. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -62,15 +68,10 @@ impl MonoiofsBuilder { impl Builder for MonoiofsBuilder { const SCHEME: Scheme = Scheme::Monoiofs; - type Accessor = MonoiofsBackend; type Config = MonoiofsConfig; - fn from_config(config: Self::Config) -> Self { - MonoiofsBuilder { config } - } - - fn build(&mut self) -> Result { - let root = self.config.root.take().map(PathBuf::from).ok_or( + fn build(self) -> Result { + let root = self.config.root.map(PathBuf::from).ok_or( Error::new(ErrorKind::ConfigInvalid, "root is not specified") .with_operation("Builder::build"), )?; diff --git a/core/src/services/mysql/backend.rs b/core/src/services/mysql/backend.rs index cc0f1b591da..3301a7368f1 100644 --- a/core/src/services/mysql/backend.rs +++ b/core/src/services/mysql/backend.rs @@ -60,6 +60,12 @@ impl Debug for MysqlConfig { } } +impl Configurator for MysqlConfig { + fn into_builder(self) -> impl Builder { + MysqlBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct MysqlBuilder { @@ -137,14 +143,9 @@ impl MysqlBuilder { impl Builder for MysqlBuilder { const SCHEME: Scheme = Scheme::Mysql; - type Accessor = MySqlBackend; type Config = MysqlConfig; - fn from_config(config: Self::Config) -> Self { - MysqlBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let conn = match self.config.connection_string.clone() { Some(v) => v, None => { diff --git a/core/src/services/obs/backend.rs b/core/src/services/obs/backend.rs index 742f4a22c3e..bd332f607c1 100644 --- a/core/src/services/obs/backend.rs +++ b/core/src/services/obs/backend.rs @@ -63,6 +63,15 @@ impl Debug for ObsConfig { } } +impl Configurator for ObsConfig { + fn into_builder(self) -> impl Builder { + ObsBuilder { + config: self, + http_client: None, + } + } +} + /// Huawei-Cloud Object Storage Service (OBS) support #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -153,20 +162,12 @@ impl ObsBuilder { impl Builder for ObsBuilder { const SCHEME: Scheme = Scheme::Obs; - type Accessor = ObsBackend; type Config = ObsConfig; - fn from_config(config: Self::Config) -> Self { - ObsBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); let bucket = match &self.config.bucket { @@ -203,7 +204,7 @@ impl Builder for ObsBuilder { }; debug!("backend use endpoint {}", &endpoint); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -216,11 +217,11 @@ impl Builder for ObsBuilder { // Load cfg from env first. cfg = cfg.from_env(); - if let Some(v) = self.config.access_key_id.take() { + if let Some(v) = self.config.access_key_id { cfg.access_key_id = Some(v); } - if let Some(v) = self.config.secret_access_key.take() { + if let Some(v) = self.config.secret_access_key { cfg.secret_access_key = Some(v); } diff --git a/core/src/services/onedrive/builder.rs b/core/src/services/onedrive/builder.rs index 7c536531ee9..32544f2d114 100644 --- a/core/src/services/onedrive/builder.rs +++ b/core/src/services/onedrive/builder.rs @@ -22,8 +22,8 @@ use log::debug; use serde::{Deserialize, Serialize}; use super::backend::OnedriveBackend; -use crate::raw::normalize_root; use crate::raw::HttpClient; +use crate::raw::{normalize_root, Access}; use crate::Scheme; use crate::*; @@ -46,6 +46,15 @@ impl Debug for OnedriveConfig { } } +impl Configurator for OnedriveConfig { + fn into_builder(self) -> impl Builder { + OnedriveBuilder { + config: self, + http_client: None, + } + } +} + /// [OneDrive](https://onedrive.com) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -91,21 +100,13 @@ impl OnedriveBuilder { impl Builder for OnedriveBuilder { const SCHEME: Scheme = Scheme::Onedrive; - type Accessor = OnedriveBackend; type Config = OnedriveConfig; - fn from_config(config: Self::Config) -> Self { - OnedriveBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/oss/backend.rs b/core/src/services/oss/backend.rs index e24e404b7c7..591851de2c5 100644 --- a/core/src/services/oss/backend.rs +++ b/core/src/services/oss/backend.rs @@ -83,6 +83,15 @@ impl Debug for OssConfig { } } +impl Configurator for OssConfig { + fn into_builder(self) -> impl Builder { + OssBuilder { + config: self, + http_client: None, + } + } +} + /// Aliyun Object Storage Service (OSS) support #[doc = include_str!("docs.md")] #[derive(Default)] @@ -278,17 +287,9 @@ impl OssBuilder { impl Builder for OssBuilder { const SCHEME: Scheme = Scheme::Oss; - type Accessor = OssBackend; type Config = OssConfig; - fn from_config(config: Self::Config) -> Self { - OssBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -303,15 +304,6 @@ impl Builder for OssBuilder { ), }?; - let client = if let Some(client) = self.http_client.take() { - client - } else { - HttpClient::new().map_err(|err| { - err.with_operation("Builder::build") - .with_context("service", Scheme::Oss) - })? - }; - // Retrieve endpoint and host by parsing the endpoint option and bucket. If presign_endpoint is not // set, take endpoint as default presign_endpoint. let (endpoint, host) = self.parse_endpoint(&self.config.endpoint, bucket)?; @@ -345,14 +337,23 @@ impl Builder for OssBuilder { // Load cfg from env first. cfg = cfg.from_env(); - if let Some(v) = self.config.access_key_id.take() { + if let Some(v) = self.config.access_key_id { cfg.access_key_id = Some(v); } - if let Some(v) = self.config.access_key_secret.take() { + if let Some(v) = self.config.access_key_secret { cfg.access_key_secret = Some(v); } + let client = if let Some(client) = self.http_client { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::Oss) + })? + }; + let loader = AliyunLoader::new(client.client(), cfg); let signer = AliyunOssSigner::new(bucket); @@ -361,7 +362,6 @@ impl Builder for OssBuilder { .config .batch_max_operations .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS); - debug!("Backend build finished"); Ok(OssBackend { core: Arc::new(OssCore { diff --git a/core/src/services/oss/core.rs b/core/src/services/oss/core.rs index 00c0e1b9559..12f5157686d 100644 --- a/core/src/services/oss/core.rs +++ b/core/src/services/oss/core.rs @@ -469,19 +469,6 @@ impl OssCore { self.send(req).await } - pub async fn oss_put_object( - &self, - path: &str, - size: Option, - args: &OpWrite, - body: Buffer, - ) -> Result> { - let mut req = self.oss_put_object_request(path, size, args, body, false)?; - - self.sign(&mut req).await?; - self.send(req).await - } - pub async fn oss_copy_object(&self, from: &str, to: &str) -> Result> { let source = build_abs_path(&self.root, from); let target = build_abs_path(&self.root, to); diff --git a/core/src/services/pcloud/backend.rs b/core/src/services/pcloud/backend.rs index 35bf1022ce7..94b051d5f89 100644 --- a/core/src/services/pcloud/backend.rs +++ b/core/src/services/pcloud/backend.rs @@ -63,6 +63,15 @@ impl Debug for PcloudConfig { } } +impl Configurator for PcloudConfig { + fn into_builder(self) -> impl Builder { + PcloudBuilder { + config: self, + http_client: None, + } + } +} + /// [pCloud](https://www.pcloud.com/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -146,18 +155,10 @@ impl PcloudBuilder { impl Builder for PcloudBuilder { const SCHEME: Scheme = Scheme::Pcloud; - type Accessor = PcloudBackend; type Config = PcloudConfig; - fn from_config(config: Self::Config) -> Self { - PcloudBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of PcloudBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -186,7 +187,7 @@ impl Builder for PcloudBuilder { .with_context("service", Scheme::Pcloud)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/persy/backend.rs b/core/src/services/persy/backend.rs index 5af2ef7e5ab..8f7b1e2847c 100644 --- a/core/src/services/persy/backend.rs +++ b/core/src/services/persy/backend.rs @@ -25,6 +25,7 @@ use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; + use crate::Builder; use crate::Error; use crate::ErrorKind; @@ -44,6 +45,12 @@ pub struct PersyConfig { pub index: Option, } +impl Configurator for PersyConfig { + fn into_builder(self) -> impl Builder { + PersyBuilder { config: self } + } +} + /// persy service support. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -73,27 +80,22 @@ impl PersyBuilder { impl Builder for PersyBuilder { const SCHEME: Scheme = Scheme::Persy; - type Accessor = PersyBackend; type Config = PersyConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { - let datafile_path = self.config.datafile.take().ok_or_else(|| { + fn build(self) -> Result { + let datafile_path = self.config.datafile.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datafile is required but not set") .with_context("service", Scheme::Persy) })?; - let segment_name = self.config.segment.take().ok_or_else(|| { + let segment_name = self.config.segment.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "segment is required but not set") .with_context("service", Scheme::Persy) })?; let segment = segment_name.clone(); - let index_name = self.config.index.take().ok_or_else(|| { + let index_name = self.config.index.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "index is required but not set") .with_context("service", Scheme::Persy) })?; diff --git a/core/src/services/postgresql/backend.rs b/core/src/services/postgresql/backend.rs index 4a112424cc0..7de85295361 100644 --- a/core/src/services/postgresql/backend.rs +++ b/core/src/services/postgresql/backend.rs @@ -28,6 +28,7 @@ use tokio_postgres::Config; use crate::raw::adapters::kv; use crate::raw::*; + use crate::*; /// Config for PostgreSQL services support. @@ -67,6 +68,12 @@ impl Debug for PostgresqlConfig { } } +impl Configurator for PostgresqlConfig { + fn into_builder(self) -> impl Builder { + PostgresqlBuilder { config: self } + } +} + /// [PostgreSQL](https://www.postgresql.org/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -162,14 +169,9 @@ impl PostgresqlBuilder { impl Builder for PostgresqlBuilder { const SCHEME: Scheme = Scheme::Postgresql; - type Accessor = PostgresqlBackend; type Config = PostgresqlConfig; - fn from_config(config: Self::Config) -> Self { - PostgresqlBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let conn = match self.config.connection_string.clone() { Some(v) => v, None => { diff --git a/core/src/services/redb/backend.rs b/core/src/services/redb/backend.rs index 8a782937ac2..53860f3c630 100644 --- a/core/src/services/redb/backend.rs +++ b/core/src/services/redb/backend.rs @@ -44,6 +44,12 @@ pub struct RedbConfig { pub table: Option, } +impl Configurator for RedbConfig { + fn into_builder(self) -> impl Builder { + RedbBuilder { config: self } + } +} + /// Redb service support. #[doc = include_str!("docs.md")] #[derive(Default, Debug)] @@ -73,20 +79,15 @@ impl RedbBuilder { impl Builder for RedbBuilder { const SCHEME: Scheme = Scheme::Redb; - type Accessor = RedbBackend; type Config = RedbConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - - fn build(&mut self) -> Result { - let datadir_path = self.config.datadir.take().ok_or_else(|| { + fn build(self) -> Result { + let datadir_path = self.config.datadir.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Redb) })?; - let table_name = self.config.table.take().ok_or_else(|| { + let table_name = self.config.table.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "table is required but not set") .with_context("service", Scheme::Redb) })?; diff --git a/core/src/services/redis/backend.rs b/core/src/services/redis/backend.rs index 84375c66fc6..62bf03c7789 100644 --- a/core/src/services/redis/backend.rs +++ b/core/src/services/redis/backend.rs @@ -97,6 +97,12 @@ impl Debug for RedisConfig { } } +impl Configurator for RedisConfig { + fn into_builder(self) -> impl Builder { + RedisBuilder { config: self } + } +} + /// [Redis](https://redis.io/) services support. #[doc = include_str!("docs.md")] #[derive(Clone, Default)] @@ -192,14 +198,9 @@ impl RedisBuilder { impl Builder for RedisBuilder { const SCHEME: Scheme = Scheme::Redis; - type Accessor = RedisBackend; type Config = RedisConfig; - fn from_config(config: Self::Config) -> Self { - RedisBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let root = normalize_root( self.config .root diff --git a/core/src/services/rocksdb/backend.rs b/core/src/services/rocksdb/backend.rs index ec14a47e71e..3cb15f491b1 100644 --- a/core/src/services/rocksdb/backend.rs +++ b/core/src/services/rocksdb/backend.rs @@ -41,6 +41,12 @@ pub struct RocksdbConfig { pub root: Option, } +impl Configurator for RocksdbConfig { + fn into_builder(self) -> impl Builder { + RocksdbBuilder { config: self } + } +} + /// RocksDB service support. #[doc = include_str!("docs.md")] #[derive(Clone, Default)] @@ -68,15 +74,10 @@ impl RocksdbBuilder { impl Builder for RocksdbBuilder { const SCHEME: Scheme = Scheme::Rocksdb; - type Accessor = RocksdbBackend; type Config = RocksdbConfig; - fn from_config(config: Self::Config) -> Self { - RocksdbBuilder { config } - } - - fn build(&mut self) -> Result { - let path = self.config.datadir.take().ok_or_else(|| { + fn build(self) -> Result { + let path = self.config.datadir.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Rocksdb) })?; diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 9113c3da481..e2893bdd83d 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -224,6 +224,16 @@ impl Debug for S3Config { } } +impl Configurator for S3Config { + fn into_builder(self) -> impl Builder { + S3Builder { + config: self, + customized_credential_load: None, + http_client: None, + } + } +} + /// Aws S3 and compatible services (including minio, digitalocean space, Tencent Cloud Object Storage(COS) and so on) support. /// For more information about s3-compatible services, refer to [Compatible Services](#compatible-services). #[doc = include_str!("docs.md")] @@ -803,18 +813,9 @@ impl S3Builder { impl Builder for S3Builder { const SCHEME: Scheme = Scheme::S3; - type Accessor = S3Backend; type Config = S3Config; - fn from_config(config: Self::Config) -> Self { - S3Builder { - config, - customized_credential_load: None, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -889,15 +890,6 @@ impl Builder for S3Builder { } }; - let client = if let Some(client) = self.http_client.take() { - client - } else { - HttpClient::new().map_err(|err| { - err.with_operation("Builder::build") - .with_context("service", Scheme::S3) - })? - }; - // This is our current config. let mut cfg = AwsConfig::default(); if !self.config.disable_config_load { @@ -908,7 +900,7 @@ impl Builder for S3Builder { } } - if let Some(v) = self.config.region.take() { + if let Some(v) = self.config.region.clone() { cfg.region = Some(v); } if cfg.region.is_none() { @@ -928,24 +920,33 @@ impl Builder for S3Builder { debug!("backend use endpoint: {endpoint}"); // Setting all value from user input if available. - if let Some(v) = self.config.access_key_id.take() { + if let Some(v) = self.config.access_key_id { cfg.access_key_id = Some(v) } - if let Some(v) = self.config.secret_access_key.take() { + if let Some(v) = self.config.secret_access_key { cfg.secret_access_key = Some(v) } - if let Some(v) = self.config.session_token.take() { + if let Some(v) = self.config.session_token { cfg.session_token = Some(v) } + let client = if let Some(client) = self.http_client { + client + } else { + HttpClient::new().map_err(|err| { + err.with_operation("Builder::build") + .with_context("service", Scheme::S3) + })? + }; + let mut loader: Option> = None; // If customized_credential_load is set, we will use it. - if let Some(v) = self.customized_credential_load.take() { + if let Some(v) = self.customized_credential_load { loader = Some(v); } // If role_arn is set, we must use AssumeRoleLoad. - if let Some(role_arn) = self.config.role_arn.take() { + if let Some(role_arn) = self.config.role_arn { // use current env as source credential loader. let default_loader = AwsDefaultLoader::new(client.client(), cfg.clone()); @@ -991,7 +992,7 @@ impl Builder for S3Builder { .config .batch_max_operations .unwrap_or(DEFAULT_BATCH_MAX_OPERATIONS); - debug!("backend build finished"); + Ok(S3Backend { core: Arc::new(S3Core { bucket: bucket.to_string(), diff --git a/core/src/services/seafile/backend.rs b/core/src/services/seafile/backend.rs index eb13f98fa2d..fcafd330ba8 100644 --- a/core/src/services/seafile/backend.rs +++ b/core/src/services/seafile/backend.rs @@ -28,15 +28,15 @@ use tokio::sync::RwLock; use super::core::parse_dir_detail; use super::core::parse_file_detail; use super::core::SeafileCore; +use super::core::SeafileSigner; use super::error::parse_error; use super::lister::SeafileLister; use super::writer::SeafileWriter; use super::writer::SeafileWriters; use crate::raw::*; -use crate::services::seafile::core::SeafileSigner; use crate::*; -/// Config for backblaze seafile services support. +/// Config for seafile services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] #[non_exhaustive] @@ -70,6 +70,15 @@ impl Debug for SeafileConfig { } } +impl Configurator for SeafileConfig { + fn into_builder(self) -> impl Builder { + SeafileBuilder { + config: self, + http_client: None, + } + } +} + /// [seafile](https://www.seafile.com) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -164,18 +173,10 @@ impl SeafileBuilder { impl Builder for SeafileBuilder { const SCHEME: Scheme = Scheme::Seafile; - type Accessor = SeafileBackend; type Config = SeafileConfig; - fn from_config(config: Self::Config) -> Self { - SeafileBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of SeafileBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -211,7 +212,7 @@ impl Builder for SeafileBuilder { .with_context("service", Scheme::Seafile)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/seafile/core.rs b/core/src/services/seafile/core.rs index fb92c67bb6b..ddb784e02e5 100644 --- a/core/src/services/seafile/core.rs +++ b/core/src/services/seafile/core.rs @@ -293,36 +293,6 @@ impl SeafileCore { } } - /// create dir - pub async fn create_dir(&self, path: &str) -> Result<()> { - let path = build_abs_path(&self.root, path); - let path = format!("/{}", &path[..path.len() - 1]); - let path = percent_encode_path(&path); - - let auth_info = self.get_auth_info().await?; - - let req = Request::post(format!( - "{}/api2/repos/{}/dir/?p={}", - self.endpoint, auth_info.repo_id, path, - )); - - let body = "operation=mkdir".to_string(); - - let req = req - .header(header::AUTHORIZATION, format!("Token {}", auth_info.token)) - .header(header::CONTENT_TYPE, "application/x-www-form-urlencoded") - .body(Buffer::from(Bytes::from(body))) - .map_err(new_request_build_error)?; - - let resp = self.send(req).await?; - let status = resp.status(); - - match status { - StatusCode::CREATED => Ok(()), - _ => Err(parse_error(resp).await?), - } - } - /// delete file or dir pub async fn delete(&self, path: &str) -> Result<()> { let path = build_abs_path(&self.root, path); diff --git a/core/src/services/sftp/backend.rs b/core/src/services/sftp/backend.rs index 76a26a1475b..7df9c5a1598 100644 --- a/core/src/services/sftp/backend.rs +++ b/core/src/services/sftp/backend.rs @@ -71,6 +71,12 @@ impl Debug for SftpConfig { } } +impl Configurator for SftpConfig { + fn into_builder(self) -> impl Builder { + SftpBuilder { config: self } + } +} + /// SFTP services support. (only works on unix) /// /// If you are interested in working on windows, pl ease refer to [this](https://github.com/apache/opendal/issues/2963) issue. @@ -167,14 +173,9 @@ impl SftpBuilder { impl Builder for SftpBuilder { const SCHEME: Scheme = Scheme::Sftp; - type Accessor = SftpBackend; type Config = SftpConfig; - fn from_config(config: Self::Config) -> Self { - SftpBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("sftp backend build started: {:?}", &self); let endpoint = match self.config.endpoint.clone() { Some(v) => v, diff --git a/core/src/services/sled/backend.rs b/core/src/services/sled/backend.rs index 26e4fee5de2..fa3ad7b83ef 100644 --- a/core/src/services/sled/backend.rs +++ b/core/src/services/sled/backend.rs @@ -24,6 +24,7 @@ use tokio::task; use crate::raw::adapters::kv; use crate::raw::*; + use crate::Builder; use crate::Error; use crate::ErrorKind; @@ -56,6 +57,12 @@ impl Debug for SledConfig { } } +impl Configurator for SledConfig { + fn into_builder(self) -> impl Builder { + SledBuilder { config: self } + } +} + /// Sled services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -93,15 +100,10 @@ impl SledBuilder { impl Builder for SledBuilder { const SCHEME: Scheme = Scheme::Sled; - type Accessor = SledBackend; type Config = SledConfig; - fn from_config(config: Self::Config) -> Self { - SledBuilder { config } - } - - fn build(&mut self) -> Result { - let datadir_path = self.config.datadir.take().ok_or_else(|| { + fn build(self) -> Result { + let datadir_path = self.config.datadir.ok_or_else(|| { Error::new(ErrorKind::ConfigInvalid, "datadir is required but not set") .with_context("service", Scheme::Sled) })?; @@ -117,7 +119,6 @@ impl Builder for SledBuilder { let tree_name = self .config .tree - .take() .unwrap_or_else(|| DEFAULT_TREE_ID.to_string()); let tree = db.open_tree(&tree_name).map_err(|e| { diff --git a/core/src/services/sqlite/backend.rs b/core/src/services/sqlite/backend.rs index 06fdddc10ee..9b4d56fe4dc 100644 --- a/core/src/services/sqlite/backend.rs +++ b/core/src/services/sqlite/backend.rs @@ -75,6 +75,12 @@ impl Debug for SqliteConfig { } } +impl Configurator for SqliteConfig { + fn into_builder(self) -> impl Builder { + SqliteBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct SqliteBuilder { @@ -150,14 +156,9 @@ impl SqliteBuilder { impl Builder for SqliteBuilder { const SCHEME: Scheme = Scheme::Sqlite; - type Accessor = SqliteBackend; type Config = SqliteConfig; - fn from_config(config: Self::Config) -> Self { - SqliteBuilder { config } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { let connection_string = match self.config.connection_string.clone() { Some(v) => v, None => { diff --git a/core/src/services/supabase/backend.rs b/core/src/services/supabase/backend.rs index d14690df4fa..324d7298e12 100644 --- a/core/src/services/supabase/backend.rs +++ b/core/src/services/supabase/backend.rs @@ -23,12 +23,11 @@ use http::StatusCode; use log::debug; use serde::{Deserialize, Serialize}; -use crate::raw::*; -use crate::*; - use super::core::*; use super::error::parse_error; use super::writer::*; +use crate::raw::*; +use crate::*; /// Config for supabase service support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -54,6 +53,15 @@ impl Debug for SupabaseConfig { } } +impl Configurator for SupabaseConfig { + fn into_builder(self) -> impl Builder { + SupabaseBuilder { + config: self, + http_client: None, + } + } +} + /// [Supabase](https://supabase.com/) service support #[doc = include_str!("docs.md")] #[derive(Default)] @@ -124,25 +132,17 @@ impl SupabaseBuilder { impl Builder for SupabaseBuilder { const SCHEME: Scheme = Scheme::Supabase; - type Accessor = SupabaseBackend; type Config = SupabaseConfig; - fn from_config(config: Self::Config) -> Self { - SupabaseBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + fn build(self) -> Result { + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", &root); let bucket = &self.config.bucket; - let endpoint = self.config.endpoint.take().unwrap_or_default(); + let endpoint = self.config.endpoint.unwrap_or_default(); - let http_client = if let Some(client) = self.http_client.take() { + let http_client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/surrealdb/backend.rs b/core/src/services/surrealdb/backend.rs index b9e2066e206..ed673713593 100644 --- a/core/src/services/surrealdb/backend.rs +++ b/core/src/services/surrealdb/backend.rs @@ -26,13 +26,8 @@ use surrealdb::Surreal; use tokio::sync::OnceCell; use crate::raw::adapters::kv; -use crate::raw::normalize_root; -use crate::Buffer; -use crate::Builder; -use crate::Capability; -use crate::Error; -use crate::ErrorKind; -use crate::Scheme; +use crate::raw::{normalize_root, Access}; +use crate::*; /// Config for Surrealdb services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -75,6 +70,13 @@ impl Debug for SurrealdbConfig { .finish() } } + +impl Configurator for SurrealdbConfig { + fn into_builder(self) -> impl Builder { + SurrealdbBuilder { config: self } + } +} + #[doc = include_str!("docs.md")] #[derive(Default)] pub struct SurrealdbBuilder { @@ -180,14 +182,9 @@ impl SurrealdbBuilder { impl Builder for SurrealdbBuilder { const SCHEME: Scheme = Scheme::Surrealdb; - type Accessor = SurrealdbBackend; type Config = SurrealdbConfig; - fn from_config(config: Self::Config) -> Self { - SurrealdbBuilder { config } - } - - fn build(&mut self) -> crate::Result { + fn build(self) -> Result { let connection_string = match self.config.connection_string.clone() { Some(v) => v, None => { diff --git a/core/src/services/swift/backend.rs b/core/src/services/swift/backend.rs index 037c3895c43..938aa13d3f8 100644 --- a/core/src/services/swift/backend.rs +++ b/core/src/services/swift/backend.rs @@ -24,13 +24,12 @@ use http::StatusCode; use log::debug; use serde::{Deserialize, Serialize}; -use crate::raw::*; -use crate::*; - use super::core::*; use super::error::parse_error; use super::lister::SwiftLister; use super::writer::SwiftWriter; +use crate::raw::*; +use crate::*; /// Config for OpenStack Swift support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -63,6 +62,12 @@ impl Debug for SwiftConfig { } } +impl Configurator for SwiftConfig { + fn into_builder(self) -> impl Builder { + SwiftBuilder { config: self } + } +} + /// [OpenStack Swift](https://docs.openstack.org/api-ref/object-store/#)'s REST API support. /// For more information about swift-compatible services, refer to [Compatible Services](#compatible-services). #[doc = include_str!("docs.md")] @@ -136,21 +141,16 @@ impl SwiftBuilder { impl Builder for SwiftBuilder { const SCHEME: Scheme = Scheme::Swift; - type Accessor = SwiftBackend; type Config = SwiftConfig; - fn from_config(config: Self::Config) -> Self { - SwiftBuilder { config } - } - /// Build a SwiftBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {}", root); - let endpoint = match self.config.endpoint.take() { + let endpoint = match self.config.endpoint { Some(endpoint) => { if endpoint.starts_with("http") { endpoint @@ -167,7 +167,7 @@ impl Builder for SwiftBuilder { }; debug!("backend use endpoint: {}", &endpoint); - let container = match self.config.container.take() { + let container = match self.config.container { Some(container) => container, None => { return Err(Error::new( @@ -178,11 +178,10 @@ impl Builder for SwiftBuilder { }; debug!("backend use container: {}", &container); - let token = self.config.token.take().unwrap_or_default(); + let token = self.config.token.unwrap_or_default(); let client = HttpClient::new()?; - debug!("backend build finished: {:?}", &self); Ok(SwiftBackend { core: Arc::new(SwiftCore { root, diff --git a/core/src/services/tikv/backend.rs b/core/src/services/tikv/backend.rs index f707719799f..87d7fc07b77 100644 --- a/core/src/services/tikv/backend.rs +++ b/core/src/services/tikv/backend.rs @@ -24,6 +24,7 @@ use tikv_client::RawClient; use tokio::sync::OnceCell; use crate::raw::adapters::kv; +use crate::raw::Access; use crate::Builder; use crate::Capability; use crate::Error; @@ -61,6 +62,12 @@ impl Debug for TikvConfig { } } +impl Configurator for TikvConfig { + fn into_builder(self) -> impl Builder { + TikvBuilder { config: self } + } +} + /// TiKV backend builder #[doc = include_str!("docs.md")] #[derive(Clone, Default)] @@ -119,15 +126,10 @@ impl TikvBuilder { impl Builder for TikvBuilder { const SCHEME: Scheme = Scheme::Tikv; - type Accessor = TikvBackend; type Config = TikvConfig; - fn from_config(config: Self::Config) -> Self { - TikvBuilder { config } - } - - fn build(&mut self) -> Result { - let endpoints = self.config.endpoints.take().ok_or_else(|| { + fn build(self) -> Result { + let endpoints = self.config.endpoints.ok_or_else(|| { Error::new( ErrorKind::ConfigInvalid, "endpoints is required but not set", diff --git a/core/src/services/upyun/backend.rs b/core/src/services/upyun/backend.rs index afa3ac39784..e6687e39fd9 100644 --- a/core/src/services/upyun/backend.rs +++ b/core/src/services/upyun/backend.rs @@ -24,14 +24,12 @@ use http::StatusCode; use log::debug; use serde::{Deserialize, Serialize}; -use super::core::parse_info; -use super::core::UpyunCore; +use super::core::*; use super::error::parse_error; use super::lister::UpyunLister; use super::writer::UpyunWriter; use super::writer::UpyunWriters; use crate::raw::*; -use crate::services::upyun::core::UpyunSigner; use crate::*; /// Config for backblaze upyun services support. @@ -63,6 +61,15 @@ impl Debug for UpyunConfig { } } +impl Configurator for UpyunConfig { + fn into_builder(self) -> impl Builder { + UpyunBuilder { + config: self, + http_client: None, + } + } +} + /// [upyun](https://www.upyun.com/products/file-storage) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -144,18 +151,10 @@ impl UpyunBuilder { impl Builder for UpyunBuilder { const SCHEME: Scheme = Scheme::Upyun; - type Accessor = UpyunBackend; type Config = UpyunConfig; - fn from_config(config: Self::Config) -> Self { - UpyunBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of UpyunBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -184,7 +183,7 @@ impl Builder for UpyunBuilder { .with_context("service", Scheme::Upyun)), }?; - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -202,7 +201,6 @@ impl Builder for UpyunBuilder { core: Arc::new(UpyunCore { root, operator, - password, bucket: self.config.bucket.clone(), signer, client, diff --git a/core/src/services/upyun/core.rs b/core/src/services/upyun/core.rs index 8f6f9f905db..aec51f4c6a1 100644 --- a/core/src/services/upyun/core.rs +++ b/core/src/services/upyun/core.rs @@ -59,8 +59,6 @@ pub struct UpyunCore { pub root: String, /// The endpoint of this backend. pub operator: String, - /// The password id of this backend. - pub password: String, /// The bucket of this backend. pub bucket: String, diff --git a/core/src/services/vercel_artifacts/builder.rs b/core/src/services/vercel_artifacts/builder.rs index 614c09df971..5b8e9e56f7c 100644 --- a/core/src/services/vercel_artifacts/builder.rs +++ b/core/src/services/vercel_artifacts/builder.rs @@ -19,12 +19,11 @@ use std::fmt::{Debug, Formatter}; use serde::{Deserialize, Serialize}; -use crate::raw::HttpClient; +use super::backend::VercelArtifactsBackend; +use crate::raw::{Access, HttpClient}; use crate::Scheme; use crate::*; -use super::backend::VercelArtifactsBackend; - /// Config for Vercel Cache support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] #[serde(default)] @@ -42,6 +41,15 @@ impl Debug for VercelArtifactsConfig { } } +impl Configurator for VercelArtifactsConfig { + fn into_builder(self) -> impl Builder { + VercelArtifactsBuilder { + config: self, + http_client: None, + } + } +} + /// [Vercel Cache](https://vercel.com/docs/concepts/monorepos/remote-caching) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -81,18 +89,10 @@ impl VercelArtifactsBuilder { impl Builder for VercelArtifactsBuilder { const SCHEME: Scheme = Scheme::VercelArtifacts; - type Accessor = VercelArtifactsBackend; type Config = VercelArtifactsConfig; - fn from_config(config: Self::Config) -> Self { - Self { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { - let client = if let Some(client) = self.http_client.take() { + fn build(self) -> Result { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/vercel_blob/backend.rs b/core/src/services/vercel_blob/backend.rs index befdf06601f..98b96536f92 100644 --- a/core/src/services/vercel_blob/backend.rs +++ b/core/src/services/vercel_blob/backend.rs @@ -58,6 +58,15 @@ impl Debug for VercelBlobConfig { } } +impl Configurator for VercelBlobConfig { + fn into_builder(self) -> impl Builder { + VercelBlobBuilder { + config: self, + http_client: None, + } + } +} + /// [VercelBlob](https://vercel.com/docs/storage/vercel-blob) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -114,18 +123,10 @@ impl VercelBlobBuilder { impl Builder for VercelBlobBuilder { const SCHEME: Scheme = Scheme::VercelBlob; - type Accessor = VercelBlobBackend; type Config = VercelBlobConfig; - fn from_config(config: Self::Config) -> Self { - VercelBlobBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of VercelBlobBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -138,7 +139,7 @@ impl Builder for VercelBlobBuilder { .with_context("service", Scheme::VercelBlob)); } - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/services/webdav/backend.rs b/core/src/services/webdav/backend.rs index 5730b26713f..caed4a02f79 100644 --- a/core/src/services/webdav/backend.rs +++ b/core/src/services/webdav/backend.rs @@ -30,6 +30,7 @@ use super::error::parse_error; use super::lister::WebdavLister; use super::writer::WebdavWriter; use crate::raw::*; + use crate::*; /// Config for [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. @@ -63,6 +64,15 @@ impl Debug for WebdavConfig { } } +impl Configurator for WebdavConfig { + fn into_builder(self) -> impl Builder { + WebdavBuilder { + config: self, + http_client: None, + } + } +} + /// [WebDAV](https://datatracker.ietf.org/doc/html/rfc4918) backend support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -150,17 +160,9 @@ impl WebdavBuilder { impl Builder for WebdavBuilder { const SCHEME: Scheme = Scheme::Webdav; - type Accessor = WebdavBackend; type Config = WebdavConfig; - fn from_config(config: Self::Config) -> Self { - WebdavBuilder { - config, - http_client: None, - } - } - - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let endpoint = match &self.config.endpoint { @@ -184,7 +186,7 @@ impl Builder for WebdavBuilder { let root = normalize_root(&self.config.root.clone().unwrap_or_default()); debug!("backend use root {}", root); - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { @@ -204,8 +206,6 @@ impl Builder for WebdavBuilder { authorization = Some(format_authorization_by_bearer(token)?) } - debug!("backend build finished: {:?}", &self); - let core = Arc::new(WebdavCore { endpoint: endpoint.to_string(), server_path, diff --git a/core/src/services/webhdfs/backend.rs b/core/src/services/webhdfs/backend.rs index 65e60610162..2fcbbe08ef5 100644 --- a/core/src/services/webhdfs/backend.rs +++ b/core/src/services/webhdfs/backend.rs @@ -29,9 +29,6 @@ use log::debug; use serde::{Deserialize, Serialize}; use tokio::sync::OnceCell; -use crate::raw::*; -use crate::*; - use super::error::parse_error; use super::lister::WebhdfsLister; use super::message::BooleanResp; @@ -39,6 +36,8 @@ use super::message::FileStatusType; use super::message::FileStatusWrapper; use super::writer::WebhdfsWriter; use super::writer::WebhdfsWriters; +use crate::raw::*; +use crate::*; const WEBHDFS_DEFAULT_ENDPOINT: &str = "http://127.0.0.1:9870"; @@ -69,6 +68,12 @@ impl Debug for WebhdfsConfig { } } +impl Configurator for WebhdfsConfig { + fn into_builder(self) -> impl Builder { + WebhdfsBuilder { config: self } + } +} + /// [WebHDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/WebHDFS.html)'s REST API support. #[doc = include_str!("docs.md")] #[derive(Default, Clone)] @@ -159,13 +164,8 @@ impl WebhdfsBuilder { impl Builder for WebhdfsBuilder { const SCHEME: Scheme = Scheme::Webhdfs; - type Accessor = WebhdfsBackend; type Config = WebhdfsConfig; - fn from_config(config: Self::Config) -> Self { - Self { config } - } - /// build the backend /// /// # Note @@ -173,14 +173,14 @@ impl Builder for WebhdfsBuilder { /// when building backend, the built backend will check if the root directory /// exits. /// if the directory does not exits, the directory will be automatically created - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("start building backend: {:?}", self); - let root = normalize_root(&self.config.root.take().unwrap_or_default()); + let root = normalize_root(&self.config.root.unwrap_or_default()); debug!("backend use root {root}"); // check scheme - let endpoint = match self.config.endpoint.take() { + let endpoint = match self.config.endpoint { Some(endpoint) => { if endpoint.starts_with("http") { endpoint @@ -192,12 +192,11 @@ impl Builder for WebhdfsBuilder { }; debug!("backend use endpoint {}", endpoint); - let atomic_write_dir = self.config.atomic_write_dir.take(); + let atomic_write_dir = self.config.atomic_write_dir; let auth = self .config .delegation - .take() .map(|dt| format!("delegation_token={dt}")); let client = HttpClient::new()?; diff --git a/core/src/services/yandex_disk/backend.rs b/core/src/services/yandex_disk/backend.rs index 710c7006b5a..d8e09f4cc95 100644 --- a/core/src/services/yandex_disk/backend.rs +++ b/core/src/services/yandex_disk/backend.rs @@ -27,14 +27,13 @@ use http::StatusCode; use log::debug; use serde::{Deserialize, Serialize}; -use crate::raw::*; -use crate::*; - use super::core::*; use super::error::parse_error; use super::lister::YandexDiskLister; use super::writer::YandexDiskWriter; use super::writer::YandexDiskWriters; +use crate::raw::*; +use crate::*; /// Config for backblaze YandexDisk services support. #[derive(Default, Serialize, Deserialize, Clone, PartialEq, Eq)] @@ -59,6 +58,15 @@ impl Debug for YandexDiskConfig { } } +impl Configurator for YandexDiskConfig { + fn into_builder(self) -> impl Builder { + YandexDiskBuilder { + config: self, + http_client: None, + } + } +} + /// [YandexDisk](https://360.yandex.com/disk/) services support. #[doc = include_str!("docs.md")] #[derive(Default)] @@ -115,18 +123,10 @@ impl YandexDiskBuilder { impl Builder for YandexDiskBuilder { const SCHEME: Scheme = Scheme::YandexDisk; - type Accessor = YandexDiskBackend; type Config = YandexDiskConfig; - fn from_config(config: Self::Config) -> Self { - YandexDiskBuilder { - config, - http_client: None, - } - } - /// Builds the backend and returns the result of YandexDiskBackend. - fn build(&mut self) -> Result { + fn build(self) -> Result { debug!("backend build started: {:?}", &self); let root = normalize_root(&self.config.root.clone().unwrap_or_default()); @@ -141,7 +141,7 @@ impl Builder for YandexDiskBuilder { ); } - let client = if let Some(client) = self.http_client.take() { + let client = if let Some(client) = self.http_client { client } else { HttpClient::new().map_err(|err| { diff --git a/core/src/types/builder.rs b/core/src/types/builder.rs index 123377e0762..e854de59ad6 100644 --- a/core/src/types/builder.rs +++ b/core/src/types/builder.rs @@ -18,61 +18,100 @@ use crate::raw::*; use crate::*; use serde::de::DeserializeOwned; -use serde::Deserialize; -use std::collections::HashMap; +use serde::Serialize; -/// Builder is used to set up a real underlying service, i.e. storage accessor. +/// Builder is used to set up underlying services. /// -/// One builder is usually used by [`Operator`] during its initialization. -/// It can be created by accepting several k-v pairs from one HashMap, one iterator and specific environment variables. +/// This trait allows the developer to define a builder struct that can: /// -/// By default each builder of underlying service must support deriving from one HashMap. -/// Besides that, according to the implementation, each builder will have its own special methods -/// to control the behavior of initialization of the underlying service. -/// It often provides semantic interface instead of using dynamic k-v strings directly. -/// Because the latter way is obscure and hard to remember how many parameters it will have. +/// - build a service via builder style API. +/// - configure in-memory options like `http_client` or `customized_credential_load`. /// -/// So it is recommended that developer should read related doc of builder carefully when you are working with one service. -/// We also promise that every public API will provide detailed documentation. +/// Usually, users don't need to use or import this trait directly, they can use `Operator` API instead. /// -/// It's recommended to use [`Operator::new`] to avoid use `Builder` trait directly. -pub trait Builder: Default { +/// For example: +/// +/// ``` +/// # use anyhow::Result; +/// use opendal::services::Fs; +/// use opendal::Operator; +/// async fn test() -> Result<()> { +/// // Create fs backend builder. +/// let mut builder = Fs::default(); +/// // Set the root for fs, all operations will happen under this root. +/// // +/// // NOTE: the root must be absolute path. +/// builder.root("/tmp"); +/// +/// // Build an `Operator` to start operating the storage. +/// let op: Operator = Operator::new(builder)?.finish(); +/// +/// Ok(()) +/// } +/// ``` +pub trait Builder: Default + 'static { /// Associated scheme for this builder. It indicates what underlying service is. const SCHEME: Scheme; - /// The accessor that built by this builder. - type Accessor: Access; - /// The config for this builder. - type Config: DeserializeOwned; - - /// Construct a builder from given config. - fn from_config(config: Self::Config) -> Self; - - /// Construct a builder from given map which contains several parameters needed by underlying service. - fn from_map(map: HashMap) -> Result { - match Self::Config::deserialize(ConfigDeserializer::new(map)) { - Ok(config) => Ok(Self::from_config(config)), - Err(err) => Err( - Error::new(ErrorKind::ConfigInvalid, "failed to deserialize config") - .set_source(err), - ), - } - } + /// Associated configuration for this builder. + type Config: Configurator; /// Consume the accessor builder to build a service. - fn build(&mut self) -> Result; + fn build(self) -> Result; } /// Dummy implementation of builder impl Builder for () { const SCHEME: Scheme = Scheme::Custom("dummy"); - - type Accessor = (); - type Config = (); - fn from_config(_: Self::Config) -> Self {} - - fn build(&mut self) -> Result { + fn build(self) -> Result { Ok(()) } } + +/// Configurator is used to configure the underlying service. +/// +/// This trait allows the developer to define a configuration struct that can: +/// +/// - deserialize from an iterator like hashmap or vector. +/// - convert into a service builder and finally build the underlying services. +/// +/// Usually, users don't need to use or import this trait directly, they can use `Operator` API instead. +/// +/// For example: +/// +/// ``` +/// # use anyhow::Result; +/// use std::collections::HashMap; +/// +/// use opendal::services::MemoryConfig; +/// use opendal::Operator; +/// async fn test() -> Result<()> { +/// let mut cfg = MemoryConfig::default(); +/// cfg.root = Some("/".to_string()); +/// +/// // Build an `Operator` to start operating the storage. +/// let op: Operator = Operator::from_config(cfg)?.finish(); +/// +/// Ok(()) +/// } +/// ``` +pub trait Configurator: Serialize + DeserializeOwned + 'static { + /// Deserialize from an iterator. + /// + /// This API is provided by opendal, developer should not implement it. + fn from_iter(iter: impl IntoIterator) -> Result { + let cfg = ConfigDeserializer::new(iter.into_iter().collect()); + + Self::deserialize(cfg).map_err(|err| { + Error::new(ErrorKind::ConfigInvalid, "failed to deserialize config").set_source(err) + }) + } + + /// Convert this configuration into a service builder. + fn into_builder(self) -> impl Builder; +} + +impl Configurator for () { + fn into_builder(self) -> impl Builder {} +} diff --git a/core/src/types/mod.rs b/core/src/types/mod.rs index 79dcc993aae..602cf8c557f 100644 --- a/core/src/types/mod.rs +++ b/core/src/types/mod.rs @@ -57,6 +57,7 @@ pub use operator::OperatorInfo; mod builder; pub use builder::Builder; +pub use builder::Configurator; mod error; pub use error::Error; diff --git a/core/src/types/operator/builder.rs b/core/src/types/operator/builder.rs index 7f0f1480dd4..36d8d699b0c 100644 --- a/core/src/types/operator/builder.rs +++ b/core/src/types/operator/builder.rs @@ -73,11 +73,36 @@ impl Operator { /// } /// ``` #[allow(clippy::new_ret_no_self)] - pub fn new(mut ab: B) -> Result> { + pub fn new(ab: B) -> Result> { let acc = ab.build()?; Ok(OperatorBuilder::new(acc)) } + /// Create a new operator from given config. + /// + /// # Examples + /// + /// ``` + /// # use anyhow::Result; + /// use std::collections::HashMap; + /// + /// use opendal::services::MemoryConfig; + /// use opendal::Operator; + /// async fn test() -> Result<()> { + /// let cfg = MemoryConfig::default(); + /// + /// // Build an `Operator` to start operating the storage. + /// let op: Operator = Operator::from_config(cfg)?.finish(); + /// + /// Ok(()) + /// } + /// ``` + pub fn from_config(cfg: C) -> Result> { + let builder = cfg.into_builder(); + let acc = builder.build()?; + Ok(OperatorBuilder::new(acc)) + } + /// Create a new operator from given map. /// /// # Notes @@ -112,7 +137,8 @@ impl Operator { pub fn from_map( map: HashMap, ) -> Result> { - let acc = B::from_map(map)?.build()?; + let builder = B::Config::from_iter(map)?.into_builder(); + let acc = builder.build()?; Ok(OperatorBuilder::new(acc)) }