Skip to content

Commit

Permalink
feat: Implement Config for part of services (#4277)
Browse files Browse the repository at this point in the history 
* add config for azdls

Signed-off-by: Xuanwo <[email protected]>

* Add config for azfile

Signed-off-by: Xuanwo <[email protected]>

* Add cacache

Signed-off-by: Xuanwo <[email protected]>

* Impl config for cloudflare kv

Signed-off-by: Xuanwo <[email protected]>

* Migrate cos

Signed-off-by: Xuanwo <[email protected]>

* Impl config for dashmap

Signed-off-by: Xuanwo <[email protected]>

* Add config for dbfs

Signed-off-by: Xuanwo <[email protected]>

* Fix build

Signed-off-by: Xuanwo <[email protected]>

* Fix cos

Signed-off-by: Xuanwo <[email protected]>

---------

Signed-off-by: Xuanwo <[email protected]>
  • Loading branch information
@Xuanwo
Xuanwo authored Feb 27, 2024
1 parent 4ce08cc commit 92b7a63
Show file tree
Hide file tree
Showing 8 changed files with 238 additions and 138 deletions.
3 changes: 2 additions & 1 deletion core/src/services/azblob/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ const KNOWN_AZBLOB_ENDPOINT_SUFFIX: &[&str] = &[
];

const AZBLOB_BATCH_LIMIT: usize = 256;

/// Azure Storage Blob services support.
#[derive(Default, Deserialize, Clone)]
pub struct AzblobConfig {
Expand Down Expand Up @@ -124,7 +125,7 @@ pub struct AzblobBuilder {

impl Debug for AzblobBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzblobBuilder");

ds.field("config", &self.config);

Expand Down
73 changes: 44 additions & 29 deletions core/src/services/azdls/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
use serde::Deserialize;

use super::core::AzdlsCore;
use super::error::parse_error;
use super::lister::AzdlsLister;
use super::writer::AzdlsWriter;
use super::writer::{AzdlsWriter, AzdlsWriters};
use crate::raw::*;
use crate::services::azdls::writer::AzdlsWriters;
use crate::*;

/// Known endpoint suffix Azure Data Lake Storage Gen2 URI syntax.
Expand All @@ -46,20 +46,18 @@ const KNOWN_AZDLS_ENDPOINT_SUFFIX: &[&str] = &[
];

/// Azure Data Lake Storage Gen2 Support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzdlsBuilder {
#[derive(Default, Deserialize, Clone)]
pub struct AzdlsConfig {
root: Option<String>,
filesystem: String,
endpoint: Option<String>,
account_name: Option<String>,
account_key: Option<String>,
http_client: Option<HttpClient>,
}

impl Debug for AzdlsBuilder {
impl Debug for AzdlsConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzdlsConfig");

ds.field("root", &self.root);
ds.field("filesystem", &self.filesystem);
Expand All @@ -76,21 +74,39 @@ impl Debug for AzdlsBuilder {
}
}

/// Azure Data Lake Storage Gen2 Support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzdlsBuilder {
config: AzdlsConfig,
http_client: Option<HttpClient>,
}

impl Debug for AzdlsBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("AzdlsBuilder");

ds.field("config", &self.config);

ds.finish()
}
}

impl AzdlsBuilder {
/// Set root of this backend.
///
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
self.root = Some(root.to_string())
self.config.root = Some(root.to_string())
}

self
}

/// Set filesystem name of this backend.
pub fn filesystem(&mut self, filesystem: &str) -> &mut Self {
self.filesystem = filesystem.to_string();
self.config.filesystem = filesystem.to_string();

self
}
Expand All @@ -104,7 +120,7 @@ impl AzdlsBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
}

self
Expand All @@ -116,7 +132,7 @@ impl AzdlsBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
self.account_name = Some(account_name.to_string());
self.config.account_name = Some(account_name.to_string());
}

self
Expand All @@ -128,7 +144,7 @@ impl AzdlsBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
self.account_key = Some(account_key.to_string());
self.config.account_key = Some(account_key.to_string());
}

self
Expand All @@ -153,19 +169,19 @@ impl Builder for AzdlsBuilder {
fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);

let root = normalize_root(&self.root.take().unwrap_or_default());
let root = normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);

// Handle endpoint, region and container name.
let filesystem = match self.filesystem.is_empty() {
false => Ok(&self.filesystem),
let filesystem = match self.config.filesystem.is_empty() {
false => Ok(&self.config.filesystem),
true => Err(Error::new(ErrorKind::ConfigInvalid, "filesystem is empty")
.with_operation("Builder::build")
.with_context("service", Scheme::Azdls)),
}?;
debug!("backend use filesystem {}", &filesystem);

let endpoint = match &self.endpoint {
let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
.with_operation("Builder::build")
Expand All @@ -184,10 +200,11 @@ impl Builder for AzdlsBuilder {

let config_loader = AzureStorageConfig {
account_name: self
.config
.account_name
.clone()
.or_else(|| infer_storage_name_from_endpoint(endpoint.as_str())),
account_key: self.account_key.clone(),
account_key: self.config.account_key.clone(),
sas_token: None,
..Default::default()
};
Expand All @@ -198,7 +215,7 @@ impl Builder for AzdlsBuilder {
debug!("backend build finished: {:?}", &self);
Ok(AzdlsBackend {
core: Arc::new(AzdlsCore {
filesystem: self.filesystem.clone(),
filesystem: self.config.filesystem.clone(),
root,
endpoint,
client,
Expand All @@ -209,15 +226,13 @@ impl Builder for AzdlsBuilder {
}

fn from_map(map: HashMap<String, String>) -> Self {
let mut builder = AzdlsBuilder::default();
let config = AzdlsConfig::deserialize(ConfigDeserializer::new(map))
.expect("config deserialize must succeed");

map.get("root").map(|v| builder.root(v));
map.get("filesystem").map(|v| builder.filesystem(v));
map.get("endpoint").map(|v| builder.endpoint(v));
map.get("account_name").map(|v| builder.account_name(v));
map.get("account_key").map(|v| builder.account_key(v));

builder
AzdlsBuilder {
config,
http_client: None,
}
}
}

Expand Down Expand Up @@ -467,7 +482,7 @@ mod tests {
assert_eq!(azdls.core.filesystem, "filesystem".to_string());

assert_eq!(
azdls_builder.account_key.unwrap(),
azdls_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
Expand All @@ -488,6 +503,6 @@ mod tests {

assert_eq!(azdls.core.filesystem, "filesystem".to_string());

assert_eq!(azdls_builder.account_key, None);
assert_eq!(azdls_builder.config.account_key, None);
}
}
74 changes: 47 additions & 27 deletions core/src/services/azfile/backend.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ use log::debug;
use reqsign::AzureStorageConfig;
use reqsign::AzureStorageLoader;
use reqsign::AzureStorageSigner;
use serde::Deserialize;

use super::core::AzfileCore;
use super::error::parse_error;
Expand All @@ -39,31 +40,51 @@ use crate::*;
const DEFAULT_AZFILE_ENDPOINT_SUFFIX: &str = "file.core.windows.net";

/// Azure File services support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzfileBuilder {
#[derive(Default, Deserialize, Clone)]
pub struct AzfileConfig {
root: Option<String>,
endpoint: Option<String>,
account_name: Option<String>,
share_name: String,
account_name: Option<String>,
account_key: Option<String>,
sas_token: Option<String>,
http_client: Option<HttpClient>,
}

impl Debug for AzfileBuilder {
impl Debug for AzfileConfig {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("Builder");
let mut ds = f.debug_struct("AzfileConfig");

ds.field("root", &self.root);
ds.field("endpoint", &self.endpoint);
ds.field("share_name", &self.share_name);
ds.field("endpoint", &self.endpoint);

if self.account_name.is_some() {
ds.field("account_name", &"<redacted>");
}
if self.account_key.is_some() {
ds.field("account_key", &"<redacted>");
}
if self.sas_token.is_some() {
ds.field("sas_token", &"<redacted>");
}

ds.finish()
}
}

/// Azure File services support.
#[doc = include_str!("docs.md")]
#[derive(Default, Clone)]
pub struct AzfileBuilder {
config: AzfileConfig,
http_client: Option<HttpClient>,
}

impl Debug for AzfileBuilder {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
let mut ds = f.debug_struct("AzfileBuilder");

ds.field("config", &self.config);

ds.finish()
}
Expand All @@ -75,7 +96,7 @@ impl AzfileBuilder {
/// All operations will happen under this root.
pub fn root(&mut self, root: &str) -> &mut Self {
if !root.is_empty() {
self.root = Some(root.to_string())
self.config.root = Some(root.to_string())
}

self
Expand All @@ -85,7 +106,7 @@ impl AzfileBuilder {
pub fn endpoint(&mut self, endpoint: &str) -> &mut Self {
if !endpoint.is_empty() {
// Trim trailing `/` so that we can accept `http://127.0.0.1:9000/`
self.endpoint = Some(endpoint.trim_end_matches('/').to_string());
self.config.endpoint = Some(endpoint.trim_end_matches('/').to_string());
}

self
Expand All @@ -97,7 +118,7 @@ impl AzfileBuilder {
/// - If not, we will try to load it from environment.
pub fn account_name(&mut self, account_name: &str) -> &mut Self {
if !account_name.is_empty() {
self.account_name = Some(account_name.to_string());
self.config.account_name = Some(account_name.to_string());
}

self
Expand All @@ -109,7 +130,7 @@ impl AzfileBuilder {
/// - If not, we will try to load it from environment.
pub fn account_key(&mut self, account_key: &str) -> &mut Self {
if !account_key.is_empty() {
self.account_key = Some(account_key.to_string());
self.config.account_key = Some(account_key.to_string());
}

self
Expand All @@ -121,7 +142,7 @@ impl AzfileBuilder {
/// You can find more about from: <https://learn.microsoft.com/en-us/rest/api/storageservices/operations-on-shares--file-service>
pub fn share_name(&mut self, share_name: &str) -> &mut Self {
if !share_name.is_empty() {
self.share_name = share_name.to_string();
self.config.share_name = share_name.to_string();
}

self
Expand All @@ -144,24 +165,22 @@ impl Builder for AzfileBuilder {
type Accessor = AzfileBackend;

fn from_map(map: HashMap<String, String>) -> Self {
let mut builder = AzfileBuilder::default();

map.get("root").map(|v| builder.root(v));
map.get("endpoint").map(|v| builder.endpoint(v));
map.get("account_name").map(|v| builder.account_name(v));
map.get("account_key").map(|v| builder.account_key(v));
map.get("share_name").map(|v| builder.share_name(v));
let config = AzfileConfig::deserialize(ConfigDeserializer::new(map))
.expect("config deserialize must succeed");

builder
AzfileBuilder {
config,
http_client: None,
}
}

fn build(&mut self) -> Result<Self::Accessor> {
debug!("backend build started: {:?}", &self);

let root = normalize_root(&self.root.take().unwrap_or_default());
let root = normalize_root(&self.config.root.take().unwrap_or_default());
debug!("backend use root {}", root);

let endpoint = match &self.endpoint {
let endpoint = match &self.config.endpoint {
Some(endpoint) => Ok(endpoint.clone()),
None => Err(Error::new(ErrorKind::ConfigInvalid, "endpoint is empty")
.with_operation("Builder::build")
Expand All @@ -179,6 +198,7 @@ impl Builder for AzfileBuilder {
};

let account_name_option = self
.config
.account_name
.clone()
.or_else(|| infer_account_name_from_endpoint(endpoint.as_str()));
Expand All @@ -194,8 +214,8 @@ impl Builder for AzfileBuilder {

let config_loader = AzureStorageConfig {
account_name: Some(account_name),
account_key: self.account_key.clone(),
sas_token: self.sas_token.clone(),
account_key: self.config.account_key.clone(),
sas_token: self.config.sas_token.clone(),
..Default::default()
};

Expand All @@ -211,7 +231,7 @@ impl Builder for AzfileBuilder {
loader: cred_loader,
client,
signer,
share_name: self.share_name.clone(),
share_name: self.config.share_name.clone(),
}),
})
}
Expand Down Expand Up @@ -435,7 +455,7 @@ mod tests {
);

assert_eq!(
azfile_builder.account_key.unwrap(),
azfile_builder.config.account_key.unwrap(),
"account-key".to_string()
);
}
Expand Down
Loading

0 comments on commit 92b7a63

Please sign in to comment.