Skip to content

Commit

Permalink
Update
Browse files Browse the repository at this point in the history
  • Loading branch information
diqiu50 committed Jan 2, 2025
1 parent 3ddb5a3 commit 61418c9
Show file tree
Hide file tree
Showing 6 changed files with 27 additions and 13 deletions.
4 changes: 2 additions & 2 deletions clients/filesystem-fuse/conf/gvfs_fuse.toml
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ metalake = "your_metalake"

# extend settings
[extend_config]
access_key_id = "your access_key"
secret_access_key = "your_secret_key"
s3-access_key_id = "your access_key"
s3-secret_access_key = "your_secret_key"
4 changes: 2 additions & 2 deletions clients/filesystem-fuse/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -309,11 +309,11 @@ mod test {
assert_eq!(config.gravitino.uri, "http://localhost:8090");
assert_eq!(config.gravitino.metalake, "test");
assert_eq!(
config.extend_config.get("access_key_id"),
config.extend_config.get("s3-access_key_id"),
Some(&"XXX_access_key".to_string())
);
assert_eq!(
config.extend_config.get("secret_access_key"),
config.extend_config.get("s3-secret_access_key"),
Some(&"XXX_secret_key".to_string())
);
}
Expand Down
16 changes: 15 additions & 1 deletion clients/filesystem-fuse/src/s3_filesystem.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ use log::error;
use opendal::layers::LoggingLayer;
use opendal::services::S3;
use opendal::{Builder, Operator};
use std::collections::HashMap;
use std::path::Path;

pub(crate) struct S3FileSystem {
Expand All @@ -37,13 +38,26 @@ pub(crate) struct S3FileSystem {
impl S3FileSystem {}

impl S3FileSystem {
const S3_CONFIG_PREFIX: &'static str = "s3-";

pub(crate) fn new(
catalog: &Catalog,
fileset: &Fileset,
config: &AppConfig,
_fs_context: &FileSystemContext,
) -> GvfsResult<Self> {
let mut opendal_config = config.extend_config.clone();
let mut opendal_config: HashMap<String, String> = config
.extend_config
.clone()
.into_iter()
.filter_map(|(k, v)| {
if k.starts_with(Self::S3_CONFIG_PREFIX) {
Some((k.to_string(), v.to_string()))
} else {
None
}
})
.collect();

let bucket = extract_bucket(&fileset.storage_location)?;
opendal_config.insert("bucket".to_string(), bucket);
Expand Down
4 changes: 2 additions & 2 deletions clients/filesystem-fuse/tests/conf/gvfs_fuse_memory.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,5 @@ metalake = "test"

# extend settings
[extend_config]
access_key_id = "XXX_access_key"
secret_access_key = "XXX_secret_key"
s3-access_key_id = "XXX_access_key"
s3-secret_access_key = "XXX_secret_key"
8 changes: 4 additions & 4 deletions clients/filesystem-fuse/tests/conf/gvfs_fuse_s3_ut.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,8 @@ metalake = "test"

# extend settings
[extend_config]
access_key_id = "XXX_access_key"
secret_access_key = "XXX_secret_key"
region = "XXX_region"
bucket = "XXX_bucket"
s3-access_key_id = "XXX_access_key"
s3-secret_access_key = "XXX_secret_key"
s3-region = "XXX_region"
s3-bucket = "XXX_bucket"

4 changes: 2 additions & 2 deletions clients/filesystem-fuse/tests/conf/gvfs_fuse_test.toml
Original file line number Diff line number Diff line change
Expand Up @@ -36,5 +36,5 @@ metalake = "test"

# extend settings
[extend_config]
access_key_id = "XXX_access_key"
secret_access_key = "XXX_secret_key"
s3-access_key_id = "XXX_access_key"
s3-secret_access_key = "XXX_secret_key"

0 comments on commit 61418c9

Please sign in to comment.