From f02521659db8ea543b803b6f095082ac96365a13 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 10:57:38 +0800 Subject: [PATCH 01/31] Add SimpleFilesystem --- clients/filesystem-fuse/.cargo/config.toml | 1 - clients/filesystem-fuse/Cargo.toml | 6 +- clients/filesystem-fuse/Makefile | 70 +++ clients/filesystem-fuse/build.gradle.kts | 32 +- clients/filesystem-fuse/src/filesystem.rs | 427 +++++++++++++++++- .../filesystem-fuse/src/fuse_api_handle.rs | 7 +- clients/filesystem-fuse/src/lib.rs | 2 + clients/filesystem-fuse/src/main.rs | 2 + .../src/opened_file_manager.rs | 65 +++ clients/filesystem-fuse/src/utils.rs | 35 ++ 10 files changed, 625 insertions(+), 22 deletions(-) create mode 100644 clients/filesystem-fuse/Makefile create mode 100644 clients/filesystem-fuse/src/opened_file_manager.rs create mode 100644 clients/filesystem-fuse/src/utils.rs diff --git a/clients/filesystem-fuse/.cargo/config.toml b/clients/filesystem-fuse/.cargo/config.toml index 78bc9f7fe48..9d5bb048edc 100644 --- a/clients/filesystem-fuse/.cargo/config.toml +++ b/clients/filesystem-fuse/.cargo/config.toml @@ -16,5 +16,4 @@ # under the License. [build] -target-dir = "build" rustflags = ["-Adead_code", "-Aclippy::redundant-field-names"] diff --git a/clients/filesystem-fuse/Cargo.toml b/clients/filesystem-fuse/Cargo.toml index 2883cecc656..3bcf20f37ef 100644 --- a/clients/filesystem-fuse/Cargo.toml +++ b/clients/filesystem-fuse/Cargo.toml @@ -30,13 +30,15 @@ name = "gvfs-fuse" path = "src/main.rs" [lib] -name="gvfs_fuse" +name = "gvfs_fuse" [dependencies] async-trait = "0.1" bytes = "1.6.0" -futures-util = "0.3.30" +dashmap = "6.1.0" fuse3 = { version = "0.8.1", "features" = ["tokio-runtime", "unprivileged"] } +futures-util = "0.3.30" +libc = "0.2.168" log = "0.4.22" tokio = { version = "1.38.0", features = ["full"] } tracing-subscriber = { version = "0.3.18", features = ["env-filter"] } diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile new file mode 100644 index 00000000000..2e48a3e1fd8 --- /dev/null +++ b/clients/filesystem-fuse/Makefile @@ -0,0 +1,70 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +.EXPORT_ALL_VARIABLES: + +.PHONY: build +build: + cargo build --all-features --workspace + +fmt: + cargo fmt --all + +cargo-sort: install-cargo-sort + cargo sort -w + +check-fmt: + cargo fmt --all -- --check + +check-clippy: + #cargo clippy --all-targets --all-features --workspace -- -D warnings + cargo clippy --all-targets --all-features --workspace -- + +install-cargo-sort: + cargo install cargo-sort@1.0.9 + +check-cargo-sort: install-cargo-sort + cargo sort -c + +install-cargo-machete: + cargo install cargo-machete + +cargo-machete: install-cargo-machete + cargo machete + +install-taplo-cli: + cargo install taplo-cli@0.9.0 + +fix-toml: install-taplo-cli + taplo fmt + +check-toml: install-taplo-cli + taplo check + +check: check-fmt check-clippy check-cargo-sort check-toml cargo-machete + +doc-test: + cargo test --no-fail-fast --doc --all-features --workspace + +unit-test: doc-test + cargo test --no-fail-fast --lib --all-features --workspace + +test: doc-test + cargo test --no-fail-fast --all-targets --all-features --workspace + +clean: + cargo clean diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 08693ddc5bd..201397c19e7 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -32,7 +32,7 @@ val buildRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) description = "Compile the Rust project" workingDir = file("$projectDir") - commandLine("bash", "-c", "cargo build --release") + commandLine("bash", "-c", "make build") } val checkRustProject by tasks.registering(Exec::class) { @@ -40,18 +40,7 @@ val checkRustProject by tasks.registering(Exec::class) { description = "Check the Rust project" workingDir = file("$projectDir") - commandLine( - "bash", - "-c", - """ - set -e - echo "Checking the code format" - cargo fmt --all -- --check - - echo "Running clippy" - cargo clippy --all-targets --all-features --workspace -- -D warnings - """.trimIndent() - ) + commandLine( "bash", "-c", "make check") } val testRustProject by tasks.registering(Exec::class) { @@ -59,7 +48,18 @@ val testRustProject by tasks.registering(Exec::class) { description = "Run tests in the Rust project" group = "verification" workingDir = file("$projectDir") - commandLine("bash", "-c", "cargo test --release") + commandLine("bash", "-c", "make test") + + standardOutput = System.out + errorOutput = System.err +} + +val cleanRustProject by tasks.registering(Exec::class) { + dependsOn(checkRustEnvironment) + description = "Run tests in the Rust project" + group = "verification" + workingDir = file("$projectDir") + commandLine("bash", "-c", "make clean") standardOutput = System.out errorOutput = System.err @@ -85,3 +85,7 @@ tasks.named("check") { tasks.named("test") { dependsOn(testRustProject) } + +tasks.named("clean") { + dependsOn(cleanRustProject) +} diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 6d1d8fa2538..6a2bf80c6dc 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -16,9 +16,15 @@ * specific language governing permissions and limitations * under the License. */ +use crate::opened_file_manager::OpenedFileManager; +use crate::utils::{join_file_path, split_file_path}; use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType, Timestamp}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use std::time::SystemTime; +use tokio::sync::RwLock; pub(crate) type Result = std::result::Result; @@ -174,9 +180,6 @@ pub struct FileStat { // file type like regular file or directory and so on pub(crate) kind: FileType, - // file permission - pub(crate) perm: u16, - // file access time pub(crate) atime: Timestamp, @@ -190,6 +193,48 @@ pub struct FileStat { pub(crate) nlink: u32, } +impl FileStat { + pub fn new_file_with_path(path: &str, size: u64) -> Self { + let (parent, name) = split_file_path(path); + Self::new_file(parent, name, size) + } + + pub fn new_dir_with_path(path: &str) -> Self { + let (parent, name) = split_file_path(path); + Self::new_dir(parent, name) + } + + pub fn new_file(parent: &str, name: &str, size: u64) -> Self { + Self::new_file_entry(parent, name, size, FileType::RegularFile) + } + + pub fn new_dir(parent: &str, name: &str) -> Self { + Self::new_file_entry(parent, name, 0, FileType::Directory) + } + + pub fn new_file_entry(parent: &str, name: &str, size: u64, kind: FileType) -> Self { + let atime = Timestamp::from(SystemTime::now()); + Self { + file_id: 0, + parent_file_id: 0, + name: name.into(), + path: join_file_path(parent, name), + size: size, + kind: kind, + atime: atime, + mtime: atime, + ctime: atime, + nlink: 1, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.parent_file_id = parent_file_id; + self.file_id = file_id; + } +} + /// Opened file for read or write, it is used to read or write the file content. pub(crate) struct OpenedFile { pub(crate) file_stat: FileStat, @@ -201,6 +246,66 @@ pub(crate) struct OpenedFile { pub writer: Option>, } +impl OpenedFile { + pub fn new(file_stat: FileStat) -> Self { + OpenedFile { + file_stat: file_stat, + handle_id: 0, + reader: None, + writer: None, + } + } + + async fn read(&mut self, offset: u64, size: u32) -> Result { + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.reader.as_mut().unwrap().read(offset, size).await + } + + async fn write(&mut self, offset: u64, data: &[u8]) -> Result { + let end = offset + data.len() as u64; + + if end > self.file_stat.size { + self.file_stat.size = end; + } + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.file_stat.mtime = self.file_stat.atime; + + self.writer.as_mut().unwrap().write(offset, data).await + } + + async fn close(&mut self) -> Result<()> { + if let Some(mut reader) = self.reader.take() { + reader.close().await?; + } + if let Some(mut writer) = self.writer.take() { + self.flush().await?; + writer.close().await? + } + Ok(()) + } + + async fn flush(&mut self) -> Result<()> { + if let Some(writer) = &mut self.writer { + writer.flush().await?; + } + Ok(()) + } + + fn file_handle(&self) -> FileHandle { + debug_assert!(self.handle_id != 0); + debug_assert!(self.file_stat.file_id != 0); + FileHandle { + file_id: self.file_stat.file_id, + handle_id: self.handle_id, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.file_stat.set_file_id(parent_file_id, file_id) + } +} + // FileHandle is the file handle for the opened file. pub(crate) struct FileHandle { pub(crate) file_id: u64, @@ -239,3 +344,319 @@ pub trait FileWriter: Sync + Send { Ok(()) } } + +/// SimpleFileSystem is a simple implementation for the file system. +/// it is used to manage the file metadata and file handle. +/// The operations of the file system are implemented by the PathFileSystem. +/// Note: This class is not use in the production code, it is used for the demo and testing +pub struct SimpleFileSystem { + /// file entries + file_entry_manager: RwLock, + /// opened files + opened_file_manager: OpenedFileManager, + /// inode id generator + inode_id_generator: AtomicU64, + + /// real system + fs: T, +} + +impl SimpleFileSystem { + const INITIAL_INODE_ID: u64 = 10000; + const ROOT_DIR_PARENT_FILE_ID: u64 = 0; + const ROOT_DIR_FILE_ID: u64 = 1; + const ROOT_DIR_NAME: &'static str = ""; + + pub(crate) fn new(fs: T) -> Self { + Self { + file_entry_manager: RwLock::new(FileEntryManager::new()), + opened_file_manager: OpenedFileManager::new(), + inode_id_generator: AtomicU64::new(Self::INITIAL_INODE_ID), + fs, + } + } + + fn next_inode_id(&self) -> u64 { + self.inode_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + async fn get_file_node(&self, file_id: u64) -> Result { + self.file_entry_manager + .read() + .await + .get_file_by_id(file_id) + .ok_or(Errno::from(libc::ENOENT)) + } + + async fn get_file_node_by_path(&self, path: &str) -> Option { + self.file_entry_manager.read().await.get_file_by_name(path) + } + + async fn fill_file_id(&self, file_stat: &mut FileStat, parent_file_id: u64) { + let mut file_manager = self.file_entry_manager.write().await; + let file = file_manager.get_file_by_name(&file_stat.path); + match file { + None => { + file_stat.set_file_id(parent_file_id, self.next_inode_id()); + file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); + } + Some(file) => { + file_stat.set_file_id(file.parent_file_id, file.file_id); + } + } + } + + async fn open_file_internal( + &self, + file_id: u64, + flags: u32, + kind: FileType, + ) -> Result { + let file_node = self.get_file_node(file_id).await?; + + let mut file = { + match kind { + FileType::Directory => { + self.fs + .open_dir(&file_node.file_name, OpenFileFlags(flags)) + .await? + } + FileType::RegularFile => { + self.fs + .open_file(&file_node.file_name, OpenFileFlags(flags)) + .await? + } + _ => return Err(Errno::from(libc::EINVAL)), + } + }; + file.set_file_id(file_node.parent_file_id, file_id); + let file = self.opened_file_manager.put_file(file); + let file = file.lock().await; + Ok(file.file_handle()) + } +} + +#[async_trait] +impl RawFileSystem for SimpleFileSystem { + async fn init(&self) -> Result<()> { + self.file_entry_manager.write().await.insert( + Self::ROOT_DIR_PARENT_FILE_ID, + Self::ROOT_DIR_FILE_ID, + Self::ROOT_DIR_NAME, + ); + self.fs.init().await + } + + async fn get_file_path(&self, file_id: u64) -> String { + let file = self.get_file_node(file_id).await; + file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) + } + + async fn valid_file_id(&self, _file_id: u64, fh: u64) -> Result<()> { + let file_id = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))? + .lock() + .await + .file_stat + .file_id; + + (file_id == _file_id) + .then_some(()) + .ok_or(Errno::from(libc::EBADF)) + } + + async fn stat(&self, file_id: u64) -> Result { + let file_node = self.get_file_node(file_id).await?; + let mut stat = self.fs.stat(&file_node.file_name).await?; + stat.set_file_id(file_node.parent_file_id, file_node.file_id); + Ok(stat) + } + + async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { + let parent_file_node = self.get_file_node(parent_file_id).await?; + let mut stat = self.fs.lookup(&parent_file_node.file_name, name).await?; + self.fill_file_id(&mut stat, parent_file_id).await; + Ok(stat) + } + + async fn read_dir(&self, file_id: u64) -> Result> { + let file_node = self.get_file_node(file_id).await?; + let mut files = self.fs.read_dir(&file_node.file_name).await?; + for file in files.iter_mut() { + self.fill_file_id(file, file_node.file_id).await; + } + Ok(files) + } + + async fn open_file(&self, file_id: u64, flags: u32) -> Result { + self.open_file_internal(file_id, flags, FileType::RegularFile) + .await + } + + async fn open_dir(&self, file_id: u64, flags: u32) -> Result { + self.open_file_internal(file_id, flags, FileType::Directory) + .await + } + + async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { + let parent_node = self.get_file_node(parent_file_id).await?; + let mut file = self + .fs + .create_file(&parent_node.file_name, name, OpenFileFlags(flags)) + .await?; + + file.set_file_id(parent_file_id, self.next_inode_id()); + { + let mut file_node_manager = self.file_entry_manager.write().await; + file_node_manager.insert( + file.file_stat.parent_file_id, + file.file_stat.file_id, + &file.file_stat.path, + ); + } + let file = self.opened_file_manager.put_file(file); + let file = file.lock().await; + Ok(file.file_handle()) + } + + async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { + let parent_node = self.get_file_node(parent_file_id).await?; + let mut file = self.fs.create_dir(&parent_node.file_name, name).await?; + + file.set_file_id(parent_file_id, self.next_inode_id()); + { + let mut file_node_manager = self.file_entry_manager.write().await; + file_node_manager.insert(file.parent_file_id, file.file_id, &file.path); + } + Ok(file.file_id) + } + + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { + let file_node = self.get_file_node(file_id).await?; + self.fs + .set_attr(&file_node.file_name, file_stat, true) + .await + } + + async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { + let parent_file_node = self.get_file_node(parent_file_id).await?; + self.fs + .remove_file(&parent_file_node.file_name, name) + .await?; + + { + let mut file_id_manager = self.file_entry_manager.write().await; + file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + } + Ok(()) + } + + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { + let parent_file_node = self.get_file_node(parent_file_id).await?; + self.fs + .remove_dir(&parent_file_node.file_name, name) + .await?; + + { + let mut file_id_manager = self.file_entry_manager.write().await; + file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + } + Ok(()) + } + + async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { + let file = self + .opened_file_manager + .remove_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut file = file.lock().await; + file.close().await?; + Ok(()) + } + + async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { + let file_stat: FileStat; + let data = { + let opened_file = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + file_stat = opened_file.file_stat.clone(); + opened_file.read(offset, size).await + }; + + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + data + } + + async fn write(&self, _file_id: u64, fh: u64, offset: u64, data: &[u8]) -> Result { + let (len, file_stat) = { + let opened_file = self + .opened_file_manager + .get_file(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + let len = opened_file.write(offset, data).await; + (len, opened_file.file_stat.clone()) + }; + + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + len + } +} + +/// File entry is represent the abstract file. +#[derive(Debug, Clone)] +struct FileEntry { + file_id: u64, + parent_file_id: u64, + file_name: String, +} + +/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. +struct FileEntryManager { + // file_id_map is a map of file_id to file name. + file_id_map: HashMap, + + // file_name_map is a map of file name to file id. + file_name_map: HashMap, +} + +impl FileEntryManager { + fn new() -> Self { + Self { + file_id_map: HashMap::new(), + file_name_map: HashMap::new(), + } + } + + fn get_file_by_id(&self, file_id: u64) -> Option { + self.file_id_map.get(&file_id).cloned() + } + + fn get_file_by_name(&self, file_name: &str) -> Option { + self.file_name_map.get(file_name).cloned() + } + + fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { + let file_node = FileEntry { + file_id, + parent_file_id, + file_name: file_name.to_string(), + }; + self.file_id_map.insert(file_id, file_node.clone()); + self.file_name_map.insert(file_name.to_string(), file_node); + } + + fn remove(&mut self, file_name: &str) { + if let Some(node) = self.file_name_map.remove(file_name) { + self.file_id_map.remove(&node.file_id); + } + } +} diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 8c065df0227..800930f4ed3 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -401,6 +401,10 @@ impl Filesystem for FuseApiHandle { const fn fstat_to_file_attr(file_st: &FileStat, context: &FileSystemContext) -> FileAttr { debug_assert!(file_st.file_id != 0 && file_st.parent_file_id != 0); + let perm = match file_st.kind { + Directory => context.default_dir_perm, + _ => context.default_file_perm, + }; FileAttr { ino: file_st.file_id, size: file_st.size, @@ -409,7 +413,7 @@ const fn fstat_to_file_attr(file_st: &FileStat, context: &FileSystemContext) -> mtime: file_st.mtime, ctime: file_st.ctime, kind: file_st.kind, - perm: file_st.perm, + perm: perm, nlink: file_st.nlink, uid: context.uid, gid: context.gid, @@ -469,7 +473,6 @@ mod test { path: "".to_string(), size: 10032, kind: FileType::RegularFile, - perm: 0, atime: Timestamp { sec: 10, nsec: 3 }, mtime: Timestamp { sec: 12, nsec: 5 }, ctime: Timestamp { sec: 15, nsec: 7 }, diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index 54fb59a5107..b531df82f79 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -18,3 +18,5 @@ */ mod filesystem; mod fuse_api_handle; +mod opened_file_manager; +mod utils; diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index f6a7e69ec67..24358840df5 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -18,6 +18,8 @@ */ mod filesystem; mod fuse_api_handle; +mod opened_file_manager; +mod utils; use log::debug; use log::info; diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs new file mode 100644 index 00000000000..664f0a89875 --- /dev/null +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::filesystem::OpenedFile; +use dashmap::DashMap; +use std::sync::atomic::AtomicU64; +use std::sync::Arc; +use tokio::sync::Mutex; + +// OpenedFileManager is a manager all the opened files. and allocate a file handle id for the opened file. +pub(crate) struct OpenedFileManager { + // file_handle_map is a map of file_handle_id to opened file. + file_handle_map: DashMap>>, + + // file_handle_id_generator is used to generate unique file handle IDs. + handle_id_generator: AtomicU64, +} + +impl OpenedFileManager { + pub fn new() -> Self { + Self { + file_handle_map: Default::default(), + handle_id_generator: AtomicU64::new(1), + } + } + + pub(crate) fn next_handle_id(&self) -> u64 { + self.handle_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + pub(crate) fn put_file(&self, mut file: OpenedFile) -> Arc> { + let file_handle_id = self.next_handle_id(); + file.handle_id = file_handle_id; + let file_handle = Arc::new(Mutex::new(file)); + self.file_handle_map + .insert(file_handle_id, file_handle.clone()); + file_handle + } + + pub(crate) fn get_file(&self, handle_id: u64) -> Option>> { + self.file_handle_map + .get(&handle_id) + .map(|x| x.value().clone()) + } + + pub(crate) fn remove_file(&self, handle_id: u64) -> Option>> { + self.file_handle_map.remove(&handle_id).map(|x| x.1) + } +} diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs new file mode 100644 index 00000000000..6118b672362 --- /dev/null +++ b/clients/filesystem-fuse/src/utils.rs @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +// join the parent and name to a path +pub fn join_file_path(parent: &str, name: &str) -> String { + if parent.is_empty() { + name.to_string() + } else { + format!("{}/{}", parent, name) + } +} + +// split the path to parent and name +pub fn split_file_path(path: &str) -> (&str, &str) { + match path.rfind('/') { + Some(pos) => (&path[..pos], &path[pos + 1..]), + None => ("", path), + } +} From 97e066d5020c535129a9e2204324de907bdec9a2 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 11:48:01 +0800 Subject: [PATCH 02/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 109 +++++++++++----------- 1 file changed, 56 insertions(+), 53 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 6a2bf80c6dc..ba0c04dc5d9 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -257,20 +257,28 @@ impl OpenedFile { } async fn read(&mut self, offset: u64, size: u32) -> Result { + let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; + let result = reader.read(offset, size).await?; + + // update the access time self.file_stat.atime = Timestamp::from(SystemTime::now()); - self.reader.as_mut().unwrap().read(offset, size).await + + Ok(result) } async fn write(&mut self, offset: u64, data: &[u8]) -> Result { - let end = offset + data.len() as u64; + let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; + let written = writer.write(offset, data).await?; + // update the file size + let end = offset + data.len() as u64; if end > self.file_stat.size { self.file_stat.size = end; } self.file_stat.atime = Timestamp::from(SystemTime::now()); self.file_stat.mtime = self.file_stat.atime; - self.writer.as_mut().unwrap().write(offset, data).await + Ok(written) } async fn close(&mut self) -> Result<()> { @@ -355,14 +363,14 @@ pub struct SimpleFileSystem { /// opened files opened_file_manager: OpenedFileManager, /// inode id generator - inode_id_generator: AtomicU64, + file_id_generator: AtomicU64, /// real system fs: T, } impl SimpleFileSystem { - const INITIAL_INODE_ID: u64 = 10000; + const INITIAL_FILE_ID: u64 = 10000; const ROOT_DIR_PARENT_FILE_ID: u64 = 0; const ROOT_DIR_FILE_ID: u64 = 1; const ROOT_DIR_NAME: &'static str = ""; @@ -371,17 +379,17 @@ impl SimpleFileSystem { Self { file_entry_manager: RwLock::new(FileEntryManager::new()), opened_file_manager: OpenedFileManager::new(), - inode_id_generator: AtomicU64::new(Self::INITIAL_INODE_ID), + file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), fs, } } - fn next_inode_id(&self) -> u64 { - self.inode_id_generator + fn next_file_id(&self) -> u64 { + self.file_id_generator .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - async fn get_file_node(&self, file_id: u64) -> Result { + async fn get_file_entry(&self, file_id: u64) -> Result { self.file_entry_manager .read() .await @@ -389,7 +397,7 @@ impl SimpleFileSystem { .ok_or(Errno::from(libc::ENOENT)) } - async fn get_file_node_by_path(&self, path: &str) -> Option { + async fn get_file_entry_by_path(&self, path: &str) -> Option { self.file_entry_manager.read().await.get_file_by_name(path) } @@ -398,7 +406,8 @@ impl SimpleFileSystem { let file = file_manager.get_file_by_name(&file_stat.path); match file { None => { - file_stat.set_file_id(parent_file_id, self.next_inode_id()); + // allocate new file id + file_stat.set_file_id(parent_file_id, self.next_file_id()); file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); } Some(file) => { @@ -413,24 +422,24 @@ impl SimpleFileSystem { flags: u32, kind: FileType, ) -> Result { - let file_node = self.get_file_node(file_id).await?; + let file_entry = self.get_file_entry(file_id).await?; let mut file = { match kind { FileType::Directory => { self.fs - .open_dir(&file_node.file_name, OpenFileFlags(flags)) + .open_dir(&file_entry.file_name, OpenFileFlags(flags)) .await? } FileType::RegularFile => { self.fs - .open_file(&file_node.file_name, OpenFileFlags(flags)) + .open_file(&file_entry.file_name, OpenFileFlags(flags)) .await? } _ => return Err(Errno::from(libc::EINVAL)), } }; - file.set_file_id(file_node.parent_file_id, file_id); + file.set_file_id(file_entry.parent_file_id, file_id); let file = self.opened_file_manager.put_file(file); let file = file.lock().await; Ok(file.file_handle()) @@ -449,7 +458,7 @@ impl RawFileSystem for SimpleFileSystem { } async fn get_file_path(&self, file_id: u64) -> String { - let file = self.get_file_node(file_id).await; + let file = self.get_file_entry(file_id).await; file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) } @@ -469,24 +478,24 @@ impl RawFileSystem for SimpleFileSystem { } async fn stat(&self, file_id: u64) -> Result { - let file_node = self.get_file_node(file_id).await?; - let mut stat = self.fs.stat(&file_node.file_name).await?; - stat.set_file_id(file_node.parent_file_id, file_node.file_id); + let file = self.get_file_entry(file_id).await?; + let mut stat = self.fs.stat(&file.file_name).await?; + stat.set_file_id(file.parent_file_id, file.file_id); Ok(stat) } async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_node = self.get_file_node(parent_file_id).await?; - let mut stat = self.fs.lookup(&parent_file_node.file_name, name).await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; self.fill_file_id(&mut stat, parent_file_id).await; Ok(stat) } async fn read_dir(&self, file_id: u64) -> Result> { - let file_node = self.get_file_node(file_id).await?; - let mut files = self.fs.read_dir(&file_node.file_name).await?; + let file = self.get_file_entry(file_id).await?; + let mut files = self.fs.read_dir(&file.file_name).await?; for file in files.iter_mut() { - self.fill_file_id(file, file_node.file_id).await; + self.fill_file_id(file, file.file_id).await; } Ok(files) } @@ -502,16 +511,16 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent_node = self.get_file_node(parent_file_id).await?; + let parent = self.get_file_entry(parent_file_id).await?; let mut file = self .fs - .create_file(&parent_node.file_name, name, OpenFileFlags(flags)) + .create_file(&parent.file_name, name, OpenFileFlags(flags)) .await?; - file.set_file_id(parent_file_id, self.next_inode_id()); + file.set_file_id(parent_file_id, self.next_file_id()); { - let mut file_node_manager = self.file_entry_manager.write().await; - file_node_manager.insert( + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert( file.file_stat.parent_file_id, file.file_stat.file_id, &file.file_stat.path, @@ -523,46 +532,40 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent_node = self.get_file_node(parent_file_id).await?; - let mut file = self.fs.create_dir(&parent_node.file_name, name).await?; + let parent = self.get_file_entry(parent_file_id).await?; + let mut file = self.fs.create_dir(&parent.file_name, name).await?; - file.set_file_id(parent_file_id, self.next_inode_id()); + file.set_file_id(parent_file_id, self.next_file_id()); { - let mut file_node_manager = self.file_entry_manager.write().await; - file_node_manager.insert(file.parent_file_id, file.file_id, &file.path); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(file.parent_file_id, file.file_id, &file.path); } Ok(file.file_id) } async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file_node = self.get_file_node(file_id).await?; - self.fs - .set_attr(&file_node.file_name, file_stat, true) - .await + let file = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file.file_name, file_stat, true).await } async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_node = self.get_file_node(parent_file_id).await?; - self.fs - .remove_file(&parent_file_node.file_name, name) - .await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file.file_name, name).await?; { let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); } Ok(()) } async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_node = self.get_file_node(parent_file_id).await?; - self.fs - .remove_dir(&parent_file_node.file_name, name) - .await?; + let parent_file = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file.file_name, name).await?; { let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file_node.file_name, name)); + file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); } Ok(()) } @@ -645,18 +648,18 @@ impl FileEntryManager { } fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { - let file_node = FileEntry { + let file = FileEntry { file_id, parent_file_id, file_name: file_name.to_string(), }; - self.file_id_map.insert(file_id, file_node.clone()); - self.file_name_map.insert(file_name.to_string(), file_node); + self.file_id_map.insert(file_id, file.clone()); + self.file_name_map.insert(file_name.to_string(), file); } fn remove(&mut self, file_name: &str) { - if let Some(node) = self.file_name_map.remove(file_name) { - self.file_id_map.remove(&node.file_id); + if let Some(file) = self.file_name_map.remove(file_name) { + self.file_id_map.remove(&file.file_id); } } } From d0b6958025d19e69f0b02e92205bffae76704d50 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:17:51 +0800 Subject: [PATCH 03/31] Add uts --- clients/filesystem-fuse/Makefile | 3 +- clients/filesystem-fuse/src/filesystem.rs | 82 +++++++++++++++++++ .../src/opened_file_manager.rs | 62 ++++++++++++++ clients/filesystem-fuse/src/utils.rs | 23 ++++++ 4 files changed, 168 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile index 2e48a3e1fd8..9a9cbe042a6 100644 --- a/clients/filesystem-fuse/Makefile +++ b/clients/filesystem-fuse/Makefile @@ -31,8 +31,7 @@ check-fmt: cargo fmt --all -- --check check-clippy: - #cargo clippy --all-targets --all-features --workspace -- -D warnings - cargo clippy --all-targets --all-features --workspace -- + cargo clippy --all-targets --all-features --workspace -- -D warnings install-cargo-sort: cargo install cargo-sort@1.0.9 diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index ba0c04dc5d9..4c0d716d91c 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -663,3 +663,85 @@ impl FileEntryManager { } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_create_file_stat() { + //test new file + let file_stat = FileStat::new_file("a", "b", 10); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 10); + assert_eq!(file_stat.kind, FileType::RegularFile); + + //test new dir + let file_stat = FileStat::new_dir("a", "b"); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 0); + assert_eq!(file_stat.kind, FileType::Directory); + + //test new file with path + let file_stat = FileStat::new_file_with_path("a/b", 10); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 10); + assert_eq!(file_stat.kind, FileType::RegularFile); + + //teset new dir with path + let file_stat = FileStat::new_dir_with_path("a/b"); + assert_eq!(file_stat.name, "b"); + assert_eq!(file_stat.path, "a/b"); + assert_eq!(file_stat.size, 0); + assert_eq!(file_stat.kind, FileType::Directory); + } + + #[test] + fn test_file_stat_set_file_id() { + let mut file_stat = FileStat::new_file("a", "b", 10); + file_stat.set_file_id(1, 2); + assert_eq!(file_stat.file_id, 2); + assert_eq!(file_stat.parent_file_id, 1); + } + + #[test] + #[should_panic(expected = "assertion failed: file_id != 0 && parent_file_id != 0")] + fn test_file_stat_set_file_id_panic() { + let mut file_stat = FileStat::new_file("a", "b", 10); + file_stat.set_file_id(1, 0); + } + + #[test] + fn test_open_file() { + let mut open_file = OpenedFile::new(FileStat::new_file("a", "b", 10)); + assert_eq!(open_file.file_stat.name, "b"); + assert_eq!(open_file.file_stat.size, 10); + + open_file.set_file_id(1, 2); + + assert_eq!(open_file.file_stat.file_id, 2); + assert_eq!(open_file.file_stat.parent_file_id, 1); + } + + #[test] + fn test_file_entry_manager() { + let mut manager = FileEntryManager::new(); + manager.insert(1, 2, "a/b"); + let file = manager.get_file_by_id(2).unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.file_name, "a/b"); + + let file = manager.get_file_by_name("a/b").unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.file_name, "a/b"); + + manager.remove("a/b"); + assert!(manager.get_file_by_id(2).is_none()); + assert!(manager.get_file_by_name("a/b").is_none()); + } +} diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 664f0a89875..03567485a4c 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -63,3 +63,65 @@ impl OpenedFileManager { self.file_handle_map.remove(&handle_id).map(|x| x.1) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::FileStat; + + #[tokio::test] + async fn test_opened_file_manager() { + let manager = OpenedFileManager::new(); + + let file1_stat = FileStat::new_file("", "a.txt", 13); + let file2_stat = FileStat::new_file("", "b.txt", 18); + + let file1 = OpenedFile::new(file1_stat.clone()); + let file2 = OpenedFile::new(file2_stat.clone()); + + let handle_id1 = manager.put_file(file1).lock().await.handle_id; + let handle_id2 = manager.put_file(file2).lock().await.handle_id; + + // Test the file handle id is assigned. + assert!(handle_id1 > 0 && handle_id2 > 0); + assert_ne!(handle_id1, handle_id2); + + // test get file by handle id + assert_eq!( + manager + .get_file(handle_id1) + .unwrap() + .lock() + .await + .file_stat + .name, + file1_stat.name + ); + + assert_eq!( + manager + .get_file(handle_id2) + .unwrap() + .lock() + .await + .file_stat + .name, + file2_stat.name + ); + + // test remove file by handle id + assert_eq!( + manager + .remove_file(handle_id1) + .unwrap() + .lock() + .await + .handle_id, + handle_id1 + ); + + // test get file by handle id after remove + assert!(manager.get_file(handle_id1).is_none()); + assert!(manager.get_file(handle_id2).is_some()); + } +} diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 6118b672362..7ac0c7212a7 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -33,3 +33,26 @@ pub fn split_file_path(path: &str) -> (&str, &str) { None => ("", path), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_join_file_path() { + assert_eq!(join_file_path("", "a"), "a"); + assert_eq!(join_file_path("", "a.txt"), "a.txt"); + assert_eq!(join_file_path("a", "b"), "a/b"); + assert_eq!(join_file_path("a/b", "c"), "a/b/c"); + assert_eq!(join_file_path("a/b", "c.txt"), "a/b/c.txt"); + } + + #[test] + fn test_split_file_path() { + assert_eq!(split_file_path("a"), ("", "a")); + assert_eq!(split_file_path("a.txt"), ("", "a.txt")); + assert_eq!(split_file_path("a/b"), ("a", "b")); + assert_eq!(split_file_path("a/b/c"), ("a/b", "c")); + assert_eq!(split_file_path("a/b/c.txt"), ("a/b", "c.txt")); + } +} From 3769400a32a0cb109323f40e46e61ae0a6a9ff7a Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:26:37 +0800 Subject: [PATCH 04/31] Fix test error --- clients/filesystem-fuse/Makefile | 6 +++--- clients/filesystem-fuse/src/filesystem.rs | 2 +- clients/filesystem-fuse/src/fuse_api_handle.rs | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/Makefile b/clients/filesystem-fuse/Makefile index 9a9cbe042a6..f4a4cef20ae 100644 --- a/clients/filesystem-fuse/Makefile +++ b/clients/filesystem-fuse/Makefile @@ -27,6 +27,9 @@ fmt: cargo-sort: install-cargo-sort cargo sort -w +fix-toml: install-taplo-cli + taplo fmt + check-fmt: cargo fmt --all -- --check @@ -48,9 +51,6 @@ cargo-machete: install-cargo-machete install-taplo-cli: cargo install taplo-cli@0.9.0 -fix-toml: install-taplo-cli - taplo fmt - check-toml: install-taplo-cli taplo check diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 4c0d716d91c..f42418fde03 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -691,7 +691,7 @@ mod tests { assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); - //teset new dir with path + //test new dir with path let file_stat = FileStat::new_dir_with_path("a/b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 800930f4ed3..e9f58f55045 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -496,7 +496,7 @@ mod test { assert_eq!(file_attr.mtime, Timestamp { sec: 12, nsec: 5 }); assert_eq!(file_attr.ctime, Timestamp { sec: 15, nsec: 7 }); assert_eq!(file_attr.kind, FileType::RegularFile); - assert_eq!(file_attr.perm, 0); + assert_eq!(file_attr.perm, context.default_file_perm); assert_eq!(file_attr.nlink, 0); assert_eq!(file_attr.uid, 1); assert_eq!(file_attr.gid, 2); From 3ecb9805ae7d0cf1fe8096595d6875591596933d Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:44:48 +0800 Subject: [PATCH 05/31] Update comments --- clients/filesystem-fuse/src/filesystem.rs | 34 ++++++++++++------- .../filesystem-fuse/src/fuse_api_handle.rs | 4 +-- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index f42418fde03..2578ab0b4d3 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -44,7 +44,7 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn get_file_path(&self, file_id: u64) -> String; /// Validate the file id and file handle, if file id and file handle is valid and it associated, return Ok - async fn valid_file_id(&self, file_id: u64, fh: u64) -> Result<()>; + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()>; /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; @@ -260,7 +260,7 @@ impl OpenedFile { let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; let result = reader.read(offset, size).await?; - // update the access time + // update the atime self.file_stat.atime = Timestamp::from(SystemTime::now()); Ok(result) @@ -270,7 +270,7 @@ impl OpenedFile { let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; let written = writer.write(offset, data).await?; - // update the file size + // update the file size ,mtime and atime let end = offset + data.len() as u64; if end > self.file_stat.size { self.file_stat.size = end; @@ -365,7 +365,7 @@ pub struct SimpleFileSystem { /// inode id generator file_id_generator: AtomicU64, - /// real system + /// real filesystem fs: T, } @@ -411,6 +411,7 @@ impl SimpleFileSystem { file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); } Some(file) => { + // use the exist file id file_stat.set_file_id(file.parent_file_id, file.file_id); } } @@ -439,6 +440,7 @@ impl SimpleFileSystem { _ => return Err(Errno::from(libc::EINVAL)), } }; + // set the exists file id file.set_file_id(file_entry.parent_file_id, file_id); let file = self.opened_file_manager.put_file(file); let file = file.lock().await; @@ -449,6 +451,7 @@ impl SimpleFileSystem { #[async_trait] impl RawFileSystem for SimpleFileSystem { async fn init(&self) -> Result<()> { + // init root directory self.file_entry_manager.write().await.insert( Self::ROOT_DIR_PARENT_FILE_ID, Self::ROOT_DIR_FILE_ID, @@ -462,8 +465,8 @@ impl RawFileSystem for SimpleFileSystem { file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) } - async fn valid_file_id(&self, _file_id: u64, fh: u64) -> Result<()> { - let file_id = self + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { + let fh_file_id = self .opened_file_manager .get_file(fh) .ok_or(Errno::from(libc::EBADF))? @@ -472,7 +475,7 @@ impl RawFileSystem for SimpleFileSystem { .file_stat .file_id; - (file_id == _file_id) + (file_id == fh_file_id) .then_some(()) .ok_or(Errno::from(libc::EBADF)) } @@ -487,6 +490,7 @@ impl RawFileSystem for SimpleFileSystem { async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { let parent_file = self.get_file_entry(parent_file_id).await?; let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; + // fill the file id to file stat self.fill_file_id(&mut stat, parent_file_id).await; Ok(stat) } @@ -518,14 +522,14 @@ impl RawFileSystem for SimpleFileSystem { .await?; file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - file.file_stat.parent_file_id, - file.file_stat.file_id, - &file.file_stat.path, - ); + file_manager.insert(parent_file_id, file.file_stat.file_id, &file.file_stat.path); } + + // put the file to the opened file manager let file = self.opened_file_manager.put_file(file); let file = file.lock().await; Ok(file.file_handle()) @@ -536,6 +540,8 @@ impl RawFileSystem for SimpleFileSystem { let mut file = self.fs.create_dir(&parent.file_name, name).await?; file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; file_manager.insert(file.parent_file_id, file.file_id, &file.path); @@ -552,6 +558,7 @@ impl RawFileSystem for SimpleFileSystem { let parent_file = self.get_file_entry(parent_file_id).await?; self.fs.remove_file(&parent_file.file_name, name).await?; + // remove the file from file entry manager { let mut file_id_manager = self.file_entry_manager.write().await; file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); @@ -563,6 +570,7 @@ impl RawFileSystem for SimpleFileSystem { let parent_file = self.get_file_entry(parent_file_id).await?; self.fs.remove_dir(&parent_file.file_name, name).await?; + // remove the dir from file entry manager { let mut file_id_manager = self.file_entry_manager.write().await; file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); @@ -592,6 +600,7 @@ impl RawFileSystem for SimpleFileSystem { opened_file.read(offset, size).await }; + // update the file atime self.fs.set_attr(&file_stat.path, &file_stat, false).await?; data @@ -608,6 +617,7 @@ impl RawFileSystem for SimpleFileSystem { (len, opened_file.file_stat.clone()) }; + // update the file size, mtime and atime self.fs.set_attr(&file_stat.path, &file_stat, false).await?; len diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index e9f58f55045..76af581446b 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -117,7 +117,7 @@ impl Filesystem for FuseApiHandle { ) -> fuse3::Result { // check the fh is associated with the file_id if let Some(fh) = fh { - self.fs.valid_file_id(inode, fh).await?; + self.fs.valid_file_handle_id(inode, fh).await?; } let file_stat = self.fs.stat(inode).await?; @@ -136,7 +136,7 @@ impl Filesystem for FuseApiHandle { ) -> fuse3::Result { // check the fh is associated with the file_id if let Some(fh) = fh { - self.fs.valid_file_id(inode, fh).await?; + self.fs.valid_file_handle_id(inode, fh).await?; } let new_file_stat = self From 4fe58f27a5637104d69046b347d8818bd5395328 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 17 Dec 2024 15:53:14 +0800 Subject: [PATCH 06/31] Fix ci error --- clients/filesystem-fuse/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 201397c19e7..7e20e13cedd 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -40,7 +40,7 @@ val checkRustProject by tasks.registering(Exec::class) { description = "Check the Rust project" workingDir = file("$projectDir") - commandLine( "bash", "-c", "make check") + commandLine("bash", "-c", "make check") } val testRustProject by tasks.registering(Exec::class) { From a55bff87a27c9a098e457ac8f24b1d83ef3b712c Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:11:33 +0800 Subject: [PATCH 07/31] Add description of gvfs-fuse filesystem struct --- clients/filesystem-fuse/src/main.rs | 48 +++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 24358840df5..e713e743fb8 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -32,3 +32,51 @@ async fn main() { debug!("Shutdown filesystem..."); exit(0); } + +async fn create_gvfs_fuse_filesystem() { + // Gvfs-fuse filesystem structure: + // FuseApiHandle + // ├─ SimpleFileSystem (RawFileSystem) + // │ └─ FileSystemLog (PathFileSystem) + // ├─ GravitinoComposedFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ └─ S3FileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ └─ HDFSFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ NasFileSystem (PathFileSystem) + // │ │ └─ JuiceFileSystem (PathFileSystem) + // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ └─ XXXFileSystem (PathFileSystem) + // + // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. + // It manages file and directory relationships, as well as file mappings. + // It delegates file operations to the PathFileSystem + // + // `FileSystemLog` is a decorator that adds extra debug logging functionality to file system APIs. + // Similar implementations include permissions, caching, and metrics. + // + // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. + // It translates the part of catalog and schema of fileset path to a signal GravitinoFilesetFileSystem path. + // If the user only mounts a fileset, this layer is not present. There will only be one below layer. + // + // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. + // and delegate the operation to the real storage. + // + // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. + // it can assess the S3, HDFS, gcs, azblob and other storage. + // + // `S3FileSystem` is a file system that use to access S3 storage. + // + // `HDFSFileSystem` is a file system that use to access HDFS storage. + + // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. + + // `JuiceFileSystem` is a file system that use to manage juice mount and access JuiceFS storage. + + // `XXXXFileSystem` is a file system that use implement access file by your extent. + + todo!("Implement the createGvfsFuseFileSystem function"); +} From 58922e14f14f276e51ecbb57b769eaed85d9ec97 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:16:13 +0800 Subject: [PATCH 08/31] Update --- clients/filesystem-fuse/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index e713e743fb8..69b0a75e054 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -74,9 +74,9 @@ async fn create_gvfs_fuse_filesystem() { // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. - // `JuiceFileSystem` is a file system that use to manage juice mount and access JuiceFS storage. + // `JuiceFileSystem` is a file system that use to manage JuiceFS mount and access JuiceFS storage. - // `XXXXFileSystem` is a file system that use implement access file by your extent. + // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. todo!("Implement the createGvfsFuseFileSystem function"); } From cda8275d86da50816d70367e142a2fde0fd27b9c Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 11:18:11 +0800 Subject: [PATCH 09/31] Update --- clients/filesystem-fuse/src/main.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 69b0a75e054..6935a6b8e2b 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -37,19 +37,19 @@ async fn create_gvfs_fuse_filesystem() { // Gvfs-fuse filesystem structure: // FuseApiHandle // ├─ SimpleFileSystem (RawFileSystem) - // │ └─ FileSystemLog (PathFileSystem) - // ├─ GravitinoComposedFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ └─ S3FileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ └─ HDFSFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ NasFileSystem (PathFileSystem) - // │ │ └─ JuiceFileSystem (PathFileSystem) - // │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ └─ XXXFileSystem (PathFileSystem) + // │ └─ FileSystemLog (PathFileSystem) + // │ ├─ GravitinoComposedFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) + // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ NasFileSystem (PathFileSystem) + // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) + // │ │ │ └─ XXXFileSystem (PathFileSystem) // // `SimpleFileSystem` is a low-level filesystem designed to communicate with FUSE APIs. // It manages file and directory relationships, as well as file mappings. From 5f9371787add08280bc66546fe2145aa817949e3 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 14:14:17 +0800 Subject: [PATCH 10/31] Update --- clients/filesystem-fuse/src/main.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 6935a6b8e2b..244de85a7bd 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -40,14 +40,14 @@ async fn create_gvfs_fuse_filesystem() { // │ └─ FileSystemLog (PathFileSystem) // │ ├─ GravitinoComposedFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ │ └─ S3FileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ OpenDALFileSystem (PathFileSystem) - // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ │ └─ HDFSFileSystem (PathFileSystem) + // │ │ │ └─ OpenDALFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) - // │ │ │ └─ NasFileSystem (PathFileSystem) - // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ │ └─ JuiceFileSystem (PathFileSystem) + // │ │ │ └─ NasFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) // │ │ │ └─ XXXFileSystem (PathFileSystem) // @@ -59,7 +59,7 @@ async fn create_gvfs_fuse_filesystem() { // Similar implementations include permissions, caching, and metrics. // // `GravitinoComposeFileSystem` is a composite file system that can combine multiple `GravitinoFilesetFileSystem`. - // It translates the part of catalog and schema of fileset path to a signal GravitinoFilesetFileSystem path. + // It use the part of catalog and schema of fileset path to a find actual GravitinoFilesetFileSystem. delegate the operation to the real storage. // If the user only mounts a fileset, this layer is not present. There will only be one below layer. // // `GravitinoFilesetFileSystem` is a file system that can access a fileset.It translates the fileset path to the real storage path. @@ -68,14 +68,14 @@ async fn create_gvfs_fuse_filesystem() { // `OpenDALFileSystem` is a file system that use the OpenDAL to access real storage. // it can assess the S3, HDFS, gcs, azblob and other storage. // - // `S3FileSystem` is a file system that use to access S3 storage. + // `S3FileSystem` is a file system that use `OpenDALFileSystem` to access S3 storage. + // + // `HDFSFileSystem` is a file system that use `OpenDALFileSystem` to access HDFS storage. // - // `HDFSFileSystem` is a file system that use to access HDFS storage. - // `NasFileSystem` is a filesystem that uses a locally accessible path mounted by NAS tools, such as JuiceFS. - - // `JuiceFileSystem` is a file system that use to manage JuiceFS mount and access JuiceFS storage. - + // + // `JuiceFileSystem` is a file that use `NasFileSystem` to access JuiceFS storage. + // // `XXXFileSystem is a filesystem that allows you to implement file access through your own extensions. todo!("Implement the createGvfsFuseFileSystem function"); From 39e55fed30507d05098858ff5751ebdfa578fb25 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 19:28:32 +0800 Subject: [PATCH 11/31] Update for review --- clients/filesystem-fuse/src/filesystem.rs | 196 ++++++++++-------- clients/filesystem-fuse/src/main.rs | 2 +- .../src/opened_file_manager.rs | 4 +- 3 files changed, 110 insertions(+), 92 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 2578ab0b4d3..3e56b9e20f6 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -194,25 +194,25 @@ pub struct FileStat { } impl FileStat { - pub fn new_file_with_path(path: &str, size: u64) -> Self { + pub fn new_file_filestat_with_path(path: &str, size: u64) -> Self { let (parent, name) = split_file_path(path); - Self::new_file(parent, name, size) + Self::new_file_filestat(parent, name, size) } - pub fn new_dir_with_path(path: &str) -> Self { + pub fn new_dir_filestat_with_path(path: &str) -> Self { let (parent, name) = split_file_path(path); - Self::new_dir(parent, name) + Self::new_dir_filestat(parent, name) } - pub fn new_file(parent: &str, name: &str, size: u64) -> Self { - Self::new_file_entry(parent, name, size, FileType::RegularFile) + pub fn new_file_filestat(parent: &str, name: &str, size: u64) -> Self { + Self::new_filestat(parent, name, size, FileType::RegularFile) } - pub fn new_dir(parent: &str, name: &str) -> Self { - Self::new_file_entry(parent, name, 0, FileType::Directory) + pub fn new_dir_filestat(parent: &str, name: &str) -> Self { + Self::new_filestat(parent, name, 0, FileType::Directory) } - pub fn new_file_entry(parent: &str, name: &str, size: u64, kind: FileType) -> Self { + pub fn new_filestat(parent: &str, name: &str, size: u64, kind: FileType) -> Self { let atime = Timestamp::from(SystemTime::now()); Self { file_id: 0, @@ -282,12 +282,24 @@ impl OpenedFile { } async fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); if let Some(mut reader) = self.reader.take() { - reader.close().await?; + if let Err(e) = reader.close().await { + errors.push(e); + } } + if let Some(mut writer) = self.writer.take() { - self.flush().await?; - writer.close().await? + if let Err(e) = self.flush().await { + errors.push(e); + } + if let Err(e) = writer.close().await { + errors.push(e); + } + } + + if !errors.is_empty() { + return Err(errors.remove(0)); } Ok(()) } @@ -357,7 +369,7 @@ pub trait FileWriter: Sync + Send { /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. /// Note: This class is not use in the production code, it is used for the demo and testing -pub struct SimpleFileSystem { +pub struct DefaultRawFileSystem { /// file entries file_entry_manager: RwLock, /// opened files @@ -369,7 +381,7 @@ pub struct SimpleFileSystem { fs: T, } -impl SimpleFileSystem { +impl DefaultRawFileSystem { const INITIAL_FILE_ID: u64 = 10000; const ROOT_DIR_PARENT_FILE_ID: u64 = 0; const ROOT_DIR_FILE_ID: u64 = 1; @@ -398,13 +410,13 @@ impl SimpleFileSystem { } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_file_by_name(path) + self.file_entry_manager.read().await.get_file_by_path(path) } - async fn fill_file_id(&self, file_stat: &mut FileStat, parent_file_id: u64) { + async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file = file_manager.get_file_by_name(&file_stat.path); - match file { + let file_entry = file_manager.get_file_by_path(&file_stat.path); + match file_entry { None => { // allocate new file id file_stat.set_file_id(parent_file_id, self.next_file_id()); @@ -425,31 +437,31 @@ impl SimpleFileSystem { ) -> Result { let file_entry = self.get_file_entry(file_id).await?; - let mut file = { + let mut opened_file = { match kind { FileType::Directory => { self.fs - .open_dir(&file_entry.file_name, OpenFileFlags(flags)) + .open_dir(&file_entry.path, OpenFileFlags(flags)) .await? } FileType::RegularFile => { self.fs - .open_file(&file_entry.file_name, OpenFileFlags(flags)) + .open_file(&file_entry.path, OpenFileFlags(flags)) .await? } _ => return Err(Errno::from(libc::EINVAL)), } }; // set the exists file id - file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put_file(file); + opened_file.set_file_id(file_entry.parent_file_id, file_id); + let file = self.opened_file_manager.put_file(opened_file); let file = file.lock().await; Ok(file.file_handle()) } } #[async_trait] -impl RawFileSystem for SimpleFileSystem { +impl RawFileSystem for DefaultRawFileSystem { async fn init(&self) -> Result<()> { // init root directory self.file_entry_manager.write().await.insert( @@ -461,8 +473,10 @@ impl RawFileSystem for SimpleFileSystem { } async fn get_file_path(&self, file_id: u64) -> String { - let file = self.get_file_entry(file_id).await; - file.map(|x| x.file_name).unwrap_or_else(|_| "".to_string()) + let file_entry = self.get_file_entry(file_id).await; + file_entry + .map(|x| x.path) + .unwrap_or_else(|_| "".to_string()) } async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { @@ -481,27 +495,28 @@ impl RawFileSystem for SimpleFileSystem { } async fn stat(&self, file_id: u64) -> Result { - let file = self.get_file_entry(file_id).await?; - let mut stat = self.fs.stat(&file.file_name).await?; - stat.set_file_id(file.parent_file_id, file.file_id); - Ok(stat) + let file_entry = self.get_file_entry(file_id).await?; + let mut file_stat = self.fs.stat(&file_entry.path).await?; + file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); + Ok(file_stat) } async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file = self.get_file_entry(parent_file_id).await?; - let mut stat = self.fs.lookup(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; // fill the file id to file stat - self.fill_file_id(&mut stat, parent_file_id).await; - Ok(stat) + self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) + .await; + Ok(file_stat) } async fn read_dir(&self, file_id: u64) -> Result> { - let file = self.get_file_entry(file_id).await?; - let mut files = self.fs.read_dir(&file.file_name).await?; - for file in files.iter_mut() { - self.fill_file_id(file, file.file_id).await; + let file_entry = self.get_file_entry(file_id).await?; + let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; + for file in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file, file.file_id).await; } - Ok(files) + Ok(child_filestats) } async fn open_file(&self, file_id: u64, flags: u32) -> Result { @@ -515,77 +530,80 @@ impl RawFileSystem for SimpleFileSystem { } async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent = self.get_file_entry(parent_file_id).await?; - let mut file = self + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut opened_file = self .fs - .create_file(&parent.file_name, name, OpenFileFlags(flags)) + .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - file.set_file_id(parent_file_id, self.next_file_id()); + opened_file.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(parent_file_id, file.file_stat.file_id, &file.file_stat.path); + file_manager.insert( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ); } // put the file to the opened file manager - let file = self.opened_file_manager.put_file(file); - let file = file.lock().await; - Ok(file.file_handle()) + let opened_file = self.opened_file_manager.put_file(opened_file); + let opened_file = opened_file.lock().await; + Ok(opened_file.file_handle()) } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent = self.get_file_entry(parent_file_id).await?; - let mut file = self.fs.create_dir(&parent.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; - file.set_file_id(parent_file_id, self.next_file_id()); + filestat.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager { let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(file.parent_file_id, file.file_id, &file.path); + file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); } - Ok(file.file_id) + Ok(filestat.file_id) } async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file = self.get_file_entry(file_id).await?; - self.fs.set_attr(&file.file_name, file_stat, true).await + let file_entry = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file_entry.path, file_stat, true).await } async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file = self.get_file_entry(parent_file_id).await?; - self.fs.remove_file(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file_entry.path, name).await?; // remove the file from file entry manager { - let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); } Ok(()) } async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file = self.get_file_entry(parent_file_id).await?; - self.fs.remove_dir(&parent_file.file_name, name).await?; + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file_entry.path, name).await?; // remove the dir from file entry manager { - let mut file_id_manager = self.file_entry_manager.write().await; - file_id_manager.remove(&join_file_path(&parent_file.file_name, name)); + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); } Ok(()) } async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { - let file = self + let opened_file = self .opened_file_manager .remove_file(fh) .ok_or(Errno::from(libc::EBADF))?; - let mut file = file.lock().await; - file.close().await?; - Ok(()) + let mut file = opened_file.lock().await; + file.close().await } async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { @@ -629,23 +647,23 @@ impl RawFileSystem for SimpleFileSystem { struct FileEntry { file_id: u64, parent_file_id: u64, - file_name: String, + path: String, } /// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. struct FileEntryManager { - // file_id_map is a map of file_id to file name. + // file_id_map is a map of file_id to file entry. file_id_map: HashMap, - // file_name_map is a map of file name to file id. - file_name_map: HashMap, + // file_path_map is a map of file path to file entry. + file_path_map: HashMap, } impl FileEntryManager { fn new() -> Self { Self { file_id_map: HashMap::new(), - file_name_map: HashMap::new(), + file_path_map: HashMap::new(), } } @@ -653,22 +671,22 @@ impl FileEntryManager { self.file_id_map.get(&file_id).cloned() } - fn get_file_by_name(&self, file_name: &str) -> Option { - self.file_name_map.get(file_name).cloned() + fn get_file_by_path(&self, path: &str) -> Option { + self.file_path_map.get(path).cloned() } - fn insert(&mut self, parent_file_id: u64, file_id: u64, file_name: &str) { + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { let file = FileEntry { file_id, parent_file_id, - file_name: file_name.to_string(), + path: path.to_string(), }; self.file_id_map.insert(file_id, file.clone()); - self.file_name_map.insert(file_name.to_string(), file); + self.file_path_map.insert(path.to_string(), file); } - fn remove(&mut self, file_name: &str) { - if let Some(file) = self.file_name_map.remove(file_name) { + fn remove(&mut self, path: &str) { + if let Some(file) = self.file_path_map.remove(path) { self.file_id_map.remove(&file.file_id); } } @@ -681,28 +699,28 @@ mod tests { #[test] fn test_create_file_stat() { //test new file - let file_stat = FileStat::new_file("a", "b", 10); + let file_stat = FileStat::new_file_filestat("a", "b", 10); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir - let file_stat = FileStat::new_dir("a", "b"); + let file_stat = FileStat::new_dir_filestat("a", "b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 0); assert_eq!(file_stat.kind, FileType::Directory); //test new file with path - let file_stat = FileStat::new_file_with_path("a/b", 10); + let file_stat = FileStat::new_file_filestat_with_path("a/b", 10); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 10); assert_eq!(file_stat.kind, FileType::RegularFile); //test new dir with path - let file_stat = FileStat::new_dir_with_path("a/b"); + let file_stat = FileStat::new_dir_filestat_with_path("a/b"); assert_eq!(file_stat.name, "b"); assert_eq!(file_stat.path, "a/b"); assert_eq!(file_stat.size, 0); @@ -711,7 +729,7 @@ mod tests { #[test] fn test_file_stat_set_file_id() { - let mut file_stat = FileStat::new_file("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 2); assert_eq!(file_stat.file_id, 2); assert_eq!(file_stat.parent_file_id, 1); @@ -720,13 +738,13 @@ mod tests { #[test] #[should_panic(expected = "assertion failed: file_id != 0 && parent_file_id != 0")] fn test_file_stat_set_file_id_panic() { - let mut file_stat = FileStat::new_file("a", "b", 10); + let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } #[test] fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file("a", "b", 10)); + let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); assert_eq!(open_file.file_stat.name, "b"); assert_eq!(open_file.file_stat.size, 10); @@ -743,15 +761,15 @@ mod tests { let file = manager.get_file_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.file_name, "a/b"); + assert_eq!(file.path, "a/b"); - let file = manager.get_file_by_name("a/b").unwrap(); + let file = manager.get_file_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); - assert_eq!(file.file_name, "a/b"); + assert_eq!(file.path, "a/b"); manager.remove("a/b"); assert!(manager.get_file_by_id(2).is_none()); - assert!(manager.get_file_by_name("a/b").is_none()); + assert!(manager.get_file_by_path("a/b").is_none()); } } diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 244de85a7bd..25a3b1579ce 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -36,7 +36,7 @@ async fn main() { async fn create_gvfs_fuse_filesystem() { // Gvfs-fuse filesystem structure: // FuseApiHandle - // ├─ SimpleFileSystem (RawFileSystem) + // ├─ DefaultRawFileSystem (RawFileSystem) // │ └─ FileSystemLog (PathFileSystem) // │ ├─ GravitinoComposedFileSystem (PathFileSystem) // │ │ ├─ GravitinoFilesetFileSystem (PathFileSystem) diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 03567485a4c..c7a47cb306e 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -73,8 +73,8 @@ mod tests { async fn test_opened_file_manager() { let manager = OpenedFileManager::new(); - let file1_stat = FileStat::new_file("", "a.txt", 13); - let file2_stat = FileStat::new_file("", "b.txt", 18); + let file1_stat = FileStat::new_file_filestat("", "a.txt", 13); + let file2_stat = FileStat::new_file_filestat("", "b.txt", 18); let file1 = OpenedFile::new(file1_stat.clone()); let file2 = OpenedFile::new(file2_stat.clone()); From 366c77a3d95db4d77835d6392aee2af238eaea66 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 19:56:20 +0800 Subject: [PATCH 12/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 3e56b9e20f6..bec9ccc004a 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -383,7 +383,7 @@ pub struct DefaultRawFileSystem { impl DefaultRawFileSystem { const INITIAL_FILE_ID: u64 = 10000; - const ROOT_DIR_PARENT_FILE_ID: u64 = 0; + const ROOT_DIR_PARENT_FILE_ID: u64 = 1; const ROOT_DIR_FILE_ID: u64 = 1; const ROOT_DIR_NAME: &'static str = ""; From 33dc3bd8142ae58e6dfda61ab3590d08ec4553df Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 21:50:43 +0800 Subject: [PATCH 13/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index bec9ccc004a..a1625504707 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -271,7 +271,7 @@ impl OpenedFile { let written = writer.write(offset, data).await?; // update the file size ,mtime and atime - let end = offset + data.len() as u64; + let end = offset + written as u64; if end > self.file_stat.size { self.file_stat.size = end; } @@ -365,7 +365,7 @@ pub trait FileWriter: Sync + Send { } } -/// SimpleFileSystem is a simple implementation for the file system. +/// DefaultRawFileSystem is a simple implementation for the file system. /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. /// Note: This class is not use in the production code, it is used for the demo and testing @@ -676,13 +676,13 @@ impl FileEntryManager { } fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { - let file = FileEntry { + let file_entry = FileEntry { file_id, parent_file_id, path: path.to_string(), }; - self.file_id_map.insert(file_id, file.clone()); - self.file_path_map.insert(path.to_string(), file); + self.file_id_map.insert(file_id, file_entry.clone()); + self.file_path_map.insert(path.to_string(), file_entry); } fn remove(&mut self, path: &str) { From 865f21315d0b4705b4bd33d370aef36da113c7a4 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 21:56:26 +0800 Subject: [PATCH 14/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 30 +++++++-------- .../src/opened_file_manager.rs | 37 +++++-------------- 2 files changed, 25 insertions(+), 42 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index a1625504707..fbc0386282c 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -405,17 +405,17 @@ impl DefaultRawFileSystem { self.file_entry_manager .read() .await - .get_file_by_id(file_id) + .get_by_id(file_id) .ok_or(Errno::from(libc::ENOENT)) } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_file_by_path(path) + self.file_entry_manager.read().await.get_by_path(path) } async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_file_by_path(&file_stat.path); + let file_entry = file_manager.get_by_path(&file_stat.path); match file_entry { None => { // allocate new file id @@ -454,7 +454,7 @@ impl DefaultRawFileSystem { }; // set the exists file id opened_file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put_file(opened_file); + let file = self.opened_file_manager.put(opened_file); let file = file.lock().await; Ok(file.file_handle()) } @@ -482,7 +482,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { let fh_file_id = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))? .lock() .await @@ -549,7 +549,7 @@ impl RawFileSystem for DefaultRawFileSystem { } // put the file to the opened file manager - let opened_file = self.opened_file_manager.put_file(opened_file); + let opened_file = self.opened_file_manager.put(opened_file); let opened_file = opened_file.lock().await; Ok(opened_file.file_handle()) } @@ -600,7 +600,7 @@ impl RawFileSystem for DefaultRawFileSystem { async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { let opened_file = self .opened_file_manager - .remove_file(fh) + .remove(fh) .ok_or(Errno::from(libc::EBADF))?; let mut file = opened_file.lock().await; file.close().await @@ -611,7 +611,7 @@ impl RawFileSystem for DefaultRawFileSystem { let data = { let opened_file = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; file_stat = opened_file.file_stat.clone(); @@ -628,7 +628,7 @@ impl RawFileSystem for DefaultRawFileSystem { let (len, file_stat) = { let opened_file = self .opened_file_manager - .get_file(fh) + .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; let len = opened_file.write(offset, data).await; @@ -667,11 +667,11 @@ impl FileEntryManager { } } - fn get_file_by_id(&self, file_id: u64) -> Option { + fn get_by_id(&self, file_id: u64) -> Option { self.file_id_map.get(&file_id).cloned() } - fn get_file_by_path(&self, path: &str) -> Option { + fn get_by_path(&self, path: &str) -> Option { self.file_path_map.get(path).cloned() } @@ -758,18 +758,18 @@ mod tests { fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); manager.insert(1, 2, "a/b"); - let file = manager.get_file_by_id(2).unwrap(); + let file = manager.get_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); - let file = manager.get_file_by_path("a/b").unwrap(); + let file = manager.get_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); manager.remove("a/b"); - assert!(manager.get_file_by_id(2).is_none()); - assert!(manager.get_file_by_path("a/b").is_none()); + assert!(manager.get_by_id(2).is_none()); + assert!(manager.get_by_path("a/b").is_none()); } } diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index c7a47cb306e..695d69adca4 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -44,7 +44,7 @@ impl OpenedFileManager { .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - pub(crate) fn put_file(&self, mut file: OpenedFile) -> Arc> { + pub(crate) fn put(&self, mut file: OpenedFile) -> Arc> { let file_handle_id = self.next_handle_id(); file.handle_id = file_handle_id; let file_handle = Arc::new(Mutex::new(file)); @@ -53,13 +53,13 @@ impl OpenedFileManager { file_handle } - pub(crate) fn get_file(&self, handle_id: u64) -> Option>> { + pub(crate) fn get(&self, handle_id: u64) -> Option>> { self.file_handle_map .get(&handle_id) .map(|x| x.value().clone()) } - pub(crate) fn remove_file(&self, handle_id: u64) -> Option>> { + pub(crate) fn remove(&self, handle_id: u64) -> Option>> { self.file_handle_map.remove(&handle_id).map(|x| x.1) } } @@ -79,8 +79,8 @@ mod tests { let file1 = OpenedFile::new(file1_stat.clone()); let file2 = OpenedFile::new(file2_stat.clone()); - let handle_id1 = manager.put_file(file1).lock().await.handle_id; - let handle_id2 = manager.put_file(file2).lock().await.handle_id; + let handle_id1 = manager.put(file1).lock().await.handle_id; + let handle_id2 = manager.put(file2).lock().await.handle_id; // Test the file handle id is assigned. assert!(handle_id1 > 0 && handle_id2 > 0); @@ -88,40 +88,23 @@ mod tests { // test get file by handle id assert_eq!( - manager - .get_file(handle_id1) - .unwrap() - .lock() - .await - .file_stat - .name, + manager.get(handle_id1).unwrap().lock().await.file_stat.name, file1_stat.name ); assert_eq!( - manager - .get_file(handle_id2) - .unwrap() - .lock() - .await - .file_stat - .name, + manager.get(handle_id2).unwrap().lock().await.file_stat.name, file2_stat.name ); // test remove file by handle id assert_eq!( - manager - .remove_file(handle_id1) - .unwrap() - .lock() - .await - .handle_id, + manager.remove(handle_id1).unwrap().lock().await.handle_id, handle_id1 ); // test get file by handle id after remove - assert!(manager.get_file(handle_id1).is_none()); - assert!(manager.get_file(handle_id2).is_some()); + assert!(manager.get(handle_id1).is_none()); + assert!(manager.get(handle_id2).is_some()); } } From 2570eb8f7845ff1d239346ed25947c8a42e23575 Mon Sep 17 00:00:00 2001 From: yuhui Date: Wed, 18 Dec 2024 22:24:13 +0800 Subject: [PATCH 15/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index fbc0386282c..219dbfae0f9 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -368,7 +368,6 @@ pub trait FileWriter: Sync + Send { /// DefaultRawFileSystem is a simple implementation for the file system. /// it is used to manage the file metadata and file handle. /// The operations of the file system are implemented by the PathFileSystem. -/// Note: This class is not use in the production code, it is used for the demo and testing pub struct DefaultRawFileSystem { /// file entries file_entry_manager: RwLock, From bfd3945c939df6566c5ead3c74280f7e8fa6d779 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 09:53:11 +0800 Subject: [PATCH 16/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 219dbfae0f9..45fa5392c99 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -404,17 +404,20 @@ impl DefaultRawFileSystem { self.file_entry_manager .read() .await - .get_by_id(file_id) + .get_file_entry_by_id(file_id) .ok_or(Errno::from(libc::ENOENT)) } async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager.read().await.get_by_path(path) + self.file_entry_manager + .read() + .await + .get_file_entry_by_path(path) } async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_by_path(&file_stat.path); + let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); match file_entry { None => { // allocate new file id @@ -666,11 +669,11 @@ impl FileEntryManager { } } - fn get_by_id(&self, file_id: u64) -> Option { + fn get_file_entry_by_id(&self, file_id: u64) -> Option { self.file_id_map.get(&file_id).cloned() } - fn get_by_path(&self, path: &str) -> Option { + fn get_file_entry_by_path(&self, path: &str) -> Option { self.file_path_map.get(path).cloned() } @@ -757,18 +760,18 @@ mod tests { fn test_file_entry_manager() { let mut manager = FileEntryManager::new(); manager.insert(1, 2, "a/b"); - let file = manager.get_by_id(2).unwrap(); + let file = manager.get_file_entry_by_id(2).unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); - let file = manager.get_by_path("a/b").unwrap(); + let file = manager.get_file_entry_by_path("a/b").unwrap(); assert_eq!(file.file_id, 2); assert_eq!(file.parent_file_id, 1); assert_eq!(file.path, "a/b"); manager.remove("a/b"); - assert!(manager.get_by_id(2).is_none()); - assert!(manager.get_by_path("a/b").is_none()); + assert!(manager.get_file_entry_by_id(2).is_none()); + assert!(manager.get_file_entry_by_path("a/b").is_none()); } } From 4fb77e1e2d0739515987cd7966c8bd9ccf48666e Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 12:05:23 +0800 Subject: [PATCH 17/31] Update --- clients/filesystem-fuse/src/utils.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 7ac0c7212a7..7619775c949 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -19,6 +19,7 @@ // join the parent and name to a path pub fn join_file_path(parent: &str, name: &str) -> String { + //TODO handle corner cases if parent.is_empty() { name.to_string() } else { From 8cf0c4a25c47170e8ec579846c208034fe617fe9 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 12:09:51 +0800 Subject: [PATCH 18/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 45fa5392c99..846f51631fb 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -94,19 +94,19 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file stat by file path, if the file is exist, return the file stat - async fn stat(&self, name: &str) -> Result; + async fn stat(&self, path: &str) -> Result; /// Get the file stat by parent file path and file name, if the file is exist, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; /// Read the directory by file path, if the file is a valid directory, return the file stat list - async fn read_dir(&self, name: &str) -> Result>; + async fn read_dir(&self, path: &str) -> Result>; /// Open the file by file path and flags, if the file is exist, return the opened file - async fn open_file(&self, name: &str, flags: OpenFileFlags) -> Result; + async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; /// Open the directory by file path and flags, if the file is exist, return the opened file - async fn open_dir(&self, name: &str, flags: OpenFileFlags) -> Result; + async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; /// Create the file by parent file path and file name and flags, if successful, return the opened file async fn create_file( @@ -120,7 +120,7 @@ pub(crate) trait PathFileSystem: Send + Sync { async fn create_dir(&self, parent: &str, name: &str) -> Result; /// Set the file attribute by file path and file stat - async fn set_attr(&self, name: &str, file_stat: &FileStat, flush: bool) -> Result<()>; + async fn set_attr(&self, path: &str, file_stat: &FileStat, flush: bool) -> Result<()>; /// Remove the file by parent file path and file name async fn remove_file(&self, parent: &str, name: &str) -> Result<()>; From 21b080d6edd7cc0fc076a337d1703ef9c63e102e Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 19:29:47 +0800 Subject: [PATCH 19/31] Update --- clients/filesystem-fuse/build.gradle.kts | 8 - .../src/default_raw_filesystem.rs | 398 +++++++++++++++ clients/filesystem-fuse/src/filesystem.rs | 480 +----------------- clients/filesystem-fuse/src/lib.rs | 2 + clients/filesystem-fuse/src/main.rs | 2 + clients/filesystem-fuse/src/opened_file.rs | 123 +++++ .../src/opened_file_manager.rs | 2 +- 7 files changed, 534 insertions(+), 481 deletions(-) create mode 100644 clients/filesystem-fuse/src/default_raw_filesystem.rs create mode 100644 clients/filesystem-fuse/src/opened_file.rs diff --git a/clients/filesystem-fuse/build.gradle.kts b/clients/filesystem-fuse/build.gradle.kts index 7e20e13cedd..7d24c86a5b0 100644 --- a/clients/filesystem-fuse/build.gradle.kts +++ b/clients/filesystem-fuse/build.gradle.kts @@ -20,8 +20,6 @@ import org.gradle.api.tasks.Exec val checkRustEnvironment by tasks.registering(Exec::class) { - description = "Check if Rust environment." - group = "verification" commandLine("bash", "-c", "cargo --version") standardOutput = System.out errorOutput = System.err @@ -30,14 +28,12 @@ val checkRustEnvironment by tasks.registering(Exec::class) { val buildRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Compile the Rust project" workingDir = file("$projectDir") commandLine("bash", "-c", "make build") } val checkRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Check the Rust project" workingDir = file("$projectDir") commandLine("bash", "-c", "make check") @@ -45,8 +41,6 @@ val checkRustProject by tasks.registering(Exec::class) { val testRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Run tests in the Rust project" - group = "verification" workingDir = file("$projectDir") commandLine("bash", "-c", "make test") @@ -56,8 +50,6 @@ val testRustProject by tasks.registering(Exec::class) { val cleanRustProject by tasks.registering(Exec::class) { dependsOn(checkRustEnvironment) - description = "Run tests in the Rust project" - group = "verification" workingDir = file("$projectDir") commandLine("bash", "-c", "make clean") diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs new file mode 100644 index 00000000000..2ea2f126305 --- /dev/null +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; +use crate::opened_file::{FileHandle, OpenFileFlags}; +use crate::opened_file_manager::OpenedFileManager; +use crate::utils::join_file_path; +use async_trait::async_trait; +use bytes::Bytes; +use fuse3::{Errno, FileType}; +use std::collections::HashMap; +use std::sync::atomic::AtomicU64; +use tokio::sync::RwLock; + +/// DefaultRawFileSystem is a simple implementation for the file system. +/// it is used to manage the file metadata and file handle. +/// The operations of the file system are implemented by the PathFileSystem. +pub struct DefaultRawFileSystem { + /// file entries + file_entry_manager: RwLock, + /// opened files + opened_file_manager: OpenedFileManager, + /// inode id generator + file_id_generator: AtomicU64, + + /// real filesystem + fs: T, +} + +impl DefaultRawFileSystem { + const INITIAL_FILE_ID: u64 = 10000; + const ROOT_DIR_PARENT_FILE_ID: u64 = 1; + const ROOT_DIR_FILE_ID: u64 = 1; + const ROOT_DIR_NAME: &'static str = ""; + + pub(crate) fn new(fs: T) -> Self { + Self { + file_entry_manager: RwLock::new(FileEntryManager::new()), + opened_file_manager: OpenedFileManager::new(), + file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), + fs, + } + } + + fn next_file_id(&self) -> u64 { + self.file_id_generator + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + } + + async fn get_file_entry(&self, file_id: u64) -> crate::filesystem::Result { + self.file_entry_manager + .read() + .await + .get_file_entry_by_id(file_id) + .ok_or(Errno::from(libc::ENOENT)) + } + + async fn get_file_entry_by_path(&self, path: &str) -> Option { + self.file_entry_manager + .read() + .await + .get_file_entry_by_path(path) + } + + async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { + let mut file_manager = self.file_entry_manager.write().await; + let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); + match file_entry { + None => { + // allocate new file id + file_stat.set_file_id(parent_file_id, self.next_file_id()); + file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); + } + Some(file) => { + // use the exist file id + file_stat.set_file_id(file.parent_file_id, file.file_id); + } + } + } + + async fn open_file_internal( + &self, + file_id: u64, + flags: u32, + kind: FileType, + ) -> crate::filesystem::Result { + let file_entry = self.get_file_entry(file_id).await?; + + let mut opened_file = { + match kind { + FileType::Directory => { + self.fs + .open_dir(&file_entry.path, OpenFileFlags(flags)) + .await? + } + FileType::RegularFile => { + self.fs + .open_file(&file_entry.path, OpenFileFlags(flags)) + .await? + } + _ => return Err(Errno::from(libc::EINVAL)), + } + }; + // set the exists file id + opened_file.set_file_id(file_entry.parent_file_id, file_id); + let file = self.opened_file_manager.put(opened_file); + let file = file.lock().await; + Ok(file.file_handle()) + } +} + +#[async_trait] +impl RawFileSystem for DefaultRawFileSystem { + async fn init(&self) -> crate::filesystem::Result<()> { + // init root directory + self.file_entry_manager.write().await.insert( + Self::ROOT_DIR_PARENT_FILE_ID, + Self::ROOT_DIR_FILE_ID, + Self::ROOT_DIR_NAME, + ); + self.fs.init().await + } + + async fn get_file_path(&self, file_id: u64) -> String { + let file_entry = self.get_file_entry(file_id).await; + file_entry + .map(|x| x.path) + .unwrap_or_else(|_| "".to_string()) + } + + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + let fh_file_id = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))? + .lock() + .await + .file_stat + .file_id; + + (file_id == fh_file_id) + .then_some(()) + .ok_or(Errno::from(libc::EBADF)) + } + + async fn stat(&self, file_id: u64) -> crate::filesystem::Result { + let file_entry = self.get_file_entry(file_id).await?; + let mut file_stat = self.fs.stat(&file_entry.path).await?; + file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); + Ok(file_stat) + } + + async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; + // fill the file id to file stat + self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) + .await; + Ok(file_stat) + } + + async fn read_dir(&self, file_id: u64) -> crate::filesystem::Result> { + let file_entry = self.get_file_entry(file_id).await?; + let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; + for file in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file, file.file_id).await; + } + Ok(child_filestats) + } + + async fn open_file(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + self.open_file_internal(file_id, flags, FileType::RegularFile) + .await + } + + async fn open_dir(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + self.open_file_internal(file_id, flags, FileType::Directory) + .await + } + + async fn create_file( + &self, + parent_file_id: u64, + name: &str, + flags: u32, + ) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut opened_file = self + .fs + .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) + .await?; + + opened_file.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ); + } + + // put the file to the opened file manager + let opened_file = self.opened_file_manager.put(opened_file); + let opened_file = opened_file.lock().await; + Ok(opened_file.file_handle()) + } + + async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; + + filestat.set_file_id(parent_file_id, self.next_file_id()); + + // insert the new file to file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); + } + Ok(filestat.file_id) + } + + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> crate::filesystem::Result<()> { + let file_entry = self.get_file_entry(file_id).await?; + self.fs.set_attr(&file_entry.path, file_stat, true).await + } + + async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_file(&parent_file_entry.path, name).await?; + + // remove the file from file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + } + Ok(()) + } + + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + let parent_file_entry = self.get_file_entry(parent_file_id).await?; + self.fs.remove_dir(&parent_file_entry.path, name).await?; + + // remove the dir from file entry manager + { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(&join_file_path(&parent_file_entry.path, name)); + } + Ok(()) + } + + async fn close_file(&self, _file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + let opened_file = self + .opened_file_manager + .remove(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut file = opened_file.lock().await; + file.close().await + } + + async fn read( + &self, + _file_id: u64, + fh: u64, + offset: u64, + size: u32, + ) -> crate::filesystem::Result { + let file_stat: FileStat; + let data = { + let opened_file = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + file_stat = opened_file.file_stat.clone(); + opened_file.read(offset, size).await + }; + + // update the file atime + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + data + } + + async fn write( + &self, + _file_id: u64, + fh: u64, + offset: u64, + data: &[u8], + ) -> crate::filesystem::Result { + let (len, file_stat) = { + let opened_file = self + .opened_file_manager + .get(fh) + .ok_or(Errno::from(libc::EBADF))?; + let mut opened_file = opened_file.lock().await; + let len = opened_file.write(offset, data).await; + (len, opened_file.file_stat.clone()) + }; + + // update the file size, mtime and atime + self.fs.set_attr(&file_stat.path, &file_stat, false).await?; + + len + } +} + +/// File entry is represent the abstract file. +#[derive(Debug, Clone)] +struct FileEntry { + file_id: u64, + parent_file_id: u64, + path: String, +} + +/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. +struct FileEntryManager { + // file_id_map is a map of file_id to file entry. + file_id_map: HashMap, + + // file_path_map is a map of file path to file entry. + file_path_map: HashMap, +} + +impl FileEntryManager { + fn new() -> Self { + Self { + file_id_map: HashMap::new(), + file_path_map: HashMap::new(), + } + } + + fn get_file_entry_by_id(&self, file_id: u64) -> Option { + self.file_id_map.get(&file_id).cloned() + } + + fn get_file_entry_by_path(&self, path: &str) -> Option { + self.file_path_map.get(path).cloned() + } + + fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { + let file_entry = FileEntry { + file_id, + parent_file_id, + path: path.to_string(), + }; + self.file_id_map.insert(file_id, file_entry.clone()); + self.file_path_map.insert(path.to_string(), file_entry); + } + + fn remove(&mut self, path: &str) { + if let Some(file) = self.file_path_map.remove(path) { + self.file_id_map.remove(&file.file_id); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_file_entry_manager() { + let mut manager = FileEntryManager::new(); + manager.insert(1, 2, "a/b"); + let file = manager.get_file_entry_by_id(2).unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + let file = manager.get_file_entry_by_path("a/b").unwrap(); + assert_eq!(file.file_id, 2); + assert_eq!(file.parent_file_id, 1); + assert_eq!(file.path, "a/b"); + + manager.remove("a/b"); + assert!(manager.get_file_entry_by_id(2).is_none()); + assert!(manager.get_file_entry_by_path("a/b").is_none()); + } +} diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 846f51631fb..ad552c16ebb 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -16,15 +16,12 @@ * specific language governing permissions and limitations * under the License. */ -use crate::opened_file_manager::OpenedFileManager; +use crate::opened_file::{FileHandle, OpenFileFlags, OpenedFile}; use crate::utils::{join_file_path, split_file_path}; use async_trait::async_trait; use bytes::Bytes; use fuse3::{Errno, FileType, Timestamp}; -use std::collections::HashMap; -use std::sync::atomic::AtomicU64; use std::time::SystemTime; -use tokio::sync::RwLock; pub(crate) type Result = std::result::Result; @@ -49,7 +46,7 @@ pub(crate) trait RawFileSystem: Send + Sync { /// Get the file stat by file id. if the file id is valid, return the file stat async fn stat(&self, file_id: u64) -> Result; - /// Lookup the file by parent file id and file name, if the file is exist, return the file stat + /// Lookup the file by parent file id and file name, if the file exists, return the file stat async fn lookup(&self, parent_file_id: u64, name: &str) -> Result; /// Read the directory by file id, if the file id is a valid directory, return the file stat list @@ -93,22 +90,22 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Init the file system async fn init(&self) -> Result<()>; - /// Get the file stat by file path, if the file is exist, return the file stat + /// Get the file stat by file path, if the file exists, return the file stat async fn stat(&self, path: &str) -> Result; - /// Get the file stat by parent file path and file name, if the file is exist, return the file stat + /// Get the file stat by parent file path and file name, if the file exists, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; - /// Read the directory by file path, if the file is a valid directory, return the file stat list + /// Read the directory by file path, if the is a valid directory, return the file stat list async fn read_dir(&self, path: &str) -> Result>; - /// Open the file by file path and flags, if the file is exist, return the opened file + /// Open the file by file path and flags, if the file exists, return the opened file async fn open_file(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Open the directory by file path and flags, if the file is exist, return the opened file + /// Open the directory by file path and flags, if the file exists, return the opened file async fn open_dir(&self, path: &str, flags: OpenFileFlags) -> Result; - /// Create the file by parent file path and file name and flags, if successful, return the opened file + /// Create the file by parent file path and file name and flags, if successful return the opened file async fn create_file( &self, parent: &str, @@ -235,107 +232,6 @@ impl FileStat { } } -/// Opened file for read or write, it is used to read or write the file content. -pub(crate) struct OpenedFile { - pub(crate) file_stat: FileStat, - - pub(crate) handle_id: u64, - - pub reader: Option>, - - pub writer: Option>, -} - -impl OpenedFile { - pub fn new(file_stat: FileStat) -> Self { - OpenedFile { - file_stat: file_stat, - handle_id: 0, - reader: None, - writer: None, - } - } - - async fn read(&mut self, offset: u64, size: u32) -> Result { - let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; - let result = reader.read(offset, size).await?; - - // update the atime - self.file_stat.atime = Timestamp::from(SystemTime::now()); - - Ok(result) - } - - async fn write(&mut self, offset: u64, data: &[u8]) -> Result { - let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; - let written = writer.write(offset, data).await?; - - // update the file size ,mtime and atime - let end = offset + written as u64; - if end > self.file_stat.size { - self.file_stat.size = end; - } - self.file_stat.atime = Timestamp::from(SystemTime::now()); - self.file_stat.mtime = self.file_stat.atime; - - Ok(written) - } - - async fn close(&mut self) -> Result<()> { - let mut errors = Vec::new(); - if let Some(mut reader) = self.reader.take() { - if let Err(e) = reader.close().await { - errors.push(e); - } - } - - if let Some(mut writer) = self.writer.take() { - if let Err(e) = self.flush().await { - errors.push(e); - } - if let Err(e) = writer.close().await { - errors.push(e); - } - } - - if !errors.is_empty() { - return Err(errors.remove(0)); - } - Ok(()) - } - - async fn flush(&mut self) -> Result<()> { - if let Some(writer) = &mut self.writer { - writer.flush().await?; - } - Ok(()) - } - - fn file_handle(&self) -> FileHandle { - debug_assert!(self.handle_id != 0); - debug_assert!(self.file_stat.file_id != 0); - FileHandle { - file_id: self.file_stat.file_id, - handle_id: self.handle_id, - } - } - - pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { - debug_assert!(file_id != 0 && parent_file_id != 0); - self.file_stat.set_file_id(parent_file_id, file_id) - } -} - -// FileHandle is the file handle for the opened file. -pub(crate) struct FileHandle { - pub(crate) file_id: u64, - - pub(crate) handle_id: u64, -} - -// OpenFileFlags is the open file flags for the file system. -pub struct OpenFileFlags(u32); - /// File reader interface for read file content #[async_trait] pub(crate) trait FileReader: Sync + Send { @@ -365,335 +261,6 @@ pub trait FileWriter: Sync + Send { } } -/// DefaultRawFileSystem is a simple implementation for the file system. -/// it is used to manage the file metadata and file handle. -/// The operations of the file system are implemented by the PathFileSystem. -pub struct DefaultRawFileSystem { - /// file entries - file_entry_manager: RwLock, - /// opened files - opened_file_manager: OpenedFileManager, - /// inode id generator - file_id_generator: AtomicU64, - - /// real filesystem - fs: T, -} - -impl DefaultRawFileSystem { - const INITIAL_FILE_ID: u64 = 10000; - const ROOT_DIR_PARENT_FILE_ID: u64 = 1; - const ROOT_DIR_FILE_ID: u64 = 1; - const ROOT_DIR_NAME: &'static str = ""; - - pub(crate) fn new(fs: T) -> Self { - Self { - file_entry_manager: RwLock::new(FileEntryManager::new()), - opened_file_manager: OpenedFileManager::new(), - file_id_generator: AtomicU64::new(Self::INITIAL_FILE_ID), - fs, - } - } - - fn next_file_id(&self) -> u64 { - self.file_id_generator - .fetch_add(1, std::sync::atomic::Ordering::SeqCst) - } - - async fn get_file_entry(&self, file_id: u64) -> Result { - self.file_entry_manager - .read() - .await - .get_file_entry_by_id(file_id) - .ok_or(Errno::from(libc::ENOENT)) - } - - async fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_entry_manager - .read() - .await - .get_file_entry_by_path(path) - } - - async fn resolve_file_id_to_filestat(&self, file_stat: &mut FileStat, parent_file_id: u64) { - let mut file_manager = self.file_entry_manager.write().await; - let file_entry = file_manager.get_file_entry_by_path(&file_stat.path); - match file_entry { - None => { - // allocate new file id - file_stat.set_file_id(parent_file_id, self.next_file_id()); - file_manager.insert(file_stat.parent_file_id, file_stat.file_id, &file_stat.path); - } - Some(file) => { - // use the exist file id - file_stat.set_file_id(file.parent_file_id, file.file_id); - } - } - } - - async fn open_file_internal( - &self, - file_id: u64, - flags: u32, - kind: FileType, - ) -> Result { - let file_entry = self.get_file_entry(file_id).await?; - - let mut opened_file = { - match kind { - FileType::Directory => { - self.fs - .open_dir(&file_entry.path, OpenFileFlags(flags)) - .await? - } - FileType::RegularFile => { - self.fs - .open_file(&file_entry.path, OpenFileFlags(flags)) - .await? - } - _ => return Err(Errno::from(libc::EINVAL)), - } - }; - // set the exists file id - opened_file.set_file_id(file_entry.parent_file_id, file_id); - let file = self.opened_file_manager.put(opened_file); - let file = file.lock().await; - Ok(file.file_handle()) - } -} - -#[async_trait] -impl RawFileSystem for DefaultRawFileSystem { - async fn init(&self) -> Result<()> { - // init root directory - self.file_entry_manager.write().await.insert( - Self::ROOT_DIR_PARENT_FILE_ID, - Self::ROOT_DIR_FILE_ID, - Self::ROOT_DIR_NAME, - ); - self.fs.init().await - } - - async fn get_file_path(&self, file_id: u64) -> String { - let file_entry = self.get_file_entry(file_id).await; - file_entry - .map(|x| x.path) - .unwrap_or_else(|_| "".to_string()) - } - - async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { - let fh_file_id = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))? - .lock() - .await - .file_stat - .file_id; - - (file_id == fh_file_id) - .then_some(()) - .ok_or(Errno::from(libc::EBADF)) - } - - async fn stat(&self, file_id: u64) -> Result { - let file_entry = self.get_file_entry(file_id).await?; - let mut file_stat = self.fs.stat(&file_entry.path).await?; - file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); - Ok(file_stat) - } - - async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; - // fill the file id to file stat - self.resolve_file_id_to_filestat(&mut file_stat, parent_file_id) - .await; - Ok(file_stat) - } - - async fn read_dir(&self, file_id: u64) -> Result> { - let file_entry = self.get_file_entry(file_id).await?; - let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; - for file in child_filestats.iter_mut() { - self.resolve_file_id_to_filestat(file, file.file_id).await; - } - Ok(child_filestats) - } - - async fn open_file(&self, file_id: u64, flags: u32) -> Result { - self.open_file_internal(file_id, flags, FileType::RegularFile) - .await - } - - async fn open_dir(&self, file_id: u64, flags: u32) -> Result { - self.open_file_internal(file_id, flags, FileType::Directory) - .await - } - - async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file = self - .fs - .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) - .await?; - - opened_file.set_file_id(parent_file_id, self.next_file_id()); - - // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, - ); - } - - // put the file to the opened file manager - let opened_file = self.opened_file_manager.put(opened_file); - let opened_file = opened_file.lock().await; - Ok(opened_file.file_handle()) - } - - async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; - - filestat.set_file_id(parent_file_id, self.next_file_id()); - - // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); - } - Ok(filestat.file_id) - } - - async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { - let file_entry = self.get_file_entry(file_id).await?; - self.fs.set_attr(&file_entry.path, file_stat, true).await - } - - async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_file(&parent_file_entry.path, name).await?; - - // remove the file from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } - Ok(()) - } - - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { - let parent_file_entry = self.get_file_entry(parent_file_id).await?; - self.fs.remove_dir(&parent_file_entry.path, name).await?; - - // remove the dir from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } - Ok(()) - } - - async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { - let opened_file = self - .opened_file_manager - .remove(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut file = opened_file.lock().await; - file.close().await - } - - async fn read(&self, _file_id: u64, fh: u64, offset: u64, size: u32) -> Result { - let file_stat: FileStat; - let data = { - let opened_file = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut opened_file = opened_file.lock().await; - file_stat = opened_file.file_stat.clone(); - opened_file.read(offset, size).await - }; - - // update the file atime - self.fs.set_attr(&file_stat.path, &file_stat, false).await?; - - data - } - - async fn write(&self, _file_id: u64, fh: u64, offset: u64, data: &[u8]) -> Result { - let (len, file_stat) = { - let opened_file = self - .opened_file_manager - .get(fh) - .ok_or(Errno::from(libc::EBADF))?; - let mut opened_file = opened_file.lock().await; - let len = opened_file.write(offset, data).await; - (len, opened_file.file_stat.clone()) - }; - - // update the file size, mtime and atime - self.fs.set_attr(&file_stat.path, &file_stat, false).await?; - - len - } -} - -/// File entry is represent the abstract file. -#[derive(Debug, Clone)] -struct FileEntry { - file_id: u64, - parent_file_id: u64, - path: String, -} - -/// FileEntryManager is manage all the file entries in memory. it is used manger the file relationship and name mapping. -struct FileEntryManager { - // file_id_map is a map of file_id to file entry. - file_id_map: HashMap, - - // file_path_map is a map of file path to file entry. - file_path_map: HashMap, -} - -impl FileEntryManager { - fn new() -> Self { - Self { - file_id_map: HashMap::new(), - file_path_map: HashMap::new(), - } - } - - fn get_file_entry_by_id(&self, file_id: u64) -> Option { - self.file_id_map.get(&file_id).cloned() - } - - fn get_file_entry_by_path(&self, path: &str) -> Option { - self.file_path_map.get(path).cloned() - } - - fn insert(&mut self, parent_file_id: u64, file_id: u64, path: &str) { - let file_entry = FileEntry { - file_id, - parent_file_id, - path: path.to_string(), - }; - self.file_id_map.insert(file_id, file_entry.clone()); - self.file_path_map.insert(path.to_string(), file_entry); - } - - fn remove(&mut self, path: &str) { - if let Some(file) = self.file_path_map.remove(path) { - self.file_id_map.remove(&file.file_id); - } - } -} - #[cfg(test)] mod tests { use super::*; @@ -743,35 +310,4 @@ mod tests { let mut file_stat = FileStat::new_file_filestat("a", "b", 10); file_stat.set_file_id(1, 0); } - - #[test] - fn test_open_file() { - let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); - assert_eq!(open_file.file_stat.name, "b"); - assert_eq!(open_file.file_stat.size, 10); - - open_file.set_file_id(1, 2); - - assert_eq!(open_file.file_stat.file_id, 2); - assert_eq!(open_file.file_stat.parent_file_id, 1); - } - - #[test] - fn test_file_entry_manager() { - let mut manager = FileEntryManager::new(); - manager.insert(1, 2, "a/b"); - let file = manager.get_file_entry_by_id(2).unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - let file = manager.get_file_entry_by_path("a/b").unwrap(); - assert_eq!(file.file_id, 2); - assert_eq!(file.parent_file_id, 1); - assert_eq!(file.path, "a/b"); - - manager.remove("a/b"); - assert!(manager.get_file_entry_by_id(2).is_none()); - assert!(manager.get_file_entry_by_path("a/b").is_none()); - } } diff --git a/clients/filesystem-fuse/src/lib.rs b/clients/filesystem-fuse/src/lib.rs index b531df82f79..c1689bac476 100644 --- a/clients/filesystem-fuse/src/lib.rs +++ b/clients/filesystem-fuse/src/lib.rs @@ -16,7 +16,9 @@ * specific language governing permissions and limitations * under the License. */ +mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod opened_file; mod opened_file_manager; mod utils; diff --git a/clients/filesystem-fuse/src/main.rs b/clients/filesystem-fuse/src/main.rs index 25a3b1579ce..3d8e9dbb953 100644 --- a/clients/filesystem-fuse/src/main.rs +++ b/clients/filesystem-fuse/src/main.rs @@ -16,8 +16,10 @@ * specific language governing permissions and limitations * under the License. */ +mod default_raw_filesystem; mod filesystem; mod fuse_api_handle; +mod opened_file; mod opened_file_manager; mod utils; diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs new file mode 100644 index 00000000000..61a9af4a3bc --- /dev/null +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -0,0 +1,123 @@ +use crate::filesystem::{FileReader, FileStat, FileWriter, Result}; +use bytes::Bytes; +use fuse3::{Errno, Timestamp}; +use std::time::SystemTime; + +/// Opened file for read or write, it is used to read or write the file content. +pub(crate) struct OpenedFile { + pub(crate) file_stat: FileStat, + + pub(crate) handle_id: u64, + + pub reader: Option>, + + pub writer: Option>, +} + +impl OpenedFile { + pub(crate) fn new(file_stat: FileStat) -> Self { + OpenedFile { + file_stat: file_stat, + handle_id: 0, + reader: None, + writer: None, + } + } + + pub(crate) async fn read(&mut self, offset: u64, size: u32) -> Result { + let reader = self.reader.as_mut().ok_or(Errno::from(libc::EBADF))?; + let result = reader.read(offset, size).await?; + + // update the atime + self.file_stat.atime = Timestamp::from(SystemTime::now()); + + Ok(result) + } + + pub(crate) async fn write(&mut self, offset: u64, data: &[u8]) -> Result { + let writer = self.writer.as_mut().ok_or(Errno::from(libc::EBADF))?; + let written = writer.write(offset, data).await?; + + // update the file size ,mtime and atime + let end = offset + written as u64; + if end > self.file_stat.size { + self.file_stat.size = end; + } + self.file_stat.atime = Timestamp::from(SystemTime::now()); + self.file_stat.mtime = self.file_stat.atime; + + Ok(written) + } + + pub(crate) async fn close(&mut self) -> Result<()> { + let mut errors = Vec::new(); + if let Some(mut reader) = self.reader.take() { + if let Err(e) = reader.close().await { + errors.push(e); + } + } + + if let Some(mut writer) = self.writer.take() { + if let Err(e) = self.flush().await { + errors.push(e); + } + if let Err(e) = writer.close().await { + errors.push(e); + } + } + + if !errors.is_empty() { + return Err(errors.remove(0)); + } + Ok(()) + } + + pub(crate) async fn flush(&mut self) -> Result<()> { + if let Some(writer) = &mut self.writer { + writer.flush().await?; + } + Ok(()) + } + + pub(crate) fn file_handle(&self) -> FileHandle { + debug_assert!(self.handle_id != 0); + debug_assert!(self.file_stat.file_id != 0); + FileHandle { + file_id: self.file_stat.file_id, + handle_id: self.handle_id, + } + } + + pub(crate) fn set_file_id(&mut self, parent_file_id: u64, file_id: u64) { + debug_assert!(file_id != 0 && parent_file_id != 0); + self.file_stat.set_file_id(parent_file_id, file_id) + } +} + +// FileHandle is the file handle for the opened file. +pub(crate) struct FileHandle { + pub(crate) file_id: u64, + + pub(crate) handle_id: u64, +} + +// OpenFileFlags is the open file flags for the file system. +pub(crate) struct OpenFileFlags(pub(crate) u32); + +#[cfg(test)] +mod tests { + use super::*; + use crate::filesystem::FileStat; + + #[test] + fn test_open_file() { + let mut open_file = OpenedFile::new(FileStat::new_file_filestat("a", "b", 10)); + assert_eq!(open_file.file_stat.name, "b"); + assert_eq!(open_file.file_stat.size, 10); + + open_file.set_file_id(1, 2); + + assert_eq!(open_file.file_stat.file_id, 2); + assert_eq!(open_file.file_stat.parent_file_id, 1); + } +} diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 695d69adca4..005ec629145 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::OpenedFile; +use crate::opened_file::OpenedFile; use dashmap::DashMap; use std::sync::atomic::AtomicU64; use std::sync::Arc; From 4fe5fe3717ef1acb71f46f4aa753448fec45df15 Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 19:35:12 +0800 Subject: [PATCH 20/31] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 2ea2f126305..c559955b8e2 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -35,7 +35,7 @@ pub struct DefaultRawFileSystem { file_entry_manager: RwLock, /// opened files opened_file_manager: OpenedFileManager, - /// inode id generator + /// file id generator file_id_generator: AtomicU64, /// real filesystem From 2517afa5fb90322505d163005504c5e8921ade5d Mon Sep 17 00:00:00 2001 From: yuhui Date: Thu, 19 Dec 2024 20:33:21 +0800 Subject: [PATCH 21/31] Update --- clients/filesystem-fuse/src/filesystem.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index ad552c16ebb..7a74180159b 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -96,7 +96,7 @@ pub(crate) trait PathFileSystem: Send + Sync { /// Get the file stat by parent file path and file name, if the file exists, return the file stat async fn lookup(&self, parent: &str, name: &str) -> Result; - /// Read the directory by file path, if the is a valid directory, return the file stat list + /// Read the directory by file path, if the directory exists, return the file stat list async fn read_dir(&self, path: &str) -> Result>; /// Open the file by file path and flags, if the file exists, return the opened file From 76305a9980c941618cda5e45b83aa46934199650 Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 10:44:35 +0800 Subject: [PATCH 22/31] Update --- clients/filesystem-fuse/src/opened_file.rs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/clients/filesystem-fuse/src/opened_file.rs b/clients/filesystem-fuse/src/opened_file.rs index 61a9af4a3bc..ba3e41595da 100644 --- a/clients/filesystem-fuse/src/opened_file.rs +++ b/clients/filesystem-fuse/src/opened_file.rs @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ use crate::filesystem::{FileReader, FileStat, FileWriter, Result}; use bytes::Bytes; use fuse3::{Errno, Timestamp}; From d444ad3c159cd90dabdb704da819049052351564 Mon Sep 17 00:00:00 2001 From: yuhui Date: Fri, 20 Dec 2024 16:49:10 +0800 Subject: [PATCH 23/31] Fix --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index c559955b8e2..d492143b18f 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -282,15 +282,14 @@ impl RawFileSystem for DefaultRawFileSystem { offset: u64, size: u32, ) -> crate::filesystem::Result { - let file_stat: FileStat; - let data = { + let (data, file_stat) = { let opened_file = self .opened_file_manager .get(fh) .ok_or(Errno::from(libc::EBADF))?; let mut opened_file = opened_file.lock().await; - file_stat = opened_file.file_stat.clone(); - opened_file.read(offset, size).await + let data = opened_file.read(offset, size).await; + (data, opened_file.file_stat.clone()) }; // update the file atime From 8fb13bb2116c637bf017191d0451bd81aef4b5be Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:02:33 +0800 Subject: [PATCH 24/31] Update --- .../src/default_raw_filesystem.rs | 34 +++++++++---------- clients/filesystem-fuse/src/filesystem.rs | 2 +- .../filesystem-fuse/src/fuse_api_handle.rs | 4 --- clients/filesystem-fuse/src/utils.rs | 8 +++++ 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d492143b18f..d08ffc77b5b 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -16,7 +16,7 @@ * specific language governing permissions and limitations * under the License. */ -use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem}; +use crate::filesystem::{FileStat, PathFileSystem, RawFileSystem, Result}; use crate::opened_file::{FileHandle, OpenFileFlags}; use crate::opened_file_manager::OpenedFileManager; use crate::utils::join_file_path; @@ -62,7 +62,7 @@ impl DefaultRawFileSystem { .fetch_add(1, std::sync::atomic::Ordering::SeqCst) } - async fn get_file_entry(&self, file_id: u64) -> crate::filesystem::Result { + async fn get_file_entry(&self, file_id: u64) -> Result { self.file_entry_manager .read() .await @@ -126,7 +126,7 @@ impl DefaultRawFileSystem { #[async_trait] impl RawFileSystem for DefaultRawFileSystem { - async fn init(&self) -> crate::filesystem::Result<()> { + async fn init(&self) -> Result<()> { // init root directory self.file_entry_manager.write().await.insert( Self::ROOT_DIR_PARENT_FILE_ID, @@ -136,14 +136,12 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.init().await } - async fn get_file_path(&self, file_id: u64) -> String { + async fn get_file_path(&self, file_id: u64) -> Result { let file_entry = self.get_file_entry(file_id).await; - file_entry - .map(|x| x.path) - .unwrap_or_else(|_| "".to_string()) + Ok(file_entry?.path) } - async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { let fh_file_id = self .opened_file_manager .get(fh) @@ -158,14 +156,14 @@ impl RawFileSystem for DefaultRawFileSystem { .ok_or(Errno::from(libc::EBADF)) } - async fn stat(&self, file_id: u64) -> crate::filesystem::Result { + async fn stat(&self, file_id: u64) -> Result { let file_entry = self.get_file_entry(file_id).await?; let mut file_stat = self.fs.stat(&file_entry.path).await?; file_stat.set_file_id(file_entry.parent_file_id, file_entry.file_id); Ok(file_stat) } - async fn lookup(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn lookup(&self, parent_file_id: u64, name: &str) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut file_stat = self.fs.lookup(&parent_file_entry.path, name).await?; // fill the file id to file stat @@ -174,7 +172,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(file_stat) } - async fn read_dir(&self, file_id: u64) -> crate::filesystem::Result> { + async fn read_dir(&self, file_id: u64) -> Result> { let file_entry = self.get_file_entry(file_id).await?; let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; for file in child_filestats.iter_mut() { @@ -183,12 +181,12 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(child_filestats) } - async fn open_file(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + async fn open_file(&self, file_id: u64, flags: u32) -> Result { self.open_file_internal(file_id, flags, FileType::RegularFile) .await } - async fn open_dir(&self, file_id: u64, flags: u32) -> crate::filesystem::Result { + async fn open_dir(&self, file_id: u64, flags: u32) -> Result { self.open_file_internal(file_id, flags, FileType::Directory) .await } @@ -223,7 +221,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(opened_file.file_handle()) } - async fn create_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result { + async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut filestat = self.fs.create_dir(&parent_file_entry.path, name).await?; @@ -237,12 +235,12 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(filestat.file_id) } - async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> crate::filesystem::Result<()> { + async fn set_attr(&self, file_id: u64, file_stat: &FileStat) -> Result<()> { let file_entry = self.get_file_entry(file_id).await?; self.fs.set_attr(&file_entry.path, file_stat, true).await } - async fn remove_file(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_file(&self, parent_file_id: u64, name: &str) -> Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; self.fs.remove_file(&parent_file_entry.path, name).await?; @@ -254,7 +252,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(()) } - async fn remove_dir(&self, parent_file_id: u64, name: &str) -> crate::filesystem::Result<()> { + async fn remove_dir(&self, parent_file_id: u64, name: &str) -> Result<()> { let parent_file_entry = self.get_file_entry(parent_file_id).await?; self.fs.remove_dir(&parent_file_entry.path, name).await?; @@ -266,7 +264,7 @@ impl RawFileSystem for DefaultRawFileSystem { Ok(()) } - async fn close_file(&self, _file_id: u64, fh: u64) -> crate::filesystem::Result<()> { + async fn close_file(&self, _file_id: u64, fh: u64) -> Result<()> { let opened_file = self .opened_file_manager .remove(fh) diff --git a/clients/filesystem-fuse/src/filesystem.rs b/clients/filesystem-fuse/src/filesystem.rs index 7a74180159b..b0d32ded233 100644 --- a/clients/filesystem-fuse/src/filesystem.rs +++ b/clients/filesystem-fuse/src/filesystem.rs @@ -38,7 +38,7 @@ pub(crate) trait RawFileSystem: Send + Sync { async fn init(&self) -> Result<()>; /// Get the file path by file id, if the file id is valid, return the file path - async fn get_file_path(&self, file_id: u64) -> String; + async fn get_file_path(&self, file_id: u64) -> Result; /// Validate the file id and file handle, if file id and file handle is valid and it associated, return Ok async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()>; diff --git a/clients/filesystem-fuse/src/fuse_api_handle.rs b/clients/filesystem-fuse/src/fuse_api_handle.rs index 76af581446b..7dc5461ce7f 100644 --- a/clients/filesystem-fuse/src/fuse_api_handle.rs +++ b/clients/filesystem-fuse/src/fuse_api_handle.rs @@ -52,10 +52,6 @@ impl FuseApiHandle { } } - pub async fn get_file_path(&self, file_id: u64) -> String { - self.fs.get_file_path(file_id).await - } - async fn get_modified_file_stat( &self, file_id: u64, diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 7619775c949..6aeb2d5e767 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -16,6 +16,7 @@ * specific language governing permissions and limitations * under the License. */ +use crate::filesystem::RawFileSystem; // join the parent and name to a path pub fn join_file_path(parent: &str, name: &str) -> String { @@ -35,6 +36,13 @@ pub fn split_file_path(path: &str) -> (&str, &str) { } } +// convert file id to string if file id is invalid return "Unknown" +pub async fn file_id_to_string(file_id: u64, fs: &impl RawFileSystem) -> String { + fs.get_file_path(file_id) + .await + .unwrap_or("Unknown".to_string()) +} + #[cfg(test)] mod tests { use super::*; From e23a5891739ed62838ad0c2c0ea17ebf598acbca Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:06:34 +0800 Subject: [PATCH 25/31] Update --- clients/filesystem-fuse/src/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/clients/filesystem-fuse/src/utils.rs b/clients/filesystem-fuse/src/utils.rs index 6aeb2d5e767..0c0cc80a162 100644 --- a/clients/filesystem-fuse/src/utils.rs +++ b/clients/filesystem-fuse/src/utils.rs @@ -36,8 +36,8 @@ pub fn split_file_path(path: &str) -> (&str, &str) { } } -// convert file id to string if file id is invalid return "Unknown" -pub async fn file_id_to_string(file_id: u64, fs: &impl RawFileSystem) -> String { +// convert file id to file path string if file id is invalid return "Unknown" +pub async fn file_id_to_file_path_string(file_id: u64, fs: &impl RawFileSystem) -> String { fs.get_file_path(file_id) .await .unwrap_or("Unknown".to_string()) From e8c74f6c703b07335a4831d28d67312c95d9582e Mon Sep 17 00:00:00 2001 From: yuhui Date: Mon, 23 Dec 2024 17:22:38 +0800 Subject: [PATCH 26/31] Update --- .../src/default_raw_filesystem.rs | 56 +++++++++---------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index d08ffc77b5b..4e3c9ed41d3 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -98,7 +98,7 @@ impl DefaultRawFileSystem { file_id: u64, flags: u32, kind: FileType, - ) -> crate::filesystem::Result { + ) -> Result { let file_entry = self.get_file_entry(file_id).await?; let mut opened_file = { @@ -122,17 +122,28 @@ impl DefaultRawFileSystem { let file = file.lock().await; Ok(file.file_handle()) } + + async fn remove_file_entry_locked(&self, path: &str) { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.remove(path); + } + + async fn insert_file_entry_locked(&self, parent_file_id: u64, file_id: u64, path: &str) { + let mut file_manager = self.file_entry_manager.write().await; + file_manager.insert(parent_file_id, file_id, path); + } } #[async_trait] impl RawFileSystem for DefaultRawFileSystem { async fn init(&self) -> Result<()> { // init root directory - self.file_entry_manager.write().await.insert( + self.insert_file_entry_locked( Self::ROOT_DIR_PARENT_FILE_ID, Self::ROOT_DIR_FILE_ID, Self::ROOT_DIR_NAME, - ); + ) + .await; self.fs.init().await } @@ -191,12 +202,7 @@ impl RawFileSystem for DefaultRawFileSystem { .await } - async fn create_file( - &self, - parent_file_id: u64, - name: &str, - flags: u32, - ) -> crate::filesystem::Result { + async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; let mut opened_file = self .fs @@ -206,14 +212,12 @@ impl RawFileSystem for DefaultRawFileSystem { opened_file.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert( - parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, - ); - } + self.insert_file_entry_locked( + parent_file_id, + opened_file.file_stat.file_id, + &opened_file.file_stat.path, + ) + .await; // put the file to the opened file manager let opened_file = self.opened_file_manager.put(opened_file); @@ -228,10 +232,8 @@ impl RawFileSystem for DefaultRawFileSystem { filestat.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.insert(filestat.parent_file_id, filestat.file_id, &filestat.path); - } + self.insert_file_entry_locked(parent_file_id, filestat.file_id, &filestat.path) + .await; Ok(filestat.file_id) } @@ -245,10 +247,8 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_file(&parent_file_entry.path, name).await?; // remove the file from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } + self.remove_file_entry_locked(&join_file_path(&parent_file_entry.path, name)) + .await; Ok(()) } @@ -257,10 +257,8 @@ impl RawFileSystem for DefaultRawFileSystem { self.fs.remove_dir(&parent_file_entry.path, name).await?; // remove the dir from file entry manager - { - let mut file_manager = self.file_entry_manager.write().await; - file_manager.remove(&join_file_path(&parent_file_entry.path, name)); - } + self.remove_file_entry_locked(&join_file_path(&parent_file_entry.path, name)) + .await; Ok(()) } From 4dd5bf4fb936c0e4fae039dfc6bd607149e3f5e3 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 09:59:40 +0800 Subject: [PATCH 27/31] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 4e3c9ed41d3..f524e1878e0 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -148,8 +148,8 @@ impl RawFileSystem for DefaultRawFileSystem { } async fn get_file_path(&self, file_id: u64) -> Result { - let file_entry = self.get_file_entry(file_id).await; - Ok(file_entry?.path) + let file_entry = self.get_file_entry(file_id).await?; + Ok(file_entry.path) } async fn valid_file_handle_id(&self, file_id: u64, fh: u64) -> Result<()> { @@ -186,8 +186,9 @@ impl RawFileSystem for DefaultRawFileSystem { async fn read_dir(&self, file_id: u64) -> Result> { let file_entry = self.get_file_entry(file_id).await?; let mut child_filestats = self.fs.read_dir(&file_entry.path).await?; - for file in child_filestats.iter_mut() { - self.resolve_file_id_to_filestat(file, file.file_id).await; + for file_stat in child_filestats.iter_mut() { + self.resolve_file_id_to_filestat(file_stat, file_stat.file_id) + .await; } Ok(child_filestats) } From 01d90a68416a68d67bd2b9eb124f9b32186920ab Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:12:34 +0800 Subject: [PATCH 28/31] Update --- .../src/default_raw_filesystem.rs | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index f524e1878e0..02a36b95985 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,25 +205,27 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file = self + let mut opened_file_with_out_file_handle_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - opened_file.set_file_id(parent_file_id, self.next_file_id()); + opened_file_with_out_file_handle_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - opened_file.file_stat.file_id, - &opened_file.file_stat.path, + opened_file_with_out_file_handle_id.file_stat.file_id, + &opened_file_with_out_file_handle_id.file_stat.path, ) .await; - // put the file to the opened file manager - let opened_file = self.opened_file_manager.put(opened_file); - let opened_file = opened_file.lock().await; - Ok(opened_file.file_handle()) + // put the openfile to the opened file manager and allocate a file handle id + let opened_file_with_file_handle_id = self + .opened_file_manager + .put(opened_file_with_out_file_handle_id); + let opened_file_with_file_handle_id = opened_file_with_file_handle_id.lock().await; + Ok(opened_file_with_file_handle_id.file_handle()) } async fn create_dir(&self, parent_file_id: u64, name: &str) -> Result { From 14609cf09d1b7cfc128ab693af9e62ea06efe1b6 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:13:54 +0800 Subject: [PATCH 29/31] Update --- clients/filesystem-fuse/src/opened_file_manager.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/clients/filesystem-fuse/src/opened_file_manager.rs b/clients/filesystem-fuse/src/opened_file_manager.rs index 005ec629145..17bfe00a397 100644 --- a/clients/filesystem-fuse/src/opened_file_manager.rs +++ b/clients/filesystem-fuse/src/opened_file_manager.rs @@ -45,6 +45,7 @@ impl OpenedFileManager { } pub(crate) fn put(&self, mut file: OpenedFile) -> Arc> { + // Put the file into the file handle map, and allocate a file handle id for the file. let file_handle_id = self.next_handle_id(); file.handle_id = file_handle_id; let file_handle = Arc::new(Mutex::new(file)); From 127cb41fd408e2cf1cafca381f454229ebc63185 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:54:34 +0800 Subject: [PATCH 30/31] Update --- .../filesystem-fuse/src/default_raw_filesystem.rs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 02a36b95985..8add88d7aa4 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,26 +205,24 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut opened_file_with_out_file_handle_id = self + let mut file_with_out_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - opened_file_with_out_file_handle_id.set_file_id(parent_file_id, self.next_file_id()); + file_with_out_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - opened_file_with_out_file_handle_id.file_stat.file_id, - &opened_file_with_out_file_handle_id.file_stat.path, + file_with_out_id.file_stat.file_id, + &file_with_out_id.file_stat.path, ) .await; // put the openfile to the opened file manager and allocate a file handle id - let opened_file_with_file_handle_id = self - .opened_file_manager - .put(opened_file_with_out_file_handle_id); - let opened_file_with_file_handle_id = opened_file_with_file_handle_id.lock().await; + let file_with_id = self.opened_file_manager.put(file_with_out_id); + let opened_file_with_file_handle_id = file_with_id.lock().await; Ok(opened_file_with_file_handle_id.file_handle()) } From df48db4622d070b2ca4080c441ad337e6333f597 Mon Sep 17 00:00:00 2001 From: yuhui Date: Tue, 24 Dec 2024 10:55:31 +0800 Subject: [PATCH 31/31] Update --- clients/filesystem-fuse/src/default_raw_filesystem.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clients/filesystem-fuse/src/default_raw_filesystem.rs b/clients/filesystem-fuse/src/default_raw_filesystem.rs index 8add88d7aa4..9a66cd551f0 100644 --- a/clients/filesystem-fuse/src/default_raw_filesystem.rs +++ b/clients/filesystem-fuse/src/default_raw_filesystem.rs @@ -205,23 +205,23 @@ impl RawFileSystem for DefaultRawFileSystem { async fn create_file(&self, parent_file_id: u64, name: &str, flags: u32) -> Result { let parent_file_entry = self.get_file_entry(parent_file_id).await?; - let mut file_with_out_id = self + let mut file_without_id = self .fs .create_file(&parent_file_entry.path, name, OpenFileFlags(flags)) .await?; - file_with_out_id.set_file_id(parent_file_id, self.next_file_id()); + file_without_id.set_file_id(parent_file_id, self.next_file_id()); // insert the new file to file entry manager self.insert_file_entry_locked( parent_file_id, - file_with_out_id.file_stat.file_id, - &file_with_out_id.file_stat.path, + file_without_id.file_stat.file_id, + &file_without_id.file_stat.path, ) .await; // put the openfile to the opened file manager and allocate a file handle id - let file_with_id = self.opened_file_manager.put(file_with_out_id); + let file_with_id = self.opened_file_manager.put(file_without_id); let opened_file_with_file_handle_id = file_with_id.lock().await; Ok(opened_file_with_file_handle_id.file_handle()) }