From 02a784ac0dd6880f4bc26bc7b145301e5b05e427 Mon Sep 17 00:00:00 2001 From: xmchx <1394466835@qq.com> Date: Thu, 24 Oct 2024 10:19:59 +0800 Subject: [PATCH 1/6] fix: add all-features flag for opendal_compat doc build (#5234) --- integrations/compat/Cargo.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/integrations/compat/Cargo.toml b/integrations/compat/Cargo.toml index 4fc457d9882..f92030fe641 100644 --- a/integrations/compat/Cargo.toml +++ b/integrations/compat/Cargo.toml @@ -27,6 +27,9 @@ repository = "https://github.com/apache/opendal" rust-version = "1.75" version = "1.0.0" +[package.metadata.docs.rs] +all-features = true + [features] v0_50_to_v0_49 = ["dep:opendal_v0_49", "dep:opendal_v0_50"] From 0ef4feab9355000bbf8791ab1b8da6e41053a5ac Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 24 Oct 2024 14:02:19 +0800 Subject: [PATCH 2/6] fix(integrations/compat): Capability has different fields (#5236) Signed-off-by: Xuanwo --- integrations/compat/Cargo.toml | 3 + integrations/compat/src/v0_50_to_v0_49.rs | 82 +++++++++++++++++++++-- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/integrations/compat/Cargo.toml b/integrations/compat/Cargo.toml index f92030fe641..a1834a07459 100644 --- a/integrations/compat/Cargo.toml +++ b/integrations/compat/Cargo.toml @@ -37,3 +37,6 @@ v0_50_to_v0_49 = ["dep:opendal_v0_49", "dep:opendal_v0_50"] async-trait = "0.1" opendal_v0_49 = { package = "opendal", version = "0.49", optional = true } opendal_v0_50 = { package = "opendal", version = "0.50", optional = true, path = "../../core" } + +[dev-dependencies] +tokio = { version = "1.41", features = ["full"] } diff --git a/integrations/compat/src/v0_50_to_v0_49.rs b/integrations/compat/src/v0_50_to_v0_49.rs index d26bfff3926..fea3744b0da 100644 --- a/integrations/compat/src/v0_50_to_v0_49.rs +++ b/integrations/compat/src/v0_50_to_v0_49.rs @@ -22,6 +22,7 @@ use opendal_v0_49::raw::{ }; use opendal_v0_49::Buffer; use std::fmt::{Debug, Formatter}; +use std::ops::Deref; use std::sync::Arc; /// Convert an opendal v0.50 `Operator` into an opendal v0.49 `Operator` for compatibility. @@ -64,7 +65,10 @@ impl opendal_v0_49::raw::Access for CompatAccesso type BlockingLister = CompatWrapper; fn info(&self) -> Arc { - convert::raw_oio_accessor_info_into(self.0.info()) + let new_info = self.0.info().deref().clone(); + let old_info = convert::raw_oio_accessor_info_into(new_info); + + Arc::new(old_info) } async fn create_dir(&self, path: &str, _: OpCreateDir) -> opendal_v0_49::Result { @@ -342,7 +346,6 @@ impl opendal_v0_49::raw::oio::Blocking /// in which we added a new field since v0.50. mod convert { use std::mem::transmute; - use std::sync::Arc; pub fn error_into(e: opendal_v0_50::Error) -> opendal_v0_49::Error { unsafe { transmute(e) } @@ -357,9 +360,61 @@ mod convert { } pub fn raw_oio_accessor_info_into( - e: Arc, - ) -> Arc { - unsafe { transmute(e) } + e: opendal_v0_50::raw::AccessorInfo, + ) -> opendal_v0_49::raw::AccessorInfo { + let mut info = opendal_v0_49::raw::AccessorInfo::default(); + info.set_name(e.name()) + .set_root(e.root()) + .set_scheme(e.scheme().into_static().parse().unwrap()) + .set_native_capability(capability_into(e.native_capability())); + + info + } + + /// opendal_v0_50 added a new field `write_with_if_none_match`. + pub fn capability_into(e: opendal_v0_50::Capability) -> opendal_v0_49::Capability { + opendal_v0_49::Capability { + stat: e.stat, + stat_with_if_match: e.stat_with_if_match, + stat_with_if_none_match: e.stat_with_if_none_match, + stat_with_override_cache_control: e.stat_with_override_cache_control, + stat_with_override_content_disposition: e.stat_with_override_content_disposition, + stat_with_override_content_type: e.stat_with_override_content_type, + read: e.read, + read_with_if_match: e.read_with_if_match, + read_with_if_none_match: e.read_with_if_none_match, + read_with_override_cache_control: e.read_with_override_cache_control, + read_with_override_content_disposition: e.read_with_override_content_disposition, + read_with_override_content_type: e.read_with_override_content_type, + write: e.write, + write_can_multi: e.write_can_multi, + write_can_empty: e.write_can_empty, + write_can_append: e.write_can_append, + write_with_content_type: e.write_with_content_type, + write_with_content_disposition: e.write_with_content_disposition, + write_with_cache_control: e.write_with_cache_control, + write_with_user_metadata: e.write_with_user_metadata, + write_multi_max_size: e.write_multi_max_size, + write_multi_min_size: e.write_multi_min_size, + write_multi_align_size: e.write_multi_align_size, + write_total_max_size: e.write_total_max_size, + create_dir: e.create_dir, + delete: e.delete, + copy: e.copy, + rename: e.rename, + list: e.list, + list_with_limit: e.list_with_limit, + list_with_start_after: e.list_with_start_after, + list_with_recursive: e.list_with_recursive, + presign: e.presign, + presign_read: e.presign_read, + presign_stat: e.presign_stat, + presign_write: e.presign_write, + batch: e.batch, + batch_delete: e.batch_delete, + batch_max_operations: e.batch_max_operations, + blocking: e.blocking, + } } pub fn raw_oio_entry_into(e: opendal_v0_50::raw::oio::Entry) -> opendal_v0_49::raw::oio::Entry { @@ -459,3 +514,20 @@ mod convert { unsafe { transmute(e) } } } + +#[cfg(test)] +mod tests { + use opendal_v0_50 as new_o; + + #[tokio::test] + async fn test_read() { + let new_op = new_o::Operator::from_config(new_o::services::MemoryConfig::default()) + .unwrap() + .finish(); + let old_op = super::v0_50_to_v0_49(new_op); + + old_op.write("test", "hello, world!").await.unwrap(); + let bs = old_op.read("test").await.unwrap(); + assert_eq!(String::from_utf8_lossy(&bs.to_vec()), "hello, world!"); + } +} From f6c5fc372971bd802823a78cf63ef97d49ebce89 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 24 Oct 2024 21:47:36 +0800 Subject: [PATCH 3/6] fix(integration/compat): Fix opendal 0.50 OpList has new field (#5238) Signed-off-by: Xuanwo --- integrations/compat/src/v0_50_to_v0_49.rs | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/integrations/compat/src/v0_50_to_v0_49.rs b/integrations/compat/src/v0_50_to_v0_49.rs index fea3744b0da..7810d69905e 100644 --- a/integrations/compat/src/v0_50_to_v0_49.rs +++ b/integrations/compat/src/v0_50_to_v0_49.rs @@ -345,6 +345,7 @@ impl opendal_v0_49::raw::oio::Blocking /// `transmute` also perform compile time checks to detect any type size mismatch like `OpWrite` /// in which we added a new field since v0.50. mod convert { + use opendal_v0_50::Metakey; use std::mem::transmute; pub fn error_into(e: opendal_v0_50::Error) -> opendal_v0_49::Error { @@ -474,8 +475,28 @@ mod convert { unsafe { transmute(e) } } + /// OpenDAL v0.50's OpList has a new field `version`. pub fn raw_op_list_from(e: opendal_v0_49::raw::OpList) -> opendal_v0_50::raw::OpList { - unsafe { transmute(e) } + let mut op = opendal_v0_50::raw::OpList::new(); + + if let Some(v) = e.limit() { + op = op.with_limit(v); + } + + if let Some(v) = e.start_after() { + op = op.with_start_after(v); + } + + if e.recursive() { + op = op.with_recursive(true); + } + + // There is no way for us to convert `metakey` without depending on `flagset`, + // let's just hardcode them. + op = op.with_metakey(Metakey::Mode | Metakey::LastModified); + op = op.with_concurrent(e.concurrent()); + + op } pub fn raw_rp_list_into(e: opendal_v0_50::raw::RpList) -> opendal_v0_49::raw::RpList { From f6e60f621dec82765a49283caf0ddc1ff8b1d260 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Thu, 24 Oct 2024 23:24:49 +0800 Subject: [PATCH 4/6] fix(integrations/compat): Fix dead loop happened during list (#5240) Signed-off-by: Xuanwo --- integrations/compat/src/v0_50_to_v0_49.rs | 70 +++++++++++++++++------ 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/integrations/compat/src/v0_50_to_v0_49.rs b/integrations/compat/src/v0_50_to_v0_49.rs index 7810d69905e..a69d22b977c 100644 --- a/integrations/compat/src/v0_50_to_v0_49.rs +++ b/integrations/compat/src/v0_50_to_v0_49.rs @@ -59,10 +59,10 @@ impl Debug for CompatAccessor { impl opendal_v0_49::raw::Access for CompatAccessor { type Reader = CompatWrapper; type Writer = CompatWrapper; - type Lister = CompatWrapper; + type Lister = CompatListWrapper; type BlockingReader = CompatWrapper; type BlockingWriter = CompatWrapper; - type BlockingLister = CompatWrapper; + type BlockingLister = CompatListWrapper; fn info(&self) -> Arc { let new_info = self.0.info().deref().clone(); @@ -131,7 +131,10 @@ impl opendal_v0_49::raw::Access for CompatAccesso .list(path, convert::raw_op_list_from(args)) .await .map_err(convert::error_into)?; - Ok((convert::raw_rp_list_into(rp), CompatWrapper(lister))) + Ok(( + convert::raw_rp_list_into(rp), + CompatListWrapper(path.to_string(), lister), + )) } async fn copy(&self, from: &str, to: &str, args: OpCopy) -> opendal_v0_49::Result { @@ -229,7 +232,10 @@ impl opendal_v0_49::raw::Access for CompatAccesso .0 .blocking_list(path, convert::raw_op_list_from(args)) .map_err(convert::error_into)?; - Ok((convert::raw_rp_list_into(rp), CompatWrapper(lister))) + Ok(( + convert::raw_rp_list_into(rp), + CompatListWrapper(path.to_string(), lister), + )) } fn blocking_copy(&self, from: &str, to: &str, args: OpCopy) -> opendal_v0_49::Result { @@ -281,16 +287,6 @@ impl opendal_v0_49::raw::oio::Write for Compa } } -impl opendal_v0_49::raw::oio::List for CompatWrapper { - async fn next(&mut self) -> opendal_v0_49::Result> { - self.0 - .next() - .await - .map(|v| v.map(convert::raw_oio_entry_into)) - .map_err(convert::error_into) - } -} - impl opendal_v0_49::raw::oio::BlockingRead for CompatWrapper { @@ -316,14 +312,50 @@ impl opendal_v0_49::raw::oio::Blockin } } +/// A wrapper to convert `List` from v0.50 to v0.49. +/// +/// The first `String` is the path of parent. We save it to check if the entry is itself. +/// In OpenDAL v0.50, lister will return itself, this behavior is different from v0.49. +struct CompatListWrapper(String, I); + +impl opendal_v0_49::raw::oio::List for CompatListWrapper { + async fn next(&mut self) -> opendal_v0_49::Result> { + loop { + let Some(de) = self + .1 + .next() + .await + .map(|v| v.map(convert::raw_oio_entry_into)) + .map_err(convert::error_into)? + else { + return Ok(None); + }; + if de.path() == self.0 { + continue; + } + return Ok(Some(de)); + } + } +} + impl opendal_v0_49::raw::oio::BlockingList - for CompatWrapper + for CompatListWrapper { fn next(&mut self) -> opendal_v0_49::Result> { - self.0 - .next() - .map(|v| v.map(convert::raw_oio_entry_into)) - .map_err(convert::error_into) + loop { + let Some(de) = self + .1 + .next() + .map(|v| v.map(convert::raw_oio_entry_into)) + .map_err(convert::error_into)? + else { + return Ok(None); + }; + if de.path() == self.0 { + continue; + } + return Ok(Some(de)); + } } } From 08d3bf4043a0a99f7f37d61754b633ca285d1f23 Mon Sep 17 00:00:00 2001 From: tison Date: Fri, 25 Oct 2024 12:22:50 +0800 Subject: [PATCH 5/6] chore: tidy up c binding build and docs (#5243) Signed-off-by: tison --- bindings/c/CMakeLists.txt | 5 ++++- bindings/c/README.md | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/bindings/c/CMakeLists.txt b/bindings/c/CMakeLists.txt index a6fe41ebd4c..41b510832be 100644 --- a/bindings/c/CMakeLists.txt +++ b/bindings/c/CMakeLists.txt @@ -16,6 +16,8 @@ # under the License. cmake_minimum_required(VERSION 3.22) +cmake_policy(SET CMP0135 NEW) + project(opendal-c) if (NOT CMAKE_BUILD_TYPE) @@ -23,7 +25,6 @@ if (NOT CMAKE_BUILD_TYPE) endif() option(TEST_ENABLE_ASAN "Enable AddressSanitizer for tests" OFF) -set(GOOGLETEST_VERSION 1.15.2) # force the compiler to support these standards set(CMAKE_C_STANDARD 11) @@ -35,6 +36,8 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) # fetch google test via GitHub include(FetchContent) + +set(GOOGLETEST_VERSION 1.15.2) FetchContent_Declare( googletest URL https://github.com/google/googletest/archive/refs/tags/v${GOOGLETEST_VERSION}.zip diff --git a/bindings/c/README.md b/bindings/c/README.md index 19bf72839b3..954405814a4 100644 --- a/bindings/c/README.md +++ b/bindings/c/README.md @@ -89,7 +89,7 @@ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - To **build the library and header file**. ```sh - mkdir build && cd build + mkdir -p build && cd build cmake .. make ``` From 3fe8caa08a7e91fe05d210154b792183bd846421 Mon Sep 17 00:00:00 2001 From: Nadeshiko Manju Date: Tue, 29 Oct 2024 00:46:22 +0800 Subject: [PATCH 6/6] build(binding/python): Support Python 3.13 (#5248) --- bindings/python/Cargo.toml | 4 ++-- bindings/python/pyproject.toml | 4 +--- bindings/python/src/file.rs | 2 +- bindings/python/src/lister.rs | 2 +- bindings/python/src/operator.rs | 4 ++-- 5 files changed, 7 insertions(+), 9 deletions(-) diff --git a/bindings/python/Cargo.toml b/bindings/python/Cargo.toml index 28f5725767d..0ecb72511ee 100644 --- a/bindings/python/Cargo.toml +++ b/bindings/python/Cargo.toml @@ -161,8 +161,8 @@ futures = "0.3.28" opendal = { version = ">=0", path = "../../core", features = [ "layers-blocking", ] } -pyo3 = { version = "0.21.2", features = ["abi3", "abi3-py311"] } -pyo3-asyncio = { package = "pyo3-asyncio-0-21", version = "0.21", features = [ +pyo3 = { version = "0.22.5", features = ["abi3", "abi3-py311"] } +pyo3-async-runtimes = { version = "0.22.0", features = [ "tokio-runtime", ] } tokio = "1" diff --git a/bindings/python/pyproject.toml b/bindings/python/pyproject.toml index cb64f80e69d..fd58e48ff34 100644 --- a/bindings/python/pyproject.toml +++ b/bindings/python/pyproject.toml @@ -29,9 +29,7 @@ description = "Apache OpenDAL™ Python Binding" license = { text = "Apache-2.0" } name = "opendal" readme = "README.md" -# PyO3 doesn't support python 3.13 yet. -# ref: https://github.com/apache/opendal/issues/4268 -requires-python = ">=3.11, < 3.13" +requires-python = ">=3.11" [project.optional-dependencies] benchmark = [ diff --git a/bindings/python/src/file.rs b/bindings/python/src/file.rs index 53a65a274f9..be7a461893e 100644 --- a/bindings/python/src/file.rs +++ b/bindings/python/src/file.rs @@ -31,7 +31,7 @@ use pyo3::buffer::PyBuffer; use pyo3::exceptions::PyIOError; use pyo3::exceptions::PyValueError; use pyo3::prelude::*; -use pyo3_asyncio::tokio::future_into_py; +use pyo3_async_runtimes::tokio::future_into_py; use tokio::sync::Mutex; use crate::*; diff --git a/bindings/python/src/lister.rs b/bindings/python/src/lister.rs index e108ec2dcc3..e2074016fae 100644 --- a/bindings/python/src/lister.rs +++ b/bindings/python/src/lister.rs @@ -20,7 +20,7 @@ use std::sync::Arc; use futures::TryStreamExt; use pyo3::exceptions::PyStopAsyncIteration; use pyo3::prelude::*; -use pyo3_asyncio::tokio::future_into_py; +use pyo3_async_runtimes::tokio::future_into_py; use tokio::sync::Mutex; use crate::*; diff --git a/bindings/python/src/operator.rs b/bindings/python/src/operator.rs index 54e7345741f..0e6c51ce107 100644 --- a/bindings/python/src/operator.rs +++ b/bindings/python/src/operator.rs @@ -22,7 +22,7 @@ use std::time::Duration; use pyo3::prelude::*; use pyo3::types::PyBytes; use pyo3::types::PyDict; -use pyo3_asyncio::tokio::future_into_py; +use pyo3_async_runtimes::tokio::future_into_py; use crate::*; @@ -32,7 +32,7 @@ fn build_operator( ) -> PyResult { let mut op = ocore::Operator::via_iter(scheme, map).map_err(format_pyerr)?; if !op.info().full_capability().blocking { - let runtime = pyo3_asyncio::tokio::get_runtime(); + let runtime = pyo3_async_runtimes::tokio::get_runtime(); let _guard = runtime.enter(); op = op .layer(ocore::layers::BlockingLayer::create().expect("blocking layer must be created"));