Merge pull request #37 from manomayam/migr_opendal_039_042

chore(deps): migrate opendal from 0.39 to 0.42.0
manomayam · Nov 15, 2023 · 7a57489 · 7a57489
2 parents 699e853 + bcb268a
commit 7a57489
Show file tree

Hide file tree

Showing 9 changed files with 24 additions and 32 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/crates/manas_repo_opendal/Cargo.toml b/crates/manas_repo_opendal/Cargo.toml
@@ -34,7 +34,7 @@ bimap = "0.6.3"
 thiserror = "1.0.43"
 flagset = "0.4.3"
 # Opendal is pinned, as it breaks frequently.
-opendal = { version = "^0.39.0", features = ["rustls"]}
+opendal = { version = "^0.42.0", git = "https://github.com/apache/incubator-opendal/", tag = "v0.42.0-rc.3", features = ["rustls"]}
 gdp_rs = { version = "0.1.1", path = "../../fcrates/gdp_rs" }
 regex = "1.10.2"
 percent-encoding = "2.3.0"

diff --git a/crates/manas_repo_opendal/src/context.rs b/crates/manas_repo_opendal/src/context.rs
@@ -45,7 +45,11 @@ where
     /// Get backend capabilities.
     #[inline]
     pub fn backend_caps(&self) -> Capability {
-        self.object_store.backend.operator().info().capability()
+        self.object_store
+            .backend
+            .operator()
+            .info()
+            .full_capability()
     }
 
     /// Create a new [`ODRContext`] from given params.

diff --git a/crates/manas_repo_opendal/src/object_store/backend/impl_/common/stat_fix_layer.rs b/crates/manas_repo_opendal/src/object_store/backend/impl_/common/stat_fix_layer.rs
@@ -32,7 +32,6 @@ impl<A: Accessor> LayeredAccessor for StatFixAccessor<A> {
     type Inner = A;
     type Reader = A::Reader;
     type BlockingReader = A::BlockingReader;
-    type Appender = A::Appender;
     type Writer = A::Writer;
     type BlockingWriter = A::BlockingWriter;
     type Pager = A::Pager;
@@ -54,10 +53,6 @@ impl<A: Accessor> LayeredAccessor for StatFixAccessor<A> {
         self.inner.write(path, args).await
     }
 
-    async fn append(&self, path: &str, args: OpAppend) -> Result<(RpAppend, Self::Appender)> {
-        self.inner.append(path, args).await
-    }
-
     fn blocking_write(&self, path: &str, args: OpWrite) -> Result<(RpWrite, Self::BlockingWriter)> {
         self.inner.blocking_write(path, args)
     }

diff --git a/crates/manas_repo_opendal/src/object_store/backend/impl_/embedded/service.rs b/crates/manas_repo_opendal/src/object_store/backend/impl_/embedded/service.rs
@@ -156,7 +156,6 @@ impl<Assets: RustEmbed + 'static> Accessor for EmbeddedAccessor<Assets> {
     type Reader = oio::Cursor;
     type BlockingReader = ();
     type Writer = ();
-    type Appender = ();
     type BlockingWriter = ();
     type Pager = NsPage<Assets>;
     type BlockingPager = ();
@@ -165,7 +164,7 @@ impl<Assets: RustEmbed + 'static> Accessor for EmbeddedAccessor<Assets> {
         let mut info = AccessorInfo::default();
         info.set_scheme(opendal::Scheme::Custom("Embedded"))
             .set_name(self.name.as_str())
-            .set_capability(Capability {
+            .set_native_capability(Capability {
                 stat: true,
                 read: true,
                 read_with_range: true,
@@ -201,7 +200,10 @@ impl<Assets: RustEmbed + 'static> Accessor for EmbeddedAccessor<Assets> {
         metadata = metadata.with_content_length(bs.len() as u64);
         // TODO content-range should be included.
 
-        Ok((RpRead::with_metadata(metadata), oio::Cursor::from(bs)))
+        Ok((
+            RpRead::new().with_size(Some(metadata.content_length())),
+            oio::Cursor::from(bs),
+        ))
     }
 
     async fn list(&self, path: &str, args: OpList) -> Result<(RpList, Self::Pager)> {

diff --git a/crates/manas_repo_opendal/src/object_store/object/invariant.rs b/crates/manas_repo_opendal/src/object_store/object/invariant.rs
@@ -11,15 +11,15 @@ use manas_http::{
     header::common::media_type::MediaType,
     representation::impl_::common::data::bytes_stream::BoxBytesStream,
 };
-use opendal::{EntryMode, Lister, Metakey, Writer};
+use opendal::{EntryMode, Lister, Writer};
 use tracing::{error, warn};
 
 use super::{
     predicate::{
         is_file_object::IsFileObject, is_namespace_object::IsNamespaceObject,
         ObjectKindBasedClassification,
     },
-    ODRObject,
+    ODRObject, ODR_OBJECT_METAKEY,
 };
 use crate::object_store::{
     backend::ODRObjectStoreBackend, ODRObjectStoreSetup, OstBackendPathDecodeError,
@@ -222,8 +222,8 @@ impl<'id, OstSetup: ODRObjectStoreSetup> ODRNamespaceObjectExt<OstSetup>
         let backend_listing: Lister = self
             .backend
             .operator()
-            .list(self.backend_entry.path())
-            // .metakey(*ODR_OBJECT_METAKEY)
+            .lister_with(self.backend_entry.path())
+            .metakey(*ODR_OBJECT_METAKEY)
             .inspect_err(|_| error!("Error in getting backend entries."))
             .await?;
 
@@ -234,8 +234,7 @@ impl<'id, OstSetup: ODRObjectStoreSetup> ODRNamespaceObjectExt<OstSetup>
             for await entry in backend_listing {
                 let entry = entry?;
                 // Get mode from metadata.
-                // let mode = entry.metadata().mode();
-                let mode = backend.operator().metadata(&entry, Metakey::Mode).await?.mode();
+                let mode = entry.metadata().mode();
 
                 // If unknown mode, yield error.
                 if mode ==EntryMode::Unknown {

diff --git a/crates/manas_repo_opendal/src/object_store/object/mod.rs b/crates/manas_repo_opendal/src/object_store/object/mod.rs
@@ -184,13 +184,7 @@ impl<'id, OstSetup: ODRObjectStoreSetup> ODRObject<'id, OstSetup> {
     pub async fn metadata(&self) -> Result<Metadata, opendal::Error> {
         match &self.backend_entry {
             OpendalBackendEntry::Simple { path } => self.backend.operator().stat(path).await,
-            OpendalBackendEntry::Cached(cached_entry) => {
-                // Ok(cached_entry.metadata().clone())
-                self.backend
-                    .operator()
-                    .metadata(cached_entry, *ODR_OBJECT_METAKEY)
-                    .await
-            }
+            OpendalBackendEntry::Cached(cached_entry) => Ok(cached_entry.metadata().clone()),
         }
     }
 

diff --git a/...opendal/src/service/resource_operator/common/status_token/variant/existing_represented.rs b/...opendal/src/service/resource_operator/common/status_token/variant/existing_represented.rs
@@ -234,7 +234,7 @@ impl<Setup: ODRSetup> ODRExistingRepresentedResourceToken<Setup> {
 
             if let Some(cty_str) = opt_cty_str {
                 if let Ok(cty) = MediaType::from_str(cty_str).map_err(|e| {
-                    warn!("Invalid base object content-type metadata. Err: {}", e);
+                    warn!("Invalid assoc object content-type metadata. Err: {}", e);
                     e
                 }) {
                     return Ok(cty);

diff --git a/crates/manas_server/Cargo.toml b/crates/manas_server/Cargo.toml
@@ -31,7 +31,7 @@ tower = { version = "0.4.13", features = ["make", "util"] }
 webid = { version = "0.1.0", path = "../../fcrates/webid" }
 rust-embed = { version = "8.0.0" }
 # Opendal is pinned, as it breaks frequently.
-opendal = {version = "^0.39.0", features = ["rustls"]}
+opendal = {version = "^0.42.0", git = "https://github.com/apache/incubator-opendal/", tag = "v0.42.0-rc.3", features = ["rustls"]}
 typed_record = { version = "0.1.1", path = "../../fcrates/typed_record", features = [
     "ext-http",
 ] }