From 2934c1d5c732f89279c8c8f5c26036525ae82e5c Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sat, 21 Sep 2024 19:02:04 +0800 Subject: [PATCH] test(core): add tests for versioning --- core/src/services/s3/backend.rs | 3 +- core/src/types/capability.rs | 5 ++- core/src/types/operator/operator_futures.rs | 2 +- core/tests/behavior/async_delete.rs | 40 ++++++++++++++++- core/tests/behavior/async_list.rs | 6 +-- core/tests/behavior/async_read.rs | 37 +++++++++++++++- core/tests/behavior/async_stat.rs | 48 ++++++++++++++++++--- 7 files changed, 127 insertions(+), 14 deletions(-) diff --git a/core/src/services/s3/backend.rs b/core/src/services/s3/backend.rs index 745b292f2b5..9b6c8f95898 100644 --- a/core/src/services/s3/backend.rs +++ b/core/src/services/s3/backend.rs @@ -938,7 +938,6 @@ impl Access for S3Backend { list_with_limit: true, list_with_start_after: true, list_with_recursive: true, - list_with_version: self.core.enable_versioning, presign: true, presign_stat: true, @@ -948,6 +947,8 @@ impl Access for S3Backend { batch: true, batch_max_operations: Some(self.core.batch_max_operations), + versioning: self.core.enable_versioning, + ..Default::default() }); diff --git a/core/src/types/capability.rs b/core/src/types/capability.rs index cae69f04ab6..b3b654feaeb 100644 --- a/core/src/types/capability.rs +++ b/core/src/types/capability.rs @@ -134,8 +134,6 @@ pub struct Capability { pub list_with_start_after: bool, /// If backend supports list with recursive. pub list_with_recursive: bool, - /// If backend supports list with object version. - pub list_with_version: bool, /// If operator supports presign. pub presign: bool, @@ -155,6 +153,9 @@ pub struct Capability { /// If operator supports blocking. pub blocking: bool, + + /// If operator supports versioning + pub versioning: bool, } impl Debug for Capability { diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 777c817e0f8..b5ddd1d4c32 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -406,7 +406,7 @@ impl>> FutureWriter { /// /// ## Notes /// - /// we don't need to include the user defined metadata prefix in the key + /// we don't need to include the user defined metadata prefix in the key. /// every service will handle it internally pub fn user_metadata(self, data: impl IntoIterator) -> Self { self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options)) diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs index ab51712d696..f1d2f35dc3b 100644 --- a/core/tests/behavior/async_delete.rs +++ b/core/tests/behavior/async_delete.rs @@ -33,7 +33,8 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_delete_with_special_chars, test_delete_not_existing, test_delete_stream, - test_remove_one_file + test_remove_one_file, + test_delete_with_version )); if cap.list_with_recursive { tests.extend(async_trials!(op, test_remove_all_basic)); @@ -212,3 +213,40 @@ pub async fn test_remove_all_with_prefix_exists(op: Operator) -> Result<()> { .expect("write must succeed"); test_blocking_remove_all_with_objects(op, parent, ["a", "a/b", "a/c", "a/b/e"]).await } + +pub async fn test_delete_with_version(op: Operator) -> Result<()> { + if !op.info().full_capability().versioning { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + //TODO: refactor these code after `write` can return metadata + op.write(path.as_str(), content) + .await + .expect("write must success"); + let meta = op.stat(path.as_str()).await.expect("stat must success"); + let version = meta.version().expect("must have version"); + + op.delete(path.as_str()).await.expect("delete must success"); + assert!(!op.is_exist(path.as_str()).await?); + + // after simple delete, we can still get the data using version + let meta = op + .stat_with(path.as_str()) + .version(version) + .await + .expect("stat must success"); + assert_eq!(version, meta.version().expect("must have version")); + + // after delete with version, we can get the object with special version + op.delete_with(path.as_str()) + .version(version) + .await + .expect("delete must success"); + let ret = op.stat_with(path.as_str()).version(version).await; + assert!(ret.is_err()); + assert_eq!(ret.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} diff --git a/core/tests/behavior/async_list.rs b/core/tests/behavior/async_list.rs index ba862f70252..f48c689ab7c 100644 --- a/core/tests/behavior/async_list.rs +++ b/core/tests/behavior/async_list.rs @@ -684,7 +684,7 @@ pub async fn test_list_only(op: Operator) -> Result<()> { } pub async fn test_list_files_with_version(op: Operator) -> Result<()> { - if !op.info().full_capability().list_with_version { + if !op.info().full_capability().versioning { return Ok(()); } @@ -722,7 +722,7 @@ pub async fn test_list_with_version_and_limit(op: Operator) -> Result<()> { if op.info().scheme() == Scheme::Gdrive { return Ok(()); } - if !op.info().full_capability().list_with_version { + if !op.info().full_capability().versioning { return Ok(()); } @@ -775,7 +775,7 @@ pub async fn test_list_with_version_and_limit(op: Operator) -> Result<()> { } pub async fn test_list_with_version_and_start_after(op: Operator) -> Result<()> { - if !op.info().full_capability().list_with_version { + if !op.info().full_capability().versioning { return Ok(()); } diff --git a/core/tests/behavior/async_read.rs b/core/tests/behavior/async_read.rs index 53e6f29a610..82f869c44cc 100644 --- a/core/tests/behavior/async_read.rs +++ b/core/tests/behavior/async_read.rs @@ -44,7 +44,8 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_read_with_special_chars, test_read_with_override_cache_control, test_read_with_override_content_disposition, - test_read_with_override_content_type + test_read_with_override_content_type, + test_read_with_version )) } @@ -553,3 +554,37 @@ pub async fn test_read_only_read_with_if_none_match(op: Operator) -> anyhow::Res Ok(()) } + +pub async fn test_read_with_version(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().versioning { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let meta = op.stat(path.as_str()).await.expect("stat must success"); + let version = meta.version().expect("must have version"); + + let data = op + .read_with(path.as_str()) + .version(version) + .await + .expect("read must success"); + assert_eq!(content, data.to_vec()); + + op.write(path.as_str(), "1") + .await + .expect("write must success"); + + // we can use read with version to get the first version data + let second_data = op + .read_with(path.as_str()) + .version(version) + .await + .expect("read must success"); + assert_eq!(content, second_data.to_vec()); + + Ok(()) +} diff --git a/core/tests/behavior/async_stat.rs b/core/tests/behavior/async_stat.rs index 2dcbde9b2df..146031c87d9 100644 --- a/core/tests/behavior/async_stat.rs +++ b/core/tests/behavior/async_stat.rs @@ -18,13 +18,12 @@ use std::str::FromStr; use std::time::Duration; +use crate::*; use anyhow::Result; use http::StatusCode; use log::warn; use reqwest::Url; -use crate::*; - pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); @@ -42,7 +41,8 @@ pub fn tests(op: &Operator, tests: &mut Vec) { test_stat_with_override_cache_control, test_stat_with_override_content_disposition, test_stat_with_override_content_type, - test_stat_root + test_stat_root, + test_stat_with_version )) } @@ -166,12 +166,12 @@ pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> { pub async fn test_stat_not_exist(op: Operator) -> Result<()> { let path = uuid::Uuid::new_v4().to_string(); - // Stat not exist file should returns NotFound. + // Stat not exist file should return NotFound. let meta = op.stat(&path).await; assert!(meta.is_err()); assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - // Stat not exist dir should also returns NotFound. + // Stat not exist dir should also return NotFound. if op.info().full_capability().create_dir { let meta = op.stat(&format!("{path}/")).await; assert!(meta.is_err()); @@ -499,3 +499,41 @@ pub async fn test_read_only_stat_root(op: Operator) -> Result<()> { Ok(()) } + +pub async fn test_stat_with_version(op: Operator) -> Result<()> { + if !op.info().full_capability().versioning { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(path.as_str(), content.clone()) + .await + .expect("write must success"); + let first_meta = op.stat(path.as_str()).await.expect("stat must success"); + let first_version = first_meta.version().expect("must have version"); + + let first_versioning_meta = op + .stat_with(path.as_str()) + .version(first_version) + .await + .expect("stat must success"); + assert_eq!(first_meta, first_versioning_meta); + + op.write(path.as_str(), content) + .await + .expect("write must success"); + let second_meta = op.stat(path.as_str()).await.expect("stat must success"); + let second_version = second_meta.version().expect("must have version"); + assert_ne!(first_version, second_version); + + // we can still `stat` with first_version after writing new data + let meta = op + .stat_with(path.as_str()) + .version(first_version) + .await + .expect("stat must success"); + assert_eq!(first_meta, meta); + + Ok(()) +}