From 62e79cae1781a96118b61c7448f3aafb8fc2e161 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Wed, 3 Jan 2024 18:29:15 +0800 Subject: [PATCH] refactor: Re-organize the layout of tests (#3904) * Refactor tests Signed-off-by: Xuanwo * refactor: Re-organize the layout of tests Signed-off-by: Xuanwo --------- Signed-off-by: Xuanwo --- core/tests/behavior/append.rs | 223 -- .../tests/behavior/{copy.rs => async_copy.rs} | 28 +- .../{list_only.rs => async_create_dir.rs} | 41 +- core/tests/behavior/async_delete.rs | 158 ++ .../tests/behavior/{fuzz.rs => async_fuzz.rs} | 6 +- .../tests/behavior/{list.rs => async_list.rs} | 65 +- .../behavior/{presign.rs => async_presign.rs} | 13 +- core/tests/behavior/async_read.rs | 794 +++++++ .../behavior/{rename.rs => async_rename.rs} | 26 +- core/tests/behavior/async_stat.rs | 501 +++++ core/tests/behavior/async_write.rs | 421 ++++ core/tests/behavior/blocking_append.rs | 220 -- core/tests/behavior/blocking_copy.rs | 26 +- core/tests/behavior/blocking_create_dir.rs | 68 + core/tests/behavior/blocking_delete.rs | 64 + core/tests/behavior/blocking_list.rs | 24 +- core/tests/behavior/blocking_read.rs | 212 ++ core/tests/behavior/blocking_read_only.rs | 112 - core/tests/behavior/blocking_rename.rs | 26 +- core/tests/behavior/blocking_stat.rs | 107 + core/tests/behavior/blocking_write.rs | 518 +---- core/tests/behavior/main.rs | 90 +- core/tests/behavior/read_only.rs | 331 --- core/tests/behavior/utils.rs | 301 +-- core/tests/behavior/write.rs | 1825 ----------------- 25 files changed, 2614 insertions(+), 3586 deletions(-) delete mode 100644 core/tests/behavior/append.rs rename core/tests/behavior/{copy.rs => async_copy.rs} (93%) rename core/tests/behavior/{list_only.rs => async_create_dir.rs} (52%) create mode 100644 core/tests/behavior/async_delete.rs rename core/tests/behavior/{fuzz.rs => async_fuzz.rs} (97%) rename core/tests/behavior/{list.rs => async_list.rs} (91%) rename core/tests/behavior/{presign.rs => async_presign.rs} (94%) create mode 100644 core/tests/behavior/async_read.rs rename core/tests/behavior/{rename.rs => async_rename.rs} (93%) create mode 100644 core/tests/behavior/async_stat.rs create mode 100644 core/tests/behavior/async_write.rs delete mode 100644 core/tests/behavior/blocking_append.rs create mode 100644 core/tests/behavior/blocking_create_dir.rs create mode 100644 core/tests/behavior/blocking_delete.rs create mode 100644 core/tests/behavior/blocking_read.rs delete mode 100644 core/tests/behavior/blocking_read_only.rs create mode 100644 core/tests/behavior/blocking_stat.rs delete mode 100644 core/tests/behavior/read_only.rs delete mode 100644 core/tests/behavior/write.rs diff --git a/core/tests/behavior/append.rs b/core/tests/behavior/append.rs deleted file mode 100644 index 5f535e4c110..00000000000 --- a/core/tests/behavior/append.rs +++ /dev/null @@ -1,223 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::vec; - -use anyhow::Result; -use futures::io::BufReader; -use futures::io::Cursor; -use sha2::Digest; -use sha2::Sha256; - -use crate::*; - -pub fn behavior_append_tests(op: &Operator) -> Vec { - let cap = op.info().full_capability(); - - if !(cap.read && cap.write && cap.write_can_append) { - return vec![]; - } - - async_trials!( - op, - test_append_create_append, - test_append_with_dir_path, - test_append_with_cache_control, - test_append_with_content_type, - test_append_with_content_disposition, - test_appender_futures_copy, - test_fuzz_appender - ) -} - -/// Test append to a file must success. -pub async fn test_append_create_append(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content_one, size_one) = gen_bytes(op.info().full_capability()); - let (content_two, size_two) = gen_bytes(op.info().full_capability()); - - op.write_with(&path, content_one.clone()) - .append(true) - .await - .expect("append file first time must success"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.content_length(), size_one as u64); - - op.write_with(&path, content_two.clone()) - .append(true) - .await - .expect("append to an existing file must success"); - - let bs = op.read(&path).await.expect("read file must success"); - - assert_eq!(bs.len(), size_one + size_two); - assert_eq!(bs[..size_one], content_one); - assert_eq!(bs[size_one..], content_two); - - op.delete(&path).await.expect("delete file must success"); - - Ok(()) -} - -/// Test append to a directory path must fail. -pub async fn test_append_with_dir_path(op: Operator) -> Result<()> { - let path = format!("{}/", uuid::Uuid::new_v4()); - let (content, _) = gen_bytes(op.info().full_capability()); - - let res = op.write_with(&path, content).append(true).await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::IsADirectory); - - Ok(()) -} - -/// Test append with cache control must success. -pub async fn test_append_with_cache_control(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_cache_control { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - let target_cache_control = "no-cache, no-store, max-age=300"; - op.write_with(&path, content) - .append(true) - .cache_control(target_cache_control) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.cache_control().expect("cache control must exist"), - target_cache_control - ); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Test append with content type must success. -pub async fn test_append_with_content_type(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_content_type { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_type = "application/json"; - op.write_with(&path, content) - .append(true) - .content_type(target_content_type) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_type().expect("content type must exist"), - target_content_type - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Write a single file with content disposition should succeed. -pub async fn test_append_with_content_disposition(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_content_disposition { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_disposition = "attachment; filename=\"filename.jpg\""; - op.write_with(&path, content) - .append(true) - .content_disposition(target_content_disposition) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_disposition().expect("content type must exist"), - target_content_disposition - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Copy data from reader to writer -pub async fn test_appender_futures_copy(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, size): (Vec, usize) = - gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); - - let mut a = op.writer_with(&path).append(true).await?; - - // Wrap a buf reader here to make sure content is read in 1MiB chunks. - let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); - futures::io::copy_buf(&mut cursor, &mut a).await?; - a.close().await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), size as u64); - - let bs = op.read(&path).await?; - assert_eq!(bs.len(), size, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content)), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Test for fuzzing appender. -pub async fn test_fuzz_appender(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let mut fuzzer = ObjectWriterFuzzer::new(&path, None); - - let mut a = op.writer_with(&path).append(true).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectWriterAction::Write(bs) => { - a.write(bs).await?; - } - } - } - a.close().await?; - - let content = op.read(&path).await?; - fuzzer.check(&content); - - op.delete(&path).await.expect("delete file must success"); - - Ok(()) -} diff --git a/core/tests/behavior/copy.rs b/core/tests/behavior/async_copy.rs similarity index 93% rename from core/tests/behavior/copy.rs rename to core/tests/behavior/async_copy.rs index 1da12af7637..cee9494ec8a 100644 --- a/core/tests/behavior/copy.rs +++ b/core/tests/behavior/async_copy.rs @@ -21,24 +21,22 @@ use sha2::Sha256; use crate::*; -pub fn behavior_copy_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.copy) { - return vec![]; + if cap.read && cap.write && cap.copy { + tests.extend(async_trials!( + op, + test_copy_file_with_ascii_name, + test_copy_file_with_non_ascii_name, + test_copy_non_existing_source, + test_copy_source_dir, + test_copy_target_dir, + test_copy_self, + test_copy_nested, + test_copy_overwrite + )) } - - async_trials!( - op, - test_copy_file_with_ascii_name, - test_copy_file_with_non_ascii_name, - test_copy_non_existing_source, - test_copy_source_dir, - test_copy_target_dir, - test_copy_self, - test_copy_nested, - test_copy_overwrite - ) } /// Copy a file with ascii name and test contents. diff --git a/core/tests/behavior/list_only.rs b/core/tests/behavior/async_create_dir.rs similarity index 52% rename from core/tests/behavior/list_only.rs rename to core/tests/behavior/async_create_dir.rs index d7b02da6d96..4b7fa9ec043 100644 --- a/core/tests/behavior/list_only.rs +++ b/core/tests/behavior/async_create_dir.rs @@ -15,40 +15,39 @@ // specific language governing permissions and limitations // under the License. -use std::collections::HashMap; - use anyhow::Result; -use futures::TryStreamExt; use crate::*; -pub fn behavior_list_only_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !cap.list || cap.write { - return vec![]; + if cap.create_dir && cap.stat { + tests.extend(async_trials!(op, test_create_dir, test_create_dir_existing)) } +} + +/// Create dir with dir path should succeed. +pub async fn test_create_dir(op: Operator) -> Result<()> { + let path = TEST_FIXTURE.new_dir_path(); - async_trials!(op, test_list_only) + op.create_dir(&path).await?; + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::DIR); + Ok(()) } -/// Stat normal file and dir should return metadata -pub async fn test_list_only(op: Operator) -> Result<()> { - let mut entries = HashMap::new(); +/// Create dir on existing dir should succeed. +pub async fn test_create_dir_existing(op: Operator) -> Result<()> { + let path = TEST_FIXTURE.new_dir_path(); - let mut ds = op.lister("/").await?; - while let Some(de) = ds.try_next().await? { - entries.insert(de.path().to_string(), op.stat(de.path()).await?.mode()); - } + op.create_dir(&path).await?; - assert_eq!(entries["normal_file.txt"], EntryMode::FILE); - assert_eq!( - entries["special_file !@#$%^&()_+-=;',.txt"], - EntryMode::FILE - ); + op.create_dir(&path).await?; - assert_eq!(entries["normal_dir/"], EntryMode::DIR); - assert_eq!(entries["special_dir !@#$%^&()_+-=;',/"], EntryMode::DIR); + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::DIR); Ok(()) } diff --git a/core/tests/behavior/async_delete.rs b/core/tests/behavior/async_delete.rs new file mode 100644 index 00000000000..11e5c2ca486 --- /dev/null +++ b/core/tests/behavior/async_delete.rs @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use futures::StreamExt; +use log::warn; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.delete && cap.write { + tests.extend(async_trials!( + op, + test_delete_file, + test_delete_empty_dir, + test_delete_with_special_chars, + test_delete_not_existing, + test_delete_stream, + test_remove_one_file + )) + } +} + +/// Delete existing file should succeed. +pub async fn test_delete_file(op: Operator) -> Result<()> { + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content).await.expect("write must succeed"); + + op.delete(&path).await?; + + // Stat it again to check. + assert!(!op.is_exist(&path).await?); + + Ok(()) +} + +/// Delete empty dir should succeed. +pub async fn test_delete_empty_dir(op: Operator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = TEST_FIXTURE.new_dir_path(); + + op.create_dir(&path).await.expect("create must succeed"); + + op.delete(&path).await?; + + Ok(()) +} + +/// Delete file with special chars should succeed. +pub async fn test_delete_with_special_chars(op: Operator) -> Result<()> { + // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. + if op.info().scheme() == opendal::Scheme::Supabase { + warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); + return Ok(()); + } + // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. + if op.info().scheme() == opendal::Scheme::Atomicserver { + warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); + return Ok(()); + } + + let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); + let (path, content, _) = TEST_FIXTURE.new_file_with_path(op.clone(), &path); + + op.write(&path, content).await.expect("write must succeed"); + + op.delete(&path).await?; + + // Stat it again to check. + assert!(!op.is_exist(&path).await?); + + Ok(()) +} + +/// Delete not existing file should also succeed. +pub async fn test_delete_not_existing(op: Operator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + op.delete(&path).await?; + + Ok(()) +} + +/// Remove one file +pub async fn test_remove_one_file(op: Operator) -> Result<()> { + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + op.remove(vec![path.clone()]).await?; + + // Stat it again to check. + assert!(!op.is_exist(&path).await?); + + op.write(&format!("/{path}"), content) + .await + .expect("write must succeed"); + + op.remove(vec![path.clone()]).await?; + + // Stat it again to check. + assert!(!op.is_exist(&path).await?); + + Ok(()) +} + +/// Delete via stream. +pub async fn test_delete_stream(op: Operator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let dir = uuid::Uuid::new_v4().to_string(); + op.create_dir(&format!("{dir}/")) + .await + .expect("creat must succeed"); + + let expected: Vec<_> = (0..100).collect(); + for path in expected.iter() { + op.write(&format!("{dir}/{path}"), "delete_stream").await?; + } + + op.with_limit(30) + .remove_via(futures::stream::iter(expected.clone()).map(|v| format!("{dir}/{v}"))) + .await?; + + // Stat it again to check. + for path in expected.iter() { + assert!( + !op.is_exist(&format!("{dir}/{path}")).await?, + "{path} should be removed" + ) + } + + Ok(()) +} diff --git a/core/tests/behavior/fuzz.rs b/core/tests/behavior/async_fuzz.rs similarity index 97% rename from core/tests/behavior/fuzz.rs rename to core/tests/behavior/async_fuzz.rs index cc97c0ea4c8..c6e27c47fe0 100644 --- a/core/tests/behavior/fuzz.rs +++ b/core/tests/behavior/async_fuzz.rs @@ -25,13 +25,13 @@ use opendal::raw::BytesRange; use crate::*; -pub fn behavior_fuzz_tests(op: &Operator) -> Vec { - async_trials!( +pub fn tests(op: &Operator, tests: &mut Vec) { + tests.extend(async_trials!( op, test_fuzz_issue_2717, test_fuzz_pr_3395_case_1, test_fuzz_pr_3395_case_2 - ) + )) } async fn test_fuzz_read( diff --git a/core/tests/behavior/list.rs b/core/tests/behavior/async_list.rs similarity index 91% rename from core/tests/behavior/list.rs rename to core/tests/behavior/async_list.rs index b421da57fdd..2ac169c20df 100644 --- a/core/tests/behavior/list.rs +++ b/core/tests/behavior/async_list.rs @@ -26,31 +26,33 @@ use log::debug; use crate::*; -pub fn behavior_list_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.list) { - return vec![]; + if cap.read && cap.write && cap.list { + tests.extend(async_trials!( + op, + test_check, + test_list_dir, + test_list_dir_with_metakey, + test_list_dir_with_metakey_complete, + test_list_prefix, + test_list_rich_dir, + test_list_empty_dir, + test_list_non_exist_dir, + test_list_sub_dir, + test_list_nested_dir, + test_list_dir_with_file_path, + test_list_with_start_after, + test_list_with_recursive, + test_list_root_with_recursive, + test_remove_all + )) } - async_trials!( - op, - test_check, - test_list_dir, - test_list_dir_with_metakey, - test_list_dir_with_metakey_complete, - test_list_prefix, - test_list_rich_dir, - test_list_empty_dir, - test_list_non_exist_dir, - test_list_sub_dir, - test_list_nested_dir, - test_list_dir_with_file_path, - test_list_with_start_after, - test_list_with_recursive, - test_list_root_with_recursive, - test_remove_all - ) + if cap.read && !cap.write && cap.list { + tests.extend(async_trials!(op, test_list_only)) + } } /// Check should be OK. @@ -530,3 +532,24 @@ pub async fn test_remove_all(op: Operator) -> Result<()> { } Ok(()) } + +/// Stat normal file and dir should return metadata +pub async fn test_list_only(op: Operator) -> Result<()> { + let mut entries = HashMap::new(); + + let mut ds = op.lister("/").await?; + while let Some(de) = ds.try_next().await? { + entries.insert(de.path().to_string(), op.stat(de.path()).await?.mode()); + } + + assert_eq!(entries["normal_file.txt"], EntryMode::FILE); + assert_eq!( + entries["special_file !@#$%^&()_+-=;',.txt"], + EntryMode::FILE + ); + + assert_eq!(entries["normal_dir/"], EntryMode::DIR); + assert_eq!(entries["special_dir !@#$%^&()_+-=;',/"], EntryMode::DIR); + + Ok(()) +} diff --git a/core/tests/behavior/presign.rs b/core/tests/behavior/async_presign.rs similarity index 94% rename from core/tests/behavior/presign.rs rename to core/tests/behavior/async_presign.rs index 87bc2365b85..e58d419fb41 100644 --- a/core/tests/behavior/presign.rs +++ b/core/tests/behavior/async_presign.rs @@ -28,14 +28,17 @@ use sha2::Sha256; use crate::*; -pub fn behavior_presign_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.list && cap.write && cap.presign) { - return vec![]; + if cap.read && cap.write && cap.presign { + tests.extend(async_trials!( + op, + test_presign_write, + test_presign_read, + test_presign_stat + )) } - - async_trials!(op, test_presign_write, test_presign_read, test_presign_stat) } /// Presign write should succeed. diff --git a/core/tests/behavior/async_read.rs b/core/tests/behavior/async_read.rs new file mode 100644 index 00000000000..e372fe62d8a --- /dev/null +++ b/core/tests/behavior/async_read.rs @@ -0,0 +1,794 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::str::FromStr; +use std::time::Duration; + +use futures::AsyncReadExt; +use futures::AsyncSeekExt; +use http::StatusCode; +use log::warn; +use reqwest::Url; +use sha2::Digest; +use sha2::Sha256; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.read && cap.write { + tests.extend(async_trials!( + op, + test_read_full, + test_read_range, + test_read_large_range, + test_reader_range, + test_reader_range_with_buffer, + test_reader_from, + test_reader_from_with_buffer, + test_reader_tail, + test_reader_tail_with_buffer, + test_read_not_exist, + test_read_with_if_match, + test_read_with_if_none_match, + test_read_with_dir_path, + test_read_with_special_chars, + test_read_with_override_cache_control, + test_read_with_override_content_disposition, + test_read_with_override_content_type, + test_read_with_invalid_seek + )) + } + + if cap.read && !cap.write { + tests.extend(async_trials!( + op, + test_read_only_read_full, + test_read_only_read_full_with_special_chars, + test_read_only_read_with_range, + test_read_only_reader_with_range, + test_read_only_reader_from, + test_read_only_reader_tail, + test_read_only_read_not_exist, + test_read_only_read_with_dir_path, + test_read_only_read_with_if_match, + test_read_only_read_with_if_none_match + )) + } +} + +/// Read full content should match. +pub async fn test_read_full(op: Operator) -> anyhow::Result<()> { + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let bs = op.read(&path).await?; + assert_eq!(size, bs.len(), "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content)), + "read content" + ); + + Ok(()) +} + +/// Read range content should match. +pub async fn test_read_range(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let bs = op.read_with(&path).range(offset..offset + length).await?; + assert_eq!(bs.len() as u64, length, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!( + "{:x}", + Sha256::digest(&content[offset as usize..(offset + length) as usize]) + ), + "read content" + ); + + Ok(()) +} + +/// Read large range content should match. +pub async fn test_read_large_range(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, _) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let bs = op.read_with(&path).range(offset..u32::MAX as u64).await?; + assert_eq!( + bs.len() as u64, + size as u64 - offset, + "read size with large range" + ); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[offset as usize..])), + "read content with large range" + ); + + Ok(()) +} + +/// Read range content should match. +pub async fn test_reader_range(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = op.reader_with(&path).range(offset..offset + length).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!( + "{:x}", + Sha256::digest(&content[offset as usize..(offset + length) as usize]) + ), + "read content" + ); + + Ok(()) +} + +/// Read range content should match. +pub async fn test_reader_range_with_buffer(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = op + .reader_with(&path) + .range(offset..offset + length) + .buffer(4096) + .await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!( + "{:x}", + Sha256::digest(&content[offset as usize..(offset + length) as usize]) + ), + "read content" + ); + + Ok(()) +} + +/// Read range from should match. +pub async fn test_reader_from(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, _) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = op.reader_with(&path).range(offset..).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), size - offset as usize, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[offset as usize..])), + "read content" + ); + + Ok(()) +} + +/// Read range from should match. +pub async fn test_reader_from_with_buffer(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (offset, _) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = op.reader_with(&path).range(offset..).buffer(4096).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), size - offset as usize, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[offset as usize..])), + "read content" + ); + + Ok(()) +} + +/// Read range tail should match. +pub async fn test_reader_tail(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (_, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = match op.reader_with(&path).range(..length).await { + Ok(r) => r, + // Not all services support range with tail range, let's tolerate this. + Err(err) if err.kind() == ErrorKind::Unsupported => { + warn!("service doesn't support range with tail"); + return Ok(()); + } + Err(err) => return Err(err.into()), + }; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), length as usize, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[size - length as usize..])), + "read content" + ); + + Ok(()) +} + +/// Read range tail should match. +pub async fn test_reader_tail_with_buffer(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + let (_, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = match op.reader_with(&path).range(..length).buffer(4096).await { + Ok(r) => r, + // Not all services support range with tail range, let's tolerate this. + Err(err) if err.kind() == ErrorKind::Unsupported => { + warn!("service doesn't support range with tail"); + return Ok(()); + } + Err(err) => return Err(err.into()), + }; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), length as usize, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[size - length as usize..])), + "read content" + ); + + Ok(()) +} + +/// Read not exist file should return NotFound +pub async fn test_read_not_exist(op: Operator) -> anyhow::Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let bs = op.read(&path).await; + assert!(bs.is_err()); + assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +/// Read with if_match should match, else get a ConditionNotMatch error. +pub async fn test_read_with_if_match(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_if_match { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&path).await?; + + let res = op.read_with(&path).if_match("\"invalid_etag\"").await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let bs = op + .read_with(&path) + .if_match(meta.etag().expect("etag must exist")) + .await + .expect("read must succeed"); + assert_eq!(bs, content); + + Ok(()) +} + +/// Read with if_none_match should match, else get a ConditionNotMatch error. +pub async fn test_read_with_if_none_match(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_if_none_match { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&path).await?; + + let res = op + .read_with(&path) + .if_none_match(meta.etag().expect("etag must exist")) + .await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let bs = op + .read_with(&path) + .if_none_match("\"invalid_etag\"") + .await + .expect("read must succeed"); + assert_eq!(bs, content); + + Ok(()) +} + +/// Read with dir path should return an error. +pub async fn test_read_with_dir_path(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = TEST_FIXTURE.new_dir_path(); + + op.create_dir(&path).await.expect("write must succeed"); + + let result = op.read(&path).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); + + Ok(()) +} + +/// Read file with special chars should succeed. +pub async fn test_read_with_special_chars(op: Operator) -> anyhow::Result<()> { + // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. + if op.info().scheme() == opendal::Scheme::Supabase { + warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); + return Ok(()); + } + // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. + if op.info().scheme() == opendal::Scheme::Atomicserver { + warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); + return Ok(()); + } + + let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); + let (path, content, size) = TEST_FIXTURE.new_file_with_path(op.clone(), &path); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let bs = op.read(&path).await?; + assert_eq!(size, bs.len(), "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content)), + "read content" + ); + + Ok(()) +} + +/// Read file with override-cache-control should succeed. +pub async fn test_read_with_override_cache_control(op: Operator) -> anyhow::Result<()> { + if !(op.info().full_capability().read_with_override_cache_control + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_cache_control = "no-cache, no-store, must-revalidate"; + let signed_req = op + .presign_read_with(&path, Duration::from_secs(60)) + .override_cache_control(target_cache_control) + .await + .expect("sign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get("cache-control") + .expect("cache-control header must exist") + .to_str() + .expect("cache-control header must be string"), + target_cache_control + ); + + Ok(()) +} + +/// Read file with override_content_disposition should succeed. +pub async fn test_read_with_override_content_disposition(op: Operator) -> anyhow::Result<()> { + if !(op + .info() + .full_capability() + .read_with_override_content_disposition + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_content_disposition = "attachment; filename=foo.txt"; + + let signed_req = op + .presign_read_with(&path, Duration::from_secs(60)) + .override_content_disposition(target_content_disposition) + .await + .expect("presign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get(http::header::CONTENT_DISPOSITION) + .expect("content-disposition header must exist") + .to_str() + .expect("content-disposition header must be string"), + target_content_disposition + ); + assert_eq!(resp.bytes().await?, content); + + Ok(()) +} + +/// Read file with override_content_type should succeed. +pub async fn test_read_with_override_content_type(op: Operator) -> anyhow::Result<()> { + if !(op.info().full_capability().read_with_override_content_type + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_content_type = "application/opendal"; + + let signed_req = op + .presign_read_with(&path, Duration::from_secs(60)) + .override_content_type(target_content_type) + .await + .expect("presign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get(http::header::CONTENT_TYPE) + .expect("content-type header must exist") + .to_str() + .expect("content-type header must be string"), + target_content_type + ); + assert_eq!(resp.bytes().await?, content); + + Ok(()) +} + +/// seeking a negative position should return a InvalidInput error +pub async fn test_read_with_invalid_seek(op: Operator) -> anyhow::Result<()> { + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let mut r = op.reader(&path).await?; + let res = r.seek(std::io::SeekFrom::Current(-1024)).await; + + assert!(res.is_err()); + + assert_eq!( + res.unwrap_err().kind(), + std::io::ErrorKind::InvalidInput, + "seeking a negative position should return a InvalidInput error" + ); + + Ok(()) +} + +/// Read full content should match. +pub async fn test_read_only_read_full(op: Operator) -> anyhow::Result<()> { + let bs = op.read("normal_file.txt").await?; + assert_eq!(bs.len(), 30482, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", + "read content" + ); + + Ok(()) +} + +/// Read full content should match. +pub async fn test_read_only_read_full_with_special_chars(op: Operator) -> anyhow::Result<()> { + let bs = op.read("special_file !@#$%^&()_+-=;',.txt").await?; + assert_eq!(bs.len(), 30482, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", + "read content" + ); + + Ok(()) +} + +/// Read full content should match. +pub async fn test_read_only_read_with_range(op: Operator) -> anyhow::Result<()> { + let bs = op.read_with("normal_file.txt").range(1024..2048).await?; + assert_eq!(bs.len(), 1024, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", + "read content" + ); + + Ok(()) +} + +/// Read range should match. +pub async fn test_read_only_reader_with_range(op: Operator) -> anyhow::Result<()> { + let mut r = op.reader_with("normal_file.txt").range(1024..2048).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), 1024, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", + "read content" + ); + + Ok(()) +} + +/// Read from should match. +pub async fn test_read_only_reader_from(op: Operator) -> anyhow::Result<()> { + let mut r = op.reader_with("normal_file.txt").range(29458..).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), 1024, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "cc9312c869238ea9410b6716e0fc3f48056f2bfb2fe06ccf5f96f2c3bf39e71b", + "read content" + ); + + Ok(()) +} + +/// Read tail should match. +pub async fn test_read_only_reader_tail(op: Operator) -> anyhow::Result<()> { + let mut r = op.reader_with("normal_file.txt").range(..1024).await?; + + let mut bs = Vec::new(); + r.read_to_end(&mut bs).await?; + + assert_eq!(bs.len(), 1024, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "cc9312c869238ea9410b6716e0fc3f48056f2bfb2fe06ccf5f96f2c3bf39e71b", + "read content" + ); + + Ok(()) +} + +/// Read not exist file should return NotFound +pub async fn test_read_only_read_not_exist(op: Operator) -> anyhow::Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let bs = op.read(&path).await; + assert!(bs.is_err()); + assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +/// Read with dir path should return an error. +pub async fn test_read_only_read_with_dir_path(op: Operator) -> anyhow::Result<()> { + let path = format!("{}/", uuid::Uuid::new_v4()); + + let result = op.read(&path).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); + + Ok(()) +} + +/// Read with if_match should match, else get a ConditionNotMatch error. +pub async fn test_read_only_read_with_if_match(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_if_match { + return Ok(()); + } + + let path = "normal_file.txt"; + + let meta = op.stat(path).await?; + + let res = op.read_with(path).if_match("invalid_etag").await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let bs = op + .read_with(path) + .if_match(meta.etag().expect("etag must exist")) + .await + .expect("read must succeed"); + assert_eq!(bs.len(), 30482, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", + "read content" + ); + + Ok(()) +} + +/// Read with if_none_match should match, else get a ConditionNotMatch error. +pub async fn test_read_only_read_with_if_none_match(op: Operator) -> anyhow::Result<()> { + if !op.info().full_capability().read_with_if_none_match { + return Ok(()); + } + + let path = "normal_file.txt"; + + let meta = op.stat(path).await?; + + let res = op + .read_with(path) + .if_none_match(meta.etag().expect("etag must exist")) + .await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let bs = op + .read_with(path) + .if_none_match("invalid_etag") + .await + .expect("read must succeed"); + assert_eq!(bs.len(), 30482, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", + "read content" + ); + + Ok(()) +} diff --git a/core/tests/behavior/rename.rs b/core/tests/behavior/async_rename.rs similarity index 93% rename from core/tests/behavior/rename.rs rename to core/tests/behavior/async_rename.rs index f155653b2e8..d3cd7f921f3 100644 --- a/core/tests/behavior/rename.rs +++ b/core/tests/behavior/async_rename.rs @@ -21,23 +21,21 @@ use sha2::Sha256; use crate::*; -pub fn behavior_rename_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.rename) { - return vec![]; + if cap.read && cap.write && cap.rename { + tests.extend(async_trials!( + op, + test_rename_file, + test_rename_non_existing_source, + test_rename_source_dir, + test_rename_target_dir, + test_rename_self, + test_rename_nested, + test_rename_overwrite + )) } - - async_trials!( - op, - test_rename_file, - test_rename_non_existing_source, - test_rename_source_dir, - test_rename_target_dir, - test_rename_self, - test_rename_nested, - test_rename_overwrite - ) } /// Rename a file and test with stat. diff --git a/core/tests/behavior/async_stat.rs b/core/tests/behavior/async_stat.rs new file mode 100644 index 00000000000..9dffb31da9b --- /dev/null +++ b/core/tests/behavior/async_stat.rs @@ -0,0 +1,501 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::str::FromStr; +use std::time::Duration; + +use anyhow::Result; +use http::StatusCode; +use log::warn; +use reqwest::Url; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.stat && cap.write { + tests.extend(async_trials!( + op, + test_stat_file, + test_stat_dir, + test_stat_nested_parent_dir, + test_stat_with_special_chars, + test_stat_not_cleaned_path, + test_stat_not_exist, + test_stat_with_if_match, + test_stat_with_if_none_match, + test_stat_with_override_cache_control, + test_stat_with_override_content_disposition, + test_stat_with_override_content_type, + test_stat_root + )) + } + + if cap.stat && !cap.write { + tests.extend(async_trials!( + op, + test_read_only_stat_file_and_dir, + test_read_only_stat_special_chars, + test_read_only_stat_not_cleaned_path, + test_read_only_stat_not_exist, + test_read_only_stat_with_if_match, + test_read_only_stat_with_if_none_match, + test_read_only_stat_root + )) + } +} + +/// Stat existing file should return metadata +pub async fn test_stat_file(op: Operator) -> Result<()> { + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content).await.expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + // Stat a file with trailing slash should return `NotFound`. + if op.info().full_capability().create_dir { + let result = op.stat(&format!("{path}/")).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), ErrorKind::NotFound); + } + + Ok(()) +} + +/// Stat existing file should return metadata +pub async fn test_stat_dir(op: Operator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = TEST_FIXTURE.new_dir_path(); + + op.create_dir(&path).await.expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + // Stat a dir without trailing slash could have two behavior. + let result = op.stat(path.trim_end_matches('/')).await; + match result { + Ok(meta) => assert_eq!(meta.mode(), EntryMode::DIR), + Err(err) => assert_eq!(err.kind(), ErrorKind::NotFound), + } + + Ok(()) +} + +/// Stat the parent dir of existing dir should return metadata +pub async fn test_stat_nested_parent_dir(op: Operator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let parent = format!("{}", uuid::Uuid::new_v4()); + let file = format!("{}", uuid::Uuid::new_v4()); + let (path, content, _) = + TEST_FIXTURE.new_file_with_path(op.clone(), &format!("{parent}/{file}")); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&format!("{parent}/")).await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat existing file with special chars should return metadata +pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> { + // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. + if op.info().scheme() == opendal::Scheme::Supabase { + warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); + return Ok(()); + } + // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. + if op.info().scheme() == opendal::Scheme::Atomicserver { + warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); + return Ok(()); + } + + let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); + let (path, content, size) = TEST_FIXTURE.new_file_with_path(op.clone(), &path); + + op.write(&path, content).await.expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Stat not cleaned path should also succeed. +pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> { + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content).await.expect("write must succeed"); + + let meta = op.stat(&format!("//{}", &path)).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Stat not exist file should return NotFound +pub async fn test_stat_not_exist(op: Operator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + // Stat not exist file should returns NotFound. + let meta = op.stat(&path).await; + assert!(meta.is_err()); + assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + + // Stat not exist dir should also returns NotFound. + if op.info().full_capability().create_dir { + let meta = op.stat(&format!("{path}/")).await; + assert!(meta.is_err()); + assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + } + + Ok(()) +} + +/// Stat with if_match should succeed, else get a ConditionNotMatch error. +pub async fn test_stat_with_if_match(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_if_match { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + let res = op.stat_with(&path).if_match("\"invalid_etag\"").await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let result = op + .stat_with(&path) + .if_match(meta.etag().expect("etag must exist")) + .await; + assert!(result.is_ok()); + + Ok(()) +} + +/// Stat with if_none_match should succeed, else get a ConditionNotMatch. +pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_if_none_match { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + let res = op + .stat_with(&path) + .if_none_match(meta.etag().expect("etag must exist")) + .await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let res = op + .stat_with(&path) + .if_none_match("\"invalid_etag\"") + .await?; + assert_eq!(res.mode(), meta.mode()); + assert_eq!(res.content_length(), meta.content_length()); + + Ok(()) +} + +/// Stat file with override-cache-control should succeed. +pub async fn test_stat_with_override_cache_control(op: Operator) -> Result<()> { + if !(op.info().full_capability().stat_with_override_cache_control + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_cache_control = "no-cache, no-store, must-revalidate"; + let signed_req = op + .presign_stat_with(&path, Duration::from_secs(60)) + .override_cache_control(target_cache_control) + .await + .expect("sign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get("cache-control") + .expect("cache-control header must exist") + .to_str() + .expect("cache-control header must be string"), + target_cache_control + ); + + Ok(()) +} + +/// Stat file with override_content_disposition should succeed. +pub async fn test_stat_with_override_content_disposition(op: Operator) -> Result<()> { + if !(op + .info() + .full_capability() + .stat_with_override_content_disposition + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_content_disposition = "attachment; filename=foo.txt"; + + let signed_req = op + .presign_stat_with(&path, Duration::from_secs(60)) + .override_content_disposition(target_content_disposition) + .await + .expect("presign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get(http::header::CONTENT_DISPOSITION) + .expect("content-disposition header must exist") + .to_str() + .expect("content-disposition header must be string"), + target_content_disposition + ); + + Ok(()) +} + +/// Stat file with override_content_type should succeed. +pub async fn test_stat_with_override_content_type(op: Operator) -> Result<()> { + if !(op.info().full_capability().stat_with_override_content_type + && op.info().full_capability().presign) + { + return Ok(()); + } + + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content.clone()) + .await + .expect("write must succeed"); + + let target_content_type = "application/opendal"; + + let signed_req = op + .presign_stat_with(&path, Duration::from_secs(60)) + .override_content_type(target_content_type) + .await + .expect("presign must succeed"); + + let client = reqwest::Client::new(); + let mut req = client.request( + signed_req.method().clone(), + Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), + ); + for (k, v) in signed_req.header() { + req = req.header(k, v); + } + + let resp = req.send().await.expect("send must succeed"); + + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + resp.headers() + .get(http::header::CONTENT_TYPE) + .expect("content-type header must exist") + .to_str() + .expect("content-type header must be string"), + target_content_type + ); + + Ok(()) +} + +/// Root should be able to stat and returns DIR. +pub async fn test_stat_root(op: Operator) -> Result<()> { + let meta = op.stat("").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + let meta = op.stat("/").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat normal file and dir should return metadata +pub async fn test_read_only_stat_file_and_dir(op: Operator) -> Result<()> { + let meta = op.stat("normal_file.txt").await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let meta = op.stat("normal_dir/").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat special file and dir should return metadata +pub async fn test_read_only_stat_special_chars(op: Operator) -> Result<()> { + let meta = op.stat("special_file !@#$%^&()_+-=;',.txt").await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let meta = op.stat("special_dir !@#$%^&()_+-=;',/").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat not cleaned path should also succeed. +pub async fn test_read_only_stat_not_cleaned_path(op: Operator) -> Result<()> { + let meta = op.stat("//normal_file.txt").await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + Ok(()) +} + +/// Stat not exist file should return NotFound +pub async fn test_read_only_stat_not_exist(op: Operator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let meta = op.stat(&path).await; + assert!(meta.is_err()); + assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +/// Stat with if_match should succeed, else get a ConditionNotMatch error. +pub async fn test_read_only_stat_with_if_match(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_if_match { + return Ok(()); + } + + let path = "normal_file.txt"; + + let meta = op.stat(path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let res = op.stat_with(path).if_match("invalid_etag").await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let result = op + .stat_with(path) + .if_match(meta.etag().expect("etag must exist")) + .await; + assert!(result.is_ok()); + + Ok(()) +} + +/// Stat with if_none_match should succeed, else get a ConditionNotMatch. +pub async fn test_read_only_stat_with_if_none_match(op: Operator) -> Result<()> { + if !op.info().full_capability().stat_with_if_none_match { + return Ok(()); + } + + let path = "normal_file.txt"; + + let meta = op.stat(path).await?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let res = op + .stat_with(path) + .if_none_match(meta.etag().expect("etag must exist")) + .await; + assert!(res.is_err()); + assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); + + let res = op.stat_with(path).if_none_match("invalid_etag").await?; + assert_eq!(res.mode(), meta.mode()); + assert_eq!(res.content_length(), meta.content_length()); + + Ok(()) +} + +/// Root should be able to stat and returns DIR. +pub async fn test_read_only_stat_root(op: Operator) -> Result<()> { + let meta = op.stat("").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + let meta = op.stat("/").await?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} diff --git a/core/tests/behavior/async_write.rs b/core/tests/behavior/async_write.rs new file mode 100644 index 00000000000..7eb99df2718 --- /dev/null +++ b/core/tests/behavior/async_write.rs @@ -0,0 +1,421 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use bytes::Buf; +use bytes::Bytes; +use futures::io::BufReader; +use futures::io::Cursor; +use futures::stream; +use futures::StreamExt; +use log::warn; +use sha2::Digest; +use sha2::Sha256; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.write && cap.stat { + tests.extend(async_trials!( + op, + test_write_only, + test_write_with_empty_content, + test_write_with_dir_path, + test_write_with_special_chars, + test_write_with_cache_control, + test_write_with_content_type, + test_write_with_content_disposition, + test_writer_write, + test_writer_sink, + test_writer_copy, + test_writer_abort, + test_writer_futures_copy + )) + } + + if cap.write && cap.write_can_append && cap.stat { + tests.extend(async_trials!( + op, + test_write_with_append, + test_writer_with_append + )) + } +} + +/// Write a single file and test with stat. +pub async fn test_write_only(op: Operator) -> Result<()> { + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + op.write(&path, content).await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Write a file with empty content. +pub async fn test_write_with_empty_content(op: Operator) -> Result<()> { + if !op.info().full_capability().write_can_empty { + return Ok(()); + } + + let path = TEST_FIXTURE.new_file_path(); + + op.write(&path, vec![]).await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), 0); + Ok(()) +} + +/// Write file with dir path should return an error +pub async fn test_write_with_dir_path(op: Operator) -> Result<()> { + let path = TEST_FIXTURE.new_dir_path(); + + let result = op.write(&path, vec![1]).await; + assert!(result.is_err()); + assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); + + Ok(()) +} + +/// Write a single file with special chars should succeed. +pub async fn test_write_with_special_chars(op: Operator) -> Result<()> { + // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. + if op.info().scheme() == opendal::Scheme::Supabase { + warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); + return Ok(()); + } + // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. + if op.info().scheme() == opendal::Scheme::Atomicserver { + warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); + return Ok(()); + } + + let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); + let (path, content, size) = TEST_FIXTURE.new_file_with_path(op.clone(), &path); + + op.write(&path, content).await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Write a single file with cache control should succeed. +pub async fn test_write_with_cache_control(op: Operator) -> Result<()> { + if !op.info().full_capability().write_with_cache_control { + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + let (content, _) = gen_bytes(op.info().full_capability()); + + let target_cache_control = "no-cache, no-store, max-age=300"; + op.write_with(&path, content) + .cache_control(target_cache_control) + .await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!( + meta.cache_control().expect("cache control must exist"), + target_cache_control + ); + + op.delete(&path).await.expect("delete must succeed"); + + Ok(()) +} + +/// Write a single file with content type should succeed. +pub async fn test_write_with_content_type(op: Operator) -> Result<()> { + if !op.info().full_capability().write_with_content_type { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + let target_content_type = "application/json"; + op.write_with(&path, content) + .content_type(target_content_type) + .await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!( + meta.content_type().expect("content type must exist"), + target_content_type + ); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Write a single file with content disposition should succeed. +pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> { + if !op.info().full_capability().write_with_content_disposition { + return Ok(()); + } + + let (path, content, size) = TEST_FIXTURE.new_file(op.clone()); + + let target_content_disposition = "attachment; filename=\"filename.jpg\""; + op.write_with(&path, content) + .content_disposition(target_content_disposition) + .await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!( + meta.content_disposition().expect("content type must exist"), + target_content_disposition + ); + assert_eq!(meta.content_length(), size as u64); + + Ok(()) +} + +/// Delete existing file should succeed. +pub async fn test_writer_abort(op: Operator) -> Result<()> { + let (path, content, _) = TEST_FIXTURE.new_file(op.clone()); + + let mut writer = match op.writer(&path).await { + Ok(writer) => writer, + Err(e) => { + assert_eq!(e.kind(), ErrorKind::Unsupported); + return Ok(()); + } + }; + + if let Err(e) = writer.write(content).await { + assert_eq!(e.kind(), ErrorKind::Unsupported); + return Ok(()); + } + + if let Err(e) = writer.abort().await { + assert_eq!(e.kind(), ErrorKind::Unsupported); + return Ok(()); + } + + // Aborted writer should not write actual file. + assert!(!op.is_exist(&path).await?); + Ok(()) +} + +/// Append data into writer +pub async fn test_writer_write(op: Operator) -> Result<()> { + if !(op.info().full_capability().write_can_multi) { + return Ok(()); + } + + let path = TEST_FIXTURE.new_file_path(); + let size = 5 * 1024 * 1024; // write file with 5 MiB + let content_a = gen_fixed_bytes(size); + let content_b = gen_fixed_bytes(size); + + let mut w = op.writer(&path).await?; + w.write(content_a.clone()).await?; + w.write(content_b.clone()).await?; + w.close().await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), (size * 2) as u64); + + let bs = op.read(&path).await?; + assert_eq!(bs.len(), size * 2, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content_a)), + "read content a" + ); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[size..])), + format!("{:x}", Sha256::digest(content_b)), + "read content b" + ); + + Ok(()) +} + +/// Streaming data into writer +pub async fn test_writer_sink(op: Operator) -> Result<()> { + let cap = op.info().full_capability(); + if !(cap.write && cap.write_can_multi) { + return Ok(()); + } + + let path = TEST_FIXTURE.new_file_path(); + let size = 5 * 1024 * 1024; // write file with 5 MiB + let content_a = gen_fixed_bytes(size); + let content_b = gen_fixed_bytes(size); + let stream = stream::iter(vec![content_a.clone(), content_b.clone()]).map(Ok); + + let mut w = op.writer_with(&path).buffer(5 * 1024 * 1024).await?; + w.sink(stream).await?; + w.close().await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), (size * 2) as u64); + + let bs = op.read(&path).await?; + assert_eq!(bs.len(), size * 2, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content_a)), + "read content a" + ); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[size..])), + format!("{:x}", Sha256::digest(content_b)), + "read content b" + ); + + Ok(()) +} + +/// Reading data into writer +pub async fn test_writer_copy(op: Operator) -> Result<()> { + let cap = op.info().full_capability(); + if !(cap.write && cap.write_can_multi) { + return Ok(()); + } + + let path = TEST_FIXTURE.new_file_path(); + let size = 5 * 1024 * 1024; // write file with 5 MiB + let content_a = gen_fixed_bytes(size); + let content_b = gen_fixed_bytes(size); + + let mut w = op.writer_with(&path).buffer(5 * 1024 * 1024).await?; + + let mut content = Bytes::from([content_a.clone(), content_b.clone()].concat()); + while !content.is_empty() { + let reader = Cursor::new(content.clone()); + let n = w.copy(reader).await?; + content.advance(n as usize); + } + w.close().await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), (size * 2) as u64); + + let bs = op.read(&path).await?; + assert_eq!(bs.len(), size * 2, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content_a)), + "read content a" + ); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[size..])), + format!("{:x}", Sha256::digest(content_b)), + "read content b" + ); + + Ok(()) +} + +/// Copy data from reader to writer +pub async fn test_writer_futures_copy(op: Operator) -> Result<()> { + if !(op.info().full_capability().write_can_multi) { + return Ok(()); + } + + let path = TEST_FIXTURE.new_file_path(); + let (content, size): (Vec, usize) = + gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); + + let mut w = op.writer_with(&path).buffer(8 * 1024 * 1024).await?; + + // Wrap a buf reader here to make sure content is read in 1MiB chunks. + let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); + futures::io::copy_buf(&mut cursor, &mut w).await?; + w.close().await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), size as u64); + + let bs = op.read(&path).await?; + assert_eq!(bs.len(), size, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content)), + "read content" + ); + + Ok(()) +} + +/// Test append to a file must success. +pub async fn test_write_with_append(op: Operator) -> Result<()> { + let path = TEST_FIXTURE.new_file_path(); + let (content_one, size_one) = gen_bytes(op.info().full_capability()); + let (content_two, size_two) = gen_bytes(op.info().full_capability()); + + op.write_with(&path, content_one.clone()) + .append(true) + .await + .expect("append file first time must success"); + + let meta = op.stat(&path).await?; + assert_eq!(meta.content_length(), size_one as u64); + + op.write_with(&path, content_two.clone()) + .append(true) + .await + .expect("append to an existing file must success"); + + let bs = op.read(&path).await.expect("read file must success"); + + assert_eq!(bs.len(), size_one + size_two); + assert_eq!(bs[..size_one], content_one); + assert_eq!(bs[size_one..], content_two); + + Ok(()) +} + +/// Copy data from reader to writer +pub async fn test_writer_with_append(op: Operator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + let (content, size): (Vec, usize) = + gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); + + let mut a = op.writer_with(&path).append(true).await?; + + // Wrap a buf reader here to make sure content is read in 1MiB chunks. + let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); + futures::io::copy_buf(&mut cursor, &mut a).await?; + a.close().await?; + + let meta = op.stat(&path).await.expect("stat must succeed"); + assert_eq!(meta.content_length(), size as u64); + + let bs = op.read(&path).await?; + assert_eq!(bs.len(), size, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content)), + "read content" + ); + + op.delete(&path).await.expect("delete must succeed"); + Ok(()) +} diff --git a/core/tests/behavior/blocking_append.rs b/core/tests/behavior/blocking_append.rs deleted file mode 100644 index 8161f825a39..00000000000 --- a/core/tests/behavior/blocking_append.rs +++ /dev/null @@ -1,220 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::io::BufReader; -use std::io::Cursor; -use std::vec; - -use anyhow::Result; -use sha2::Digest; -use sha2::Sha256; - -use crate::*; - -pub fn behavior_blocking_append_tests(op: &Operator) -> Vec { - let cap = op.info().full_capability(); - - if !(cap.read && cap.write && cap.blocking && cap.write_can_append) { - return vec![]; - } - - blocking_trials!( - op, - test_blocking_append_create_append, - test_blocking_append_with_dir_path, - test_blocking_append_with_cache_control, - test_blocking_append_with_content_type, - test_blocking_append_with_content_disposition, - test_blocking_appender_std_copy, - test_blocking_fuzz_appender - ) -} - -/// Test append to a file must success. -pub fn test_blocking_append_create_append(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content_one, size_one) = gen_bytes(op.info().full_capability()); - let (content_two, size_two) = gen_bytes(op.info().full_capability()); - - op.write_with(&path, content_one.clone()) - .append(true) - .call() - .expect("append file first time must success"); - - op.write_with(&path, content_two.clone()) - .append(true) - .call() - .expect("append to an existing file must success"); - - let bs = op.read(&path).expect("read file must success"); - - assert_eq!(bs.len(), size_one + size_two); - assert_eq!(bs[..size_one], content_one); - assert_eq!(bs[size_one..], content_two); - - op.delete(&path).expect("delete file must success"); - - Ok(()) -} - -/// Test append to a directory path must fail. -pub fn test_blocking_append_with_dir_path(op: BlockingOperator) -> Result<()> { - let path = format!("{}/", uuid::Uuid::new_v4()); - let (content, _) = gen_bytes(op.info().full_capability()); - - let res = op.write_with(&path, content).append(true).call(); - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::IsADirectory); - - Ok(()) -} - -/// Test append with cache control must success. -pub fn test_blocking_append_with_cache_control(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().write_with_cache_control { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - let target_cache_control = "no-cache, no-store, max-age=300"; - op.write_with(&path, content) - .append(true) - .cache_control(target_cache_control) - .call()?; - - let meta = op.stat(&path).expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.cache_control().expect("cache control must exist"), - target_cache_control - ); - - op.delete(&path).expect("delete must succeed"); - - Ok(()) -} - -/// Test append with content type must success. -pub fn test_blocking_append_with_content_type(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().write_with_content_type { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_type = "application/json"; - op.write_with(&path, content) - .append(true) - .content_type(target_content_type) - .call()?; - - let meta = op.stat(&path).expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_type().expect("content type must exist"), - target_content_type - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).expect("delete must succeed"); - - Ok(()) -} - -/// Write a single file with content disposition should succeed. -pub fn test_blocking_append_with_content_disposition(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().write_with_content_disposition { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_disposition = "attachment; filename=\"filename.jpg\""; - op.write_with(&path, content) - .append(true) - .content_disposition(target_content_disposition) - .call()?; - - let meta = op.stat(&path).expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_disposition().expect("content type must exist"), - target_content_disposition - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).expect("delete must succeed"); - - Ok(()) -} - -/// Copy data from reader to writer -pub fn test_blocking_appender_std_copy(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, size): (Vec, usize) = - gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); - - let mut a = op.writer_with(&path).append(true).call()?; - - // Wrap a buf reader here to make sure content is read in 1MiB chunks. - let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); - std::io::copy(&mut cursor, &mut a)?; - a.close()?; - - let meta = op.stat(&path).expect("stat must succeed"); - assert_eq!(meta.content_length(), size as u64); - - let bs = op.read(&path)?; - assert_eq!(bs.len(), size, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content)), - "read content" - ); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// Test for fuzzing appender. -pub fn test_blocking_fuzz_appender(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let mut fuzzer = ObjectWriterFuzzer::new(&path, None); - - let mut a = op.writer_with(&path).append(true).call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectWriterAction::Write(bs) => { - a.write(bs)?; - } - } - } - a.close()?; - - let content = op.read(&path)?; - fuzzer.check(&content); - - op.delete(&path).expect("delete file must success"); - - Ok(()) -} diff --git a/core/tests/behavior/blocking_copy.rs b/core/tests/behavior/blocking_copy.rs index 421721cf3fa..c8a8a811bd3 100644 --- a/core/tests/behavior/blocking_copy.rs +++ b/core/tests/behavior/blocking_copy.rs @@ -21,23 +21,21 @@ use sha2::Sha256; use crate::*; -pub fn behavior_blocking_copy_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.copy && cap.blocking) { - return vec![]; + if cap.read && cap.write && cap.copy && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_copy_file, + test_blocking_copy_non_existing_source, + test_blocking_copy_source_dir, + test_blocking_copy_target_dir, + test_blocking_copy_self, + test_blocking_copy_nested, + test_blocking_copy_overwrite + )) } - - blocking_trials!( - op, - test_blocking_copy_file, - test_blocking_copy_non_existing_source, - test_blocking_copy_source_dir, - test_blocking_copy_target_dir, - test_blocking_copy_self, - test_blocking_copy_nested, - test_blocking_copy_overwrite - ) } /// Copy a file and test with stat. diff --git a/core/tests/behavior/blocking_create_dir.rs b/core/tests/behavior/blocking_create_dir.rs new file mode 100644 index 00000000000..52f8c4d0738 --- /dev/null +++ b/core/tests/behavior/blocking_create_dir.rs @@ -0,0 +1,68 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.stat && cap.write && cap.create_dir && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_create_dir, + test_blocking_create_dir_existing + )) + } +} + +/// Create dir with dir path should succeed. +pub fn test_blocking_create_dir(op: BlockingOperator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = format!("{}/", uuid::Uuid::new_v4()); + + op.create_dir(&path)?; + + let meta = op.stat(&path)?; + assert_eq!(meta.mode(), EntryMode::DIR); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Create dir on existing dir should succeed. +pub fn test_blocking_create_dir_existing(op: BlockingOperator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = format!("{}/", uuid::Uuid::new_v4()); + + op.create_dir(&path)?; + + op.create_dir(&path)?; + + let meta = op.stat(&path)?; + assert_eq!(meta.mode(), EntryMode::DIR); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} diff --git a/core/tests/behavior/blocking_delete.rs b/core/tests/behavior/blocking_delete.rs new file mode 100644 index 00000000000..111beff2795 --- /dev/null +++ b/core/tests/behavior/blocking_delete.rs @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use log::debug; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.stat && cap.write && cap.delete && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_delete_file, + test_blocking_remove_one_file + )) + } +} + +// Delete existing file should succeed. +pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, _) = gen_bytes(op.info().full_capability()); + + op.write(&path, content).expect("write must succeed"); + + op.delete(&path)?; + + // Stat it again to check. + assert!(!op.is_exist(&path)?); + + Ok(()) +} + +/// Remove one file +pub fn test_blocking_remove_one_file(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + let (content, _) = gen_bytes(op.info().full_capability()); + + op.write(&path, content).expect("write must succeed"); + + op.remove(vec![path.clone()])?; + + // Stat it again to check. + assert!(!op.is_exist(&path)?); + + Ok(()) +} diff --git a/core/tests/behavior/blocking_list.rs b/core/tests/behavior/blocking_list.rs index 174f9907b38..9208929287a 100644 --- a/core/tests/behavior/blocking_list.rs +++ b/core/tests/behavior/blocking_list.rs @@ -23,22 +23,20 @@ use log::debug; use crate::*; -pub fn behavior_blocking_list_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.copy && cap.blocking && cap.list) { - return vec![]; + if cap.read && cap.write && cap.copy && cap.blocking && cap.list { + tests.extend(blocking_trials!( + op, + test_blocking_list_dir, + test_blocking_list_dir_with_metakey, + test_blocking_list_dir_with_metakey_complete, + test_blocking_list_non_exist_dir, + test_blocking_scan, + test_blocking_remove_all + )) } - - blocking_trials!( - op, - test_blocking_list_dir, - test_blocking_list_dir_with_metakey, - test_blocking_list_dir_with_metakey_complete, - test_blocking_list_non_exist_dir, - test_blocking_scan, - test_blocking_remove_all - ) } /// List dir should return newly created file. diff --git a/core/tests/behavior/blocking_read.rs b/core/tests/behavior/blocking_read.rs new file mode 100644 index 00000000000..4b5de2d7e36 --- /dev/null +++ b/core/tests/behavior/blocking_read.rs @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use log::debug; +use sha2::Digest; +use sha2::Sha256; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.read && cap.write && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_read_full, + test_blocking_read_range, + test_blocking_read_large_range, + test_blocking_read_not_exist + )) + } + + if cap.read && !cap.write && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_read_only_stat_file_and_dir, + test_blocking_read_only_stat_special_chars, + test_blocking_read_only_stat_not_exist, + test_blocking_read_only_read_full, + test_blocking_read_only_read_with_range, + test_blocking_read_only_read_not_exist + )) + } +} + +/// Read full content should match. +pub fn test_blocking_read_full(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(op.info().full_capability()); + + op.write(&path, content.clone()) + .expect("write must succeed"); + + let bs = op.read(&path)?; + assert_eq!(size, bs.len(), "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content)), + "read content" + ); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Read range content should match. +pub fn test_blocking_read_range(op: BlockingOperator) -> Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(op.info().full_capability()); + let (offset, length) = gen_offset_length(size); + + op.write(&path, content.clone()) + .expect("write must succeed"); + + let bs = op.read_with(&path).range(offset..offset + length).call()?; + assert_eq!(bs.len() as u64, length, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!( + "{:x}", + Sha256::digest(&content[offset as usize..(offset + length) as usize]) + ), + "read content" + ); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Read large range content should match. +pub fn test_blocking_read_large_range(op: BlockingOperator) -> Result<()> { + if !op.info().full_capability().read_with_range { + return Ok(()); + } + + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(op.info().full_capability()); + let (offset, _) = gen_offset_length(size); + + op.write(&path, content.clone()) + .expect("write must succeed"); + + let bs = op.read_with(&path).range(offset..u32::MAX as u64).call()?; + assert_eq!( + bs.len() as u64, + size as u64 - offset, + "read size with large range" + ); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + format!("{:x}", Sha256::digest(&content[offset as usize..])), + "read content with large range" + ); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Read not exist file should return NotFound +pub fn test_blocking_read_not_exist(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let bs = op.read(&path); + assert!(bs.is_err()); + assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +/// Stat normal file and dir should return metadata +pub fn test_blocking_read_only_stat_file_and_dir(op: BlockingOperator) -> Result<()> { + let meta = op.stat("normal_file.txt")?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let meta = op.stat("normal_dir/")?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat special file and dir should return metadata +pub fn test_blocking_read_only_stat_special_chars(op: BlockingOperator) -> Result<()> { + let meta = op.stat("special_file !@#$%^&()_+-=;',.txt")?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), 30482); + + let meta = op.stat("special_dir !@#$%^&()_+-=;',/")?; + assert_eq!(meta.mode(), EntryMode::DIR); + + Ok(()) +} + +/// Stat not exist file should return NotFound +pub fn test_blocking_read_only_stat_not_exist(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let meta = op.stat(&path); + assert!(meta.is_err()); + assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} + +/// Read full content should match. +pub fn test_blocking_read_only_read_full(op: BlockingOperator) -> Result<()> { + let bs = op.read("normal_file.txt")?; + assert_eq!(bs.len(), 30482, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", + "read content" + ); + + Ok(()) +} + +/// Read full content should match. +pub fn test_blocking_read_only_read_with_range(op: BlockingOperator) -> Result<()> { + let bs = op.read_with("normal_file.txt").range(1024..2048).call()?; + assert_eq!(bs.len(), 1024, "read size"); + assert_eq!( + format!("{:x}", Sha256::digest(&bs)), + "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", + "read content" + ); + + Ok(()) +} + +/// Read not exist file should return NotFound +pub fn test_blocking_read_only_read_not_exist(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let bs = op.read(&path); + assert!(bs.is_err()); + assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} diff --git a/core/tests/behavior/blocking_read_only.rs b/core/tests/behavior/blocking_read_only.rs deleted file mode 100644 index 5b5fb6840ae..00000000000 --- a/core/tests/behavior/blocking_read_only.rs +++ /dev/null @@ -1,112 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::Result; -use sha2::Digest; -use sha2::Sha256; - -use crate::*; - -pub fn behavior_blocking_read_only_tests(op: &Operator) -> Vec { - let cap = op.info().full_capability(); - - if !(cap.read && !cap.write && cap.blocking) { - return vec![]; - } - - blocking_trials!( - op, - test_blocking_read_only_stat_file_and_dir, - test_blocking_read_only_stat_special_chars, - test_blocking_read_only_stat_not_exist, - test_blocking_read_only_read_full, - test_blocking_read_only_read_with_range, - test_blocking_read_only_read_not_exist - ) -} - -/// Stat normal file and dir should return metadata -pub fn test_blocking_read_only_stat_file_and_dir(op: BlockingOperator) -> Result<()> { - let meta = op.stat("normal_file.txt")?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let meta = op.stat("normal_dir/")?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Stat special file and dir should return metadata -pub fn test_blocking_read_only_stat_special_chars(op: BlockingOperator) -> Result<()> { - let meta = op.stat("special_file !@#$%^&()_+-=;',.txt")?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let meta = op.stat("special_dir !@#$%^&()_+-=;',/")?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Stat not exist file should return NotFound -pub fn test_blocking_read_only_stat_not_exist(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let meta = op.stat(&path); - assert!(meta.is_err()); - assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} - -/// Read full content should match. -pub fn test_blocking_read_only_read_full(op: BlockingOperator) -> Result<()> { - let bs = op.read("normal_file.txt")?; - assert_eq!(bs.len(), 30482, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", - "read content" - ); - - Ok(()) -} - -/// Read full content should match. -pub fn test_blocking_read_only_read_with_range(op: BlockingOperator) -> Result<()> { - let bs = op.read_with("normal_file.txt").range(1024..2048).call()?; - assert_eq!(bs.len(), 1024, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", - "read content" - ); - - Ok(()) -} - -/// Read not exist file should return NotFound -pub fn test_blocking_read_only_read_not_exist(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let bs = op.read(&path); - assert!(bs.is_err()); - assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} diff --git a/core/tests/behavior/blocking_rename.rs b/core/tests/behavior/blocking_rename.rs index 28565de864d..90b737aeb5c 100644 --- a/core/tests/behavior/blocking_rename.rs +++ b/core/tests/behavior/blocking_rename.rs @@ -21,23 +21,21 @@ use sha2::Sha256; use crate::*; -pub fn behavior_blocking_rename_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.copy && cap.blocking && cap.rename) { - return vec![]; + if cap.read && cap.write && cap.copy && cap.blocking && cap.rename { + tests.extend(blocking_trials!( + op, + test_blocking_rename_file, + test_blocking_rename_non_existing_source, + test_blocking_rename_source_dir, + test_blocking_rename_target_dir, + test_blocking_rename_self, + test_blocking_rename_nested, + test_blocking_rename_overwrite + )) } - - blocking_trials!( - op, - test_blocking_rename_file, - test_blocking_rename_non_existing_source, - test_blocking_rename_source_dir, - test_blocking_rename_target_dir, - test_blocking_rename_self, - test_blocking_rename_nested, - test_blocking_rename_overwrite - ) } /// Rename a file and test with stat. diff --git a/core/tests/behavior/blocking_stat.rs b/core/tests/behavior/blocking_stat.rs new file mode 100644 index 00000000000..fcbab34089b --- /dev/null +++ b/core/tests/behavior/blocking_stat.rs @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use anyhow::Result; +use log::debug; +use log::warn; + +use crate::*; + +pub fn tests(op: &Operator, tests: &mut Vec) { + let cap = op.info().full_capability(); + + if cap.stat && cap.write && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_stat_file, + test_blocking_stat_dir, + test_blocking_stat_with_special_chars, + test_blocking_stat_not_exist + )) + } +} + +/// Stat existing file should return metadata +pub fn test_blocking_stat_file(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(op.info().full_capability()); + + op.write(&path, content).expect("write must succeed"); + + let meta = op.stat(&path)?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Stat existing file should return metadata +pub fn test_blocking_stat_dir(op: BlockingOperator) -> Result<()> { + if !op.info().full_capability().create_dir { + return Ok(()); + } + + let path = format!("{}/", uuid::Uuid::new_v4()); + + op.create_dir(&path).expect("write must succeed"); + + let meta = op.stat(&path)?; + assert_eq!(meta.mode(), EntryMode::DIR); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Stat existing file with special chars should return metadata +pub fn test_blocking_stat_with_special_chars(op: BlockingOperator) -> Result<()> { + // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. + if op.info().scheme() == opendal::Scheme::Supabase { + warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); + return Ok(()); + } + // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. + if op.info().scheme() == opendal::Scheme::Atomicserver { + warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); + return Ok(()); + } + + let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); + debug!("Generate a random file: {}", &path); + let (content, size) = gen_bytes(op.info().full_capability()); + + op.write(&path, content).expect("write must succeed"); + + let meta = op.stat(&path)?; + assert_eq!(meta.mode(), EntryMode::FILE); + assert_eq!(meta.content_length(), size as u64); + + op.delete(&path).expect("delete must succeed"); + Ok(()) +} + +/// Stat not exist file should return NotFound +pub fn test_blocking_stat_not_exist(op: BlockingOperator) -> Result<()> { + let path = uuid::Uuid::new_v4().to_string(); + + let meta = op.stat(&path); + assert!(meta.is_err()); + assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + + Ok(()) +} diff --git a/core/tests/behavior/blocking_write.rs b/core/tests/behavior/blocking_write.rs index 3b890655ab0..9e29a8274c2 100644 --- a/core/tests/behavior/blocking_write.rs +++ b/core/tests/behavior/blocking_write.rs @@ -15,84 +15,34 @@ // specific language governing permissions and limitations // under the License. -use std::io::Read; -use std::io::Seek; - use anyhow::Result; use log::debug; use log::warn; use sha2::Digest; use sha2::Sha256; +use std::io::{BufReader, Cursor}; use crate::*; -pub fn behavior_blocking_write_tests(op: &Operator) -> Vec { +pub fn tests(op: &Operator, tests: &mut Vec) { let cap = op.info().full_capability(); - if !(cap.read && cap.write && cap.blocking) { - return vec![]; + if cap.stat && cap.write && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_write_file, + test_blocking_write_with_dir_path, + test_blocking_write_with_special_chars + )) } - blocking_trials!( - op, - test_blocking_create_dir, - test_blocking_create_dir_existing, - test_blocking_write_file, - test_blocking_write_with_dir_path, - test_blocking_write_with_special_chars, - test_blocking_stat_file, - test_blocking_stat_dir, - test_blocking_stat_with_special_chars, - test_blocking_stat_not_exist, - test_blocking_read_full, - test_blocking_read_range, - test_blocking_read_large_range, - test_blocking_read_not_exist, - test_blocking_fuzz_range_reader, - test_blocking_fuzz_range_reader_with_buffer, - test_blocking_fuzz_offset_reader, - test_blocking_fuzz_offset_reader_with_buffer, - test_blocking_fuzz_part_reader, - test_blocking_fuzz_part_reader_with_buffer, - test_blocking_delete_file, - test_blocking_remove_one_file - ) -} - -/// Create dir with dir path should succeed. -pub fn test_blocking_create_dir(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); + if cap.write && cap.write_can_append && cap.stat && cap.blocking { + tests.extend(blocking_trials!( + op, + test_blocking_write_with_append, + test_blocking_writer_with_append + )) } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path)?; - - let meta = op.stat(&path)?; - assert_eq!(meta.mode(), EntryMode::DIR); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// Create dir on existing dir should succeed. -pub fn test_blocking_create_dir_existing(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path)?; - - op.create_dir(&path)?; - - let meta = op.stat(&path)?; - assert_eq!(meta.mode(), EntryMode::DIR); - - op.delete(&path).expect("delete must succeed"); - Ok(()) } /// Write a single file and test with stat. @@ -148,433 +98,57 @@ pub fn test_blocking_write_with_special_chars(op: BlockingOperator) -> Result<() Ok(()) } -/// Stat existing file should return metadata -pub fn test_blocking_stat_file(op: BlockingOperator) -> Result<()> { +/// Test append to a file must success. +pub fn test_blocking_write_with_append(op: BlockingOperator) -> Result<()> { let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - let meta = op.stat(&path)?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); + let (content_one, size_one) = gen_bytes(op.info().full_capability()); + let (content_two, size_two) = gen_bytes(op.info().full_capability()); - op.delete(&path).expect("delete must succeed"); - Ok(()) -} + op.write_with(&path, content_one.clone()) + .append(true) + .call() + .expect("append file first time must success"); -/// Stat existing file should return metadata -pub fn test_blocking_stat_dir(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } + op.write_with(&path, content_two.clone()) + .append(true) + .call() + .expect("append to an existing file must success"); - let path = format!("{}/", uuid::Uuid::new_v4()); + let bs = op.read(&path).expect("read file must success"); - op.create_dir(&path).expect("write must succeed"); + assert_eq!(bs.len(), size_one + size_two); + assert_eq!(bs[..size_one], content_one); + assert_eq!(bs[size_one..], content_two); - let meta = op.stat(&path)?; - assert_eq!(meta.mode(), EntryMode::DIR); + op.delete(&path).expect("delete file must success"); - op.delete(&path).expect("delete must succeed"); Ok(()) } -/// Stat existing file with special chars should return metadata -pub fn test_blocking_stat_with_special_chars(op: BlockingOperator) -> Result<()> { - // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. - if op.info().scheme() == opendal::Scheme::Supabase { - warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); - return Ok(()); - } - // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. - if op.info().scheme() == opendal::Scheme::Atomicserver { - warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); - return Ok(()); - } - - let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - let meta = op.stat(&path)?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// Stat not exist file should return NotFound -pub fn test_blocking_stat_not_exist(op: BlockingOperator) -> Result<()> { +/// Copy data from reader to writer +pub fn test_blocking_writer_with_append(op: BlockingOperator) -> Result<()> { let path = uuid::Uuid::new_v4().to_string(); + let (content, size): (Vec, usize) = + gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); - let meta = op.stat(&path); - assert!(meta.is_err()); - assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); + let mut a = op.writer_with(&path).append(true).call()?; - Ok(()) -} - -/// Read full content should match. -pub fn test_blocking_read_full(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); + // Wrap a buf reader here to make sure content is read in 1MiB chunks. + let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); + std::io::copy(&mut cursor, &mut a)?; + a.close()?; - op.write(&path, content.clone()) - .expect("write must succeed"); + let meta = op.stat(&path).expect("stat must succeed"); + assert_eq!(meta.content_length(), size as u64); let bs = op.read(&path)?; - assert_eq!(size, bs.len(), "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content)), - "read content" - ); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// Read range content should match. -pub fn test_blocking_read_range(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let bs = op.read_with(&path).range(offset..offset + length).call()?; - assert_eq!(bs.len() as u64, length, "read size"); + assert_eq!(bs.len(), size, "read size"); assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!( - "{:x}", - Sha256::digest(&content[offset as usize..(offset + length) as usize]) - ), + format!("{:x}", Sha256::digest(&bs[..size])), + format!("{:x}", Sha256::digest(content)), "read content" ); op.delete(&path).expect("delete must succeed"); Ok(()) } - -/// Read large range content should match. -pub fn test_blocking_read_large_range(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, _) = gen_offset_length(size); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let bs = op.read_with(&path).range(offset..u32::MAX as u64).call()?; - assert_eq!( - bs.len() as u64, - size as u64 - offset, - "read size with large range" - ); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[offset as usize..])), - "read content with large range" - ); - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -/// Read not exist file should return NotFound -pub fn test_blocking_read_not_exist(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let bs = op.read(&path); - assert!(bs.is_err()); - assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} - -pub fn test_blocking_fuzz_range_reader(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op - .reader_with(&path) - .range(0..content.len() as u64) - .call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -pub fn test_blocking_fuzz_range_reader_with_buffer(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op - .reader_with(&path) - .range(0..content.len() as u64) - .buffer(4096) - .call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -pub fn test_blocking_fuzz_offset_reader(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op.reader_with(&path).range(0..).call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -pub fn test_blocking_fuzz_offset_reader_with_buffer(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op.reader_with(&path).range(0..).buffer(4096).call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -pub fn test_blocking_fuzz_part_reader(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content, offset as usize, length as usize); - let mut o = op - .reader_with(&path) - .range(offset..offset + length) - .call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -pub fn test_blocking_fuzz_part_reader_with_buffer(op: BlockingOperator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content, offset as usize, length as usize); - let mut o = op - .reader_with(&path) - .range(offset..offset + length) - .buffer(4096) - .call()?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs)?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos)?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o.next().map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).expect("delete must succeed"); - Ok(()) -} - -// Delete existing file should succeed. -pub fn test_blocking_delete_file(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - op.delete(&path)?; - - // Stat it again to check. - assert!(!op.is_exist(&path)?); - - Ok(()) -} - -/// Remove one file -pub fn test_blocking_remove_one_file(op: BlockingOperator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).expect("write must succeed"); - - op.remove(vec![path.clone()])?; - - // Stat it again to check. - assert!(!op.is_exist(&path)?); - - Ok(()) -} diff --git a/core/tests/behavior/main.rs b/core/tests/behavior/main.rs index a47d17c6ef8..c8ae40fe929 100644 --- a/core/tests/behavior/main.rs +++ b/core/tests/behavior/main.rs @@ -21,45 +21,35 @@ mod utils; pub use utils::*; -// Async test cases -mod append; -mod copy; -mod fuzz; -mod list; -mod list_only; -mod presign; -mod read_only; -mod rename; -mod write; -use append::behavior_append_tests; -use copy::behavior_copy_tests; -use fuzz::behavior_fuzz_tests; -use list::behavior_list_tests; -use list_only::behavior_list_only_tests; -use presign::behavior_presign_tests; -use read_only::behavior_read_only_tests; -use rename::behavior_rename_tests; -use write::behavior_write_tests; +mod async_copy; +mod async_create_dir; +mod async_delete; +mod async_fuzz; +mod async_list; +mod async_presign; +mod async_read; +mod async_rename; +mod async_stat; +mod async_write; // Blocking test cases -mod blocking_append; mod blocking_copy; +mod blocking_create_dir; +mod blocking_delete; mod blocking_list; -mod blocking_read_only; +mod blocking_read; mod blocking_rename; +mod blocking_stat; mod blocking_write; -use blocking_append::behavior_blocking_append_tests; -use blocking_copy::behavior_blocking_copy_tests; -use blocking_list::behavior_blocking_list_tests; -use blocking_read_only::behavior_blocking_read_only_tests; -use blocking_rename::behavior_blocking_rename_tests; -use blocking_write::behavior_blocking_write_tests; + // External dependencies use libtest_mimic::Arguments; use libtest_mimic::Trial; -use opendal::raw::tests::init_test_service; +use opendal::raw::tests::{init_test_service, TEST_RUNTIME}; use opendal::*; +pub static TEST_FIXTURE: Fixture = Fixture::new(); + fn main() -> anyhow::Result<()> { let args = Arguments::from_args(); @@ -70,23 +60,26 @@ fn main() -> anyhow::Result<()> { }; let mut tests = Vec::new(); - // Blocking tests - tests.extend(behavior_blocking_append_tests(&op)); - tests.extend(behavior_blocking_copy_tests(&op)); - tests.extend(behavior_blocking_list_tests(&op)); - tests.extend(behavior_blocking_read_only_tests(&op)); - tests.extend(behavior_blocking_rename_tests(&op)); - tests.extend(behavior_blocking_write_tests(&op)); - // Async tests - tests.extend(behavior_append_tests(&op)); - tests.extend(behavior_copy_tests(&op)); - tests.extend(behavior_list_only_tests(&op)); - tests.extend(behavior_list_tests(&op)); - tests.extend(behavior_presign_tests(&op)); - tests.extend(behavior_read_only_tests(&op)); - tests.extend(behavior_rename_tests(&op)); - tests.extend(behavior_write_tests(&op)); - tests.extend(behavior_fuzz_tests(&op)); + + async_copy::tests(&op, &mut tests); + async_create_dir::tests(&op, &mut tests); + async_delete::tests(&op, &mut tests); + async_fuzz::tests(&op, &mut tests); + async_list::tests(&op, &mut tests); + async_presign::tests(&op, &mut tests); + async_read::tests(&op, &mut tests); + async_rename::tests(&op, &mut tests); + async_stat::tests(&op, &mut tests); + async_write::tests(&op, &mut tests); + + blocking_copy::tests(&op, &mut tests); + blocking_create_dir::tests(&op, &mut tests); + blocking_delete::tests(&op, &mut tests); + blocking_list::tests(&op, &mut tests); + blocking_read::tests(&op, &mut tests); + blocking_rename::tests(&op, &mut tests); + blocking_stat::tests(&op, &mut tests); + blocking_write::tests(&op, &mut tests); // Don't init logging while building operator which may break cargo // nextest output @@ -96,5 +89,10 @@ fn main() -> anyhow::Result<()> { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .try_init(); - libtest_mimic::run(&args, tests).exit(); + let conclusion = libtest_mimic::run(&args, tests); + + // Cleanup the fixtures. + TEST_RUNTIME.block_on(TEST_FIXTURE.cleanup(op)); + + conclusion.exit() } diff --git a/core/tests/behavior/read_only.rs b/core/tests/behavior/read_only.rs deleted file mode 100644 index e888a278ad8..00000000000 --- a/core/tests/behavior/read_only.rs +++ /dev/null @@ -1,331 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use anyhow::Result; -use futures::AsyncReadExt; -use sha2::Digest; -use sha2::Sha256; - -use crate::*; - -pub fn behavior_read_only_tests(op: &Operator) -> Vec { - let cap = op.info().full_capability(); - - if !cap.read || cap.write { - return vec![]; - } - - async_trials!( - op, - test_read_only_stat_file_and_dir, - test_read_only_stat_special_chars, - test_read_only_stat_not_cleaned_path, - test_read_only_stat_not_exist, - test_read_only_stat_with_if_match, - test_read_only_stat_with_if_none_match, - test_read_only_stat_root, - test_read_only_read_full, - test_read_only_read_full_with_special_chars, - test_read_only_read_with_range, - test_read_only_reader_with_range, - test_read_only_reader_from, - test_read_only_reader_tail, - test_read_only_read_not_exist, - test_read_only_read_with_dir_path, - test_read_only_read_with_if_match, - test_read_only_read_with_if_none_match - ) -} - -/// Stat normal file and dir should return metadata -pub async fn test_read_only_stat_file_and_dir(op: Operator) -> Result<()> { - let meta = op.stat("normal_file.txt").await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let meta = op.stat("normal_dir/").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Stat special file and dir should return metadata -pub async fn test_read_only_stat_special_chars(op: Operator) -> Result<()> { - let meta = op.stat("special_file !@#$%^&()_+-=;',.txt").await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let meta = op.stat("special_dir !@#$%^&()_+-=;',/").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Stat not cleaned path should also succeed. -pub async fn test_read_only_stat_not_cleaned_path(op: Operator) -> Result<()> { - let meta = op.stat("//normal_file.txt").await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - Ok(()) -} - -/// Stat not exist file should return NotFound -pub async fn test_read_only_stat_not_exist(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let meta = op.stat(&path).await; - assert!(meta.is_err()); - assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} - -/// Stat with if_match should succeed, else get a ConditionNotMatch error. -pub async fn test_read_only_stat_with_if_match(op: Operator) -> Result<()> { - if !op.info().full_capability().stat_with_if_match { - return Ok(()); - } - - let path = "normal_file.txt"; - - let meta = op.stat(path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let res = op.stat_with(path).if_match("invalid_etag").await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let result = op - .stat_with(path) - .if_match(meta.etag().expect("etag must exist")) - .await; - assert!(result.is_ok()); - - Ok(()) -} - -/// Stat with if_none_match should succeed, else get a ConditionNotMatch. -pub async fn test_read_only_stat_with_if_none_match(op: Operator) -> Result<()> { - if !op.info().full_capability().stat_with_if_none_match { - return Ok(()); - } - - let path = "normal_file.txt"; - - let meta = op.stat(path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), 30482); - - let res = op - .stat_with(path) - .if_none_match(meta.etag().expect("etag must exist")) - .await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let res = op.stat_with(path).if_none_match("invalid_etag").await?; - assert_eq!(res.mode(), meta.mode()); - assert_eq!(res.content_length(), meta.content_length()); - - Ok(()) -} - -/// Root should be able to stat and returns DIR. -pub async fn test_read_only_stat_root(op: Operator) -> Result<()> { - let meta = op.stat("").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - let meta = op.stat("/").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Read full content should match. -pub async fn test_read_only_read_full(op: Operator) -> Result<()> { - let bs = op.read("normal_file.txt").await?; - assert_eq!(bs.len(), 30482, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", - "read content" - ); - - Ok(()) -} - -/// Read full content should match. -pub async fn test_read_only_read_full_with_special_chars(op: Operator) -> Result<()> { - let bs = op.read("special_file !@#$%^&()_+-=;',.txt").await?; - assert_eq!(bs.len(), 30482, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", - "read content" - ); - - Ok(()) -} - -/// Read full content should match. -pub async fn test_read_only_read_with_range(op: Operator) -> Result<()> { - let bs = op.read_with("normal_file.txt").range(1024..2048).await?; - assert_eq!(bs.len(), 1024, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", - "read content" - ); - - Ok(()) -} - -/// Read range should match. -pub async fn test_read_only_reader_with_range(op: Operator) -> Result<()> { - let mut r = op.reader_with("normal_file.txt").range(1024..2048).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), 1024, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "330c6d57fdc1119d6021b37714ca5ad0ede12edd484f66be799a5cff59667034", - "read content" - ); - - Ok(()) -} - -/// Read from should match. -pub async fn test_read_only_reader_from(op: Operator) -> Result<()> { - let mut r = op.reader_with("normal_file.txt").range(29458..).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), 1024, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "cc9312c869238ea9410b6716e0fc3f48056f2bfb2fe06ccf5f96f2c3bf39e71b", - "read content" - ); - - Ok(()) -} - -/// Read tail should match. -pub async fn test_read_only_reader_tail(op: Operator) -> Result<()> { - let mut r = op.reader_with("normal_file.txt").range(..1024).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), 1024, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "cc9312c869238ea9410b6716e0fc3f48056f2bfb2fe06ccf5f96f2c3bf39e71b", - "read content" - ); - - Ok(()) -} - -/// Read not exist file should return NotFound -pub async fn test_read_only_read_not_exist(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let bs = op.read(&path).await; - assert!(bs.is_err()); - assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} - -/// Read with dir path should return an error. -pub async fn test_read_only_read_with_dir_path(op: Operator) -> Result<()> { - let path = format!("{}/", uuid::Uuid::new_v4()); - - let result = op.read(&path).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); - - Ok(()) -} - -/// Read with if_match should match, else get a ConditionNotMatch error. -pub async fn test_read_only_read_with_if_match(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_if_match { - return Ok(()); - } - - let path = "normal_file.txt"; - - let meta = op.stat(path).await?; - - let res = op.read_with(path).if_match("invalid_etag").await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let bs = op - .read_with(path) - .if_match(meta.etag().expect("etag must exist")) - .await - .expect("read must succeed"); - assert_eq!(bs.len(), 30482, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", - "read content" - ); - - Ok(()) -} - -/// Read with if_none_match should match, else get a ConditionNotMatch error. -pub async fn test_read_only_read_with_if_none_match(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_if_none_match { - return Ok(()); - } - - let path = "normal_file.txt"; - - let meta = op.stat(path).await?; - - let res = op - .read_with(path) - .if_none_match(meta.etag().expect("etag must exist")) - .await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let bs = op - .read_with(path) - .if_none_match("invalid_etag") - .await - .expect("read must succeed"); - assert_eq!(bs.len(), 30482, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - "943048ba817cdcd786db07d1f42d5500da7d10541c2f9353352cd2d3f66617e5", - "read content" - ); - - Ok(()) -} diff --git a/core/tests/behavior/utils.rs b/core/tests/behavior/utils.rs index 0bb0ed607f1..f8ebae75a25 100644 --- a/core/tests/behavior/utils.rs +++ b/core/tests/behavior/utils.rs @@ -15,23 +15,16 @@ // specific language governing permissions and limitations // under the License. -use std::fmt; -use std::fmt::Debug; -use std::fmt::Formatter; -use std::io::SeekFrom; -use std::usize; +use std::sync::Mutex; +use std::{mem, usize}; -use bytes::Bytes; use futures::Future; use libtest_mimic::Failed; use libtest_mimic::Trial; -use log::debug; use opendal::raw::tests::TEST_RUNTIME; use opendal::*; use rand::distributions::uniform::SampleRange; use rand::prelude::*; -use sha2::Digest; -use sha2::Sha256; pub fn gen_bytes_with_range(range: impl SampleRange) -> (Vec, usize) { let mut rng = thread_rng(); @@ -110,255 +103,89 @@ macro_rules! blocking_trials { }; } -/// ObjectReaderFuzzer is the fuzzer for object readers. -/// -/// We will generate random read/seek/next operations to operate on object -/// reader to check if the output is expected. -/// -/// # TODO -/// -/// This fuzzer only generate valid operations. -/// -/// In the future, we need to generate invalid operations to check if we -/// handled correctly. -pub struct ObjectReaderFuzzer { - name: String, - bs: Vec, - - offset: usize, - size: usize, - cur: usize, - rng: ThreadRng, - actions: Vec, +pub struct Fixture { + pub paths: Mutex>, } -#[derive(Debug, Clone, Copy)] -pub enum ObjectReaderAction { - Read(usize), - Seek(SeekFrom), - Next, -} - -impl ObjectReaderFuzzer { - /// Create a new fuzzer. - pub fn new(name: &str, bs: Vec, offset: usize, size: usize) -> Self { +impl Fixture { + /// Create a new fixture + pub const fn new() -> Self { Self { - name: name.to_string(), - bs, - - offset, - size, - cur: 0, - - rng: thread_rng(), - actions: vec![], + paths: Mutex::new(vec![]), } } - /// Generate a new action. - pub fn fuzz(&mut self) -> ObjectReaderAction { - let action = match self.rng.gen_range(0..3) { - // Generate a read action. - 0 => { - if self.cur >= self.size { - ObjectReaderAction::Read(0) - } else { - let size = self.rng.gen_range(0..self.size - self.cur); - ObjectReaderAction::Read(size) - } - } - // Generate a seek action. - 1 => match self.rng.gen_range(0..3) { - // Generate a SeekFrom::Start action. - 0 => { - let offset = self.rng.gen_range(0..self.size as u64); - ObjectReaderAction::Seek(SeekFrom::Start(offset)) - } - // Generate a SeekFrom::End action. - 1 => { - let offset = self.rng.gen_range(-(self.size as i64)..0); - ObjectReaderAction::Seek(SeekFrom::End(offset)) - } - // Generate a SeekFrom::Current action. - 2 => { - let offset = self - .rng - .gen_range(-(self.cur as i64)..(self.size - self.cur) as i64); - ObjectReaderAction::Seek(SeekFrom::Current(offset)) - } - _ => unreachable!(), - }, - // Generate a next action. - 2 => ObjectReaderAction::Next, - _ => unreachable!(), - }; - - debug!("{} perform fuzz action: {:?}", self.name, action); - self.actions.push(action); - - action - } + /// Create a new dir path + pub fn new_dir_path(&self) -> String { + let path = format!("{}/", uuid::Uuid::new_v4()); + self.paths.lock().unwrap().push(path.clone()); - /// Check if read operation is expected. - pub fn check_read(&mut self, output_n: usize, output_bs: &[u8]) { - assert!( - self.cur + output_n <= self.size, - "check read failed: output bs is larger than remaining bs: actions: {:?}", - self.actions - ); - - let current_size = self.offset + self.cur; - let expected_bs = &self.bs[current_size..current_size + output_n]; - - assert_eq!( - format!("{:x}", Sha256::digest(output_bs)), - format!("{:x}", Sha256::digest(expected_bs)), - "check read failed: output bs is different with expected bs, actions: {:?}", - self.actions, - ); - - // Update current pos. - self.cur += output_n; + path } - /// Check if seek operation is expected. - pub fn check_seek(&mut self, input_pos: SeekFrom, output_pos: u64) { - let expected_pos = match input_pos { - SeekFrom::Start(offset) => offset as i64, - SeekFrom::End(offset) => self.size as i64 + offset, - SeekFrom::Current(offset) => self.cur as i64 + offset, - }; - - assert_eq!( - output_pos, expected_pos as u64, - "check seek failed: output pos is different with expected pos, actions: {:?}", - self.actions - ); - - // Update current pos. - self.cur = expected_pos as usize; - } + /// Create a new file path + pub fn new_file_path(&self) -> String { + let path = format!("{}", uuid::Uuid::new_v4()); + self.paths.lock().unwrap().push(path.clone()); - /// Check if next operation is expected. - pub fn check_next(&mut self, output_bs: Option) { - if let Some(output_bs) = output_bs { - assert!( - self.cur + output_bs.len() <= self.size, - "check next failed: output bs is larger than remaining bs, actions: {:?}", - self.actions - ); - - let current_size = self.offset + self.cur; - let expected_bs = &self.bs[current_size..current_size + output_bs.len()]; - - assert_eq!( - format!("{:x}", Sha256::digest(&output_bs)), - format!("{:x}", Sha256::digest(expected_bs)), - "check next failed: output bs is different with expected bs, actions: {:?}", - self.actions - ); - - // Update current pos. - self.cur += output_bs.len(); - } else { - assert!( - self.cur >= self.size, - "check next failed: output bs is None, we still have bytes to read, actions: {:?}", - self.actions - ) - } + path } -} -/// ObjectWriterFuzzer is the fuzzer for object writer. -/// -/// We will generate random write operations to operate on object -/// write to check if the output is expected. -/// -/// # TODO -/// -/// This fuzzer only generate valid operations. -/// -/// In the future, we need to generate invalid operations to check if we -/// handled correctly. -pub struct ObjectWriterFuzzer { - name: String, - bs: Vec, - - size: Option, - cur: usize, - rng: ThreadRng, - actions: Vec, -} - -#[derive(Clone)] -pub enum ObjectWriterAction { - Write(Bytes), -} + /// Create a new file with random content + pub fn new_file(&self, op: impl Into) -> (String, Vec, usize) { + let max_size = op + .into() + .info() + .full_capability() + .write_total_max_size + .unwrap_or(4 * 1024 * 1024); -impl Debug for ObjectWriterAction { - fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { - match self { - ObjectWriterAction::Write(bs) => write!(f, "Write({})", bs.len()), - } + self.new_file_with_range(uuid::Uuid::new_v4().to_string(), 1..max_size) } -} - -impl ObjectWriterFuzzer { - /// Create a new fuzzer. - pub fn new(name: &str, size: Option) -> Self { - Self { - name: name.to_string(), - bs: Vec::new(), - - size, - cur: 0, - rng: thread_rng(), - actions: vec![], - } + pub fn new_file_with_path( + &self, + op: impl Into, + path: &str, + ) -> (String, Vec, usize) { + let max_size = op + .into() + .info() + .full_capability() + .write_total_max_size + .unwrap_or(4 * 1024 * 1024); + + self.new_file_with_range(path, 1..max_size) } - /// Generate a new action. - pub fn fuzz(&mut self) -> ObjectWriterAction { - let max = if let Some(size) = self.size { - size - self.cur - } else { - // Set max to 1MiB - 1024 * 1024 - }; - - let size = self.rng.gen_range(0..max); - - let mut bs = vec![0; size]; - self.rng.fill_bytes(&mut bs); + /// Create a new file with random content in range. + fn new_file_with_range( + &self, + path: impl Into, + range: impl SampleRange, + ) -> (String, Vec, usize) { + let path = path.into(); + self.paths.lock().unwrap().push(path.clone()); - let bs = Bytes::from(bs); - self.bs.extend_from_slice(&bs); - self.cur += bs.len(); + let mut rng = thread_rng(); - let action = ObjectWriterAction::Write(bs); - debug!("{} perform fuzz action: {:?}", self.name, action); + let size = rng.gen_range(range); + let mut content = vec![0; size]; + rng.fill_bytes(&mut content); - self.actions.push(action.clone()); - - action + (path, content, size) } - /// Check if read operation is expected. - pub fn check(&mut self, actual_bs: &[u8]) { - assert_eq!( - self.bs.len(), - actual_bs.len(), - "check failed: expected len is different with actual len, actions: {:?}", - self.actions - ); - - assert_eq!( - format!("{:x}", Sha256::digest(&self.bs)), - format!("{:x}", Sha256::digest(actual_bs)), - "check failed: expected bs is different with actual bs, actions: {:?}", - self.actions, - ); + /// Perform cleanup + pub async fn cleanup(&self, op: impl Into) { + let op = op.into(); + let paths: Vec<_> = mem::take(self.paths.lock().unwrap().as_mut()); + for path in paths.iter() { + // We try our best to cleanup fixtures, but won't panic if failed. + let _ = op.delete(path).await.map_err(|err| { + log::error!("fixture cleanup path {path} failed: {:?}", err); + }); + log::info!("fixture cleanup path {path} succeeded") + } } } diff --git a/core/tests/behavior/write.rs b/core/tests/behavior/write.rs deleted file mode 100644 index 6c3114820f2..00000000000 --- a/core/tests/behavior/write.rs +++ /dev/null @@ -1,1825 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -use std::str::FromStr; -use std::time::Duration; - -use anyhow::Result; -use bytes::Buf; -use bytes::Bytes; -use futures::io::BufReader; -use futures::io::Cursor; -use futures::stream; -use futures::AsyncReadExt; -use futures::AsyncSeekExt; -use futures::StreamExt; -use http::StatusCode; -use log::debug; -use log::warn; -use reqwest::Url; -use sha2::Digest; -use sha2::Sha256; - -use crate::*; - -pub fn behavior_write_tests(op: &Operator) -> Vec { - let cap = op.info().full_capability(); - - if !(cap.read && cap.write) { - return vec![]; - } - - async_trials!( - op, - test_create_dir, - test_create_dir_existing, - test_write_only, - test_write_with_empty_content, - test_write_with_dir_path, - test_write_with_special_chars, - test_write_with_cache_control, - test_write_with_content_type, - test_write_with_content_disposition, - test_stat_file, - test_stat_dir, - test_stat_nested_parent_dir, - test_stat_with_special_chars, - test_stat_not_cleaned_path, - test_stat_not_exist, - test_stat_with_if_match, - test_stat_with_if_none_match, - test_stat_with_override_cache_control, - test_stat_with_override_content_disposition, - test_stat_with_override_content_type, - test_stat_root, - test_read_full, - test_read_range, - test_read_large_range, - test_reader_range, - test_reader_range_with_buffer, - test_reader_from, - test_reader_from_with_buffer, - test_reader_tail, - test_reader_tail_with_buffer, - test_read_not_exist, - test_read_with_if_match, - test_read_with_if_none_match, - test_fuzz_reader_with_range, - test_fuzz_reader_with_range_and_buffer, - test_fuzz_offset_reader, - test_fuzz_offset_reader_with_buffer, - test_fuzz_part_reader, - test_fuzz_part_reader_with_buffer, - test_read_with_dir_path, - test_read_with_special_chars, - test_read_with_override_cache_control, - test_read_with_override_content_disposition, - test_read_with_override_content_type, - test_delete_file, - test_delete_empty_dir, - test_delete_with_special_chars, - test_delete_not_existing, - test_delete_stream, - test_remove_one_file, - test_writer_write, - test_writer_sink, - test_writer_copy, - test_writer_abort, - test_writer_futures_copy, - test_fuzz_unsized_writer, - test_invalid_reader_seek - ) -} - -/// Create dir with dir path should succeed. -pub async fn test_create_dir(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path).await?; - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Create dir on existing dir should succeed. -pub async fn test_create_dir_existing(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path).await?; - - op.create_dir(&path).await?; - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Write a single file and test with stat. -pub async fn test_write_only(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Write a file with empty content. -pub async fn test_write_with_empty_content(op: Operator) -> Result<()> { - if !op.info().full_capability().write_can_empty { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - - op.write(&path, vec![]).await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), 0); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Write file with dir path should return an error -pub async fn test_write_with_dir_path(op: Operator) -> Result<()> { - let path = format!("{}/", uuid::Uuid::new_v4()); - let (content, _) = gen_bytes(op.info().full_capability()); - - let result = op.write(&path, content).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); - - Ok(()) -} - -/// Write a single file with special chars should succeed. -pub async fn test_write_with_special_chars(op: Operator) -> Result<()> { - // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. - if op.info().scheme() == opendal::Scheme::Supabase { - warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); - return Ok(()); - } - // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. - if op.info().scheme() == opendal::Scheme::Atomicserver { - warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); - return Ok(()); - } - - let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Write a single file with cache control should succeed. -pub async fn test_write_with_cache_control(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_cache_control { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - let target_cache_control = "no-cache, no-store, max-age=300"; - op.write_with(&path, content) - .cache_control(target_cache_control) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.cache_control().expect("cache control must exist"), - target_cache_control - ); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Write a single file with content type should succeed. -pub async fn test_write_with_content_type(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_content_type { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_type = "application/json"; - op.write_with(&path, content) - .content_type(target_content_type) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_type().expect("content type must exist"), - target_content_type - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Write a single file with content disposition should succeed. -pub async fn test_write_with_content_disposition(op: Operator) -> Result<()> { - if !op.info().full_capability().write_with_content_disposition { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - let target_content_disposition = "attachment; filename=\"filename.jpg\""; - op.write_with(&path, content) - .content_disposition(target_content_disposition) - .await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!( - meta.content_disposition().expect("content type must exist"), - target_content_disposition - ); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Stat existing file should return metadata -pub async fn test_stat_file(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - // Stat a file with trailing slash should return `NotFound`. - if op.info().full_capability().create_dir { - let result = op.stat(&format!("{path}/")).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().kind(), ErrorKind::NotFound); - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat existing file should return metadata -pub async fn test_stat_dir(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path).await.expect("write must succeed"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - // Stat a dir without trailing slash could have two behavior. - let result = op.stat(path.trim_end_matches('/')).await; - match result { - Ok(meta) => assert_eq!(meta.mode(), EntryMode::DIR), - Err(err) => assert_eq!(err.kind(), ErrorKind::NotFound), - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat the parent dir of existing dir should return metadata -pub async fn test_stat_nested_parent_dir(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let parent = format!("{}", uuid::Uuid::new_v4()); - let file = format!("{}", uuid::Uuid::new_v4()); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&format!("{parent}/{file}"), content.clone()) - .await - .expect("write must succeed"); - - let meta = op.stat(&format!("{parent}/")).await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - op.delete(&format!("{parent}/{file}")) - .await - .expect("delete must succeed"); - Ok(()) -} - -/// Stat existing file with special chars should return metadata -pub async fn test_stat_with_special_chars(op: Operator) -> Result<()> { - // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. - if op.info().scheme() == opendal::Scheme::Supabase { - warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); - return Ok(()); - } - // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. - if op.info().scheme() == opendal::Scheme::Atomicserver { - warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); - return Ok(()); - } - - let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat not cleaned path should also succeed. -pub async fn test_stat_not_cleaned_path(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - let meta = op.stat(&format!("//{}", &path)).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat not exist file should return NotFound -pub async fn test_stat_not_exist(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - // Stat not exist file should returns NotFound. - let meta = op.stat(&path).await; - assert!(meta.is_err()); - assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - - // Stat not exist dir should also returns NotFound. - if op.info().full_capability().create_dir { - let meta = op.stat(&format!("{path}/")).await; - assert!(meta.is_err()); - assert_eq!(meta.unwrap_err().kind(), ErrorKind::NotFound); - } - - Ok(()) -} - -/// Stat with if_match should succeed, else get a ConditionNotMatch error. -pub async fn test_stat_with_if_match(op: Operator) -> Result<()> { - if !op.info().full_capability().stat_with_if_match { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - let res = op.stat_with(&path).if_match("\"invalid_etag\"").await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let result = op - .stat_with(&path) - .if_match(meta.etag().expect("etag must exist")) - .await; - assert!(result.is_ok()); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat with if_none_match should succeed, else get a ConditionNotMatch. -pub async fn test_stat_with_if_none_match(op: Operator) -> Result<()> { - if !op.info().full_capability().stat_with_if_none_match { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let meta = op.stat(&path).await?; - assert_eq!(meta.mode(), EntryMode::FILE); - assert_eq!(meta.content_length(), size as u64); - - let res = op - .stat_with(&path) - .if_none_match(meta.etag().expect("etag must exist")) - .await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let res = op - .stat_with(&path) - .if_none_match("\"invalid_etag\"") - .await?; - assert_eq!(res.mode(), meta.mode()); - assert_eq!(res.content_length(), meta.content_length()); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat file with override-cache-control should succeed. -pub async fn test_stat_with_override_cache_control(op: Operator) -> Result<()> { - if !(op.info().full_capability().stat_with_override_cache_control - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_cache_control = "no-cache, no-store, must-revalidate"; - let signed_req = op - .presign_stat_with(&path, Duration::from_secs(60)) - .override_cache_control(target_cache_control) - .await - .expect("sign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get("cache-control") - .expect("cache-control header must exist") - .to_str() - .expect("cache-control header must be string"), - target_cache_control - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Stat file with override_content_disposition should succeed. -pub async fn test_stat_with_override_content_disposition(op: Operator) -> Result<()> { - if !(op - .info() - .full_capability() - .stat_with_override_content_disposition - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_content_disposition = "attachment; filename=foo.txt"; - - let signed_req = op - .presign_stat_with(&path, Duration::from_secs(60)) - .override_content_disposition(target_content_disposition) - .await - .expect("presign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get(http::header::CONTENT_DISPOSITION) - .expect("content-disposition header must exist") - .to_str() - .expect("content-disposition header must be string"), - target_content_disposition - ); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Stat file with override_content_type should succeed. -pub async fn test_stat_with_override_content_type(op: Operator) -> Result<()> { - if !(op.info().full_capability().stat_with_override_content_type - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_content_type = "application/opendal"; - - let signed_req = op - .presign_stat_with(&path, Duration::from_secs(60)) - .override_content_type(target_content_type) - .await - .expect("presign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get(http::header::CONTENT_TYPE) - .expect("content-type header must exist") - .to_str() - .expect("content-type header must be string"), - target_content_type - ); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Root should be able to stat and returns DIR. -pub async fn test_stat_root(op: Operator) -> Result<()> { - let meta = op.stat("").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - let meta = op.stat("/").await?; - assert_eq!(meta.mode(), EntryMode::DIR); - - Ok(()) -} - -/// Read full content should match. -pub async fn test_read_full(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let bs = op.read(&path).await?; - assert_eq!(size, bs.len(), "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content)), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range content should match. -pub async fn test_read_range(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let bs = op.read_with(&path).range(offset..offset + length).await?; - assert_eq!(bs.len() as u64, length, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!( - "{:x}", - Sha256::digest(&content[offset as usize..(offset + length) as usize]) - ), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read large range content should match. -pub async fn test_read_large_range(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, _) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let bs = op.read_with(&path).range(offset..u32::MAX as u64).await?; - assert_eq!( - bs.len() as u64, - size as u64 - offset, - "read size with large range" - ); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[offset as usize..])), - "read content with large range" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range content should match. -pub async fn test_reader_range(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = op.reader_with(&path).range(offset..offset + length).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!( - "{:x}", - Sha256::digest(&content[offset as usize..(offset + length) as usize]) - ), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range content should match. -pub async fn test_reader_range_with_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = op - .reader_with(&path) - .range(offset..offset + length) - .buffer(4096) - .await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!( - "{:x}", - Sha256::digest(&content[offset as usize..(offset + length) as usize]) - ), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range from should match. -pub async fn test_reader_from(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, _) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = op.reader_with(&path).range(offset..).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), size - offset as usize, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[offset as usize..])), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range from should match. -pub async fn test_reader_from_with_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, _) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = op.reader_with(&path).range(offset..).buffer(4096).await?; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), size - offset as usize, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[offset as usize..])), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range tail should match. -pub async fn test_reader_tail(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (_, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = match op.reader_with(&path).range(..length).await { - Ok(r) => r, - // Not all services support range with tail range, let's tolerate this. - Err(err) if err.kind() == ErrorKind::Unsupported => { - warn!("service doesn't support range with tail"); - return Ok(()); - } - Err(err) => return Err(err.into()), - }; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), length as usize, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[size - length as usize..])), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read range tail should match. -pub async fn test_reader_tail_with_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (_, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = match op.reader_with(&path).range(..length).buffer(4096).await { - Ok(r) => r, - // Not all services support range with tail range, let's tolerate this. - Err(err) if err.kind() == ErrorKind::Unsupported => { - warn!("service doesn't support range with tail"); - return Ok(()); - } - Err(err) => return Err(err.into()), - }; - - let mut bs = Vec::new(); - r.read_to_end(&mut bs).await?; - - assert_eq!(bs.len(), length as usize, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content[size - length as usize..])), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read not exist file should return NotFound -pub async fn test_read_not_exist(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - let bs = op.read(&path).await; - assert!(bs.is_err()); - assert_eq!(bs.unwrap_err().kind(), ErrorKind::NotFound); - - Ok(()) -} - -/// Read with if_match should match, else get a ConditionNotMatch error. -pub async fn test_read_with_if_match(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_if_match { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let meta = op.stat(&path).await?; - - let res = op.read_with(&path).if_match("\"invalid_etag\"").await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let bs = op - .read_with(&path) - .if_match(meta.etag().expect("etag must exist")) - .await - .expect("read must succeed"); - assert_eq!(bs, content); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read with if_none_match should match, else get a ConditionNotMatch error. -pub async fn test_read_with_if_none_match(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_if_none_match { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let meta = op.stat(&path).await?; - - let res = op - .read_with(&path) - .if_none_match(meta.etag().expect("etag must exist")) - .await; - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - - let bs = op - .read_with(&path) - .if_none_match("\"invalid_etag\"") - .await - .expect("read must succeed"); - assert_eq!(bs, content); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_reader_with_range(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op.reader_with(&path).range(0..content.len() as u64).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_reader_with_range_and_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op - .reader_with(&path) - .range(0..content.len() as u64) - .buffer(4096) - .await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_offset_reader(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op.reader_with(&path).range(0..).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_offset_reader_with_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = ObjectReaderFuzzer::new(&path, content.clone(), 0, content.len()); - let mut o = op.reader_with(&path).range(0..).buffer(4096).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_part_reader(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = - ObjectReaderFuzzer::new(&path, content.clone(), offset as usize, length as usize); - let mut o = op.reader_with(&path).range(offset..offset + length).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -pub async fn test_fuzz_part_reader_with_buffer(op: Operator) -> Result<()> { - if !op.info().full_capability().read_with_range { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - let (offset, length) = gen_offset_length(size); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut fuzzer = - ObjectReaderFuzzer::new(&path, content.clone(), offset as usize, length as usize); - let mut o = op - .reader_with(&path) - .range(offset..offset + length) - .buffer(4096) - .await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectReaderAction::Read(size) => { - let mut bs = vec![0; size]; - let n = o.read(&mut bs).await?; - fuzzer.check_read(n, &bs[..n]) - } - ObjectReaderAction::Seek(input_pos) => { - let actual_pos = o.seek(input_pos).await?; - fuzzer.check_seek(input_pos, actual_pos) - } - ObjectReaderAction::Next => { - let actual_bs = o - .next() - .await - .map(|v| v.expect("next should not return error")); - fuzzer.check_next(actual_bs) - } - } - } - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read with dir path should return an error. -pub async fn test_read_with_dir_path(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path).await.expect("write must succeed"); - - let result = op.read(&path).await; - assert!(result.is_err()); - assert_eq!(result.unwrap_err().kind(), ErrorKind::IsADirectory); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read file with special chars should succeed. -pub async fn test_read_with_special_chars(op: Operator) -> Result<()> { - // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. - if op.info().scheme() == opendal::Scheme::Supabase { - warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); - return Ok(()); - } - // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. - if op.info().scheme() == opendal::Scheme::Atomicserver { - warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); - return Ok(()); - } - - let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, size) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let bs = op.read(&path).await?; - assert_eq!(size, bs.len(), "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs)), - format!("{:x}", Sha256::digest(&content)), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read file with override-cache-control should succeed. -pub async fn test_read_with_override_cache_control(op: Operator) -> Result<()> { - if !(op.info().full_capability().read_with_override_cache_control - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_cache_control = "no-cache, no-store, must-revalidate"; - let signed_req = op - .presign_read_with(&path, Duration::from_secs(60)) - .override_cache_control(target_cache_control) - .await - .expect("sign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get("cache-control") - .expect("cache-control header must exist") - .to_str() - .expect("cache-control header must be string"), - target_cache_control - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Read file with override_content_disposition should succeed. -pub async fn test_read_with_override_content_disposition(op: Operator) -> Result<()> { - if !(op - .info() - .full_capability() - .read_with_override_content_disposition - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_content_disposition = "attachment; filename=foo.txt"; - - let signed_req = op - .presign_read_with(&path, Duration::from_secs(60)) - .override_content_disposition(target_content_disposition) - .await - .expect("presign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get(http::header::CONTENT_DISPOSITION) - .expect("content-disposition header must exist") - .to_str() - .expect("content-disposition header must be string"), - target_content_disposition - ); - assert_eq!(resp.bytes().await?, content); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Read file with override_content_type should succeed. -pub async fn test_read_with_override_content_type(op: Operator) -> Result<()> { - if !(op.info().full_capability().read_with_override_content_type - && op.info().full_capability().presign) - { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let target_content_type = "application/opendal"; - - let signed_req = op - .presign_read_with(&path, Duration::from_secs(60)) - .override_content_type(target_content_type) - .await - .expect("presign must succeed"); - - let client = reqwest::Client::new(); - let mut req = client.request( - signed_req.method().clone(), - Url::from_str(&signed_req.uri().to_string()).expect("must be valid url"), - ); - for (k, v) in signed_req.header() { - req = req.header(k, v); - } - - let resp = req.send().await.expect("send must succeed"); - - assert_eq!(resp.status(), StatusCode::OK); - assert_eq!( - resp.headers() - .get(http::header::CONTENT_TYPE) - .expect("content-type header must exist") - .to_str() - .expect("content-type header must be string"), - target_content_type - ); - assert_eq!(resp.bytes().await?, content); - - op.delete(&path).await.expect("delete must succeed"); - - Ok(()) -} - -/// Delete existing file should succeed. -pub async fn test_writer_abort(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - let mut writer = match op.writer(&path).await { - Ok(writer) => writer, - Err(e) => { - assert_eq!(e.kind(), ErrorKind::Unsupported); - return Ok(()); - } - }; - - if let Err(e) = writer.write(content).await { - assert_eq!(e.kind(), ErrorKind::Unsupported); - return Ok(()); - } - - if let Err(e) = writer.abort().await { - assert_eq!(e.kind(), ErrorKind::Unsupported); - return Ok(()); - } - - // Aborted writer should not write actual file. - assert!(!op.is_exist(&path).await?); - Ok(()) -} - -/// Delete existing file should succeed. -pub async fn test_delete_file(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - op.delete(&path).await?; - - // Stat it again to check. - assert!(!op.is_exist(&path).await?); - - Ok(()) -} - -/// Delete empty dir should succeed. -pub async fn test_delete_empty_dir(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let path = format!("{}/", uuid::Uuid::new_v4()); - - op.create_dir(&path).await.expect("create must succeed"); - - op.delete(&path).await?; - - Ok(()) -} - -/// Delete file with special chars should succeed. -pub async fn test_delete_with_special_chars(op: Operator) -> Result<()> { - // Ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 addressed. - if op.info().scheme() == opendal::Scheme::Supabase { - warn!("ignore test for supabase until https://github.com/apache/incubator-opendal/issues/2194 is resolved"); - return Ok(()); - } - // Ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 addressed. - if op.info().scheme() == opendal::Scheme::Atomicserver { - warn!("ignore test for atomicserver until https://github.com/atomicdata-dev/atomic-server/issues/663 is resolved"); - return Ok(()); - } - - let path = format!("{} !@#$%^&()_+-=;',.txt", uuid::Uuid::new_v4()); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content).await.expect("write must succeed"); - - op.delete(&path).await?; - - // Stat it again to check. - assert!(!op.is_exist(&path).await?); - - Ok(()) -} - -/// Delete not existing file should also succeed. -pub async fn test_delete_not_existing(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - - op.delete(&path).await?; - - Ok(()) -} - -/// Remove one file -pub async fn test_remove_one_file(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - op.remove(vec![path.clone()]).await?; - - // Stat it again to check. - assert!(!op.is_exist(&path).await?); - - op.write(&format!("/{path}"), content) - .await - .expect("write must succeed"); - - op.remove(vec![path.clone()]).await?; - - // Stat it again to check. - assert!(!op.is_exist(&path).await?); - - Ok(()) -} - -/// Delete via stream. -pub async fn test_delete_stream(op: Operator) -> Result<()> { - if !op.info().full_capability().create_dir { - return Ok(()); - } - - let dir = uuid::Uuid::new_v4().to_string(); - op.create_dir(&format!("{dir}/")) - .await - .expect("creat must succeed"); - - let expected: Vec<_> = (0..100).collect(); - for path in expected.iter() { - op.write(&format!("{dir}/{path}"), "delete_stream").await?; - } - - op.with_limit(30) - .remove_via(futures::stream::iter(expected.clone()).map(|v| format!("{dir}/{v}"))) - .await?; - - // Stat it again to check. - for path in expected.iter() { - assert!( - !op.is_exist(&format!("{dir}/{path}")).await?, - "{path} should be removed" - ) - } - - Ok(()) -} - -/// Append data into writer -pub async fn test_writer_write(op: Operator) -> Result<()> { - if !(op.info().full_capability().write_can_multi) { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let size = 5 * 1024 * 1024; // write file with 5 MiB - let content_a = gen_fixed_bytes(size); - let content_b = gen_fixed_bytes(size); - - let mut w = op.writer(&path).await?; - w.write(content_a.clone()).await?; - w.write(content_b.clone()).await?; - w.close().await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), (size * 2) as u64); - - let bs = op.read(&path).await?; - assert_eq!(bs.len(), size * 2, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content_a)), - "read content a" - ); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[size..])), - format!("{:x}", Sha256::digest(content_b)), - "read content b" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Streaming data into writer -pub async fn test_writer_sink(op: Operator) -> Result<()> { - let cap = op.info().full_capability(); - if !(cap.write && cap.write_can_multi) { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let size = 5 * 1024 * 1024; // write file with 5 MiB - let content_a = gen_fixed_bytes(size); - let content_b = gen_fixed_bytes(size); - let stream = stream::iter(vec![content_a.clone(), content_b.clone()]).map(Ok); - - let mut w = op.writer_with(&path).buffer(5 * 1024 * 1024).await?; - w.sink(stream).await?; - w.close().await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), (size * 2) as u64); - - let bs = op.read(&path).await?; - assert_eq!(bs.len(), size * 2, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content_a)), - "read content a" - ); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[size..])), - format!("{:x}", Sha256::digest(content_b)), - "read content b" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Reading data into writer -pub async fn test_writer_copy(op: Operator) -> Result<()> { - let cap = op.info().full_capability(); - if !(cap.write && cap.write_can_multi) { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let size = 5 * 1024 * 1024; // write file with 5 MiB - let content_a = gen_fixed_bytes(size); - let content_b = gen_fixed_bytes(size); - - let mut w = op.writer_with(&path).buffer(5 * 1024 * 1024).await?; - - let mut content = Bytes::from([content_a.clone(), content_b.clone()].concat()); - while !content.is_empty() { - let reader = Cursor::new(content.clone()); - let n = w.copy(reader).await?; - content.advance(n as usize); - } - w.close().await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), (size * 2) as u64); - - let bs = op.read(&path).await?; - assert_eq!(bs.len(), size * 2, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content_a)), - "read content a" - ); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[size..])), - format!("{:x}", Sha256::digest(content_b)), - "read content b" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Copy data from reader to writer -pub async fn test_writer_futures_copy(op: Operator) -> Result<()> { - if !(op.info().full_capability().write_can_multi) { - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - let (content, size): (Vec, usize) = - gen_bytes_with_range(10 * 1024 * 1024..20 * 1024 * 1024); - - let mut w = op.writer_with(&path).buffer(8 * 1024 * 1024).await?; - - // Wrap a buf reader here to make sure content is read in 1MiB chunks. - let mut cursor = BufReader::with_capacity(1024 * 1024, Cursor::new(content.clone())); - futures::io::copy_buf(&mut cursor, &mut w).await?; - w.close().await?; - - let meta = op.stat(&path).await.expect("stat must succeed"); - assert_eq!(meta.content_length(), size as u64); - - let bs = op.read(&path).await?; - assert_eq!(bs.len(), size, "read size"); - assert_eq!( - format!("{:x}", Sha256::digest(&bs[..size])), - format!("{:x}", Sha256::digest(content)), - "read content" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// Add test for unsized writer -pub async fn test_fuzz_unsized_writer(op: Operator) -> Result<()> { - if !op.info().full_capability().write_can_multi { - warn!("{op:?} doesn't support write without content length, test skip"); - return Ok(()); - } - - let path = uuid::Uuid::new_v4().to_string(); - - let mut fuzzer = ObjectWriterFuzzer::new(&path, None); - - let mut w = op.writer_with(&path).buffer(8 * 1024 * 1024).await?; - - for _ in 0..100 { - match fuzzer.fuzz() { - ObjectWriterAction::Write(bs) => w.write(bs).await?, - } - } - w.close().await?; - - let content = op.read(&path).await?; - fuzzer.check(&content); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -} - -/// seeking a negative position should return a InvalidInput error -pub async fn test_invalid_reader_seek(op: Operator) -> Result<()> { - let path = uuid::Uuid::new_v4().to_string(); - debug!("Generate a random file: {}", &path); - let (content, _) = gen_bytes(op.info().full_capability()); - - op.write(&path, content.clone()) - .await - .expect("write must succeed"); - - let mut r = op.reader(&path).await?; - let res = r.seek(std::io::SeekFrom::Current(-1024)).await; - - assert!(res.is_err()); - - assert_eq!( - res.unwrap_err().kind(), - std::io::ErrorKind::InvalidInput, - "seeking a negative position should return a InvalidInput error" - ); - - op.delete(&path).await.expect("delete must succeed"); - Ok(()) -}