From 3dde35b1cff07bf7be6f2ff46b51dc33f9ee577b Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 10:12:49 +0800 Subject: [PATCH 1/6] fix(services/fs,hdfs): fix poll_close when retry --- .github/workflows/test_edge.yml | 25 +++++++++ core/Cargo.lock | 10 ++++ .../Cargo.toml | 30 +++++++++++ .../README.md | 14 +++++ .../src/main.rs | 48 +++++++++++++++++ core/src/services/fs/writer.rs | 51 ++++++++++++++----- core/src/services/hdfs/writer.rs | 22 +++++--- 7 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 core/edge/file_close_with_retry_on_full_disk/Cargo.toml create mode 100644 core/edge/file_close_with_retry_on_full_disk/README.md create mode 100644 core/edge/file_close_with_retry_on_full_disk/src/main.rs diff --git a/.github/workflows/test_edge.yml b/.github/workflows/test_edge.yml index a09f1de8b6d..c8888ee1e1b 100644 --- a/.github/workflows/test_edge.yml +++ b/.github/workflows/test_edge.yml @@ -33,6 +33,31 @@ on: - ".github/workflows/edge_test.yml" jobs: + test_file_close_with_retry_on_full_disk: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create disk image + run: | + fallocate -l 512K disk.img + mkfs disk.img + + - name: Mount disk image + run: | + mkdir /tmp/test_dir + sudo mount -o loop disk.img /tmp/test_dir + + - name: Set permissions + run: sudo chmod a+wr /tmp/test_dir + + - name: Test + working-directory: core/edge/file_close_with_retry_on_full_disk + run: cargo run + env: + OPENDAL_FS_ROOT: /tmp/test_dir test_file_write_on_full_disk: runs-on: ubuntu-latest diff --git a/core/Cargo.lock b/core/Cargo.lock index fbaaf3f2a8a..e087911ad03 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2003,6 +2003,16 @@ dependencies = [ "signature 1.6.4", ] +[[package]] +name = "edge_file_close_with_retry_on_full_disk" +version = "0.0.0" +dependencies = [ + "futures", + "opendal", + "rand 0.8.5", + "tokio", +] + [[package]] name = "edge_test_aws_s3_assume_role_with_web_identity" version = "0.0.0" diff --git a/core/edge/file_close_with_retry_on_full_disk/Cargo.toml b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml new file mode 100644 index 00000000000..bdc2a648b85 --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = "2021" +license = "Apache-2.0" +name = "edge_file_close_with_retry_on_full_disk" +publish = false +rust-version = "1.67" +version = "0.0.0" + +[dependencies] +futures = "0.3" +opendal = { path = "../../" } +rand = "0.8" +tokio = { version = "1", features = ["full"] } diff --git a/core/edge/file_close_with_retry_on_full_disk/README.md b/core/edge/file_close_with_retry_on_full_disk/README.md new file mode 100644 index 00000000000..51cc9641afd --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/README.md @@ -0,0 +1,14 @@ +# File Close with Retry on Full Disk + +Reported by [fs::Writer::poll_close can't be retried multiple times when error occurs](https://github.com/apache/opendal/issues/4058). + +Setup: + +```shell +fallocate -l 512K disk.img +mkfs disk.img +mkdir ./td +sudo mount -o loop td.img ./td +chmod a+wr ./td +``` + diff --git a/core/edge/file_close_with_retry_on_full_disk/src/main.rs b/core/edge/file_close_with_retry_on_full_disk/src/main.rs new file mode 100644 index 00000000000..abadfab1bc7 --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/src/main.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::env; + +use opendal::layers::RetryLayer; +use opendal::services::Fs; +use opendal::Operator; +use opendal::Result; +use rand::prelude::*; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Fs::default(); + builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); + let op = Operator::new(builder)? + .layer(RetryLayer::new().with_max_times(3)) + .finish(); + + let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024); + let mut bs = vec![0; size]; + thread_rng().fill_bytes(&mut bs); + + let mut w = op.writer("/test").await?; + w.write(bs).await?; + let result = w.close().await; + // Write file with size > 512KB should fail + assert!( + result.is_err(), + "close file with retry on full disk should return error" + ); + + Ok(()) +} diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs index 12e5a4fffe7..a8ebb4fd3c3 100644 --- a/core/src/services/fs/writer.rs +++ b/core/src/services/fs/writer.rs @@ -35,7 +35,7 @@ pub struct FsWriter { tmp_path: Option, f: Option, - fut: Option>>, + fut: Option)>>, } impl FsWriter { @@ -69,23 +69,35 @@ impl oio::Write for FsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } let mut f = self.f.take().expect("FsWriter must be initialized"); let tmp_path = self.tmp_path.clone(); let target_path = self.target_path.clone(); self.fut = Some(Box::pin(async move { - f.flush().await.map_err(new_std_io_error)?; - f.sync_all().await.map_err(new_std_io_error)?; + if let Err(e) = f.flush().await.map_err(new_std_io_error) { + // Reserve the original error for retry. + return (f, Err(e)); + } + if let Err(e) = f.sync_all().await.map_err(new_std_io_error) { + return (f, Err(e)); + } if let Some(tmp_path) = &tmp_path { - tokio::fs::rename(tmp_path, &target_path) + if let Err(e) = tokio::fs::rename(tmp_path, &target_path) .await - .map_err(new_std_io_error)?; + .map_err(new_std_io_error) + { + return (f, Err(e)); + } } - Ok(()) + (f, Ok(())) })); } } @@ -95,21 +107,32 @@ impl oio::Write for FsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } - let _ = self.f.take().expect("FsWriter must be initialized"); + let f = self.f.take().expect("FsWriter must be initialized"); let tmp_path = self.tmp_path.clone(); self.fut = Some(Box::pin(async move { if let Some(tmp_path) = &tmp_path { - tokio::fs::remove_file(tmp_path) + if let Err(e) = tokio::fs::remove_file(tmp_path) .await .map_err(new_std_io_error) + { + return (f, Err(e)); + } + (f, Ok(())) } else { - Err(Error::new( - ErrorKind::Unsupported, - "Fs doesn't support abort if atomic_write_dir is not set", - )) + ( + f, + Err(Error::new( + ErrorKind::Unsupported, + "Fs doesn't support abort if atomic_write_dir is not set", + )), + ) } })); } diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs index 6c77097d842..535e97c38b5 100644 --- a/core/src/services/hdfs/writer.rs +++ b/core/src/services/hdfs/writer.rs @@ -36,7 +36,7 @@ pub struct HdfsWriter { tmp_path: Option, f: Option, client: Arc, - fut: Option>>, + fut: Option)>>, } /// # Safety @@ -76,7 +76,11 @@ impl oio::Write for HdfsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } let mut f = self.f.take().expect("HdfsWriter must be initialized"); @@ -86,15 +90,21 @@ impl oio::Write for HdfsWriter { let client = self.client.clone(); self.fut = Some(Box::pin(async move { - f.close().await.map_err(new_std_io_error)?; + if let Err(e) = f.close().await.map_err(new_std_io_error) { + // Reserve the original error for retry. + return (f, Err(e)); + } if let Some(tmp_path) = tmp_path { - client + if let Err(e) = client .rename_file(&tmp_path, &target_path) - .map_err(new_std_io_error)?; + .map_err(new_std_io_error) + { + return (f, Err(e)); + } } - Ok(()) + (f, Ok(())) })); } } From 6f1b80ca7652e39c9355705d3099e64273078436 Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 10:12:49 +0800 Subject: [PATCH 2/6] fix(services/fs,hdfs): fix poll_close when retry --- .github/workflows/test_edge.yml | 25 +++++++++ core/Cargo.lock | 10 ++++ .../Cargo.toml | 30 +++++++++++ .../README.md | 14 +++++ .../src/main.rs | 48 +++++++++++++++++ core/src/services/fs/writer.rs | 51 ++++++++++++++----- core/src/services/hdfs/writer.rs | 22 +++++--- 7 files changed, 180 insertions(+), 20 deletions(-) create mode 100644 core/edge/file_close_with_retry_on_full_disk/Cargo.toml create mode 100644 core/edge/file_close_with_retry_on_full_disk/README.md create mode 100644 core/edge/file_close_with_retry_on_full_disk/src/main.rs diff --git a/.github/workflows/test_edge.yml b/.github/workflows/test_edge.yml index a09f1de8b6d..c8888ee1e1b 100644 --- a/.github/workflows/test_edge.yml +++ b/.github/workflows/test_edge.yml @@ -33,6 +33,31 @@ on: - ".github/workflows/edge_test.yml" jobs: + test_file_close_with_retry_on_full_disk: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create disk image + run: | + fallocate -l 512K disk.img + mkfs disk.img + + - name: Mount disk image + run: | + mkdir /tmp/test_dir + sudo mount -o loop disk.img /tmp/test_dir + + - name: Set permissions + run: sudo chmod a+wr /tmp/test_dir + + - name: Test + working-directory: core/edge/file_close_with_retry_on_full_disk + run: cargo run + env: + OPENDAL_FS_ROOT: /tmp/test_dir test_file_write_on_full_disk: runs-on: ubuntu-latest diff --git a/core/Cargo.lock b/core/Cargo.lock index fbaaf3f2a8a..e087911ad03 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2003,6 +2003,16 @@ dependencies = [ "signature 1.6.4", ] +[[package]] +name = "edge_file_close_with_retry_on_full_disk" +version = "0.0.0" +dependencies = [ + "futures", + "opendal", + "rand 0.8.5", + "tokio", +] + [[package]] name = "edge_test_aws_s3_assume_role_with_web_identity" version = "0.0.0" diff --git a/core/edge/file_close_with_retry_on_full_disk/Cargo.toml b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml new file mode 100644 index 00000000000..bdc2a648b85 --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml @@ -0,0 +1,30 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +[package] +edition = "2021" +license = "Apache-2.0" +name = "edge_file_close_with_retry_on_full_disk" +publish = false +rust-version = "1.67" +version = "0.0.0" + +[dependencies] +futures = "0.3" +opendal = { path = "../../" } +rand = "0.8" +tokio = { version = "1", features = ["full"] } diff --git a/core/edge/file_close_with_retry_on_full_disk/README.md b/core/edge/file_close_with_retry_on_full_disk/README.md new file mode 100644 index 00000000000..51cc9641afd --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/README.md @@ -0,0 +1,14 @@ +# File Close with Retry on Full Disk + +Reported by [fs::Writer::poll_close can't be retried multiple times when error occurs](https://github.com/apache/opendal/issues/4058). + +Setup: + +```shell +fallocate -l 512K disk.img +mkfs disk.img +mkdir ./td +sudo mount -o loop td.img ./td +chmod a+wr ./td +``` + diff --git a/core/edge/file_close_with_retry_on_full_disk/src/main.rs b/core/edge/file_close_with_retry_on_full_disk/src/main.rs new file mode 100644 index 00000000000..abadfab1bc7 --- /dev/null +++ b/core/edge/file_close_with_retry_on_full_disk/src/main.rs @@ -0,0 +1,48 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::env; + +use opendal::layers::RetryLayer; +use opendal::services::Fs; +use opendal::Operator; +use opendal::Result; +use rand::prelude::*; + +#[tokio::main] +async fn main() -> Result<()> { + let mut builder = Fs::default(); + builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); + let op = Operator::new(builder)? + .layer(RetryLayer::new().with_max_times(3)) + .finish(); + + let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024); + let mut bs = vec![0; size]; + thread_rng().fill_bytes(&mut bs); + + let mut w = op.writer("/test").await?; + w.write(bs).await?; + let result = w.close().await; + // Write file with size > 512KB should fail + assert!( + result.is_err(), + "close file with retry on full disk should return error" + ); + + Ok(()) +} diff --git a/core/src/services/fs/writer.rs b/core/src/services/fs/writer.rs index 12e5a4fffe7..a8ebb4fd3c3 100644 --- a/core/src/services/fs/writer.rs +++ b/core/src/services/fs/writer.rs @@ -35,7 +35,7 @@ pub struct FsWriter { tmp_path: Option, f: Option, - fut: Option>>, + fut: Option)>>, } impl FsWriter { @@ -69,23 +69,35 @@ impl oio::Write for FsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } let mut f = self.f.take().expect("FsWriter must be initialized"); let tmp_path = self.tmp_path.clone(); let target_path = self.target_path.clone(); self.fut = Some(Box::pin(async move { - f.flush().await.map_err(new_std_io_error)?; - f.sync_all().await.map_err(new_std_io_error)?; + if let Err(e) = f.flush().await.map_err(new_std_io_error) { + // Reserve the original error for retry. + return (f, Err(e)); + } + if let Err(e) = f.sync_all().await.map_err(new_std_io_error) { + return (f, Err(e)); + } if let Some(tmp_path) = &tmp_path { - tokio::fs::rename(tmp_path, &target_path) + if let Err(e) = tokio::fs::rename(tmp_path, &target_path) .await - .map_err(new_std_io_error)?; + .map_err(new_std_io_error) + { + return (f, Err(e)); + } } - Ok(()) + (f, Ok(())) })); } } @@ -95,21 +107,32 @@ impl oio::Write for FsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } - let _ = self.f.take().expect("FsWriter must be initialized"); + let f = self.f.take().expect("FsWriter must be initialized"); let tmp_path = self.tmp_path.clone(); self.fut = Some(Box::pin(async move { if let Some(tmp_path) = &tmp_path { - tokio::fs::remove_file(tmp_path) + if let Err(e) = tokio::fs::remove_file(tmp_path) .await .map_err(new_std_io_error) + { + return (f, Err(e)); + } + (f, Ok(())) } else { - Err(Error::new( - ErrorKind::Unsupported, - "Fs doesn't support abort if atomic_write_dir is not set", - )) + ( + f, + Err(Error::new( + ErrorKind::Unsupported, + "Fs doesn't support abort if atomic_write_dir is not set", + )), + ) } })); } diff --git a/core/src/services/hdfs/writer.rs b/core/src/services/hdfs/writer.rs index 6c77097d842..535e97c38b5 100644 --- a/core/src/services/hdfs/writer.rs +++ b/core/src/services/hdfs/writer.rs @@ -36,7 +36,7 @@ pub struct HdfsWriter { tmp_path: Option, f: Option, client: Arc, - fut: Option>>, + fut: Option)>>, } /// # Safety @@ -76,7 +76,11 @@ impl oio::Write for HdfsWriter { if let Some(fut) = self.fut.as_mut() { let res = ready!(fut.poll_unpin(cx)); self.fut = None; - return Poll::Ready(res); + if let Err(e) = res.1 { + self.f = Some(res.0); + return Poll::Ready(Err(e)); + } + return Poll::Ready(Ok(())); } let mut f = self.f.take().expect("HdfsWriter must be initialized"); @@ -86,15 +90,21 @@ impl oio::Write for HdfsWriter { let client = self.client.clone(); self.fut = Some(Box::pin(async move { - f.close().await.map_err(new_std_io_error)?; + if let Err(e) = f.close().await.map_err(new_std_io_error) { + // Reserve the original error for retry. + return (f, Err(e)); + } if let Some(tmp_path) = tmp_path { - client + if let Err(e) = client .rename_file(&tmp_path, &target_path) - .map_err(new_std_io_error)?; + .map_err(new_std_io_error) + { + return (f, Err(e)); + } } - Ok(()) + (f, Ok(())) })); } } From ca947b4b1adae01beb518d589dd6dadaa168ff0c Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 13:01:17 +0800 Subject: [PATCH 3/6] fix(services/fs,hdfs): fix poll_close when retry --- .github/workflows/test_edge.yml | 52 +++++++++++++++++---------------- core/Cargo.lock | 10 ------- 2 files changed, 27 insertions(+), 35 deletions(-) diff --git a/.github/workflows/test_edge.yml b/.github/workflows/test_edge.yml index c8888ee1e1b..6056d768a84 100644 --- a/.github/workflows/test_edge.yml +++ b/.github/workflows/test_edge.yml @@ -33,31 +33,6 @@ on: - ".github/workflows/edge_test.yml" jobs: - test_file_close_with_retry_on_full_disk: - runs-on: ubuntu-latest - - steps: - - name: Checkout code - uses: actions/checkout@v4 - - - name: Create disk image - run: | - fallocate -l 512K disk.img - mkfs disk.img - - - name: Mount disk image - run: | - mkdir /tmp/test_dir - sudo mount -o loop disk.img /tmp/test_dir - - - name: Set permissions - run: sudo chmod a+wr /tmp/test_dir - - - name: Test - working-directory: core/edge/file_close_with_retry_on_full_disk - run: cargo run - env: - OPENDAL_FS_ROOT: /tmp/test_dir test_file_write_on_full_disk: runs-on: ubuntu-latest @@ -162,3 +137,30 @@ jobs: OPENDAL_S3_BUCKET: opendal-testing OPENDAL_S3_ROLE_ARN: arn:aws:iam::952853449216:role/opendal-testing OPENDAL_S3_REGION: ap-northeast-1 + + test_file_close_with_retry_on_full_disk: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Create disk image + run: | + fallocate -l 512K disk.img + mkfs disk.img + + - name: Mount disk image + run: | + mkdir /tmp/test_dir + sudo mount -o loop disk.img /tmp/test_dir + + - name: Set permissions + run: sudo chmod a+wr /tmp/test_dir + + - name: Test + working-directory: core/edge/file_close_with_retry_on_full_disk + run: cargo run + env: + OPENDAL_FS_ROOT: /tmp/test_dir + RUST_BACKTRACE: 1 diff --git a/core/Cargo.lock b/core/Cargo.lock index e087911ad03..fbaaf3f2a8a 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2003,16 +2003,6 @@ dependencies = [ "signature 1.6.4", ] -[[package]] -name = "edge_file_close_with_retry_on_full_disk" -version = "0.0.0" -dependencies = [ - "futures", - "opendal", - "rand 0.8.5", - "tokio", -] - [[package]] name = "edge_test_aws_s3_assume_role_with_web_identity" version = "0.0.0" From d1b400af53c916e8f7bbc1add1c811eb70d5ed0b Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 13:13:03 +0800 Subject: [PATCH 4/6] fix(services/fs,hdfs): fix poll_close when retry --- .github/workflows/test_edge.yml | 1 + core/Cargo.lock | 10 ++++++++++ .../file_close_with_retry_on_full_disk/src/main.rs | 2 ++ 3 files changed, 13 insertions(+) diff --git a/.github/workflows/test_edge.yml b/.github/workflows/test_edge.yml index 6056d768a84..ae0ddac30d7 100644 --- a/.github/workflows/test_edge.yml +++ b/.github/workflows/test_edge.yml @@ -164,3 +164,4 @@ jobs: env: OPENDAL_FS_ROOT: /tmp/test_dir RUST_BACKTRACE: 1 + RUST_LOG: debug diff --git a/core/Cargo.lock b/core/Cargo.lock index fbaaf3f2a8a..e087911ad03 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2003,6 +2003,16 @@ dependencies = [ "signature 1.6.4", ] +[[package]] +name = "edge_file_close_with_retry_on_full_disk" +version = "0.0.0" +dependencies = [ + "futures", + "opendal", + "rand 0.8.5", + "tokio", +] + [[package]] name = "edge_test_aws_s3_assume_role_with_web_identity" version = "0.0.0" diff --git a/core/edge/file_close_with_retry_on_full_disk/src/main.rs b/core/edge/file_close_with_retry_on_full_disk/src/main.rs index abadfab1bc7..c29d9ae8890 100644 --- a/core/edge/file_close_with_retry_on_full_disk/src/main.rs +++ b/core/edge/file_close_with_retry_on_full_disk/src/main.rs @@ -17,6 +17,7 @@ use std::env; +use opendal::layers::LoggingLayer; use opendal::layers::RetryLayer; use opendal::services::Fs; use opendal::Operator; @@ -29,6 +30,7 @@ async fn main() -> Result<()> { builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); let op = Operator::new(builder)? .layer(RetryLayer::new().with_max_times(3)) + .layer(LoggingLayer::default()) .finish(); let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024); From ea53221dc0d2f8ddcff3dbd8f1b6cf064deacadf Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 13:36:16 +0800 Subject: [PATCH 5/6] fix(services/fs,hdfs): fix poll_close when retry --- core/Cargo.lock | 1 + core/edge/file_close_with_retry_on_full_disk/Cargo.toml | 1 + core/edge/file_close_with_retry_on_full_disk/src/main.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/core/Cargo.lock b/core/Cargo.lock index e087911ad03..ca77fed5bd9 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2011,6 +2011,7 @@ dependencies = [ "opendal", "rand 0.8.5", "tokio", + "tracing-subscriber", ] [[package]] diff --git a/core/edge/file_close_with_retry_on_full_disk/Cargo.toml b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml index bdc2a648b85..79fb4c2016d 100644 --- a/core/edge/file_close_with_retry_on_full_disk/Cargo.toml +++ b/core/edge/file_close_with_retry_on_full_disk/Cargo.toml @@ -28,3 +28,4 @@ futures = "0.3" opendal = { path = "../../" } rand = "0.8" tokio = { version = "1", features = ["full"] } +tracing-subscriber = "0.3" \ No newline at end of file diff --git a/core/edge/file_close_with_retry_on_full_disk/src/main.rs b/core/edge/file_close_with_retry_on_full_disk/src/main.rs index c29d9ae8890..bb14636865d 100644 --- a/core/edge/file_close_with_retry_on_full_disk/src/main.rs +++ b/core/edge/file_close_with_retry_on_full_disk/src/main.rs @@ -23,9 +23,11 @@ use opendal::services::Fs; use opendal::Operator; use opendal::Result; use rand::prelude::*; +use tracing_subscriber; #[tokio::main] async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); let mut builder = Fs::default(); builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); let op = Operator::new(builder)? From 86583a3c13dbe088b76f09570643aaa04bd0285c Mon Sep 17 00:00:00 2001 From: hoslo Date: Sun, 4 Feb 2024 13:44:24 +0800 Subject: [PATCH 6/6] fix(services/fs,hdfs): fix poll_close when retry --- .github/workflows/test_edge.yml | 2 +- core/edge/file_close_with_retry_on_full_disk/src/main.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_edge.yml b/.github/workflows/test_edge.yml index ae0ddac30d7..673d2c9c0e1 100644 --- a/.github/workflows/test_edge.yml +++ b/.github/workflows/test_edge.yml @@ -164,4 +164,4 @@ jobs: env: OPENDAL_FS_ROOT: /tmp/test_dir RUST_BACKTRACE: 1 - RUST_LOG: debug + RUST_LOG: trace diff --git a/core/edge/file_close_with_retry_on_full_disk/src/main.rs b/core/edge/file_close_with_retry_on_full_disk/src/main.rs index bb14636865d..d23fec56c85 100644 --- a/core/edge/file_close_with_retry_on_full_disk/src/main.rs +++ b/core/edge/file_close_with_retry_on_full_disk/src/main.rs @@ -31,8 +31,8 @@ async fn main() -> Result<()> { let mut builder = Fs::default(); builder.root(&env::var("OPENDAL_FS_ROOT").expect("root must be set for this test")); let op = Operator::new(builder)? - .layer(RetryLayer::new().with_max_times(3)) .layer(LoggingLayer::default()) + .layer(RetryLayer::new().with_max_times(3)) .finish(); let size = thread_rng().gen_range(512 * 1024 + 1..4 * 1024 * 1024);