diff --git a/README.md b/README.md index f2ce58d..f97e8c5 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,10 @@ EOF bench: v0.2.6 Processing time: 4457.868372796s v0.2.7 Processing time: 4441.127406732s +v0.2.8 Processing time: 4650.917674426s - https://github.com/hyperium/hyper/issues/1358 - https://github.com/hyperium/hyper/issues/1358#issuecomment-366550636 +- https://gist.github.com/klausi/f94b9aff7d36a1cb4ebbca746f0a099f - https://gist.github.com/mustafaturan/47268d8ad6d56cadda357e4c438f51ca diff --git a/hfd-cli/Cargo.toml b/hfd-cli/Cargo.toml index 279491e..8cce2ed 100644 --- a/hfd-cli/Cargo.toml +++ b/hfd-cli/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "hfd-cli" -version = "0.2.8" +version = "0.2.9" edition = "2021" [dependencies] diff --git a/hfd-cli/src/lib.rs b/hfd-cli/src/lib.rs index 9e8ef50..795174f 100644 --- a/hfd-cli/src/lib.rs +++ b/hfd-cli/src/lib.rs @@ -105,6 +105,36 @@ async fn download_chunk( Ok(()) } +async fn download_chunk_with_retry( + headers: HeaderMap, + url: String, + path: PathBuf, + s: usize, + e: usize, +) -> Result<(), Box> { + const MAX_RETRIES: usize = 100; + let mut retries = 0; + + let mut error: Option> = None; + + while retries < MAX_RETRIES { + match download_chunk(headers.clone(), url.clone(), path.clone(), s, e).await { + Ok(_) => return Ok(()), + Err(e) => { + println!("Retry {:#?} with {:?} times", e, retries); + error = Some(e); + retries += 1; + std::thread::sleep(Duration::from_millis(10)); + } + } + } + + Err(error.unwrap_or_else(|| Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + "Exhausted all retries", + )))) +} + async fn download( headers: HeaderMap, url: String, @@ -135,14 +165,20 @@ async fn download( for s in (0..length).step_by(chunk_size) { let e = std::cmp::min(s + chunk_size - 1, length); - tasks.push(download_chunk(headers.clone(), url.clone(), path.clone(), s, e)); + tasks.push(download_chunk_with_retry(headers.clone(), url.clone(), path.clone(), s, e)); } while let Some(handle) = tasks.next().await { let res = match handle { - Ok(socket) => socket, - Err(e) => println!("{:?}", e), - }; + Ok(socket) => { + socket + }, + Err(e) => { + println!("Chunk Error {:#?}", e); + std::thread::sleep(Duration::from_millis(10)); + continue; + } + }; } Ok(()) } @@ -235,9 +271,13 @@ impl HfClient { while let Some(handle) = tasks.next().await { let res = match handle { - Ok(socket) => socket, - Err(e) => println!("{:?}", e), - }; + Ok(socket) => socket, + Err(e) => { + println!("File Error {:#?}", e); + std::thread::sleep(Duration::from_millis(10)); + continue; + } + }; } } Ok(())