From 242f42ce38fdf79c0806420e35bef182f3e2625a Mon Sep 17 00:00:00 2001 From: Antonio Yang Date: Thu, 11 Feb 2021 16:07:01 +0800 Subject: [PATCH 1/2] refact download pool --- Cargo.toml | 4 ++-- test_shell.sh | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9e3cd8d..f0a582c 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s3rs" -version = "0.4.5" +version = "0.4.6" authors = ["Antonio Yang "] description = "A s3 cli client with multi configs with diffent provider" keywords = ["S3", "Amazon", "CEPH", "AWS"] @@ -34,7 +34,7 @@ serde_json = "1.0" regex = "0.2" quick-xml = "0.12" colored = "1.6" -s3handler = "0.6.6" +s3handler = "0.7" clap = "2.33" hex = "0.4.2" blake2-rfc = "0.2" diff --git a/test_shell.sh b/test_shell.sh index c809bec..43518ba 100755 --- a/test_shell.sh +++ b/test_shell.sh @@ -9,6 +9,7 @@ set timeout 180 spawn rm -f /tmp/test spawn rm -f /tmp/test-orig +spawn cp README.md test spawn dd if=/dev/urandom bs=1024 count=11264 of=/tmp/test-orig spawn cargo run @@ -79,6 +80,7 @@ expect -re $prompt send "exit\r" expect "cya~" +spawn rm -f test spawn md5sum /tmp/test-orig /tmp/test interact From 2e769859e0b1fe6ef3c1ba7c8e0929ce02c29e90 Mon Sep 17 00:00:00 2001 From: Antonio Yang Date: Wed, 10 Feb 2021 11:10:38 +0800 Subject: [PATCH 2/2] async feature --- Cargo.toml | 12 ++++++++-- src/command/mod.rs | 56 ++++++++++++++++++++++++++++++++++++++++++++++ test_command.sh | 3 ++- test_shell.sh | 10 ++++++++- 4 files changed, 77 insertions(+), 4 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f0a582c..11264e2 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "s3rs" -version = "0.4.6" +version = "0.4.7" authors = ["Antonio Yang "] description = "A s3 cli client with multi configs with diffent provider" keywords = ["S3", "Amazon", "CEPH", "AWS"] @@ -34,9 +34,17 @@ serde_json = "1.0" regex = "0.2" quick-xml = "0.12" colored = "1.6" -s3handler = "0.7" +s3handler = "0.7.1" clap = "2.33" hex = "0.4.2" blake2-rfc = "0.2" rand = "0.7" humansize = "1.1" +tokio = { version = "0.2", optional = true } + +[features] +default = ["async"] +async = [ + "s3handler/tokio-async", + "tokio" +] diff --git a/src/command/mod.rs b/src/command/mod.rs index aeca0ab..c150e2e 100644 --- a/src/command/mod.rs +++ b/src/command/mod.rs @@ -1,8 +1,17 @@ +#[cfg(feature = "async")] +use std::path::Path; +#[cfg(feature = "async")] +use tokio::runtime::Runtime; use regex::Regex; use humansize::{FileSize, file_size_opts}; use crate::logger::change_log_type; use colored::{self, *}; +#[cfg(feature = "async")] +use s3handler::{ + S3Object, + none_blocking::primitives::S3Pool +}; pub mod secret; @@ -206,6 +215,30 @@ pub fn do_command(handler: &mut s3handler::Handler, s3_type: &String, command: & } }; } else if command.starts_with("put") { + #[cfg(feature = "async")] + { + let mut rt = Runtime::new().unwrap(); + let file_path = command.split_whitespace().nth(1).unwrap_or(""); + let mut s3_object: S3Object = command.split_whitespace().nth(2).unwrap_or("").into(); + if s3_object.key.is_none() { + s3_object.key = Some( + Path::new(file_path) + .file_name() + .map(|s|format!("/{}",s.to_string_lossy())).unwrap_or_else(||"/".to_string()) + ) + } + let s3_pool = S3Pool::from(&*handler); + rt.block_on(async { + match s3_pool + .resource(s3_object) + .upload_file(file_path).await { + Err(e) => println!("{}", e), + Ok(_) => println!("upload completed"), + }; + }); + } + + #[cfg(not(feature = "async"))] match handler.put( command.split_whitespace().nth(1).unwrap_or(""), command.split_whitespace().nth(2).unwrap_or(""), @@ -214,6 +247,29 @@ pub fn do_command(handler: &mut s3handler::Handler, s3_type: &String, command: & Ok(_) => println!("upload completed"), }; } else if command.starts_with("get") { + #[cfg(feature = "async")] + { + let s3_pool = S3Pool::from(&*handler); + let s3_object: S3Object = command.split_whitespace().nth(1).unwrap_or("").into(); + if s3_object.key.is_none() { + println!("please specify the object you want to download"); + return; + } + + let mut rt = Runtime::new().unwrap(); + rt.block_on( + async { + match s3_pool + .resource(s3_object) + .download_file(command.split_whitespace().nth(2).unwrap_or("")).await { + Err(e) => println!("{}", e), + Ok(_) => println!("download completed"), + }; + } + ); + } + + #[cfg(not(feature = "async"))] match handler.get( command.split_whitespace().nth(1).unwrap_or(""), command.split_whitespace().nth(2), diff --git a/test_command.sh b/test_command.sh index c909eb2..f851516 100755 --- a/test_command.sh +++ b/test_command.sh @@ -9,6 +9,7 @@ set timeout 120 spawn dd if=/dev/urandom bs=1024 count=7000 of=/tmp/7M spawn cargo build +spawn cp README.md test expect $prompt spawn target/debug/s3rs --config=$config ls @@ -65,4 +66,4 @@ expect $prompt spawn md5sum /tmp/7M /tmp/7 interact - +spawn rm -f test diff --git a/test_shell.sh b/test_shell.sh index 43518ba..87a8b20 100755 --- a/test_shell.sh +++ b/test_shell.sh @@ -19,9 +19,15 @@ send $item\r expect -re $prompt send ls\r +expect -re $prompt +send "log debug\r" + expect -re $prompt send "put test s3://$bucket\r" +expect -re $prompt +send "log error\r" + expect -re $prompt send "ls s3://$bucket\r" @@ -79,8 +85,10 @@ send "get s3://$bucket/test-orig /tmp/test\r" expect -re $prompt send "exit\r" +send "rm -f test\r" + expect "cya~" spawn rm -f test spawn md5sum /tmp/test-orig /tmp/test - interact +spawn rm -f test