From b50d073c83382aa83066205f3c6a2bc50ff9082f Mon Sep 17 00:00:00 2001 From: durch Date: Sun, 1 Sep 2024 10:46:35 +0200 Subject: [PATCH] v0.36.0-alpha.3 + Removes `no_ssl_verify` feature, moves the functionality into Bucket configuration + Adds support for `reqwest::Proxy` with the `tokio` backend + Removes `futures_utils` and `futures_io` explicit dependencies --- s3/Cargo.toml | 26 +++-- s3/Makefile | 12 +- s3/src/bucket.rs | 167 +++++++++++++++++++++++----- s3/src/request/async_std_backend.rs | 4 +- s3/src/request/request_trait.rs | 4 +- s3/src/request/tokio_backend.rs | 32 ++++-- 6 files changed, 188 insertions(+), 57 deletions(-) diff --git a/s3/Cargo.toml b/s3/Cargo.toml index 786a31b209..b303ecae8a 100644 --- a/s3/Cargo.toml +++ b/s3/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "rust-s3" -version = "0.36.0-alpha.2" +version = "0.36.0-alpha.3" authors = ["Drazen Urch"] description = "Rust library for working with AWS S3 and compatible object storage APIs" repository = "https://github.com/durch/rust-s3" @@ -51,9 +51,7 @@ base64 = "0.22" block_on_proc = { version = "0.2", optional = true } bytes = { version = "1.2" } cfg-if = "1" -futures = { version = "0.3", optional = true } -futures-io = { version = "0.3", optional = true } -futures-util = { version = "0.3", optional = true, features = ["io"] } +futures = { version = "0.3", optional = true, default-features = false } hex = "0.4" hmac = "0.12" http = "1" @@ -63,15 +61,24 @@ md5 = "0.7" minidom = { version = "0.16", optional = true } percent-encoding = "2" quick-xml = { version = "0.36", features = ["serialize"] } -reqwest = { version = "0.12", optional = true, features = ["stream",], default-features = false } +reqwest = { version = "0.12", optional = true, features = [ + "stream", +], default-features = false } serde = "1" serde_derive = "1" serde_json = "1" sha2 = "0.10" -surf = { version = "2", optional = true, default-features = false, features = ["h1-client-rustls",] } +surf = { version = "2", optional = true, default-features = false, features = [ + "h1-client-rustls", +] } thiserror = "1" -time = { version = "^0.3.6", features = ["formatting", "macros"], default-features = false} -tokio = { version = "1", features = ["io-util",], optional = true, default-features = false } +time = { version = "^0.3.6", features = [ + "formatting", + "macros", +], default-features = false } +tokio = { version = "1", features = [ + "io-util", +], optional = true, default-features = false } tokio-native-tls = { version = "0.3", optional = true } tokio-rustls = { version = "0.26", optional = true } tokio-stream = { version = "0.1", optional = true } @@ -81,12 +88,11 @@ url = "2" default = ["tags", "tokio-native-tls", "fail-on-err"] sync = ["attohttpc", "maybe-async/is_sync"] -with-async-std = ["async-std", "surf", "futures-io", "futures-util", "futures"] +with-async-std = ["async-std", "surf", "futures"] with-tokio = ["reqwest", "tokio", "tokio/fs", "tokio-stream", "futures"] blocking = ["block_on_proc", "tokio/rt", "tokio/rt-multi-thread"] fail-on-err = [] -no-verify-ssl = [] tags = ["minidom"] http-credentials = ["aws-creds/http-credentials"] diff --git a/s3/Makefile b/s3/Makefile index d20fc18c90..d5c96de275 100644 --- a/s3/Makefile +++ b/s3/Makefile @@ -4,12 +4,12 @@ sync-all: sync-nativetls sync-rustlstls sync-nossl ci: clippy fmt-check clippy: tokio-clippy async-std-clippy sync-clippy -tokio-clippy: tokio-nativetls-clippy tokio-nossl-clippy tokio-noverify-clippy tokio-rustlstls-clippy +tokio-clippy: tokio-nativetls-clippy tokio-nossl-clippy tokio-rustlstls-clippy sync-clippy: sync-nativetls-clippy sync-nossl-clippy sync-rustlstls-clippy # tokio -tokio: tokio-nativetls tokio-nossl tokio-noverify tokio-rustlstls -tokio-not-ignored: tokio-nativetls-test-not-ignored tokio-nossl-test-not-ignored tokio-noverify-test-not-ignored tokio-rustlstls-test-not-ignored +tokio: tokio-nativetls tokio-nossl tokio-rustlstls +tokio-not-ignored: tokio-nativetls-test-not-ignored tokio-nossl-test-not-ignored-not-ignored tokio-rustlstls-test-not-ignored tokio-nativetls: tokio-nativetls-clippy tokio-nativetls-test tokio-nativetls-blocking-test-ignored tokio-nativetls-clippy: @@ -28,12 +28,6 @@ tokio-nossl-clippy: tokio-nossl-test-not-ignored: cargo test --no-default-features --features with-tokio --features aws-creds/http-credentials -tokio-noverify: tokio-noverify-test-not-ignored tokio-noverify-clippy -tokio-noverify-clippy: - cargo clippy --no-default-features --features with-tokio --features no-verify-ssl --features aws-creds/http-credentials -- -D warnings -tokio-noverify-test-not-ignored: - cargo test --no-default-features --features with-tokio --features no-verify-ssl --features aws-creds/http-credentials - tokio-rustlstls: tokio-rustlstls-test-not-ignored tokio-rustlstls-test-ignored tokio-rustlstls-clippy tokio-rustlstls-clippy: cargo clippy --no-default-features --features with-tokio --features tokio-rustls-tls --features aws-creds/http-credentials -- -D warnings diff --git a/s3/src/bucket.rs b/s3/src/bucket.rs index f14cf7014b..7a241a6c7d 100644 --- a/s3/src/bucket.rs +++ b/s3/src/bucket.rs @@ -9,8 +9,10 @@ use crate::bucket_ops::{BucketConfiguration, CreateBucketResponse}; use crate::command::{Command, Multipart}; use crate::creds::Credentials; use crate::region::Region; -#[cfg(any(feature = "with-tokio", feature = "tokio-native-tls"))] +#[cfg(feature = "with-tokio")] use crate::request::tokio_backend::client; +#[cfg(feature = "with-tokio")] +use crate::request::tokio_backend::ClientOptions; #[cfg(any(feature = "with-tokio", feature = "with-async-std"))] use crate::request::ResponseDataStream; use crate::request::{Request as _, ResponseData}; @@ -105,14 +107,10 @@ pub struct Bucket { pub request_timeout: Option, path_style: bool, listobjects_v2: bool, - #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] - http_client: Arc, - #[cfg(all( - feature = "with-tokio", - not(feature = "tokio-native-tls"), - not(feature = "tokio-rustls-tls") - ))] - http_client: Arc, + #[cfg(feature = "with-tokio")] + http_client: reqwest::Client, + #[cfg(feature = "with-tokio")] + client_options: crate::request::tokio_backend::ClientOptions, } impl Bucket { @@ -132,8 +130,8 @@ impl Bucket { } #[cfg(feature = "with-tokio")] - pub fn http_client(&self) -> Arc { - Arc::clone(&self.http_client) + pub fn http_client(&self) -> reqwest::Client { + self.http_client.clone() } } @@ -567,6 +565,9 @@ impl Bucket { region: Region, credentials: Credentials, ) -> Result, S3Error> { + #[cfg(feature = "with-tokio")] + let options = ClientOptions::default(); + Ok(Box::new(Bucket { name: name.into(), region, @@ -576,8 +577,10 @@ impl Bucket { request_timeout: DEFAULT_REQUEST_TIMEOUT, path_style: false, listobjects_v2: true, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: Arc::new(client(DEFAULT_REQUEST_TIMEOUT)?), + #[cfg(feature = "with-tokio")] + http_client: client(&options)?, + #[cfg(feature = "with-tokio")] + client_options: options, })) } @@ -593,6 +596,9 @@ impl Bucket { /// let bucket = Bucket::new_public(bucket_name, region).unwrap(); /// ``` pub fn new_public(name: &str, region: Region) -> Result { + #[cfg(feature = "with-tokio")] + let options = ClientOptions::default(); + Ok(Bucket { name: name.into(), region, @@ -602,8 +608,10 @@ impl Bucket { request_timeout: DEFAULT_REQUEST_TIMEOUT, path_style: false, listobjects_v2: true, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: Arc::new(client(DEFAULT_REQUEST_TIMEOUT)?), + #[cfg(feature = "with-tokio")] + http_client: client(&options)?, + #[cfg(feature = "with-tokio")] + client_options: options, }) } @@ -617,8 +625,10 @@ impl Bucket { request_timeout: self.request_timeout, path_style: true, listobjects_v2: self.listobjects_v2, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: self.http_client.clone(), + #[cfg(feature = "with-tokio")] + http_client: self.http_client(), + #[cfg(feature = "with-tokio")] + client_options: self.client_options.clone(), }) } @@ -632,8 +642,10 @@ impl Bucket { request_timeout: self.request_timeout, path_style: self.path_style, listobjects_v2: self.listobjects_v2, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: self.http_client.clone(), + #[cfg(feature = "with-tokio")] + http_client: self.http_client(), + #[cfg(feature = "with-tokio")] + client_options: self.client_options.clone(), }) } @@ -650,12 +662,34 @@ impl Bucket { request_timeout: self.request_timeout, path_style: self.path_style, listobjects_v2: self.listobjects_v2, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: self.http_client.clone(), + #[cfg(feature = "with-tokio")] + http_client: self.http_client(), + #[cfg(feature = "with-tokio")] + client_options: self.client_options.clone(), }) } + #[cfg(not(feature = "with-tokio"))] + pub fn with_request_timeout(&self, request_timeout: Duration) -> Result, S3Error> { + Ok(Box::new(Bucket { + name: self.name.clone(), + region: self.region.clone(), + credentials: self.credentials.clone(), + extra_headers: self.extra_headers.clone(), + extra_query: self.extra_query.clone(), + request_timeout: Some(request_timeout), + path_style: self.path_style, + listobjects_v2: self.listobjects_v2, + })) + } + + #[cfg(feature = "with-tokio")] pub fn with_request_timeout(&self, request_timeout: Duration) -> Result, S3Error> { + let options = ClientOptions { + request_timeout: Some(request_timeout), + ..Default::default() + }; + Ok(Box::new(Bucket { name: self.name.clone(), region: self.region.clone(), @@ -665,8 +699,10 @@ impl Bucket { request_timeout: Some(request_timeout), path_style: self.path_style, listobjects_v2: self.listobjects_v2, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: Arc::new(client(Some(request_timeout))?), + #[cfg(feature = "with-tokio")] + http_client: client(&options)?, + #[cfg(feature = "with-tokio")] + client_options: options, })) } @@ -680,11 +716,88 @@ impl Bucket { request_timeout: self.request_timeout, path_style: self.path_style, listobjects_v2: false, - #[cfg(any(feature = "tokio-native-tls", feature = "with-tokio"))] - http_client: self.http_client.clone(), + #[cfg(feature = "with-tokio")] + http_client: self.http_client(), + #[cfg(feature = "with-tokio")] + client_options: self.client_options.clone(), } } + /// Configures a bucket to accept invalid SSL certificates and hostnames. + /// + /// This method is available only when either the `tokio-native-tls` or `tokio-rustls-tls` feature is enabled. + /// + /// # Parameters + /// + /// - `accept_invalid_certs`: A boolean flag that determines whether the client should accept invalid SSL certificates. + /// - `accept_invalid_hostnames`: A boolean flag that determines whether the client should accept invalid hostnames. + /// + /// # Returns + /// + /// Returns a `Result` containing the newly configured `Bucket` instance if successful, or an `S3Error` if an error occurs during client configuration. + /// + /// # Errors + /// + /// This function returns an `S3Error` if the HTTP client configuration fails. + /// + /// # Example + /// + /// ```rust + /// # use s3::bucket::Bucket; + /// # use s3::error::S3Error; + /// # use s3::creds::Credentials; + /// # use s3::Region; + /// # use std::str::FromStr; + /// + /// # fn example() -> Result<(), S3Error> { + /// let bucket = Bucket::new("my-bucket", Region::from_str("us-east-1")?, Credentials::default()?)? + /// .set_dangereous_config(true, true)?; + /// # Ok(()) + /// # } + /// + #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] + pub fn set_dangereous_config( + &self, + accept_invalid_certs: bool, + accept_invalid_hostnames: bool, + ) -> Result { + let mut options = self.client_options.clone(); + options.accept_invalid_certs = accept_invalid_certs; + options.accept_invalid_hostnames = accept_invalid_hostnames; + + Ok(Bucket { + name: self.name.clone(), + region: self.region.clone(), + credentials: self.credentials.clone(), + extra_headers: self.extra_headers.clone(), + extra_query: self.extra_query.clone(), + request_timeout: self.request_timeout, + path_style: self.path_style, + listobjects_v2: self.listobjects_v2, + http_client: client(&options)?, + client_options: options, + }) + } + + #[cfg(feature = "with-tokio")] + pub fn set_proxy(&self, proxy: reqwest::Proxy) -> Result { + let mut options = self.client_options.clone(); + options.proxy = Some(proxy); + + Ok(Bucket { + name: self.name.clone(), + region: self.region.clone(), + credentials: self.credentials.clone(), + extra_headers: self.extra_headers.clone(), + extra_query: self.extra_query.clone(), + request_timeout: self.request_timeout, + path_style: self.path_style, + listobjects_v2: self.listobjects_v2, + http_client: client(&options)?, + client_options: options, + }) + } + /// Copy file from an S3 path, internally within the same bucket. /// /// # Example: @@ -1056,9 +1169,9 @@ impl Bucket { /// #[cfg(feature = "with-tokio")] /// use tokio::io::AsyncWriteExt; /// #[cfg(feature = "with-async-std")] - /// use futures_util::StreamExt; + /// use async_std::stream::StreamExt; /// #[cfg(feature = "with-async-std")] - /// use futures_util::AsyncWriteExt; + /// use async_std::io::WriteExt; /// /// # #[tokio::main] /// # async fn main() -> Result<()> { diff --git a/s3/src/request/async_std_backend.rs b/s3/src/request/async_std_backend.rs index 7609c71c46..5a932c0c41 100644 --- a/s3/src/request/async_std_backend.rs +++ b/s3/src/request/async_std_backend.rs @@ -1,8 +1,8 @@ +use async_std::io::Write as AsyncWrite; use async_std::io::{ReadExt, WriteExt}; use async_std::stream::StreamExt; use bytes::Bytes; -use futures_io::AsyncWrite; -use futures_util::FutureExt; +use futures::FutureExt; use std::collections::HashMap; use crate::bucket::Bucket; diff --git a/s3/src/request/request_trait.rs b/s3/src/request/request_trait.rs index 0b59da720b..e80f0f0572 100644 --- a/s3/src/request/request_trait.rs +++ b/s3/src/request/request_trait.rs @@ -22,7 +22,7 @@ use http::HeaderMap; use std::fmt::Write as _; #[cfg(feature = "with-async-std")] -use futures_util::Stream; +use async_std::stream::Stream; #[cfg(feature = "with-tokio")] use tokio_stream::Stream; @@ -132,7 +132,7 @@ pub trait Request { writer: &mut T, ) -> Result; #[cfg(feature = "with-async-std")] - async fn response_data_to_writer( + async fn response_data_to_writer( &self, writer: &mut T, ) -> Result; diff --git a/s3/src/request/tokio_backend.rs b/s3/src/request/tokio_backend.rs index d833b99cd8..9d40db3ed7 100644 --- a/s3/src/request/tokio_backend.rs +++ b/s3/src/request/tokio_backend.rs @@ -17,23 +17,41 @@ use crate::utils::now_utc; use tokio_stream::StreamExt; +#[derive(Clone, Debug, Default)] +pub(crate) struct ClientOptions { + pub request_timeout: Option, + pub proxy: Option, + #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] + pub accept_invalid_certs: bool, + #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] + pub accept_invalid_hostnames: bool, +} + #[cfg(feature = "with-tokio")] -pub fn client(request_timeout: Option) -> Result { +pub(crate) fn client(options: &ClientOptions) -> Result { let client = reqwest::Client::builder(); - let client = if let Some(timeout) = request_timeout { + let client = if let Some(timeout) = options.request_timeout { client.timeout(timeout) } else { client }; + let client = if let Some(ref proxy) = options.proxy { + client.proxy(proxy.clone()) + } else { + client + }; + cfg_if::cfg_if! { if #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] { - let client = if cfg!(feature = "no-verify-ssl") { - client.danger_accept_invalid_certs(true) - } else { - client - }; + let client = client.danger_accept_invalid_certs(options.accept_invalid_certs); + } + } + + cfg_if::cfg_if! { + if #[cfg(any(feature = "tokio-native-tls", feature = "tokio-rustls-tls"))] { + let client = client.danger_accept_invalid_hostnames(options.accept_invalid_hostnames); } }