diff --git a/core/Cargo.lock b/core/Cargo.lock index d25c6e29f74..8149ddc7a2f 100644 --- a/core/Cargo.lock +++ b/core/Cargo.lock @@ -2637,6 +2637,17 @@ dependencies = [ "rtrb", ] +[[package]] +name = "fastrace-jaeger" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290bef417c054e1780b06518808326c49129667e6907840fb583c7f340a98614" +dependencies = [ + "fastrace", + "log", + "thrift_codec", +] + [[package]] name = "fastrace-macro" version = "0.7.2" @@ -4705,6 +4716,7 @@ dependencies = [ "dotenvy", "etcd-client", "fastrace", + "fastrace-jaeger", "flagset", "flume", "foundationdb", @@ -4729,6 +4741,8 @@ dependencies = [ "openssh", "openssh-sftp-client", "opentelemetry", + "opentelemetry-otlp", + "opentelemetry_sdk", "percent-encoding", "persy", "pretty_assertions", @@ -4755,6 +4769,7 @@ dependencies = [ "tikv-client", "tokio", "tracing", + "tracing-opentelemetry", "tracing-subscriber", "uuid", ] @@ -4979,6 +4994,55 @@ dependencies = [ "thiserror", ] +[[package]] +name = "opentelemetry-otlp" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b925a602ffb916fb7421276b86756027b37ee708f9dce2dbdcc51739f07e727" +dependencies = [ + "async-trait", + "futures-core", + "http 1.1.0", + "opentelemetry", + "opentelemetry-proto", + "opentelemetry_sdk", + "prost 0.13.1", + "thiserror", + "tokio", + "tonic 0.12.1", +] + +[[package]] +name = "opentelemetry-proto" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ee9f20bff9c984511a02f082dc8ede839e4a9bf15cc2487c8d6fea5ad850d9" +dependencies = [ + "opentelemetry", + "opentelemetry_sdk", + "prost 0.13.1", + "tonic 0.12.1", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eac490ec80f24a17828d49b40b60f5aeaccdfe6a503f939713afd22bc28df" +dependencies = [ + "async-trait", + "futures-channel", + "futures-executor", + "futures-util", + "glob", + "once_cell", + "opentelemetry", + "percent-encoding", + "rand 0.8.5", + "serde_json", + "thiserror", +] + [[package]] name = "ordered-multimap" version = "0.7.3" @@ -7799,6 +7863,16 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "thrift_codec" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83d957f535b242b91aa9f47bde08080f9a6fef276477e55b0079979d002759d5" +dependencies = [ + "byteorder", + "trackable", +] + [[package]] name = "tikv-client" version = "0.3.0" @@ -8190,6 +8264,24 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-opentelemetry" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9784ed4da7d921bc8df6963f8c80a0e4ce34ba6ba76668acadd3edbd985ff3b" +dependencies = [ + "js-sys", + "once_cell", + "opentelemetry", + "opentelemetry_sdk", + "smallvec", + "tracing", + "tracing-core", + "tracing-log", + "tracing-subscriber", + "web-time", +] + [[package]] name = "tracing-subscriber" version = "0.3.18" @@ -8208,6 +8300,25 @@ dependencies = [ "tracing-log", ] +[[package]] +name = "trackable" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b15bd114abb99ef8cee977e517c8f37aee63f184f2d08e3e6ceca092373369ae" +dependencies = [ + "trackable_derive", +] + +[[package]] +name = "trackable_derive" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb235c5847e2f82cfe0f07eb971d1e5f6804b18dac2ae16349cc604380f82f" +dependencies = [ + "quote", + "syn 1.0.109", +] + [[package]] name = "trice" version = "0.4.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index 90a02fef814..63a3d6b5d4b 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -154,6 +154,7 @@ services-icloud = ["internal-path-cache"] services-ipfs = ["dep:prost"] services-ipmfs = [] services-koofr = [] +services-lakefs = [] services-libsql = ["dep:hrana-client-proto"] services-memcached = ["dep:bb8"] services-memory = [] @@ -200,7 +201,7 @@ services-vercel-blob = [] services-webdav = [] services-webhdfs = [] services-yandex-disk = [] -services-lakefs = [] + [lib] bench = false @@ -374,22 +375,26 @@ probe = { version = "0.5.1", optional = true } [target.'cfg(target_arch = "wasm32")'.dependencies] backon = { version = "1.0.1", features = ["gloo-timers-sleep"] } -tokio = { version = "1.27", features = ["time"] } getrandom = { version = "0.2", features = ["js"] } +tokio = { version = "1.27", features = ["time"] } [dev-dependencies] criterion = { version = "0.5", features = ["async", "async_tokio"] } dotenvy = "0.15" fastrace = { version = "0.7", features = ["enable"] } +fastrace-jaeger = "0.7" libtest-mimic = "0.7" opentelemetry = { version = "0.24", default-features = false, features = [ "trace", ] } +opentelemetry-otlp = "0.17" +opentelemetry_sdk = "0.24" pretty_assertions = "1" rand = "0.8" sha2 = "0.10" size = "0.4" tokio = { version = "1.27", features = ["fs", "macros", "rt-multi-thread"] } +tracing-opentelemetry = "0.25.0" tracing-subscriber = { version = "0.3", features = [ "env-filter", "tracing-log", diff --git a/core/src/layers/async_backtrace.rs b/core/src/layers/async_backtrace.rs index 290171c29c6..0dbe3bae19d 100644 --- a/core/src/layers/async_backtrace.rs +++ b/core/src/layers/async_backtrace.rs @@ -30,18 +30,19 @@ use crate::*; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::AsyncBacktraceLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::AsyncBacktraceLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(AsyncBacktraceLayer::default()) /// .finish(); +/// Ok(()) +/// # } /// ``` - #[derive(Clone, Default)] pub struct AsyncBacktraceLayer; diff --git a/core/src/layers/await_tree.rs b/core/src/layers/await_tree.rs index 4fb680ef528..e5425a79f21 100644 --- a/core/src/layers/await_tree.rs +++ b/core/src/layers/await_tree.rs @@ -22,7 +22,7 @@ use futures::FutureExt; use crate::raw::*; use crate::*; -/// Add a Instrument await-tree for actor-based applications to the underlying services. +/// Add an Instrument await-tree for actor-based applications to the underlying services. /// /// # AwaitTree /// @@ -33,16 +33,18 @@ use crate::*; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::AwaitTreeLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::AwaitTreeLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(AwaitTreeLayer::new()) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Clone, Default)] pub struct AwaitTreeLayer {} diff --git a/core/src/layers/blocking.rs b/core/src/layers/blocking.rs index 3b592a53068..8deb767e6c1 100644 --- a/core/src/layers/blocking.rs +++ b/core/src/layers/blocking.rs @@ -35,16 +35,16 @@ use crate::*; /// BlockingLayer will use current async context's runtime to handle the async calls. /// /// ```rust,no_run -/// # use anyhow::Result; -/// use opendal::layers::BlockingLayer; -/// use opendal::services::S3; -/// use opendal::BlockingOperator; -/// use opendal::Operator; +/// # use opendal::layers::BlockingLayer; +/// # use opendal::services; +/// # use opendal::BlockingOperator; +/// # use opendal::Operator; +/// # use opendal::Result; /// /// #[tokio::main] /// async fn main() -> Result<()> { /// // Create fs backend builder. -/// let mut builder = S3::default().bucket("test").region("us-east-1"); +/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); /// /// // Build an `BlockingOperator` with blocking layer to start operating the storage. /// let _: BlockingOperator = Operator::new(builder)? @@ -63,11 +63,11 @@ use crate::*; /// This often happens in the case that async function calls blocking function. /// /// ```rust,no_run -/// use opendal::layers::BlockingLayer; -/// use opendal::services::S3; -/// use opendal::BlockingOperator; -/// use opendal::Operator; -/// use opendal::Result; +/// # use opendal::layers::BlockingLayer; +/// # use opendal::services; +/// # use opendal::BlockingOperator; +/// # use opendal::Operator; +/// # use opendal::Result; /// /// #[tokio::main] /// async fn main() -> Result<()> { @@ -77,7 +77,7 @@ use crate::*; /// /// fn blocking_fn() -> Result { /// // Create fs backend builder. -/// let mut builder = S3::default().bucket("test").region("us-east-1"); +/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); /// /// let handle = tokio::runtime::Handle::try_current().unwrap(); /// let _guard = handle.enter(); @@ -98,12 +98,12 @@ use crate::*; /// > runtime on demand. /// /// ```rust,no_run -/// use once_cell::sync::Lazy; -/// use opendal::layers::BlockingLayer; -/// use opendal::services::S3; -/// use opendal::BlockingOperator; -/// use opendal::Operator; -/// use opendal::Result; +/// # use once_cell::sync::Lazy; +/// # use opendal::layers::BlockingLayer; +/// # use opendal::services; +/// # use opendal::BlockingOperator; +/// # use opendal::Operator; +/// # use opendal::Result; /// /// static RUNTIME: Lazy = Lazy::new(|| { /// tokio::runtime::Builder::new_multi_thread() @@ -111,11 +111,10 @@ use crate::*; /// .build() /// .unwrap() /// }); -/// /// /// /// fn main() -> Result<()> { /// // Create fs backend builder. -/// let mut builder = S3::default().bucket("test").region("us-east-1"); +/// let mut builder = services::S3::default().bucket("test").region("us-east-1"); /// /// // Fetch the `EnterGuard` from global runtime. /// let _guard = RUNTIME.enter(); diff --git a/core/src/layers/chaos.rs b/core/src/layers/chaos.rs index 083e28c825f..126fcfef0d1 100644 --- a/core/src/layers/chaos.rs +++ b/core/src/layers/chaos.rs @@ -45,16 +45,18 @@ use crate::*; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::ChaosLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::ChaosLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(ChaosLayer::new(0.1)) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Debug, Clone)] pub struct ChaosLayer { diff --git a/core/src/layers/concurrent_limit.rs b/core/src/layers/concurrent_limit.rs index 87ad19b50c9..bd17666442b 100644 --- a/core/src/layers/concurrent_limit.rs +++ b/core/src/layers/concurrent_limit.rs @@ -34,16 +34,18 @@ use crate::*; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::ConcurrentLimitLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::ConcurrentLimitLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(ConcurrentLimitLayer::new(1024)) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Clone)] pub struct ConcurrentLimitLayer { diff --git a/core/src/layers/dtrace.rs b/core/src/layers/dtrace.rs index 51b001313b5..7cfa8ac55dd 100644 --- a/core/src/layers/dtrace.rs +++ b/core/src/layers/dtrace.rs @@ -111,31 +111,27 @@ use crate::*; /// /// Example: /// -/// ```no_build -/// use anyhow::Result; -/// use opendal::layers::DTraceLayer; -/// use opendal::services::Fs; -/// use opendal::Operator; +/// ```no_run +/// # use opendal::layers::DtraceLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; /// -/// #[tokio::main] -/// async fn main() -> Result<()> { -/// let mut builder = Fs::default(); +/// # #[tokio::main] +/// # async fn main() -> Result<()> { +/// // `Accessor` provides the low level APIs, we will use `Operator` normally. +/// let op: Operator = Operator::new(services::Fs::default().root("/tmp"))? +/// .layer(DtraceLayer::default()) +/// .finish(); /// -/// builder.root("/tmp"); -/// -/// // `Accessor` provides the low level APIs, we will use `Operator` normally. -/// let op: Operator = Operator::new(builder)? -/// .layer(DtraceLayer::default()) -/// .finish(); -/// -/// let path = "/tmp/test.txt"; -/// for _ in 1..100000 { -/// let bs = vec![0; 64 * 1024 * 1024]; -/// op.write(path, bs).await?; -/// op.read(path).await?; -/// } -/// Ok(()) +/// let path = "/tmp/test.txt"; +/// for _ in 1..100000 { +/// let bs = vec![0; 64 * 1024 * 1024]; +/// op.write(path, bs).await?; +/// op.read(path).await?; /// } +/// Ok(()) +/// # } /// ``` /// /// Then you can use `readelf -n target/debug/examples/dtrace` to see the probes: diff --git a/core/src/layers/error_context.rs b/core/src/layers/error_context.rs index 43a8fc224e8..8f4be55bc77 100644 --- a/core/src/layers/error_context.rs +++ b/core/src/layers/error_context.rs @@ -28,7 +28,7 @@ use crate::*; /// /// # Notes /// -/// This layer will adding the following error context into all errors: +/// This layer will add the following error context into all errors: /// /// - `service`: The [`Scheme`] of underlying service. /// - `operation`: The [`Operation`] of this operation diff --git a/core/src/layers/fastrace.rs b/core/src/layers/fastrace.rs index caae7b939ac..513d91b2b10 100644 --- a/core/src/layers/fastrace.rs +++ b/core/src/layers/fastrace.rs @@ -31,61 +31,55 @@ use crate::*; /// /// ## Basic Setup /// -/// ```no_build -/// use anyhow::Result; -/// use opendal::layers::FastraceLayer; -/// use opendal::services; -/// use opendal::Operator; +/// ```no_run +/// # use opendal::layers::FastraceLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(FastraceLayer) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// ## Real usage /// -/// ```no_build -/// use std::error::Error; +/// ```no_run +/// # use anyhow::Result; +/// # use fastrace::prelude::*; +/// # use opendal::layers::FastraceLayer; +/// # use opendal::services; +/// # use opendal::Operator; /// -/// use anyhow::Result; -/// use futures::executor::block_on; -/// use fastrace::collector::Config; -/// use fastrace::prelude::*; -/// use opendal::layers::FastraceLayer; -/// use opendal::services; -/// use opendal::Operator; +/// # fn main() -> Result<()> { +/// let reporter = fastrace_jaeger::JaegerReporter::new("127.0.0.1:6831".parse()?, "opendal").unwrap(); +/// fastrace::set_reporter(reporter, fastrace::collector::Config::default()); /// -/// fn main() -> Result<(), Box> { -/// let reporter = -/// fastrace_jaeger::JaegerReporter::new("127.0.0.1:6831".parse().unwrap(), "opendal") -/// .unwrap(); -/// fastrace::set_reporter(reporter, Config::default()); -/// -/// { -/// let root = Span::root("op", SpanContext::random()); -/// let runtime = tokio::runtime::Runtime::new()?; -/// runtime.block_on( -/// async { -/// let _ = dotenvy::dotenv(); -/// let op = Operator::new(services::Memory::default()) -/// .expect("init operator must succeed") -/// .layer(FastraceLayer) -/// .finish(); -/// op.write("test", "0".repeat(16 * 1024 * 1024).into_bytes()) -/// .await -/// .expect("must succeed"); -/// op.stat("test").await.expect("must succeed"); -/// op.read("test").await.expect("must succeed"); -/// } -/// .in_span(Span::enter_with_parent("test", &root)), -/// ); -/// } +/// { +/// let root = Span::root("op", SpanContext::random()); +/// let runtime = tokio::runtime::Runtime::new()?; +/// runtime.block_on( +/// async { +/// let _ = dotenvy::dotenv(); +/// let op = Operator::new(services::Memory::default())? +/// .layer(FastraceLayer) +/// .finish(); +/// op.write("test", "0".repeat(16 * 1024 * 1024).into_bytes()).await?; +/// op.stat("test").await?; +/// op.read("test").await?; +/// Ok::<(), opendal::Error>(()) +/// } +/// .in_span(Span::enter_with_parent("test", &root)), +/// )?; +/// } /// -/// fastrace::flush(); +/// fastrace::flush(); /// -/// Ok(()) -/// } +/// Ok(()) +/// # } /// ``` /// /// # Output @@ -96,15 +90,14 @@ use crate::*; /// /// For example: /// -/// ```no_build -/// extern crate fastrace_jaeger; -/// -/// use fastrace::collector::Config; +/// ```no_run +/// # use anyhow::Result; /// -/// let reporter = -/// fastrace_jaeger::JaegerReporter::new("127.0.0.1:6831".parse().unwrap(), "opendal") -/// .unwrap(); -/// fastrace::set_reporter(reporter, Config::default()); +/// # fn main() -> Result<()> { +/// let reporter = fastrace_jaeger::JaegerReporter::new("127.0.0.1:6831".parse()?, "opendal").unwrap(); +/// fastrace::set_reporter(reporter, fastrace::collector::Config::default()); +/// Ok(()) +/// # } /// ``` /// /// For real-world usage, please take a look at [`fastrace-datadog`](https://crates.io/crates/fastrace-datadog) or [`fastrace-jaeger`](https://crates.io/crates/fastrace-jaeger) . diff --git a/core/src/layers/immutable_index.rs b/core/src/layers/immutable_index.rs index 7cbb88ca8b8..5e02472fab6 100644 --- a/core/src/layers/immutable_index.rs +++ b/core/src/layers/immutable_index.rs @@ -30,22 +30,25 @@ use crate::*; /// # Examples /// /// ```rust, no_run -/// use std::collections::HashMap; +/// # use std::collections::HashMap; /// -/// use opendal::layers::ImmutableIndexLayer; -/// use opendal::services; -/// use opendal::Operator; +/// # use opendal::layers::ImmutableIndexLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; /// +/// # fn main() -> Result<()> { /// let mut iil = ImmutableIndexLayer::default(); /// /// for i in ["file", "dir/", "dir/file", "dir_without_prefix/file"] { /// iil.insert(i.to_string()) /// } /// -/// let op = Operator::from_map::(HashMap::default()) -/// .unwrap() +/// let op = Operator::from_iter::(HashMap::<_, _>::default())? /// .layer(iil) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Default, Debug, Clone)] pub struct ImmutableIndexLayer { diff --git a/core/src/layers/logging.rs b/core/src/layers/logging.rs index d3dfdbdfeb6..100a3951ef9 100644 --- a/core/src/layers/logging.rs +++ b/core/src/layers/logging.rs @@ -43,16 +43,18 @@ use crate::*; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::LoggingLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::LoggingLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(LoggingLayer::default()) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// # Output @@ -76,13 +78,14 @@ use crate::*; /// You can implement your own logging interceptor to customize the logging behavior. /// /// ```no_run -/// use opendal::layers::LoggingInterceptor; -/// use opendal::layers::LoggingLayer; -/// use opendal::raw; -/// use opendal::services; -/// use opendal::Error; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::LoggingInterceptor; +/// # use opendal::layers::LoggingLayer; +/// # use opendal::raw; +/// # use opendal::services; +/// # use opendal::Error; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// /// #[derive(Debug, Clone)] /// struct MyLoggingInterceptor; @@ -100,10 +103,12 @@ use crate::*; /// } /// } /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(LoggingLayer::new(MyLoggingInterceptor)) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Debug)] pub struct LoggingLayer { diff --git a/core/src/layers/metrics.rs b/core/src/layers/metrics.rs index c7fc75fa1f1..94ec8ba7ad5 100644 --- a/core/src/layers/metrics.rs +++ b/core/src/layers/metrics.rs @@ -41,16 +41,16 @@ use crate::*; /// # Examples /// /// ```no_run -/// # use anyhow::Result; /// # use opendal::layers::MetricsLayer; /// # use opendal::services; /// # use opendal::Operator; +/// # use opendal::Result; /// /// # fn main() -> Result<()> { -/// let _ = Operator::new(services::Memory::default())? -/// .layer(MetricsLayer::default()) -/// .finish(); -/// Ok(()) +/// let _ = Operator::new(services::Memory::default())? +/// .layer(MetricsLayer::default()) +/// .finish(); +/// Ok(()) /// # } /// ``` /// diff --git a/core/src/layers/mime_guess.rs b/core/src/layers/mime_guess.rs index 5b7f0712468..7a4bc08b8aa 100644 --- a/core/src/layers/mime_guess.rs +++ b/core/src/layers/mime_guess.rs @@ -45,16 +45,18 @@ use crate::Result; /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::MimeGuessLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::MimeGuessLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(MimeGuessLayer::default()) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Debug, Clone, Default)] #[non_exhaustive] diff --git a/core/src/layers/oteltrace.rs b/core/src/layers/oteltrace.rs index 659134377c9..ab2044d4029 100644 --- a/core/src/layers/oteltrace.rs +++ b/core/src/layers/oteltrace.rs @@ -37,16 +37,18 @@ use crate::*; /// /// ## Basic Setup /// -/// ```no_build -/// use anyhow::Result; -/// use opendal::layers::OtelTraceLayer; -/// use opendal::services; -/// use opendal::Operator; +/// ```no_run +/// # use opendal::layers::OtelTraceLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(OtelTraceLayer) /// .finish(); +/// Ok(()) +/// # } /// ``` pub struct OtelTraceLayer; diff --git a/core/src/layers/prometheus.rs b/core/src/layers/prometheus.rs index 458afd36b96..f374c24a0b5 100644 --- a/core/src/layers/prometheus.rs +++ b/core/src/layers/prometheus.rs @@ -51,32 +51,31 @@ use crate::*; /// /// # #[tokio::main] /// # async fn main() -> Result<()> { -/// // Pick a builder and configure it. -/// let builder = services::Memory::default(); -/// let registry = prometheus::default_registry(); +/// let registry = prometheus::default_registry(); /// -/// let op = Operator::new(builder)? -/// .layer(PrometheusLayer::builder().register(registry).expect("register metrics successfully")) -/// .finish(); -/// debug!("operator: {op:?}"); +/// let op = Operator::new(services::Memory::default())? +/// .layer(PrometheusLayer::builder().register(registry).expect("register metrics successfully")) +/// .finish(); +/// debug!("operator: {op:?}"); /// -/// // Write data into object test. -/// op.write("test", "Hello, World!").await?; -/// // Read data from object. -/// let bs = op.read("test").await?; -/// info!("content: {}", String::from_utf8_lossy(&bs.to_bytes())); +/// // Write data into object test. +/// op.write("test", "Hello, World!").await?; +/// // Read data from object. +/// let bs = op.read("test").await?; +/// info!("content: {}", String::from_utf8_lossy(&bs.to_bytes())); /// -/// // Get object metadata. -/// let meta = op.stat("test").await?; -/// info!("meta: {:?}", meta); +/// // Get object metadata. +/// let meta = op.stat("test").await?; +/// info!("meta: {:?}", meta); /// -/// // Export prometheus metrics. -/// let mut buffer = Vec::::new(); -/// let encoder = prometheus::TextEncoder::new(); -/// encoder.encode(&prometheus::gather(), &mut buffer).unwrap(); -/// println!("## Prometheus Metrics"); -/// println!("{}", String::from_utf8(buffer.clone()).unwrap()); -/// Ok(()) +/// // Export prometheus metrics. +/// let mut buffer = Vec::::new(); +/// let encoder = prometheus::TextEncoder::new(); +/// encoder.encode(&prometheus::gather(), &mut buffer).unwrap(); +/// println!("## Prometheus Metrics"); +/// println!("{}", String::from_utf8(buffer.clone()).unwrap()); +/// +/// Ok(()) /// # } /// ``` #[derive(Clone, Debug)] diff --git a/core/src/layers/prometheus_client.rs b/core/src/layers/prometheus_client.rs index dc61d910d42..e888248c29e 100644 --- a/core/src/layers/prometheus_client.rs +++ b/core/src/layers/prometheus_client.rs @@ -52,31 +52,30 @@ use crate::*; /// /// # #[tokio::main] /// # async fn main() -> Result<()> { -/// // Pick a builder and configure it. -/// let builder = services::Memory::default(); -/// let mut registry = prometheus_client::registry::Registry::default(); +/// let mut registry = prometheus_client::registry::Registry::default(); /// -/// let op = Operator::new(builder)? -/// .layer(PrometheusClientLayer::builder().register(&mut registry)) -/// .finish(); -/// debug!("operator: {op:?}"); +/// let op = Operator::new(services::Memory::default())? +/// .layer(PrometheusClientLayer::builder().register(&mut registry)) +/// .finish(); +/// debug!("operator: {op:?}"); /// -/// // Write data into object test. -/// op.write("test", "Hello, World!").await?; -/// // Read data from object. -/// let bs = op.read("test").await?; -/// info!("content: {}", String::from_utf8_lossy(&bs.to_bytes())); +/// // Write data into object test. +/// op.write("test", "Hello, World!").await?; +/// // Read data from object. +/// let bs = op.read("test").await?; +/// info!("content: {}", String::from_utf8_lossy(&bs.to_bytes())); /// -/// // Get object metadata. -/// let meta = op.stat("test").await?; -/// info!("meta: {:?}", meta); +/// // Get object metadata. +/// let meta = op.stat("test").await?; +/// info!("meta: {:?}", meta); /// -/// // Export prometheus metrics. -/// let mut buf = String::new(); -/// prometheus_client::encoding::text::encode(&mut buf, ®istry).unwrap(); -/// println!("## Prometheus Metrics"); -/// println!("{}", buf); -/// Ok(()) +/// // Export prometheus metrics. +/// let mut buf = String::new(); +/// prometheus_client::encoding::text::encode(&mut buf, ®istry).unwrap(); +/// println!("## Prometheus Metrics"); +/// println!("{}", buf); +/// +/// Ok(()) /// # } /// ``` #[derive(Clone, Debug)] diff --git a/core/src/layers/retry.rs b/core/src/layers/retry.rs index dc8905af654..b98fd7eecce 100644 --- a/core/src/layers/retry.rs +++ b/core/src/layers/retry.rs @@ -48,30 +48,42 @@ use crate::*; /// /// For example, while composing `RetryLayer` with `TimeoutLayer`. The order of layer is sensitive. /// -/// ```no_build -/// let op = Operator::new(builder.clone()) -/// .unwrap() +/// ```no_run +/// # use std::time::Duration; +/// +/// # use opendal::layers::RetryLayer; +/// # use opendal::layers::TimeoutLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// +/// # fn main() -> Result<()> { +/// let op = Operator::new(services::Memory::default())? /// // This is fine, since timeout happen during retry. /// .layer(TimeoutLayer::new().with_io_timeout(Duration::from_nanos(1))) /// .layer(RetryLayer::new()) /// // This is wrong. Since timeout layer will drop future, leaving retry layer in a bad state. /// .layer(TimeoutLayer::new().with_io_timeout(Duration::from_nanos(1))) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// # Examples /// /// ```no_run -/// use anyhow::Result; -/// use opendal::layers::RetryLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::RetryLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(RetryLayer::new()) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// ## Customize retry interceptor @@ -80,15 +92,15 @@ use crate::*; /// their own retry interceptor logic. /// /// ```no_run -/// use std::time::Duration; +/// # use std::time::Duration; /// -/// use anyhow::Result; -/// use opendal::layers::RetryInterceptor; -/// use opendal::layers::RetryLayer; -/// use opendal::services; -/// use opendal::Error; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::RetryInterceptor; +/// # use opendal::layers::RetryLayer; +/// # use opendal::services; +/// # use opendal::Error; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// /// struct MyRetryInterceptor; /// @@ -98,10 +110,12 @@ use crate::*; /// } /// } /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(RetryLayer::new().with_notify(MyRetryInterceptor)) /// .finish(); +/// Ok(()) +/// # } /// ``` pub struct RetryLayer { builder: ExponentialBuilder, diff --git a/core/src/layers/throttle.rs b/core/src/layers/throttle.rs index f73f33d3caa..06a2f631320 100644 --- a/core/src/layers/throttle.rs +++ b/core/src/layers/throttle.rs @@ -50,17 +50,20 @@ use crate::*; /// /// This example limits bandwidth to 10 KiB/s and burst size to 10 MiB. /// -/// ```no_build -/// use anyhow::Result; -/// use opendal::layers::ThrottleLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// ```no_run +/// # use opendal::layers::ThrottleLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// +/// # fn main() -> Result<()> { /// let _ = Operator::new(services::Memory::default()) /// .expect("must init") /// .layer(ThrottleLayer::new(10 * 1024, 10000 * 1024)) /// .finish(); +/// Ok(()) +/// # } /// ``` #[derive(Clone)] pub struct ThrottleLayer { diff --git a/core/src/layers/timeout.rs b/core/src/layers/timeout.rs index 3adc6878532..2bda1292b03 100644 --- a/core/src/layers/timeout.rs +++ b/core/src/layers/timeout.rs @@ -51,15 +51,25 @@ use crate::*; /// For example, while using `TimeoutLayer` with `RetryLayer` at the same time, please make sure /// timeout layer showed up before retry layer. /// -/// ```no_build -/// let op = Operator::new(builder.clone()) -/// .unwrap() +/// ```no_run +/// # use std::time::Duration; +/// +/// # use opendal::layers::RetryLayer; +/// # use opendal::layers::TimeoutLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// +/// # fn main() -> Result<()> { +/// let op = Operator::new(services::Memory::default())? /// // This is fine, since timeout happen during retry. /// .layer(TimeoutLayer::new().with_io_timeout(Duration::from_nanos(1))) /// .layer(RetryLayer::new()) /// // This is wrong. Since timeout layer will drop future, leaving retry layer in a bad state. /// .layer(TimeoutLayer::new().with_io_timeout(Duration::from_nanos(1))) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// # Examples @@ -68,22 +78,24 @@ use crate::*; /// operations, 3 seconds timeout for all io operations. /// /// ```no_run -/// use std::time::Duration; +/// # use std::time::Duration; /// -/// use anyhow::Result; -/// use opendal::layers::TimeoutLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opendal::Scheme; +/// # use opendal::layers::TimeoutLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; +/// # use opendal::Scheme; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer( /// TimeoutLayer::default() /// .with_timeout(Duration::from_secs(10)) /// .with_io_timeout(Duration::from_secs(3)), /// ) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// # Implementation Notes diff --git a/core/src/layers/tracing.rs b/core/src/layers/tracing.rs index f95ac60b6bb..aef0dfb59cb 100644 --- a/core/src/layers/tracing.rs +++ b/core/src/layers/tracing.rs @@ -31,68 +31,76 @@ use crate::*; /// /// ## Basic Setup /// -/// ```no_build -/// use anyhow::Result; -/// use opendal::layers::TracingLayer; -/// use opendal::services; -/// use opendal::Operator; +/// ```no_run +/// # use opendal::layers::TracingLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opendal::Result; /// -/// let _ = Operator::new(services::Memory::default()) -/// .expect("must init") +/// # fn main() -> Result<()> { +/// let _ = Operator::new(services::Memory::default())? /// .layer(TracingLayer) /// .finish(); +/// Ok(()) +/// # } /// ``` /// /// ## Real usage /// -/// ```no_build -/// use std::error::Error; +/// ```no_run +/// # use anyhow::Result; +/// # use opendal::layers::TracingLayer; +/// # use opendal::services; +/// # use opendal::Operator; +/// # use opentelemetry::KeyValue; +/// # use opentelemetry_sdk::trace; +/// # use opentelemetry_sdk::Resource; +/// # use tracing_subscriber::prelude::*; +/// # use tracing_subscriber::EnvFilter; /// -/// use anyhow::Result; -/// use opendal::layers::TracingLayer; -/// use opendal::services; -/// use opendal::Operator; -/// use opentelemetry::global; -/// use tracing::span; -/// use tracing_subscriber::prelude::*; -/// use tracing_subscriber::EnvFilter; +/// # fn main() -> Result<()> { +/// use opentelemetry::trace::TracerProvider; +/// let tracer_provider = opentelemetry_otlp::new_pipeline() +/// .tracing() +/// .with_exporter(opentelemetry_otlp::new_exporter().tonic()) +/// .with_trace_config( +/// trace::Config::default() +/// .with_resource(Resource::new(vec![KeyValue::new("service.name", "opendal_example")])) +/// ) +/// .install_simple()?; +/// let tracer = tracer_provider.tracer("opendal_tracer"); +/// let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); /// -/// fn main() -> Result<(), Box> { -/// let tracer = opentelemetry_jaeger::new_pipeline() -/// .with_service_name("opendal_example") -/// .install_simple()?; -/// let opentelemetry = tracing_opentelemetry::layer().with_tracer(tracer); -/// tracing_subscriber::registry() -/// .with(EnvFilter::from_default_env()) -/// .with(opentelemetry) -/// .try_init()?; +/// tracing_subscriber::registry() +/// .with(EnvFilter::from_default_env()) +/// .with(opentelemetry) +/// .try_init()?; /// +/// { /// let runtime = tokio::runtime::Runtime::new()?; -/// /// runtime.block_on(async { -/// let root = span!(tracing::Level::INFO, "app_start", work_units = 2); +/// let root = tracing::span!(tracing::Level::INFO, "app_start", work_units = 2); /// let _enter = root.enter(); /// /// let _ = dotenvy::dotenv(); -/// let op = Operator::from_env::() -/// .expect("init operator must succeed") +/// let op = Operator::new(services::Memory::default())? /// .layer(TracingLayer) /// .finish(); /// -/// op.object("test") -/// .write("0".repeat(16 * 1024 * 1024).into_bytes()) -/// .await -/// .expect("must succeed"); -/// op.stat("test").await.expect("must succeed"); -/// op.read("test").await.expect("must succeed"); -/// }); -/// -/// // Shut down the current tracer provider. This will invoke the shutdown -/// // method on all span processors. span processors should export remaining -/// // spans before return. -/// global::shutdown_tracer_provider(); -/// Ok(()) +/// op.write("test", "0".repeat(16 * 1024 * 1024).into_bytes()).await?; +/// op.stat("test").await?; +/// op.read("test").await?; +/// Ok::<(), opendal::Error>(()) +/// })?; /// } +/// +/// // Shut down the current tracer provider. +/// // This will invoke the shutdown method on all span processors. +/// // span processors should export remaining spans before return. +/// opentelemetry::global::shutdown_tracer_provider(); +/// +/// Ok(()) +/// # } /// ``` /// /// # Output @@ -103,8 +111,26 @@ use crate::*; /// /// For example: /// -/// ```no_build -/// extern crate tracing; +/// ```no_run +/// # use tracing::dispatcher; +/// # use tracing::Event; +/// # use tracing::Metadata; +/// # use tracing::span::Attributes; +/// # use tracing::span::Id; +/// # use tracing::span::Record; +/// # use tracing::subscriber::Subscriber; +/// +/// # pub struct FooSubscriber; +/// # impl Subscriber for FooSubscriber { +/// # fn enabled(&self, _: &Metadata) -> bool { false } +/// # fn new_span(&self, _: &Attributes) -> Id { Id::from_u64(0) } +/// # fn record(&self, _: &Id, _: &Record) {} +/// # fn record_follows_from(&self, _: &Id, _: &Id) {} +/// # fn event(&self, _: &Event) {} +/// # fn enter(&self, _: &Id) {} +/// # fn exit(&self, _: &Id) {} +/// # } +/// # impl FooSubscriber { fn new() -> Self { FooSubscriber } } /// /// let my_subscriber = FooSubscriber::new(); /// tracing::subscriber::set_global_default(my_subscriber) diff --git a/integrations/cloudfilter/Cargo.toml b/integrations/cloudfilter/Cargo.toml index 845331396d6..119e18eb9b8 100644 --- a/integrations/cloudfilter/Cargo.toml +++ b/integrations/cloudfilter/Cargo.toml @@ -40,17 +40,17 @@ serde = { version = "1.0.203", features = ["derive"] } [dev-dependencies] env_logger = "0.11.2" +libtest-mimic = "0.7.3" opendal = { version = "0.49.0", path = "../../core", features = [ "services-fs", "tests", ] } +powershell_script = "1.1.0" tokio = { version = "1.38.0", features = [ "macros", "rt-multi-thread", "signal", ] } -libtest-mimic = "0.7.3" -powershell_script = "1.1.0" [[test]] harness = false