diff --git a/Cargo.lock b/Cargo.lock index f268df68..e3ef5bef 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -857,6 +857,43 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console-api" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd326812b3fd01da5bb1af7d340d0d555fd3d4b641e7f1dfcf5962a902952787" +dependencies = [ + "futures-core", + "prost 0.12.3", + "prost-types 0.12.3", + "tonic 0.10.2", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7481d4c57092cd1c19dd541b92bdce883de840df30aa5d03fd48a3935c01842e" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils", + "futures-task", + "hdrhistogram", + "humantime", + "prost-types 0.12.3", + "serde", + "serde_json", + "thread_local", + "tokio", + "tokio-stream", + "tonic 0.10.2", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.7.1" @@ -2018,6 +2055,19 @@ dependencies = [ "allocator-api2", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.4", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.4.1" @@ -2094,6 +2144,12 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + [[package]] name = "hyper" version = "0.14.27" @@ -2584,6 +2640,7 @@ name = "keramik-common" version = "0.1.0" dependencies = [ "anyhow", + "console-subscriber", "gethostname", "opentelemetry", "opentelemetry-otlp", @@ -2640,6 +2697,7 @@ dependencies = [ "goose", "keramik-common", "libipld 0.16.0", + "multibase 0.9.1", "multihash 0.17.0", "opentelemetry", "rand 0.8.5", @@ -3185,6 +3243,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.1" @@ -3412,6 +3476,16 @@ dependencies = [ "version_check", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -3611,10 +3685,10 @@ dependencies = [ "http", "opentelemetry", "opentelemetry-proto", - "prost", + "prost 0.11.9", "thiserror", "tokio", - "tonic", + "tonic 0.8.3", ] [[package]] @@ -3626,8 +3700,8 @@ dependencies = [ "futures", "futures-util", "opentelemetry", - "prost", - "tonic", + "prost 0.11.9", + "tonic 0.8.3", "tonic-build", ] @@ -3992,7 +4066,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b82eaa1d779e9a4bc1c3217db8ffbeabaae1dca241bf70183242128d48681cd" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.11.9", +] + +[[package]] +name = "prost" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c289cda302b98a28d40c8b3b90498d6e526dd24ac2ecea73e4e491685b94a" +dependencies = [ + "bytes", + "prost-derive 0.12.3", ] [[package]] @@ -4009,8 +4093,8 @@ dependencies = [ "multimap", "petgraph", "prettyplease", - "prost", - "prost-types", + "prost 0.11.9", + "prost-types 0.11.9", "regex", "syn 1.0.109", "tempfile", @@ -4030,13 +4114,35 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "prost-derive" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efb6c9a1dd1def8e2124d17e83a20af56f1570d6c2d2bd9e266ccb768df3840e" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.37", +] + [[package]] name = "prost-types" version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "213622a1460818959ac1181aaeb2dc9c7f63df720db7d788b3e24eacd1983e13" dependencies = [ - "prost", + "prost 0.11.9", +] + +[[package]] +name = "prost-types" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "193898f59edcf43c26227dcd4c8427f00d99d61e95dcde58dabd49fa291d470e" +dependencies = [ + "prost 0.12.3", ] [[package]] @@ -5601,6 +5707,7 @@ dependencies = [ "signal-hook-registry", "socket2 0.5.4", "tokio-macros", + "tracing", "windows-sys", ] @@ -5746,8 +5853,8 @@ dependencies = [ "hyper-timeout", "percent-encoding", "pin-project", - "prost", - "prost-derive", + "prost 0.11.9", + "prost-derive 0.11.9", "tokio", "tokio-stream", "tokio-util", @@ -5758,6 +5865,33 @@ dependencies = [ "tracing-futures", ] +[[package]] +name = "tonic" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.21.4", + "bytes", + "h2", + "http", + "http-body", + "hyper", + "hyper-timeout", + "percent-encoding", + "pin-project", + "prost 0.12.3", + "tokio", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "tonic-build" version = "0.8.4" diff --git a/Cargo.toml b/Cargo.toml index 01a397fb..f853ad43 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ resolver = "2" [workspace.dependencies] anyhow = "1" clap = { version = "4", features = ["derive", "env"] } +console-subscriber = "0.2" env_logger = "0.10.0" expect-patch = { path = "./expect-patch/" } keramik-common = { path = "./common/", default-features = false } @@ -25,7 +26,7 @@ reqwest = { version = "0.11", features = ["json", "multipart"] } schemars = "0.8" serde = { version = "1", features = ["derive"] } serde_json = "1" -tokio = { version = "1", features = ["full"] } +tokio = { version = "1", features = ["full", "tracing"] } tonic = { version = "0.8" } tracing = "0.1.37" tracing-opentelemetry = "0.18" diff --git a/Makefile b/Makefile index 97036e62..587087d1 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -CARGO = CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUSTFLAGS="-D warnings" cargo +CARGO = CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse RUSTFLAGS="--cfg tokio_unstable -D warnings" cargo .PHONY: all all: build check-fmt check-clippy test diff --git a/common/Cargo.toml b/common/Cargo.toml index f1bba462..84a73f3d 100644 --- a/common/Cargo.toml +++ b/common/Cargo.toml @@ -7,15 +7,17 @@ edition = "2021" [features] default = [] telemetry = [ - "opentelemetry", - "opentelemetry-otlp", - "tracing-opentelemetry", - "tracing-subscriber", - "tracing", + "dep:opentelemetry", + "dep:opentelemetry-otlp", + "dep:tracing-opentelemetry", + "dep:tracing-subscriber", + "dep:tracing", ] +tokio-console = ["telemetry", "dep:console-subscriber"] [dependencies] anyhow.workspace = true +console-subscriber = { workspace = true, optional = true } gethostname = "0.4.2" opentelemetry-otlp = { workspace = true, optional = true } opentelemetry = { workspace = true, optional = true } diff --git a/common/src/telemetry.rs b/common/src/telemetry.rs index d4ec53ca..fa85d660 100644 --- a/common/src/telemetry.rs +++ b/common/src/telemetry.rs @@ -10,7 +10,7 @@ use opentelemetry::{ }, }; use opentelemetry_otlp::WithExportConfig; -use tracing_subscriber::{prelude::*, EnvFilter, Registry}; +use tracing_subscriber::{filter::LevelFilter, prelude::*, EnvFilter, Registry}; /// Initialize tracing and metrics pub async fn init(otlp_endpoint: String) -> Result { @@ -60,15 +60,32 @@ pub async fn init(otlp_endpoint: String) -> Result { // Build starts the meter and sets it as the global meter provider .build()?; + // Setup filters + // Default to INFO if no env is specified + let log_filter = EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env()?; + let otlp_filter = EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env()?; + // Setup tracing layers - let telemetry = tracing_opentelemetry::layer().with_tracer(tracer); - let logger = tracing_subscriber::fmt::layer().with_ansi(true).compact(); - let env_filter = EnvFilter::try_from_default_env().or_else(|_| EnvFilter::try_new("info"))?; + let telemetry = tracing_opentelemetry::layer() + .with_tracer(tracer) + .with_filter(otlp_filter); + let logger = tracing_subscriber::fmt::layer() + .with_ansi(true) + .compact() + .with_filter(log_filter); + + let collector = Registry::default().with(telemetry).with(logger); - let collector = Registry::default() - .with(telemetry) - .with(logger) - .with(env_filter); + #[cfg(feature = "tokio-console")] + let collector = { + let console_filter = EnvFilter::builder().parse("tokio=trace,runtime=trace")?; + let console_layer = console_subscriber::spawn().with_filter(console_filter); + collector.with(console_layer) + }; // Initialize tracing tracing::subscriber::set_global_default(collector)?; diff --git a/k8s/operator/manifests/operator.yaml b/k8s/operator/manifests/operator.yaml index 27163371..c7fa333b 100644 --- a/k8s/operator/manifests/operator.yaml +++ b/k8s/operator/manifests/operator.yaml @@ -124,7 +124,7 @@ spec: - name: OPERATOR_OTLP_ENDPOINT value: "https://otel:4317" - name: RUST_LOG - value: "info,kube=debug,keramik_operator=debug" + value: "info" #readinessProbe: # httpGet: # path: /health diff --git a/operator/src/network/cas.rs b/operator/src/network/cas.rs index a0f6fe06..ff973fd6 100644 --- a/operator/src/network/cas.rs +++ b/operator/src/network/cas.rs @@ -547,6 +547,10 @@ pub fn cas_ipfs_stateful_set_spec( template: PodTemplateSpec { metadata: Some(ObjectMeta { labels: selector_labels(CAS_IPFS_APP), + annotations: Some(BTreeMap::from_iter(vec![( + "prometheus/path".to_owned(), + "/metrics".to_owned(), + )])), ..Default::default() }), spec: Some(PodSpec { diff --git a/operator/src/network/controller.rs b/operator/src/network/controller.rs index 33cf737b..0fb21731 100644 --- a/operator/src/network/controller.rs +++ b/operator/src/network/controller.rs @@ -156,7 +156,7 @@ pub async fn run() { .for_each(|rec_res| async move { match rec_res { Ok((network, _)) => { - debug!(network.name, "reconcile success"); + info!(network.name, "reconcile success"); } Err(err) => { error!(?err, "reconcile error") @@ -242,7 +242,6 @@ async fn reconcile( let total_weight = ceramic_configs.0.iter().fold(0, |acc, c| acc + c.weight) as f64; let mut ceramics = Vec::with_capacity(ceramic_configs.0.len()); for i in 0..MAX_CERAMICS { - debug!(i, "ceramic check"); let suffix = format!("{}", i); if let Some(config) = ceramic_configs.0.get(i) { let replicas = ((config.weight as f64 / total_weight) * spec.replicas as f64) as i32; @@ -256,7 +255,7 @@ async fn reconcile( }) } else { let info = CeramicInfo::new(&suffix, 0); - debug!(?info, "deleting extra ceramic"); + trace!(?info, "deleting extra ceramic"); delete_ceramic(cx.clone(), &ns, &info).await?; } } @@ -2586,7 +2585,7 @@ mod tests { stub.cas_ipfs_stateful_set.patch(expect![[r#" --- original +++ modified - @@ -92,14 +92,14 @@ + @@ -95,14 +95,14 @@ ], "resources": { "limits": { diff --git a/operator/src/network/testdata/default_stubs/cas_ipfs_stateful_set b/operator/src/network/testdata/default_stubs/cas_ipfs_stateful_set index 56240f9d..5e33c1ff 100644 --- a/operator/src/network/testdata/default_stubs/cas_ipfs_stateful_set +++ b/operator/src/network/testdata/default_stubs/cas_ipfs_stateful_set @@ -25,6 +25,9 @@ Request { "serviceName": "cas-ipfs", "template": { "metadata": { + "annotations": { + "prometheus/path": "/metrics" + }, "labels": { "app": "cas-ipfs" } diff --git a/operator/src/simulation/controller.rs b/operator/src/simulation/controller.rs index 9d93093f..1a7887bc 100644 --- a/operator/src/simulation/controller.rs +++ b/operator/src/simulation/controller.rs @@ -23,7 +23,7 @@ use kube::{ }; use rand::{thread_rng, Rng, RngCore}; -use tracing::{debug, error}; +use tracing::{debug, error, info}; use crate::{ labels::MANAGED_BY_LABEL_SELECTOR, @@ -118,7 +118,7 @@ pub async fn run() { .for_each(|rec_res| async move { match rec_res { Ok((simulation, _)) => { - debug!(simulation.name, "reconcile success"); + info!(simulation.name, "reconcile success"); } Err(err) => { error!(?err, "reconcile error") diff --git a/operator/src/simulation/spec.rs b/operator/src/simulation/spec.rs index 997295f4..7156dca1 100644 --- a/operator/src/simulation/spec.rs +++ b/operator/src/simulation/spec.rs @@ -19,7 +19,7 @@ pub struct SimulationSpec { pub scenario: String, /// Number of users pub users: u32, - /// Time in seconds to run the simulation + /// Time in minutes to run the simulation pub run_time: u32, /// Image for all jobs created by the simulation. pub image: Option, diff --git a/runner/Cargo.toml b/runner/Cargo.toml index 29ca54c2..e25b1f6e 100644 --- a/runner/Cargo.toml +++ b/runner/Cargo.toml @@ -13,7 +13,7 @@ ceramic-http-client = { git = "https://github.com/3box/ceramic-http-client-rs.gi cid = "0.9" clap.workspace = true goose = { version = "0.16", features = ["gaggle"] } -keramik-common = { workspace = true, features = ["telemetry"] } +keramik-common = { workspace = true, features = ["telemetry", "tokio-console"] } libipld = "0.16.0" multihash.workspace = true opentelemetry.workspace = true @@ -26,3 +26,4 @@ serde_json.workspace = true tokio.workspace = true tracing-log.workspace = true tracing.workspace = true +multibase.workspace = true diff --git a/runner/src/simulate.rs b/runner/src/simulate.rs index 8d14c24e..6d6d678b 100644 --- a/runner/src/simulate.rs +++ b/runner/src/simulate.rs @@ -178,6 +178,8 @@ fn manager_config(count: usize, users: usize, run_time: String) -> GooseConfigur } fn worker_config(target_peer_addr: String, throttle_requests: Option) -> GooseConfiguration { let mut config = GooseConfiguration::default(); + config.scenario_log = "scenario.log".to_owned(); + config.transaction_log = "transaction.log".to_owned(); config.request_log = "request.log".to_owned(); config.log_level = 2; config.worker = true;