diff --git a/Cargo.lock b/Cargo.lock index 6b2ec4c..4c7e671 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -415,25 +415,6 @@ dependencies = [ "syn 2.0.60", ] -[[package]] -name = "axum-otel-metrics" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11b5bd67776dca9326650fc2e2ddd15ddaca16a3c8e80a9a874ba111afab82bd" -dependencies = [ - "axum", - "futures-util", - "http 1.1.0", - "http-body 1.0.0", - "opentelemetry", - "opentelemetry-prometheus", - "opentelemetry-semantic-conventions", - "opentelemetry_sdk", - "pin-project-lite", - "prometheus", - "tower", -] - [[package]] name = "backon" version = "0.4.3" @@ -1076,15 +1057,6 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab3db02a9c5b5121e1e42fbdb1aeb65f5e02624cc58c43f2884c6ccac0b82f95" -dependencies = [ - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.5" @@ -1397,19 +1369,21 @@ dependencies = [ "async-utility", "axum", "axum-macros", - "axum-otel-metrics", "bitcoin 0.29.2", "bitcoin_hashes 0.13.0", "chrono", "clap 3.2.25", "dotenv", "fedimint", + "futures", "futures-util", "hex", "itertools 0.12.1", "lazy_static", "lightning-invoice", "lnurl-rs", + "metrics", + "metrics-exporter-prometheus", "multimint 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", "reqwest 0.12.4", "serde", @@ -2939,6 +2913,45 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "metrics" +version = "0.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884adb57038347dfbaf2d5065887b6cf4312330dc8e94bc30a1a839bd79d3261" +dependencies = [ + "ahash 0.8.11", + "portable-atomic", +] + +[[package]] +name = "metrics-exporter-prometheus" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4f0c8427b39666bf970460908b213ec09b3b350f20c0c2eabcbba51704a08e6" +dependencies = [ + "base64 0.22.0", + "indexmap 2.2.6", + "metrics", + "metrics-util", + "quanta", + "thiserror", +] + +[[package]] +name = "metrics-util" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4259040465c955f9f2f1a4a8a16dc46726169bca0f88e8fb2dbeced487c3e828" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.14.3", + "metrics", + "num_cpus", + "quanta", + "sketches-ddsketch", +] + [[package]] name = "mime" version = "0.3.17" @@ -3289,71 +3302,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381" -[[package]] -name = "opentelemetry" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "900d57987be3f2aeb70d385fff9b27fb74c5723cc9a52d904d4f9c807a0667bf" -dependencies = [ - "futures-core", - "futures-sink", - "js-sys", - "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - -[[package]] -name = "opentelemetry-prometheus" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30bbcf6341cab7e2193e5843f0ac36c446a5b3fccb28747afaeda17996dcd02e" -dependencies = [ - "once_cell", - "opentelemetry", - "opentelemetry_sdk", - "prometheus", - "protobuf", -] - -[[package]] -name = "opentelemetry-semantic-conventions" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9ab5bd6c42fb9349dcf28af2ba9a0667f697f9bdcca045d39f2cec5543e2910" - -[[package]] -name = "opentelemetry_sdk" -version = "0.22.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e90c7113be649e31e9a0f8b5ee24ed7a16923b322c3c5ab6367469c049d6b7e" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "glob", - "once_cell", - "opentelemetry", - "ordered-float", - "percent-encoding", - "rand", - "thiserror", - "tokio", - "tokio-stream", -] - -[[package]] -name = "ordered-float" -version = "4.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a76df7075c7d4d01fdcb46c912dd17fba5b60c78ea480b475f2b6ab6f666584e" -dependencies = [ - "num-traits", -] - [[package]] name = "os_str_bytes" version = "6.6.1" @@ -3535,6 +3483,12 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d30538d42559de6b034bc76fd6dd4c38961b1ee5c6c56e3808c50128fdbc22ce" + [[package]] name = "powerfmt" version = "0.2.0" @@ -3590,27 +3544,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "prometheus" -version = "0.13.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" -dependencies = [ - "cfg-if", - "fnv", - "lazy_static", - "memchr", - "parking_lot", - "protobuf", - "thiserror", -] - -[[package]] -name = "protobuf" -version = "2.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" - [[package]] name = "qoi" version = "0.4.1" @@ -3629,6 +3562,21 @@ dependencies = [ "image", ] +[[package]] +name = "quanta" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e5167a477619228a0b284fac2674e3c388cba90631d7b7de620e6f1fcd08da5" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "2.0.1" @@ -3740,6 +3688,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "11.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb9ee317cfe3fbd54b36a511efc1edd42e216903c9cd575e686dd68a2ba90d8d" +dependencies = [ + "bitflags 2.5.0", +] + [[package]] name = "rayon" version = "1.10.0" @@ -4393,6 +4350,12 @@ dependencies = [ "quote", ] +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" + [[package]] name = "slab" version = "0.4.9" @@ -5061,12 +5024,6 @@ dependencies = [ "serde", ] -[[package]] -name = "urlencoding" -version = "2.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" - [[package]] name = "utf-8" version = "0.7.6" diff --git a/fedimint-clientd/Cargo.toml b/fedimint-clientd/Cargo.toml index 58ff9a4..1f57b3a 100644 --- a/fedimint-clientd/Cargo.toml +++ b/fedimint-clientd/Cargo.toml @@ -39,5 +39,8 @@ futures-util = "0.3.30" clap = { version = "3", features = ["derive", "env"] } multimint = { version = "0.3.8" } # multimint = { path = "../multimint" } -axum-otel-metrics = "0.8.0" hex = "0.4.3" + +futures = "0.3" +metrics = { version = "0.23", default-features = false } +metrics-exporter-prometheus = { version = "0.15.3", default-features = false } diff --git a/fedimint-clientd/src/main.rs b/fedimint-clientd/src/main.rs index ed2b46a..5ff8c5e 100644 --- a/fedimint-clientd/src/main.rs +++ b/fedimint-clientd/src/main.rs @@ -1,8 +1,16 @@ +use std::future::ready; use std::path::PathBuf; use std::str::FromStr; +use std::time::Instant; use anyhow::Result; +use axum::extract::{MatchedPath, Request}; use axum::http::Method; +use axum::middleware::{self, Next}; +use axum::response::IntoResponse; +use futures::future::TryFutureExt; +use futures::try_join; +use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle}; use multimint::fedimint_core::api::InviteCode; use router::handlers::{admin, ln, mint, onchain}; use router::ws::websocket_handler; @@ -17,7 +25,6 @@ mod utils; use axum::routing::{get, post}; use axum::Router; -use axum_otel_metrics::HttpMetricsLayerBuilder; use clap::{Parser, Subcommand, ValueEnum}; use state::AppState; // use tower_http::cors::{Any, CorsLayer}; @@ -66,6 +73,10 @@ struct Cli { #[clap(long, env = "FEDIMINT_CLIENTD_ADDR", required = true)] addr: String, + /// Prometheus addr + #[clap(long, env = "PROMETHEUS_ADDR", default_value = "127.0.0.1:3001")] + prometheus_addr: String, + /// Manual secret #[clap(long, env = "FEDIMINT_CLIENTD_MANUAL_SECRET", required = false)] manual_secret: Option, @@ -114,50 +125,102 @@ async fn main() -> Result<()> { return Err(anyhow::anyhow!("No clients found, must have at least one client to start the server. Try providing a federation invite code with the `--invite-code` flag or setting the `FEDIMINT_CLIENTD_INVITE_CODE` environment variable.")); } - let app = match cli.mode { + let main_server = start_main_server(&cli.addr, &cli.password, cli.mode, state) + .map_err(|e| e.context("main server has failed")); + let metrics_server = start_metrics_server(&cli.prometheus_addr) + .map_err(|e| e.context("metrics server has failed")); + + try_join!(main_server, metrics_server)?; + Ok(()) +} + +async fn start_main_server( + addr: &str, + password: &str, + mode: Mode, + state: AppState, +) -> anyhow::Result<()> { + let app = match mode { Mode::Rest => Router::new() .nest("/v2", fedimint_v2_rest()) .with_state(state) - .layer(ValidateRequestHeaderLayer::bearer(&cli.password)), + .layer(ValidateRequestHeaderLayer::bearer(password)), Mode::Ws => Router::new() .route("/ws", get(websocket_handler)) .with_state(state) - .layer(ValidateRequestHeaderLayer::bearer(&cli.password)), + .layer(ValidateRequestHeaderLayer::bearer(password)), }; - info!("Starting server in {:?} mode", cli.mode); + info!("Starting server in {mode:?} mode"); let cors = CorsLayer::new() - // allow `GET` and `POST` when accessing the resource .allow_methods([Method::GET, Method::POST]) - // allow requests from any origin .allow_origin(Any) - // allow auth header .allow_headers(Any); - let metrics = HttpMetricsLayerBuilder::new() - .with_service_name("fedimint-clientd".to_string()) - .build(); - let app = app .layer(cors) - .layer(TraceLayer::new_for_http()) // tracing requests - // no traces for routes bellow - .route("/health", get(|| async { "Server is up and running!" })) // for health check - // metrics for all routes above - .merge(metrics.routes()) - .layer(metrics); - - let listener = tokio::net::TcpListener::bind(cli.addr.clone()) - .await - .map_err(|e| anyhow::anyhow!("Failed to bind to address, should be a valid address and port like 127.0.0.1:3333: {e}"))?; - info!("fedimint-clientd Listening on {}", &cli.addr); - axum::serve(listener, app) - .await - .map_err(|e| anyhow::anyhow!("Failed to start server: {e}"))?; + .layer(TraceLayer::new_for_http()) + .route("/health", get(|| async { "Server is up and running!" })) + .route_layer(middleware::from_fn(track_metrics)); + + let listener = tokio::net::TcpListener::bind(addr).await?; + info!("fedimint-clientd listening on {addr:?}"); + axum::serve(listener, app).await?; + Ok(()) +} +async fn start_metrics_server(bind: &str) -> anyhow::Result<()> { + let app = metrics_app()?; + let listener = tokio::net::TcpListener::bind(bind).await?; + tracing::info!("Prometheus listening on {}", listener.local_addr()?); + axum::serve(listener, app).await?; Ok(()) } +fn metrics_app() -> anyhow::Result { + let recorder_handle = setup_metrics_recorder()?; + Ok(Router::new().route("/metrics", get(move || ready(recorder_handle.render())))) +} + +fn setup_metrics_recorder() -> anyhow::Result { + const EXPONENTIAL_SECONDS: &[f64] = &[ + 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0, + ]; + + Ok(PrometheusBuilder::new() + .set_buckets_for_metric( + Matcher::Full("http_requests_duration_seconds".to_string()), + EXPONENTIAL_SECONDS, + )? + .install_recorder()?) +} + +async fn track_metrics(req: Request, next: Next) -> impl IntoResponse { + let start = Instant::now(); + let path = if let Some(matched_path) = req.extensions().get::() { + matched_path.as_str().to_owned() + } else { + req.uri().path().to_owned() + }; + let method = req.method().clone(); + + let response = next.run(req).await; + + let latency = start.elapsed().as_secs_f64(); + let status = response.status().as_u16().to_string(); + + let labels = [ + ("method", method.to_string()), + ("path", path), + ("status", status), + ]; + + metrics::counter!("http_requests_total", &labels).increment(1); + metrics::histogram!("http_requests_duration_seconds", &labels).record(latency); + + response +} + /// Implements Fedimint V0.2 API Route matching against CLI commands: /// - `/fedimint/v2/admin/backup`: Upload the (encrypted) snapshot of mint notes /// to federation. diff --git a/fedimint-nwc/src/services/nostr.rs b/fedimint-nwc/src/services/nostr.rs index d924c38..740fbf0 100644 --- a/fedimint-nwc/src/services/nostr.rs +++ b/fedimint-nwc/src/services/nostr.rs @@ -51,7 +51,7 @@ impl NostrService { .map(|line| line.to_string()) .collect::>(); - let client = Client::new(&Keys::new(server_key.into())); + let client = Client::new(Keys::new(server_key.into())); let service = Self { client, server_key, diff --git a/flake.lock b/flake.lock index 69697c7..8b3450d 100644 --- a/flake.lock +++ b/flake.lock @@ -27,17 +27,17 @@ ] }, "locked": { - "lastModified": 1719001124, - "narHash": "sha256-JXrMwYlQarZPyjN5UkD4fS9mrHSE1PUa7P//1Z5Sqr0=", + "lastModified": 1727381935, + "narHash": "sha256-G2fOYRZM7bXK5eBb+GK3k/WmO+q5JA/GtFwSPc3kdc8=", "owner": "tadfisher", "repo": "android-nixpkgs", - "rev": "7fa1348249564e43185d3053f579f9fa923d46cc", + "rev": "522d86121cbd413aff922c54f38106ecf8740107", "type": "github" }, "original": { "owner": "tadfisher", "repo": "android-nixpkgs", - "rev": "7fa1348249564e43185d3053f579f9fa923d46cc", + "rev": "522d86121cbd413aff922c54f38106ecf8740107", "type": "github" } }, @@ -188,11 +188,11 @@ "nixpkgs": "nixpkgs_4" }, "locked": { - "lastModified": 1728332776, - "narHash": "sha256-9sXaZxnrRk/8I2PHUYxKzWuLdL8pOIHXIZoVjrv0kmA=", + "lastModified": 1728337685, + "narHash": "sha256-BJyPRvLR0pHVWkWsJAa0rcUydtUtavScLA+CEg+IoWk=", "owner": "fedimint", "repo": "fedimint", - "rev": "ab014f58ea178a5b9a36b6426cb270b815b6d630", + "rev": "5d49668f83d291c4185d4f39ac40e6218f503fb2", "type": "github" }, "original": { @@ -427,17 +427,17 @@ "systems": "systems_4" }, "locked": { - "lastModified": 1719004469, - "narHash": "sha256-TZSHiEJ3qYgA46vikQKT2bwGCEF2LrJVw7cettqa+/g=", + "lastModified": 1727478500, + "narHash": "sha256-nrDYdwIAI1nskNEE/r9rhDJDaouZ4tpSyURfndRsPho=", "owner": "dpc", "repo": "flakebox", - "rev": "12d5ee4f6c47bc01f07ec6f5848a83db265902d3", + "rev": "ee39d59b2c3779e5827f8fa2d269610c556c04c8", "type": "github" }, "original": { "owner": "dpc", "repo": "flakebox", - "rev": "12d5ee4f6c47bc01f07ec6f5848a83db265902d3", + "rev": "ee39d59b2c3779e5827f8fa2d269610c556c04c8", "type": "github" } },