ingestion: add histogram metric for request latency

ingestion/src/lib.rs

@@ -1,6 +1,7 @@
 use axum::{
-    extract::{FromRef, State},
-    response::Json,
+    extract::{FromRef, MatchedPath, Request, State},
+    middleware::{self, Next},
+    response::{IntoResponse, Json},
     routing::post,
     Router,
 };
@@ -430,6 +431,32 @@ fn get_conversions(filename: &str) -> Result<ParamConversions, csv::Error> {
     ))
 }
 
+/// Middleware function that runs around a request, so we can record how long it took
+async fn track_request_duration(req: Request, next: Next) -> impl IntoResponse {
+    let start = std::time::Instant::now();
+    let path = if let Some(matched_path) = req.extensions().get::<MatchedPath>() {
+        matched_path.as_str().to_owned()
+    } else {
+        req.uri().path().to_owned()
+    };
+    let method = req.method().clone();
+
+    let response = next.run(req).await;
+
+    let latency = start.elapsed().as_secs_f64();
+    let status = response.status().as_u16().to_string();
+
+    let labels = [
+        ("method", method.to_string()),
+        ("path", path),
+        ("status", status),
+    ];
+
+    metrics::histogram!("http_requests_duration_seconds", &labels).record(latency);
+
+    response
+}
+
 pub async fn run(
     db_pool: PgConnectionPool,
     param_conversion_path: &str,
@@ -448,6 +475,7 @@ pub async fn run(
     // build our application with a single route
     let app = Router::new()
         .route("/kldata", post(handle_kldata))
+        .route_layer(middleware::from_fn(track_request_duration))
         .with_state(IngestorState {
             db_pool,
             param_conversions,

ingestion/src/main.rs

@@ -1,5 +1,5 @@
 use bb8_postgres::PostgresConnectionManager;
-use metrics_exporter_prometheus::PrometheusBuilder;
+use metrics_exporter_prometheus::{Matcher, PrometheusBuilder};
 use rove_connector::Connector;
 use std::sync::{Arc, RwLock};
 use tokio_postgres::NoTls;
@@ -79,9 +79,17 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
 
     // Set up prometheus metrics exporter
     PrometheusBuilder::new()
+        .set_buckets_for_metric(
+            Matcher::Full("http_requests_duration_seconds".to_string()),
+            &[
+                0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
+            ],
+        )
+        .expect("Failed to set metric buckets")
         .install()
         .expect("Failed to set up metrics exporter");
 
+    // Register metrics so they're guaranteed to show in exporter output
     let _ = metrics::counter!("kldata_messages_received");
     let _ = metrics::counter!("kldata_failures");
     let _ = metrics::counter!("kafka_messages_received");