From 8e06bfc973b4f754d87f1561768f5e8fa986eae3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Trygve=20Laugst=C3=B8l?= Date: Thu, 2 Jan 2025 08:43:45 +0100 Subject: [PATCH] ops: Working Loki integration for Scaleway --- Cargo.lock | 93 ++++++++++++++++++++++++++++++++++++++++++++- backend/Cargo.toml | 1 + backend/src/main.rs | 74 +++++++++++++++++++++++++++++++----- 3 files changed, 157 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b0056b..a1d3aa7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1546,6 +1546,16 @@ version = "0.4.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" +[[package]] +name = "loki-api" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "674883a98273598ac3aad4301724c56734bea90574c5033af067e8f9fb5eb399" +dependencies = [ + "prost", + "prost-types", +] + [[package]] name = "matchers" version = "0.1.0" @@ -1954,6 +1964,38 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn 2.0.91", +] + +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost", +] + [[package]] name = "quote" version = "1.0.37" @@ -2460,6 +2502,7 @@ dependencies = [ "tokio", "tower-http", "tracing", + "tracing-loki", "tracing-subscriber", "url", ] @@ -2510,6 +2553,12 @@ dependencies = [ "serde", ] +[[package]] +name = "snap" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b6b67fb9a61334225b5b790716f609cd58395f895b3fe8b328786812a40bc3b" + [[package]] name = "socket2" version = "0.5.8" @@ -3138,6 +3187,17 @@ dependencies = [ "valuable", ] +[[package]] +name = "tracing-log" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f751112709b4e791d8ce53e32c4ed2d353565a795ce84da2285393f41557bdf2" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-log" version = "0.2.0" @@ -3149,6 +3209,37 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-loki" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fea6023f9fe4b69267ccd3ed7d203d931c43c5f82dbaa0f07202bc17193a5f43" +dependencies = [ + "loki-api", + "reqwest 0.12.9", + "serde", + "serde_json", + "snap", + "tokio", + "tokio-stream", + "tracing", + "tracing-core", + "tracing-log 0.1.4", + "tracing-serde", + "tracing-subscriber", + "url", +] + +[[package]] +name = "tracing-serde" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc6b213177105856957181934e4920de57730fc69bf42c37ee5bb664d406d9e1" +dependencies = [ + "serde", + "tracing-core", +] + [[package]] name = "tracing-subscriber" version = "0.3.19" @@ -3164,7 +3255,7 @@ dependencies = [ "thread_local", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", ] [[package]] diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 911e6ae..113c4a6 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -23,5 +23,6 @@ time = "0.3.37" tokio = { version = "1.42.0", features = ["rt", "rt-multi-thread", "macros"] } tower-http = { version = "0.6.2", features = ["fs", "trace"] } tracing = { version = "0.1.41", features = ["log"] } +tracing-loki = "0.2.5" tracing-subscriber = { version = "0.3.19", features = ["env-filter"] } url = "2.5.4" diff --git a/backend/src/main.rs b/backend/src/main.rs index e71c992..5ce0a80 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -27,31 +27,30 @@ use reqwest::Client as ReqwestClient; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgConnectOptions; use std::path::Path; +use std::process; use std::process::exit; use tokio::net::TcpListener; use tokio::signal; use tower_http::{services::ServeDir, trace::TraceLayer}; use tracing::{debug, info, span, warn, Level}; +use tracing_loki; use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; +use url::Url; static COOKIE_NAME: &str = "SESSION"; #[tokio::main] async fn main() { - tracing_subscriber::registry() - .with( - tracing_subscriber::EnvFilter::try_from_default_env() - .unwrap_or_else(|_| format!("{}=debug", env!("CARGO_CRATE_NAME")).into()), - ) - .with(tracing_subscriber::fmt::layer()) - .init(); - - warn!("skjera starting"); - // We don't care if there is a problem here let env = dotenv::dotenv(); let is_local = env.is_ok(); + if let Err(err) = configure_logging() { + println!("{}", err); + } + + warn!("skjera starting"); + let options = match std::env::var("DATABASE_URL") { Ok(url) => match url.parse::() { Ok(options) => options, @@ -98,6 +97,61 @@ async fn main() { start_server(server_impl, "0.0.0.0:8080").await } +fn configure_logging() -> Result<(), anyhow::Error> { + let mut configure_console = true; + + if let Ok(loki_url) = std::env::var("LOKI_URL") { + let loki_token = std::env::var("LOKI_TOKEN").ok(); + + configure_loki(Url::parse(&loki_url)?, loki_token)?; + configure_console = false; + } + + if configure_console { + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| format!("{}=debug", env!("CARGO_CRATE_NAME")).into()), + ) + .with(tracing_subscriber::fmt::layer()) + .init(); + }; + + Ok(()) +} + +fn configure_loki(loki_url: Url, loki_token: Option) -> Result<(), anyhow::Error> { + let mut b = tracing_loki::builder() + .label("host", "mine")? + .extra_field("pid", format!("{}", process::id()))?; + + if let Some(loki_token) = loki_token { + b = b.http_header("Authorization", format!("Bearer {}", loki_token))? + // b = b.http_header("X-Token", loki_token)? + } + + let (layer, task) = b.build_url(loki_url.clone())?; + + // We need to register our layer with `tracing`. + tracing_subscriber::registry() + .with(layer) + .with(tracing_subscriber::fmt::layer()) + .init(); + + // The background task needs to be spawned so the logs actually get + // delivered. + tokio::spawn(task); + + println!("Logging to Loki: {}", loki_url); + + info!( + task = "tracing_setup", + result = "success", + "tracing successfully set up", + ); + + Ok(()) +} // async fn run_migrations(pg_options: PgConnectOptions) -> Result<(), anyhow::Error> { // let postgres_pool = PgPoolOptions::new() // .max_connections(1)