From 6cc8cda042e7a01db0404be9ba73a1678ee00f9c Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Tue, 28 Mar 2023 19:50:30 +1030 Subject: [PATCH 01/16] feat: add opentelemetry metrics and host functions --- Cargo.lock | 150 +++++++++ Cargo.toml | 4 + crates/lunatic-metrics-api/Cargo.toml | 11 +- crates/lunatic-metrics-api/src/lib.rs | 316 +++++++++++++++++- crates/lunatic-process/src/lib.rs | 3 +- .../lunatic-process/src/runtimes/wasmtime.rs | 4 + crates/lunatic-process/src/state.rs | 3 + flake.lock | 77 +++++ flake.nix | 54 +++ src/main.rs | 8 +- src/mode/cargo_test.rs | 12 + src/mode/common.rs | 22 ++ src/mode/execution.rs | 5 +- src/mode/node.rs | 1 + src/mode/run.rs | 28 ++ src/state.rs | 127 ++++++- 16 files changed, 797 insertions(+), 28 deletions(-) create mode 100644 flake.lock create mode 100644 flake.nix diff --git a/Cargo.lock b/Cargo.lock index 0abab3bac..1b79b05f7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -995,6 +995,7 @@ checksum = "531ac96c6ff5fd7c62263c5e3c67a603af4fcaee2e1a0ae5565ba3a11e69e549" dependencies = [ "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", @@ -1017,12 +1018,34 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86d7a0c1aa76363dac491de0ee99faf6941128376f1cf96f07db7603b7de69dd" +[[package]] +name = "futures-executor" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1997dd9df74cdac935c76252744c1ed5794fac083242ea4fe77ef3ed60ba0f83" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + [[package]] name = "futures-io" version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89d422fa3cbe3b40dca574ab087abb5bc98258ea57eea3fd6f1fa7162c778b91" +[[package]] +name = "futures-macro" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eb14ed937631bd8b8b8977f2c198443447a8355b6e3ca599f38c975e5a963b6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "futures-sink" version = "0.3.27" @@ -1041,11 +1064,16 @@ version = "0.3.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ef6b17e481503ec85211fed8f39d1970f128935ca1f814cd32ac4a6842e84ab" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -1300,6 +1328,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "integer-encoding" +version = "3.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" + [[package]] name = "io-extras" version = "0.17.2" @@ -1550,9 +1584,14 @@ name = "lunatic-metrics-api" version = "0.12.0" dependencies = [ "anyhow", + "hash-map-id", "log", "lunatic-common-api", + "lunatic-process", + "lunatic-process-api", "metrics", + "opentelemetry", + "serde_json", "wasmtime", ] @@ -1647,6 +1686,8 @@ dependencies = [ "lunatic-version-api", "lunatic-wasi-api", "metrics-exporter-prometheus", + "opentelemetry", + "opentelemetry-jaeger", "regex", "reqwest", "serde", @@ -2019,6 +2060,76 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "opentelemetry" +version = "0.19.0" +dependencies = [ + "opentelemetry_api", + "opentelemetry_sdk", +] + +[[package]] +name = "opentelemetry-jaeger" +version = "0.18.0" +dependencies = [ + "async-trait", + "futures", + "futures-executor", + "once_cell", + "opentelemetry", + "opentelemetry-semantic-conventions", + "thiserror", + "thrift", + "tokio", +] + +[[package]] +name = "opentelemetry-semantic-conventions" +version = "0.11.0" +dependencies = [ + "opentelemetry", +] + +[[package]] +name = "opentelemetry_api" +version = "0.19.0" +dependencies = [ + "futures-channel", + "futures-util", + "indexmap", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", +] + +[[package]] +name = "opentelemetry_sdk" +version = "0.19.0" +dependencies = [ + "async-trait", + "crossbeam-channel", + "futures-channel", + "futures-executor", + "futures-util", + "once_cell", + "opentelemetry_api", + "percent-encoding", + "rand", + "thiserror", + "tokio", + "tokio-stream", +] + +[[package]] +name = "ordered-float" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7940cf2ca942593318d07fcf2596cdca60a85c9e7fab408a5e21a4f9dcd40d87" +dependencies = [ + "num-traits", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -2879,6 +2990,28 @@ dependencies = [ "syn", ] +[[package]] +name = "threadpool" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d050e60b33d41c19108b32cea32164033a9013fe3b46cbd4457559bfbf77afaa" +dependencies = [ + "num_cpus", +] + +[[package]] +name = "thrift" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e54bc85fc7faa8bc175c4bab5b92ba8d9a3ce893d0e9f42cc455c8ab16a9e09" +dependencies = [ + "byteorder", + "integer-encoding", + "log", + "ordered-float", + "threadpool", +] + [[package]] name = "time" version = "0.3.20" @@ -2981,6 +3114,17 @@ dependencies = [ "webpki", ] +[[package]] +name = "tokio-stream" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fb52b74f05dbf495a8fba459fdc331812b96aa086d9eb78101fa0d4569c3313" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "tokio-util" version = "0.7.7" @@ -3154,6 +3298,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + [[package]] name = "uuid" version = "1.3.0" diff --git a/Cargo.toml b/Cargo.toml index 3c0f69ce1..4516bd5b2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ metrics = [ "lunatic-registry-api/metrics", "lunatic-timer-api/metrics", "dep:lunatic-metrics-api", + "dep:opentelemetry", ] prometheus = ["dep:metrics-exporter-prometheus", "metrics"] @@ -61,6 +62,8 @@ dashmap = { workspace = true } env_logger = "0.9" log = { workspace = true } metrics-exporter-prometheus = { version = "0.11.0", optional = true } +opentelemetry = { workspace = true, features = ["rt-tokio"], optional = true } +opentelemetry-jaeger = { version = "0.18.0", features = ["rt-tokio"], path = "../../tqwewe/opentelemetry-rust/opentelemetry-jaeger" } regex = "1.7" reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -127,6 +130,7 @@ anyhow = "1.0" bincode = "1.3" dashmap = "5.4" log = "0.4" +opentelemetry = { version = "0.19", path = "../../tqwewe/opentelemetry-rust/opentelemetry" } metrics = "0.20.1" reqwest = "0.11" rustls-pemfile = "1.0" diff --git a/crates/lunatic-metrics-api/Cargo.toml b/crates/lunatic-metrics-api/Cargo.toml index 72860bf1e..ecb206a60 100644 --- a/crates/lunatic-metrics-api/Cargo.toml +++ b/crates/lunatic-metrics-api/Cargo.toml @@ -9,7 +9,12 @@ license = "Apache-2.0/MIT" [dependencies] anyhow = { workspace = true } -wasmtime = { workspace = true } -metrics = { workspace = true } -lunatic-common-api = { workspace = true } +hash-map-id = { workspace = true } log = { workspace = true } +lunatic-common-api = { workspace = true } +lunatic-process = { workspace = true } +lunatic-process-api = { workspace = true } +opentelemetry = { workspace = true } +serde_json = "1.0" +metrics = { workspace = true } +wasmtime = { workspace = true } diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 2da8d4271..78ab8dd51 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -1,10 +1,44 @@ +use std::borrow::Cow; + use anyhow::Result; +use hash_map_id::HashMapId; +use log::{Level, Record}; use lunatic_common_api::{get_memory, IntoTrap}; +use lunatic_process::state::ProcessState; +use lunatic_process_api::ProcessCtx; use metrics::{counter, decrement_gauge, gauge, histogram, increment_counter, increment_gauge}; +use opentelemetry::{ + trace::{Span, SpanRef, Tracer}, + Context, KeyValue, StringValue, +}; +use serde_json::Map; use wasmtime::{Caller, Linker}; +pub type SpanResources = HashMapId; +pub type TracerSpan = ::Span; + +pub trait MetricsCtx { + type Tracer: Tracer + Send + Sync; + + fn log(&self, record: &Record); + fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option + where + T: Into>, + I: IntoIterator; + fn drop_span(&mut self, id: u64); + fn get_span(&self, id: u64) -> Option>; + fn get_last_span(&self) -> SpanRef; +} + /// Links the [Metrics](https://crates.io/crates/metrics) APIs -pub fn register(linker: &mut Linker) -> anyhow::Result<()> { +pub fn register(linker: &mut Linker) -> anyhow::Result<()> +where + T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync + 'static, + <::Tracer as Tracer>::Span: Send + Sync, +{ + linker.func_wrap("lunatic::metrics", "start_span", start_span)?; + linker.func_wrap("lunatic::metrics", "add_event", add_event)?; + linker.func_wrap("lunatic::metrics", "drop_span", drop_span)?; linker.func_wrap("lunatic::metrics", "counter", counter)?; linker.func_wrap("lunatic::metrics", "increment_counter", increment_counter)?; linker.func_wrap("lunatic::metrics", "gauge", gauge)?; @@ -14,14 +48,12 @@ pub fn register(linker: &mut Linker) -> anyhow::Result<()> { Ok(()) } -fn get_string_arg( - caller: &mut Caller, +fn get_string_arg( + memory_slice: &[u8], name_str_ptr: u32, name_str_len: u32, func_name: &str, ) -> Result { - let memory = get_memory(caller)?; - let memory_slice = memory.data(caller); let name = memory_slice .get(name_str_ptr as usize..(name_str_ptr + name_str_len) as usize) .or_trap(func_name)?; @@ -29,6 +61,157 @@ fn get_string_arg( Ok(name) } +/// TODO docs +fn start_span( + mut caller: Caller<'_, T>, + parent_span: u64, + name_str_ptr: u32, + name_str_len: u32, + attributes_ptr: u32, + attributes_len: u32, +) -> Result +where + T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, + <::Tracer as Tracer>::Span: Send + Sync, +{ + let memory = get_memory(&mut caller)?; + let (memory_slice, state) = memory.data_and_store_mut(&mut caller); + + let parent = if parent_span != u64::MAX { + Some(parent_span) + } else { + None + }; + + let name = get_string_arg( + memory_slice, + name_str_ptr, + name_str_len, + "lunatic::metrics::start_span", + )?; + let attributes = if attributes_len > 0 { + let attributes_data = memory_slice + .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) + .or_trap("lunatic::metrics::start_span")?; + let attributes_json = + serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::start_span")?; + data_to_opentelemetry(attributes_json) + } else { + vec![] + }; + + let id = state + .add_span(parent, name, attributes) + .or_trap("lunatic::metrics::start_span")?; + + Ok(id) +} + +/// TODO docs +fn add_event( + mut caller: Caller<'_, T>, + span_id: u64, + name_str_ptr: u32, + name_str_len: u32, + attributes_ptr: u32, + attributes_len: u32, +) -> Result<()> +where + T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, + <::Tracer as Tracer>::Span: Send + Sync, +{ + let memory = get_memory(&mut caller)?; + let (memory_slice, state) = memory.data_and_store_mut(&mut caller); + + let name = get_string_arg( + memory_slice, + name_str_ptr, + name_str_len, + "lunatic::metrics::add_event", + )?; + + let attributes = if attributes_len > 0 { + let attributes_data = memory_slice + .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) + .or_trap("lunatic::metrics::add_event")?; + let attributes_json: Map = + serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::add_event")?; + + let level = attributes_json + .get("level") + .and_then(|level| level.as_str()) + .and_then(|level| level.parse().ok()) + .unwrap_or(Level::Info); + let message = attributes_json + .get("message") + .and_then(|message| message.as_str()) + .unwrap_or(&name); + let target = attributes_json + .get("target") + .and_then(|target| target.as_str()) + .or(state.module().module().name()) + .unwrap_or(""); + let file = attributes_json.get("file").and_then(|file| file.as_str()); + let line = attributes_json + .get("line") + .and_then(|line| line.as_u64().map(|line| line as u32)); + let module_path = attributes_json + .get("module_path") + .and_then(|module_path| module_path.as_str()); + + state.log( + &Record::builder() + .args(format_args!("{message}")) + .level(level) + .target(target) + .file(file) + .line(line) + .module_path(module_path) + .build(), + ); + + data_to_opentelemetry(attributes_json) + } else { + let message = &name; + let target = state.module().module().name().unwrap_or(""); + + state.log( + &Record::builder() + .args(format_args!("{message}")) + .target(target) + .build(), + ); + + vec![] + }; + + let span = if span_id != u64::MAX { + state + .get_span(span_id) + .or_trap("lunatic::metrics::add_event")? + } else { + state.get_last_span() + }; + + span.add_event(name, attributes); + + Ok(()) +} + +/// TODO docs +fn drop_span(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, + <::Tracer as Tracer>::Span: Send + Sync, +{ + let memory = get_memory(&mut caller)?; + let (_data, state) = memory.data_and_store_mut(&mut caller); + + state.drop_span(id); + + Ok(()) +} + /// Sets a counter. /// /// Traps: @@ -40,8 +223,11 @@ fn counter( name_str_len: u32, value: u64, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::counter", @@ -61,8 +247,11 @@ fn increment_counter( name_str_ptr: u32, name_str_len: u32, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::increment_counter", @@ -83,8 +272,11 @@ fn gauge( name_str_len: u32, value: f64, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::gauge", @@ -105,8 +297,11 @@ fn increment_gauge( name_str_len: u32, value: f64, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::increment_gauge", @@ -127,8 +322,11 @@ fn decrement_gauge( name_str_len: u32, value: f64, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::decrement_gauge", @@ -149,8 +347,11 @@ fn histogram( name_str_len: u32, value: f64, ) -> Result<()> { + let memory = get_memory(&mut caller)?; + let memory_slice = memory.data(&mut caller); + let name = get_string_arg( - &mut caller, + memory_slice, name_str_ptr, name_str_len, "lunatic::metrics::histogram", @@ -159,3 +360,96 @@ fn histogram( histogram!(name, value); Ok(()) } + +fn data_to_opentelemetry(data: Map) -> Vec { + data.into_iter() + .map(|(k, v)| KeyValue { + key: k.to_string().into(), + value: json_to_opentelemetry(v), // TODO: Remove this clone + }) + .collect() +} + +fn json_to_opentelemetry(value: serde_json::Value) -> opentelemetry::Value { + match value { + serde_json::Value::Null => "null".into(), + serde_json::Value::Bool(b) => opentelemetry::Value::Bool(b), + serde_json::Value::Number(n) => n + .as_f64() + .map(opentelemetry::Value::F64) + .or_else(|| n.as_i64().map(opentelemetry::Value::I64)) + .unwrap(), + serde_json::Value::String(s) => s.into(), + serde_json::Value::Array(a) => { + let first_type = a.first(); + let valid_ot_array = a.iter().skip(1).all(|value| match first_type { + Some(serde_json::Value::Null) => false, + Some(serde_json::Value::Bool(_)) => value.is_boolean(), + Some(serde_json::Value::Number(n)) if n.is_f64() => value.is_f64(), + Some(serde_json::Value::Number(n)) if n.is_i64() => value.is_i64(), + Some(serde_json::Value::String(_)) => value.is_string(), + Some(serde_json::Value::Array(_)) => false, + Some(serde_json::Value::Object(_)) => false, + _ => false, + }); + // if the json array can be represented as a opentelemetry array, then convert + // accoridngly. Otherwise, just convert each value to a string. + if valid_ot_array { + match first_type.unwrap() { + serde_json::Value::Bool(_) => opentelemetry::Value::Array( + a.into_iter() + .map(|value| match value { + serde_json::Value::Bool(_) => value.as_bool(), + _ => None, + }) + .collect::>>() + .unwrap() + .into(), + ), + serde_json::Value::Number(n) if n.is_f64() => opentelemetry::Value::Array( + a.into_iter() + .map(|value| match value { + serde_json::Value::Number(_) => value.as_f64(), + _ => None, + }) + .collect::>>() + .unwrap() + .into(), + ), + serde_json::Value::Number(n) if n.is_i64() => opentelemetry::Value::Array( + a.into_iter() + .map(|value| match value { + serde_json::Value::Number(_) => value.as_i64(), + _ => None, + }) + .collect::>>() + .unwrap() + .into(), + ), + serde_json::Value::String(_) => opentelemetry::Value::Array( + a.into_iter() + .map(|value| match value { + serde_json::Value::String(_) => match value { + serde_json::Value::String(s) => Some(StringValue::from(s)), + _ => None, + }, + _ => None, + }) + .collect::>>() + .unwrap() + .into(), + ), + _ => unreachable!("we already checked for other types"), + } + } else { + opentelemetry::Value::Array( + a.into_iter() + .map(|value| StringValue::from(value.to_string())) + .collect::>() + .into(), + ) + } + } + serde_json::Value::Object(o) => serde_json::to_string(&o).unwrap().into(), + } +} diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index 719921238..9928f3fb8 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -378,8 +378,6 @@ where } }; - env.remove_process(id); - match result { Finished::Normal(result) => { let result: ExecutionResult<_> = result.into(); @@ -539,4 +537,5 @@ pub enum ResultValue { Ok, Failed(String), SpawnError(String), + Killed, } diff --git a/crates/lunatic-process/src/runtimes/wasmtime.rs b/crates/lunatic-process/src/runtimes/wasmtime.rs index 094a08181..3a7829a5f 100644 --- a/crates/lunatic-process/src/runtimes/wasmtime.rs +++ b/crates/lunatic-process/src/runtimes/wasmtime.rs @@ -101,6 +101,10 @@ impl WasmtimeCompiledModule { &self.inner.source } + pub fn module(&self) -> &wasmtime::Module { + &self.inner.module + } + pub fn instantiator(&self) -> &wasmtime::InstancePre { &self.inner.instance_pre } diff --git a/crates/lunatic-process/src/state.rs b/crates/lunatic-process/src/state.rs index 6d13d673a..a3e395124 100644 --- a/crates/lunatic-process/src/state.rs +++ b/crates/lunatic-process/src/state.rs @@ -66,4 +66,7 @@ pub trait ProcessState: Sized { fn registry_atomic_put( &mut self, ) -> &mut Option>>; + + // Spans + // fn span_resources(&self) -> &SpanResources<> } diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..09540df62 --- /dev/null +++ b/flake.lock @@ -0,0 +1,77 @@ +{ + "nodes": { + "flake-utils": { + "locked": { + "lastModified": 1659877975, + "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1671636183, + "narHash": "sha256-dboEYqb7vnH9pVEwgaWz7dzVi7eh6N5tRuhJ/nluoCg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "60ff1ccd98a2f81347457a473c7a96b9b6166c88", + "type": "github" + }, + "original": { + "owner": "NixOS", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs_2": { + "locked": { + "lastModified": 1665296151, + "narHash": "sha256-uOB0oxqxN9K7XGF1hcnY+PQnlQJ+3bP2vCn/+Ru/bbc=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "14ccaaedd95a488dd7ae142757884d8e125b3363", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "nixpkgs": "nixpkgs", + "rust-overlay": "rust-overlay" + } + }, + "rust-overlay": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1671589280, + "narHash": "sha256-FmJ4SC+Ewi1iMhdtRcrwirMfvW7h2jakT7ILLo9BVws=", + "owner": "oxalica", + "repo": "rust-overlay", + "rev": "bfc54bcf98dacdc649c88a82bf14d00b399aa3bb", + "type": "github" + }, + "original": { + "owner": "oxalica", + "repo": "rust-overlay", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..993fb7a00 --- /dev/null +++ b/flake.nix @@ -0,0 +1,54 @@ +{ + description = "Example Rust development environment for Zero to Nix"; + + # Flake inputs + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs"; # also valid: "nixpkgs" + rust-overlay.url = "github:oxalica/rust-overlay"; # A helper for Rust + Nix + }; + + # Flake outputs + outputs = { self, nixpkgs, rust-overlay }: + let + # Overlays enable you to customize the Nixpkgs attribute set + overlays = [ + # Makes a `rust-bin` attribute available in Nixpkgs + (import rust-overlay) + # Provides a `rustToolchain` attribute for Nixpkgs that we can use to + # create a Rust environment + (self: super: { + rustToolchain = super.rust-bin.stable.latest.default.override { + extensions = [ "rust-src" ]; + }; + }) + ]; + + # Systems supported + allSystems = [ + "x86_64-linux" # 64-bit Intel/AMD Linux + "aarch64-linux" # 64-bit ARM Linux + "x86_64-darwin" # 64-bit Intel macOS + "aarch64-darwin" # 64-bit ARM macOS + ]; + + # Helper to provide system-specific attributes + forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f { + pkgs = import nixpkgs { inherit overlays system; }; + }); + in + { + # Development environment output + devShells = forAllSystems ({ pkgs }: { + default = pkgs.mkShell { + # The Nix packages provided in the environment + packages = (with pkgs; [ + # The package provided by our custom overlay. Includes cargo, Clippy, cargo-fmt, + # rustdoc, rustfmt, and other tools. + rustToolchain + openssl + pkg-config + ]) ++ pkgs.lib.optionals pkgs.stdenv.isDarwin (with pkgs; [ libiconv ]); + }; + }); + }; +} diff --git a/src/main.rs b/src/main.rs index 6a81385c3..f5f881dc4 100644 --- a/src/main.rs +++ b/src/main.rs @@ -74,9 +74,13 @@ async fn main() -> Result<()> { Err(_) => false, }; - if cargo_test { + let result = if cargo_test { cargo_test::test(augmented_args).await } else { execution::execute(augmented_args).await - } + }; + + opentelemetry::global::shutdown_tracer_provider(); + + result } diff --git a/src/mode/cargo_test.rs b/src/mode/cargo_test.rs index 56a4fde15..e463ba6f4 100644 --- a/src/mode/cargo_test.rs +++ b/src/mode/cargo_test.rs @@ -2,6 +2,7 @@ use std::{collections::HashMap, env, fs, path::Path, sync::Arc, time::Instant}; use anyhow::{Context, Result}; use clap::Parser; +use env_logger::filter; use lunatic_process::{ env::{Environment, LunaticEnvironment}, runtimes, @@ -11,6 +12,7 @@ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use lunatic_stdout_capture::StdoutCapture; use lunatic_wasi_api::LunaticWasiCtx; +use opentelemetry::{global::BoxedTracer, sdk::export::trace::stdout, trace::noop::NoopTracer}; use tokio::sync::RwLock; #[derive(Parser, Debug)] @@ -208,6 +210,13 @@ pub(crate) async fn test(augmented_args: Option>) -> Result<()> { let (sender, mut receiver) = tokio::sync::mpsc::unbounded_channel(); let config = Arc::new(config); + let tracer = Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))); + let tracer_context = Arc::new(opentelemetry::Context::new()); + let logger = Arc::new( + env_logger::Builder::from_env("LUNATIC_LOG") + .default_format() + .build(), + ); for test_function in test_functions { // Skip over filtered out functions @@ -236,6 +245,9 @@ pub(crate) async fn test(augmented_args: Option>) -> Result<()> { module.clone(), config.clone(), registry, + tracer.clone(), + tracer_context.clone(), + logger.clone(), ) .unwrap(); diff --git a/src/mode/common.rs b/src/mode/common.rs index a1fc0183f..553ec5104 100644 --- a/src/mode/common.rs +++ b/src/mode/common.rs @@ -3,6 +3,7 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::{anyhow, Context, Result}; use clap::Args; +use env_logger::filter::{self, Filter}; use lunatic_distributed::DistributedProcessState; use lunatic_process::{ env::{Environment, LunaticEnvironment, LunaticEnvironments}, @@ -11,6 +12,12 @@ use lunatic_process::{ }; use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; +use opentelemetry::{ + global::BoxedTracer, + sdk::export::trace::stdout, + trace::{noop::NoopTracer, Span, SpanBuilder, TraceContextExt, Tracer}, + KeyValue, +}; #[derive(Args, Debug)] pub struct WasmArgs {} @@ -24,6 +31,7 @@ pub struct RunWasm { pub envs: Arc, pub env: Arc, pub distributed: Option, + pub tracer: Arc, } pub async fn run_wasm(args: RunWasm) -> Result<()> { @@ -65,6 +73,16 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { }; let module = Arc::new(args.runtime.compile_module::(module)?); + let mut root_span = args.tracer.start( + "app_start", // SpanBuilder::from_name("app_start") + // .with_attributes([KeyValue::new("path", path.to_string_lossy().to_string())]), + ); + let tracer_context = Arc::new(opentelemetry::Context::current_with_span(root_span)); + let logger = Arc::new( + env_logger::Builder::from_env("LUNATIC_LOG") + .default_format() + .build(), + ); let state = DefaultProcessState::new( args.env.clone(), args.distributed, @@ -72,10 +90,14 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { module.clone(), Arc::new(config), Default::default(), + args.tracer.clone(), + tracer_context.clone(), + logger, ) .unwrap(); args.env.can_spawn_next_process().await?; + let (task, _) = spawn_wasm( args.env, args.runtime, diff --git a/src/mode/execution.rs b/src/mode/execution.rs index eb69cfbad..6ae2f1da8 100644 --- a/src/mode/execution.rs +++ b/src/mode/execution.rs @@ -30,7 +30,10 @@ enum Commands { } pub(crate) async fn execute(augmented_args: Option>) -> Result<()> { - env_logger::Builder::from_env(env_logger::Env::default().default_filter_or("info")).init(); + env_logger::Builder::from_env( + env_logger::Env::default().default_filter_or("warn,crane*=info,lunatic=info"), + ) + .init(); let args = match augmented_args { Some(a) => Args::parse_from(a), diff --git a/src/mode/node.rs b/src/mode/node.rs index 5a54d1e16..2f23651d2 100644 --- a/src/mode/node.rs +++ b/src/mode/node.rs @@ -129,6 +129,7 @@ pub(crate) async fn start(args: Args) -> Result<()> { envs, env, distributed: Some(dist), + tracer: todo!(), }) .await { diff --git a/src/mode/run.rs b/src/mode/run.rs index 172cab44d..c4bde06c1 100644 --- a/src/mode/run.rs +++ b/src/mode/run.rs @@ -6,11 +6,21 @@ use lunatic_process::{ env::{Environments, LunaticEnvironments}, runtimes::{self}, }; +use opentelemetry::{ + global::BoxedTracer, + runtime::Tokio, + trace::{noop::NoopTracer, Span, Tracer}, + Context, KeyValue, +}; use super::common::{run_wasm, RunWasm}; #[derive(Parser, Debug)] #[command(version)] +#[command(group( + clap::ArgGroup::new("tracer") + .args(["jaeger"]), +))] pub struct Args { /// Grant access to the given host directories #[arg(long, value_name = "DIRECTORY")] @@ -20,6 +30,10 @@ pub struct Args { #[arg(long)] pub bench: bool, + /// Jaeger connection url for tracing. + #[arg(long)] + pub jaeger: Option, + /// Entry .wasm file #[arg(index = 1)] pub path: PathBuf, @@ -48,6 +62,19 @@ pub(crate) async fn start(mut args: Args) -> Result<()> { if args.bench { args.wasm_args.push("--bench".to_owned()); } + + let tracer = match args.jaeger { + Some(jaeger_url) => { + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + let tracer = opentelemetry_jaeger::new_agent_pipeline() + .with_endpoint(jaeger_url) + .with_service_name("lunatic") + .install_batch(Tokio)?; + Arc::new(BoxedTracer::new(Box::new(tracer))) + } + None => Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), + }; + run_wasm(RunWasm { path: args.path, wasm_args: args.wasm_args, @@ -56,6 +83,7 @@ pub(crate) async fn start(mut args: Args) -> Result<()> { envs, env, distributed: None, + tracer, }) .await } diff --git a/src/state.rs b/src/state.rs index a274b84d8..87841f136 100644 --- a/src/state.rs +++ b/src/state.rs @@ -3,9 +3,12 @@ use std::fmt::Debug; use std::sync::Arc; use anyhow::Result; +use env_logger::Logger; use hash_map_id::HashMapId; +use log::{Log, Record}; use lunatic_distributed::{DistributedCtx, DistributedProcessState}; use lunatic_error_api::{ErrorCtx, ErrorResource}; +use lunatic_metrics_api::{MetricsCtx, SpanResources, TracerSpan}; use lunatic_networking_api::{DnsIterator, TlsConnection, TlsListener}; use lunatic_networking_api::{NetworkingCtx, TcpConnection}; use lunatic_process::env::{Environment, LunaticEnvironment}; @@ -21,6 +24,9 @@ use lunatic_sqlite_api::{SQLiteConnections, SQLiteCtx, SQLiteGuestAllocators, SQ use lunatic_stdout_capture::StdoutCapture; use lunatic_timer_api::{TimerCtx, TimerResources}; use lunatic_wasi_api::{build_wasi, LunaticWasiCtx}; +use opentelemetry::global::BoxedTracer; +use opentelemetry::trace::{FutureExt, Span, SpanRef, TraceContextExt, Tracer}; +use opentelemetry::{Context, KeyValue}; use tokio::net::{TcpListener, UdpSocket}; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::{Mutex, RwLock, RwLockWriteGuard}; @@ -75,6 +81,12 @@ pub struct DefaultProcessState { // `RwLock` struct. The lifetime of the lock will need to be extended to `'static`, but this // is a safe operation, because it references a global registry that outlives all processes. registry_atomic_put: Option>>, + // Metrics + // TODO: Does this need to be in an Arc? + tracer: Arc, + tracer_context: Arc, + last_span_id: u64, + logger: Arc, } impl DefaultProcessState { @@ -85,12 +97,16 @@ impl DefaultProcessState { module: Arc>, config: Arc, registry: Arc>>, + tracer: Arc, + tracer_context: Arc, + log_filter: Arc, ) -> Result { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); + let id = environment.get_next_process_id(); let state = Self { - id: environment.get_next_process_id(), + id, environment, distributed, runtime: Some(runtime), @@ -99,7 +115,7 @@ impl DefaultProcessState { message: None, signal_mailbox, message_mailbox, - resources: Resources::default(), + resources: Resources::new(id, &tracer, &tracer_context), wasi: build_wasi( Some(config.command_line_arguments()), Some(config.environment_variables()), @@ -111,6 +127,10 @@ impl DefaultProcessState { registry, db_resources: DbResources::default(), registry_atomic_put: None, + tracer, + tracer_context, + last_span_id: 0, + logger: log_filter, }; Ok(state) } @@ -127,8 +147,9 @@ impl ProcessState for DefaultProcessState { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); + let id = self.environment.get_next_process_id(); let state = Self { - id: self.environment.get_next_process_id(), + id, environment: self.environment.clone(), distributed: self.distributed.clone(), runtime: self.runtime.clone(), @@ -137,7 +158,7 @@ impl ProcessState for DefaultProcessState { message: None, signal_mailbox, message_mailbox, - resources: Resources::default(), + resources: Resources::new(id, &self.tracer, &self.tracer_context), wasi: build_wasi( Some(config.command_line_arguments()), Some(config.environment_variables()), @@ -149,6 +170,10 @@ impl ProcessState for DefaultProcessState { registry: self.registry.clone(), db_resources: DbResources::default(), registry_atomic_put: None, + tracer: Arc::clone(&self.tracer), + tracer_context: Arc::clone(&self.tracer_context), + last_span_id: 0, + logger: self.logger.clone(), }; Ok(state) } @@ -406,10 +431,64 @@ impl SQLiteCtx for DefaultProcessState { } } -#[derive(Default, Debug)] +impl MetricsCtx for DefaultProcessState { + type Tracer = BoxedTracer; + + fn log(&self, record: &Record) { + self.logger.log(record); + } + + fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option + where + T: Into>, + I: IntoIterator, + { + let parent_ctx = if let Some(parent_id) = parent { + self.resources.spans.get(parent_id)? + } else { + &self.resources.process_context + }; + let mut span = self.tracer.start_with_context(name, parent_ctx); + span.set_attributes(attributes); + let context = parent_ctx.with_span(span); + let id = self.resources.spans.add(context); + self.last_span_id = id; + Some(id) + } + + fn drop_span(&mut self, id: u64) { + self.resources.spans.remove(id); + self.last_span_id -= 1; + } + + fn get_span(&self, id: u64) -> Option> { + self.resources.spans.get(id).map(|ctx| ctx.span()) + } + + fn get_last_span(&self) -> SpanRef<'_> { + self.resources + .spans + .get(self.last_span_id) + .map(|ctx| ctx.span()) + .unwrap_or_else(|| self.resources.process_context.span()) + } +} + +impl DefaultProcessState { + fn get_last_context(&self) -> &Context { + self.resources + .spans + .get(self.last_span_id) + .unwrap_or(&self.tracer_context) + } +} + +#[derive(Debug)] pub(crate) struct Resources { pub(crate) configs: HashMapId, pub(crate) modules: HashMapId>>, + pub(crate) spans: SpanResources, + pub(crate) process_context: Context, pub(crate) timers: TimerResources, pub(crate) dns_iterators: HashMapId, pub(crate) tcp_listeners: HashMapId, @@ -420,6 +499,32 @@ pub(crate) struct Resources { pub(crate) errors: HashMapId, } +impl Resources { + fn new(process_id: u64, tracer: &BoxedTracer, tracer_context: &Context) -> Self { + let mut process_span = tracer.start_with_context("process_spawn", tracer_context); + process_span.set_attribute(opentelemetry::KeyValue::new( + "process.id", + process_id as i64, + )); + let process_context = tracer_context.with_span(process_span); + + Resources { + configs: Default::default(), + modules: Default::default(), + spans: Default::default(), + process_context, + timers: Default::default(), + dns_iterators: Default::default(), + tcp_listeners: Default::default(), + tcp_streams: Default::default(), + tls_listeners: Default::default(), + tls_streams: Default::default(), + udp_sockets: Default::default(), + errors: Default::default(), + } + } +} + impl DistributedCtx for DefaultProcessState { fn distributed_mut(&mut self) -> Result<&mut DistributedProcessState> { match self.distributed.as_mut() { @@ -456,6 +561,7 @@ impl DistributedCtx for DefaultProcessState { runtime: WasmtimeRuntime, module: Arc>, config: Arc, + // tracer: Arc, ) -> Result { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); @@ -470,7 +576,7 @@ impl DistributedCtx for DefaultProcessState { message: None, signal_mailbox, message_mailbox, - resources: Resources::default(), + resources: todo!(), // Resources::default(), wasi: build_wasi( Some(config.command_line_arguments()), Some(config.environment_variables()), @@ -482,13 +588,16 @@ impl DistributedCtx for DefaultProcessState { registry: Default::default(), // TODO move registry into env? db_resources: DbResources::default(), registry_atomic_put: None, + tracer: todo!(), // TODO this should not be hard coded like this + tracer_context: todo!(), + last_span_id: 0, + logger: todo!(), }; Ok(state) } } -mod tests { - +/* mod tests { #[tokio::test] async fn import_filter_signature_matches() { use std::collections::HashMap; @@ -529,4 +638,4 @@ mod tests { .await .unwrap(); } -} +} */ From a79aff9ba8ee003050cdd61fbc5900f3b5d5275d Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Thu, 30 Mar 2023 20:01:19 +1030 Subject: [PATCH 02/16] feat: add graceful shutdown --- Cargo.lock | 10 ++++ Cargo.toml | 5 +- crates/lunatic-metrics-api/src/lib.rs | 27 ++++++--- crates/lunatic-process/src/env.rs | 65 ++++++++++++++++++++- crates/lunatic-process/src/lib.rs | 2 + src/main.rs | 2 - src/mode/cargo_test.rs | 3 +- src/mode/common.rs | 29 +++++++--- src/mode/node.rs | 5 +- src/mode/run.rs | 9 +-- src/state.rs | 82 ++++++++++++++------------- 11 files changed, 165 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1b79b05f7..eff65c894 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2800,6 +2800,15 @@ dependencies = [ "dirs", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +dependencies = [ + "libc", +] + [[package]] name = "sketches-ddsketch" version = "0.2.0" @@ -3077,6 +3086,7 @@ dependencies = [ "mio", "num_cpus", "pin-project-lite", + "signal-hook-registry", "socket2", "tokio-macros", "windows-sys 0.45.0", diff --git a/Cargo.toml b/Cargo.toml index 4516bd5b2..d92a8fad2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ metrics = [ "lunatic-registry-api/metrics", "lunatic-timer-api/metrics", "dep:lunatic-metrics-api", - "dep:opentelemetry", ] prometheus = ["dep:metrics-exporter-prometheus", "metrics"] @@ -62,12 +61,12 @@ dashmap = { workspace = true } env_logger = "0.9" log = { workspace = true } metrics-exporter-prometheus = { version = "0.11.0", optional = true } -opentelemetry = { workspace = true, features = ["rt-tokio"], optional = true } +opentelemetry = { workspace = true, features = ["rt-tokio"] } opentelemetry-jaeger = { version = "0.18.0", features = ["rt-tokio"], path = "../../tqwewe/opentelemetry-rust/opentelemetry-jaeger" } regex = "1.7" reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } -tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net", "signal"] } toml = "0.5" uuid = { workspace = true } wasmtime = { workspace = true } diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 78ab8dd51..c7f3eee64 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -8,7 +8,7 @@ use lunatic_process::state::ProcessState; use lunatic_process_api::ProcessCtx; use metrics::{counter, decrement_gauge, gauge, histogram, increment_counter, increment_gauge}; use opentelemetry::{ - trace::{Span, SpanRef, Tracer}, + trace::{SpanRef, Tracer}, Context, KeyValue, StringValue, }; use serde_json::Map; @@ -138,12 +138,19 @@ where serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::add_event")?; let level = attributes_json - .get("level") - .and_then(|level| level.as_str()) - .and_then(|level| level.parse().ok()) + .get("severityNumber") + .and_then(|level| level.as_u64()) + .and_then(|level| match level { + 1..=4 => Some(Level::Trace), + 5..=8 => Some(Level::Debug), + 9..=12 => Some(Level::Info), + 13..=16 => Some(Level::Warn), + 17..=20 => Some(Level::Error), + _ => None, + }) .unwrap_or(Level::Info); let message = attributes_json - .get("message") + .get("body") .and_then(|message| message.as_str()) .unwrap_or(&name); let target = attributes_json @@ -151,12 +158,14 @@ where .and_then(|target| target.as_str()) .or(state.module().module().name()) .unwrap_or(""); - let file = attributes_json.get("file").and_then(|file| file.as_str()); + let file = attributes_json + .get("code.filepath") + .and_then(|file| file.as_str()); let line = attributes_json - .get("line") + .get("code.lineno") .and_then(|line| line.as_u64().map(|line| line as u32)); let module_path = attributes_json - .get("module_path") + .get("code.namespace") .and_then(|module_path| module_path.as_str()); state.log( @@ -365,7 +374,7 @@ fn data_to_opentelemetry(data: Map) -> Vec data.into_iter() .map(|(k, v)| KeyValue { key: k.to_string().into(), - value: json_to_opentelemetry(v), // TODO: Remove this clone + value: json_to_opentelemetry(v), }) .collect() } diff --git a/crates/lunatic-process/src/env.rs b/crates/lunatic-process/src/env.rs index fae17c4a6..e52c2bff5 100644 --- a/crates/lunatic-process/src/env.rs +++ b/crates/lunatic-process/src/env.rs @@ -1,10 +1,16 @@ use anyhow::Result; use async_trait::async_trait; use dashmap::DashMap; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, +use std::{ + future::Future, + pin::Pin, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, + task::{Context, Poll}, }; +use tokio::time::{interval, Interval}; use crate::{Process, Signal}; @@ -18,6 +24,9 @@ pub trait Environment: Send + Sync { fn process_count(&self) -> usize; async fn can_spawn_next_process(&self) -> Result>; fn send(&self, id: u64, signal: Signal); + fn shutdown(&self) -> ShutdownFuture<'_, Self> + where + Self: Sized; } #[async_trait] @@ -101,6 +110,17 @@ impl Environment for LunaticEnvironment { // Don't impose any limits to process spawning Ok(Some(())) } + + fn shutdown(&self) -> ShutdownFuture<'_, Self> + where + Self: Sized, + { + for proc in self.processes.iter() { + proc.send(Signal::Kill); + } + + ShutdownFuture::new(self) + } } #[derive(Clone, Default)] @@ -123,3 +143,42 @@ impl Environments for LunaticEnvironments { self.envs.get(&id).map(|e| e.clone()) } } + +#[derive(Debug)] +pub struct ShutdownFuture<'a, E> { + environment: &'a E, + interval: Interval, +} + +impl<'a, E> ShutdownFuture<'a, E> { + fn new(environment: &'a E) -> Self { + ShutdownFuture { + environment, + interval: interval(tokio::time::Duration::from_millis(50)), + } + } +} + +impl<'a, E> Future for ShutdownFuture<'a, E> +where + E: Environment, +{ + type Output = (); + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + if self.environment.process_count() == 0 { + Poll::Ready(()) + } else { + match self.interval.poll_tick(cx) { + Poll::Ready(_) => { + if self.environment.process_count() == 0 { + Poll::Ready(()) + } else { + Poll::Pending + } + } + Poll::Pending => Poll::Pending, + } + } + } +} diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index 9928f3fb8..aa3f19ab5 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -378,6 +378,8 @@ where } }; + env.remove_process(id); + match result { Finished::Normal(result) => { let result: ExecutionResult<_> = result.into(); diff --git a/src/main.rs b/src/main.rs index f5f881dc4..c7bd75a03 100644 --- a/src/main.rs +++ b/src/main.rs @@ -80,7 +80,5 @@ async fn main() -> Result<()> { execution::execute(augmented_args).await }; - opentelemetry::global::shutdown_tracer_provider(); - result } diff --git a/src/mode/cargo_test.rs b/src/mode/cargo_test.rs index e463ba6f4..287f99f16 100644 --- a/src/mode/cargo_test.rs +++ b/src/mode/cargo_test.rs @@ -2,7 +2,6 @@ use std::{collections::HashMap, env, fs, path::Path, sync::Arc, time::Instant}; use anyhow::{Context, Result}; use clap::Parser; -use env_logger::filter; use lunatic_process::{ env::{Environment, LunaticEnvironment}, runtimes, @@ -12,7 +11,7 @@ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use lunatic_stdout_capture::StdoutCapture; use lunatic_wasi_api::LunaticWasiCtx; -use opentelemetry::{global::BoxedTracer, sdk::export::trace::stdout, trace::noop::NoopTracer}; +use opentelemetry::{global::BoxedTracer, trace::noop::NoopTracer}; use tokio::sync::RwLock; #[derive(Parser, Debug)] diff --git a/src/mode/common.rs b/src/mode/common.rs index 553ec5104..4fc984fe0 100644 --- a/src/mode/common.rs +++ b/src/mode/common.rs @@ -3,7 +3,6 @@ use std::{path::PathBuf, sync::Arc}; use anyhow::{anyhow, Context, Result}; use clap::Args; -use env_logger::filter::{self, Filter}; use lunatic_distributed::DistributedProcessState; use lunatic_process::{ env::{Environment, LunaticEnvironment, LunaticEnvironments}, @@ -14,8 +13,7 @@ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use opentelemetry::{ global::BoxedTracer, - sdk::export::trace::stdout, - trace::{noop::NoopTracer, Span, SpanBuilder, TraceContextExt, Tracer}, + trace::{Span, TraceContextExt, Tracer}, KeyValue, }; @@ -77,11 +75,18 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { "app_start", // SpanBuilder::from_name("app_start") // .with_attributes([KeyValue::new("path", path.to_string_lossy().to_string())]), ); + root_span.set_attribute(KeyValue::new( + "lunatic.path", + path.to_string_lossy().to_string(), + )); let tracer_context = Arc::new(opentelemetry::Context::current_with_span(root_span)); let logger = Arc::new( - env_logger::Builder::from_env("LUNATIC_LOG") - .default_format() - .build(), + env_logger::Builder::from_env( + env_logger::Env::new() + .filter_or("LUNATIC_LOG", "info") + .write_style("LUNATIC_LOG_STYLE"), + ) + .build(), ); let state = DefaultProcessState::new( args.env.clone(), @@ -90,14 +95,22 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { module.clone(), Arc::new(config), Default::default(), - args.tracer.clone(), - tracer_context.clone(), + args.tracer, + tracer_context, logger, ) .unwrap(); args.env.can_spawn_next_process().await?; + let env = args.env.clone(); + tokio::spawn(async move { + if tokio::signal::ctrl_c().await.is_ok() { + let _ = tokio::time::timeout(tokio::time::Duration::from_secs(5), env.shutdown()).await; + std::process::exit(0); + } + }); + let (task, _) = spawn_wasm( args.env, args.runtime, diff --git a/src/mode/node.rs b/src/mode/node.rs index 2f23651d2..882633809 100644 --- a/src/mode/node.rs +++ b/src/mode/node.rs @@ -4,12 +4,13 @@ use std::{ }; use clap::Parser; +use opentelemetry::{global::BoxedTracer, trace::noop::NoopTracer}; use std::{collections::HashMap, sync::Arc}; use anyhow::{anyhow, Context, Result}; use lunatic_distributed::{ - control::{self}, + control, distributed::{self, server::ServerCtx}, quic, }; @@ -129,7 +130,7 @@ pub(crate) async fn start(args: Args) -> Result<()> { envs, env, distributed: Some(dist), - tracer: todo!(), + tracer: Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), }) .await { diff --git a/src/mode/run.rs b/src/mode/run.rs index c4bde06c1..48b15d83e 100644 --- a/src/mode/run.rs +++ b/src/mode/run.rs @@ -4,14 +4,9 @@ use anyhow::Result; use clap::Parser; use lunatic_process::{ env::{Environments, LunaticEnvironments}, - runtimes::{self}, -}; -use opentelemetry::{ - global::BoxedTracer, - runtime::Tokio, - trace::{noop::NoopTracer, Span, Tracer}, - Context, KeyValue, + runtimes, }; +use opentelemetry::{global::BoxedTracer, runtime::Tokio, trace::noop::NoopTracer}; use super::common::{run_wasm, RunWasm}; diff --git a/src/state.rs b/src/state.rs index 87841f136..93e3a547c 100644 --- a/src/state.rs +++ b/src/state.rs @@ -8,7 +8,7 @@ use hash_map_id::HashMapId; use log::{Log, Record}; use lunatic_distributed::{DistributedCtx, DistributedProcessState}; use lunatic_error_api::{ErrorCtx, ErrorResource}; -use lunatic_metrics_api::{MetricsCtx, SpanResources, TracerSpan}; +use lunatic_metrics_api::{MetricsCtx, SpanResources}; use lunatic_networking_api::{DnsIterator, TlsConnection, TlsListener}; use lunatic_networking_api::{NetworkingCtx, TcpConnection}; use lunatic_process::env::{Environment, LunaticEnvironment}; @@ -25,7 +25,8 @@ use lunatic_stdout_capture::StdoutCapture; use lunatic_timer_api::{TimerCtx, TimerResources}; use lunatic_wasi_api::{build_wasi, LunaticWasiCtx}; use opentelemetry::global::BoxedTracer; -use opentelemetry::trace::{FutureExt, Span, SpanRef, TraceContextExt, Tracer}; +use opentelemetry::trace::noop::NoopTracer; +use opentelemetry::trace::{Span, SpanRef, TraceContextExt, Tracer}; use opentelemetry::{Context, KeyValue}; use tokio::net::{TcpListener, UdpSocket}; use tokio::sync::mpsc::unbounded_channel; @@ -105,22 +106,23 @@ impl DefaultProcessState { let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); let id = environment.get_next_process_id(); + let wasi = build_wasi( + Some(config.command_line_arguments()), + Some(config.environment_variables()), + config.preopened_dirs(), + )?; let state = Self { id, environment, distributed, runtime: Some(runtime), module: Some(module), - config: config.clone(), + config, message: None, signal_mailbox, message_mailbox, resources: Resources::new(id, &tracer, &tracer_context), - wasi: build_wasi( - Some(config.command_line_arguments()), - Some(config.environment_variables()), - config.preopened_dirs(), - )?, + wasi, wasi_stdout: None, wasi_stderr: None, initialized: false, @@ -148,30 +150,31 @@ impl ProcessState for DefaultProcessState { let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); let id = self.environment.get_next_process_id(); + let wasi = build_wasi( + Some(config.command_line_arguments()), + Some(config.environment_variables()), + config.preopened_dirs(), + )?; let state = Self { id, environment: self.environment.clone(), distributed: self.distributed.clone(), runtime: self.runtime.clone(), module: Some(module), - config: config.clone(), + config, message: None, signal_mailbox, message_mailbox, resources: Resources::new(id, &self.tracer, &self.tracer_context), - wasi: build_wasi( - Some(config.command_line_arguments()), - Some(config.environment_variables()), - config.preopened_dirs(), - )?, + wasi, wasi_stdout: None, wasi_stderr: None, initialized: false, registry: self.registry.clone(), db_resources: DbResources::default(), registry_atomic_put: None, - tracer: Arc::clone(&self.tracer), - tracer_context: Arc::clone(&self.tracer_context), + tracer: self.tracer.clone(), + tracer_context: self.tracer_context.clone(), last_span_id: 0, logger: self.logger.clone(), }; @@ -448,11 +451,15 @@ impl MetricsCtx for DefaultProcessState { } else { &self.resources.process_context }; + let mut span = self.tracer.start_with_context(name, parent_ctx); span.set_attributes(attributes); + let context = parent_ctx.with_span(span); + let id = self.resources.spans.add(context); self.last_span_id = id; + Some(id) } @@ -474,16 +481,7 @@ impl MetricsCtx for DefaultProcessState { } } -impl DefaultProcessState { - fn get_last_context(&self) -> &Context { - self.resources - .spans - .get(self.last_span_id) - .unwrap_or(&self.tracer_context) - } -} - -#[derive(Debug)] +#[derive(Debug, Default)] pub(crate) struct Resources { pub(crate) configs: HashMapId, pub(crate) modules: HashMapId>>, @@ -561,37 +559,45 @@ impl DistributedCtx for DefaultProcessState { runtime: WasmtimeRuntime, module: Arc>, config: Arc, - // tracer: Arc, + // tracer: Arc, + // tracer_context: Arc, + // logger: Arc, ) -> Result { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); + let id = environment.get_next_process_id(); + let wasi = build_wasi( + Some(config.command_line_arguments()), + Some(config.environment_variables()), + config.preopened_dirs(), + )?; let state = Self { - id: environment.get_next_process_id(), + id, environment, distributed: Some(distributed), runtime: Some(runtime), module: Some(module), - config: config.clone(), + config, message: None, signal_mailbox, message_mailbox, - resources: todo!(), // Resources::default(), - wasi: build_wasi( - Some(config.command_line_arguments()), - Some(config.environment_variables()), - config.preopened_dirs(), - )?, + resources: Resources::default(), + wasi, wasi_stdout: None, wasi_stderr: None, initialized: false, registry: Default::default(), // TODO move registry into env? db_resources: DbResources::default(), registry_atomic_put: None, - tracer: todo!(), // TODO this should not be hard coded like this - tracer_context: todo!(), + tracer: Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), + tracer_context: Arc::new(Context::new()), last_span_id: 0, - logger: todo!(), + logger: Arc::new( + env_logger::Builder::new() + .filter_level(log::LevelFilter::Off) + .build(), + ), }; Ok(state) } From ab619bdf46955e9a55f7f69db666b5c9de7195d4 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Wed, 5 Apr 2023 17:05:23 +0930 Subject: [PATCH 03/16] feat: add basic meters support --- Cargo.lock | 133 ++++---- Cargo.toml | 9 +- crates/hash-map-id/src/lib.rs | 7 + crates/lunatic-messaging-api/src/lib.rs | 12 +- crates/lunatic-metrics-api/src/lib.rs | 417 ++++++++++++++++-------- crates/lunatic-process/src/env.rs | 62 +--- crates/lunatic-process/src/lib.rs | 10 +- crates/lunatic-process/src/message.rs | 75 +---- src/main.rs | 6 + src/mode/cargo_test.rs | 8 +- src/mode/common.rs | 126 +++++-- src/mode/node.rs | 18 +- src/mode/run.rs | 52 ++- src/state.rs | 187 +++++++---- 14 files changed, 657 insertions(+), 465 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index eff65c894..cbcc7ea5a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1103,7 +1103,7 @@ checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] @@ -1667,6 +1667,7 @@ dependencies = [ "dashmap", "env_logger 0.9.3", "hash-map-id", + "hyper", "log", "lunatic-control", "lunatic-control-axum", @@ -1685,9 +1686,10 @@ dependencies = [ "lunatic-trap-api", "lunatic-version-api", "lunatic-wasi-api", - "metrics-exporter-prometheus", "opentelemetry", "opentelemetry-jaeger", + "opentelemetry-prometheus", + "prometheus", "regex", "reqwest", "serde", @@ -1831,25 +1833,6 @@ dependencies = [ "portable-atomic", ] -[[package]] -name = "metrics-exporter-prometheus" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8603921e1f54ef386189335f288441af761e0fc61bcb552168d9cedfe63ebc70" -dependencies = [ - "hyper", - "indexmap", - "ipnet", - "metrics", - "metrics-util", - "parking_lot", - "portable-atomic", - "quanta", - "thiserror", - "tokio", - "tracing", -] - [[package]] name = "metrics-macros" version = "0.6.0" @@ -1861,23 +1844,6 @@ dependencies = [ "syn", ] -[[package]] -name = "metrics-util" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7d24dc2dbae22bff6f1f9326ffce828c9f07ef9cc1e8002e5279f845432a30a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", - "hashbrown", - "metrics", - "num_cpus", - "parking_lot", - "portable-atomic", - "quanta", - "sketches-ddsketch", -] - [[package]] name = "mime" version = "0.3.16" @@ -1898,7 +1864,7 @@ checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" dependencies = [ "libc", "log", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", "windows-sys 0.45.0", ] @@ -2063,6 +2029,7 @@ dependencies = [ [[package]] name = "opentelemetry" version = "0.19.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" dependencies = [ "opentelemetry_api", "opentelemetry_sdk", @@ -2071,6 +2038,7 @@ dependencies = [ [[package]] name = "opentelemetry-jaeger" version = "0.18.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" dependencies = [ "async-trait", "futures", @@ -2083,9 +2051,22 @@ dependencies = [ "tokio", ] +[[package]] +name = "opentelemetry-prometheus" +version = "0.12.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" +dependencies = [ + "once_cell", + "opentelemetry_api", + "opentelemetry_sdk", + "prometheus", + "protobuf", +] + [[package]] name = "opentelemetry-semantic-conventions" version = "0.11.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" dependencies = [ "opentelemetry", ] @@ -2093,7 +2074,9 @@ dependencies = [ [[package]] name = "opentelemetry_api" version = "0.19.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" dependencies = [ + "fnv", "futures-channel", "futures-util", "indexmap", @@ -2106,6 +2089,7 @@ dependencies = [ [[package]] name = "opentelemetry_sdk" version = "0.19.0" +source = "git+https://github.com/tqwewe/opentelemetry-rust?branch=cow#d59370647fd98699fb7668f532b0022db7f4b7e5" dependencies = [ "async-trait", "crossbeam-channel", @@ -2114,8 +2098,10 @@ dependencies = [ "futures-util", "once_cell", "opentelemetry_api", + "ordered-float 3.6.0", "percent-encoding", "rand", + "regex", "thiserror", "tokio", "tokio-stream", @@ -2130,6 +2116,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a384337e997e6860ffbaa83708b2ef329fd8c54cb67a5f64d421e0f943254f" +dependencies = [ + "num-traits", +] + [[package]] name = "os_str_bytes" version = "6.4.1" @@ -2291,6 +2286,27 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prometheus" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "449811d15fbdf5ceb5c1144416066429cf82316e2ec8ce0c1f6f8a02e7bbcf8c" +dependencies = [ + "cfg-if", + "fnv", + "lazy_static", + "memchr", + "parking_lot", + "protobuf", + "thiserror", +] + +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "psm" version = "0.1.21" @@ -2311,22 +2327,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "quanta" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7e31331286705f455e56cca62e0e717158474ff02b7936c1fa596d983f4ae27" -dependencies = [ - "crossbeam-utils", - "libc", - "mach", - "once_cell", - "raw-cpuid", - "wasi 0.10.2+wasi-snapshot-preview1", - "web-sys", - "winapi", -] - [[package]] name = "quinn" version = "0.9.3" @@ -2416,15 +2416,6 @@ dependencies = [ "getrandom", ] -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags", -] - [[package]] name = "rayon" version = "1.7.0" @@ -2809,12 +2800,6 @@ dependencies = [ "libc", ] -[[package]] -name = "sketches-ddsketch" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceb945e54128e09c43d8e4f1277851bd5044c6fc540bbaa2ad888f60b3da9ae7" - [[package]] name = "slab" version = "0.4.8" @@ -3017,7 +3002,7 @@ dependencies = [ "byteorder", "integer-encoding", "log", - "ordered-float", + "ordered-float 2.10.0", "threadpool", ] @@ -3357,12 +3342,6 @@ dependencies = [ "try-lock", ] -[[package]] -name = "wasi" -version = "0.10.2+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index d92a8fad2..781718536 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ metrics = [ "lunatic-timer-api/metrics", "dep:lunatic-metrics-api", ] -prometheus = ["dep:metrics-exporter-prometheus", "metrics"] [dependencies] hash-map-id = { workspace = true } @@ -59,10 +58,12 @@ async-ctrlc = "1.2.0" clap = { version = "4.0", features = ["cargo", "derive"] } dashmap = { workspace = true } env_logger = "0.9" +hyper = { version = "0.14", features = ["full"] } log = { workspace = true } -metrics-exporter-prometheus = { version = "0.11.0", optional = true } opentelemetry = { workspace = true, features = ["rt-tokio"] } -opentelemetry-jaeger = { version = "0.18.0", features = ["rt-tokio"], path = "../../tqwewe/opentelemetry-rust/opentelemetry-jaeger" } +opentelemetry-jaeger = { version = "0.18.0", git = "https://github.com/tqwewe/opentelemetry-rust", branch = "cow", features = ["rt-tokio"] } +opentelemetry-prometheus = { version = "0.12.0", git = "https://github.com/tqwewe/opentelemetry-rust", branch = "cow" } +prometheus = "0.13" regex = "1.7" reqwest = { workspace = true } serde = { workspace = true, features = ["derive"] } @@ -129,7 +130,7 @@ anyhow = "1.0" bincode = "1.3" dashmap = "5.4" log = "0.4" -opentelemetry = { version = "0.19", path = "../../tqwewe/opentelemetry-rust/opentelemetry" } +opentelemetry = { version = "0.19", git = "https://github.com/tqwewe/opentelemetry-rust", branch = "cow", features = ["metrics"] } metrics = "0.20.1" reqwest = "0.11" rustls-pemfile = "1.0" diff --git a/crates/hash-map-id/src/lib.rs b/crates/hash-map-id/src/lib.rs index 0baf7fc2a..02bf4635f 100644 --- a/crates/hash-map-id/src/lib.rs +++ b/crates/hash-map-id/src/lib.rs @@ -35,6 +35,13 @@ where pub fn get(&self, id: u64) -> Option<&T> { self.store.get(&id) } + + pub fn get_last(&self) -> Option<&T> { + self.store + .iter() + .max_by_key(|(k, _)| **k) + .map(|(_, value)| value) + } } impl Default for HashMapId diff --git a/crates/lunatic-messaging-api/src/lib.rs b/crates/lunatic-messaging-api/src/lib.rs index 41e0bf521..42900ca6e 100644 --- a/crates/lunatic-messaging-api/src/lib.rs +++ b/crates/lunatic-messaging-api/src/lib.rs @@ -306,7 +306,7 @@ fn take_module + NetworkingCtx + 'static>( .or_trap("lunatic::message::take_module")?; let module = match message { Message::Data(data) => data - .take_module(index as usize) + .take_resource(index as usize) .or_trap("lunatic::message::take_module")?, Message::LinkDied(_) => { return Err(anyhow!("Unexpected `Message::LinkDied` in scratch area")) @@ -316,10 +316,10 @@ fn take_module + NetworkingCtx + 'static>( } // Adds a tcp stream resource to the message that is currently in the scratch area and returns -// the new location of it. This will remove the tcp stream from the current process' resources. +// the new location of it. This will remove the tcp stream from the current process' resources. // // Traps: -// * If TCP stream ID doesn't exist +// * If TCP stream ID doesn't exist. // * If no data message is in the scratch area. fn push_tcp_stream + NetworkingCtx>( mut caller: Caller, @@ -361,7 +361,7 @@ fn take_tcp_stream + NetworkingCtx>( .or_trap("lunatic::message::take_tcp_stream")?; let tcp_stream = match message { Message::Data(data) => data - .take_tcp_stream(index as usize) + .take_resource(index as usize) .or_trap("lunatic::message::take_tcp_stream")?, Message::LinkDied(_) => { return Err(anyhow!("Unexpected `Message::LinkDied` in scratch area")) @@ -417,7 +417,7 @@ fn take_tls_stream + NetworkingCtx>( .or_trap("lunatic::message::take_tls_stream")?; let tls_stream = match message { Message::Data(data) => data - .take_tls_stream(index as usize) + .take_resource(index as usize) .or_trap("lunatic::message::take_tls_stream")?, Message::LinkDied(_) => { return Err(anyhow!("Unexpected `Message::LinkDied` in scratch area")) @@ -608,7 +608,7 @@ fn take_udp_socket + NetworkingCtx>( .or_trap("lunatic::message::take_udp_socket")?; let udp_socket = match message { Message::Data(data) => data - .take_udp_socket(index as usize) + .take_resource(index as usize) .or_trap("lunatic::message::take_udp_socket")?, Message::LinkDied(_) => { return Err(anyhow!("Unexpected `Message::LinkDied` in scratch area")) diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index c7f3eee64..9c6cf6782 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -1,4 +1,4 @@ -use std::borrow::Cow; +use std::{borrow::Cow, sync::Arc}; use anyhow::Result; use hash_map_id::HashMapId; @@ -8,26 +8,53 @@ use lunatic_process::state::ProcessState; use lunatic_process_api::ProcessCtx; use metrics::{counter, decrement_gauge, gauge, histogram, increment_counter, increment_gauge}; use opentelemetry::{ - trace::{SpanRef, Tracer}, + global, + metrics::{Counter, Histogram, Meter, MeterProvider, Unit}, + sdk::metrics::data::Gauge, + trace::{Span, SpanRef, TraceContextExt, Tracer, TracerProvider}, Context, KeyValue, StringValue, }; use serde_json::Map; use wasmtime::{Caller, Linker}; -pub type SpanResources = HashMapId; -pub type TracerSpan = ::Span; +pub type ContextResources = HashMapId; pub trait MetricsCtx { - type Tracer: Tracer + Send + Sync; + type Tracer: Tracer; + type MeterProvider: MeterProvider; + + fn tracer(&self) -> &Self::Tracer; + fn meter_provider(&self) -> &Self::MeterProvider; + + // fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option + // where + // T: Into>, + // I: IntoIterator; + // fn get_span(&self, id: u64) -> Option>; + // fn get_last_span(&self) -> SpanRef; + // fn drop_span(&mut self, id: u64); + fn add_context(&mut self, context: Context) -> u64; + fn get_context(&self, id: u64) -> Option<&Context>; + fn get_last_context(&self) -> &Context; + fn drop_context(&mut self, id: u64); + + fn add_meter(&mut self, meter: Meter) -> u64; + fn get_meter(&self, id: u64) -> Option<&Meter>; + fn drop_meter(&mut self, id: u64) -> Option; fn log(&self, record: &Record); - fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option - where - T: Into>, - I: IntoIterator; - fn drop_span(&mut self, id: u64); - fn get_span(&self, id: u64) -> Option>; - fn get_last_span(&self) -> SpanRef; + + fn add_counter(&mut self, counter: Counter) -> u64; + fn get_counter(&self, id: u64) -> Option<&Counter>; + fn drop_counter(&mut self, id: u64) -> Option>; + + fn add_gauge(&mut self, gauge: Gauge) -> u64; + fn get_gauge(&self, id: u64) -> Option<&Gauge>; + fn drop_gauge(&mut self, id: u64) -> Option>; + + fn add_histogram(&mut self, histogram: Histogram) -> u64; + fn get_histogram(&self, id: u64) -> Option<&Histogram>; + fn drop_histogram(&mut self, id: u64) -> Option>; } /// Links the [Metrics](https://crates.io/crates/metrics) APIs @@ -37,10 +64,17 @@ where <::Tracer as Tracer>::Span: Send + Sync, { linker.func_wrap("lunatic::metrics", "start_span", start_span)?; - linker.func_wrap("lunatic::metrics", "add_event", add_event)?; linker.func_wrap("lunatic::metrics", "drop_span", drop_span)?; + + linker.func_wrap("lunatic::metrics", "meter", meter)?; + linker.func_wrap("lunatic::metrics", "drop_meter", drop_meter)?; + + linker.func_wrap("lunatic::metrics", "add_event", add_event)?; + linker.func_wrap("lunatic::metrics", "counter", counter)?; linker.func_wrap("lunatic::metrics", "increment_counter", increment_counter)?; + linker.func_wrap("lunatic::metrics", "drop_counter", drop_counter)?; + linker.func_wrap("lunatic::metrics", "gauge", gauge)?; linker.func_wrap("lunatic::metrics", "increment_gauge", increment_gauge)?; linker.func_wrap("lunatic::metrics", "decrement_gauge", decrement_gauge)?; @@ -48,49 +82,40 @@ where Ok(()) } -fn get_string_arg( - memory_slice: &[u8], - name_str_ptr: u32, - name_str_len: u32, - func_name: &str, -) -> Result { - let name = memory_slice - .get(name_str_ptr as usize..(name_str_ptr + name_str_len) as usize) - .or_trap(func_name)?; - let name = String::from_utf8(name.to_vec()).or_trap(func_name)?; - Ok(name) -} - -/// TODO docs +/// Starts a new span of work, used for recording metrics including log events and meters. +/// +/// `parent` is the ID of another span. If it is set to u64::MAX, then the last created span +/// will be used. +/// +/// Traps: +/// * If the name is not a valid utf8 string. +/// * If the attributes is not valid json. +/// * If the parent span does not exist. +/// * If any memory outside the guest heap space is referenced. fn start_span( mut caller: Caller<'_, T>, - parent_span: u64, - name_str_ptr: u32, - name_str_len: u32, + parent: u64, + name_ptr: u32, + name_len: u32, attributes_ptr: u32, attributes_len: u32, ) -> Result where T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, - <::Tracer as Tracer>::Span: Send + Sync, + <::Tracer as Tracer>::Span: Send + Sync + 'static, { let memory = get_memory(&mut caller)?; - let (memory_slice, state) = memory.data_and_store_mut(&mut caller); + let (data, state) = memory.data_and_store_mut(&mut caller); - let parent = if parent_span != u64::MAX { - Some(parent_span) + let parent = if parent != u64::MAX { + Some(parent) } else { None }; - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, - "lunatic::metrics::start_span", - )?; + let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::start_span")?; let attributes = if attributes_len > 0 { - let attributes_data = memory_slice + let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) .or_trap("lunatic::metrics::start_span")?; let attributes_json = @@ -100,19 +125,103 @@ where vec![] }; - let id = state - .add_span(parent, name, attributes) - .or_trap("lunatic::metrics::start_span")?; + let parent_ctx = if let Some(id) = parent { + state + .get_context(id) + .or_trap("lunatic::metrics::start_span")? + } else { + state.get_last_context() + }; + + let mut span = state.tracer().start_with_context(name, parent_ctx); + span.set_attributes(attributes); + + let context = parent_ctx.with_span(span); + + let id = state.add_context(context); + + Ok(id) +} + +/// Drops a span, marking it as finished. +fn drop_span(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, + <::Tracer as Tracer>::Span: Send + Sync, +{ + let memory = get_memory(&mut caller)?; + let (_data, state) = memory.data_and_store_mut(&mut caller); + + state.drop_context(id); + + Ok(()) +} + +/// Adds a new meter with a given name. +/// +/// Traps: +/// * If the name attribute is not a valid utf8 string. +/// * If any memory outside the guest heap space is referenced. +fn meter(mut caller: Caller<'_, T>, name_ptr: u32, name_len: u32) -> Result +where + T: MetricsCtx, +{ + let memory = get_memory(&mut caller)?; + let (data, state) = memory.data_and_store_mut(&mut caller); + + let name = get_string_arg(&data, name_ptr, name_len, "lunatic::metrics::meter")?; + + let meter = state.meter_provider().meter(name.into()); + let id = state.add_meter(meter); Ok(id) } -/// TODO docs +/// Drops a meter. +fn drop_meter(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: MetricsCtx, +{ + let memory = get_memory(&mut caller)?; + let (_data, state) = memory.data_and_store_mut(&mut caller); + + state.drop_meter(id); + + Ok(()) +} + +/// Adds a log event in span, containing a name and optional attributes. +/// +/// `span` is the ID of the parent span. If it is set to u64::MAX, then the last created span +/// will be used. +/// +/// The following attributes are optional, and used for logging to the terminal: +/// * `target`: a string describing the part of the system where the span or event that this +/// metadata describes occurred. +/// * `severityNumber`: the numerical severity of the log. +/// The following are numbers that map to differnt log levels: +/// * 1..=4 => Trace +/// * 5..=8 => Debug +/// * 9..=12 => Info +/// * 13..=16 => Warn +/// * 17..=20 => Error +/// Defaults to Info if omitted or an invalid number. +/// * `body`: the log message body. Defaults to the event name. +/// * `code.filepath`: the filepath where the log occurred. +/// * `code.lineno`: the line number where the log occurred. +/// * `code.column`: the column number where the log occurred. +/// * `code.namespace`: the module name where the log occurred. +/// +/// Traps: +/// * If the name is not a valid utf8 string. +/// * If the attributes is not valid json. +/// * If the parent span does not exist. +/// * If any memory outside the guest heap space is referenced. fn add_event( mut caller: Caller<'_, T>, - span_id: u64, - name_str_ptr: u32, - name_str_len: u32, + span: u64, + name_ptr: u32, + name_len: u32, attributes_ptr: u32, attributes_len: u32, ) -> Result<()> @@ -121,17 +230,12 @@ where <::Tracer as Tracer>::Span: Send + Sync, { let memory = get_memory(&mut caller)?; - let (memory_slice, state) = memory.data_and_store_mut(&mut caller); + let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, - "lunatic::metrics::add_event", - )?; + let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::add_event")?; let attributes = if attributes_len > 0 { - let attributes_data = memory_slice + let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) .or_trap("lunatic::metrics::add_event")?; let attributes_json: Map = @@ -194,12 +298,13 @@ where vec![] }; - let span = if span_id != u64::MAX { + let span = if span != u64::MAX { state - .get_span(span_id) + .get_context(span) .or_trap("lunatic::metrics::add_event")? + .span() } else { - state.get_last_span() + state.get_last_context().span() }; span.add_event(name, attributes); @@ -207,20 +312,6 @@ where Ok(()) } -/// TODO docs -fn drop_span(mut caller: Caller<'_, T>, id: u64) -> Result<()> -where - T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, - <::Tracer as Tracer>::Span: Send + Sync, -{ - let memory = get_memory(&mut caller)?; - let (_data, state) = memory.data_and_store_mut(&mut caller); - - state.drop_span(id); - - Ok(()) -} - /// Sets a counter. /// /// Traps: @@ -228,22 +319,54 @@ where /// * If any memory outside the guest heap space is referenced. fn counter( mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, - value: u64, -) -> Result<()> { + meter: u64, + name_ptr: u32, + name_len: u32, + description_ptr: u32, + description_len: u32, + unit_ptr: u32, + unit_len: u32, +) -> Result +where + T: MetricsCtx, +{ let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, + let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::counter")?; + let description = get_string_arg( + data, + description_ptr, + description_len, "lunatic::metrics::counter", )?; + let unit = + get_string_arg(data, unit_ptr, unit_len, "lunatic::metrics::counter").map(|unit| { + if unit.is_empty() { + None + } else { + Some(Unit::new(unit)) + } + })?; + + let mut counter_builder = state + .get_meter(meter) + .or_trap("lunatic::metrics::counter")? + .f64_counter(name); + if !description.is_empty() { + counter_builder = counter_builder.with_description(description); + } + if let Some(unit) = unit { + counter_builder = counter_builder.with_unit(unit); + } - counter!(name, value); - Ok(()) + let counter = counter_builder + .try_init() + .or_trap("lunatic::metrics::counter")?; + + let id = state.add_counter(counter); + + Ok(id) } /// Increments a counter. @@ -253,20 +376,57 @@ fn counter( /// * If any memory outside the guest heap space is referenced. fn increment_counter( mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, -) -> Result<()> { + span: u64, + counter: u64, + amount: f64, + attributes_ptr: u32, + attributes_len: u32, +) -> Result<()> +where + T: MetricsCtx, +{ let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, - "lunatic::metrics::increment_counter", - )?; + let cx = if span != u64::MAX { + state + .get_context(span) + .or_trap("lunatic::metrics::increment_counter")? + } else { + state.get_last_context() + }; + + let attributes = if attributes_len > 0 { + let attributes_data = data + .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) + .or_trap("lunatic::metrics::add_event")?; + let attributes_json: Map = + serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::add_event")?; + + data_to_opentelemetry(attributes_json) + } else { + vec![] + }; + + let counter = state + .get_counter(counter) + .or_trap("lunatic::metrics::increment_counter")?; + + counter.add(cx, amount, &attributes); + + Ok(()) +} + +/// Drops a counter. +fn drop_counter(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: MetricsCtx, +{ + let memory = get_memory(&mut caller)?; + let (_data, state) = memory.data_and_store_mut(&mut caller); + + state.drop_counter(id); - increment_counter!(name); Ok(()) } @@ -275,21 +435,11 @@ fn increment_counter( /// Traps: /// * If the name is not a valid utf8 string. /// * If any memory outside the guest heap space is referenced. -fn gauge( - mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, - value: f64, -) -> Result<()> { +fn gauge(mut caller: Caller<'_, T>, name_ptr: u32, name_len: u32, value: f64) -> Result<()> { let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let data = memory.data(&mut caller); - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, - "lunatic::metrics::gauge", - )?; + let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::gauge")?; gauge!(name, value); Ok(()) @@ -302,17 +452,17 @@ fn gauge( /// * If any memory outside the guest heap space is referenced. fn increment_gauge( mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, + name_ptr: u32, + name_len: u32, value: f64, ) -> Result<()> { let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let data = memory.data(&mut caller); let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, + data, + name_ptr, + name_len, "lunatic::metrics::increment_gauge", )?; @@ -327,17 +477,17 @@ fn increment_gauge( /// * If any memory outside the guest heap space is referenced. fn decrement_gauge( mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, + name_ptr: u32, + name_len: u32, value: f64, ) -> Result<()> { let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let data = memory.data(&mut caller); let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, + data, + name_ptr, + name_len, "lunatic::metrics::decrement_gauge", )?; @@ -350,26 +500,27 @@ fn decrement_gauge( /// Traps: /// * If the name is not a valid utf8 string. /// * If any memory outside the guest heap space is referenced. -fn histogram( - mut caller: Caller<'_, T>, - name_str_ptr: u32, - name_str_len: u32, - value: f64, -) -> Result<()> { +fn histogram(mut caller: Caller<'_, T>, name_ptr: u32, name_len: u32, value: f64) -> Result<()> { let memory = get_memory(&mut caller)?; - let memory_slice = memory.data(&mut caller); + let data = memory.data(&mut caller); - let name = get_string_arg( - memory_slice, - name_str_ptr, - name_str_len, - "lunatic::metrics::histogram", - )?; + let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::histogram")?; histogram!(name, value); Ok(()) } +fn get_string_arg(data: &[u8], name_ptr: u32, name_len: u32, func_name: &str) -> Result { + if name_len == 0 { + return Ok(String::new()); + } + let name = data + .get(name_ptr as usize..(name_ptr + name_len) as usize) + .or_trap(func_name)?; + let name = String::from_utf8(name.to_vec()).or_trap(func_name)?; + Ok(name) +} + fn data_to_opentelemetry(data: Map) -> Vec { data.into_iter() .map(|(k, v)| KeyValue { diff --git a/crates/lunatic-process/src/env.rs b/crates/lunatic-process/src/env.rs index e52c2bff5..ae58455b1 100644 --- a/crates/lunatic-process/src/env.rs +++ b/crates/lunatic-process/src/env.rs @@ -1,16 +1,10 @@ use anyhow::Result; use async_trait::async_trait; use dashmap::DashMap; -use std::{ - future::Future, - pin::Pin, - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - task::{Context, Poll}, +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, }; -use tokio::time::{interval, Interval}; use crate::{Process, Signal}; @@ -24,9 +18,7 @@ pub trait Environment: Send + Sync { fn process_count(&self) -> usize; async fn can_spawn_next_process(&self) -> Result>; fn send(&self, id: u64, signal: Signal); - fn shutdown(&self) -> ShutdownFuture<'_, Self> - where - Self: Sized; + fn shutdown(&self); } #[async_trait] @@ -111,15 +103,10 @@ impl Environment for LunaticEnvironment { Ok(Some(())) } - fn shutdown(&self) -> ShutdownFuture<'_, Self> - where - Self: Sized, - { + fn shutdown(&self) { for proc in self.processes.iter() { proc.send(Signal::Kill); } - - ShutdownFuture::new(self) } } @@ -143,42 +130,3 @@ impl Environments for LunaticEnvironments { self.envs.get(&id).map(|e| e.clone()) } } - -#[derive(Debug)] -pub struct ShutdownFuture<'a, E> { - environment: &'a E, - interval: Interval, -} - -impl<'a, E> ShutdownFuture<'a, E> { - fn new(environment: &'a E) -> Self { - ShutdownFuture { - environment, - interval: interval(tokio::time::Duration::from_millis(50)), - } - } -} - -impl<'a, E> Future for ShutdownFuture<'a, E> -where - E: Environment, -{ - type Output = (); - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - if self.environment.process_count() == 0 { - Poll::Ready(()) - } else { - match self.interval.poll_tick(cx) { - Poll::Ready(_) => { - if self.environment.process_count() == 0 { - Poll::Ready(()) - } else { - Poll::Pending - } - } - Poll::Pending => Poll::Pending, - } - } - } -} diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index aa3f19ab5..b97dde7ca 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -378,9 +378,7 @@ where } }; - env.remove_process(id); - - match result { + let result = match result { Finished::Normal(result) => { let result: ExecutionResult<_> = result.into(); @@ -429,7 +427,11 @@ where }); Err(anyhow!("Process received Kill signal")) } - } + }; + + env.remove_process(id); + + result } /// A process spawned from a native Rust closure. diff --git a/crates/lunatic-process/src/message.rs b/crates/lunatic-process/src/message.rs index 648bfb714..0b6868040 100644 --- a/crates/lunatic-process/src/message.rs +++ b/crates/lunatic-process/src/message.rs @@ -11,10 +11,7 @@ use std::{ sync::Arc, }; -use lunatic_networking_api::{TcpConnection, TlsConnection}; -use tokio::net::UdpSocket; - -use crate::runtimes::wasmtime::WasmtimeCompiledModule; +use smallvec::SmallVec; pub type Resource = dyn Any + Send + Sync; @@ -59,7 +56,7 @@ pub struct DataMessage { pub tag: Option, pub read_ptr: usize, pub buffer: Vec, - pub resources: Vec>>, + pub resources: SmallVec<[Option>; 3]>, } impl DataMessage { @@ -69,7 +66,7 @@ impl DataMessage { tag, read_ptr: 0, buffer: Vec::with_capacity(buffer_capacity), - resources: Vec::new(), + resources: SmallVec::new(), } } @@ -79,7 +76,7 @@ impl DataMessage { tag, read_ptr: 0, buffer, - resources: Vec::new(), + resources: SmallVec::new(), } } @@ -91,39 +88,21 @@ impl DataMessage { self.resources.len() - 1 } - /// Takes a module from the message, but preserves the indexes of all others. - /// - /// If the index is out of bound or the resource is not a module the function will return - /// None. - pub fn take_module( - &mut self, - index: usize, - ) -> Option>> { - self.take_downcast(index) - } - - /// Takes a TCP stream from the message, but preserves the indexes of all others. - /// - /// If the index is out of bound or the resource is not a tcp stream the function will return - /// None. - pub fn take_tcp_stream(&mut self, index: usize) -> Option> { - self.take_downcast(index) - } - - /// Takes a UDP Socket from the message, but preserves the indexes of all others. - /// - /// If the index is out of bound or the resource is not a tcp stream the function will return - /// None. - pub fn take_udp_socket(&mut self, index: usize) -> Option> { - self.take_downcast(index) - } - - /// Takes a TLS stream from the message, but preserves the indexes of all others. + /// Takes a resource from the message, downcasting to `T`. /// - /// If the index is out of bound or the resource is not a tcp stream the function will return - /// None. - pub fn take_tls_stream(&mut self, index: usize) -> Option> { - self.take_downcast(index) + /// If the index is out of bounds, or the downcast fails, None is returned and the resource + /// will remain in the message. + pub fn take_resource(&mut self, index: usize) -> Option> { + let resource_ref = self.resources.get_mut(index)?; + let resource_any = std::mem::take(resource_ref).map(|resource| resource.downcast())?; + match resource_any { + Ok(resource) => Some(resource), + Err(resource) => { + // Downcast failed, return the resource back to the message. + *resource_ref = Some(resource); + None + } + } } /// Moves read pointer to index. @@ -144,24 +123,6 @@ impl DataMessage { ); metrics::histogram!("lunatic.process.messages.data.size", self.size() as f64); } - - fn take_downcast(&mut self, index: usize) -> Option> { - let resource = self.resources.get_mut(index); - match resource { - Some(resource_ref) => { - let resource_any = std::mem::take(resource_ref).map(|resource| resource.downcast()); - match resource_any { - Some(Ok(resource)) => Some(resource), - Some(Err(resource)) => { - *resource_ref = Some(resource); - None - } - None => None, - } - } - None => None, - } - } } impl Write for DataMessage { diff --git a/src/main.rs b/src/main.rs index c7bd75a03..b69b5c76a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,6 +25,10 @@ fn is_run_implied() -> bool { #[tokio::main] async fn main() -> Result<()> { + if env::var("OTEL_SERVICE_NAME").is_err() { + env::set_var("OTEL_SERVICE_NAME", "lunatic"); + } + // Run is implied from lunatic 0.12 let augmented_args = if is_run_implied() { let mut augmented_args: VecDeque = std::env::args().collect(); @@ -80,5 +84,7 @@ async fn main() -> Result<()> { execution::execute(augmented_args).await }; + opentelemetry::global::shutdown_tracer_provider(); + result } diff --git a/src/mode/cargo_test.rs b/src/mode/cargo_test.rs index 287f99f16..e67a01656 100644 --- a/src/mode/cargo_test.rs +++ b/src/mode/cargo_test.rs @@ -11,7 +11,11 @@ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use lunatic_stdout_capture::StdoutCapture; use lunatic_wasi_api::LunaticWasiCtx; -use opentelemetry::{global::BoxedTracer, trace::noop::NoopTracer}; +use opentelemetry::{ + global::{BoxedTracer, GlobalMeterProvider}, + metrics::noop::NoopMeterProvider, + trace::noop::NoopTracer, +}; use tokio::sync::RwLock; #[derive(Parser, Debug)] @@ -211,6 +215,7 @@ pub(crate) async fn test(augmented_args: Option>) -> Result<()> { let config = Arc::new(config); let tracer = Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))); let tracer_context = Arc::new(opentelemetry::Context::new()); + let meter_provider = GlobalMeterProvider::new(NoopMeterProvider::new()); let logger = Arc::new( env_logger::Builder::from_env("LUNATIC_LOG") .default_format() @@ -246,6 +251,7 @@ pub(crate) async fn test(augmented_args: Option>) -> Result<()> { registry, tracer.clone(), tracer_context.clone(), + meter_provider.clone(), logger.clone(), ) .unwrap(); diff --git a/src/mode/common.rs b/src/mode/common.rs index 4fc984fe0..1a8007662 100644 --- a/src/mode/common.rs +++ b/src/mode/common.rs @@ -1,8 +1,14 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{anyhow, Context, Result}; use clap::Args; +use hyper::{ + header::CONTENT_TYPE, + http, + service::{make_service_fn, service_fn}, + Body, Method, Request, Response, Server, +}; use lunatic_distributed::DistributedProcessState; use lunatic_process::{ env::{Environment, LunaticEnvironment, LunaticEnvironments}, @@ -12,10 +18,13 @@ use lunatic_process::{ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use opentelemetry::{ - global::BoxedTracer, + global::{BoxedTracer, GlobalMeterProvider}, + metrics::MetricsError, + sdk::metrics::MeterProvider, trace::{Span, TraceContextExt, Tracer}, KeyValue, }; +use prometheus::{Encoder, Registry, TextEncoder}; #[derive(Args, Debug)] pub struct WasmArgs {} @@ -30,6 +39,7 @@ pub struct RunWasm { pub env: Arc, pub distributed: Option, pub tracer: Arc, + pub meter_provider: GlobalMeterProvider, } pub async fn run_wasm(args: RunWasm) -> Result<()> { @@ -75,11 +85,12 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { "app_start", // SpanBuilder::from_name("app_start") // .with_attributes([KeyValue::new("path", path.to_string_lossy().to_string())]), ); - root_span.set_attribute(KeyValue::new( - "lunatic.path", - path.to_string_lossy().to_string(), - )); - let tracer_context = Arc::new(opentelemetry::Context::current_with_span(root_span)); + root_span.set_attributes([ + KeyValue::new("service.name", "lunatic"), + KeyValue::new("lunatic.path", path.to_string_lossy().to_string()), + KeyValue::new("lunatic.environment", args.env.id() as i64), + ]); + let tracer_context = Arc::new(opentelemetry::Context::new().with_span(root_span)); let logger = Arc::new( env_logger::Builder::from_env( env_logger::Env::new() @@ -97,22 +108,15 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { Default::default(), args.tracer, tracer_context, + args.meter_provider, logger, ) .unwrap(); args.env.can_spawn_next_process().await?; - let env = args.env.clone(); - tokio::spawn(async move { - if tokio::signal::ctrl_c().await.is_ok() { - let _ = tokio::time::timeout(tokio::time::Duration::from_secs(5), env.shutdown()).await; - std::process::exit(0); - } - }); - let (task, _) = spawn_wasm( - args.env, + args.env.clone(), args.runtime, &module, state, @@ -126,27 +130,77 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { path.to_string_lossy() ))?; - // Wait on the main process to finish - task.await.map(|_| ()).map_err(|e| anyhow!(e.to_string())) + // Wait on the main process to finish, or ctrl c signal + tokio::select! { + result = task => { + result.map(|_| ()).map_err(|err| anyhow!(err.to_string())) + }, + Ok(_) = tokio::signal::ctrl_c() => { + args.env.shutdown(); + tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; + Ok(()) + }, + } } -#[cfg(feature = "prometheus")] -#[derive(Args, Debug)] -pub struct PrometheusArgs { - /// Enables the prometheus metrics exporter - #[arg(long)] - pub prometheus: bool, - - /// Address to bind the prometheus http listener to - #[arg(long, value_name = "PROMETHEUS_HTTP_ADDRESS", requires = "prometheus")] - pub prometheus_http: Option, -} +pub fn prometheus(addr: &SocketAddr, _node_id: Option) -> Result { + let registry = Registry::new(); + let exporter = opentelemetry_prometheus::exporter() + .with_registry(registry.clone()) + .build()?; + let provider = MeterProvider::builder().with_reader(exporter).build(); + + async fn handle_request( + req: Request, + registry: Registry, + ) -> Result, http::Error> { + match (req.method(), req.uri().path()) { + (&Method::GET, "/metrics") => { + let mut buffer = vec![]; + let encoder = TextEncoder::new(); + let metric_families = registry.gather(); + if let Err(err) = encoder.encode(&metric_families, &mut buffer) { + log::error!("failed to encode prometheus metrics: {err}"); + return Response::builder().status(400).body(Body::empty()); + } + + Response::builder() + .status(200) + .header(CONTENT_TYPE, encoder.format_type()) + .body(Body::from(buffer)) + } + _ => Response::builder() + .status(404) + .body(Body::from("not found")), + } + } + + let server = + Server::bind(addr).serve(make_service_fn(move |_conn| { + let registry = registry.clone(); + + async move { + Ok::<_, http::Error>(service_fn(move |req| handle_request(req, registry.clone()))) + } + })); + + if addr.ip().is_loopback() || addr.ip().is_unspecified() { + log::info!( + "prometheus metrics available at http://localhost:{}/metrics", + addr.port() + ); + } else { + log::info!( + "prometheus metrics available at http://{}:{}/metrics", + addr.ip(), + addr.port() + ); + } -#[cfg(feature = "prometheus")] -pub fn prometheus(http_socket: Option, node_id: Option) -> Result<()> { - metrics_exporter_prometheus::PrometheusBuilder::new() - .with_http_listener(http_socket.unwrap_or_else(|| "0.0.0.0:9927".parse().unwrap())) - .add_global_label("node_id", node_id.unwrap_or(0).to_string()) - .install()?; - Ok(()) + tokio::spawn(server); + // metrics_exporter_prometheus::PrometheusBuilder::new() + // .with_http_listener(http_socket.unwrap_or_else(|| "0.0.0.0:9927".parse().unwrap())) + // .add_global_label("node_id", node_id.unwrap_or(0).to_string()) + // .install()?; + Ok(provider) } diff --git a/src/mode/node.rs b/src/mode/node.rs index 882633809..7ce76f0c2 100644 --- a/src/mode/node.rs +++ b/src/mode/node.rs @@ -4,7 +4,11 @@ use std::{ }; use clap::Parser; -use opentelemetry::{global::BoxedTracer, trace::noop::NoopTracer}; +use opentelemetry::{ + global::{BoxedTracer, GlobalMeterProvider}, + metrics::noop::NoopMeterProvider, + trace::noop::NoopTracer, +}; use std::{collections::HashMap, sync::Arc}; @@ -42,18 +46,9 @@ pub(crate) struct Args { /// Define key=value variable to store as node information #[arg(long, value_parser = parse_key_val, action = clap::ArgAction::Append)] tag: Vec<(String, String)>, - - #[cfg(feature = "prometheus")] - #[command(flatten)] - prometheus: super::common::PrometheusArgs, } pub(crate) async fn start(args: Args) -> Result<()> { - #[cfg(feature = "prometheus")] - if args.prometheus.prometheus { - super::common::prometheus(args.prometheus.prometheus_http, None)?; - } - let socket = args .bind_socket .or_else(get_available_localhost) @@ -130,7 +125,8 @@ pub(crate) async fn start(args: Args) -> Result<()> { envs, env, distributed: Some(dist), - tracer: Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), + tracer: Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), // TODO: Shoul this be noop? + meter_provider: GlobalMeterProvider::new(NoopMeterProvider::new()), // TODO: Should this be noop? }) .await { diff --git a/src/mode/run.rs b/src/mode/run.rs index 48b15d83e..aa59d666e 100644 --- a/src/mode/run.rs +++ b/src/mode/run.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, sync::Arc}; +use std::{net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::Result; use clap::Parser; @@ -6,16 +6,17 @@ use lunatic_process::{ env::{Environments, LunaticEnvironments}, runtimes, }; -use opentelemetry::{global::BoxedTracer, runtime::Tokio, trace::noop::NoopTracer}; +use opentelemetry::{ + global::{BoxedTracer, GlobalMeterProvider}, + metrics::noop::NoopMeterProvider, + runtime::Tokio, + trace::noop::NoopTracer, +}; -use super::common::{run_wasm, RunWasm}; +use super::common::{prometheus, run_wasm, RunWasm}; #[derive(Parser, Debug)] #[command(version)] -#[command(group( - clap::ArgGroup::new("tracer") - .args(["jaeger"]), -))] pub struct Args { /// Grant access to the given host directories #[arg(long, value_name = "DIRECTORY")] @@ -26,8 +27,24 @@ pub struct Args { pub bench: bool, /// Jaeger connection url for tracing. - #[arg(long)] - pub jaeger: Option, + #[arg( + long, + value_name = "JAEGER_HTTP_ADDRESS", + num_args(0..=1), + require_equals(true), + default_missing_value("127.0.0.1:6831") + )] + pub jaeger: Option, + + /// Address to bind the prometheus http listener to + #[arg( + long, + value_name = "PROMETHEUS_HTTP_ADDRESS", + num_args(0..=1), + require_equals(true), + default_missing_value("0.0.0.0:9927") + )] + pub prometheus: Option, /// Entry .wasm file #[arg(index = 1)] @@ -36,18 +53,9 @@ pub struct Args { /// Arguments passed to the guest #[arg(index = 2)] pub wasm_args: Vec, - - #[cfg(feature = "prometheus")] - #[command(flatten)] - prometheus: super::common::PrometheusArgs, } pub(crate) async fn start(mut args: Args) -> Result<()> { - #[cfg(feature = "prometheus")] - if args.prometheus.prometheus { - super::common::prometheus(args.prometheus.prometheus_http, None)?; - } - // Create wasmtime runtime let wasmtime_config = runtimes::wasmtime::default_config(); let runtime = runtimes::wasmtime::WasmtimeRuntime::new(&wasmtime_config)?; @@ -70,6 +78,13 @@ pub(crate) async fn start(mut args: Args) -> Result<()> { None => Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), }; + let meter_provider = match &args.prometheus { + Some(url) => GlobalMeterProvider::new( + prometheus(url, None).expect("failed to create prometheus registry"), + ), + None => GlobalMeterProvider::new(NoopMeterProvider::new()), + }; + run_wasm(RunWasm { path: args.path, wasm_args: args.wasm_args, @@ -79,6 +94,7 @@ pub(crate) async fn start(mut args: Args) -> Result<()> { env, distributed: None, tracer, + meter_provider, }) .await } diff --git a/src/state.rs b/src/state.rs index 93e3a547c..8b6c1d2a5 100644 --- a/src/state.rs +++ b/src/state.rs @@ -8,7 +8,7 @@ use hash_map_id::HashMapId; use log::{Log, Record}; use lunatic_distributed::{DistributedCtx, DistributedProcessState}; use lunatic_error_api::{ErrorCtx, ErrorResource}; -use lunatic_metrics_api::{MetricsCtx, SpanResources}; +use lunatic_metrics_api::{ContextResources, MetricsCtx}; use lunatic_networking_api::{DnsIterator, TlsConnection, TlsListener}; use lunatic_networking_api::{NetworkingCtx, TcpConnection}; use lunatic_process::env::{Environment, LunaticEnvironment}; @@ -24,9 +24,12 @@ use lunatic_sqlite_api::{SQLiteConnections, SQLiteCtx, SQLiteGuestAllocators, SQ use lunatic_stdout_capture::StdoutCapture; use lunatic_timer_api::{TimerCtx, TimerResources}; use lunatic_wasi_api::{build_wasi, LunaticWasiCtx}; -use opentelemetry::global::BoxedTracer; +use opentelemetry::global::{BoxedTracer, GlobalMeterProvider}; +use opentelemetry::metrics::noop::NoopMeterProvider; +use opentelemetry::metrics::{Counter, Histogram, Meter}; +use opentelemetry::sdk::metrics::data::Gauge; use opentelemetry::trace::noop::NoopTracer; -use opentelemetry::trace::{Span, SpanRef, TraceContextExt, Tracer}; +use opentelemetry::trace::{Span, TraceContextExt, Tracer}; use opentelemetry::{Context, KeyValue}; use tokio::net::{TcpListener, UdpSocket}; use tokio::sync::mpsc::unbounded_channel; @@ -86,7 +89,8 @@ pub struct DefaultProcessState { // TODO: Does this need to be in an Arc? tracer: Arc, tracer_context: Arc, - last_span_id: u64, + process_context: Context, + meter_provider: GlobalMeterProvider, logger: Arc, } @@ -100,12 +104,14 @@ impl DefaultProcessState { registry: Arc>>, tracer: Arc, tracer_context: Arc, + meter_provider: GlobalMeterProvider, log_filter: Arc, ) -> Result { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); let id = environment.get_next_process_id(); + let process_context = new_process_context(id, &tracer, &tracer_context); let wasi = build_wasi( Some(config.command_line_arguments()), Some(config.environment_variables()), @@ -121,7 +127,7 @@ impl DefaultProcessState { message: None, signal_mailbox, message_mailbox, - resources: Resources::new(id, &tracer, &tracer_context), + resources: Resources::default(), wasi, wasi_stdout: None, wasi_stderr: None, @@ -131,7 +137,8 @@ impl DefaultProcessState { registry_atomic_put: None, tracer, tracer_context, - last_span_id: 0, + process_context, + meter_provider, logger: log_filter, }; Ok(state) @@ -150,6 +157,7 @@ impl ProcessState for DefaultProcessState { let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); let message_mailbox = MessageMailbox::default(); let id = self.environment.get_next_process_id(); + let process_context = new_process_context(id, &self.tracer, &self.tracer_context); let wasi = build_wasi( Some(config.command_line_arguments()), Some(config.environment_variables()), @@ -165,7 +173,7 @@ impl ProcessState for DefaultProcessState { message: None, signal_mailbox, message_mailbox, - resources: Resources::new(id, &self.tracer, &self.tracer_context), + resources: Resources::default(), wasi, wasi_stdout: None, wasi_stderr: None, @@ -175,7 +183,8 @@ impl ProcessState for DefaultProcessState { registry_atomic_put: None, tracer: self.tracer.clone(), tracer_context: self.tracer_context.clone(), - last_span_id: 0, + process_context, + meter_provider: self.meter_provider.clone(), logger: self.logger.clone(), }; Ok(state) @@ -436,48 +445,113 @@ impl SQLiteCtx for DefaultProcessState { impl MetricsCtx for DefaultProcessState { type Tracer = BoxedTracer; + type MeterProvider = GlobalMeterProvider; + + fn tracer(&self) -> &Self::Tracer { + &self.tracer + } + + fn meter_provider(&self) -> &Self::MeterProvider { + &self.meter_provider + } fn log(&self, record: &Record) { self.logger.log(record); } - fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option - where - T: Into>, - I: IntoIterator, - { - let parent_ctx = if let Some(parent_id) = parent { - self.resources.spans.get(parent_id)? - } else { - &self.resources.process_context - }; + fn add_context(&mut self, context: Context) -> u64 { + context.span().set_attribute(KeyValue::new("hey", "there")); + self.resources.contexts.add(context) + } + + fn get_context(&self, id: u64) -> Option<&Context> { + self.resources.contexts.get(id) + } + + fn get_last_context(&self) -> &Context { + self.resources + .contexts + .get_last() + .unwrap_or(&self.process_context) + } + + fn drop_context(&mut self, id: u64) { + self.resources.contexts.remove(id); + } + + fn add_meter(&mut self, meter: Meter) -> u64 { + self.resources.meters.add(meter) + } + + fn get_meter(&self, id: u64) -> Option<&Meter> { + self.resources.meters.get(id) + } + + fn drop_meter(&mut self, id: u64) -> Option { + self.resources.meters.remove(id) + } + + fn add_counter(&mut self, counter: Counter) -> u64 { + self.resources.metrics.add(Metric::Counter(counter)) + } - let mut span = self.tracer.start_with_context(name, parent_ctx); - span.set_attributes(attributes); + fn get_counter(&self, id: u64) -> Option<&Counter> { + match self.resources.metrics.get(id)? { + Metric::Counter(counter) => Some(counter), + _ => None, + } + } + + fn drop_counter(&mut self, id: u64) -> Option> { + self.resources + .metrics + .remove(id) + .and_then(|metric| match metric { + Metric::Counter(counter) => Some(counter), + _ => None, + }) + } - let context = parent_ctx.with_span(span); + fn add_gauge(&mut self, gauge: Gauge) -> u64 { + self.resources.metrics.add(Metric::Gauge(gauge)) + } - let id = self.resources.spans.add(context); - self.last_span_id = id; + fn get_gauge(&self, id: u64) -> Option<&Gauge> { + match self.resources.metrics.get(id)? { + Metric::Gauge(gauge) => Some(gauge), + _ => None, + } + } - Some(id) + fn drop_gauge(&mut self, id: u64) -> Option> { + self.resources + .metrics + .remove(id) + .and_then(|metric| match metric { + Metric::Gauge(gauge) => Some(gauge), + _ => None, + }) } - fn drop_span(&mut self, id: u64) { - self.resources.spans.remove(id); - self.last_span_id -= 1; + fn add_histogram(&mut self, histogram: Histogram) -> u64 { + self.resources.metrics.add(Metric::Histogram(histogram)) } - fn get_span(&self, id: u64) -> Option> { - self.resources.spans.get(id).map(|ctx| ctx.span()) + fn get_histogram(&self, id: u64) -> Option<&Histogram> { + match self.resources.metrics.get(id)? { + Metric::Histogram(histogram) => Some(histogram), + _ => None, + } } - fn get_last_span(&self) -> SpanRef<'_> { + fn drop_histogram(&mut self, id: u64) -> Option> { self.resources - .spans - .get(self.last_span_id) - .map(|ctx| ctx.span()) - .unwrap_or_else(|| self.resources.process_context.span()) + .metrics + .remove(id) + .and_then(|metric| match metric { + Metric::Histogram(histogram) => Some(histogram), + _ => None, + }) } } @@ -485,8 +559,9 @@ impl MetricsCtx for DefaultProcessState { pub(crate) struct Resources { pub(crate) configs: HashMapId, pub(crate) modules: HashMapId>>, - pub(crate) spans: SpanResources, - pub(crate) process_context: Context, + pub(crate) contexts: ContextResources, + pub(crate) meters: HashMapId, + pub(crate) metrics: HashMapId>, pub(crate) timers: TimerResources, pub(crate) dns_iterators: HashMapId, pub(crate) tcp_listeners: HashMapId, @@ -497,30 +572,10 @@ pub(crate) struct Resources { pub(crate) errors: HashMapId, } -impl Resources { - fn new(process_id: u64, tracer: &BoxedTracer, tracer_context: &Context) -> Self { - let mut process_span = tracer.start_with_context("process_spawn", tracer_context); - process_span.set_attribute(opentelemetry::KeyValue::new( - "process.id", - process_id as i64, - )); - let process_context = tracer_context.with_span(process_span); - - Resources { - configs: Default::default(), - modules: Default::default(), - spans: Default::default(), - process_context, - timers: Default::default(), - dns_iterators: Default::default(), - tcp_listeners: Default::default(), - tcp_streams: Default::default(), - tls_listeners: Default::default(), - tls_streams: Default::default(), - udp_sockets: Default::default(), - errors: Default::default(), - } - } +pub(crate) enum Metric { + Counter(Counter), + Gauge(Gauge), + Histogram(Histogram), } impl DistributedCtx for DefaultProcessState { @@ -592,7 +647,8 @@ impl DistributedCtx for DefaultProcessState { registry_atomic_put: None, tracer: Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))), tracer_context: Arc::new(Context::new()), - last_span_id: 0, + process_context: Context::new(), + meter_provider: GlobalMeterProvider::new(NoopMeterProvider::new()), logger: Arc::new( env_logger::Builder::new() .filter_level(log::LevelFilter::Off) @@ -603,6 +659,15 @@ impl DistributedCtx for DefaultProcessState { } } +fn new_process_context(process_id: u64, tracer: &BoxedTracer, tracer_context: &Context) -> Context { + let mut process_span = tracer.start_with_context("process_spawn", tracer_context); + process_span.set_attribute(opentelemetry::KeyValue::new( + "process.id", + process_id as i64, + )); + tracer_context.with_span(process_span) +} + /* mod tests { #[tokio::test] async fn import_filter_signature_matches() { From 80f7827a7ac207b953e351a620f382fc27094c17 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Fri, 7 Apr 2023 14:38:12 +0930 Subject: [PATCH 04/16] feat: add metric types and rename metric host functions --- crates/lunatic-metrics-api/src/lib.rs | 429 +++++++++++++++++--------- src/state.rs | 53 ++-- 2 files changed, 323 insertions(+), 159 deletions(-) diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 9c6cf6782..71721202c 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -1,17 +1,15 @@ -use std::{borrow::Cow, sync::Arc}; - -use anyhow::Result; +use anyhow::{Context as _, Result}; use hash_map_id::HashMapId; use log::{Level, Record}; use lunatic_common_api::{get_memory, IntoTrap}; use lunatic_process::state::ProcessState; use lunatic_process_api::ProcessCtx; -use metrics::{counter, decrement_gauge, gauge, histogram, increment_counter, increment_gauge}; use opentelemetry::{ - global, - metrics::{Counter, Histogram, Meter, MeterProvider, Unit}, - sdk::metrics::data::Gauge, - trace::{Span, SpanRef, TraceContextExt, Tracer, TracerProvider}, + metrics::{ + Counter, Histogram, InstrumentBuilder, Meter, MeterProvider, MetricsError, Unit, + UpDownCounter, + }, + trace::{Span, TraceContextExt, Tracer}, Context, KeyValue, StringValue, }; use serde_json::Map; @@ -48,9 +46,9 @@ pub trait MetricsCtx { fn get_counter(&self, id: u64) -> Option<&Counter>; fn drop_counter(&mut self, id: u64) -> Option>; - fn add_gauge(&mut self, gauge: Gauge) -> u64; - fn get_gauge(&self, id: u64) -> Option<&Gauge>; - fn drop_gauge(&mut self, id: u64) -> Option>; + fn add_up_down_counter(&mut self, up_down_counter: UpDownCounter) -> u64; + fn get_up_down_counter(&self, id: u64) -> Option<&UpDownCounter>; + fn drop_up_down_counter(&mut self, id: u64) -> Option>; fn add_histogram(&mut self, histogram: Histogram) -> u64; fn get_histogram(&self, id: u64) -> Option<&Histogram>; @@ -63,22 +61,34 @@ where T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync + 'static, <::Tracer as Tracer>::Span: Send + Sync, { - linker.func_wrap("lunatic::metrics", "start_span", start_span)?; - linker.func_wrap("lunatic::metrics", "drop_span", drop_span)?; + linker.func_wrap("lunatic::metrics", "span_start", span_start)?; + linker.func_wrap("lunatic::metrics", "span_drop", span_drop)?; linker.func_wrap("lunatic::metrics", "meter", meter)?; - linker.func_wrap("lunatic::metrics", "drop_meter", drop_meter)?; + linker.func_wrap("lunatic::metrics", "meter_drop", meter_drop)?; - linker.func_wrap("lunatic::metrics", "add_event", add_event)?; + linker.func_wrap("lunatic::metrics", "event", event)?; linker.func_wrap("lunatic::metrics", "counter", counter)?; - linker.func_wrap("lunatic::metrics", "increment_counter", increment_counter)?; - linker.func_wrap("lunatic::metrics", "drop_counter", drop_counter)?; + linker.func_wrap("lunatic::metrics", "counter_add", counter_add)?; + linker.func_wrap("lunatic::metrics", "counter_drop", counter_drop)?; + + linker.func_wrap("lunatic::metrics", "up_down_counter", up_down_counter)?; + linker.func_wrap( + "lunatic::metrics", + "up_down_counter_add", + up_down_counter_add, + )?; + linker.func_wrap( + "lunatic::metrics", + "up_down_counter_drop", + up_down_counter_drop, + )?; - linker.func_wrap("lunatic::metrics", "gauge", gauge)?; - linker.func_wrap("lunatic::metrics", "increment_gauge", increment_gauge)?; - linker.func_wrap("lunatic::metrics", "decrement_gauge", decrement_gauge)?; linker.func_wrap("lunatic::metrics", "histogram", histogram)?; + linker.func_wrap("lunatic::metrics", "histogram_record", histogram_record)?; + linker.func_wrap("lunatic::metrics", "histogram_drop", histogram_drop)?; + Ok(()) } @@ -92,7 +102,7 @@ where /// * If the attributes is not valid json. /// * If the parent span does not exist. /// * If any memory outside the guest heap space is referenced. -fn start_span( +fn span_start( mut caller: Caller<'_, T>, parent: u64, name_ptr: u32, @@ -113,13 +123,13 @@ where None }; - let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::start_span")?; + let name = get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::span_start")?; let attributes = if attributes_len > 0 { let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) - .or_trap("lunatic::metrics::start_span")?; + .or_trap("lunatic::metrics::span_start")?; let attributes_json = - serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::start_span")?; + serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::span_start")?; data_to_opentelemetry(attributes_json) } else { vec![] @@ -128,7 +138,7 @@ where let parent_ctx = if let Some(id) = parent { state .get_context(id) - .or_trap("lunatic::metrics::start_span")? + .or_trap("lunatic::metrics::span_start")? } else { state.get_last_context() }; @@ -144,7 +154,7 @@ where } /// Drops a span, marking it as finished. -fn drop_span(mut caller: Caller<'_, T>, id: u64) -> Result<()> +fn span_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, <::Tracer as Tracer>::Span: Send + Sync, @@ -169,7 +179,7 @@ where let memory = get_memory(&mut caller)?; let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg(&data, name_ptr, name_len, "lunatic::metrics::meter")?; + let name = get_string_arg(&data, name_ptr, name_len).or_trap("lunatic::metrics::meter")?; let meter = state.meter_provider().meter(name.into()); let id = state.add_meter(meter); @@ -178,7 +188,7 @@ where } /// Drops a meter. -fn drop_meter(mut caller: Caller<'_, T>, id: u64) -> Result<()> +fn meter_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where T: MetricsCtx, { @@ -217,7 +227,7 @@ where /// * If the attributes is not valid json. /// * If the parent span does not exist. /// * If any memory outside the guest heap space is referenced. -fn add_event( +fn event( mut caller: Caller<'_, T>, span: u64, name_ptr: u32, @@ -232,14 +242,14 @@ where let memory = get_memory(&mut caller)?; let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::add_event")?; + let name = get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::event")?; let attributes = if attributes_len > 0 { let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) - .or_trap("lunatic::metrics::add_event")?; + .or_trap("lunatic::metrics::event")?; let attributes_json: Map = - serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::add_event")?; + serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::event")?; let level = attributes_json .get("severityNumber") @@ -301,7 +311,7 @@ where let span = if span != u64::MAX { state .get_context(span) - .or_trap("lunatic::metrics::add_event")? + .or_trap("lunatic::metrics::event")? .span() } else { state.get_last_context().span() @@ -330,39 +340,18 @@ fn counter( where T: MetricsCtx, { - let memory = get_memory(&mut caller)?; - let (data, state) = memory.data_and_store_mut(&mut caller); - - let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::counter")?; - let description = get_string_arg( - data, + let (state, counter) = create_metric( + &mut caller, + meter, + name_ptr, + name_len, description_ptr, description_len, - "lunatic::metrics::counter", - )?; - let unit = - get_string_arg(data, unit_ptr, unit_len, "lunatic::metrics::counter").map(|unit| { - if unit.is_empty() { - None - } else { - Some(Unit::new(unit)) - } - })?; - - let mut counter_builder = state - .get_meter(meter) - .or_trap("lunatic::metrics::counter")? - .f64_counter(name); - if !description.is_empty() { - counter_builder = counter_builder.with_description(description); - } - if let Some(unit) = unit { - counter_builder = counter_builder.with_unit(unit); - } - - let counter = counter_builder - .try_init() - .or_trap("lunatic::metrics::counter")?; + unit_ptr, + unit_len, + |meter, name| meter.f64_counter(name), + ) + .or_trap("lunatic::metrics::counter")?; let id = state.add_counter(counter); @@ -374,7 +363,7 @@ where /// Traps: /// * If the name is not a valid utf8 string. /// * If any memory outside the guest heap space is referenced. -fn increment_counter( +fn counter_add( mut caller: Caller<'_, T>, span: u64, counter: u64, @@ -385,32 +374,12 @@ fn increment_counter( where T: MetricsCtx, { - let memory = get_memory(&mut caller)?; - let (data, state) = memory.data_and_store_mut(&mut caller); - - let cx = if span != u64::MAX { - state - .get_context(span) - .or_trap("lunatic::metrics::increment_counter")? - } else { - state.get_last_context() - }; - - let attributes = if attributes_len > 0 { - let attributes_data = data - .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) - .or_trap("lunatic::metrics::add_event")?; - let attributes_json: Map = - serde_json::from_slice(attributes_data).or_trap("lunatic::metrics::add_event")?; - - data_to_opentelemetry(attributes_json) - } else { - vec![] - }; + let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) + .or_trap("lunatic::metrics::counter_add")?; let counter = state .get_counter(counter) - .or_trap("lunatic::metrics::increment_counter")?; + .or_trap("lunatic::metrics::counter_add")?; counter.add(cx, amount, &attributes); @@ -418,7 +387,7 @@ where } /// Drops a counter. -fn drop_counter(mut caller: Caller<'_, T>, id: u64) -> Result<()> +fn counter_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where T: MetricsCtx, { @@ -430,97 +399,275 @@ where Ok(()) } -/// Sets a gauge. -/// -/// Traps: -/// * If the name is not a valid utf8 string. -/// * If any memory outside the guest heap space is referenced. -fn gauge(mut caller: Caller<'_, T>, name_ptr: u32, name_len: u32, value: f64) -> Result<()> { +fn up_down_counter( + mut caller: Caller<'_, T>, + meter: u64, + name_ptr: u32, + name_len: u32, + description_ptr: u32, + description_len: u32, + unit_ptr: u32, + unit_len: u32, +) -> Result +where + T: MetricsCtx, +{ + let (state, up_down_counter) = create_metric( + &mut caller, + meter, + name_ptr, + name_len, + description_ptr, + description_len, + unit_ptr, + unit_len, + |meter, name| meter.f64_up_down_counter(name), + ) + .or_trap("lunatic::metrics::up_down_counter")?; + + let id = state.add_up_down_counter(up_down_counter); + + Ok(id) +} + +fn up_down_counter_add( + mut caller: Caller<'_, T>, + span: u64, + counter: u64, + amount: f64, + attributes_ptr: u32, + attributes_len: u32, +) -> Result<()> +where + T: MetricsCtx, +{ + let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) + .or_trap("lunatic::metrics::up_down_counter_add")?; + + let counter = state + .get_up_down_counter(counter) + .or_trap("lunatic::metrics::up_down_counter_add")?; + + counter.add(cx, amount, &attributes); + + Ok(()) +} + +fn up_down_counter_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: MetricsCtx, +{ let memory = get_memory(&mut caller)?; - let data = memory.data(&mut caller); + let (_data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::gauge")?; + state.drop_up_down_counter(id); - gauge!(name, value); Ok(()) } -/// Increments a gauge. +// /// Increments a gauge. +// /// +// /// Traps: +// /// * If the name is not a valid utf8 string. +// /// * If any memory outside the guest heap space is referenced. +// fn increment_gauge( +// mut caller: Caller<'_, T>, +// name_ptr: u32, +// name_len: u32, +// value: f64, +// ) -> Result<()> { +// todo!() +// // let memory = get_memory(&mut caller)?; +// // let data = memory.data(&mut caller); +// // +// // let name = +// // get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::increment_gauge")?; +// // +// // increment_gauge!(name, value); +// // Ok(()) +// } +// +// /// Decrements a gauge. +// /// +// /// Traps: +// /// * If the name is not a valid utf8 string. +// /// * If any memory outside the guest heap space is referenced. +// fn decrement_gauge( +// mut caller: Caller<'_, T>, +// name_ptr: u32, +// name_len: u32, +// value: f64, +// ) -> Result<()> { +// todo!() +// // let memory = get_memory(&mut caller)?; +// // let data = memory.data(&mut caller); +// // +// // let name = +// // get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::decrement_gauge")?; +// // +// // decrement_gauge!(name, value); +// // Ok(()) +// } + +/// Sets a histogram. /// /// Traps: /// * If the name is not a valid utf8 string. /// * If any memory outside the guest heap space is referenced. -fn increment_gauge( +fn histogram( mut caller: Caller<'_, T>, + meter: u64, name_ptr: u32, name_len: u32, - value: f64, -) -> Result<()> { - let memory = get_memory(&mut caller)?; - let data = memory.data(&mut caller); - - let name = get_string_arg( - data, + description_ptr: u32, + description_len: u32, + unit_ptr: u32, + unit_len: u32, +) -> Result +where + T: MetricsCtx, +{ + let (state, histogram) = create_metric( + &mut caller, + meter, name_ptr, name_len, - "lunatic::metrics::increment_gauge", - )?; + description_ptr, + description_len, + unit_ptr, + unit_len, + |meter, name| meter.f64_histogram(name), + ) + .or_trap("lunatic::metrics::histogram")?; - increment_gauge!(name, value); - Ok(()) + let id = state.add_histogram(histogram); + + Ok(id) } -/// Decrements a gauge. -/// -/// Traps: -/// * If the name is not a valid utf8 string. -/// * If any memory outside the guest heap space is referenced. -fn decrement_gauge( +fn histogram_record( mut caller: Caller<'_, T>, - name_ptr: u32, - name_len: u32, + span: u64, + histogram: u64, value: f64, -) -> Result<()> { - let memory = get_memory(&mut caller)?; - let data = memory.data(&mut caller); + attributes_ptr: u32, + attributes_len: u32, +) -> Result<()> +where + T: MetricsCtx, +{ + let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) + .or_trap("lunatic::metrics::histogram_record")?; - let name = get_string_arg( - data, - name_ptr, - name_len, - "lunatic::metrics::decrement_gauge", - )?; + let histogram = state + .get_histogram(histogram) + .or_trap("lunatic::metrics::histogram_record")?; + + histogram.record(cx, value, &attributes); - decrement_gauge!(name, value); Ok(()) } -/// Sets a histogram. -/// -/// Traps: -/// * If the name is not a valid utf8 string. -/// * If any memory outside the guest heap space is referenced. -fn histogram(mut caller: Caller<'_, T>, name_ptr: u32, name_len: u32, value: f64) -> Result<()> { +fn histogram_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> +where + T: MetricsCtx, +{ let memory = get_memory(&mut caller)?; - let data = memory.data(&mut caller); + let (_data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg(data, name_ptr, name_len, "lunatic::metrics::histogram")?; + state.drop_histogram(id); - histogram!(name, value); Ok(()) } -fn get_string_arg(data: &[u8], name_ptr: u32, name_len: u32, func_name: &str) -> Result { +fn get_string_arg(data: &[u8], name_ptr: u32, name_len: u32) -> Result { if name_len == 0 { return Ok(String::new()); } let name = data .get(name_ptr as usize..(name_ptr + name_len) as usize) - .or_trap(func_name)?; - let name = String::from_utf8(name.to_vec()).or_trap(func_name)?; + .context("invalid memory region")?; + let name = String::from_utf8(name.to_vec())?; Ok(name) } +fn create_metric<'a, T, M, F>( + caller: &'a mut Caller<'_, T>, + meter: u64, + name_ptr: u32, + name_len: u32, + description_ptr: u32, + description_len: u32, + unit_ptr: u32, + unit_len: u32, + builder: F, +) -> Result<(&'a mut T, M)> +where + T: MetricsCtx, + M: for<'b> TryFrom, Error = MetricsError>, + F: for<'b> Fn(&'b Meter, String) -> InstrumentBuilder<'b, M>, +{ + let memory = get_memory(caller)?; + let (data, state) = memory.data_and_store_mut(caller); + + let name = get_string_arg(data, name_ptr, name_len)?; + let description = get_string_arg(data, description_ptr, description_len)?; + let unit = get_string_arg(data, unit_ptr, unit_len).map(|unit| { + if unit.is_empty() { + None + } else { + Some(Unit::new(unit)) + } + })?; + + let meter = state.get_meter(meter).context("meter does not exist")?; + + let mut metric_builder = builder(meter, name); + if !description.is_empty() { + metric_builder = metric_builder.with_description(description); + } + if let Some(unit) = unit { + metric_builder = metric_builder.with_unit(unit); + } + + let metric = metric_builder.try_init()?; + + Ok((state, metric)) +} + +fn update_metric<'a, T>( + caller: &'a mut Caller<'_, T>, + span: u64, + attributes_ptr: u32, + attributes_len: u32, +) -> Result<(&'a T, &'a Context, Vec)> +where + T: MetricsCtx, +{ + let memory = get_memory(caller)?; + let (data, state) = memory.data_and_store_mut(caller); + + let cx = if span != u64::MAX { + state.get_context(span).context("span does not exist")? + } else { + state.get_last_context() + }; + + let attributes = if attributes_len > 0 { + let attributes_data = data + .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) + .context("invalid memory region for attributes")?; + let attributes_json: Map = + serde_json::from_slice(attributes_data)?; + + data_to_opentelemetry(attributes_json) + } else { + vec![] + }; + + Ok((state, cx, attributes)) +} + fn data_to_opentelemetry(data: Map) -> Vec { data.into_iter() .map(|(k, v)| KeyValue { diff --git a/src/state.rs b/src/state.rs index 8b6c1d2a5..20cdfe19c 100644 --- a/src/state.rs +++ b/src/state.rs @@ -26,8 +26,7 @@ use lunatic_timer_api::{TimerCtx, TimerResources}; use lunatic_wasi_api::{build_wasi, LunaticWasiCtx}; use opentelemetry::global::{BoxedTracer, GlobalMeterProvider}; use opentelemetry::metrics::noop::NoopMeterProvider; -use opentelemetry::metrics::{Counter, Histogram, Meter}; -use opentelemetry::sdk::metrics::data::Gauge; +use opentelemetry::metrics::{Counter, Histogram, Meter, UpDownCounter}; use opentelemetry::trace::noop::NoopTracer; use opentelemetry::trace::{Span, TraceContextExt, Tracer}; use opentelemetry::{Context, KeyValue}; @@ -512,23 +511,25 @@ impl MetricsCtx for DefaultProcessState { }) } - fn add_gauge(&mut self, gauge: Gauge) -> u64 { - self.resources.metrics.add(Metric::Gauge(gauge)) + fn add_up_down_counter(&mut self, up_down_counter: UpDownCounter) -> u64 { + self.resources + .metrics + .add(Metric::UpDownCounter(up_down_counter)) } - fn get_gauge(&self, id: u64) -> Option<&Gauge> { + fn get_up_down_counter(&self, id: u64) -> Option<&UpDownCounter> { match self.resources.metrics.get(id)? { - Metric::Gauge(gauge) => Some(gauge), + Metric::UpDownCounter(up_down_counter) => Some(up_down_counter), _ => None, } } - fn drop_gauge(&mut self, id: u64) -> Option> { + fn drop_up_down_counter(&mut self, id: u64) -> Option> { self.resources .metrics .remove(id) .and_then(|metric| match metric { - Metric::Gauge(gauge) => Some(gauge), + Metric::UpDownCounter(up_down_counter) => Some(up_down_counter), _ => None, }) } @@ -574,7 +575,7 @@ pub(crate) struct Resources { pub(crate) enum Metric { Counter(Counter), - Gauge(Gauge), + UpDownCounter(UpDownCounter), Histogram(Histogram), } @@ -614,9 +615,6 @@ impl DistributedCtx for DefaultProcessState { runtime: WasmtimeRuntime, module: Arc>, config: Arc, - // tracer: Arc, - // tracer_context: Arc, - // logger: Arc, ) -> Result { let signal_mailbox = unbounded_channel(); let signal_mailbox = (signal_mailbox.0, Arc::new(Mutex::new(signal_mailbox.1))); @@ -668,18 +666,25 @@ fn new_process_context(process_id: u64, tracer: &BoxedTracer, tracer_context: &C tracer_context.with_span(process_span) } -/* mod tests { +mod tests { #[tokio::test] async fn import_filter_signature_matches() { use std::collections::HashMap; - use tokio::sync::RwLock; + use std::sync::Arc; - use crate::state::DefaultProcessState; - use crate::DefaultProcessConfig; use lunatic_process::env::Environment; use lunatic_process::runtimes::wasmtime::WasmtimeRuntime; use lunatic_process::wasm::spawn_wasm; - use std::sync::Arc; + use opentelemetry::{ + global::{BoxedTracer, GlobalMeterProvider}, + metrics::noop::NoopMeterProvider, + trace::noop::NoopTracer, + Context, + }; + use tokio::sync::RwLock; + + use crate::state::DefaultProcessState; + use crate::DefaultProcessConfig; // The default configuration includes both, the "lunatic::*" and "wasi_*" namespaces. let config = DefaultProcessConfig::default(); @@ -693,6 +698,14 @@ fn new_process_context(process_id: u64, tracer: &BoxedTracer, tracer_context: &C let module = Arc::new(runtime.compile_module(raw_module.into()).unwrap()); let env = Arc::new(lunatic_process::env::LunaticEnvironment::new(0)); let registry = Arc::new(RwLock::new(HashMap::new())); + let tracer = Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))); + let tracer_context = Arc::new(Context::new()); + let meter_provider = GlobalMeterProvider::new(NoopMeterProvider::new()); + let logger = Arc::new( + env_logger::Builder::new() + .filter_level(log::LevelFilter::Off) + .build(), + ); let state = DefaultProcessState::new( env.clone(), None, @@ -700,6 +713,10 @@ fn new_process_context(process_id: u64, tracer: &BoxedTracer, tracer_context: &C module.clone(), Arc::new(config), registry, + tracer, + tracer_context, + meter_provider, + logger, ) .unwrap(); @@ -709,4 +726,4 @@ fn new_process_context(process_id: u64, tracer: &BoxedTracer, tracer_context: &C .await .unwrap(); } -} */ +} From dac04b82c032b171ab57ffdd7a2a4f7ea816031e Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Fri, 7 Apr 2023 14:40:00 +0930 Subject: [PATCH 05/16] chore: remove unused prometheus argument --- src/mode/execution.rs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mode/execution.rs b/src/mode/execution.rs index 6ae2f1da8..d2f7294d4 100644 --- a/src/mode/execution.rs +++ b/src/mode/execution.rs @@ -6,10 +6,6 @@ use clap::{Parser, Subcommand}; pub struct Args { #[command(subcommand)] command: Commands, - - #[cfg(feature = "prometheus")] - #[command(flatten)] - prometheus: super::common::PrometheusArgs, } #[derive(Debug, Subcommand)] From 7f940fc8a43e86d454b45858ce838715c5863e3a Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 16:55:32 +0930 Subject: [PATCH 06/16] feat: use opentelemetry metrics instead of metrics crate --- Cargo.lock | 44 +--- Cargo.toml | 18 +- crates/lunatic-common-api/Cargo.toml | 2 + crates/lunatic-common-api/src/lib.rs | 21 +- crates/lunatic-metrics-api/Cargo.toml | 1 - crates/lunatic-metrics-api/src/lib.rs | 7 - crates/lunatic-process-api/Cargo.toml | 6 +- crates/lunatic-process-api/src/lib.rs | 186 +++++++++------ crates/lunatic-process/Cargo.toml | 12 +- crates/lunatic-process/src/env.rs | 54 ++--- crates/lunatic-process/src/lib.rs | 311 ++++++++++++++++--------- crates/lunatic-process/src/message.rs | 22 +- crates/lunatic-registry-api/Cargo.toml | 8 +- crates/lunatic-registry-api/src/lib.rs | 99 ++++---- crates/lunatic-timer-api/Cargo.toml | 6 +- crates/lunatic-timer-api/src/lib.rs | 103 +++++--- src/mode/common.rs | 3 +- src/state.rs | 2 - 18 files changed, 525 insertions(+), 380 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbcc7ea5a..7b392410c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1482,6 +1482,8 @@ name = "lunatic-common-api" version = "0.13.0" dependencies = [ "anyhow", + "once_cell", + "opentelemetry", "wasmtime", ] @@ -1589,7 +1591,6 @@ dependencies = [ "lunatic-common-api", "lunatic-process", "lunatic-process-api", - "metrics", "opentelemetry", "serde_json", "wasmtime", @@ -1619,8 +1620,10 @@ dependencies = [ "dashmap", "hash-map-id", "log", + "lunatic-common-api", "lunatic-networking-api", - "metrics", + "once_cell", + "opentelemetry", "serde", "smallvec", "tokio", @@ -1638,7 +1641,8 @@ dependencies = [ "lunatic-error-api", "lunatic-process", "lunatic-wasi-api", - "metrics", + "once_cell", + "opentelemetry", "tokio", "wasmtime", ] @@ -1651,7 +1655,8 @@ dependencies = [ "lunatic-common-api", "lunatic-process", "lunatic-process-api", - "metrics", + "once_cell", + "opentelemetry", "tokio", "wasmtime", ] @@ -1734,7 +1739,8 @@ dependencies = [ "lunatic-common-api", "lunatic-process", "lunatic-process-api", - "metrics", + "once_cell", + "opentelemetry", "tokio", "wasmtime", ] @@ -1822,28 +1828,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "metrics" -version = "0.20.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b9b8653cec6897f73b519a43fba5ee3d50f62fe9af80b428accdcc093b4a849" -dependencies = [ - "ahash", - "metrics-macros", - "portable-atomic", -] - -[[package]] -name = "metrics-macros" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "731f8ecebd9f3a4aa847dfe75455e4757a45da40a7793d2f0b1f9b6ed18b23f3" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "mime" version = "0.3.16" @@ -2241,12 +2225,6 @@ dependencies = [ "plotters-backend", ] -[[package]] -name = "portable-atomic" -version = "0.3.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" - [[package]] name = "ppv-lite86" version = "0.2.17" diff --git a/Cargo.toml b/Cargo.toml index 781718536..faaf4975c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,16 +23,6 @@ path = "src/main.rs" name = "cargo-lunatic" path = "src/cargo_lunatic.rs" -[features] -default = ["metrics"] -metrics = [ - "lunatic-process-api/metrics", - "lunatic-process/metrics", - "lunatic-registry-api/metrics", - "lunatic-timer-api/metrics", - "dep:lunatic-metrics-api", -] - [dependencies] hash-map-id = { workspace = true } lunatic-control = { workspace = true } @@ -41,17 +31,17 @@ lunatic-distributed = { workspace = true } lunatic-distributed-api = { workspace = true } lunatic-error-api = { workspace = true } lunatic-messaging-api = { workspace = true } +lunatic-metrics-api = { workspace = true } lunatic-networking-api = { workspace = true } lunatic-process = { workspace = true } lunatic-process-api = { workspace = true } lunatic-registry-api = { workspace = true } +lunatic-sqlite-api = { workspace = true } lunatic-stdout-capture = { workspace = true } lunatic-timer-api = { workspace = true } +lunatic-trap-api = { workspace = true } lunatic-version-api = { workspace = true } -lunatic-metrics-api = { workspace = true, optional = true } lunatic-wasi-api = { workspace = true } -lunatic-trap-api = { workspace = true } -lunatic-sqlite-api = { workspace = true } anyhow = { workspace = true } async-ctrlc = "1.2.0" @@ -130,8 +120,8 @@ anyhow = "1.0" bincode = "1.3" dashmap = "5.4" log = "0.4" +once_cell = "1.17" opentelemetry = { version = "0.19", git = "https://github.com/tqwewe/opentelemetry-rust", branch = "cow", features = ["metrics"] } -metrics = "0.20.1" reqwest = "0.11" rustls-pemfile = "1.0" serde = "1.0" diff --git a/crates/lunatic-common-api/Cargo.toml b/crates/lunatic-common-api/Cargo.toml index b1c4a8c72..0a61f6e22 100644 --- a/crates/lunatic-common-api/Cargo.toml +++ b/crates/lunatic-common-api/Cargo.toml @@ -9,4 +9,6 @@ license = "Apache-2.0/MIT" [dependencies] anyhow = { workspace = true } +once_cell = { workspace = true } +opentelemetry = { workspace = true } wasmtime = { workspace = true } diff --git a/crates/lunatic-common-api/src/lib.rs b/crates/lunatic-common-api/src/lib.rs index 277e4791c..9ce907195 100644 --- a/crates/lunatic-common-api/src/lib.rs +++ b/crates/lunatic-common-api/src/lib.rs @@ -1,5 +1,7 @@ -use anyhow::{anyhow, Context, Result}; use std::{fmt::Display, future::Future, io::Write, pin::Pin}; + +use anyhow::{anyhow, Context, Result}; +use once_cell::sync::OnceCell; use wasmtime::{Caller, Memory, Val}; const ALLOCATOR_FUNCTION_NAME: &str = "lunatic_alloc"; @@ -78,6 +80,23 @@ pub async fn write_to_guest_vec( Ok(alloc_ptr) } +pub trait MetricsExt { + fn with_current_context(&self, f: F) + where + F: Fn(&T, opentelemetry::Context); +} + +impl MetricsExt for OnceCell { + fn with_current_context(&self, f: F) + where + F: Fn(&T, opentelemetry::Context), + { + if let Some(v) = self.get() { + f(v, opentelemetry::Context::current()); + } + } +} + pub trait IntoTrap { fn or_trap(self, info: S) -> Result; } diff --git a/crates/lunatic-metrics-api/Cargo.toml b/crates/lunatic-metrics-api/Cargo.toml index ecb206a60..be0e1af80 100644 --- a/crates/lunatic-metrics-api/Cargo.toml +++ b/crates/lunatic-metrics-api/Cargo.toml @@ -16,5 +16,4 @@ lunatic-process = { workspace = true } lunatic-process-api = { workspace = true } opentelemetry = { workspace = true } serde_json = "1.0" -metrics = { workspace = true } wasmtime = { workspace = true } diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 71721202c..3fc37f689 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -24,13 +24,6 @@ pub trait MetricsCtx { fn tracer(&self) -> &Self::Tracer; fn meter_provider(&self) -> &Self::MeterProvider; - // fn add_span(&mut self, parent: Option, name: T, attributes: I) -> Option - // where - // T: Into>, - // I: IntoIterator; - // fn get_span(&self, id: u64) -> Option>; - // fn get_last_span(&self) -> SpanRef; - // fn drop_span(&mut self, id: u64); fn add_context(&mut self, context: Context) -> u64; fn get_context(&self, id: u64) -> Option<&Context>; fn get_last_context(&self) -> &Context; diff --git a/crates/lunatic-process-api/Cargo.toml b/crates/lunatic-process-api/Cargo.toml index 2bea8fd57..89ae3108c 100644 --- a/crates/lunatic-process-api/Cargo.toml +++ b/crates/lunatic-process-api/Cargo.toml @@ -7,9 +7,6 @@ homepage = "https://lunatic.solutions" repository = "https://github.com/lunatic-solutions/lunatic/tree/main/crates/lunatic-process-api" license = "Apache-2.0/MIT" -[features] -metrics = ["dep:metrics"] - [dependencies] hash-map-id = { workspace = true } lunatic-common-api = { workspace = true } @@ -18,6 +15,7 @@ lunatic-process = { workspace = true } lunatic-wasi-api = { workspace = true } anyhow = { workspace = true } -metrics = { workspace = true, optional = true } +once_cell = { workspace = true } +opentelemetry = { workspace = true } tokio = { workspace = true, features = ["time"] } wasmtime = { workspace = true } diff --git a/crates/lunatic-process-api/src/lib.rs b/crates/lunatic-process-api/src/lib.rs index 228d9f5b5..be9e8fe87 100644 --- a/crates/lunatic-process-api/src/lib.rs +++ b/crates/lunatic-process-api/src/lib.rs @@ -8,7 +8,7 @@ use std::{ use anyhow::{anyhow, Result}; use hash_map_id::HashMapId; -use lunatic_common_api::{get_memory, IntoTrap}; +use lunatic_common_api::{get_memory, IntoTrap, MetricsExt}; use lunatic_error_api::ErrorCtx; use lunatic_process::{ config::ProcessConfig, @@ -20,6 +20,11 @@ use lunatic_process::{ DeathReason, Process, Signal, WasmProcess, }; use lunatic_wasi_api::LunaticWasiCtx; +use once_cell::sync::OnceCell; +use opentelemetry::{ + global, + metrics::{Counter, Histogram, Meter, Unit, UpDownCounter}, +}; use wasmtime::{Caller, Linker, ResourceLimiter, Val}; pub type ProcessResources = HashMapId>; @@ -43,6 +48,24 @@ pub trait ProcessCtx { fn environment(&self) -> Arc; } +struct ModulesMetrics { + _meter: Meter, + compiled: Counter, + dropped: Counter, + active: UpDownCounter, + compiled_duration: Histogram, +} + +struct ConfigsMetrics { + _meter: Meter, + created: Counter, + dropped: Counter, + active: UpDownCounter, +} + +static MODULES_METRICS: OnceCell = OnceCell::new(); +static CONFIGS_METRICS: OnceCell = OnceCell::new(); + // Register the process APIs to the linker pub fn register(linker: &mut Linker) -> Result<()> where @@ -57,61 +80,71 @@ where for<'a> &'a T: Send, T::Config: ProcessConfigCtx, { - #[cfg(feature = "metrics")] - lunatic_process::describe_metrics(); - - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.process.modules.compiled", - metrics::Unit::Count, - "number of modules compiled since startup" - ); - - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.process.modules.dropped", - metrics::Unit::Count, - "number of modules dropped since startup" - ); - - #[cfg(feature = "metrics")] - metrics::describe_gauge!( - "lunatic.process.modules.active", - metrics::Unit::Count, - "number of modules currently in memory" - ); - - #[cfg(feature = "metrics")] - metrics::describe_histogram!( - "lunatic.process.modules.compiled.duration", - metrics::Unit::Seconds, - "Duration of module compilation" - ); + lunatic_process::init_metrics(); + + MODULES_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.modules"); + + let compiled = meter + .u64_counter("compiled") + .with_unit(Unit::new("count")) + .with_description("Number of modules compiled since startup") + .init(); + let dropped = meter + .u64_counter("dropped") + .with_unit(Unit::new("count")) + .with_description("Number of modules dropped since startup") + .init(); + let active = meter + .i64_up_down_counter("active") + .with_unit(Unit::new("count")) + .with_description("Number of modules currently in memory") + .init(); + let compiled_duration = meter + .f64_histogram("compiled_duration") + .with_unit(Unit::new("seconds")) + .with_description("Duration of module compilation") + .init(); + + ModulesMetrics { + _meter: meter, + compiled, + dropped, + active, + compiled_duration, + } + }); + + CONFIGS_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.configs"); + + let created = meter + .u64_counter("created") + .with_unit(Unit::new("count")) + .with_description("Number of configs created since startup") + .init(); + let dropped = meter + .u64_counter("dropped") + .with_unit(Unit::new("count")) + .with_description("Number of configs dropped since startup") + .init(); + let active = meter + .i64_up_down_counter("active") + .with_unit(Unit::new("count")) + .with_description("Number of configs currently in memory") + .init(); + + ConfigsMetrics { + _meter: meter, + created, + dropped, + active, + } + }); linker.func_wrap("lunatic::process", "compile_module", compile_module)?; linker.func_wrap("lunatic::process", "drop_module", drop_module)?; - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.process.configs.created", - metrics::Unit::Count, - "number of configs created since startup" - ); - - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.process.configs.dropped", - metrics::Unit::Count, - "number of configs dropped since startup" - ); - - #[cfg(feature = "metrics")] - metrics::describe_gauge!( - "lunatic.process.configs.active", - metrics::Unit::Count, - "number of configs currently in memory" - ); - linker.func_wrap("lunatic::process", "create_config", create_config)?; linker.func_wrap("lunatic::process", "drop_config", drop_config)?; linker.func_wrap( @@ -202,13 +235,6 @@ where return Ok(-1); } - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.modules.compiled"); - - #[cfg(feature = "metrics")] - metrics::increment_gauge!("lunatic.process.modules.active", 1.0); - - #[cfg(feature = "metrics")] let start = Instant::now(); let mut module = vec![0; module_data_len as usize]; @@ -229,10 +255,14 @@ where Err(error) => (caller.data_mut().error_resources_mut().add(error), 1), }; - #[cfg(feature = "metrics")] let duration = Instant::now() - start; - #[cfg(feature = "metrics")] - metrics::histogram!("lunatic.process.modules.compiled.duration", duration); + MODULES_METRICS.with_current_context(|metrics, cx| { + metrics.compiled.add(&cx, 1, &[]); + metrics.active.add(&cx, 1, &[]); + metrics + .compiled_duration + .record(&cx, duration.as_secs_f64(), &[]); + }); memory .write(&mut caller, id_ptr as usize, &mod_or_error_id.to_le_bytes()) @@ -248,17 +278,17 @@ fn drop_module>( mut caller: Caller, module_id: u64, ) -> Result<()> { - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.modules.dropped"); - - #[cfg(feature = "metrics")] - metrics::decrement_gauge!("lunatic.process.modules.active", 1.0); - caller .data_mut() .module_resources_mut() .remove(module_id) .or_trap("lunatic::process::drop_module: Module ID doesn't exist")?; + + MODULES_METRICS.with_current_context(|metrics, cx| { + metrics.dropped.add(&cx, 1, &[]); + metrics.active.add(&cx, -1, &[]); + }); + Ok(()) } @@ -277,11 +307,13 @@ where if !caller.data().config().can_create_configs() { return -1; } + + CONFIGS_METRICS.with_current_context(|metrics, cx| { + metrics.created.add(&cx, 1, &[]); + metrics.active.add(&cx, 1, &[]); + }); + let config = T::Config::default(); - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.configs.created"); - #[cfg(feature = "metrics")] - metrics::increment_gauge!("lunatic.process.configs.active", 1.0); caller.data_mut().config_resources_mut().add(config) as i64 } @@ -298,10 +330,12 @@ fn drop_config>( .config_resources_mut() .remove(config_id) .or_trap("lunatic::process::drop_config: Config ID doesn't exist")?; - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.configs.dropped"); - #[cfg(feature = "metrics")] - metrics::decrement_gauge!("lunatic.process.configs.active", 1.0); + + CONFIGS_METRICS.with_current_context(|metrics, cx| { + metrics.dropped.add(&cx, 1, &[]); + metrics.active.add(&cx, -1, &[]); + }); + Ok(()) } diff --git a/crates/lunatic-process/Cargo.toml b/crates/lunatic-process/Cargo.toml index 235c8fdfb..ac5c79d62 100644 --- a/crates/lunatic-process/Cargo.toml +++ b/crates/lunatic-process/Cargo.toml @@ -7,21 +7,17 @@ homepage = "https://lunatic.solutions" repository = "https://github.com/lunatic-solutions/lunatic/tree/main/crates/lunatic-process" license = "Apache-2.0/MIT" -[features] -metrics = ["dep:metrics"] - -# Disabled by default as it will usually lead to giant metrics exports -detailed_metrics = ["metrics"] - [dependencies] hash-map-id = { workspace = true } +lunatic-common-api = { workspace = true } lunatic-networking-api = { workspace = true } -async-trait = "0.1.58" anyhow = { workspace = true } +async-trait = "0.1.58" dashmap = { workspace = true } log = { workspace = true } -metrics = { workspace = true, optional = true } +once_cell = { workspace = true } +opentelemetry = { workspace = true } serde = { workspace = true } smallvec = "1.10" tokio = { workspace = true, features = [ diff --git a/crates/lunatic-process/src/env.rs b/crates/lunatic-process/src/env.rs index ae58455b1..431626bd1 100644 --- a/crates/lunatic-process/src/env.rs +++ b/crates/lunatic-process/src/env.rs @@ -1,12 +1,15 @@ -use anyhow::Result; -use async_trait::async_trait; -use dashmap::DashMap; use std::sync::{ atomic::{AtomicU64, Ordering}, Arc, }; -use crate::{Process, Signal}; +use anyhow::Result; +use async_trait::async_trait; +use dashmap::DashMap; +use lunatic_common_api::MetricsExt; +use opentelemetry::KeyValue; + +use crate::{Process, Signal, ENVIRONMENT_METRICS}; #[async_trait] pub trait Environment: Send + Sync { @@ -54,30 +57,26 @@ impl Environment for LunaticEnvironment { fn add_process(&self, id: u64, proc: Arc) { self.processes.insert(id, proc); - #[cfg(all(feature = "metrics", not(feature = "detailed_metrics")))] - let labels: [(String, String); 0] = []; - #[cfg(all(feature = "metrics", feature = "detailed_metrics"))] - let labels = [("environment_id", self.id().to_string())]; - #[cfg(feature = "metrics")] - metrics::gauge!( - "lunatic.process.environment.process.count", - self.processes.len() as f64, - &labels - ); + + ENVIRONMENT_METRICS.with_current_context(|metrics, cx| { + metrics.process_count.add( + &cx, + self.processes.len() as i64, + &[KeyValue::new("environment_id", self.id() as i64)], + ); + }); } fn remove_process(&self, id: u64) { self.processes.remove(&id); - #[cfg(all(feature = "metrics", not(feature = "detailed_metrics")))] - let labels: [(String, String); 0] = []; - #[cfg(all(feature = "metrics", feature = "detailed_metrics"))] - let labels = [("environment_id", self.id().to_string())]; - #[cfg(feature = "metrics")] - metrics::gauge!( - "lunatic.process.environment.process.count", - self.processes.len() as f64, - &labels - ); + + ENVIRONMENT_METRICS.with_current_context(|metrics, cx| { + metrics.process_count.add( + &cx, + -(self.processes.len() as i64), + &[KeyValue::new("environment_id", self.id() as i64)], + ); + }); } fn process_count(&self) -> usize { @@ -121,8 +120,11 @@ impl Environments for LunaticEnvironments { async fn create(&self, id: u64) -> Arc { let env = Arc::new(LunaticEnvironment::new(id)); self.envs.insert(id, env.clone()); - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.environment.count", self.envs.len() as f64); + + ENVIRONMENT_METRICS.with_current_context(|metrics, cx| { + metrics.count.add(&cx, 1, &[]); + }); + env } diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index b97dde7ca..4b5aad65e 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -12,6 +12,13 @@ use anyhow::{anyhow, Result}; use env::Environment; use log::{debug, log_enabled, trace, warn, Level}; +use lunatic_common_api::MetricsExt; +use once_cell::sync::OnceCell; +use opentelemetry::{ + global, + metrics::{Counter, Histogram, Meter, Unit, UpDownCounter}, + KeyValue, +}; use smallvec::SmallVec; use state::ProcessState; use tokio::{ @@ -24,77 +31,43 @@ use tokio::{ use crate::{mailbox::MessageMailbox, message::Message}; -#[cfg(feature = "metrics")] -pub fn describe_metrics() { - use metrics::{describe_counter, describe_gauge, describe_histogram, Unit}; - - describe_counter!( - "lunatic.process.signals.send", - Unit::Count, - "Number of signals sent to processes since startup" - ); - - describe_counter!( - "lunatic.process.signals.received", - Unit::Count, - "Number of signals received by processes since startup" - ); - - describe_counter!( - "lunatic.process.messages.send", - Unit::Count, - "Number of messages sent to processes since startup" - ); - - describe_gauge!( - "lunatic.process.messages.outstanding", - Unit::Count, - "Current number of messages that are ready to be consumed by the process" - ); - - describe_gauge!( - "lunatic.process.links.alive", - Unit::Count, - "Number of links currently alive" - ); - - describe_counter!( - "lunatic.process.messages.data.count", - Unit::Count, - "Number of data messages send since startup" - ); - - describe_histogram!( - "lunatic.process.messages.data.resources.count", - Unit::Count, - "Number of resources used by each individual data message" - ); - - describe_histogram!( - "lunatic.process.messages.data.size", - Unit::Bytes, - "Number of bytes used by each individual data message" - ); - - describe_counter!( - "lunatic.process.messages.link_died.count", - Unit::Count, - "Number of LinkDied messages send since startup" - ); - - describe_gauge!( - "lunatic.process.environment.process.count", - Unit::Count, - "Number of currently registered processes" - ); - - describe_gauge!( - "lunatic.process.environment.count", - Unit::Count, - "Number of currently active environments" - ); +struct SignalsMetrics { + _meter: Meter, + sent: Counter, + received: Counter, + link_died: Counter, +} + +struct MessagesMetrics { + _meter: Meter, + sent: Counter, + outstanding: UpDownCounter, +} + +struct LinksMetrics { + _meter: Meter, + alive: UpDownCounter, +} + +struct DataMessagesMetrics { + _meter: Meter, + sent: Counter, + resources_count: Histogram, + size: Histogram, } +struct EnvironmentMetrics { + _meter: Meter, + count: UpDownCounter, + process_count: UpDownCounter, +} + +static SIGNALS_METRICS: OnceCell = OnceCell::new(); +static MESSAGES_METRICS: OnceCell = OnceCell::new(); +static LINKS_METRICS: OnceCell = OnceCell::new(); +static DATA_MESSAGES_METRICS: OnceCell = OnceCell::new(); +static ENVIRONMENT_METRICS: OnceCell = OnceCell::new(); + /// The `Process` is the main abstraction in lunatic. /// /// It usually represents some code that is being executed (Wasm instance or V8 isolate), but it @@ -195,21 +168,22 @@ impl Process for WasmProcess { } fn send(&self, signal: Signal) { - #[cfg(all(feature = "metrics", not(feature = "detailed_metrics")))] - let labels = [("process_kind", "wasm")]; - #[cfg(all(feature = "metrics", feature = "detailed_metrics"))] - let labels = [ - ("process_kind", "wasm"), - ("process_id", self.id().to_string()), - ]; - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.signals.send", &labels); - // If the receiver doesn't exist or is closed, just ignore it and drop the `signal`. // lunatic can't guarantee that a message was successfully seen by the receiving side even // if this call succeeds. We deliberately don't expose this API, as it would not make sense // to relay on it and could signal wrong guarantees to users. let _ = self.signal_mailbox.send(signal); + + SIGNALS_METRICS.with_current_context(|metrics, cx| { + metrics.sent.add( + &cx, + 1, + &[ + KeyValue::new("process_kind", "wasm"), + KeyValue::new("process_id", self.id() as i64), + ], + ) + }); } } @@ -295,47 +269,46 @@ where // Currently a panic would just kill the task, but not notify linked processes. let mut signal_mailbox = signal_mailbox.lock().await; let mut has_sender = true; - #[cfg(all(feature = "metrics", not(feature = "detailed_metrics")))] - let labels: [(String, String); 0] = []; - #[cfg(all(feature = "metrics", feature = "detailed_metrics"))] - let labels = [("process_id", id.to_string())]; let result = loop { tokio::select! { biased; // Handle signals first signal = signal_mailbox.recv(), if has_sender => { - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.signals.received", &labels); + SIGNALS_METRICS.with_current_context(|metrics, cx| { + metrics.received.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) + }); match signal.ok_or(()) { Ok(Signal::Message(message)) => { - #[cfg(feature = "metrics")] message.write_metrics(); message_mailbox.push(message); // process metrics - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.messages.send", &labels); - - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.messages.outstanding", message_mailbox.len() as f64, &labels); + MESSAGES_METRICS.with_current_context(|metrics, cx| { + let attrs= [KeyValue::new("process_id", id as i64)]; + metrics.sent.add(&cx, 1, &attrs); + metrics.outstanding.add(&cx, message_mailbox.len() as i64, &attrs); + }); }, Ok(Signal::DieWhenLinkDies(value)) => die_when_link_dies = value, // Put process into list of linked processes Ok(Signal::Link(tag, proc)) => { links.insert(proc.id(), (proc, tag)); - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.links.alive", links.len() as f64, &labels); + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) + }); + }, // Remove process from list Ok(Signal::UnLink { process_id }) => { links.remove(&process_id); - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.links.alive", links.len() as f64, &labels); + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) + }); } // Exit loop and don't poll anymore the future if Signal::Kill received. Ok(Signal::Kill) => break Finished::KillSignal, @@ -344,8 +317,10 @@ where Ok(Signal::LinkDied(id, tag, reason)) => { links.remove(&id); - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.links.alive", links.len() as f64, &labels); + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) + }); + match reason { DeathReason::Failure | DeathReason::NoProcess => { if die_when_link_dies { @@ -355,11 +330,10 @@ where } else { let message = Message::LinkDied(tag); - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.messages.send", &labels); + MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.outstanding.add(&cx, -(message_mailbox.len() as i64), &[KeyValue::new("process_id", id as i64)]); + }); - #[cfg(feature = "metrics")] - metrics::gauge!("lunatic.process.messages.outstanding", message_mailbox.len() as f64, &labels); message_mailbox.push(message); } }, @@ -471,21 +445,22 @@ impl Process for NativeProcess { } fn send(&self, signal: Signal) { - #[cfg(all(feature = "metrics", not(feature = "detailed_metrics")))] - let labels = [("process_kind", "native")]; - #[cfg(all(feature = "metrics", feature = "detailed_metrics"))] - let labels = [ - ("process_kind", "native"), - ("process_id", self.id().to_string()), - ]; - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.process.signals.send", &labels); - // If the receiver doesn't exist or is closed, just ignore it and drop the `signal`. // lunatic can't guarantee that a message was successfully seen by the receiving side even // if this call succeeds. We deliberately don't expose this API, as it would not make sense // to relay on it and could signal wrong guarantees to users. let _ = self.signal_mailbox.send(signal); + + SIGNALS_METRICS.with_current_context(|metrics, cx| { + metrics.sent.add( + &cx, + 1, + &[ + KeyValue::new("process_kind", "native"), + KeyValue::new("process_id", self.id() as i64), + ], + ); + }); } } @@ -543,3 +518,117 @@ pub enum ResultValue { SpawnError(String), Killed, } +pub fn init_metrics() { + SIGNALS_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.signals"); + + let sent = meter + .u64_counter("sent") + .with_unit(Unit::new("count")) + .with_description("Number of signals sent to processes since startup") + .init(); + let received = meter + .u64_counter("received") + .with_unit(Unit::new("count")) + .with_description("Number of received by processes since startup") + .init(); + let link_died = meter + .u64_counter("link_died") + .with_unit(Unit::new("count")) + .with_description("Number of LinkDied messages send since startup") + .init(); + + SignalsMetrics { + _meter: meter, + sent, + received, + link_died, + } + }); + + MESSAGES_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.messages"); + + let sent = meter + .u64_counter("sent") + .with_unit(Unit::new("count")) + .with_description("Number of messages sent to processes since startup") + .init(); + let outstanding = meter + .i64_up_down_counter("outstanding") + .with_unit(Unit::new("count")) + .with_description( + "Current number of messages that are ready to be consumed by processes", + ) + .init(); + + MessagesMetrics { + _meter: meter, + sent, + outstanding, + } + }); + + LINKS_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.links"); + + let alive = meter + .i64_up_down_counter("alive") + .with_unit(Unit::new("count")) + .with_description("Number of links currently alive") + .init(); + + LinksMetrics { + _meter: meter, + alive, + } + }); + + DATA_MESSAGES_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.process.messages.data"); + + let sent = meter + .u64_counter("sent") + .with_unit(Unit::new("count")) + .with_description("Number of data messages sent since startup") + .init(); + let resources_count = meter + .u64_histogram("resources_count") + .with_unit(Unit::new("count")) + .with_description("Number of resources used by each individual message") + .init(); + let size = meter + .u64_histogram("size") + .with_unit(Unit::new("bytes")) + .with_description("Number of bytes used by each individual process") + .init(); + + DataMessagesMetrics { + _meter: meter, + sent, + resources_count, + size, + } + }); + + ENVIRONMENT_METRICS.get_or_init(|| { + let meter = global::meter("lunatic.environment"); + + let count = meter + .i64_up_down_counter("count") + .with_unit(Unit::new("count")) + .with_description("Number of currently active environment") + .init(); + let process_count = meter + .i64_up_down_counter("process_count") + .with_unit(Unit::new("count")) + .with_description("Number of currently registered processes") + .init(); + + EnvironmentMetrics { + _meter: meter, + count, + process_count, + } + }); +} diff --git a/crates/lunatic-process/src/message.rs b/crates/lunatic-process/src/message.rs index 0b6868040..149817604 100644 --- a/crates/lunatic-process/src/message.rs +++ b/crates/lunatic-process/src/message.rs @@ -11,8 +11,11 @@ use std::{ sync::Arc, }; +use lunatic_common_api::MetricsExt; use smallvec::SmallVec; +use crate::{DATA_MESSAGES_METRICS, SIGNALS_METRICS}; + pub type Resource = dyn Any + Send + Sync; /// Can be sent between processes by being embedded into a [`Signal::Message`][0] @@ -36,12 +39,13 @@ impl Message { } } - #[cfg(feature = "metrics")] pub fn write_metrics(&self) { match self { Message::Data(message) => message.write_metrics(), Message::LinkDied(_) => { - metrics::increment_counter!("lunatic.process.messages.link_died.count"); + SIGNALS_METRICS.with_current_context(|metrics, cx| { + metrics.link_died.add(&cx, 1, &[]); + }); } } } @@ -114,14 +118,14 @@ impl DataMessage { self.buffer.len() } - #[cfg(feature = "metrics")] pub fn write_metrics(&self) { - metrics::increment_counter!("lunatic.process.messages.data.count"); - metrics::histogram!( - "lunatic.process.messages.data.resources.count", - self.resources.len() as f64 - ); - metrics::histogram!("lunatic.process.messages.data.size", self.size() as f64); + DATA_MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.sent.add(&cx, 1, &[]); + metrics + .resources_count + .record(&cx, self.resources.len() as u64, &[]); + metrics.size.record(&cx, self.size() as u64, &[]); + }); } } diff --git a/crates/lunatic-registry-api/Cargo.toml b/crates/lunatic-registry-api/Cargo.toml index 3befbb1a2..770eb3dc8 100644 --- a/crates/lunatic-registry-api/Cargo.toml +++ b/crates/lunatic-registry-api/Cargo.toml @@ -7,15 +7,13 @@ homepage = "https://lunatic.solutions" repository = "https://github.com/lunatic-solutions/lunatic/tree/main/crates" license = "Apache-2.0/MIT" -[features] -metrics = ["dep:metrics"] - [dependencies] lunatic-common-api = { workspace = true } lunatic-process = { workspace = true } lunatic-process-api = { workspace = true } -tokio = { workspace = true, features = ["sync"] } anyhow = { workspace = true } -metrics = { workspace = true, optional = true } +once_cell = { workspace = true } +opentelemetry = { workspace = true } +tokio = { workspace = true, features = ["sync"] } wasmtime = { workspace = true } diff --git a/crates/lunatic-registry-api/src/lib.rs b/crates/lunatic-registry-api/src/lib.rs index 55993561f..34416a9b4 100644 --- a/crates/lunatic-registry-api/src/lib.rs +++ b/crates/lunatic-registry-api/src/lib.rs @@ -1,46 +1,65 @@ use std::{collections::HashMap, future::Future, mem::transmute}; use anyhow::{anyhow, Result}; -use lunatic_common_api::{get_memory, IntoTrap}; +use lunatic_common_api::{get_memory, IntoTrap, MetricsExt}; use lunatic_process::state::ProcessState; use lunatic_process_api::ProcessCtx; +use once_cell::sync::OnceCell; +use opentelemetry::{ + global, + metrics::{Counter, Meter, UpDownCounter}, +}; use tokio::sync::RwLockWriteGuard; use wasmtime::{Caller, Linker}; +struct Metrics { + _meter: Meter, + registered: UpDownCounter, + read: Counter, + write: Counter, + deletion: Counter, +} + +static METRICS: OnceCell = OnceCell::new(); + // Register the registry APIs to the linker pub fn register + Send + Sync + 'static>( linker: &mut Linker, ) -> Result<()> { + METRICS.get_or_init(|| { + let meter = global::meter("lunatic.registry"); + + let registered = meter + .i64_up_down_counter("registered") + .with_description("Number or processes currently registered") + .init(); + let read = meter + .u64_counter("read") + .with_description("Number of entries read from the registry") + .init(); + let write = meter + .u64_counter("write") + .with_description("Number of entries written to the registry") + .init(); + let deletion = meter + .u64_counter("deletion") + .with_description("Number of entries deleted from the registry") + .init(); + + Metrics { + _meter: meter, + registered, + read, + write, + deletion, + } + }); + linker.func_wrap4_async("lunatic::registry", "put", put)?; linker.func_wrap4_async("lunatic::registry", "get", get)?; linker.func_wrap4_async("lunatic::registry", "get_or_put_later", get_or_put_later)?; linker.func_wrap2_async("lunatic::registry", "remove", remove)?; - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.registry.write", - metrics::Unit::Count, - "number of new entries written to the registry" - ); - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.timers.read", - metrics::Unit::Count, - "number of entries read from the registry" - ); - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.timers.deletion", - metrics::Unit::Count, - "number of entries deleted from the registry" - ); - #[cfg(feature = "metrics")] - metrics::describe_gauge!( - "lunatic.timers.registered", - metrics::Unit::Count, - "number of processes currently registered" - ); - Ok(()) } @@ -79,11 +98,10 @@ fn put + Send + Sync>( } } - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.registry.write"); - - #[cfg(feature = "metrics")] - metrics::increment_gauge!("lunatic.registry.registered", 1.0); + METRICS.with_current_context(|metrics, cx| { + metrics.registered.add(&cx, 1, &[]); + metrics.write.add(&cx, 1, &[]); + }); Ok(()) }) @@ -115,8 +133,9 @@ fn get + Send + Sync>( )); } - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.registry.read"); + METRICS.with_current_context(|metrics, cx| { + metrics.read.add(&cx, 1, &[]); + }); let (node_id, process_id) = if let Some(process) = state.registry().read().await.get(name) { *process @@ -168,8 +187,9 @@ fn get_or_put_later + Send + Sync>( .or_trap("lunatic::registry::get")?; let name = std::str::from_utf8(name).or_trap("lunatic::registry::get")?; - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.registry.read"); + METRICS.with_current_context(|metrics, cx| { + metrics.read.add(&cx, 1, &[]); + }); // Lock the registry for every other process before lookup, to make sure // nobody else can insert anything before us. @@ -230,11 +250,10 @@ fn remove + Send + Sync>( state.registry().write().await.remove(name); - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.registry.deletion"); - - #[cfg(feature = "metrics")] - metrics::decrement_gauge!("lunatic.registry.registered", 1.0); + METRICS.with_current_context(|metrics, cx| { + metrics.deletion.add(&cx, 1, &[]); + metrics.registered.add(&cx, -1, &[]); + }); Ok(()) }) diff --git a/crates/lunatic-timer-api/Cargo.toml b/crates/lunatic-timer-api/Cargo.toml index 1afe85034..877850a5f 100644 --- a/crates/lunatic-timer-api/Cargo.toml +++ b/crates/lunatic-timer-api/Cargo.toml @@ -7,9 +7,6 @@ homepage = "https://lunatic.solutions" repository = "https://github.com/lunatic-solutions/lunatic/tree/main/crates/lunatic-timer-api" license = "Apache-2.0/MIT" -[features] -metrics = ["dep:metrics"] - [dependencies] hash-map-id = { workspace = true } lunatic-common-api = { workspace = true } @@ -17,6 +14,7 @@ lunatic-process = { workspace = true } lunatic-process-api = { workspace = true } anyhow = { workspace = true } -metrics = { workspace = true, optional = true } +once_cell = { workspace = true } +opentelemetry = { workspace = true } tokio = { workspace = true, features = ["time", "rt"] } wasmtime = { workspace = true } diff --git a/crates/lunatic-timer-api/src/lib.rs b/crates/lunatic-timer-api/src/lib.rs index b43795fa5..d1e1f2233 100644 --- a/crates/lunatic-timer-api/src/lib.rs +++ b/crates/lunatic-timer-api/src/lib.rs @@ -7,9 +7,14 @@ use std::{ use anyhow::Result; use hash_map_id::HashMapId; -use lunatic_common_api::IntoTrap; +use lunatic_common_api::{IntoTrap, MetricsExt}; use lunatic_process::{state::ProcessState, Signal}; use lunatic_process_api::ProcessCtx; +use once_cell::sync::OnceCell; +use opentelemetry::{ + global, + metrics::{Counter, Meter, Unit, UpDownCounter}, +}; use tokio::task::JoinHandle; use wasmtime::{Caller, Linker}; @@ -84,37 +89,55 @@ pub trait TimerCtx { fn timer_resources_mut(&mut self) -> &mut TimerResources; } +struct Metrics { + _meter: Meter, + started: Counter, + completed: Counter, + canceled: Counter, + active: UpDownCounter, +} + +static METRICS: OnceCell = OnceCell::new(); + pub fn register + TimerCtx + Send + 'static>( linker: &mut Linker, ) -> Result<()> { + METRICS.get_or_init(|| { + let meter = global::meter("lunatic.timers"); + + let started = meter + .u64_counter("started") + .with_unit(Unit::new("count")) + .with_description("Number of timers set since startup") + .init(); + let completed = meter + .u64_counter("completed") + .with_unit(Unit::new("count")) + .with_description("Number of timners completed since startup") + .init(); + let canceled = meter + .u64_counter("canceled") + .with_unit(Unit::new("count")) + .with_description("Number of timers cancelled since startup") + .init(); + let active = meter + .i64_up_down_counter("active") + .with_unit(Unit::new("count")) + .with_description("Number of timers currently active") + .init(); + + Metrics { + _meter: meter, + started, + completed, + canceled, + active, + } + }); + linker.func_wrap("lunatic::timer", "send_after", send_after)?; linker.func_wrap1_async("lunatic::timer", "cancel_timer", cancel_timer)?; - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.timers.started", - metrics::Unit::Count, - "number of timers set since startup, will usually be completed + canceled + active" - ); - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.timers.completed", - metrics::Unit::Count, - "number of timers completed since startup" - ); - #[cfg(feature = "metrics")] - metrics::describe_counter!( - "lunatic.timers.canceled", - metrics::Unit::Count, - "number of timers canceled since startup" - ); - #[cfg(feature = "metrics")] - metrics::describe_gauge!( - "lunatic.timers.active", - metrics::Unit::Count, - "number of timers currently active" - ); - Ok(()) } @@ -140,20 +163,22 @@ fn send_after + TimerCtx>( let target_time = Instant::now() + Duration::from_millis(delay); let timer_handle = tokio::task::spawn(async move { - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.timers.started"); - #[cfg(feature = "metrics")] - metrics::increment_gauge!("lunatic.timers.active", 1.0); + METRICS.with_current_context(|metrics, cx| { + metrics.started.add(&cx, 1, &[]); + metrics.active.add(&cx, 1, &[]); + }); + let duration_remaining = target_time - Instant::now(); if duration_remaining != Duration::ZERO { tokio::time::sleep(duration_remaining).await; } if let Some(process) = process { - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.timers.completed"); - #[cfg(feature = "metrics")] - metrics::decrement_gauge!("lunatic.timers.active", 1.0); process.send(Signal::Message(message)); + + METRICS.with_current_context(|metrics, cx| { + metrics.completed.add(&cx, 1, &[]); + metrics.active.add(&cx, -1, &[]); + }); } }); @@ -181,10 +206,12 @@ fn cancel_timer( match timer_handle { Some(timer_handle) => { timer_handle.abort(); - #[cfg(feature = "metrics")] - metrics::increment_counter!("lunatic.timers.canceled"); - #[cfg(feature = "metrics")] - metrics::decrement_gauge!("lunatic.timers.active", 1.0); + + METRICS.with_current_context(|metrics, cx| { + metrics.canceled.add(&cx, 1, &[]); + metrics.active.add(&cx, -1, &[]); + }); + Ok(1) } None => Ok(0), diff --git a/src/mode/common.rs b/src/mode/common.rs index 1a8007662..a2c62c8bc 100644 --- a/src/mode/common.rs +++ b/src/mode/common.rs @@ -18,7 +18,7 @@ use lunatic_process::{ use lunatic_process_api::ProcessConfigCtx; use lunatic_runtime::{DefaultProcessConfig, DefaultProcessState}; use opentelemetry::{ - global::{BoxedTracer, GlobalMeterProvider}, + global::{self, BoxedTracer, GlobalMeterProvider}, metrics::MetricsError, sdk::metrics::MeterProvider, trace::{Span, TraceContextExt, Tracer}, @@ -149,6 +149,7 @@ pub fn prometheus(addr: &SocketAddr, _node_id: Option) -> Result, diff --git a/src/state.rs b/src/state.rs index 20cdfe19c..7b3c22913 100644 --- a/src/state.rs +++ b/src/state.rs @@ -85,7 +85,6 @@ pub struct DefaultProcessState { // is a safe operation, because it references a global registry that outlives all processes. registry_atomic_put: Option>>, // Metrics - // TODO: Does this need to be in an Arc? tracer: Arc, tracer_context: Arc, process_context: Context, @@ -200,7 +199,6 @@ impl ProcessState for DefaultProcessState { lunatic_registry_api::register(linker)?; lunatic_distributed_api::register(linker)?; lunatic_sqlite_api::register(linker)?; - #[cfg(feature = "metrics")] lunatic_metrics_api::register(linker)?; lunatic_trap_api::register(linker)?; Ok(()) From edbd303458a4d5a43938d8b09b802b8449e00c8d Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 18:39:25 +0930 Subject: [PATCH 07/16] fix: metrics issues --- crates/lunatic-messaging-api/src/lib.rs | 11 ++++- crates/lunatic-process/src/env.rs | 10 ++-- crates/lunatic-process/src/lib.rs | 62 ++++++++++++++----------- crates/lunatic-registry-api/src/lib.rs | 16 ++++--- 4 files changed, 59 insertions(+), 40 deletions(-) diff --git a/crates/lunatic-messaging-api/src/lib.rs b/crates/lunatic-messaging-api/src/lib.rs index 42900ca6e..2781c7d2d 100644 --- a/crates/lunatic-messaging-api/src/lib.rs +++ b/crates/lunatic-messaging-api/src/lib.rs @@ -14,7 +14,7 @@ use wasmtime::{Caller, Linker}; use lunatic_process::{ message::{DataMessage, Message}, state::ProcessState, - Signal, + Signal, MESSAGES_METRICS, }; // Register the mailbox APIs to the linker @@ -544,6 +544,15 @@ fn receive + Send>( }; let pop = caller.data_mut().mailbox().pop(tags.as_deref()); + + MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.outstanding.add( + &cx, + -1, + &[KeyValue::new("process_id", caller.data().id() as i64)], + ); + }); + if let Ok(message) = match timeout_duration { // Without timeout u64::MAX => Ok(pop.await), diff --git a/crates/lunatic-process/src/env.rs b/crates/lunatic-process/src/env.rs index 431626bd1..f8ab598b3 100644 --- a/crates/lunatic-process/src/env.rs +++ b/crates/lunatic-process/src/env.rs @@ -59,11 +59,9 @@ impl Environment for LunaticEnvironment { self.processes.insert(id, proc); ENVIRONMENT_METRICS.with_current_context(|metrics, cx| { - metrics.process_count.add( - &cx, - self.processes.len() as i64, - &[KeyValue::new("environment_id", self.id() as i64)], - ); + metrics + .process_count + .add(&cx, 1, &[KeyValue::new("environment_id", self.id() as i64)]); }); } @@ -73,7 +71,7 @@ impl Environment for LunaticEnvironment { ENVIRONMENT_METRICS.with_current_context(|metrics, cx| { metrics.process_count.add( &cx, - -(self.processes.len() as i64), + -1, &[KeyValue::new("environment_id", self.id() as i64)], ); }); diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index 4b5aad65e..92e5fa977 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -31,42 +31,42 @@ use tokio::{ use crate::{mailbox::MessageMailbox, message::Message}; -struct SignalsMetrics { +pub struct SignalsMetrics { _meter: Meter, - sent: Counter, - received: Counter, - link_died: Counter, + pub sent: Counter, + pub received: Counter, + pub link_died: Counter, } -struct MessagesMetrics { +pub struct MessagesMetrics { _meter: Meter, - sent: Counter, - outstanding: UpDownCounter, + pub sent: Counter, + pub outstanding: UpDownCounter, } -struct LinksMetrics { +pub struct LinksMetrics { _meter: Meter, - alive: UpDownCounter, + pub alive: UpDownCounter, } -struct DataMessagesMetrics { +pub struct DataMessagesMetrics { _meter: Meter, - sent: Counter, - resources_count: Histogram, - size: Histogram, + pub sent: Counter, + pub resources_count: Histogram, + pub size: Histogram, } -struct EnvironmentMetrics { +pub struct EnvironmentMetrics { _meter: Meter, - count: UpDownCounter, - process_count: UpDownCounter, + pub count: UpDownCounter, + pub process_count: UpDownCounter, } -static SIGNALS_METRICS: OnceCell = OnceCell::new(); -static MESSAGES_METRICS: OnceCell = OnceCell::new(); -static LINKS_METRICS: OnceCell = OnceCell::new(); -static DATA_MESSAGES_METRICS: OnceCell = OnceCell::new(); -static ENVIRONMENT_METRICS: OnceCell = OnceCell::new(); +pub static SIGNALS_METRICS: OnceCell = OnceCell::new(); +pub static MESSAGES_METRICS: OnceCell = OnceCell::new(); +pub static LINKS_METRICS: OnceCell = OnceCell::new(); +pub static DATA_MESSAGES_METRICS: OnceCell = OnceCell::new(); +pub static ENVIRONMENT_METRICS: OnceCell = OnceCell::new(); /// The `Process` is the main abstraction in lunatic. /// @@ -287,9 +287,9 @@ where // process metrics MESSAGES_METRICS.with_current_context(|metrics, cx| { - let attrs= [KeyValue::new("process_id", id as i64)]; + let attrs = [KeyValue::new("process_id", id as i64)]; metrics.sent.add(&cx, 1, &attrs); - metrics.outstanding.add(&cx, message_mailbox.len() as i64, &attrs); + metrics.outstanding.add(&cx, 1, &attrs); }); }, Ok(Signal::DieWhenLinkDies(value)) => die_when_link_dies = value, @@ -329,12 +329,12 @@ where break Finished::KillSignal } else { let message = Message::LinkDied(tag); + message_mailbox.push(message); MESSAGES_METRICS.with_current_context(|metrics, cx| { - metrics.outstanding.add(&cx, -(message_mailbox.len() as i64), &[KeyValue::new("process_id", id as i64)]); + metrics.outstanding.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]); }); - message_mailbox.push(message); } }, // In case a linked process finishes normally, don't do anything. @@ -352,6 +352,14 @@ where } }; + MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.outstanding.add( + &cx, + -(message_mailbox.len() as i64), + &[KeyValue::new("process_id", id as i64)], + ); + }); + let result = match result { Finished::Normal(result) => { let result: ExecutionResult<_> = result.into(); @@ -530,7 +538,7 @@ pub fn init_metrics() { let received = meter .u64_counter("received") .with_unit(Unit::new("count")) - .with_description("Number of received by processes since startup") + .with_description("Number of signals received by processes since startup") .init(); let link_died = meter .u64_counter("link_died") @@ -617,7 +625,7 @@ pub fn init_metrics() { let count = meter .i64_up_down_counter("count") .with_unit(Unit::new("count")) - .with_description("Number of currently active environment") + .with_description("Number of currently active environments") .init(); let process_count = meter .i64_up_down_counter("process_count") diff --git a/crates/lunatic-registry-api/src/lib.rs b/crates/lunatic-registry-api/src/lib.rs index 34416a9b4..23a4c085b 100644 --- a/crates/lunatic-registry-api/src/lib.rs +++ b/crates/lunatic-registry-api/src/lib.rs @@ -7,7 +7,7 @@ use lunatic_process_api::ProcessCtx; use once_cell::sync::OnceCell; use opentelemetry::{ global, - metrics::{Counter, Meter, UpDownCounter}, + metrics::{Counter, Meter, Unit, UpDownCounter}, }; use tokio::sync::RwLockWriteGuard; use wasmtime::{Caller, Linker}; @@ -17,7 +17,7 @@ struct Metrics { registered: UpDownCounter, read: Counter, write: Counter, - deletion: Counter, + delete: Counter, } static METRICS: OnceCell = OnceCell::new(); @@ -31,18 +31,22 @@ pub fn register + Send + Sync + 'static>( let registered = meter .i64_up_down_counter("registered") + .with_unit(Unit::new("count")) .with_description("Number or processes currently registered") .init(); let read = meter .u64_counter("read") + .with_unit(Unit::new("count")) .with_description("Number of entries read from the registry") .init(); let write = meter .u64_counter("write") + .with_unit(Unit::new("count")) .with_description("Number of entries written to the registry") .init(); - let deletion = meter - .u64_counter("deletion") + let delete = meter + .u64_counter("delete") + .with_unit(Unit::new("count")) .with_description("Number of entries deleted from the registry") .init(); @@ -51,7 +55,7 @@ pub fn register + Send + Sync + 'static>( registered, read, write, - deletion, + delete, } }); @@ -251,7 +255,7 @@ fn remove + Send + Sync>( state.registry().write().await.remove(name); METRICS.with_current_context(|metrics, cx| { - metrics.deletion.add(&cx, 1, &[]); + metrics.delete.add(&cx, 1, &[]); metrics.registered.add(&cx, -1, &[]); }); From 2248f8f9e8aa0e0eefe2ad8d2c2ee8aa91302e03 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 18:39:34 +0930 Subject: [PATCH 08/16] docs: add codedocs to metrics host functions --- crates/lunatic-metrics-api/src/lib.rs | 104 ++++++++++++-------------- 1 file changed, 49 insertions(+), 55 deletions(-) diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 3fc37f689..bf85ec70c 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -48,7 +48,7 @@ pub trait MetricsCtx { fn drop_histogram(&mut self, id: u64) -> Option>; } -/// Links the [Metrics](https://crates.io/crates/metrics) APIs +/// Links the [Metrics](https://crates.io/crates/metrics) APIs. pub fn register(linker: &mut Linker) -> anyhow::Result<()> where T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync + 'static, @@ -85,10 +85,9 @@ where Ok(()) } -/// Starts a new span of work, used for recording metrics including log events and meters. +/// Starts a new span of work, used for recording metrics including log events and meters such as counters, gauges, and histograms. /// -/// `parent` is the ID of another span. If it is set to u64::MAX, then the last created span -/// will be used. +/// If parent is set to u64::MAX, then the last created span will be used. /// /// Traps: /// * If the name is not a valid utf8 string. @@ -195,8 +194,7 @@ where /// Adds a log event in span, containing a name and optional attributes. /// -/// `span` is the ID of the parent span. If it is set to u64::MAX, then the last created span -/// will be used. +/// If span is set to u64::MAX, then the last created span will be used. /// /// The following attributes are optional, and used for logging to the terminal: /// * `target`: a string describing the part of the system where the span or event that this @@ -315,10 +313,13 @@ where Ok(()) } -/// Sets a counter. +/// Creates a counter with a given name, and optional description and unit. /// /// Traps: -/// * If the name is not a valid utf8 string. +/// * If the name is not a valid utf8 string, or contains invalid characters. +/// * If the description is not a valid utf8 string. +/// * If the unit is not a valid utf8 string, or exceeds 63 characters. +/// * If the meter does not exist. /// * If any memory outside the guest heap space is referenced. fn counter( mut caller: Caller<'_, T>, @@ -353,8 +354,12 @@ where /// Increments a counter. /// +/// If span is set to u64::MAX, then the last created span will be used. +/// /// Traps: -/// * If the name is not a valid utf8 string. +/// * If the span does not exist. +/// * If the counter does not exist. +/// * If the attributes is not valid json. /// * If any memory outside the guest heap space is referenced. fn counter_add( mut caller: Caller<'_, T>, @@ -392,6 +397,14 @@ where Ok(()) } +/// Creates an up/down counter with a given name, and optional description and unit. +/// +/// Traps: +/// * If the name is not a valid utf8 string, or contains invalid characters. +/// * If the description is not a valid utf8 string. +/// * If the unit is not a valid utf8 string, or exceeds 63 characters. +/// * If the meter does not exist. +/// * If any memory outside the guest heap space is referenced. fn up_down_counter( mut caller: Caller<'_, T>, meter: u64, @@ -423,6 +436,15 @@ where Ok(id) } +/// Increments an up/down counter. The amount can be negative to decrement. +/// +/// If span is set to u64::MAX, then the last created span will be used. +/// +/// Traps: +/// * If the span does not exist. +/// * If the counter does not exist. +/// * If the attributes is not valid json. +/// * If any memory outside the guest heap space is referenced. fn up_down_counter_add( mut caller: Caller<'_, T>, span: u64, @@ -446,6 +468,7 @@ where Ok(()) } +/// Drops an up/down counter. fn up_down_counter_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where T: MetricsCtx, @@ -458,54 +481,13 @@ where Ok(()) } -// /// Increments a gauge. -// /// -// /// Traps: -// /// * If the name is not a valid utf8 string. -// /// * If any memory outside the guest heap space is referenced. -// fn increment_gauge( -// mut caller: Caller<'_, T>, -// name_ptr: u32, -// name_len: u32, -// value: f64, -// ) -> Result<()> { -// todo!() -// // let memory = get_memory(&mut caller)?; -// // let data = memory.data(&mut caller); -// // -// // let name = -// // get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::increment_gauge")?; -// // -// // increment_gauge!(name, value); -// // Ok(()) -// } -// -// /// Decrements a gauge. -// /// -// /// Traps: -// /// * If the name is not a valid utf8 string. -// /// * If any memory outside the guest heap space is referenced. -// fn decrement_gauge( -// mut caller: Caller<'_, T>, -// name_ptr: u32, -// name_len: u32, -// value: f64, -// ) -> Result<()> { -// todo!() -// // let memory = get_memory(&mut caller)?; -// // let data = memory.data(&mut caller); -// // -// // let name = -// // get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::decrement_gauge")?; -// // -// // decrement_gauge!(name, value); -// // Ok(()) -// } - -/// Sets a histogram. +/// Creates a histogram with a given name, and optional description and unit. /// /// Traps: -/// * If the name is not a valid utf8 string. +/// * If the name is not a valid utf8 string, or contains invalid characters. +/// * If the description is not a valid utf8 string. +/// * If the unit is not a valid utf8 string, or exceeds 63 characters. +/// * If the meter does not exist. /// * If any memory outside the guest heap space is referenced. fn histogram( mut caller: Caller<'_, T>, @@ -538,6 +520,15 @@ where Ok(id) } +/// Records a value to a histogram. +/// +/// If span is set to u64::MAX, then the last created span will be used. +/// +/// Traps: +/// * If the span does not exist. +/// * If the histogram does not exist. +/// * If the attributes is not valid json. +/// * If any memory outside the guest heap space is referenced. fn histogram_record( mut caller: Caller<'_, T>, span: u64, @@ -561,6 +552,7 @@ where Ok(()) } +/// Drops a histogram. fn histogram_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where T: MetricsCtx, @@ -573,6 +565,8 @@ where Ok(()) } +// === Helper functions === + fn get_string_arg(data: &[u8], name_ptr: u32, name_len: u32) -> Result { if name_len == 0 { return Ok(String::new()); From fb78e96847d161ef723cbdd8e1c3859d7302adac Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 18:51:19 +0930 Subject: [PATCH 09/16] refactor: remove trait bounds from metrics functions --- Cargo.lock | 2 +- crates/lunatic-messaging-api/Cargo.toml | 1 + crates/lunatic-messaging-api/src/lib.rs | 20 ++++++++++---------- crates/lunatic-metrics-api/Cargo.toml | 6 +++--- crates/lunatic-metrics-api/src/lib.rs | 11 ++++------- 5 files changed, 19 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7b392410c..cc4ad5674 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1577,6 +1577,7 @@ dependencies = [ "lunatic-networking-api", "lunatic-process", "lunatic-process-api", + "opentelemetry", "tokio", "wasmtime", ] @@ -1590,7 +1591,6 @@ dependencies = [ "log", "lunatic-common-api", "lunatic-process", - "lunatic-process-api", "opentelemetry", "serde_json", "wasmtime", diff --git a/crates/lunatic-messaging-api/Cargo.toml b/crates/lunatic-messaging-api/Cargo.toml index 9cb797fed..9824bf9e3 100644 --- a/crates/lunatic-messaging-api/Cargo.toml +++ b/crates/lunatic-messaging-api/Cargo.toml @@ -14,5 +14,6 @@ lunatic-process = { workspace = true } lunatic-process-api = { workspace = true } anyhow = { workspace = true } +opentelemetry = { workspace = true } tokio = { workspace = true, features = ["time"] } wasmtime = { workspace = true } diff --git a/crates/lunatic-messaging-api/src/lib.rs b/crates/lunatic-messaging-api/src/lib.rs index 2781c7d2d..af0493315 100644 --- a/crates/lunatic-messaging-api/src/lib.rs +++ b/crates/lunatic-messaging-api/src/lib.rs @@ -5,17 +5,17 @@ use std::{ }; use anyhow::{anyhow, Result}; -use lunatic_common_api::{get_memory, IntoTrap}; +use lunatic_common_api::{get_memory, IntoTrap, MetricsExt}; use lunatic_networking_api::NetworkingCtx; -use lunatic_process_api::ProcessCtx; -use tokio::time::{timeout, Duration}; -use wasmtime::{Caller, Linker}; - use lunatic_process::{ message::{DataMessage, Message}, state::ProcessState, Signal, MESSAGES_METRICS, }; +use lunatic_process_api::ProcessCtx; +use opentelemetry::KeyValue; +use tokio::time::{timeout, Duration}; +use wasmtime::{Caller, Linker}; // Register the mailbox APIs to the linker pub fn register + NetworkingCtx + Send + 'static>( @@ -526,6 +526,8 @@ fn receive + Send>( timeout_duration: u64, ) -> Box> + Send + '_> { Box::new(async move { + let id = caller.data().id(); + let tags = if tag_len > 0 { let memory = get_memory(&mut caller)?; let buffer = memory @@ -546,11 +548,9 @@ fn receive + Send>( let pop = caller.data_mut().mailbox().pop(tags.as_deref()); MESSAGES_METRICS.with_current_context(|metrics, cx| { - metrics.outstanding.add( - &cx, - -1, - &[KeyValue::new("process_id", caller.data().id() as i64)], - ); + metrics + .outstanding + .add(&cx, -1, &[KeyValue::new("process_id", id as i64)]); }); if let Ok(message) = match timeout_duration { diff --git a/crates/lunatic-metrics-api/Cargo.toml b/crates/lunatic-metrics-api/Cargo.toml index be0e1af80..71353fd19 100644 --- a/crates/lunatic-metrics-api/Cargo.toml +++ b/crates/lunatic-metrics-api/Cargo.toml @@ -8,12 +8,12 @@ repository = "https://github.com/lunatic-solutions/lunatic/tree/main/crates/luna license = "Apache-2.0/MIT" [dependencies] -anyhow = { workspace = true } hash-map-id = { workspace = true } -log = { workspace = true } lunatic-common-api = { workspace = true } lunatic-process = { workspace = true } -lunatic-process-api = { workspace = true } + +anyhow = { workspace = true } +log = { workspace = true } opentelemetry = { workspace = true } serde_json = "1.0" wasmtime = { workspace = true } diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index bf85ec70c..6ea5e9311 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -3,7 +3,6 @@ use hash_map_id::HashMapId; use log::{Level, Record}; use lunatic_common_api::{get_memory, IntoTrap}; use lunatic_process::state::ProcessState; -use lunatic_process_api::ProcessCtx; use opentelemetry::{ metrics::{ Counter, Histogram, InstrumentBuilder, Meter, MeterProvider, MetricsError, Unit, @@ -51,7 +50,7 @@ pub trait MetricsCtx { /// Links the [Metrics](https://crates.io/crates/metrics) APIs. pub fn register(linker: &mut Linker) -> anyhow::Result<()> where - T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync + 'static, + T: ProcessState + MetricsCtx + 'static, <::Tracer as Tracer>::Span: Send + Sync, { linker.func_wrap("lunatic::metrics", "span_start", span_start)?; @@ -103,7 +102,7 @@ fn span_start( attributes_len: u32, ) -> Result where - T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, + T: MetricsCtx, <::Tracer as Tracer>::Span: Send + Sync + 'static, { let memory = get_memory(&mut caller)?; @@ -148,8 +147,7 @@ where /// Drops a span, marking it as finished. fn span_drop(mut caller: Caller<'_, T>, id: u64) -> Result<()> where - T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, - <::Tracer as Tracer>::Span: Send + Sync, + T: MetricsCtx, { let memory = get_memory(&mut caller)?; let (_data, state) = memory.data_and_store_mut(&mut caller); @@ -227,8 +225,7 @@ fn event( attributes_len: u32, ) -> Result<()> where - T: ProcessState + ProcessCtx + MetricsCtx + Send + Sync, - <::Tracer as Tracer>::Span: Send + Sync, + T: ProcessState + MetricsCtx, { let memory = get_memory(&mut caller)?; let (data, state) = memory.data_and_store_mut(&mut caller); From e7eb0042a1edc4f2c9950b323a2c451e4f667209 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 19:04:25 +0930 Subject: [PATCH 10/16] refactor: fix clippy lints --- crates/lunatic-metrics-api/src/lib.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 6ea5e9311..6ff89667a 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -169,7 +169,7 @@ where let memory = get_memory(&mut caller)?; let (data, state) = memory.data_and_store_mut(&mut caller); - let name = get_string_arg(&data, name_ptr, name_len).or_trap("lunatic::metrics::meter")?; + let name = get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::meter")?; let meter = state.meter_provider().meter(name.into()); let id = state.add_meter(meter); @@ -318,6 +318,7 @@ where /// * If the unit is not a valid utf8 string, or exceeds 63 characters. /// * If the meter does not exist. /// * If any memory outside the guest heap space is referenced. +#[allow(clippy::too_many_arguments)] fn counter( mut caller: Caller<'_, T>, meter: u64, @@ -402,6 +403,7 @@ where /// * If the unit is not a valid utf8 string, or exceeds 63 characters. /// * If the meter does not exist. /// * If any memory outside the guest heap space is referenced. +#[allow(clippy::too_many_arguments)] fn up_down_counter( mut caller: Caller<'_, T>, meter: u64, @@ -486,6 +488,7 @@ where /// * If the unit is not a valid utf8 string, or exceeds 63 characters. /// * If the meter does not exist. /// * If any memory outside the guest heap space is referenced. +#[allow(clippy::too_many_arguments)] fn histogram( mut caller: Caller<'_, T>, meter: u64, @@ -575,6 +578,7 @@ fn get_string_arg(data: &[u8], name_ptr: u32, name_len: u32) -> Result { Ok(name) } +#[allow(clippy::too_many_arguments)] fn create_metric<'a, T, M, F>( caller: &'a mut Caller<'_, T>, meter: u64, @@ -655,7 +659,7 @@ where fn data_to_opentelemetry(data: Map) -> Vec { data.into_iter() .map(|(k, v)| KeyValue { - key: k.to_string().into(), + key: k.into(), value: json_to_opentelemetry(v), }) .collect() From e2a393ef5f628c39779fbb8c96a1a450052ecaaa Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 19:12:13 +0930 Subject: [PATCH 11/16] refactor: fix clippy lint --- src/state.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/state.rs b/src/state.rs index 7b3c22913..cb732097f 100644 --- a/src/state.rs +++ b/src/state.rs @@ -93,6 +93,7 @@ pub struct DefaultProcessState { } impl DefaultProcessState { + #[allow(clippy::too_many_arguments)] pub fn new( environment: Arc, distributed: Option, From c812311454cb5aeef3a4f8920d8845f7cb5d05f1 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 19:25:20 +0930 Subject: [PATCH 12/16] fix: benchmarks --- benches/benchmark.rs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/benches/benchmark.rs b/benches/benchmark.rs index bf536b70f..84108afd7 100644 --- a/benches/benchmark.rs +++ b/benches/benchmark.rs @@ -7,6 +7,12 @@ use lunatic_process::{ runtimes::wasmtime::{default_config, WasmtimeRuntime}, }; use lunatic_runtime::{state::DefaultProcessState, DefaultProcessConfig}; +use opentelemetry::{ + global::{BoxedTracer, GlobalMeterProvider}, + metrics::noop::NoopMeterProvider, + trace::noop::NoopTracer, + Context, +}; use tokio::sync::RwLock; fn criterion_benchmark(c: &mut Criterion) { @@ -23,6 +29,15 @@ fn criterion_benchmark(c: &mut Criterion) { .unwrap(), ); + let tracer = Arc::new(BoxedTracer::new(Box::new(NoopTracer::new()))); + let tracer_context = Arc::new(Context::new()); + let meter_provider = GlobalMeterProvider::new(NoopMeterProvider::new()); + let logger = Arc::new( + env_logger::Builder::new() + .filter_level(log::LevelFilter::Off) + .build(), + ); + let env = Arc::new(LunaticEnvironment::new(0)); c.bench_function("spawn process", |b| { b.to_async(&rt).iter(|| async { @@ -34,6 +49,10 @@ fn criterion_benchmark(c: &mut Criterion) { module.clone(), config.clone(), registry, + tracer.clone(), + tracer_context.clone(), + meter_provider.clone(), + logger.clone(), ) .unwrap(); lunatic_process::wasm::spawn_wasm( From 36a84e2a8561e5c0046eb95b2d953167547f9966 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 22:28:34 +0930 Subject: [PATCH 13/16] fix: metric host functions in all_imports.wat --- wat/all_imports.wat | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/wat/all_imports.wat b/wat/all_imports.wat index ffe99457e..d99e82d19 100644 --- a/wat/all_imports.wat +++ b/wat/all_imports.wat @@ -114,12 +114,20 @@ (import "lunatic::distributed" "send" (func (param i64 i64) (result i32))) (import "lunatic::distributed" "send_receive_skip_search" (func (param i64 i64 i64 i64) (result i32))) - (import "lunatic::metrics" "counter" (func (param i32 i32 i64))) - (import "lunatic::metrics" "increment_counter" (func (param i32 i32))) - (import "lunatic::metrics" "gauge" (func (param i32 i32 f64))) - (import "lunatic::metrics" "increment_gauge" (func (param i32 i32 f64))) - (import "lunatic::metrics" "decrement_gauge" (func (param i32 i32 f64))) - (import "lunatic::metrics" "histogram" (func (param i32 i32 f64))) + (import "lunatic::metrics" "span_start" (func (param i64 i32 i32 i32 i32) (result i64))) + (import "lunatic::metrics" "span_drop" (func (param i64))) + (import "lunatic::metrics" "meter" (func (param i32 i32) (result i64))) + (import "lunatic::metrics" "meter_drop" (func (param i64))) + (import "lunatic::metrics" "event" (func (param i64 i32 i32 i32 i32))) + (import "lunatic::metrics" "counter" (func (param i64 i32 i32 i32 i32 i32 i32) (result i64))) + (import "lunatic::metrics" "counter_add" (func (param i64 i64 f64 i32 i32))) + (import "lunatic::metrics" "counter_drop" (func (param i64))) + (import "lunatic::metrics" "up_down_counter" (func (param i64 i32 i32 i32 i32 i32 i32) (result i64))) + (import "lunatic::metrics" "up_down_counter_add" (func (param i64 i64 f64 i32 i32))) + (import "lunatic::metrics" "up_down_counter_drop" (func (param i64))) + (import "lunatic::metrics" "histogram" (func (param i64 i32 i32 i32 i32 i32 i32) (result i64))) + (import "lunatic::metrics" "histogram_record" (func (param i64 i64 f64 i32 i32))) + (import "lunatic::metrics" "histogram_drop" (func (param i64))) (func (export "hello") nop) ) From 08d258be21c617fddb87a7a3d508658e0aeffb6c Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 10 Apr 2023 22:36:34 +0930 Subject: [PATCH 14/16] chore: delete flake files --- flake.lock | 77 ------------------------------------------------------ flake.nix | 54 -------------------------------------- 2 files changed, 131 deletions(-) delete mode 100644 flake.lock delete mode 100644 flake.nix diff --git a/flake.lock b/flake.lock deleted file mode 100644 index 09540df62..000000000 --- a/flake.lock +++ /dev/null @@ -1,77 +0,0 @@ -{ - "nodes": { - "flake-utils": { - "locked": { - "lastModified": 1659877975, - "narHash": "sha256-zllb8aq3YO3h8B/U0/J1WBgAL8EX5yWf5pMj3G0NAmc=", - "owner": "numtide", - "repo": "flake-utils", - "rev": "c0e246b9b83f637f4681389ecabcb2681b4f3af0", - "type": "github" - }, - "original": { - "owner": "numtide", - "repo": "flake-utils", - "type": "github" - } - }, - "nixpkgs": { - "locked": { - "lastModified": 1671636183, - "narHash": "sha256-dboEYqb7vnH9pVEwgaWz7dzVi7eh6N5tRuhJ/nluoCg=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "60ff1ccd98a2f81347457a473c7a96b9b6166c88", - "type": "github" - }, - "original": { - "owner": "NixOS", - "repo": "nixpkgs", - "type": "github" - } - }, - "nixpkgs_2": { - "locked": { - "lastModified": 1665296151, - "narHash": "sha256-uOB0oxqxN9K7XGF1hcnY+PQnlQJ+3bP2vCn/+Ru/bbc=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "14ccaaedd95a488dd7ae142757884d8e125b3363", - "type": "github" - }, - "original": { - "owner": "NixOS", - "ref": "nixpkgs-unstable", - "repo": "nixpkgs", - "type": "github" - } - }, - "root": { - "inputs": { - "nixpkgs": "nixpkgs", - "rust-overlay": "rust-overlay" - } - }, - "rust-overlay": { - "inputs": { - "flake-utils": "flake-utils", - "nixpkgs": "nixpkgs_2" - }, - "locked": { - "lastModified": 1671589280, - "narHash": "sha256-FmJ4SC+Ewi1iMhdtRcrwirMfvW7h2jakT7ILLo9BVws=", - "owner": "oxalica", - "repo": "rust-overlay", - "rev": "bfc54bcf98dacdc649c88a82bf14d00b399aa3bb", - "type": "github" - }, - "original": { - "owner": "oxalica", - "repo": "rust-overlay", - "type": "github" - } - } - }, - "root": "root", - "version": 7 -} diff --git a/flake.nix b/flake.nix deleted file mode 100644 index 993fb7a00..000000000 --- a/flake.nix +++ /dev/null @@ -1,54 +0,0 @@ -{ - description = "Example Rust development environment for Zero to Nix"; - - # Flake inputs - inputs = { - nixpkgs.url = "github:NixOS/nixpkgs"; # also valid: "nixpkgs" - rust-overlay.url = "github:oxalica/rust-overlay"; # A helper for Rust + Nix - }; - - # Flake outputs - outputs = { self, nixpkgs, rust-overlay }: - let - # Overlays enable you to customize the Nixpkgs attribute set - overlays = [ - # Makes a `rust-bin` attribute available in Nixpkgs - (import rust-overlay) - # Provides a `rustToolchain` attribute for Nixpkgs that we can use to - # create a Rust environment - (self: super: { - rustToolchain = super.rust-bin.stable.latest.default.override { - extensions = [ "rust-src" ]; - }; - }) - ]; - - # Systems supported - allSystems = [ - "x86_64-linux" # 64-bit Intel/AMD Linux - "aarch64-linux" # 64-bit ARM Linux - "x86_64-darwin" # 64-bit Intel macOS - "aarch64-darwin" # 64-bit ARM macOS - ]; - - # Helper to provide system-specific attributes - forAllSystems = f: nixpkgs.lib.genAttrs allSystems (system: f { - pkgs = import nixpkgs { inherit overlays system; }; - }); - in - { - # Development environment output - devShells = forAllSystems ({ pkgs }: { - default = pkgs.mkShell { - # The Nix packages provided in the environment - packages = (with pkgs; [ - # The package provided by our custom overlay. Includes cargo, Clippy, cargo-fmt, - # rustdoc, rustfmt, and other tools. - rustToolchain - openssl - pkg-config - ]) ++ pkgs.lib.optionals pkgs.stdenv.isDarwin (with pkgs; [ libiconv ]); - }; - }); - }; -} From 4abbdd69560a63c2620564399d91eb842463d837 Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Mon, 17 Apr 2023 14:02:27 +0930 Subject: [PATCH 15/16] fix: environment shutdown --- crates/lunatic-process/src/env.rs | 21 ++- crates/lunatic-process/src/lib.rs | 283 ++++++++++++++-------------- crates/lunatic-process/src/state.rs | 3 - src/mode/common.rs | 29 +-- src/mode/run.rs | 1 + src/state.rs | 3 +- 6 files changed, 176 insertions(+), 164 deletions(-) diff --git a/crates/lunatic-process/src/env.rs b/crates/lunatic-process/src/env.rs index f8ab598b3..a8afafb62 100644 --- a/crates/lunatic-process/src/env.rs +++ b/crates/lunatic-process/src/env.rs @@ -8,6 +8,7 @@ use async_trait::async_trait; use dashmap::DashMap; use lunatic_common_api::MetricsExt; use opentelemetry::KeyValue; +use tokio::sync::{mpsc, Mutex}; use crate::{Process, Signal, ENVIRONMENT_METRICS}; @@ -21,7 +22,7 @@ pub trait Environment: Send + Sync { fn process_count(&self) -> usize; async fn can_spawn_next_process(&self) -> Result>; fn send(&self, id: u64, signal: Signal); - fn shutdown(&self); + async fn shutdown(&self); } #[async_trait] @@ -32,19 +33,21 @@ pub trait Environments: Send + Sync { async fn get(&self, id: u64) -> Option>; } -#[derive(Clone)] pub struct LunaticEnvironment { environment_id: u64, next_process_id: Arc, processes: Arc>>, + all_processes_finished: (mpsc::Sender<()>, Mutex>), } impl LunaticEnvironment { pub fn new(id: u64) -> Self { + let (tx, rx) = mpsc::channel(1); Self { environment_id: id, processes: Arc::new(DashMap::new()), next_process_id: Arc::new(AtomicU64::new(1)), + all_processes_finished: (tx, Mutex::new(rx)), } } } @@ -75,6 +78,11 @@ impl Environment for LunaticEnvironment { &[KeyValue::new("environment_id", self.id() as i64)], ); }); + + if self.process_count() == 0 { + let tx = self.all_processes_finished.0.clone(); + tokio::spawn(async move { tx.send(()).await }); + } } fn process_count(&self) -> usize { @@ -100,9 +108,12 @@ impl Environment for LunaticEnvironment { Ok(Some(())) } - fn shutdown(&self) { - for proc in self.processes.iter() { - proc.send(Signal::Kill); + async fn shutdown(&self) { + if self.process_count() > 0 { + for proc in self.processes.iter() { + proc.send(Signal::Kill); + } + self.all_processes_finished.1.lock().await.recv().await; } } } diff --git a/crates/lunatic-process/src/lib.rs b/crates/lunatic-process/src/lib.rs index 92e5fa977..877eb5343 100644 --- a/crates/lunatic-process/src/lib.rs +++ b/crates/lunatic-process/src/lib.rs @@ -255,160 +255,161 @@ where R: Into>, F: Future + Send + 'static, { - trace!("Process {} spawned", id); - tokio::pin!(fut); - - // Defines what happens if one of the linked processes dies. - // If the value is set to false, instead of dying too the process will receive a message about - // the linked process' death. - let mut die_when_link_dies = true; - // Process linked to this one - let mut links = HashMap::new(); - // TODO: Maybe wrapping this in some kind of `std::panic::catch_unwind` wold be a good idea, - // to protect against panics in host function calls that unwind through Wasm code. - // Currently a panic would just kill the task, but not notify linked processes. - let mut signal_mailbox = signal_mailbox.lock().await; - let mut has_sender = true; - let result = loop { - tokio::select! { - biased; - // Handle signals first - signal = signal_mailbox.recv(), if has_sender => { - SIGNALS_METRICS.with_current_context(|metrics, cx| { - metrics.received.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) - }); - - match signal.ok_or(()) { - Ok(Signal::Message(message)) => { - - message.write_metrics(); - - message_mailbox.push(message); - - // process metrics - MESSAGES_METRICS.with_current_context(|metrics, cx| { - let attrs = [KeyValue::new("process_id", id as i64)]; - metrics.sent.add(&cx, 1, &attrs); - metrics.outstanding.add(&cx, 1, &attrs); - }); - }, - Ok(Signal::DieWhenLinkDies(value)) => die_when_link_dies = value, - // Put process into list of linked processes - Ok(Signal::Link(tag, proc)) => { - links.insert(proc.id(), (proc, tag)); - - LINKS_METRICS.with_current_context(|metrics, cx| { - metrics.alive.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) - }); - - }, - // Remove process from list - Ok(Signal::UnLink { process_id }) => { - links.remove(&process_id); - - LINKS_METRICS.with_current_context(|metrics, cx| { - metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) - }); - } - // Exit loop and don't poll anymore the future if Signal::Kill received. - Ok(Signal::Kill) => break Finished::KillSignal, - // Depending if `die_when_link_dies` is set, process will die or turn the - // signal into a message - Ok(Signal::LinkDied(id, tag, reason)) => { - links.remove(&id); - - LINKS_METRICS.with_current_context(|metrics, cx| { - metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) - }); - - match reason { - DeathReason::Failure | DeathReason::NoProcess => { - if die_when_link_dies { - // Even this was not a **kill** signal it has the same effect on - // this process and should be propagated as such. - break Finished::KillSignal - } else { - let message = Message::LinkDied(tag); - message_mailbox.push(message); - - MESSAGES_METRICS.with_current_context(|metrics, cx| { - metrics.outstanding.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]); - }); - - } - }, - // In case a linked process finishes normally, don't do anything. - DeathReason::Normal => {}, + let result = { + trace!("Process {} spawned", id); + tokio::pin!(fut); + + // Defines what happens if one of the linked processes dies. + // If the value is set to false, instead of dying too the process will receive a message about + // the linked process' death. + let mut die_when_link_dies = true; + // Process linked to this one + let mut links = HashMap::new(); + // TODO: Maybe wrapping this in some kind of `std::panic::catch_unwind` wold be a good idea, + // to protect against panics in host function calls that unwind through Wasm code. + // Currently a panic would just kill the task, but not notify linked processes. + let mut signal_mailbox = signal_mailbox.lock().await; + let mut has_sender = true; + let result = loop { + tokio::select! { + biased; + // Handle signals first + signal = signal_mailbox.recv(), if has_sender => { + SIGNALS_METRICS.with_current_context(|metrics, cx| { + metrics.received.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) + }); + + match signal.ok_or(()) { + Ok(Signal::Message(message)) => { + message.write_metrics(); + message_mailbox.push(message); + + MESSAGES_METRICS.with_current_context(|metrics, cx| { + let attrs = [KeyValue::new("process_id", id as i64)]; + metrics.sent.add(&cx, 1, &attrs); + metrics.outstanding.add(&cx, 1, &attrs); + }); + }, + Ok(Signal::DieWhenLinkDies(value)) => die_when_link_dies = value, + // Put process into list of linked processes + Ok(Signal::Link(tag, proc)) => { + links.insert(proc.id(), (proc, tag)); + + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]) + }); + + }, + // Remove process from list + Ok(Signal::UnLink { process_id }) => { + links.remove(&process_id); + + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) + }); + } + // Exit loop and don't poll anymore the future if Signal::Kill received. + Ok(Signal::Kill) => break Finished::KillSignal, + // Depending if `die_when_link_dies` is set, process will die or turn the + // signal into a message + Ok(Signal::LinkDied(id, tag, reason)) => { + links.remove(&id); + + LINKS_METRICS.with_current_context(|metrics, cx| { + metrics.alive.add(&cx, -1, &[KeyValue::new("process_id", id as i64)]) + }); + + match reason { + DeathReason::Failure | DeathReason::NoProcess => { + if die_when_link_dies { + // Even this was not a **kill** signal it has the same effect on + // this process and should be propagated as such. + break Finished::KillSignal; + } else { + let message = Message::LinkDied(tag); + message_mailbox.push(message); + + MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.outstanding.add(&cx, 1, &[KeyValue::new("process_id", id as i64)]); + }); + + } + }, + // In case a linked process finishes normally, don't do anything. + DeathReason::Normal => {}, + } + }, + Err(_) => { + debug_assert!(has_sender); + has_sender = false; } - }, - Err(_) => { - debug_assert!(has_sender); - has_sender = false; } } + // Run process + output = &mut fut => { break Finished::Normal(output); } } - // Run process - output = &mut fut => { break Finished::Normal(output); } - } - }; + }; - MESSAGES_METRICS.with_current_context(|metrics, cx| { - metrics.outstanding.add( - &cx, - -(message_mailbox.len() as i64), - &[KeyValue::new("process_id", id as i64)], - ); - }); + MESSAGES_METRICS.with_current_context(|metrics, cx| { + metrics.outstanding.add( + &cx, + -(message_mailbox.len() as i64), + &[KeyValue::new("process_id", id as i64)], + ); + }); - let result = match result { - Finished::Normal(result) => { - let result: ExecutionResult<_> = result.into(); - - if let Some(failure) = result.failure() { - let registry = result.state().registry().read().await; - let name = registry - .iter() - .filter(|(_, (_, process_id))| process_id == &id) - .map(|(name, _)| name.splitn(4, '/').last().unwrap_or(name.as_str())) - .collect::() - .or_id(id); + match result { + Finished::Normal(result) => { + let result: ExecutionResult<_> = result.into(); + + if let Some(failure) = result.failure() { + let registry = result.state().registry().read().await; + let name = registry + .iter() + .filter(|(_, (_, process_id))| process_id == &id) + .map(|(name, _)| name.splitn(4, '/').last().unwrap_or(name.as_str())) + .collect::() + .or_id(id); + warn!( + "Process {} failed, notifying: {} links {}", + name, + links.len(), + // If the log level is WARN instruct user how to display the stacktrace + if !log_enabled!(Level::Debug) { + "\n\t\t\t (Set ENV variable `RUST_LOG=lunatic=debug` to show stacktrace)" + } else { + "" + } + ); + debug!("{}", failure); + // Notify all links that we finished with an error + links.iter().for_each(|(_, (proc, tag))| { + proc.send(Signal::LinkDied(id, *tag, DeathReason::Failure)); + }); + Err(anyhow!(failure.to_string())) + } else { + // Notify all links that we finished normally + links.iter().for_each(|(_, (proc, tag))| { + proc.send(Signal::LinkDied(id, *tag, DeathReason::Normal)); + }); + Ok(result.into_state()) + } + } + Finished::KillSignal => { warn!( - "Process {} failed, notifying: {} links {}", - name, - links.len(), - // If the log level is WARN instruct user how to display the stacktrace - if !log_enabled!(Level::Debug) { - "\n\t\t\t (Set ENV variable `RUST_LOG=lunatic=debug` to show stacktrace)" - } else { - "" - } + "Process {} was killed, notifying: {} links", + id, + links.len() ); - debug!("{}", failure); - // Notify all links that we finished with an error + + // Notify all links that we finished because of a kill signal links.iter().for_each(|(_, (proc, tag))| { proc.send(Signal::LinkDied(id, *tag, DeathReason::Failure)); }); - Err(anyhow!(failure.to_string())) - } else { - // Notify all links that we finished normally - links.iter().for_each(|(_, (proc, tag))| { - proc.send(Signal::LinkDied(id, *tag, DeathReason::Normal)); - }); - Ok(result.into_state()) + + Err(anyhow!("Process received Kill signal")) } } - Finished::KillSignal => { - warn!( - "Process {} was killed, notifying: {} links", - id, - links.len() - ); - // Notify all links that we finished because of a kill signal - links.iter().for_each(|(_, (proc, tag))| { - proc.send(Signal::LinkDied(id, *tag, DeathReason::Failure)); - }); - Err(anyhow!("Process received Kill signal")) - } }; env.remove_process(id); diff --git a/crates/lunatic-process/src/state.rs b/crates/lunatic-process/src/state.rs index a3e395124..6d13d673a 100644 --- a/crates/lunatic-process/src/state.rs +++ b/crates/lunatic-process/src/state.rs @@ -66,7 +66,4 @@ pub trait ProcessState: Sized { fn registry_atomic_put( &mut self, ) -> &mut Option>>; - - // Spans - // fn span_resources(&self) -> &SpanResources<> } diff --git a/src/mode/common.rs b/src/mode/common.rs index a2c62c8bc..8a2cfa51e 100644 --- a/src/mode/common.rs +++ b/src/mode/common.rs @@ -9,6 +9,7 @@ use hyper::{ service::{make_service_fn, service_fn}, Body, Method, Request, Response, Server, }; +use log::LevelFilter; use lunatic_distributed::DistributedProcessState; use lunatic_process::{ env::{Environment, LunaticEnvironment, LunaticEnvironments}, @@ -25,6 +26,7 @@ use opentelemetry::{ KeyValue, }; use prometheus::{Encoder, Registry, TextEncoder}; +use tokio::time::{timeout, Duration}; #[derive(Args, Debug)] pub struct WasmArgs {} @@ -81,10 +83,7 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { }; let module = Arc::new(args.runtime.compile_module::(module)?); - let mut root_span = args.tracer.start( - "app_start", // SpanBuilder::from_name("app_start") - // .with_attributes([KeyValue::new("path", path.to_string_lossy().to_string())]), - ); + let mut root_span = args.tracer.start("app_start"); root_span.set_attributes([ KeyValue::new("service.name", "lunatic"), KeyValue::new("lunatic.path", path.to_string_lossy().to_string()), @@ -107,7 +106,7 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { Arc::new(config), Default::default(), args.tracer, - tracer_context, + tracer_context.clone(), args.meter_provider, logger, ) @@ -131,16 +130,23 @@ pub async fn run_wasm(args: RunWasm) -> Result<()> { ))?; // Wait on the main process to finish, or ctrl c signal - tokio::select! { + let result = tokio::select! { result = task => { result.map(|_| ()).map_err(|err| anyhow!(err.to_string())) }, Ok(_) = tokio::signal::ctrl_c() => { - args.env.shutdown(); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; Ok(()) }, - } + }; + + // Kill remaining processes + log::set_max_level(LevelFilter::Off); + let _ = timeout(Duration::from_millis(100), args.env.shutdown()).await; + + // End the app span + tracer_context.span().end(); + + result } pub fn prometheus(addr: &SocketAddr, _node_id: Option) -> Result { @@ -199,9 +205,6 @@ pub fn prometheus(addr: &SocketAddr, _node_id: Option) -> Result Result<()> { let tracer = opentelemetry_jaeger::new_agent_pipeline() .with_endpoint(jaeger_url) .with_service_name("lunatic") + .with_auto_split_batch(true) .install_batch(Tokio)?; Arc::new(BoxedTracer::new(Box::new(tracer))) } diff --git a/src/state.rs b/src/state.rs index cb732097f..1ee3d17af 100644 --- a/src/state.rs +++ b/src/state.rs @@ -29,7 +29,7 @@ use opentelemetry::metrics::noop::NoopMeterProvider; use opentelemetry::metrics::{Counter, Histogram, Meter, UpDownCounter}; use opentelemetry::trace::noop::NoopTracer; use opentelemetry::trace::{Span, TraceContextExt, Tracer}; -use opentelemetry::{Context, KeyValue}; +use opentelemetry::Context; use tokio::net::{TcpListener, UdpSocket}; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::{Mutex, RwLock, RwLockWriteGuard}; @@ -458,7 +458,6 @@ impl MetricsCtx for DefaultProcessState { } fn add_context(&mut self, context: Context) -> u64 { - context.span().set_attribute(KeyValue::new("hey", "there")); self.resources.contexts.add(context) } From 4019049d31bd4d2a8c75f9875a9b1be243d64e8f Mon Sep 17 00:00:00 2001 From: Ari Seyhun Date: Tue, 18 Apr 2023 14:59:35 +0930 Subject: [PATCH 16/16] feat: add process_id and environment_id to traces --- Cargo.lock | 1 + crates/lunatic-metrics-api/Cargo.toml | 1 + crates/lunatic-metrics-api/src/lib.rs | 50 +++++++++++++++++++++------ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cc4ad5674..d13d12dae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1591,6 +1591,7 @@ dependencies = [ "log", "lunatic-common-api", "lunatic-process", + "lunatic-process-api", "opentelemetry", "serde_json", "wasmtime", diff --git a/crates/lunatic-metrics-api/Cargo.toml b/crates/lunatic-metrics-api/Cargo.toml index 71353fd19..8a7c9e4fe 100644 --- a/crates/lunatic-metrics-api/Cargo.toml +++ b/crates/lunatic-metrics-api/Cargo.toml @@ -11,6 +11,7 @@ license = "Apache-2.0/MIT" hash-map-id = { workspace = true } lunatic-common-api = { workspace = true } lunatic-process = { workspace = true } +lunatic-process-api = { workspace = true } anyhow = { workspace = true } log = { workspace = true } diff --git a/crates/lunatic-metrics-api/src/lib.rs b/crates/lunatic-metrics-api/src/lib.rs index 6ff89667a..4470ef0ab 100644 --- a/crates/lunatic-metrics-api/src/lib.rs +++ b/crates/lunatic-metrics-api/src/lib.rs @@ -3,6 +3,7 @@ use hash_map_id::HashMapId; use log::{Level, Record}; use lunatic_common_api::{get_memory, IntoTrap}; use lunatic_process::state::ProcessState; +use lunatic_process_api::ProcessCtx; use opentelemetry::{ metrics::{ Counter, Histogram, InstrumentBuilder, Meter, MeterProvider, MetricsError, Unit, @@ -14,6 +15,9 @@ use opentelemetry::{ use serde_json::Map; use wasmtime::{Caller, Linker}; +pub const OTEL_LUNATIC_ENVIRONMENT_ID_KEY: &str = "lunatic.environment_id"; +pub const OTEL_LUNATIC_PROCESS_ID_KEY: &str = "lunatic.process_id"; + pub type ContextResources = HashMapId; pub trait MetricsCtx { @@ -50,7 +54,7 @@ pub trait MetricsCtx { /// Links the [Metrics](https://crates.io/crates/metrics) APIs. pub fn register(linker: &mut Linker) -> anyhow::Result<()> where - T: ProcessState + MetricsCtx + 'static, + T: ProcessCtx + ProcessState + MetricsCtx + 'static, <::Tracer as Tracer>::Span: Send + Sync, { linker.func_wrap("lunatic::metrics", "span_start", span_start)?; @@ -102,7 +106,7 @@ fn span_start( attributes_len: u32, ) -> Result where - T: MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, <::Tracer as Tracer>::Span: Send + Sync + 'static, { let memory = get_memory(&mut caller)?; @@ -115,7 +119,7 @@ where }; let name = get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::span_start")?; - let attributes = if attributes_len > 0 { + let mut attributes = if attributes_len > 0 { let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) .or_trap("lunatic::metrics::span_start")?; @@ -125,6 +129,7 @@ where } else { vec![] }; + inject_lunatic_attributes(state, &mut attributes); let parent_ctx = if let Some(id) = parent { state @@ -225,14 +230,14 @@ fn event( attributes_len: u32, ) -> Result<()> where - T: ProcessState + MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, { let memory = get_memory(&mut caller)?; let (data, state) = memory.data_and_store_mut(&mut caller); let name = get_string_arg(data, name_ptr, name_len).or_trap("lunatic::metrics::event")?; - let attributes = if attributes_len > 0 { + let mut attributes = if attributes_len > 0 { let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) .or_trap("lunatic::metrics::event")?; @@ -295,6 +300,7 @@ where vec![] }; + inject_lunatic_attributes(state, &mut attributes); let span = if span != u64::MAX { state @@ -368,7 +374,7 @@ fn counter_add( attributes_len: u32, ) -> Result<()> where - T: MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, { let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) .or_trap("lunatic::metrics::counter_add")?; @@ -453,7 +459,7 @@ fn up_down_counter_add( attributes_len: u32, ) -> Result<()> where - T: MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, { let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) .or_trap("lunatic::metrics::up_down_counter_add")?; @@ -538,7 +544,7 @@ fn histogram_record( attributes_len: u32, ) -> Result<()> where - T: MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, { let (state, cx, attributes) = update_metric(&mut caller, span, attributes_ptr, attributes_len) .or_trap("lunatic::metrics::histogram_record")?; @@ -630,7 +636,7 @@ fn update_metric<'a, T>( attributes_len: u32, ) -> Result<(&'a T, &'a Context, Vec)> where - T: MetricsCtx, + T: ProcessCtx + ProcessState + MetricsCtx, { let memory = get_memory(caller)?; let (data, state) = memory.data_and_store_mut(caller); @@ -641,7 +647,7 @@ where state.get_last_context() }; - let attributes = if attributes_len > 0 { + let mut attributes = if attributes_len > 0 { let attributes_data = data .get(attributes_ptr as usize..(attributes_ptr + attributes_len) as usize) .context("invalid memory region for attributes")?; @@ -652,10 +658,34 @@ where } else { vec![] }; + inject_lunatic_attributes(state, &mut attributes); Ok((state, cx, attributes)) } +fn inject_lunatic_attributes + ProcessState>( + state: &T, + attributes: &mut Vec, +) { + let environment_id = opentelemetry::Value::I64(state.environment().id() as i64); + let process_id = opentelemetry::Value::I64(state.id() as i64); + let lunatic_attrs = [ + (OTEL_LUNATIC_ENVIRONMENT_ID_KEY, environment_id), + (OTEL_LUNATIC_PROCESS_ID_KEY, process_id), + ]; + for (key, value) in lunatic_attrs { + match attributes + .iter_mut() + .find(|key_value| key_value.key.as_str() == key) + { + Some(key_value) => key_value.value = value, + None => { + attributes.push(KeyValue::new(key, value)); + } + } + } +} + fn data_to_opentelemetry(data: Map) -> Vec { data.into_iter() .map(|(k, v)| KeyValue {