use grafana otlp collector

near · Nov 1, 2023 · 50373cb · 50373cb
1 parent 62e3134
commit 50373cb
Show file tree

Hide file tree

Showing 8 changed files with 61 additions and 9 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/infra/main.tf b/infra/main.tf
@@ -148,6 +148,9 @@ module "leader" {
 
   jwt_signature_pk_url = var.jwt_signature_pk_url
 
+  otlp_endpoint       = var.otlp_endpoint
+  opentelemetry_level = var.opentelemetry_level
+
   depends_on = [
     google_secret_manager_secret_iam_member.account_creator_secret_access,
     google_secret_manager_secret_iam_member.fast_auth_partners_secret_access,

diff --git a/infra/modules/leader/main.tf b/infra/modules/leader/main.tf
@@ -61,11 +61,18 @@ resource "google_cloud_run_v2_service" "leader" {
           }
         }
       }
-
       env {
         name  = "MPC_RECOVERY_JWT_SIGNATURE_PK_URL"
         value = var.jwt_signature_pk_url
       }
+      env {
+        name  = "MPC_RECOVERY_OTLP_ENDPOINT"
+        value = var.otlp_endpoint
+      }
+      env {
+        name  = "MPC_RECOVERY_OPENTELEMETRY_LEVEL"
+        value = var.opentelemetry_level
+      }
 
       env {
         name  = "RUST_LOG"

diff --git a/infra/modules/leader/variables.tf b/infra/modules/leader/variables.tf
@@ -51,3 +51,11 @@ variable "fast_auth_partners_secret_id" {
 variable "jwt_signature_pk_url" {
   type = string
 }
+
+variable "otlp_endpoint" {
+  type = string
+}
+
+variable "opentelemetry_level" {
+  type = string
+}
diff --git a/infra/terraform-dev.tfvars b/infra/terraform-dev.tfvars
@@ -20,3 +20,5 @@ signer_configs = [
   }
 ]
 jwt_signature_pk_url = "https://www.googleapis.com/robot/v1/metadata/x509/[email protected]"
+otlp_endpoint        = "http://otel.dev.api.pagoda.co:443"
+opentelemetry_level  = "debug"
diff --git a/infra/variables.tf b/infra/variables.tf
@@ -53,3 +53,13 @@ variable "signer_configs" {
 variable "jwt_signature_pk_url" {
   type = string
 }
+
+variable "otlp_endpoint" {
+  type    = string
+  default = "http://localhost:4317"
+}
+
+variable "opentelemetry_level" {
+  type    = string
+  default = "off"
+}
diff --git a/mpc-recovery/Cargo.toml b/mpc-recovery/Cargo.toml
@@ -28,7 +28,7 @@ hyper-rustls = { version = "=0.23", features = ["http2"] }
 jsonwebtoken = "8.3.0"
 lazy_static = "1.4.0"
 opentelemetry = { version = "0.20.0", features = ["rt-tokio", "trace"] }
-opentelemetry-otlp = "0.13.0"
+opentelemetry-otlp = { version = "0.13.0", features = ["http-proto"] }
 opentelemetry-semantic-conventions = "0.12.0"
 prometheus = { version = "0.13.3", features = ["process"] }
 rand = "0.7"

diff --git a/mpc-recovery/src/logging.rs b/mpc-recovery/src/logging.rs
@@ -82,11 +82,20 @@ impl Display for ColorOutput {
 #[derive(Debug, Default, clap::Parser)]
 pub struct Options {
     /// Enables export of span data using opentelemetry exporters.
-    #[clap(long, value_enum, default_value = "off")]
-    opentelemetry: OpenTelemetryLevel,
+    #[clap(
+        long,
+        env("MPC_RECOVERY_OPENTELEMETRY_LEVEL"),
+        value_enum,
+        default_value = "off"
+    )]
+    opentelemetry_level: OpenTelemetryLevel,
 
     /// Opentelemetry gRPC collector endpoint.
-    #[clap(long, default_value = "http://localhost:4317")]
+    #[clap(
+        long,
+        env("MPC_RECOVERY_OTLP_ENDPOINT"),
+        default_value = "http://localhost:4317"
+    )]
     otlp_endpoint: String,
 
     /// Whether the log needs to be colored.
@@ -103,7 +112,7 @@ impl Options {
     pub fn into_str_args(self) -> Vec<String> {
         let mut buf = vec![
             "--opentelemetry".to_string(),
-            self.opentelemetry.to_string(),
+            self.opentelemetry_level.to_string(),
             "--otlp-endpoint".to_string(),
             self.otlp_endpoint,
             "--color".to_string(),
@@ -193,7 +202,7 @@ where
         .tracing()
         .with_exporter(
             opentelemetry_otlp::new_exporter()
-                .tonic()
+                .http()
                 .with_endpoint(otlp_endpoint),
         )
         .with_trace_config(
@@ -213,7 +222,7 @@ where
 fn set_default_otlp_level(options: &Options) {
     // Record the initial tracing level specified as a command-line flag. Use this recorded value to
     // reset opentelemetry filter when the LogConfig file gets deleted.
-    DEFAULT_OTLP_LEVEL.set(options.opentelemetry).unwrap();
+    DEFAULT_OTLP_LEVEL.set(options.opentelemetry_level).unwrap();
 }
 
 /// The resource representing a registered subscriber.
@@ -287,7 +296,7 @@ pub async fn default_subscriber_with_opentelemetry(
         .unwrap_or_else(|_| panic!("Failed to set Log Layer Filter"));
 
     let (subscriber, handle) = add_opentelemetry_layer(
-        options.opentelemetry,
+        options.opentelemetry_level,
         &options.otlp_endpoint,
         env,
         node_id,