feat: add some basic open telemetry setup and metrics

diode-server/cmd/ingester/main.go

@@ -5,18 +5,56 @@ import (
 	"os"
 
 	"github.com/getsentry/sentry-go"
+	"github.com/kelseyhightower/envconfig"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric"
 
 	"github.com/netboxlabs/diode/diode-server/ingester"
 	"github.com/netboxlabs/diode/diode-server/server"
+	"github.com/netboxlabs/diode/diode-server/telemetry"
+)
+
+const (
+	applicationName = "diode-ingester"
+	metricStartup   = "diode-ingester.startup_count"
 )
 
 func main() {
 	ctx := context.Background()
-	s := server.New(ctx, "diode-ingester")
+	s := server.New(ctx, applicationName)
 
 	defer s.Recover(sentry.CurrentHub())
 
-	ingesterComponent, err := ingester.New(ctx, s.Logger())
+	// Load configuration
+	var cfg ingester.Config
+	envconfig.MustProcess("", &cfg)
+
+	// Set default telemetry configuration if not provided
+	if cfg.Telemetry.ServiceName == "" {
+		cfg.Telemetry.ServiceName = applicationName
+	}
+
+	shutdown, err := telemetry.Setup(ctx, cfg.Telemetry)
+	if err != nil {
+		s.Logger().Error("failed to initialize telemetry", "error", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := shutdown(ctx); err != nil {
+			s.Logger().Error("failed to shutdown telemetry", "error", err)
+		}
+	}()
+
+	meter := otel.GetMeterProvider().Meter(applicationName)
+	startupCounter, err := meter.Int64Counter(metricStartup,
+		metric.WithDescription("Number of times the ingester service has started"))
+	if err != nil {
+		s.Logger().Error("failed to create startup metric", "error", err)
+		os.Exit(1)
+	}
+	startupCounter.Add(ctx, 1)
+
+	ingesterComponent, err := ingester.New(ctx, s.Logger(), cfg, meter)
 	if err != nil {
 		s.Logger().Error("failed to instantiate ingester component", "error", err)
 		os.Exit(1)
@@ -27,7 +65,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	// TODO: instantiate prometheus server
+	telemetry.ServePrometheusMetricsIfNecessary(cfg.Telemetry, s.Logger())
 
 	if err := s.Run(); err != nil {
 		s.Logger().Error("server failure", "serverName", s.Name(), "error", err)

diode-server/cmd/reconciler/main.go

@@ -11,23 +11,58 @@ import (
 	_ "github.com/jackc/pgx/v5/stdlib" // pgx to database/sql compatibility
 	"github.com/kelseyhightower/envconfig"
 	"github.com/pressly/goose/v3"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/metric"
 
 	"github.com/netboxlabs/diode/diode-server/dbstore/postgres"
 	"github.com/netboxlabs/diode/diode-server/migrator"
 	"github.com/netboxlabs/diode/diode-server/netboxdiodeplugin"
 	"github.com/netboxlabs/diode/diode-server/reconciler"
 	"github.com/netboxlabs/diode/diode-server/server"
+	"github.com/netboxlabs/diode/diode-server/telemetry"
+)
+
+const (
+	applicationName             = "diode-reconciler"
+	ingestionProcessorMeterName = "diode-reconciler.ingestion-processor"
+	metricStartup               = "diode-reconciler.startup_count"
 )
 
 func main() {
 	ctx := context.Background()
-	s := server.New(ctx, "diode-reconciler")
+	s := server.New(ctx, applicationName)
 
 	defer s.Recover(sentry.CurrentHub())
 
 	var cfg reconciler.Config
 	envconfig.MustProcess("", &cfg)
 
+	// Set default telemetry configuration if not provided
+	if cfg.Telemetry.ServiceName == "" {
+		cfg.Telemetry.ServiceName = applicationName
+	}
+
+	// Initialize telemetry
+	shutdown, err := telemetry.Setup(ctx, cfg.Telemetry)
+	if err != nil {
+		s.Logger().Error("failed to initialize telemetry", "error", err)
+		os.Exit(1)
+	}
+	defer func() {
+		if err := shutdown(ctx); err != nil {
+			s.Logger().Error("failed to shutdown telemetry", "error", err)
+		}
+	}()
+
+	appMeter := otel.GetMeterProvider().Meter(applicationName)
+	startupCounter, err := appMeter.Int64Counter(metricStartup,
+		metric.WithDescription("Number of times the reconciler service has started"))
+	if err != nil {
+		s.Logger().Error("failed to create startup metric", "error", err)
+		os.Exit(1)
+	}
+	startupCounter.Add(ctx, 1)
+
 	dbURL := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", cfg.PostgresHost, cfg.PostgresPort, cfg.PostgresUser, cfg.PostgresPassword, cfg.PostgresDBName)
 
 	if cfg.MigrationEnabled {
@@ -50,8 +85,15 @@ func main() {
 	if err != nil {
 		s.Logger().Error("failed to create netbox diode plugin client", "error", err)
 	}
+
 	ops := reconciler.NewOps(repository, nbClient, s.Logger())
-	ingestionProcessor, err := reconciler.NewIngestionProcessor(ctx, s.Logger(), ops)
+	ingestionMeter := otel.GetMeterProvider().Meter(ingestionProcessorMeterName)
+	ingestionMetrics, err := reconciler.NewOtelIngestionProcessorMetrics(ingestionMeter, applicationName)
+	if err != nil {
+		s.Logger().Error("failed to create ingestion processor metrics", "error", err)
+		os.Exit(1)
+	}
+	ingestionProcessor, err := reconciler.NewIngestionProcessor(ctx, s.Logger(), ops, ingestionMetrics)
 	if err != nil {
 		s.Logger().Error("failed to instantiate ingestion processor", "error", err)
 		os.Exit(1)
@@ -73,7 +115,7 @@ func main() {
 		os.Exit(1)
 	}
 
-	// TODO: instantiate prometheus server
+	telemetry.ServePrometheusMetricsIfNecessary(cfg.Telemetry, s.Logger())
 
 	if err := s.Run(); err != nil {
 		s.Logger().Error("server failure", "serverName", s.Name(), "error", err)

diode-server/docker/docker-compose.yaml

@@ -9,6 +9,12 @@ services:
       upstream diode-reconciler {
         server diode-reconciler:8081;
       }
+      upstream diode-reconcilermetrics {
+        server diode-reconciler:9090;
+      }
+      upstream diode-ingestermetrics {
+        server diode-ingester:9090;
+      }
       server {
         listen 80;
         http2 on;
@@ -22,6 +28,14 @@ services:
           rewrite /diode/(.*) /$$1 break;
           grpc_pass grpc://diode-reconciler;
         }
+        location /diode/reconciler/metrics {
+          rewrite /diode/reconciler/(.*) /$$1 break;
+          proxy_pass http://diode-reconcilermetrics;
+        }
+        location /diode/ingester/metrics {
+          rewrite /diode/ingester/(.*) /$$1 break;
+          proxy_pass http://diode-ingestermetrics;
+        }
       }'
       > /etc/nginx/conf.d/default.conf && nginx -g 'daemon off;'"
     restart: always
@@ -43,6 +57,9 @@ services:
       - RECONCILER_GRPC_PORT=${RECONCILER_GRPC_PORT}
       - DIODE_API_KEY=${DIODE_API_KEY}
       - SENTRY_DSN=${SENTRY_DSN}
+      - TELEMETRY_METRICS_EXPORTER=${TELEMETRY_METRICS_EXPORTER}
+      - TELEMETRY_TRACES_EXPORTER=${TELEMETRY_TRACES_EXPORTER}
+      - TELEMETRY_ENVIRONMENT=${TELEMETRY_ENVIRONMENT}
     restart: always
     ports: [ ]
     depends_on:
@@ -69,6 +86,9 @@ services:
       - POSTGRES_DB_NAME=${POSTGRES_DB_NAME}
       - POSTGRES_USER=${POSTGRES_USER}
       - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
+      - TELEMETRY_METRICS_EXPORTER=${TELEMETRY_METRICS_EXPORTER}
+      - TELEMETRY_TRACES_EXPORTER=${TELEMETRY_TRACES_EXPORTER}
+      - TELEMETRY_ENVIRONMENT=${TELEMETRY_ENVIRONMENT}
     restart: always
     ports: [ ]
     volumes:

diode-server/docker/sample.env

@@ -19,3 +19,6 @@ POSTGRES_PORT=5432
 POSTGRES_DB_NAME=diode
 POSTGRES_USER=diode
 POSTGRES_PASSWORD=CHANGE.ME
+TELEMETRY_ENVIRONMENT=dev
+TELEMETRY_METRICS_EXPORTER=prometheus
+TELEMETRY_TRACES_EXPORTER=none

diode-server/go.mod

@@ -5,7 +5,7 @@ go 1.23.4
 require (
 	github.com/alicebob/miniredis/v2 v2.33.0
 	github.com/andybalholm/brotli v1.1.1
-	github.com/envoyproxy/protoc-gen-validate v1.0.4
+	github.com/envoyproxy/protoc-gen-validate v1.1.0
 	github.com/getsentry/sentry-go v0.27.0
 	github.com/google/uuid v1.6.0
 	github.com/gosimple/slug v1.14.0
@@ -17,43 +17,67 @@ require (
 	github.com/mitchellh/mapstructure v1.5.0
 	github.com/oklog/run v1.1.0
 	github.com/pressly/goose/v3 v3.23.0
+	github.com/prometheus/client_golang v1.20.5
 	github.com/redis/go-redis/v9 v9.5.1
-	github.com/stretchr/testify v1.9.0
+	github.com/stretchr/testify v1.10.0
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0
+	go.opentelemetry.io/otel v1.34.0
+	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.34.0
+	go.opentelemetry.io/otel/exporters/prometheus v0.56.0
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.34.0
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.34.0
+	go.opentelemetry.io/otel/metric v1.34.0
+	go.opentelemetry.io/otel/sdk v1.34.0
+	go.opentelemetry.io/otel/sdk/metric v1.34.0
 	golang.org/x/time v0.5.0
-	google.golang.org/grpc v1.64.1
-	google.golang.org/protobuf v1.34.2
+	google.golang.org/grpc v1.69.4
+	google.golang.org/protobuf v1.36.3
 	modernc.org/sqlite v1.34.1
 )
 
 require (
 	github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.4.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/gosimple/unidecode v1.0.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.25.1 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	github.com/kr/text v0.2.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mfridman/interpolate v0.0.2 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.6.1 // indirect
+	github.com/prometheus/common v0.61.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
-	github.com/rogpeppe/go-internal v1.13.1 // indirect
 	github.com/sethvargo/go-retry v0.3.0 // indirect
 	github.com/stretchr/objx v0.5.2 // indirect
 	github.com/yuin/gopher-lua v1.1.1 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.34.0 // indirect
+	go.opentelemetry.io/otel/trace v1.34.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
-	golang.org/x/net v0.33.0 // indirect
+	golang.org/x/crypto v0.32.0 // indirect
+	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.org/x/tools v0.28.0 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20250115164207-1a7da9e5054f // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20250115164207-1a7da9e5054f // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/gc/v3 v3.0.0-20240107210532-573471604cb6 // indirect
 	modernc.org/libc v1.55.3 // indirect