From 258fe2927c6fecccc2e6c178d71e825939fea895 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Fri, 28 Jun 2024 11:28:06 +0200 Subject: [PATCH] metrics: refactor prometheus metric registration (#3132) Register prometheus metrics right after creation to reduce likelihood of missing registration. Follow up on #3129 Signed-off-by: Alexander Yastrebov --- metrics/prometheus.go | 165 +++++++++++++++--------------------------- 1 file changed, 60 insertions(+), 105 deletions(-) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 1126759317..929e8570be 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -59,105 +59,114 @@ type Prometheus struct { func NewPrometheus(opts Options) *Prometheus { opts = applyCompatibilityDefaults(opts) + p := &Prometheus{ + registry: opts.PrometheusRegistry, + opts: opts, + } + + if p.registry == nil { + p.registry = prometheus.NewRegistry() + } + namespace := promNamespace if opts.Prefix != "" { namespace = strings.TrimSuffix(opts.Prefix, ".") } - routeLookup := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.routeLookupM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promRouteSubsystem, Name: "lookup_duration_seconds", Help: "Duration in seconds of a route lookup.", Buckets: opts.HistogramBuckets, - }, []string{}) + }, []string{})) - routeErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ + p.routeErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promRouteSubsystem, Name: "error_total", Help: "The total of route lookup errors.", - }, []string{}) + }, []string{})) - response := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.responseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promResponseSubsystem, Name: "duration_seconds", Help: "Duration in seconds of a response.", Buckets: opts.HistogramBuckets, - }, []string{"code", "method", "route"}) + }, []string{"code", "method", "route"})) - filterCreate := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterCreateM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "create_duration_seconds", Help: "Duration in seconds of filter creation.", Buckets: opts.HistogramBuckets, - }, []string{"filter"}) + }, []string{"filter"})) - filterRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "request_duration_seconds", Help: "Duration in seconds of a filter request.", Buckets: opts.HistogramBuckets, - }, []string{"filter"}) + }, []string{"filter"})) - filterAllRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterAllRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "all_request_duration_seconds", Help: "Duration in seconds of a filter request by all filters.", Buckets: opts.HistogramBuckets, - }, []string{"route"}) + }, []string{"route"})) - filterAllCombinedRequest := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterAllCombinedRequestM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "all_combined_request_duration_seconds", Help: "Duration in seconds of a filter request combined by all filters.", Buckets: opts.HistogramBuckets, - }, []string{}) + }, []string{})) - proxyBackend := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.proxyBackendM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promProxySubsystem, Name: "duration_seconds", Help: "Duration in seconds of a proxy backend.", Buckets: opts.HistogramBuckets, - }, []string{"route", "host"}) + }, []string{"route", "host"})) - proxyBackendCombined := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.proxyBackendCombinedM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promProxySubsystem, Name: "combined_duration_seconds", Help: "Duration in seconds of a proxy backend combined.", Buckets: opts.HistogramBuckets, - }, []string{}) + }, []string{})) - filterResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "response_duration_seconds", Help: "Duration in seconds of a filter request.", Buckets: opts.HistogramBuckets, - }, []string{"filter"}) + }, []string{"filter"})) - filterAllResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterAllResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "all_response_duration_seconds", Help: "Duration in seconds of a filter response by all filters.", Buckets: opts.HistogramBuckets, - }, []string{"route"}) + }, []string{"route"})) - filterAllCombinedResponse := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.filterAllCombinedResponseM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promFilterSubsystem, Name: "all_combined_response_duration_seconds", Help: "Duration in seconds of a filter response combined by all filters.", Buckets: opts.HistogramBuckets, - }, []string{}) + }, []string{})) metrics := []string{} if opts.EnableServeStatusCodeMetric { @@ -166,108 +175,80 @@ func NewPrometheus(opts Options) *Prometheus { if opts.EnableServeMethodMetric { metrics = append(metrics, "method") } - serveRoute := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.serveRouteM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promServeSubsystem, Name: "route_duration_seconds", Help: "Duration in seconds of serving a route.", Buckets: opts.HistogramBuckets, - }, append(metrics, "route")) - serveRouteCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + }, append(metrics, "route"))) + p.serveRouteCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promServeSubsystem, Name: "route_count", Help: "Total number of requests of serving a route.", - }, []string{"code", "method", "route"}) + }, []string{"code", "method", "route"})) - serveHost := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.serveHostM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promServeSubsystem, Name: "host_duration_seconds", Help: "Duration in seconds of serving a host.", Buckets: opts.HistogramBuckets, - }, append(metrics, "host")) - serveHostCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + }, append(metrics, "host"))) + p.serveHostCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promServeSubsystem, Name: "host_count", Help: "Total number of requests of serving a host.", - }, []string{"code", "method", "host"}) + }, []string{"code", "method", "host"})) - proxyBackend5xx := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + p.proxyBackend5xxM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promProxySubsystem, Name: "5xx_duration_seconds", Help: "Duration in seconds of backend 5xx.", Buckets: opts.HistogramBuckets, - }, []string{}) - proxyBackendErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ + }, []string{})) + p.proxyBackendErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promProxySubsystem, Name: "error_total", Help: "Total number of backend route errors.", - }, []string{"route"}) - proxyStreamingErrors := prometheus.NewCounterVec(prometheus.CounterOpts{ + }, []string{"route"})) + p.proxyStreamingErrorsM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promStreamingSubsystem, Name: "error_total", Help: "Total number of streaming route errors.", - }, []string{"route"}) + }, []string{"route"})) - customCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + p.customCounterM = register(p, prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: namespace, Subsystem: promCustomSubsystem, Name: "total", Help: "Total number of custom metrics.", - }, []string{"key"}) - customGauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + }, []string{"key"})) + p.customGaugeM = register(p, prometheus.NewGaugeVec(prometheus.GaugeOpts{ Namespace: namespace, Subsystem: promCustomSubsystem, Name: "gauges", Help: "Gauges number of custom metrics.", - }, []string{"key"}) - customHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + }, []string{"key"})) + p.customHistogramM = register(p, prometheus.NewHistogramVec(prometheus.HistogramOpts{ Namespace: namespace, Subsystem: promCustomSubsystem, Name: "duration_seconds", Help: "Duration in seconds of custom metrics.", Buckets: opts.HistogramBuckets, - }, []string{"key"}) - - p := &Prometheus{ - routeLookupM: routeLookup, - routeErrorsM: routeErrors, - responseM: response, - filterCreateM: filterCreate, - filterRequestM: filterRequest, - filterAllRequestM: filterAllRequest, - filterAllCombinedRequestM: filterAllCombinedRequest, - proxyBackendM: proxyBackend, - proxyBackendCombinedM: proxyBackendCombined, - filterResponseM: filterResponse, - filterAllResponseM: filterAllResponse, - filterAllCombinedResponseM: filterAllCombinedResponse, - serveRouteM: serveRoute, - serveRouteCounterM: serveRouteCounter, - serveHostM: serveHost, - serveHostCounterM: serveHostCounter, - proxyBackend5xxM: proxyBackend5xx, - proxyBackendErrorsM: proxyBackendErrors, - proxyStreamingErrorsM: proxyStreamingErrors, - customCounterM: customCounter, - customGaugeM: customGauge, - customHistogramM: customHistogram, + }, []string{"key"})) - registry: opts.PrometheusRegistry, - opts: opts, - } - - if p.registry == nil { - p.registry = prometheus.NewRegistry() + // Register prometheus runtime collectors if required. + if opts.EnableRuntimeMetrics { + register(p, collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) + register(p, collectors.NewGoCollector()) } - // Register all metrics. - p.registerMetrics() return p } @@ -276,35 +257,9 @@ func (p *Prometheus) sinceS(start time.Time) float64 { return time.Since(start).Seconds() } -func (p *Prometheus) registerMetrics() { - p.registry.MustRegister(p.routeLookupM) - p.registry.MustRegister(p.responseM) - p.registry.MustRegister(p.routeErrorsM) - p.registry.MustRegister(p.filterCreateM) - p.registry.MustRegister(p.filterRequestM) - p.registry.MustRegister(p.filterAllRequestM) - p.registry.MustRegister(p.filterAllCombinedRequestM) - p.registry.MustRegister(p.proxyBackendM) - p.registry.MustRegister(p.proxyBackendCombinedM) - p.registry.MustRegister(p.filterResponseM) - p.registry.MustRegister(p.filterAllResponseM) - p.registry.MustRegister(p.filterAllCombinedResponseM) - p.registry.MustRegister(p.serveRouteM) - p.registry.MustRegister(p.serveRouteCounterM) - p.registry.MustRegister(p.serveHostM) - p.registry.MustRegister(p.serveHostCounterM) - p.registry.MustRegister(p.proxyBackend5xxM) - p.registry.MustRegister(p.proxyBackendErrorsM) - p.registry.MustRegister(p.proxyStreamingErrorsM) - p.registry.MustRegister(p.customCounterM) - p.registry.MustRegister(p.customHistogramM) - p.registry.MustRegister(p.customGaugeM) - - // Register prometheus runtime collectors if required. - if p.opts.EnableRuntimeMetrics { - p.registry.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{})) - p.registry.MustRegister(collectors.NewGoCollector()) - } +func register[T prometheus.Collector](p *Prometheus, cs T) T { + p.registry.MustRegister(cs) + return cs } func (p *Prometheus) CreateHandler() http.Handler {