diff --git a/test/e2e/logs_self_monitor_healthy_test.go b/test/e2e/logs_self_monitor_healthy_test.go index 3ad696705..d7c239bb1 100644 --- a/test/e2e/logs_self_monitor_healthy_test.go +++ b/test/e2e/logs_self_monitor_healthy_test.go @@ -101,17 +101,18 @@ var _ = Describe(suite.ID(), Label(suite.LabelSelfMonitoringLogsHealthy), Ordere // Pushing metrics to the metric gateway triggers an alert. // It makes the self-monitor call the webhook, which in turn increases the counter. assert.ManagerEmitsMetric(proxyClient, - Equal("controller_runtime_webhook_requests_total"), + HaveName(Equal("controller_runtime_webhook_requests_total")), SatisfyAll( - WithLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), - WithValue(BeNumerically(">", 0)), + HaveLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), + HaveMetricValue(BeNumerically(">", 0)), )) }) It("Ensures that telemetry_self_monitor_prober_requests_total is emitted", func() { - assert.ManagerEmitsMetric(proxyClient, - Equal("telemetry_self_monitor_prober_requests_total"), - WithValue(BeNumerically(">", 0)), + assert.ManagerEmitsMetric( + proxyClient, + HaveName(Equal("telemetry_self_monitor_prober_requests_total")), + HaveMetricValue(BeNumerically(">", 0)), ) }) }) diff --git a/test/e2e/metrics_self_monitor_healthy_test.go b/test/e2e/metrics_self_monitor_healthy_test.go index 10f7b8e14..933f32893 100644 --- a/test/e2e/metrics_self_monitor_healthy_test.go +++ b/test/e2e/metrics_self_monitor_healthy_test.go @@ -124,17 +124,17 @@ var _ = Describe(suite.ID(), Label(suite.LabelSelfMonitoringMetricsHealthy), Ord // Pushing metrics to the metric gateway triggers an alert. // It makes the self-monitor call the webhook, which in turn increases the counter. assert.ManagerEmitsMetric(proxyClient, - Equal("controller_runtime_webhook_requests_total"), + HaveName(Equal("controller_runtime_webhook_requests_total")), SatisfyAll( - WithLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), - WithValue(BeNumerically(">", 0)), + HaveLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), + HaveMetricValue(BeNumerically(">", 0)), )) }) It("Ensures that telemetry_self_monitor_prober_requests_total is emitted", func() { assert.ManagerEmitsMetric(proxyClient, - Equal("telemetry_self_monitor_prober_requests_total"), - WithValue(BeNumerically(">", 0)), + HaveName(Equal("telemetry_self_monitor_prober_requests_total")), + HaveMetricValue(BeNumerically(">", 0)), ) }) }) diff --git a/test/e2e/traces_self_monitor_healthy_test.go b/test/e2e/traces_self_monitor_healthy_test.go index f73986489..e1957810a 100644 --- a/test/e2e/traces_self_monitor_healthy_test.go +++ b/test/e2e/traces_self_monitor_healthy_test.go @@ -119,17 +119,17 @@ var _ = Describe(suite.ID(), Label(suite.LabelSelfMonitoringTracesHealthy), Orde // Pushing metrics to the metric gateway triggers an alert. // It makes the self-monitor call the webhook, which in turn increases the counter. assert.ManagerEmitsMetric(proxyClient, - Equal("controller_runtime_webhook_requests_total"), + HaveName(Equal("controller_runtime_webhook_requests_total")), SatisfyAll( - WithLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), - WithValue(BeNumerically(">", 0)), + HaveLabels(HaveKeyWithValue("webhook", "/api/v2/alerts")), + HaveMetricValue(BeNumerically(">", 0)), )) }) It("Ensures that telemetry_self_monitor_prober_requests_total is emitted", func() { assert.ManagerEmitsMetric(proxyClient, - Equal("telemetry_self_monitor_prober_requests_total"), - WithValue(BeNumerically(">", 0)), + HaveName(Equal("telemetry_self_monitor_prober_requests_total")), + HaveMetricValue(BeNumerically(">", 0)), ) }) }) diff --git a/test/testkit/assert/monitoring.go b/test/testkit/assert/monitoring.go index a9f4d42f7..051416849 100644 --- a/test/testkit/assert/monitoring.go +++ b/test/testkit/assert/monitoring.go @@ -18,7 +18,7 @@ func EmitsOTelCollectorMetrics(proxyClient *apiserverproxy.Client, metricsURL st resp, err := proxyClient.Get(metricsURL) g.Expect(err).NotTo(HaveOccurred()) g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) - g.Expect(resp).To(HaveHTTPBody(ContainMetricFamily(WithName(ContainSubstring("otelcol"))))) + g.Expect(resp).To(HaveHTTPBody(HaveFlatMetricFamilies(ContainElement(HaveName(ContainSubstring("otelcol")))))) err = resp.Body.Close() g.Expect(err).NotTo(HaveOccurred()) @@ -27,8 +27,7 @@ func EmitsOTelCollectorMetrics(proxyClient *apiserverproxy.Client, metricsURL st func ManagerEmitsMetric( proxyClient *apiserverproxy.Client, - nameMatcher types.GomegaMatcher, - metricMatcher types.GomegaMatcher) { + matchers ...types.GomegaMatcher) { Eventually(func(g Gomega) { telemetryManagerMetricsURL := proxyClient.ProxyURLForService( kitkyma.TelemetryManagerMetricsServiceName.Namespace, @@ -39,10 +38,7 @@ func ManagerEmitsMetric( g.Expect(err).NotTo(HaveOccurred()) g.Expect(resp).To(HaveHTTPStatus(http.StatusOK)) - g.Expect(resp).To(HaveHTTPBody(ContainMetricFamily(SatisfyAll( - WithName(nameMatcher), - ContainMetric(metricMatcher), - )))) + g.Expect(resp).To(HaveHTTPBody(HaveFlatMetricFamilies(ContainElement(SatisfyAll(matchers...))))) err = resp.Body.Close() g.Expect(err).NotTo(HaveOccurred()) diff --git a/test/testkit/matchers/prometheus/prometheus_matcher.go b/test/testkit/matchers/prometheus/prometheus_matcher.go index 294e836bf..76f487c93 100644 --- a/test/testkit/matchers/prometheus/prometheus_matcher.go +++ b/test/testkit/matchers/prometheus/prometheus_matcher.go @@ -5,70 +5,34 @@ import ( "github.com/onsi/gomega" "github.com/onsi/gomega/types" - prommodel "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" ) -func WithMetricFamilies(matcher types.GomegaMatcher) types.GomegaMatcher { - return gomega.WithTransform(func(responseBody []byte) ([]*prommodel.MetricFamily, error) { +func HaveFlatMetricFamilies(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(responseBody []byte) ([]FlatMetricFamily, error) { var parser expfmt.TextParser mfs, _ := parser.TextToMetricFamilies(bytes.NewReader(responseBody)) //nolint:errcheck // ignore duplicate metrics parsing error and try extract metric - var result []*prommodel.MetricFamily + fmfs := flattenAllMetricFamilies(mfs) - for _, mf := range mfs { - result = append(result, mf) - } - - return result, nil - }, matcher) -} - -func ContainMetricFamily(matcher types.GomegaMatcher) types.GomegaMatcher { - return WithMetricFamilies(gomega.ContainElement(matcher)) -} - -func WithName(matcher types.GomegaMatcher) types.GomegaMatcher { - return gomega.WithTransform(func(mf *prommodel.MetricFamily) string { - return mf.GetName() + return fmfs, nil }, matcher) } -func WithMetrics(matcher types.GomegaMatcher) types.GomegaMatcher { - return gomega.WithTransform(func(mf *prommodel.MetricFamily) []*prommodel.Metric { - return mf.GetMetric() +func HaveName(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(fmf FlatMetricFamily) string { + return fmf.Name }, matcher) } -func ContainMetric(matcher types.GomegaMatcher) types.GomegaMatcher { - return WithMetrics(gomega.ContainElement(matcher)) -} - -func WithValue(matcher types.GomegaMatcher) types.GomegaMatcher { - return gomega.WithTransform(func(m *prommodel.Metric) (float64, error) { - if m.Gauge != nil { - return m.Gauge.GetValue(), nil - } - - if m.Counter != nil { - return m.Counter.GetValue(), nil - } - - if m.Untyped != nil { - return m.Untyped.GetValue(), nil - } - - return 0, nil +func HaveMetricValue(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(fmf FlatMetricFamily) float64 { + return fmf.MetricValue }, matcher) } -func WithLabels(matcher types.GomegaMatcher) types.GomegaMatcher { - return gomega.WithTransform(func(m *prommodel.Metric) (map[string]string, error) { - labels := make(map[string]string) - for _, l := range m.Label { - labels[l.GetName()] = l.GetValue() - } - - return labels, nil +func HaveLabels(matcher types.GomegaMatcher) types.GomegaMatcher { + return gomega.WithTransform(func(fmf FlatMetricFamily) map[string]string { + return fmf.Labels }, matcher) } diff --git a/test/testkit/matchers/prometheus/prometheus_matcher_test.go b/test/testkit/matchers/prometheus/prometheus_matcher_test.go index ee5cec712..d5656ce4b 100644 --- a/test/testkit/matchers/prometheus/prometheus_matcher_test.go +++ b/test/testkit/matchers/prometheus/prometheus_matcher_test.go @@ -5,10 +5,10 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("ContainMetric", Label("metrics"), func() { +var _ = Describe("HaveFlatMetricFamilies", Label("metrics"), func() { Context("with nil input", func() { - It("should fail", func() { - success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match(nil) + It("should error", func() { + success, err := HaveFlatMetricFamilies(ContainElement(HaveName(Equal("foo_metric")))).Match(nil) Expect(err).Should(HaveOccurred()) Expect(success).Should(BeFalse()) }) @@ -16,7 +16,7 @@ var _ = Describe("ContainMetric", Label("metrics"), func() { Context("with empty input", func() { It("should fail", func() { - success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match([]byte{}) + success, err := HaveFlatMetricFamilies(ContainElement(HaveName(Equal("foo_metric")))).Match([]byte{}) Expect(err).ShouldNot(HaveOccurred()) Expect(success).Should(BeFalse()) }) @@ -24,14 +24,14 @@ var _ = Describe("ContainMetric", Label("metrics"), func() { Context("with invalid input", func() { It("should fail", func() { - success, err := ContainMetricFamily(WithName(Equal("foo_metric"))).Match([]byte{1, 2, 3}) + success, err := HaveFlatMetricFamilies(ContainElement(HaveName(Equal("foo_metric")))).Match([]byte{1, 2, 3}) Expect(err).ShouldNot(HaveOccurred()) Expect(success).Should(BeFalse()) }) }) - Context("with having metrics", func() { - It("should succeed", func() { + Context("with HaveName", func() { + It("should apply matcher", func() { fileBytes := ` # HELP fluentbit_uptime Number of seconds that Fluent Bit has been running. # TYPE fluentbit_uptime counter @@ -39,12 +39,12 @@ fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 # HELP fluentbit_input_bytes_total Number of input bytes. # TYPE fluentbit_input_bytes_total counter fluentbit_input_bytes_total{name="tele-tail"} 5217998` - Expect([]byte(fileBytes)).Should(ContainMetricFamily(WithName(Equal("fluentbit_uptime")))) + Expect([]byte(fileBytes)).Should(HaveFlatMetricFamilies(ContainElement(HaveName(Equal("fluentbit_uptime"))))) }) }) }) -var _ = Describe("WithLabels", func() { +var _ = Describe("with HaveLabels", func() { It("should apply matcher", func() { fileBytes := ` # HELP fluentbit_uptime Number of seconds that Fluent Bit has been running. @@ -54,14 +54,14 @@ fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 # TYPE fluentbit_input_bytes_total counter fluentbit_input_bytes_total{name="tele-tail"} 5000 ` - Expect([]byte(fileBytes)).Should(ContainMetricFamily(SatisfyAll( - WithName(Equal("fluentbit_input_bytes_total")), - ContainMetric(WithLabels(HaveKeyWithValue("name", "tele-tail"))), - ))) + Expect([]byte(fileBytes)).Should(HaveFlatMetricFamilies(ContainElement(SatisfyAll( + HaveName(Equal("fluentbit_input_bytes_total")), + HaveLabels(HaveKeyWithValue("name", "tele-tail")), + )))) }) }) -var _ = Describe("WithValue", func() { +var _ = Describe("with HaveValue", func() { It("should apply matcher", func() { fileBytes := ` # HELP fluentbit_uptime Number of seconds that Fluent Bit has been running. @@ -71,12 +71,10 @@ fluentbit_uptime{hostname="telemetry-fluent-bit-dglkf"} 5489 # TYPE fluentbit_input_bytes_total counter fluentbit_input_bytes_total{name="tele-tail"} 5000 ` - Expect([]byte(fileBytes)).Should(ContainMetricFamily(SatisfyAll( - WithName(Equal("fluentbit_input_bytes_total")), - ContainMetric(SatisfyAll( - WithLabels(HaveKeyWithValue("name", "tele-tail")), - WithValue(BeNumerically(">=", 0)), - )), - ))) + Expect([]byte(fileBytes)).Should(HaveFlatMetricFamilies(ContainElement(SatisfyAll( + HaveName(Equal("fluentbit_input_bytes_total")), + HaveLabels(HaveKeyWithValue("name", "tele-tail")), + HaveMetricValue(BeNumerically(">=", 0)), + )))) }) }) diff --git a/test/testkit/matchers/prometheus/prommetric_utils.go b/test/testkit/matchers/prometheus/prommetric_utils.go new file mode 100644 index 000000000..d40e42fd6 --- /dev/null +++ b/test/testkit/matchers/prometheus/prommetric_utils.go @@ -0,0 +1,76 @@ +package prometheus + +import prommodel "github.com/prometheus/client_model/go" + +// FlatMetricFamily holds all necessary information about a prometheus MetricFamily. +// Gomega doesn't handle deeply nested data structure very well and generates large, unreadable diffs when paired with +// the deeply nested structure of MetricFamily. +// +// FlatMetricFamily is a flat data structure that provides necessary information from different levels of +// MetricFamily, making accessing the information easier and improves readability of the Gomega output. +type FlatMetricFamily struct { + Name string + MetricType string + MetricValue float64 + Labels map[string]string +} + +// flattenAllMetricFamilies flattens an array of prometheus MetricFamily to a slice of FlatMetricFamily. +// It converts the deeply nested MetricFamily to a flat struct, making it more readable in the test output. +func flattenAllMetricFamilies(mfs map[string]*prommodel.MetricFamily) []FlatMetricFamily { + var fmf []FlatMetricFamily + for _, mf := range mfs { + fmf = append(fmf, flattenMetricFamily(mf)...) + } + + return fmf +} + +// flattenMetricFamily converts a single MetricFamily into a slice of FlatMetricFamily +// It loops through all the metrics in a MetricFamily and appends it to the FlatMetricFamily slice. +func flattenMetricFamily(mf *prommodel.MetricFamily) []FlatMetricFamily { + var fmf []FlatMetricFamily + + for _, m := range mf.Metric { + t, v := getTypeAndValuePerMetric(m) + fmf = append(fmf, FlatMetricFamily{ + Name: mf.GetName(), + MetricType: t, + MetricValue: v, + Labels: labelsToMap(m.GetLabel()), + }) + } + + return fmf +} + +func labelsToMap(l []*prommodel.LabelPair) map[string]string { + labels := make(map[string]string) + for _, l := range l { + labels[l.GetName()] = l.GetValue() + } + + return labels +} + +const ( + gaugeType = "Gauge" + counterType = "Counter" + untypedType = "Untyped" +) + +func getTypeAndValuePerMetric(m *prommodel.Metric) (string, float64) { + if m.Gauge != nil { + return gaugeType, m.Gauge.GetValue() + } + + if m.Counter != nil { + return counterType, m.Counter.GetValue() + } + + if m.Untyped != nil { + return untypedType, m.Untyped.GetValue() + } + + return "", 0 +}