From fc6811c7c0e5060f10ad9f4fb89d056561195a4e Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Wed, 29 Jan 2025 16:27:18 -0800 Subject: [PATCH 1/4] try to match against metric received by the sink --- tests/receivers/envoy/bundled_k8s_test.go | 63 ++++++++++++----------- tests/testutils/golden.go | 14 +++-- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/tests/receivers/envoy/bundled_k8s_test.go b/tests/receivers/envoy/bundled_k8s_test.go index 4d3480e8bd..1effa6373e 100644 --- a/tests/receivers/envoy/bundled_k8s_test.go +++ b/tests/receivers/envoy/bundled_k8s_test.go @@ -18,6 +18,7 @@ package tests import ( "context" + "errors" "fmt" "os" "path/filepath" @@ -124,39 +125,43 @@ func TestEnvoyK8sObserver(t *testing.T) { expected, err := golden.ReadMetrics(filepath.Join("testdata", "expected_k8s.yaml")) require.NoError(t, err) - i := 0 require.EventuallyWithT(t, func(tt *assert.CollectT) { if len(sink.AllMetrics()) == 0 { assert.Fail(tt, "No metrics collected") return } - - i++ - err := pmetrictest.CompareMetrics(expected, sink.AllMetrics()[len(sink.AllMetrics())-1], - pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), - pmetrictest.IgnoreResourceAttributeValue("net.host.port"), - pmetrictest.IgnoreResourceAttributeValue("net.host.name"), - pmetrictest.IgnoreResourceAttributeValue("server.address"), - pmetrictest.IgnoreResourceAttributeValue("container.name"), - pmetrictest.IgnoreResourceAttributeValue("server.port"), - pmetrictest.IgnoreResourceAttributeValue("service.name"), - pmetrictest.IgnoreResourceAttributeValue("service_instance_id"), - pmetrictest.IgnoreResourceAttributeValue("service_version"), - pmetrictest.IgnoreResourceAttributeValue("discovery.endpoint.id"), - pmetrictest.IgnoreMetricAttributeValue("service_version"), - pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), - pmetrictest.IgnoreResourceAttributeValue("server.address"), - pmetrictest.IgnoreResourceAttributeValue("k8s.pod.name"), - pmetrictest.IgnoreResourceAttributeValue("k8s.pod.uid"), - pmetrictest.IgnoreTimestamp(), - pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreMetricDataPointsOrder(), - pmetrictest.IgnoreScopeMetricsOrder(), - pmetrictest.IgnoreScopeVersion(), - pmetrictest.IgnoreResourceMetricsOrder(), - pmetrictest.IgnoreMetricsOrder(), - pmetrictest.IgnoreMetricValues(), - ) - assert.NoError(tt, err) + var errs error + for _, m := range sink.AllMetrics() { + err := pmetrictest.CompareMetrics(expected, m, + pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), + pmetrictest.IgnoreResourceAttributeValue("net.host.port"), + pmetrictest.IgnoreResourceAttributeValue("net.host.name"), + pmetrictest.IgnoreResourceAttributeValue("server.address"), + pmetrictest.IgnoreResourceAttributeValue("container.name"), + pmetrictest.IgnoreResourceAttributeValue("server.port"), + pmetrictest.IgnoreResourceAttributeValue("service.name"), + pmetrictest.IgnoreResourceAttributeValue("service_instance_id"), + pmetrictest.IgnoreResourceAttributeValue("service_version"), + pmetrictest.IgnoreResourceAttributeValue("discovery.endpoint.id"), + pmetrictest.IgnoreMetricAttributeValue("service_version"), + pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), + pmetrictest.IgnoreResourceAttributeValue("server.address"), + pmetrictest.IgnoreResourceAttributeValue("k8s.pod.name"), + pmetrictest.IgnoreResourceAttributeValue("k8s.pod.uid"), + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreScopeMetricsOrder(), + pmetrictest.IgnoreScopeVersion(), + pmetrictest.IgnoreResourceMetricsOrder(), + pmetrictest.IgnoreMetricsOrder(), + pmetrictest.IgnoreMetricValues(), + ) + if err == nil { + return + } + errs = errors.Join(errs, err) + } + assert.NoError(tt, errs) }, 120*time.Second, 1*time.Second) } diff --git a/tests/testutils/golden.go b/tests/testutils/golden.go index 587a231a3d..d975797904 100644 --- a/tests/testutils/golden.go +++ b/tests/testutils/golden.go @@ -17,6 +17,7 @@ package testutils import ( "context" + "errors" "fmt" "path/filepath" "runtime" @@ -128,8 +129,15 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath assert.Fail(tt, "No metrics collected") return } - err := pmetrictest.CompareMetrics(expected, sink.AllMetrics()[len(sink.AllMetrics())-1], - opts.compareMetricsOptions...) - assert.NoError(tt, err) + var errs error + for _, m := range sink.AllMetrics() { + err := pmetrictest.CompareMetrics(expected, m, + opts.compareMetricsOptions...) + if err == nil { + return + } + errs = errors.Join(errs, err) + } + assert.NoError(tt, errs) }, 30*time.Second, 1*time.Second) } From 3338e6f1e6c783b56bab0fb117b2a84fdebf5ea7 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Wed, 29 Jan 2025 17:06:36 -0800 Subject: [PATCH 2/4] flexible match on postgres test as well --- .../smartagent/postgresql/postgresql_test.go | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/tests/receivers/smartagent/postgresql/postgresql_test.go b/tests/receivers/smartagent/postgresql/postgresql_test.go index a20e38a13a..47abc337e7 100644 --- a/tests/receivers/smartagent/postgresql/postgresql_test.go +++ b/tests/receivers/smartagent/postgresql/postgresql_test.go @@ -17,6 +17,7 @@ package tests import ( + "errors" "path" "path/filepath" "testing" @@ -26,7 +27,6 @@ import ( "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/pdatatest/pmetrictest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/pdata/pmetric" "github.com/signalfx/splunk-otel-collector/tests/testutils" ) @@ -60,38 +60,37 @@ func TestPostgresReceiverProvidesAllMetrics(t *testing.T) { assert.Fail(tt, "No metrics collected") return } - var selected *pmetric.Metrics + var errs error for i := len(tc.OTLPReceiverSink.AllMetrics()) - 1; i >= 0; i-- { m := tc.OTLPReceiverSink.AllMetrics()[i] if m.MetricCount() == expected.MetricCount() { - selected = &m - break + err := pmetrictest.CompareMetrics(expected, m, + pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), + pmetrictest.IgnoreResourceAttributeValue("net.host.port"), + pmetrictest.IgnoreResourceAttributeValue("server.port"), + pmetrictest.IgnoreResourceAttributeValue("service.name"), + pmetrictest.IgnoreResourceAttributeValue("service_instance_id"), + pmetrictest.IgnoreResourceAttributeValue("service_version"), + pmetrictest.IgnoreMetricAttributeValue("service_version"), + pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), + pmetrictest.IgnoreMetricAttributeValue("queryid"), + pmetrictest.IgnoreMetricAttributeValue("table"), + pmetrictest.IgnoreSubsequentDataPoints(), + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreMetricDataPointsOrder(), + pmetrictest.IgnoreScopeMetricsOrder(), + pmetrictest.IgnoreMetricsOrder(), + pmetrictest.IgnoreScopeVersion(), + pmetrictest.IgnoreResourceMetricsOrder(), + pmetrictest.IgnoreMetricValues(), + ) + if err == nil { + return + } + errs = errors.Join(errs, err) } } - - require.NotNil(tt, selected) - - err := pmetrictest.CompareMetrics(expected, *selected, - pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), - pmetrictest.IgnoreResourceAttributeValue("net.host.port"), - pmetrictest.IgnoreResourceAttributeValue("server.port"), - pmetrictest.IgnoreResourceAttributeValue("service.name"), - pmetrictest.IgnoreResourceAttributeValue("service_instance_id"), - pmetrictest.IgnoreResourceAttributeValue("service_version"), - pmetrictest.IgnoreMetricAttributeValue("service_version"), - pmetrictest.IgnoreMetricAttributeValue("service_instance_id"), - pmetrictest.IgnoreMetricAttributeValue("queryid"), - pmetrictest.IgnoreMetricAttributeValue("table"), - pmetrictest.IgnoreSubsequentDataPoints(), - pmetrictest.IgnoreTimestamp(), - pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreMetricDataPointsOrder(), - pmetrictest.IgnoreScopeMetricsOrder(), - pmetrictest.IgnoreMetricsOrder(), - pmetrictest.IgnoreScopeVersion(), - pmetrictest.IgnoreResourceMetricsOrder(), - pmetrictest.IgnoreMetricValues(), - ) - assert.NoError(tt, err) + assert.NoError(tt, errs) }, 30*time.Second, 1*time.Second) } From 7ce8f15c091a2a2b088921905ce5b00e13ff3e1e Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Thu, 30 Jan 2025 17:22:24 -0800 Subject: [PATCH 3/4] retry only new batches --- tests/receivers/envoy/bundled_k8s_test.go | 5 ++++- tests/receivers/smartagent/postgresql/postgresql_test.go | 5 ++++- tests/testutils/golden.go | 6 +++++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/receivers/envoy/bundled_k8s_test.go b/tests/receivers/envoy/bundled_k8s_test.go index 1effa6373e..92028b6fdc 100644 --- a/tests/receivers/envoy/bundled_k8s_test.go +++ b/tests/receivers/envoy/bundled_k8s_test.go @@ -125,13 +125,15 @@ func TestEnvoyK8sObserver(t *testing.T) { expected, err := golden.ReadMetrics(filepath.Join("testdata", "expected_k8s.yaml")) require.NoError(t, err) + index := 0 require.EventuallyWithT(t, func(tt *assert.CollectT) { if len(sink.AllMetrics()) == 0 { assert.Fail(tt, "No metrics collected") return } var errs error - for _, m := range sink.AllMetrics() { + newIndex := len(sink.AllMetrics()) + for i := index; i < newIndex; i++ { err := pmetrictest.CompareMetrics(expected, m, pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), pmetrictest.IgnoreResourceAttributeValue("net.host.port"), @@ -162,6 +164,7 @@ func TestEnvoyK8sObserver(t *testing.T) { } errs = errors.Join(errs, err) } + index = newIndex assert.NoError(tt, errs) }, 120*time.Second, 1*time.Second) } diff --git a/tests/receivers/smartagent/postgresql/postgresql_test.go b/tests/receivers/smartagent/postgresql/postgresql_test.go index 47abc337e7..4583b5b50f 100644 --- a/tests/receivers/smartagent/postgresql/postgresql_test.go +++ b/tests/receivers/smartagent/postgresql/postgresql_test.go @@ -55,13 +55,15 @@ func TestPostgresReceiverProvidesAllMetrics(t *testing.T) { expected, err := golden.ReadMetrics(filepath.Join("testdata", "expected", "all.yaml")) require.NoError(t, err) + lastIndex := 0 assert.EventuallyWithT(t, func(tt *assert.CollectT) { if len(tc.OTLPReceiverSink.AllMetrics()) == 0 { assert.Fail(tt, "No metrics collected") return } var errs error - for i := len(tc.OTLPReceiverSink.AllMetrics()) - 1; i >= 0; i-- { + newIndex := len(tc.OTLPReceiverSink.AllMetrics()) - 1 + for i := newIndex; i >= lastIndex; i-- { m := tc.OTLPReceiverSink.AllMetrics()[i] if m.MetricCount() == expected.MetricCount() { err := pmetrictest.CompareMetrics(expected, m, @@ -91,6 +93,7 @@ func TestPostgresReceiverProvidesAllMetrics(t *testing.T) { errs = errors.Join(errs, err) } } + lastIndex = newIndex assert.NoError(tt, errs) }, 30*time.Second, 1*time.Second) } diff --git a/tests/testutils/golden.go b/tests/testutils/golden.go index d975797904..cf5d45e0d0 100644 --- a/tests/testutils/golden.go +++ b/tests/testutils/golden.go @@ -124,13 +124,16 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath expected, err := golden.ReadMetrics(filepath.Join("testdata", expectedFilePath)) require.NoError(t, err) + index := 0 assert.EventuallyWithT(t, func(tt *assert.CollectT) { if len(sink.AllMetrics()) == 0 { assert.Fail(tt, "No metrics collected") return } var errs error - for _, m := range sink.AllMetrics() { + newIndex := len(sink.AllMetrics()) + for i := index; i < newIndex; i++ { + m := sink.AllMetrics()[i] err := pmetrictest.CompareMetrics(expected, m, opts.compareMetricsOptions...) if err == nil { @@ -138,6 +141,7 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath } errs = errors.Join(errs, err) } + index = newIndex assert.NoError(tt, errs) }, 30*time.Second, 1*time.Second) } From bdf9c07cb2b7c8281d08f42a267fea6836608930 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Fri, 31 Jan 2025 08:23:45 -0800 Subject: [PATCH 4/4] fix and remove errors.Join --- tests/receivers/envoy/bundled_k8s_test.go | 8 +++----- tests/receivers/smartagent/postgresql/postgresql_test.go | 8 +++----- tests/testutils/golden.go | 8 +++----- 3 files changed, 9 insertions(+), 15 deletions(-) diff --git a/tests/receivers/envoy/bundled_k8s_test.go b/tests/receivers/envoy/bundled_k8s_test.go index 92028b6fdc..316b04d182 100644 --- a/tests/receivers/envoy/bundled_k8s_test.go +++ b/tests/receivers/envoy/bundled_k8s_test.go @@ -18,7 +18,6 @@ package tests import ( "context" - "errors" "fmt" "os" "path/filepath" @@ -131,10 +130,10 @@ func TestEnvoyK8sObserver(t *testing.T) { assert.Fail(tt, "No metrics collected") return } - var errs error + var err error newIndex := len(sink.AllMetrics()) for i := index; i < newIndex; i++ { - err := pmetrictest.CompareMetrics(expected, m, + err = pmetrictest.CompareMetrics(expected, sink.AllMetrics()[i], pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), pmetrictest.IgnoreResourceAttributeValue("net.host.port"), pmetrictest.IgnoreResourceAttributeValue("net.host.name"), @@ -162,9 +161,8 @@ func TestEnvoyK8sObserver(t *testing.T) { if err == nil { return } - errs = errors.Join(errs, err) } index = newIndex - assert.NoError(tt, errs) + assert.NoError(tt, err) }, 120*time.Second, 1*time.Second) } diff --git a/tests/receivers/smartagent/postgresql/postgresql_test.go b/tests/receivers/smartagent/postgresql/postgresql_test.go index 4583b5b50f..1ac33ae5ac 100644 --- a/tests/receivers/smartagent/postgresql/postgresql_test.go +++ b/tests/receivers/smartagent/postgresql/postgresql_test.go @@ -17,7 +17,6 @@ package tests import ( - "errors" "path" "path/filepath" "testing" @@ -61,12 +60,12 @@ func TestPostgresReceiverProvidesAllMetrics(t *testing.T) { assert.Fail(tt, "No metrics collected") return } - var errs error + var err error newIndex := len(tc.OTLPReceiverSink.AllMetrics()) - 1 for i := newIndex; i >= lastIndex; i-- { m := tc.OTLPReceiverSink.AllMetrics()[i] if m.MetricCount() == expected.MetricCount() { - err := pmetrictest.CompareMetrics(expected, m, + err = pmetrictest.CompareMetrics(expected, m, pmetrictest.IgnoreResourceAttributeValue("service.instance.id"), pmetrictest.IgnoreResourceAttributeValue("net.host.port"), pmetrictest.IgnoreResourceAttributeValue("server.port"), @@ -90,10 +89,9 @@ func TestPostgresReceiverProvidesAllMetrics(t *testing.T) { if err == nil { return } - errs = errors.Join(errs, err) } } lastIndex = newIndex - assert.NoError(tt, errs) + assert.NoError(tt, err) }, 30*time.Second, 1*time.Second) } diff --git a/tests/testutils/golden.go b/tests/testutils/golden.go index cf5d45e0d0..135e2c3ef9 100644 --- a/tests/testutils/golden.go +++ b/tests/testutils/golden.go @@ -17,7 +17,6 @@ package testutils import ( "context" - "errors" "fmt" "path/filepath" "runtime" @@ -130,18 +129,17 @@ func RunMetricsCollectionTest(t *testing.T, configFile string, expectedFilePath assert.Fail(tt, "No metrics collected") return } - var errs error + var err error newIndex := len(sink.AllMetrics()) for i := index; i < newIndex; i++ { m := sink.AllMetrics()[i] - err := pmetrictest.CompareMetrics(expected, m, + err = pmetrictest.CompareMetrics(expected, m, opts.compareMetricsOptions...) if err == nil { return } - errs = errors.Join(errs, err) } index = newIndex - assert.NoError(tt, errs) + assert.NoError(tt, err) }, 30*time.Second, 1*time.Second) }