From 2abc6d30077e12b7dd7a5895cd11f990b66e4cd0 Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Wed, 12 Feb 2025 14:18:30 +0100 Subject: [PATCH] [chore] fix flaky e2e test in k8sclusterreceiver (#37830) #### Description This PR addresses the flaky E2E test in the k8sclusterreceiver. Will need to run this multiple times to make sure this solves the issue #### Link to tracking issue Fixes #37756 #### Testing Adapted the assertion logic in the existing test --------- Signed-off-by: Florian Bacher --- receiver/k8sclusterreceiver/e2e_test.go | 168 ++++++++++++------------ 1 file changed, 86 insertions(+), 82 deletions(-) diff --git a/receiver/k8sclusterreceiver/e2e_test.go b/receiver/k8sclusterreceiver/e2e_test.go index 09682f1e7ff9..3dbb239b948e 100644 --- a/receiver/k8sclusterreceiver/e2e_test.go +++ b/receiver/k8sclusterreceiver/e2e_test.go @@ -76,47 +76,49 @@ func TestE2EClusterScoped(t *testing.T) { // golden.WriteMetrics(t, expectedFileClusterScoped, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1]) waitForData(t, wantEntries, metricsConsumer) - require.NoError(t, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1], - pmetrictest.IgnoreTimestamp(), - pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreMetricValues( - "k8s.container.cpu_request", - "k8s.container.memory_limit", - "k8s.container.memory_request", - "k8s.container.restarts", - "k8s.cronjob.active_jobs", - "k8s.deployment.available", - "k8s.deployment.desired", - "k8s.job.active_pods", - "k8s.job.desired_successful_pods", - "k8s.job.failed_pods", - "k8s.job.max_parallel_pods", - "k8s.hpa.current_replicas", - "k8s.job.successful_pods"), - pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten), - pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar), - pmetrictest.IgnoreScopeVersion(), - pmetrictest.IgnoreResourceMetricsOrder(), - pmetrictest.IgnoreMetricsOrder(), - pmetrictest.IgnoreScopeMetricsOrder(), - pmetrictest.IgnoreMetricDataPointsOrder(), - ), - ) + require.EventuallyWithT(t, func(tt *assert.CollectT) { + assert.NoError(tt, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1], + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreMetricValues( + "k8s.container.cpu_request", + "k8s.container.memory_limit", + "k8s.container.memory_request", + "k8s.container.restarts", + "k8s.cronjob.active_jobs", + "k8s.deployment.available", + "k8s.deployment.desired", + "k8s.job.active_pods", + "k8s.job.desired_successful_pods", + "k8s.job.failed_pods", + "k8s.job.max_parallel_pods", + "k8s.hpa.current_replicas", + "k8s.job.successful_pods"), + pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten), + pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar), + pmetrictest.IgnoreScopeVersion(), + pmetrictest.IgnoreResourceMetricsOrder(), + pmetrictest.IgnoreMetricsOrder(), + pmetrictest.IgnoreScopeMetricsOrder(), + pmetrictest.IgnoreMetricDataPointsOrder(), + ), + ) + }, 3*time.Minute, 1*time.Second) } // TestE2ENamespaceScoped tests the k8s cluster receiver with a real k8s cluster. @@ -160,47 +162,49 @@ func TestE2ENamespaceScoped(t *testing.T) { // golden.WriteMetrics(t, expectedFileNamespaceScoped, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1]) waitForData(t, wantEntries, metricsConsumer) - require.NoError(t, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1], - pmetrictest.IgnoreTimestamp(), - pmetrictest.IgnoreStartTimestamp(), - pmetrictest.IgnoreMetricValues( - "k8s.container.cpu_request", - "k8s.container.memory_limit", - "k8s.container.memory_request", - "k8s.container.restarts", - "k8s.cronjob.active_jobs", - "k8s.deployment.available", - "k8s.deployment.desired", - "k8s.job.active_pods", - "k8s.job.desired_successful_pods", - "k8s.job.failed_pods", - "k8s.job.max_parallel_pods", - "k8s.hpa.current_replicas", - "k8s.job.successful_pods"), - pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten), - pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames), - pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar), - pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar), - pmetrictest.IgnoreScopeVersion(), - pmetrictest.IgnoreResourceMetricsOrder(), - pmetrictest.IgnoreMetricsOrder(), - pmetrictest.IgnoreScopeMetricsOrder(), - pmetrictest.IgnoreMetricDataPointsOrder(), - ), - ) + require.EventuallyWithT(t, func(tt *assert.CollectT) { + assert.NoError(tt, pmetrictest.CompareMetrics(expected, metricsConsumer.AllMetrics()[len(metricsConsumer.AllMetrics())-1], + pmetrictest.IgnoreTimestamp(), + pmetrictest.IgnoreStartTimestamp(), + pmetrictest.IgnoreMetricValues( + "k8s.container.cpu_request", + "k8s.container.memory_limit", + "k8s.container.memory_request", + "k8s.container.restarts", + "k8s.cronjob.active_jobs", + "k8s.deployment.available", + "k8s.deployment.desired", + "k8s.job.active_pods", + "k8s.job.desired_successful_pods", + "k8s.job.failed_pods", + "k8s.job.max_parallel_pods", + "k8s.hpa.current_replicas", + "k8s.job.successful_pods"), + pmetrictest.ChangeResourceAttributeValue("container.id", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("container.image.name", containerImageShorten), + pmetrictest.ChangeResourceAttributeValue("container.image.tag", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.cronjob.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.daemonset.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.deployment.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.deployment.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.hpa.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.job.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.job.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.namespace.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.node.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.pod.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.pod.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.name", shortenNames), + pmetrictest.ChangeResourceAttributeValue("k8s.replicaset.uid", replaceWithStar), + pmetrictest.ChangeResourceAttributeValue("k8s.statefulset.uid", replaceWithStar), + pmetrictest.IgnoreScopeVersion(), + pmetrictest.IgnoreResourceMetricsOrder(), + pmetrictest.IgnoreMetricsOrder(), + pmetrictest.IgnoreScopeMetricsOrder(), + pmetrictest.IgnoreMetricDataPointsOrder(), + ), + ) + }, 3*time.Minute, 1*time.Second) } func shortenNames(value string) string {