test: Fix flakiness in metrics e2e tests (#1543)

Co-authored-by: Korbinian Stoemmer <[email protected]>
kyma-project · Oct 23, 2024 · c0c174d · c0c174d
1 parent 2895ffd
commit c0c174d
Show file tree

Hide file tree

Showing 6 changed files with 41 additions and 25 deletions.
diff --git a/test/e2e/metrics_runtime_input_resources_test.go b/test/e2e/metrics_runtime_input_resources_test.go
@@ -184,7 +184,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetA),
 					g.Expect(resp).To(HaveHTTPBody(
 						HaveFlatMetrics(ContainElement(HaveResourceAttributes(HaveKeys(ConsistOf(runtime.PodMetricsResourceAttributes))))),
 					))
-				}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
+				}, 3*periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 			})
 
 			It("Should have expected metric attributes in runtime pod metrics", func() {
@@ -404,6 +404,11 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetA),
 			assert.ServiceReady(ctx, k8sClient, types.NamespacedName{Name: backendOnlyVolumeMetricsEnabledName, Namespace: mockNs})
 		})
 
+		It("Should have pods mounting volumes running", func() {
+			assert.PodReady(ctx, k8sClient, types.NamespacedName{Name: podMountingPVCName, Namespace: mockNs})
+			assert.PodReady(ctx, k8sClient, types.NamespacedName{Name: podMountingEmptyDirName, Namespace: mockNs})
+		})
+
 		Context("Runtime volume metrics", func() {
 			It("Should deliver ONLY runtime volume metrics to volume-metrics backend", func() {
 				Eventually(func(g Gomega) {
@@ -414,7 +419,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelMetrics), Label(suite.LabelSetA),
 					g.Expect(resp).To(HaveHTTPBody(
 						HaveFlatMetrics(HaveUniqueNamesForRuntimeScope(ConsistOf(runtime.VolumeMetricsNames))),
 					))
-				}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
+				}, 2*periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 			})
 
 			It("Should have expected resource attributes in runtime volume metrics", func() {

diff --git a/test/integration/istio/access_logs_test.go b/test/integration/istio/access_logs_test.go
@@ -76,15 +76,12 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
 		})
 
 		It("Should have sample app running", func() {
-			Eventually(func(g Gomega) {
-				listOptions := client.ListOptions{
-					LabelSelector: labels.SelectorFromSet(map[string]string{"app": "sample-metrics"}),
-					Namespace:     sampleAppNs,
-				}
-				ready, err := assert.PodsReady(ctx, k8sClient, listOptions)
-				g.Expect(err).NotTo(HaveOccurred())
-				g.Expect(ready).To(BeTrueBecause("sample app is not ready"))
-			}, periodic.EventuallyTimeout*2, periodic.DefaultInterval).Should(Succeed())
+			listOptions := client.ListOptions{
+				LabelSelector: labels.SelectorFromSet(map[string]string{"app": "sample-metrics"}),
+				Namespace:     sampleAppNs,
+			}
+
+			assert.PodsReady(ctx, k8sClient, listOptions)
 		})
 
 		It("Should have the log pipeline running", func() {

diff --git a/test/integration/istio/traces_test.go b/test/integration/istio/traces_test.go
@@ -175,17 +175,12 @@ func verifySidecarPresent(namespace string, labelSelector map[string]string) {
 }
 
 func verifyAppIsRunning(namespace string, labelSelector map[string]string) {
-	Eventually(func(g Gomega) {
-		listOptions := client.ListOptions{
-			LabelSelector: labels.SelectorFromSet(labelSelector),
-			Namespace:     namespace,
-		}
-
-		ready, err := assert.PodsReady(ctx, k8sClient, listOptions)
-		g.Expect(err).NotTo(HaveOccurred())
-		g.Expect(ready).To(BeTrueBecause("Pods not ready"))
+	listOptions := client.ListOptions{
+		LabelSelector: labels.SelectorFromSet(labelSelector),
+		Namespace:     namespace,
+	}
 
-	}, periodic.EventuallyTimeout*2, periodic.DefaultInterval).Should(Succeed())
+	assert.PodsReady(ctx, k8sClient, listOptions)
 }
 
 func verifyIstioSpans(backendURL, namespace string) {

diff --git a/test/testkit/assert/daemon_set.go b/test/testkit/assert/daemon_set.go
@@ -34,5 +34,5 @@ func isDaemonSetReady(ctx context.Context, k8sClient client.Client, name types.N
 		Namespace:     name.Namespace,
 	}
 
-	return PodsReady(ctx, k8sClient, listOptions)
+	return arePodsReady(ctx, k8sClient, listOptions)
 }
diff --git a/test/testkit/assert/deployment.go b/test/testkit/assert/deployment.go
@@ -34,7 +34,7 @@ func isDeploymentReady(ctx context.Context, k8sClient client.Client, name types.
 		Namespace:     name.Namespace,
 	}
 
-	return PodsReady(ctx, k8sClient, listOptions)
+	return arePodsReady(ctx, k8sClient, listOptions)
 }
 
 func DeploymentHasPriorityClass(ctx context.Context, k8sClient client.Client, name types.NamespacedName, expectedPriorityClassName string) {

diff --git a/test/testkit/assert/pod.go b/test/testkit/assert/pod.go
@@ -4,12 +4,31 @@ import (
 	"context"
 	"fmt"
 
+	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/kyma-project/telemetry-manager/test/testkit/periodic"
 )
 
-func PodReady(ctx context.Context, k8sClient client.Client, name types.NamespacedName) (bool, error) {
+func PodReady(ctx context.Context, k8sClient client.Client, name types.NamespacedName) {
+	Eventually(func(g Gomega) {
+		ready, err := isPodReady(ctx, k8sClient, name)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(ready).To(BeTrueBecause("Pod not ready"))
+	}, periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed())
+}
+
+func PodsReady(ctx context.Context, k8sClient client.Client, listOptions client.ListOptions) {
+	Eventually(func(g Gomega) {
+		ready, err := arePodsReady(ctx, k8sClient, listOptions)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(ready).To(BeTrueBecause("Pods are not ready"))
+	}, 2*periodic.EventuallyTimeout, periodic.DefaultInterval).Should(Succeed())
+}
+
+func isPodReady(ctx context.Context, k8sClient client.Client, name types.NamespacedName) (bool, error) {
 	var pod corev1.Pod
 
 	err := k8sClient.Get(ctx, name, &pod)
@@ -26,7 +45,7 @@ func PodReady(ctx context.Context, k8sClient client.Client, name types.Namespace
 	return true, nil
 }
 
-func PodsReady(ctx context.Context, k8sClient client.Client, listOptions client.ListOptions) (bool, error) {
+func arePodsReady(ctx context.Context, k8sClient client.Client, listOptions client.ListOptions) (bool, error) {
 	var pods corev1.PodList
 
 	err := k8sClient.List(ctx, &pods, &listOptions)