chore: Improve logs e2e outputs with FlatLogs (#1513)

Co-authored-by: Hisar Balik <[email protected]>

test/e2e/logs_dedot_test.go

@@ -86,7 +86,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					ContainLd(ContainLogRecord(WithKubernetesLabels(HaveKeyWithValue("dedot_label", "logging-dedot-value")))),
+					HaveFlatLogs(ContainElement(HaveKubernetesLabels(HaveKeyWithValue("dedot_label", "logging-dedot-value")))),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/e2e/logs_drop_labels_test.go

@@ -87,8 +87,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ContainLd(ContainLogRecord(
-					WithKubernetesLabels(HaveKeyWithValue("env", "dev")))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(
+					ContainElement(HaveKubernetesLabels(HaveKeyWithValue("env", "dev")))),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
@@ -98,8 +98,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
-					WithKubernetesAnnotations(Not(BeEmpty()))))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(Not(ContainElement(
+					HaveKubernetesAnnotations(Not(BeEmpty()))))),
 				))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/e2e/logs_exclude_container_test.go

@@ -85,7 +85,7 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ContainLd(WithLogRecords(Not(BeEmpty())))))
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(Not(BeEmpty()))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
 
@@ -98,9 +98,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				defer resp.Body.Close()
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(bodyContent).To(Not(BeEmpty()), "No logs found in the backend")
-				g.Expect(bodyContent).To(Not(ContainLd(ContainLogRecord(
-					WithContainerName(ContainSubstring(loggen.DefaultName)))),
-				))
+				g.Expect(bodyContent).To(HaveFlatLogs(Not(ContainElement(
+					HaveContainerName(ContainSubstring(loggen.DefaultName)),
+				))))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
 	})

test/e2e/logs_keep_annotation_test.go

@@ -85,8 +85,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ContainLd(ContainLogRecord(
-					WithKubernetesAnnotations(HaveKeyWithValue("release", "v1.0.0")))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(
+					ContainElement(HaveKubernetesAnnotations(HaveKeyWithValue("release", "v1.0.0"))),
+				),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
@@ -96,8 +97,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
-					WithKubernetesLabels(Not(BeEmpty()))))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(Not(ContainElement(
+					HaveKubernetesLabels(Not(BeEmpty()))))),
 				))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/e2e/logs_keep_original_body_test.go

@@ -106,8 +106,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backend1ExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ConsistOfLds(ConsistOfLogRecords(
-					WithLogBody(BeEmpty()),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(HaveEach(
+					HaveLogBody(BeEmpty()),
 				))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
@@ -118,8 +118,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backend2ExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ConsistOfLds(ConsistOfLogRecords(
-					WithLogBody(Not(BeEmpty())),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(HaveEach(
+					HaveLogBody(Not(BeEmpty())),
 				))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/e2e/logs_multi_pipeline_namespace_filter_test.go

@@ -99,9 +99,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendIncludeNamespaceExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(
-					ContainLd(ContainLogRecord(WithNamespace(Equal(mock1Ns)))),
-				))
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(ContainElement(
+					HaveNamespace(Equal(mock1Ns)),
+				))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
 
@@ -110,9 +110,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backendIncludeNamespaceExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
-					WithNamespace(Equal(mock2Ns))))),
-				))
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(Not(ContainElement(
+					HaveNamespace(Equal(mock2Ns)),
+				)))))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
 
@@ -121,8 +121,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backend2ExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(
-					ContainLd(ContainLogRecord(WithNamespace(Equal(mock2Ns)))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(ContainElement(
+					HaveNamespace(Equal(mock2Ns)),
+				)),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
@@ -132,9 +133,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelLogs), Ordered, func() {
 				resp, err := proxyClient.Get(backend2ExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(Not(ContainLd(ContainLogRecord(
-					WithNamespace(Equal(mock1Ns))))),
-				))
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(Not(ContainElement(
+					HaveNamespace(Equal(mock1Ns)),
+				)))))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})
 	})

test/e2e/logs_parser_test.go

@@ -94,9 +94,9 @@ Types user:string pass:string`
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ContainLd(ContainLogRecord(SatisfyAll(
-					WithLogRecordAttrs(HaveKeyWithValue("user", "foo")),
-					WithLogRecordAttrs(HaveKeyWithValue("pass", "bar")),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(ContainElement(SatisfyAll(
+					HaveLogRecordAttributes(HaveKeyWithValue("user", "foo")),
+					HaveLogRecordAttributes(HaveKeyWithValue("pass", "bar")),
 				)))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/e2e/overrides_test.go

@@ -127,9 +127,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetry), Ordered, func() {
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-manager")),
-						WithLevel(Equal("INFO")),
+					HaveFlatLogs(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-manager")),
+						HaveLevel(Equal("INFO")),
 					))),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
@@ -141,10 +141,10 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetry), Ordered, func() {
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					Not(ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-manager")),
-						WithLevel(Equal("DEBUG")),
-						WithTimestamp(BeTemporally(">=", now)),
+					HaveFlatLogs(Not(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-manager")),
+						HaveLevel(Equal("DEBUG")),
+						HaveTimestamp(BeTemporally(">=", now)),
 					)))),
 				))
 			}, periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
@@ -177,10 +177,10 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetry), Ordered, func() {
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-manager")),
-						WithLevel(Equal("DEBUG")),
-						WithTimestamp(BeTemporally(">=", now)),
+					HaveFlatLogs(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-manager")),
+						HaveLevel(Equal("DEBUG")),
+						HaveTimestamp(BeTemporally(">=", now)),
 					))),
 				))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())

test/e2e/telemetry_logs_analysis_test.go

@@ -211,10 +211,10 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetryLogAnalysis), Ordered, fu
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					Not(ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-")),
-						WithLevel(MatchRegexp(logLevelsRegexp)),
-						WithLogBody(Not( // whitelist possible (flaky/expected) errors
+					HaveFlatLogs(Not(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-")),
+						HaveLevel(MatchRegexp(logLevelsRegexp)),
+						HaveLogBody(Not( // whitelist possible (flaky/expected) errors
 							Or(
 								ContainSubstring("grpc: addrConn.createTransport failed to connect"),
 								ContainSubstring("rpc error: code = Unavailable desc = no healthy upstream"),
@@ -232,9 +232,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetryLogAnalysis), Ordered, fu
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					Not(ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-")),
-						WithLogBody(MatchRegexp(logLevelsRegexp)), // fluenbit does not log in JSON, so we need to check the body for errors
+					HaveFlatLogs(Not(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-")),
+						HaveLogBody(MatchRegexp(logLevelsRegexp)), // fluenbit does not log in JSON, so we need to check the body for errors
 					)))),
 				))
 			}, consistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
@@ -246,9 +246,9 @@ var _ = Describe(suite.ID(), Label(suite.LabelTelemetryLogAnalysis), Ordered, fu
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 				g.Expect(resp).To(HaveHTTPBody(
-					Not(ContainLd(ContainLogRecord(SatisfyAll(
-						WithPodName(ContainSubstring("telemetry-")),
-						WithLevel(MatchRegexp(logLevelsRegexp)),
+					HaveFlatLogs(Not(ContainElement(SatisfyAll(
+						HavePodName(ContainSubstring("telemetry-")),
+						HaveLevel(MatchRegexp(logLevelsRegexp)),
 					)))),
 				))
 			}, consistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())

test/integration/istio/access_logs_test.go

@@ -108,8 +108,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
 				resp, err := proxyClient.Get(backendExportURL)
 				g.Expect(err).NotTo(HaveOccurred())
 				g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-				g.Expect(resp).To(HaveHTTPBody(ContainLd(ContainLogRecord(
-					WithLogRecordAttrs(HaveKey(BeElementOf(istio.AccessLogAttributeKeys))),
+				g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(ContainElement(
+					HaveLogRecordAttributes(HaveKey(BeElementOf(istio.AccessLogAttributeKeys))),
 				))))
 			}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 		})

test/testkit/assert/logs.go

@@ -26,8 +26,8 @@ func LogsDelivered(proxyClient *apiserverproxy.Client, expectedPodNamePrefix str
 		g.Expect(err).NotTo(HaveOccurred())
 		defer resp.Body.Close()
 		g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
-		g.Expect(resp).To(HaveHTTPBody(ContainLd(ContainLogRecord(
-			WithPodName(ContainSubstring(expectedPodNamePrefix))),
+		g.Expect(resp).To(HaveHTTPBody(HaveFlatLogs(ContainElement(
+			HavePodName(ContainSubstring(expectedPodNamePrefix))),
 		)))
 	}, periodic.EventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 }