phc: add new metric to counter number of unique requests mitigated (#…

…3059) Signed-off-by: Mustafa Abdelrahman <[email protected]>
zalando · May 8, 2024 · 0f1e776 · 0f1e776
1 parent b778f2b
commit 0f1e776
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 0 deletions.
diff --git a/docs/operation/operation.md b/docs/operation/operation.md
@@ -932,6 +932,7 @@ while choosing the endpoint for the given request
 A set of metrics will be exposed to track passive health check:
 
 * `passive-health-check.endpoints.dropped`: Number of all endpoints dropped before load balancing a request, so after N requests and M endpoints are being dropped this counter would be N*M.
+* `passive-health-check.requests.passed`: Number of unique requests where PHC was able to avoid sending them to unhealthy endpoints.
 
 ## Memory consumption
 

diff --git a/proxy/healthy_endpoints.go b/proxy/healthy_endpoints.go
@@ -34,5 +34,10 @@ func (h *healthyEndpoints) filterHealthyEndpoints(ctx *context, endpoints []rout
 	if len(filtered) == 0 {
 		return endpoints
 	}
+
+	if len(filtered) < len(endpoints) {
+		metrics.IncCounter("passive-health-check.requests.passed")
+	}
+
 	return filtered
 }
diff --git a/proxy/healthy_endpoints_test.go b/proxy/healthy_endpoints_test.go
@@ -197,6 +197,7 @@ func TestPHCForMultipleHealthyAndOneUnhealthyEndpoints(t *testing.T) {
 			assert.InDelta(t, 0, float64(failedRequests), 0.1*float64(va.TotalRequests()))
 			mockMetrics.WithCounters(func(counters map[string]int64) {
 				assert.InDelta(t, float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(va.TotalRequests())) // allow 30% error
+				assert.InDelta(t, float64(va.TotalRequests()), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(va.TotalRequests()))   // allow 30% error
 			})
 		})
 	}
@@ -215,6 +216,7 @@ func TestPHCForMultipleHealthyAndOneUnhealthyEndpoints(t *testing.T) {
 		assert.InDelta(t, 0, failedReqs, 0.1*float64(nRequests))
 		mockMetrics.WithCounters(func(counters map[string]int64) {
 			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)) // allow 30% error
+			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests))   // allow 30% error
 		})
 	})
 
@@ -225,6 +227,7 @@ func TestPHCForMultipleHealthyAndOneUnhealthyEndpoints(t *testing.T) {
 		assert.InDelta(t, 0, failedReqs, 0.1*float64(nRequests))
 		mockMetrics.WithCounters(func(counters map[string]int64) {
 			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.3*float64(nRequests)) // allow 30% error
+			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests))   // allow 30% error
 		})
 	})
 }
@@ -251,6 +254,7 @@ func TestPHCForMultipleHealthyAndMultipleUnhealthyEndpoints(t *testing.T) {
 			assert.InDelta(t, 0, float64(failedRequests), 0.3*float64(va.TotalRequests()))
 			mockMetrics.WithCounters(func(counters map[string]int64) {
 				assert.InDelta(t, 2*float64(va.TotalRequests()), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(va.TotalRequests()))
+				assert.InDelta(t, float64(va.TotalRequests()), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(va.TotalRequests())) // allow 30% error
 			})
 		})
 	}
@@ -269,6 +273,7 @@ func TestPHCForMultipleHealthyAndMultipleUnhealthyEndpoints(t *testing.T) {
 		assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests))
 		mockMetrics.WithCounters(func(counters map[string]int64) {
 			assert.InDelta(t, 2*float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(nRequests))
+			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests)) // allow 30% error
 		})
 	})
 
@@ -286,6 +291,7 @@ func TestPHCForMultipleHealthyAndMultipleUnhealthyEndpoints(t *testing.T) {
 		assert.InDelta(t, 0, failedReqs, 0.2*float64(nRequests))
 		mockMetrics.WithCounters(func(counters map[string]int64) {
 			assert.InDelta(t, 2*float64(nRequests), float64(counters["passive-health-check.endpoints.dropped"]), 0.6*float64(nRequests))
+			assert.InDelta(t, float64(nRequests), float64(counters["passive-health-check.requests.passed"]), 0.3*float64(nRequests)) // allow 30% error
 		})
 	})
 }