Measure percentiles in benchmarks

Signed-off-by: Farasath Ahamed <[email protected]>
zalando · Oct 20, 2024 · 4cbf4ab · 4cbf4ab
1 parent f1da8d0
commit 4cbf4ab
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 20 deletions.
diff --git a/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go b/filters/openpolicyagent/opaauthorizerequest/opaauthorizerequest_test.go
@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/benburkert/pbench"
 	"github.com/golang-jwt/jwt/v4"
 	opasdktest "github.com/open-policy-agent/opa/sdk/test"
 	"github.com/stretchr/testify/assert"
@@ -630,7 +631,7 @@ const (
 // Usage:
 // To run a specific scenario with or without decision logging:
 //
-//	go test -bench=BenchmarkAuthorizeRequest/<scenario>/{without-decision-logging|with-decision-logging}
+//	go test -bench=BenchmarkAuthorizeRequest/<scenario>/{no-decision-logs|with-decision-logs}
 //
 // Example:
 //
@@ -639,33 +640,39 @@ const (
 // Running with multiple GOMAXPROCS values (parallel CPU execution):
 // Use the -cpu flag to simulate the benchmark under different levels of CPU parallelism:
 //
-//	go test -bench 'BenchmarkAuthorizeRequest/minimal/without-decision-logging' -cpu 1,2,4,8
+//	go test -bench 'BenchmarkAuthorizeRequest/minimal/no-decision-logs' -cpu 1,2,4,8
 //
 // Note: Refer to the code for the latest available scenarios.
 func BenchmarkAuthorizeRequest(b *testing.B) {
+
 	scenarios := []struct {
 		name          string
-		benchmarkFunc func(b *testing.B, decisionLogging bool)
+		benchmarkFunc func(pb *pbench.B, decisionLogging bool)
 	}{
 		{"minimal", benchmarkMinimal},
 		{"with-body", benchmarkAllowWithReqBody},
 		{"jwt-validation", benchmarkJwtValidation},
 	}
 
 	for _, scenario := range scenarios {
-		b.Run(scenario.name, func(b *testing.B) {
-			b.Run("without-decision-logging", func(b *testing.B) {
-				scenario.benchmarkFunc(b, false)
+		b.Run(scenario.name, func(tb *testing.B) {
+			pb := pbench.New(tb)
+			pb.ReportPercentile(0.5)
+			pb.ReportPercentile(0.95)
+			pb.ReportPercentile(0.99)
+
+			pb.Run("no-decision-logs", func(pb *pbench.B) {
+				scenario.benchmarkFunc(pb, false)
 			})
 
-			b.Run("with-decision-logging", func(b *testing.B) {
-				scenario.benchmarkFunc(b, true)
+			pb.Run("with-decision-logs", func(pb *pbench.B) {
+				scenario.benchmarkFunc(pb, true)
 			})
 		})
 	}
 }
 
-func benchmarkMinimal(b *testing.B, decisionLogging bool) {
+func benchmarkMinimal(b *pbench.B, decisionLogging bool) {
 	opaControlPlane := opasdktest.MustNewServer(
 		opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{
 			"main.rego": `
@@ -702,16 +709,14 @@ func benchmarkMinimal(b *testing.B, decisionLogging bool) {
 	}
 
 	b.ResetTimer()
-	b.ReportAllocs()
-
-	b.RunParallel(func(pb *testing.PB) {
+	b.RunParallel(func(pb *pbench.PB) {
 		for pb.Next() {
 			f.Request(ctx)
 		}
 	})
 }
 
-func benchmarkAllowWithReqBody(b *testing.B, decisionLogging bool) {
+func benchmarkAllowWithReqBody(b *pbench.B, decisionLogging bool) {
 	opaControlPlane := opasdktest.MustNewServer(
 		opasdktest.MockBundle("/bundles/somebundle.tar.gz", map[string]string{
 			"main.rego": `
@@ -755,16 +760,14 @@ func benchmarkAllowWithReqBody(b *testing.B, decisionLogging bool) {
 	}
 
 	b.ResetTimer()
-	b.ReportAllocs()
-
-	b.RunParallel(func(pb *testing.PB) {
+	b.RunParallel(func(pb *pbench.PB) {
 		for pb.Next() {
 			f.Request(ctx)
 		}
 	})
 }
 
-func benchmarkJwtValidation(b *testing.B, decisionLogging bool) {
+func benchmarkJwtValidation(b *pbench.B, decisionLogging bool) {
 
 	publicKey, err := os.ReadFile(certPath)
 	if err != nil {
@@ -851,9 +854,7 @@ func benchmarkJwtValidation(b *testing.B, decisionLogging bool) {
 	}
 
 	b.ResetTimer()
-	b.ReportAllocs()
-
-	b.RunParallel(func(pb *testing.PB) {
+	b.RunParallel(func(pb *pbench.PB) {
 		for pb.Next() {
 			f.Request(ctx)
 			assert.False(b, ctx.FServed)

diff --git a/go.mod b/go.mod
@@ -6,6 +6,7 @@ require (
 	github.com/abbot/go-http-auth v0.4.0
 	github.com/andybalholm/brotli v1.1.0
 	github.com/aryszka/jobqueue v0.0.3
+	github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef
 	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/cespare/xxhash/v2 v2.3.0
 	github.com/cjoudrey/gluahttp v0.0.0-20201111170219-25003d9adfa9
@@ -94,6 +95,7 @@ require (
 	github.com/envoyproxy/protoc-gen-validate v1.0.4 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 // indirect
 	github.com/go-ini/ini v1.67.0 // indirect
 	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
 	github.com/go-logr/logr v1.4.2 // indirect

diff --git a/go.sum b/go.sum
@@ -42,6 +42,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ
 github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/aryszka/jobqueue v0.0.3 h1:O5YbgzQCjRomudwnDTY5BrHUNJhvPHQHq7GfGpE+ybs=
 github.com/aryszka/jobqueue v0.0.3/go.mod h1:SdxqI6HZ4E1Lss94tey5OfjcAu3bdCDWS1AQzzIN4m4=
+github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef h1:+7ZJvJGiV4hUBdjhEDhfGdjBCOmhVi0YQ5n+6g/ei+k=
+github.com/benburkert/pbench v0.0.0-20160623210926-4ec5821845ef/go.mod h1:hrhDSsc41bBqGejYXbvMh6qexfcC2vXjodP5gufwWyI=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
@@ -151,6 +153,8 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo
 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
 github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 h1:Vh7rylVZRZCj6W41lRlP17xPk4Nq260H4Xo/DDYmEZk=
+github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424/go.mod h1:vmp8DIyckQMXOPl0AQVHt+7n5h7Gb7hS6CUydiV8QeA=
 github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=