[testbed] Include CPU and memory limits to benchmark results file (op…

…en-telemetry#36753)

<!--Ex. Fixing a bug - Describe the bug and how this fixes the issue.
Ex. Adding a feature - Explain what this achieves.-->
#### Description

This PR adds the specified limits for the CPU and memory consumption to
the benchmark results file. This way, these values can also be included
in a timeseries chart, next to the max and average consumption.

For now, this is disabled by default in the `PerfTestValidator` to not
introduce any unwanted changes to the charts generated from tests using
this component

<!-- Issue number (e.g. open-telemetry#1234) or full URL to issue, if applicable. -->
#### Link to tracking issue
Fixes open-telemetry#36720

---------

Signed-off-by: Florian Bacher <[email protected]>

.chloggen/testbed-add-resource-limits.yaml

@@ -0,0 +1,27 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: enhancement
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: testbed
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Include the specified resource limits for CPU and memory in the benchmark results
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [36720]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

testbed/testbed/child_process_collector.go

@@ -466,7 +466,10 @@ func (cp *childProcessCollector) GetResourceConsumption() string {
 
 // GetTotalConsumption returns total resource consumption since start of process
 func (cp *childProcessCollector) GetTotalConsumption() *ResourceConsumption {
-	rc := &ResourceConsumption{}
+	rc := &ResourceConsumption{
+		CPUPercentLimit: float64(cp.resourceSpec.ExpectedMaxCPU),
+		RAMMiBLimit:     cp.resourceSpec.ExpectedMaxRAM,
+	}
 
 	if cp.processMon != nil {
 		// Get total elapsed time since process start

testbed/testbed/in_process_collector.go

@@ -116,10 +116,12 @@ func (ipp *inProcessCollector) GetProcessMon() *process.Process {
 
 func (ipp *inProcessCollector) GetTotalConsumption() *ResourceConsumption {
 	return &ResourceConsumption{
-		CPUPercentAvg: 0,
-		CPUPercentMax: 0,
-		RAMMiBAvg:     0,
-		RAMMiBMax:     0,
+		CPUPercentAvg:   0,
+		CPUPercentMax:   0,
+		CPUPercentLimit: 0,
+		RAMMiBAvg:       0,
+		RAMMiBMax:       0,
+		RAMMiBLimit:     0,
 	}
 }
 

testbed/testbed/otelcol_runner.go

@@ -15,10 +15,12 @@ type StartParams struct {
 }
 
 type ResourceConsumption struct {
-	CPUPercentAvg float64
-	CPUPercentMax float64
-	RAMMiBAvg     uint32
-	RAMMiBMax     uint32
+	CPUPercentAvg   float64
+	CPUPercentMax   float64
+	CPUPercentLimit float64
+	RAMMiBAvg       uint32
+	RAMMiBMax       uint32
+	RAMMiBLimit     uint32
 }
 
 // OtelcolRunner defines the interface for configuring, starting and stopping one or more instances of

testbed/testbed/results.go

@@ -52,16 +52,18 @@ type PerformanceResults struct {
 
 // PerformanceTestResult reports the results of a single performance test.
 type PerformanceTestResult struct {
-	testName          string
-	result            string
-	duration          time.Duration
-	cpuPercentageAvg  float64
-	cpuPercentageMax  float64
-	ramMibAvg         uint32
-	ramMibMax         uint32
-	sentSpanCount     uint64
-	receivedSpanCount uint64
-	errorCause        string
+	testName           string
+	result             string
+	duration           time.Duration
+	cpuPercentageAvg   float64
+	cpuPercentageMax   float64
+	cpuPercentageLimit float64
+	ramMibAvg          uint32
+	ramMibMax          uint32
+	ramMibLimit        uint32
+	sentSpanCount      uint64
+	receivedSpanCount  uint64
+	errorCause         string
 }
 
 func (r *PerformanceResults) Init(resultsDir string) {
@@ -83,8 +85,8 @@ func (r *PerformanceResults) Init(resultsDir string) {
 	_, _ = io.WriteString(r.resultsFile,
 		"# Test PerformanceResults\n"+
 			fmt.Sprintf("Started: %s\n\n", time.Now().Format(time.RFC1123Z))+
-			"Test                                    |Result|Duration|CPU Avg%|CPU Max%|RAM Avg MiB|RAM Max MiB|Sent Items|Received Items|\n"+
-			"----------------------------------------|------|-------:|-------:|-------:|----------:|----------:|---------:|-------------:|\n")
+			"Test                                    |Result|Duration|CPU Avg%|CPU Max%|CPU Limit|RAM Avg MiB|RAM Max MiB|RAM Limit MiB|Sent Items|Received Items|\n"+
+			"----------------------------------------|------|-------:|-------:|-------:|--------:|----------:|----------:|------------:|---------:|-------------:|\n")
 }
 
 // Save the total results and close the file.
@@ -103,14 +105,16 @@ func (r *PerformanceResults) Add(_ string, result any) {
 	}
 
 	_, _ = io.WriteString(r.resultsFile,
-		fmt.Sprintf("%-40s|%-6s|%7.0fs|%8.1f|%8.1f|%11d|%11d|%10d|%14d|%s\n",
+		fmt.Sprintf("%-40s|%-6s|%7.0fs|%8.1f|%8.1f|%8.1f|%11d|%11d|%11d|%10d|%14d|%s\n",
 			testResult.testName,
 			testResult.result,
 			testResult.duration.Seconds(),
 			testResult.cpuPercentageAvg,
 			testResult.cpuPercentageMax,
+			testResult.cpuPercentageLimit,
 			testResult.ramMibAvg,
 			testResult.ramMibMax,
+			testResult.ramMibLimit,
 			testResult.sentSpanCount,
 			testResult.receivedSpanCount,
 			testResult.errorCause,
@@ -135,6 +139,14 @@ func (r *PerformanceResults) Add(_ string, result any) {
 		Unit:  "%",
 		Extra: cpuChartName,
 	})
+	if testResult.cpuPercentageLimit > 0 {
+		r.benchmarkResults = append(r.benchmarkResults, &benchmarkResult{
+			Name:  "cpu_percentage_limit",
+			Value: testResult.cpuPercentageLimit,
+			Unit:  "%",
+			Extra: cpuChartName,
+		})
+	}
 	r.benchmarkResults = append(r.benchmarkResults, &benchmarkResult{
 		Name:  "ram_mib_avg",
 		Value: float64(testResult.ramMibAvg),
@@ -147,6 +159,14 @@ func (r *PerformanceResults) Add(_ string, result any) {
 		Unit:  "MiB",
 		Extra: memoryChartName,
 	})
+	if testResult.ramMibLimit > 0 {
+		r.benchmarkResults = append(r.benchmarkResults, &benchmarkResult{
+			Name:  "ram_mib_limit",
+			Value: float64(testResult.ramMibLimit),
+			Unit:  "MiB",
+			Extra: memoryChartName,
+		})
+	}
 	r.benchmarkResults = append(r.benchmarkResults, &benchmarkResult{
 		Name:  "dropped_span_count",
 		Value: float64(testResult.sentSpanCount - testResult.receivedSpanCount),

testbed/testbed/validator.go

@@ -66,7 +66,10 @@ func (v *LogPresentValidator) RecordResults(tc *TestCase) {
 }
 
 // PerfTestValidator implements TestCaseValidator for test suites using PerformanceResults for summarizing results.
-type PerfTestValidator struct{}
+type PerfTestValidator struct {
+	// IncludeLimitsInReport determines whether the summary generated by PerformanceResults should include the specified resource limits
+	IncludeLimitsInReport bool
+}
 
 func (v *PerfTestValidator) Validate(tc *TestCase) {
 	if assert.EqualValues(tc.t,
@@ -90,7 +93,7 @@ func (v *PerfTestValidator) RecordResults(tc *TestCase) {
 	// Remove "Test" prefix from test name.
 	testName := tc.t.Name()[4:]
 
-	tc.resultsSummary.Add(tc.t.Name(), &PerformanceTestResult{
+	performanceResults := &PerformanceTestResult{
 		testName:          testName,
 		result:            result,
 		receivedSpanCount: tc.MockBackend.DataItemsReceived(),
@@ -101,7 +104,14 @@ func (v *PerfTestValidator) RecordResults(tc *TestCase) {
 		ramMibAvg:         rc.RAMMiBAvg,
 		ramMibMax:         rc.RAMMiBMax,
 		errorCause:        tc.errorCause,
-	})
+	}
+
+	if v.IncludeLimitsInReport {
+		performanceResults.cpuPercentageLimit = rc.CPUPercentLimit
+		performanceResults.ramMibLimit = rc.RAMMiBLimit
+	}
+
+	tc.resultsSummary.Add(tc.t.Name(), performanceResults)
 }
 
 // CorrectnessTestValidator implements TestCaseValidator for test suites using CorrectnessResults for summarizing results.