From e59009da9b49360661cb5b47be7d21c529501c86 Mon Sep 17 00:00:00 2001 From: Franz Busch Date: Thu, 15 Aug 2024 10:51:52 +0100 Subject: [PATCH] Align benchmark scaling and minimum samples (#2839) # Motivation We want to have an aligned strategy across all of our benchmarks in how we use the scaling factor and samples. # Modification This PR makes sure that any inner loop is scaled by the scaling factor. Moreover, we want to run more than 1 iteration. To do this we set an almost infinite maximum duration and a maximum iteration count. # Result More consistent benchmarks --- Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift | 7 ++++++- Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift | 8 +++++--- .../5.10/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 4 ++-- .../Thresholds/5.10/NIOPosixBenchmarks.TCPEcho.p90.json | 2 +- .../5.10/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json | 4 ++-- .../5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 4 ++-- .../Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json | 2 +- .../5.9/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 4 ++-- .../Thresholds/5.9/NIOPosixBenchmarks.TCPEcho.p90.json | 2 +- .../5.9/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json | 4 ++-- .../NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 4 ++-- .../nightly-6.0/NIOPosixBenchmarks.TCPEcho.p90.json | 2 +- .../NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json | 4 ++-- .../NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json | 4 ++-- .../nightly-main/NIOPosixBenchmarks.TCPEcho.p90.json | 2 +- .../NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json | 4 ++-- 16 files changed, 34 insertions(+), 27 deletions(-) diff --git a/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift index 562de24eba..cea2e10af5 100644 --- a/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/NIOCoreBenchmarks/Benchmarks.swift @@ -23,7 +23,12 @@ let benchmarks = { Benchmark( "NIOAsyncChannel.init", - configuration: Benchmark.Configuration(metrics: defaultMetrics) + configuration: .init( + metrics: defaultMetrics, + scalingFactor: .kilo, + maxDuration: .seconds(10_000_000), + maxIterations: 10 + ) ) { benchmark in // Elide the cost of the 'EmbeddedChannel'. It's only used for its pipeline. var channels: [EmbeddedChannel] = [] diff --git a/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift index 4e1e3b50a6..149c51dd48 100644 --- a/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/NIOPosixBenchmarks/Benchmarks.swift @@ -26,8 +26,9 @@ let benchmarks = { "TCPEcho", configuration: .init( metrics: defaultMetrics, - timeUnits: .milliseconds, - scalingFactor: .mega + scalingFactor: .mega, + maxDuration: .seconds(10_000_000), + maxIterations: 5 ) ) { benchmark in try runTCPEcho( @@ -43,8 +44,9 @@ let benchmarks = { "TCPEchoAsyncChannel", configuration: .init( metrics: defaultMetrics, - timeUnits: .milliseconds, scalingFactor: .mega, + maxDuration: .seconds(10_000_000), + maxIterations: 5, // We are expecting a bit of allocation variance due to an allocation // in the Concurrency runtime which happens when resuming a continuation. thresholds: [.mallocCountTotal: .init(absolute: [.p90: 2000])], diff --git a/Benchmarks/Thresholds/5.10/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/5.10/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json index 882a60c306..ec2d0ad0f8 100644 --- a/Benchmarks/Thresholds/5.10/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ b/Benchmarks/Thresholds/5.10/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 8 -} + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEcho.p90.json index c6a93680d0..d7233c85b9 100644 --- a/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEcho.p90.json +++ b/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEcho.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 108 + "mallocCountTotal" : 555 } diff --git a/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json b/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json index 361ee44ab6..d773610fe1 100644 --- a/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json +++ b/Benchmarks/Thresholds/5.10/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 165000 -} + "mallocCountTotal" : 164375 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json index 882a60c306..ec2d0ad0f8 100644 --- a/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ b/Benchmarks/Thresholds/5.8/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 8 -} + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json index 248bd96061..8517c2fe45 100644 --- a/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json +++ b/Benchmarks/Thresholds/5.8/NIOPosixBenchmarks.TCPEcho.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 110 + "mallocCountTotal" : 556 } diff --git a/Benchmarks/Thresholds/5.9/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/5.9/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json index 882a60c306..ec2d0ad0f8 100644 --- a/Benchmarks/Thresholds/5.9/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ b/Benchmarks/Thresholds/5.9/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 8 -} + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEcho.p90.json index 248bd96061..8517c2fe45 100644 --- a/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEcho.p90.json +++ b/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEcho.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 110 + "mallocCountTotal" : 556 } diff --git a/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json b/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json index 2b218b490e..97ae879fef 100644 --- a/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json +++ b/Benchmarks/Thresholds/5.9/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 164000 -} + "mallocCountTotal" : 164381 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/nightly-6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/nightly-6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json index 882a60c306..ec2d0ad0f8 100644 --- a/Benchmarks/Thresholds/nightly-6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ b/Benchmarks/Thresholds/nightly-6.0/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 8 -} + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEcho.p90.json index c6a93680d0..67abcf36ac 100644 --- a/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEcho.p90.json +++ b/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEcho.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 108 + "mallocCountTotal" : 548 } diff --git a/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json b/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json index 361ee44ab6..390ed2415a 100644 --- a/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json +++ b/Benchmarks/Thresholds/nightly-6.0/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 165000 -} + "mallocCountTotal" : 164376 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/nightly-main/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json b/Benchmarks/Thresholds/nightly-main/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json index 882a60c306..ec2d0ad0f8 100644 --- a/Benchmarks/Thresholds/nightly-main/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json +++ b/Benchmarks/Thresholds/nightly-main/NIOCoreBenchmarks.NIOAsyncChannel.init.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 8 -} + "mallocCountTotal" : 8000 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEcho.p90.json b/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEcho.p90.json index c6a93680d0..67abcf36ac 100644 --- a/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEcho.p90.json +++ b/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEcho.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 108 + "mallocCountTotal" : 548 } diff --git a/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json b/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json index 361ee44ab6..8ac9bb7e07 100644 --- a/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json +++ b/Benchmarks/Thresholds/nightly-main/NIOPosixBenchmarks.TCPEchoAsyncChannel.p90.json @@ -1,3 +1,3 @@ { - "mallocCountTotal" : 165000 -} + "mallocCountTotal" : 164371 +} \ No newline at end of file