Add seed parameter

Project.toml

@@ -7,6 +7,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
 

docs/src/manual.md

@@ -85,6 +85,7 @@ You can pass the following keyword arguments to `@benchmark`, `@benchmarkable`,
 - `gcsample`: If `true`, run `gc()` before each sample. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.gcsample = false`.
 - `time_tolerance`: The noise tolerance for the benchmark's time estimate, as a percentage. This is utilized after benchmark execution, when analyzing results. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = 0.05`.
 - `memory_tolerance`: The noise tolerance for the benchmark's memory estimate, as a percentage. This is utilized after benchmark execution, when analyzing results. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = 0.01`.
+- `seed`: A seed number to which the global RNG is reset every benchmark run, if it is non-negative. This ensures that comparing two benchmarks gives actionable results, even if the running time depends on random numbers. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.seed = -1` (indicating no seed reset)
 
 To change the default values of the above fields, one can mutate the fields of `BenchmarkTools.DEFAULT_PARAMETERS`, for example:
 
@@ -255,6 +256,20 @@ julia> @btime exp(x) setup = (x=1,)  # errors
 ERROR: UndefVarError: `x` not defined
 ```
 
+### Consistent random numbers between runs
+
+You can supply the `seed` parameter to have the seed reset between runs, giving a consistent series of pseudorandom numbers.
+This is useful for comparing benchmarks - to know that they are operating on the same datasets while not needing to create those datasets manually.
+
+```julia
+julia> bg = BenchmarkGroup(
+           "a" => @benchmarkable(sleep(rand([0, 0.5]))),
+           "b" => @benchmarkable(sleep(rand([0, 0.5]))),
+           );
+julia> run(bg); # shows different results for "a" and "b", as the sleep time varies
+julia> run(bg; seed=42); # shows similar results for "a" and "b"
+```
+
 ### Understanding compiler optimizations
 
 It's possible for LLVM and Julia's compiler to perform optimizations on `@benchmarkable` expressions. In some cases, these optimizations can elide a computation altogether, resulting in unexpectedly "fast" benchmarks. For example, the following expression is non-allocating:

src/BenchmarkTools.jl

@@ -8,6 +8,7 @@ using Statistics
 using UUIDs: uuid4
 using Printf
 using Profile
+using Random
 
 const BENCHMARKTOOLS_VERSION = v"1.0.0"
 

src/execution.jl

@@ -103,6 +103,7 @@ function _run(b::Benchmark, p::Parameters; verbose=false, pad="", kwargs...)
     params = Parameters(p; kwargs...)
     @assert params.seconds > 0.0 "time limit must be greater than 0.0"
     params.gctrial && gcscrub()
+    params.seed >= 0 && Random.seed!(params.seed)
     start_time = Base.time()
     trial = Trial(params)
     params.gcsample && gcscrub()

src/parameters.jl

@@ -15,9 +15,10 @@ mutable struct Parameters
     gcsample::Bool
     time_tolerance::Float64
     memory_tolerance::Float64
+    seed::Int
 end
 
-const DEFAULT_PARAMETERS = Parameters(5.0, 10000, 1, false, 0, true, false, 0.05, 0.01)
+const DEFAULT_PARAMETERS = Parameters(5.0, 10000, 1, false, 0, true, false, 0.05, 0.01, -1)
 
 function Parameters(;
     seconds=DEFAULT_PARAMETERS.seconds,
@@ -29,6 +30,7 @@ function Parameters(;
     gcsample=DEFAULT_PARAMETERS.gcsample,
     time_tolerance=DEFAULT_PARAMETERS.time_tolerance,
     memory_tolerance=DEFAULT_PARAMETERS.memory_tolerance,
+    seed=DEFAULT_PARAMETERS.seed,
 )
     return Parameters(
         seconds,
@@ -40,6 +42,7 @@ function Parameters(;
         gcsample,
         time_tolerance,
         memory_tolerance,
+        seed,
     )
 end
 
@@ -53,6 +56,7 @@ function Parameters(
     gcsample=nothing,
     time_tolerance=nothing,
     memory_tolerance=nothing,
+    seed=nothing,
 )
     params = Parameters()
     params.seconds = seconds != nothing ? seconds : default.seconds
@@ -65,6 +69,7 @@ function Parameters(
         time_tolerance != nothing ? time_tolerance : default.time_tolerance
     params.memory_tolerance =
         memory_tolerance != nothing ? memory_tolerance : default.memory_tolerance
+    params.seed = seed != nothing ? seed : default.seed
     return params::BenchmarkTools.Parameters
 end
 
@@ -76,7 +81,8 @@ function Base.:(==)(a::Parameters, b::Parameters)
            a.gctrial == b.gctrial &&
            a.gcsample == b.gcsample &&
            a.time_tolerance == b.time_tolerance &&
-           a.memory_tolerance == b.memory_tolerance
+           a.memory_tolerance == b.memory_tolerance &&
+           a.seed == b.seed
 end
 
 function Base.copy(p::Parameters)
@@ -90,6 +96,7 @@ function Base.copy(p::Parameters)
         p.gcsample,
         p.time_tolerance,
         p.memory_tolerance,
+        p.seed,
     )
 end
 

test/ExecutionTests.jl

@@ -357,4 +357,26 @@ b = x = nothing
 GC.gc()
 @test x_finalized
 
+# Set seed
+results = Dict("a" => Int[], "b" => Int[], "c" => Int[], "d" => Int[])
+bg = BenchmarkGroup(
+    "a" => @benchmarkable(
+        push!(results["a"], rand(Int)), samples = 10, evals = 1, seed = 1234
+    ),
+    "b" => @benchmarkable(
+        push!(results["b"], rand(Int)), samples = 10, evals = 1, seed = 1234
+    ),
+    "c" => @benchmarkable(
+        push!(results["c"], rand(Int)), samples = 10, evals = 1, seed = 1235
+    ),
+    "d" => @benchmarkable(push!(results["d"], rand(Int)), samples = 10, evals = 1),
+)
+run(bg)
+@test results["a"] == results["b"]
+@test results["a"] != results["c"]
+@test results["a"] != results["d"]
+results = Dict("a" => Int[], "b" => Int[], "c" => Int[], "d" => Int[])
+run(bg; seed=1)
+@test results["a"] == results["b"] == results["c"] == results["d"]
+
 end # module