diff --git a/Project.toml b/Project.toml index ab6249c9..b7b1338b 100644 --- a/Project.toml +++ b/Project.toml @@ -7,6 +7,7 @@ JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" Logging = "56ddb016-857b-54e1-b83d-db4d58db5568" Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7" Profile = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" +Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" UUIDs = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" diff --git a/docs/src/manual.md b/docs/src/manual.md index 62a90805..1fcda7f6 100644 --- a/docs/src/manual.md +++ b/docs/src/manual.md @@ -85,6 +85,7 @@ You can pass the following keyword arguments to `@benchmark`, `@benchmarkable`, - `gcsample`: If `true`, run `gc()` before each sample. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.gcsample = false`. - `time_tolerance`: The noise tolerance for the benchmark's time estimate, as a percentage. This is utilized after benchmark execution, when analyzing results. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.time_tolerance = 0.05`. - `memory_tolerance`: The noise tolerance for the benchmark's memory estimate, as a percentage. This is utilized after benchmark execution, when analyzing results. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.memory_tolerance = 0.01`. +- `seed`: A seed number to which the global RNG is reset every benchmark run, if it is non-negative. This ensures that comparing two benchmarks gives actionable results, even if the running time depends on random numbers. Defaults to `BenchmarkTools.DEFAULT_PARAMETERS.seed = -1` (indicating no seed reset) To change the default values of the above fields, one can mutate the fields of `BenchmarkTools.DEFAULT_PARAMETERS`, for example: @@ -255,6 +256,20 @@ julia> @btime exp(x) setup = (x=1,) # errors ERROR: UndefVarError: `x` not defined ``` +### Consistent random numbers between runs + +You can supply the `seed` parameter to have the seed reset between runs, giving a consistent series of pseudorandom numbers. +This is useful for comparing benchmarks - to know that they are operating on the same datasets while not needing to create those datasets manually. + +```julia +julia> bg = BenchmarkGroup( + "a" => @benchmarkable(sleep(rand([0, 0.5]))), + "b" => @benchmarkable(sleep(rand([0, 0.5]))), + ); +julia> run(bg); # shows different results for "a" and "b", as the sleep time varies +julia> run(bg; seed=42); # shows similar results for "a" and "b" +``` + ### Understanding compiler optimizations It's possible for LLVM and Julia's compiler to perform optimizations on `@benchmarkable` expressions. In some cases, these optimizations can elide a computation altogether, resulting in unexpectedly "fast" benchmarks. For example, the following expression is non-allocating: diff --git a/src/BenchmarkTools.jl b/src/BenchmarkTools.jl index ab2a1ec4..56612535 100644 --- a/src/BenchmarkTools.jl +++ b/src/BenchmarkTools.jl @@ -8,6 +8,7 @@ using Statistics using UUIDs: uuid4 using Printf using Profile +using Random const BENCHMARKTOOLS_VERSION = v"1.0.0" diff --git a/src/execution.jl b/src/execution.jl index a031e652..59ab97b9 100644 --- a/src/execution.jl +++ b/src/execution.jl @@ -103,6 +103,7 @@ function _run(b::Benchmark, p::Parameters; verbose=false, pad="", kwargs...) params = Parameters(p; kwargs...) @assert params.seconds > 0.0 "time limit must be greater than 0.0" params.gctrial && gcscrub() + params.seed >= 0 && Random.seed!(params.seed) start_time = Base.time() trial = Trial(params) params.gcsample && gcscrub() diff --git a/src/parameters.jl b/src/parameters.jl index ff1bc615..e076abf4 100644 --- a/src/parameters.jl +++ b/src/parameters.jl @@ -15,9 +15,10 @@ mutable struct Parameters gcsample::Bool time_tolerance::Float64 memory_tolerance::Float64 + seed::Int end -const DEFAULT_PARAMETERS = Parameters(5.0, 10000, 1, false, 0, true, false, 0.05, 0.01) +const DEFAULT_PARAMETERS = Parameters(5.0, 10000, 1, false, 0, true, false, 0.05, 0.01, -1) function Parameters(; seconds=DEFAULT_PARAMETERS.seconds, @@ -29,6 +30,7 @@ function Parameters(; gcsample=DEFAULT_PARAMETERS.gcsample, time_tolerance=DEFAULT_PARAMETERS.time_tolerance, memory_tolerance=DEFAULT_PARAMETERS.memory_tolerance, + seed=DEFAULT_PARAMETERS.seed, ) return Parameters( seconds, @@ -40,6 +42,7 @@ function Parameters(; gcsample, time_tolerance, memory_tolerance, + seed, ) end @@ -53,6 +56,7 @@ function Parameters( gcsample=nothing, time_tolerance=nothing, memory_tolerance=nothing, + seed=nothing, ) params = Parameters() params.seconds = seconds != nothing ? seconds : default.seconds @@ -65,6 +69,7 @@ function Parameters( time_tolerance != nothing ? time_tolerance : default.time_tolerance params.memory_tolerance = memory_tolerance != nothing ? memory_tolerance : default.memory_tolerance + params.seed = seed != nothing ? seed : default.seed return params::BenchmarkTools.Parameters end @@ -76,7 +81,8 @@ function Base.:(==)(a::Parameters, b::Parameters) a.gctrial == b.gctrial && a.gcsample == b.gcsample && a.time_tolerance == b.time_tolerance && - a.memory_tolerance == b.memory_tolerance + a.memory_tolerance == b.memory_tolerance && + a.seed == b.seed end function Base.copy(p::Parameters) @@ -90,6 +96,7 @@ function Base.copy(p::Parameters) p.gcsample, p.time_tolerance, p.memory_tolerance, + p.seed, ) end diff --git a/test/ExecutionTests.jl b/test/ExecutionTests.jl index 235794ff..9f40d376 100644 --- a/test/ExecutionTests.jl +++ b/test/ExecutionTests.jl @@ -357,4 +357,26 @@ b = x = nothing GC.gc() @test x_finalized +# Set seed +results = Dict("a" => Int[], "b" => Int[], "c" => Int[], "d" => Int[]) +bg = BenchmarkGroup( + "a" => @benchmarkable( + push!(results["a"], rand(Int)), samples = 10, evals = 1, seed = 1234 + ), + "b" => @benchmarkable( + push!(results["b"], rand(Int)), samples = 10, evals = 1, seed = 1234 + ), + "c" => @benchmarkable( + push!(results["c"], rand(Int)), samples = 10, evals = 1, seed = 1235 + ), + "d" => @benchmarkable(push!(results["d"], rand(Int)), samples = 10, evals = 1), +) +run(bg) +@test results["a"] == results["b"] +@test results["a"] != results["c"] +@test results["a"] != results["d"] +results = Dict("a" => Int[], "b" => Int[], "c" => Int[], "d" => Int[]) +run(bg; seed=1) +@test results["a"] == results["b"] == results["c"] == results["d"] + end # module