Skip to content

Commit 03c62fa

Browse files
AzzaareAzzaare
committed
Basic bench and chairs using PerfChecker (#50)
* Basic bench and chairs Update plots New plots New plots 2 * FIx format
1 parent 6fc9f5c commit 03c62fa

File tree

65 files changed

+536
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+536
-0
lines changed

‎perf/Project.toml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[deps]
2+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
3+
CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
4+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
5+
ConstraintCommons = "e37357d9-0691-492f-a822-e5ea6a920954"
6+
ConstraintDomains = "5800fd60-8556-4464-8d61-84ebf7a0bedb"
7+
Intervals = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
8+
PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424"

‎perf/checks.jl

Lines changed: 264 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,264 @@
1+
using BenchmarkTools
2+
using CairoMakie
3+
using Chairmarks
4+
using PerfChecker
5+
6+
@info "Defining utilities"
7+
8+
d_commons = Dict(
9+
:targets => ["ConstraintDomains"],
10+
:path => @__DIR__,
11+
:pkgs => ("ConstraintDomains", :custom, [v"0.2.5", v"0.3.0", v"0.3.10"], true),
12+
:seconds => 100,
13+
:samples => 10,
14+
:evals => 10,
15+
)
16+
17+
## SECTION - Utilities
18+
tags(d) = mapreduce(x -> string(x), (y, z) -> y * "_" * z, d[:tags])
19+
20+
function visu(x, d, ::Val{:allocs})
21+
mkpath(joinpath(@__DIR__, "visuals"))
22+
c = checkres_to_scatterlines(x, Val(:alloc))
23+
save(joinpath(@__DIR__, "visuals", "allocs_evolution_$(tags(d)).png"), c)
24+
25+
for (name, c2) in checkres_to_pie(x, Val(:alloc))
26+
save(joinpath(@__DIR__, "visuals", "allocs_pie_$(name)_$(tags(d)).png"), c2)
27+
end
28+
end
29+
30+
function visu(x, d, ::Val{:benchmark})
31+
mkpath(joinpath(d[:path], "visuals"))
32+
c = checkres_to_scatterlines(x, Val(:benchmark))
33+
save(joinpath(d[:path], "visuals", "bench_evolution_$(tags(d)).png"), c)
34+
35+
for kwarg in [:times, :gctimes, :memory, :allocs]
36+
c2 = checkres_to_boxplots(x, Val(:benchmark); kwarg)
37+
save(joinpath(d[:path], "visuals", "bench_boxplots_$(kwarg)_$(tags(d)).png"), c2)
38+
end
39+
end
40+
41+
function visu(x, d, ::Val{:chairmark})
42+
mkpath(joinpath(d[:path], "visuals"))
43+
c = checkres_to_scatterlines(x, Val(:chairmark))
44+
save(joinpath(d[:path], "visuals", "chair_evolution_$(tags(d)).png"), c)
45+
46+
for kwarg in [:times, :gctimes, :bytes, :allocs]
47+
c2 = checkres_to_boxplots(x, Val(:chairmark); kwarg)
48+
save(joinpath(d[:path], "visuals", "chair_boxplots_$(kwarg)_$(tags(d)).png"), c2)
49+
end
50+
end
51+
52+
## SECTION - Commons: benchmarks and chairmarks
53+
54+
@info "Running checks: Commons"
55+
56+
d = deepcopy(d_commons)
57+
d[:tags] = [:common]
58+
59+
x = @check :benchmark d begin
60+
using ConstraintDomains
61+
end begin
62+
ed = domain()
63+
domain_size(ed) == 0 == length(ed)
64+
isempty(ed)
65+
π ∉ ed
66+
end
67+
68+
visu(x, d, Val(:benchmark))
69+
70+
x = @check :chairmark d begin
71+
using ConstraintDomains
72+
end begin
73+
ed = domain()
74+
domain_size(ed) == 0 == length(ed)
75+
isempty(ed)
76+
π ∉ ed
77+
end
78+
79+
visu(x, d, Val(:chairmark))
80+
81+
## SECTION - Continuous: benchmarks and chairmarks
82+
83+
@info "Running checks: Continuous"
84+
85+
d = deepcopy(d_commons)
86+
d[:tags] = [:continuous]
87+
88+
x = @check :benchmark d begin
89+
using ConstraintDomains
90+
using Intervals
91+
end begin
92+
if d[:current_version] < v"0.3.0"
93+
d1 = domain((1.0, true), (3.15, true))
94+
d2 = domain((-42.42, false), (5.0, false))
95+
else
96+
d1 = domain(1.0 .. 3.15)
97+
d2 = domain(Interval{Open,Open}(-42.42, 5.0))
98+
end
99+
domains = [d1, d2]
100+
for d in domains
101+
for x in [1, 2.3, π]
102+
x ∈ d
103+
end
104+
for x in [5.1, π^π, Inf]
105+
x ∉ d
106+
end
107+
rand(d) ∈ d
108+
rand(d, 1) ∈ d
109+
domain_size(d) > 0.0
110+
end
111+
end
112+
113+
visu(x, d, Val(:benchmark))
114+
115+
x = @check :chairmark d begin
116+
using ConstraintDomains
117+
using Intervals
118+
end begin
119+
if d[:current_version] < v"0.3.0"
120+
d1 = domain((1.0, true), (3.15, true))
121+
d2 = domain((-42.42, false), (5.0, false))
122+
else
123+
d1 = domain(1.0 .. 3.15)
124+
d2 = domain(Interval{Open,Open}(-42.42, 5.0))
125+
end
126+
domains = [d1, d2]
127+
for d in domains
128+
for x in [1, 2.3, π]
129+
x ∈ d
130+
end
131+
for x in [5.1, π^π, Inf]
132+
x ∉ d
133+
end
134+
rand(d) ∈ d
135+
rand(d, 1) ∈ d
136+
domain_size(d) > 0.0
137+
end
138+
end
139+
140+
visu(x, d, Val(:chairmark))
141+
142+
## SECTION - Discrete: benchmarks and chairmarks
143+
144+
@info "Running checks: Discrete"
145+
146+
d = deepcopy(d_commons)
147+
d[:tags] = [:discrete]
148+
149+
x = @check :benchmark d begin
150+
using ConstraintDomains
151+
end begin
152+
d1 = domain([4, 3, 2, 1])
153+
d2 = domain(1)
154+
foreach(i -> add!(d2, i), 2:4)
155+
domains = [d1, d2]
156+
for d in domains
157+
for x in [1, 2, 3, 4]
158+
x ∈ d
159+
end
160+
length(d) == 4
161+
rand(d) ∈ d
162+
add!(d, 5)
163+
5 ∈ d
164+
delete!(d, 5)
165+
5 ∉ d
166+
domain_size(d) == 3
167+
end
168+
169+
d3 = domain(1:5)
170+
d4 = domain(1:0.5:5)
171+
domains2 = [d3, d4]
172+
for d in domains2
173+
for x in [1, 2, 3, 4, 5]
174+
x ∈ d
175+
end
176+
for x in [42]
177+
x ∉ d
178+
end
179+
rand(d) ∈ d
180+
end
181+
end
182+
183+
visu(x, d, Val(:benchmark))
184+
185+
x = @check :chairmark d begin
186+
using ConstraintDomains
187+
end begin
188+
d1 = domain([4, 3, 2, 1])
189+
d2 = domain(1)
190+
foreach(i -> add!(d2, i), 2:4)
191+
domains = [d1, d2]
192+
for d in domains
193+
for x in [1, 2, 3, 4]
194+
x ∈ d
195+
end
196+
length(d) == 4
197+
rand(d) ∈ d
198+
add!(d, 5)
199+
5 ∈ d
200+
delete!(d, 5)
201+
5 ∉ d
202+
domain_size(d) == 3
203+
end
204+
205+
d3 = domain(1:5)
206+
d4 = domain(1:0.5:5)
207+
domains2 = [d3, d4]
208+
for d in domains2
209+
for x in [1, 2, 3, 4, 5]
210+
x ∈ d
211+
end
212+
for x in [42]
213+
x ∉ d
214+
end
215+
rand(d) ∈ d
216+
end
217+
end
218+
219+
visu(x, d, Val(:chairmark))
220+
221+
## SECTION - Explore: benchmarks and chairmarks
222+
223+
@info "Running checks: Explore"
224+
225+
d = deepcopy(d_commons)
226+
d[:tags] = [:explore]
227+
d[:pkgs] = ("ConstraintDomains", :custom, [v"0.3.1", v"0.3.10"], true)
228+
229+
# x = @check :allocs d begin
230+
# using ConstraintDomains
231+
# end begin
232+
# domains = [domain([1, 2, 3, 4]) for i = 1:4]
233+
# X, X̅ = explore(domains, allunique)
234+
# length(X) == factorial(4)
235+
# length(X̅) == 4^4 - factorial(4)
236+
# end
237+
238+
# visu(x, d, Val(:allocs))
239+
240+
x = @check :benchmark d begin
241+
using ConstraintDomains
242+
end begin
243+
domains = [domain([1, 2, 3, 4]) for i = 1:4]
244+
X, X̅ = explore(domains, allunique)
245+
length(X) == factorial(4)
246+
length(X̅) == 4^4 - factorial(4)
247+
end
248+
249+
visu(x, d, Val(:benchmark))
250+
251+
x = @check :chairmark d begin
252+
using ConstraintDomains
253+
end begin
254+
domains = [domain([1, 2, 3, 4]) for i = 1:4]
255+
X, X̅ = explore(domains, allunique)
256+
length(X) == factorial(4)
257+
length(X̅) == 4^4 - factorial(4)
258+
end
259+
260+
visu(x, d, Val(:chairmark))
261+
262+
# TODO: add more checks for parameters.jl
263+
264+
@info "All checks have been successfully run!"

‎perf/metadata/metadata.csv

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
benchmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
2+
benchmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
3+
benchmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
4+
chairmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
5+
chairmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
6+
chairmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
7+
benchmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
8+
benchmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
9+
benchmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
10+
chairmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
11+
chairmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
12+
chairmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
13+
benchmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
14+
benchmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
15+
benchmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
16+
chairmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
17+
chairmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
18+
chairmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
19+
benchmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
20+
benchmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
21+
chairmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
22+
chairmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62

‎perf/output/01d3078e-3e06-59b8-b7a8-13146a1184de.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
4.7375e-6,0.0,1632.0,19.0
3+
3.3084000000000004e-6,0.0,1632.0,19.0
4+
1.9851e-6,0.0,1632.0,19.0
5+
2.0482e-6,0.0,1632.0,19.0
6+
1.8231000000000002e-6,0.0,1632.0,19.0
7+
1.7413000000000002e-6,0.0,1632.0,19.0
8+
1.7787e-6,0.0,1632.0,19.0
9+
1.7996e-6,0.0,1632.0,19.0
10+
1.8254000000000001e-6,0.0,1632.0,19.0
11+
1.7064e-6,0.0,1632.0,19.0

‎perf/output/039713ce-acec-5012-a235-f497b94aca3c.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
4785.7,0.0,1632,19
3+
3461.3,0.0,1632,19
4+
2710.9,0.0,1632,19
5+
1916.1,0.0,1632,19
6+
2028.3,0.0,1632,19
7+
2050.8,0.0,1632,19
8+
1767.3,0.0,1632,19
9+
1749.7,0.0,1632,19
10+
1824.1,0.0,1632,19
11+
1808.7,0.0,1632,19

‎perf/output/06768c48-4ab9-5bb9-a034-980e923d7fac.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
4.9568000000000006e-6,0.0,864.0,27.0
3+
2.5383e-6,0.0,864.0,27.0
4+
1.3226e-6,0.0,864.0,27.0
5+
1.2310000000000002e-6,0.0,864.0,27.0
6+
1.1021e-6,0.0,864.0,27.0
7+
1.2353e-6,0.0,864.0,27.0
8+
1.1795e-6,0.0,864.0,27.0
9+
1.052e-6,0.0,864.0,27.0
10+
1.0814e-6,0.0,864.0,27.0
11+
1.0822e-6,0.0,864.0,27.0

‎perf/output/0d9e0caa-8170-543c-9628-2ad63c457e1f.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
8.7,0.0,0,0
3+
24.0,0.0,0,0
4+
4.3,0.0,0,0
5+
4.9,0.0,0,0
6+
5.4,0.0,0,0
7+
3.8,0.0,0,0
8+
3.7,0.0,0,0
9+
3.8,0.0,0,0
10+
3.8,0.0,0,0
11+
4.6,0.0,0,0

‎perf/output/19661d6c-8c80-5bd8-bfd1-220f92ef21fb.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
5.193300000000001e-6,0.0,864.0,27.0
3+
2.4154e-6,0.0,864.0,27.0
4+
1.0913e-6,0.0,864.0,27.0
5+
1.1673000000000002e-6,0.0,864.0,27.0
6+
1.1393e-6,0.0,864.0,27.0
7+
1.1409e-6,0.0,864.0,27.0
8+
1.1716000000000001e-6,0.0,864.0,27.0
9+
1.1899e-6,0.0,864.0,27.0
10+
1.0990000000000002e-6,0.0,864.0,27.0
11+
1.1807e-6,0.0,864.0,27.0

‎perf/output/1c54877a-7fa4-53db-8195-00f829bc16a0.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
40823.0,0.0,41392,309
3+
42830.1,0.0,41392,309
4+
38844.4,0.0,41392,309
5+
37552.7,0.0,41392,309
6+
36134.9,0.0,41392,309
7+
36296.1,0.0,41392,309
8+
35766.0,0.0,41392,309
9+
36251.5,0.0,41392,309
10+
35749.3,0.0,41392,309
11+
35883.3,0.0,41392,309

‎perf/output/3615ebfa-aa5f-5fc4-b45d-28558621f4bb.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
2.5900000000000004e-8,0.0,0.0,0.0
3+
2.49e-8,0.0,0.0,0.0
4+
5.2e-9,0.0,0.0,0.0
5+
5.4e-9,0.0,0.0,0.0
6+
3.9e-9,0.0,0.0,0.0
7+
3.8e-9,0.0,0.0,0.0
8+
3.9e-9,0.0,0.0,0.0
9+
3.8e-9,0.0,0.0,0.0
10+
3.7e-9,0.0,0.0,0.0
11+
3.8e-9,0.0,0.0,0.0

‎perf/output/431f4cb4-beae-56d3-b340-80762efacf8a.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
1.2562700000000001e-5,0.0,784.0,9.0
3+
1.14067e-5,0.0,784.0,9.0
4+
1.04715e-5,0.0,784.0,9.0
5+
1.04492e-5,0.0,784.0,9.0
6+
1.0432300000000002e-5,0.0,784.0,9.0
7+
1.04338e-5,0.0,784.0,9.0
8+
1.04493e-5,0.0,784.0,9.0
9+
1.13867e-5,0.0,784.0,9.0
10+
1.0136800000000001e-5,0.0,784.0,9.0
11+
1.0012500000000001e-5,0.0,784.0,9.0

0 commit comments

Comments
 (0)