Skip to content

Commit a7c990e

Browse files
AzzaareAzzaare
committed
Basic bench and chairs
Update plots New plots New plots 2
1 parent aa87732 commit a7c990e

File tree

65 files changed

+534
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+534
-0
lines changed

‎perf/Project.toml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
[deps]
2+
BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf"
3+
CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
4+
Chairmarks = "0ca39b1e-fe0b-4e98-acfc-b1656634c4de"
5+
ConstraintCommons = "e37357d9-0691-492f-a822-e5ea6a920954"
6+
ConstraintDomains = "5800fd60-8556-4464-8d61-84ebf7a0bedb"
7+
Intervals = "d8418881-c3e1-53bb-8760-2df7ec849ed5"
8+
PerfChecker = "6309bf6b-a531-4b08-891e-8ee981e5c424"

‎perf/checks.jl

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,262 @@
1+
using BenchmarkTools
2+
using CairoMakie
3+
using Chairmarks
4+
using PerfChecker
5+
6+
@info "Defining utilities"
7+
8+
d_commons = Dict(
9+
:targets => ["ConstraintDomains"],
10+
:path => @__DIR__,
11+
:pkgs => ("ConstraintDomains", :custom, [v"0.2.5", v"0.3.0", v"0.3.10"], true),
12+
:seconds => 100, :samples => 10, :evals => 10,
13+
)
14+
15+
## SECTION - Utilities
16+
tags(d) = mapreduce(x -> string(x), (y, z) -> y * "_" * z, d[:tags])
17+
18+
function visu(x, d, ::Val{:allocs})
19+
mkpath(joinpath(@__DIR__, "visuals"))
20+
c = checkres_to_scatterlines(x, Val(:alloc))
21+
save(joinpath(@__DIR__, "visuals", "allocs_evolution_$(tags(d)).png"), c)
22+
23+
for (name, c2) in checkres_to_pie(x, Val(:alloc))
24+
save(joinpath(@__DIR__, "visuals", "allocs_pie_$(name)_$(tags(d)).png"), c2)
25+
end
26+
end
27+
28+
function visu(x, d, ::Val{:benchmark})
29+
mkpath(joinpath(d[:path], "visuals"))
30+
c = checkres_to_scatterlines(x, Val(:benchmark))
31+
save(joinpath(d[:path], "visuals", "bench_evolution_$(tags(d)).png"), c)
32+
33+
for kwarg in [:times, :gctimes, :memory, :allocs]
34+
c2 = checkres_to_boxplots(x, Val(:benchmark); kwarg)
35+
save(joinpath(d[:path], "visuals", "bench_boxplots_$(kwarg)_$(tags(d)).png"), c2)
36+
end
37+
end
38+
39+
function visu(x, d, ::Val{:chairmark})
40+
mkpath(joinpath(d[:path], "visuals"))
41+
c = checkres_to_scatterlines(x, Val(:chairmark))
42+
save(joinpath(d[:path], "visuals", "chair_evolution_$(tags(d)).png"), c)
43+
44+
for kwarg in [:times, :gctimes, :bytes, :allocs]
45+
c2 = checkres_to_boxplots(x, Val(:chairmark); kwarg)
46+
save(joinpath(d[:path], "visuals", "chair_boxplots_$(kwarg)_$(tags(d)).png"), c2)
47+
end
48+
end
49+
50+
## SECTION - Commons: benchmarks and chairmarks
51+
52+
@info "Running checks: Commons"
53+
54+
d = deepcopy(d_commons)
55+
d[:tags] = [:common]
56+
57+
x = @check :benchmark d begin
58+
using ConstraintDomains
59+
end begin
60+
ed = domain()
61+
domain_size(ed) == 0 == length(ed)
62+
isempty(ed)
63+
π ∉ ed
64+
end
65+
66+
visu(x, d, Val(:benchmark))
67+
68+
x = @check :chairmark d begin
69+
using ConstraintDomains
70+
end begin
71+
ed = domain()
72+
domain_size(ed) == 0 == length(ed)
73+
isempty(ed)
74+
π ∉ ed
75+
end
76+
77+
visu(x, d, Val(:chairmark))
78+
79+
## SECTION - Continuous: benchmarks and chairmarks
80+
81+
@info "Running checks: Continuous"
82+
83+
d = deepcopy(d_commons)
84+
d[:tags] = [:continuous]
85+
86+
x = @check :benchmark d begin
87+
using ConstraintDomains
88+
using Intervals
89+
end begin
90+
if d[:current_version] < v"0.3.0"
91+
d1 = domain((1.0, true), (3.15, true))
92+
d2 = domain((-42.42, false), (5.0, false))
93+
else
94+
d1 = domain(1.0 .. 3.15)
95+
d2 = domain(Interval{Open,Open}(-42.42, 5.0))
96+
end
97+
domains = [d1, d2]
98+
for d in domains
99+
for x in [1, 2.3, π]
100+
x ∈ d
101+
end
102+
for x in [5.1, π^π, Inf]
103+
x ∉ d
104+
end
105+
rand(d) ∈ d
106+
rand(d, 1) ∈ d
107+
domain_size(d) > 0.0
108+
end
109+
end
110+
111+
visu(x, d, Val(:benchmark))
112+
113+
x = @check :chairmark d begin
114+
using ConstraintDomains
115+
using Intervals
116+
end begin
117+
if d[:current_version] < v"0.3.0"
118+
d1 = domain((1.0, true), (3.15, true))
119+
d2 = domain((-42.42, false), (5.0, false))
120+
else
121+
d1 = domain(1.0 .. 3.15)
122+
d2 = domain(Interval{Open,Open}(-42.42, 5.0))
123+
end
124+
domains = [d1, d2]
125+
for d in domains
126+
for x in [1, 2.3, π]
127+
x ∈ d
128+
end
129+
for x in [5.1, π^π, Inf]
130+
x ∉ d
131+
end
132+
rand(d) ∈ d
133+
rand(d, 1) ∈ d
134+
domain_size(d) > 0.0
135+
end
136+
end
137+
138+
visu(x, d, Val(:chairmark))
139+
140+
## SECTION - Discrete: benchmarks and chairmarks
141+
142+
@info "Running checks: Discrete"
143+
144+
d = deepcopy(d_commons)
145+
d[:tags] = [:discrete]
146+
147+
x = @check :benchmark d begin
148+
using ConstraintDomains
149+
end begin
150+
d1 = domain([4, 3, 2, 1])
151+
d2 = domain(1)
152+
foreach(i -> add!(d2, i), 2:4)
153+
domains = [d1, d2]
154+
for d in domains
155+
for x in [1, 2, 3, 4]
156+
x ∈ d
157+
end
158+
length(d) == 4
159+
rand(d) ∈ d
160+
add!(d, 5)
161+
5 ∈ d
162+
delete!(d, 5)
163+
5 ∉ d
164+
domain_size(d) == 3
165+
end
166+
167+
d3 = domain(1:5)
168+
d4 = domain(1:0.5:5)
169+
domains2 = [d3, d4]
170+
for d in domains2
171+
for x in [1, 2, 3, 4, 5]
172+
x ∈ d
173+
end
174+
for x in [42]
175+
x ∉ d
176+
end
177+
rand(d) ∈ d
178+
end
179+
end
180+
181+
visu(x, d, Val(:benchmark))
182+
183+
x = @check :chairmark d begin
184+
using ConstraintDomains
185+
end begin
186+
d1 = domain([4, 3, 2, 1])
187+
d2 = domain(1)
188+
foreach(i -> add!(d2, i), 2:4)
189+
domains = [d1, d2]
190+
for d in domains
191+
for x in [1, 2, 3, 4]
192+
x ∈ d
193+
end
194+
length(d) == 4
195+
rand(d) ∈ d
196+
add!(d, 5)
197+
5 ∈ d
198+
delete!(d, 5)
199+
5 ∉ d
200+
domain_size(d) == 3
201+
end
202+
203+
d3 = domain(1:5)
204+
d4 = domain(1:0.5:5)
205+
domains2 = [d3, d4]
206+
for d in domains2
207+
for x in [1, 2, 3, 4, 5]
208+
x ∈ d
209+
end
210+
for x in [42]
211+
x ∉ d
212+
end
213+
rand(d) ∈ d
214+
end
215+
end
216+
217+
visu(x, d, Val(:chairmark))
218+
219+
## SECTION - Explore: benchmarks and chairmarks
220+
221+
@info "Running checks: Explore"
222+
223+
d = deepcopy(d_commons)
224+
d[:tags] = [:explore]
225+
d[:pkgs] = ("ConstraintDomains", :custom, [v"0.3.1", v"0.3.10"], true)
226+
227+
# x = @check :allocs d begin
228+
# using ConstraintDomains
229+
# end begin
230+
# domains = [domain([1, 2, 3, 4]) for i = 1:4]
231+
# X, X̅ = explore(domains, allunique)
232+
# length(X) == factorial(4)
233+
# length(X̅) == 4^4 - factorial(4)
234+
# end
235+
236+
# visu(x, d, Val(:allocs))
237+
238+
x = @check :benchmark d begin
239+
using ConstraintDomains
240+
end begin
241+
domains = [domain([1, 2, 3, 4]) for i = 1:4]
242+
X, X̅ = explore(domains, allunique)
243+
length(X) == factorial(4)
244+
length(X̅) == 4^4 - factorial(4)
245+
end
246+
247+
visu(x, d, Val(:benchmark))
248+
249+
x = @check :chairmark d begin
250+
using ConstraintDomains
251+
end begin
252+
domains = [domain([1, 2, 3, 4]) for i = 1:4]
253+
X, X̅ = explore(domains, allunique)
254+
length(X) == factorial(4)
255+
length(X̅) == 4^4 - factorial(4)
256+
end
257+
258+
visu(x, d, Val(:chairmark))
259+
260+
# TODO: add more checks for parameters.jl
261+
262+
@info "All checks have been successfully run!"

‎perf/metadata/metadata.csv

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
benchmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
2+
benchmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
3+
benchmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
4+
chairmark_ConstraintDomains_v0.2.5_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
5+
chairmark_ConstraintDomains_v0.3.0_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
6+
chairmark_ConstraintDomains_v0.3.10_common,36c9068f-2e67-410b-b8bb-d57c1b476f62
7+
benchmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
8+
benchmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
9+
benchmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
10+
chairmark_ConstraintDomains_v0.2.5_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
11+
chairmark_ConstraintDomains_v0.3.0_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
12+
chairmark_ConstraintDomains_v0.3.10_continuous,36c9068f-2e67-410b-b8bb-d57c1b476f62
13+
benchmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
14+
benchmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
15+
benchmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
16+
chairmark_ConstraintDomains_v0.2.5_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
17+
chairmark_ConstraintDomains_v0.3.0_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
18+
chairmark_ConstraintDomains_v0.3.10_discrete,36c9068f-2e67-410b-b8bb-d57c1b476f62
19+
benchmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
20+
benchmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
21+
chairmark_ConstraintDomains_v0.3.1_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62
22+
chairmark_ConstraintDomains_v0.3.10_explore,36c9068f-2e67-410b-b8bb-d57c1b476f62

‎perf/output/01d3078e-3e06-59b8-b7a8-13146a1184de.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
4.7375e-6,0.0,1632.0,19.0
3+
3.3084000000000004e-6,0.0,1632.0,19.0
4+
1.9851e-6,0.0,1632.0,19.0
5+
2.0482e-6,0.0,1632.0,19.0
6+
1.8231000000000002e-6,0.0,1632.0,19.0
7+
1.7413000000000002e-6,0.0,1632.0,19.0
8+
1.7787e-6,0.0,1632.0,19.0
9+
1.7996e-6,0.0,1632.0,19.0
10+
1.8254000000000001e-6,0.0,1632.0,19.0
11+
1.7064e-6,0.0,1632.0,19.0

‎perf/output/039713ce-acec-5012-a235-f497b94aca3c.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
4785.7,0.0,1632,19
3+
3461.3,0.0,1632,19
4+
2710.9,0.0,1632,19
5+
1916.1,0.0,1632,19
6+
2028.3,0.0,1632,19
7+
2050.8,0.0,1632,19
8+
1767.3,0.0,1632,19
9+
1749.7,0.0,1632,19
10+
1824.1,0.0,1632,19
11+
1808.7,0.0,1632,19

‎perf/output/06768c48-4ab9-5bb9-a034-980e923d7fac.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
4.9568000000000006e-6,0.0,864.0,27.0
3+
2.5383e-6,0.0,864.0,27.0
4+
1.3226e-6,0.0,864.0,27.0
5+
1.2310000000000002e-6,0.0,864.0,27.0
6+
1.1021e-6,0.0,864.0,27.0
7+
1.2353e-6,0.0,864.0,27.0
8+
1.1795e-6,0.0,864.0,27.0
9+
1.052e-6,0.0,864.0,27.0
10+
1.0814e-6,0.0,864.0,27.0
11+
1.0822e-6,0.0,864.0,27.0

‎perf/output/0d9e0caa-8170-543c-9628-2ad63c457e1f.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
8.7,0.0,0,0
3+
24.0,0.0,0,0
4+
4.3,0.0,0,0
5+
4.9,0.0,0,0
6+
5.4,0.0,0,0
7+
3.8,0.0,0,0
8+
3.7,0.0,0,0
9+
3.8,0.0,0,0
10+
3.8,0.0,0,0
11+
4.6,0.0,0,0

‎perf/output/19661d6c-8c80-5bd8-bfd1-220f92ef21fb.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
5.193300000000001e-6,0.0,864.0,27.0
3+
2.4154e-6,0.0,864.0,27.0
4+
1.0913e-6,0.0,864.0,27.0
5+
1.1673000000000002e-6,0.0,864.0,27.0
6+
1.1393e-6,0.0,864.0,27.0
7+
1.1409e-6,0.0,864.0,27.0
8+
1.1716000000000001e-6,0.0,864.0,27.0
9+
1.1899e-6,0.0,864.0,27.0
10+
1.0990000000000002e-6,0.0,864.0,27.0
11+
1.1807e-6,0.0,864.0,27.0

‎perf/output/1c54877a-7fa4-53db-8195-00f829bc16a0.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,memory,allocs
2+
40823.0,0.0,41392,309
3+
42830.1,0.0,41392,309
4+
38844.4,0.0,41392,309
5+
37552.7,0.0,41392,309
6+
36134.9,0.0,41392,309
7+
36296.1,0.0,41392,309
8+
35766.0,0.0,41392,309
9+
36251.5,0.0,41392,309
10+
35749.3,0.0,41392,309
11+
35883.3,0.0,41392,309

‎perf/output/3615ebfa-aa5f-5fc4-b45d-28558621f4bb.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
2.5900000000000004e-8,0.0,0.0,0.0
3+
2.49e-8,0.0,0.0,0.0
4+
5.2e-9,0.0,0.0,0.0
5+
5.4e-9,0.0,0.0,0.0
6+
3.9e-9,0.0,0.0,0.0
7+
3.8e-9,0.0,0.0,0.0
8+
3.9e-9,0.0,0.0,0.0
9+
3.8e-9,0.0,0.0,0.0
10+
3.7e-9,0.0,0.0,0.0
11+
3.8e-9,0.0,0.0,0.0

‎perf/output/431f4cb4-beae-56d3-b340-80762efacf8a.csv

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
times,gctimes,bytes,allocs
2+
1.2562700000000001e-5,0.0,784.0,9.0
3+
1.14067e-5,0.0,784.0,9.0
4+
1.04715e-5,0.0,784.0,9.0
5+
1.04492e-5,0.0,784.0,9.0
6+
1.0432300000000002e-5,0.0,784.0,9.0
7+
1.04338e-5,0.0,784.0,9.0
8+
1.04493e-5,0.0,784.0,9.0
9+
1.13867e-5,0.0,784.0,9.0
10+
1.0136800000000001e-5,0.0,784.0,9.0
11+
1.0012500000000001e-5,0.0,784.0,9.0

0 commit comments

Comments
 (0)