Our perf thresholds have changed (#178)

* Our perf thresholds have changed * Apparently local runs don't match * Stop using retain/release thresholds * Remove retain/release counts
apple · Jun 7, 2024 · 4688f24 · 4688f24
1 parent d93756b
commit 4688f24
Show file tree

Hide file tree

Showing 26 changed files with 30 additions and 76 deletions.
diff --git a/Benchmarks/Benchmarks/CertificatesBenchmark/Benchmarks.swift b/Benchmarks/Benchmarks/CertificatesBenchmark/Benchmarks.swift
@@ -23,16 +23,10 @@ let benchmarks = {
             .readSyscalls,
             .writeSyscalls,
             .memoryLeaked,
-            .retainCount,
-            .releaseCount,
         ]
     )
 
-    var configWithoutRetainRelease = Benchmark.defaultConfiguration
-    configWithoutRetainRelease.metrics.removeAll(where: { $0 == .retainCount || $0 == .releaseCount })
-
-    // async code is currently still quite flaky in the number of retain/release it does so we don't measure them today
-    Benchmark("Verifier", configuration: configWithoutRetainRelease) { benchmark in
+    Benchmark("Verifier") { benchmark in
         for _ in benchmark.scaledIterations {
             await verifier()
         }

diff --git a/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json b/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 5506,
+  "mallocCountTotal" : 4851,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 120084,
-  "retainCount" : 108506,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json b/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6465,
+  "mallocCountTotal" : 5810,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 133927,
-  "retainCount" : 120568,
   "writeSyscalls" : 0
 }
diff --git a/...rks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json b/...rks/Thresholds/5.10/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6473,
+  "mallocCountTotal" : 5818,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 133256,
-  "retainCount" : 120160,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.10/CertificatesBenchmark.TinyArray.append.p90.json b/Benchmarks/Thresholds/5.10/CertificatesBenchmark.TinyArray.append.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 10,
+  "mallocCountTotal" : 26,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 12,
-  "retainCount" : 0,
   "writeSyscalls" : 0
-}
+}
diff --git a/Benchmarks/Thresholds/5.10/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.10/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 0,
+  "mallocCountTotal" : 16,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 0,
-  "retainCount" : 0,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.10/CertificatesBenchmark.Verifier.p90.json
@@ -1,5 +1,5 @@
 {
-  "mallocCountTotal" : 1233,
+  "mallocCountTotal" : 1249,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
   "writeSyscalls" : 0

diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 5506,
+  "mallocCountTotal" : 4851,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 190789,
-  "retainCount" : 180211,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6465,
+  "mallocCountTotal" : 5810,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 208611,
-  "retainCount" : 196252,
   "writeSyscalls" : 0
 }
diff --git a/...arks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json b/...arks/Thresholds/5.8/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6473,
+  "mallocCountTotal" : 5818,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 207940,
-  "retainCount" : 195844,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray.append.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray.append.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 10,
+  "mallocCountTotal" : 26,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 12,
-  "retainCount" : 0,
   "writeSyscalls" : 0
-}
+}
diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 0,
+  "mallocCountTotal" : 16,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 1,
-  "retainCount" : 0,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.8/CertificatesBenchmark.Verifier.p90.json
@@ -1,5 +1,5 @@
 {
-  "mallocCountTotal" : 1234,
+  "mallocCountTotal" : 1250,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
   "writeSyscalls" : 0

diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 5506,
+  "mallocCountTotal" : 4851,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 185130,
-  "retainCount" : 174552,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6465,
+  "mallocCountTotal" : 5810,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 199110,
-  "retainCount" : 186751,
   "writeSyscalls" : 0
 }
diff --git a/...arks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json b/...arks/Thresholds/5.9/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6473,
+  "mallocCountTotal" : 5818,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 198439,
-  "retainCount" : 186343,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray.append.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray.append.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 10,
+  "mallocCountTotal" : 26,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 12,
-  "retainCount" : 0,
   "writeSyscalls" : 0
-}
+}
diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 0,
+  "mallocCountTotal" : 16,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 1,
-  "retainCount" : 0,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/5.9/CertificatesBenchmark.Verifier.p90.json
@@ -1,5 +1,5 @@
 {
-  "mallocCountTotal" : 1233,
+  "mallocCountTotal" : 1249,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
   "writeSyscalls" : 0

diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_DER.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 5506,
+  "mallocCountTotal" : 4851,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 121084,
-  "retainCount" : 109506,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_PEM_files.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6465,
+  "mallocCountTotal" : 5810,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 134927,
-  "retainCount" : 121568,
   "writeSyscalls" : 0
 }
diff --git a/...rks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json b/...rks/Thresholds/main/CertificatesBenchmark.Parse_WebPKI_Roots_from_multi_PEM_file.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 6473,
+  "mallocCountTotal" : 5818,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 134256,
-  "retainCount" : 121160,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray.append.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray.append.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 10,
+  "mallocCountTotal" : 26,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 12,
-  "retainCount" : 0,
   "writeSyscalls" : 0
-}
+}
diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.TinyArray_non-allocating_functions.p90.json
@@ -1,8 +1,6 @@
 {
-  "mallocCountTotal" : 0,
+  "mallocCountTotal" : 16,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
-  "releaseCount" : 0,
-  "retainCount" : 0,
   "writeSyscalls" : 0
 }
diff --git a/Benchmarks/Thresholds/main/CertificatesBenchmark.Verifier.p90.json b/Benchmarks/Thresholds/main/CertificatesBenchmark.Verifier.p90.json
@@ -1,5 +1,5 @@
 {
-  "mallocCountTotal" : 1233,
+  "mallocCountTotal" : 1251,
   "memoryLeaked" : 0,
   "readSyscalls" : 0,
   "writeSyscalls" : 0

diff --git a/dev/update-benchmark-thresholds b/dev/update-benchmark-thresholds
@@ -19,7 +19,7 @@ set -o pipefail
 here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 target_repo=${2-"$here/.."}
 
-for f in 57 58 59 510 -nightly; do
+for f in 58 59 510 -nightly; do
     echo "swift$f"
 
     docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml")