From 28b9c4404b7e1e2087546fbc5c1f3dc91dd03796 Mon Sep 17 00:00:00 2001 From: Paul Toffoloni <69189821+ptoffy@users.noreply.github.com> Date: Sat, 30 Mar 2024 19:23:26 +0100 Subject: [PATCH] Update benchmarks (#154) * Update CI * Add command * Remove quiet options * Update benchmark.yml with package path * Update benchmark workflow * Update benchmark thresholds --- .github/workflows/benchmark.yml | 14 ++++--- Benchmarks/Benchmarks/Signing/Signing.swift | 26 ++++++++++-- .../TokenLifecycle/TokenLifecycle.swift | 40 ++++++++++++++++--- .../Benchmarks/Verifying/Verifying.swift | 19 ++++++++- 4 files changed, 83 insertions(+), 16 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 703942ae..d74e22b7 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -20,15 +20,17 @@ jobs: - name: Run benchmarks for PR branch continue-on-error: true run: | - cd Benchmarks - swift package -c release --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update pull_request --no-progress --quiet + swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update pull_request + - name: Run benchmarks for 'main' branch + run: | git stash git checkout main - swift package -c release --allow-writing-to-directory .benchmarkBaselines/ benchmark baseline update main --no-progress --quiet + swift package -c release --package-path Benchmarks --disable-sandbox benchmark baseline update main + - name: Compare benchmarks + continue-on-error: true + run: | date >> "${GITHUB_STEP_SUMMARY}" - swift package -c release benchmark baseline check main pull_request --format markdown >> "${GITHUB_STEP_SUMMARY}" - echo '---' >> "${GITHUB_STEP_SUMMARY}" - swift package -c release benchmark baseline compare main pull_request --no-progress --quiet --format markdown >> "${GITHUB_STEP_SUMMARY}" + swift package -c release --package-path Benchmarks benchmark baseline check main pull_request --format markdown >> "${GITHUB_STEP_SUMMARY}" - name: Get formatted date id: get-date run: echo "date=$(date +'%Y-%m-%d %H:%M:%S')" >> $GITHUB_OUTPUT diff --git a/Benchmarks/Benchmarks/Signing/Signing.swift b/Benchmarks/Benchmarks/Signing/Signing.swift index f95a39c0..a8ee3b9b 100644 --- a/Benchmarks/Benchmarks/Signing/Signing.swift +++ b/Benchmarks/Benchmarks/Signing/Signing.swift @@ -2,8 +2,18 @@ import Benchmark import Foundation import JWTKit +let customThresholds = BenchmarkThresholds( + relative: [.p25: 15.0, .p50: 15.0, .p75: 15.0, .p90: 15.0, .p99: 15.0], + absolute: [:] +) + let benchmarks = { - Benchmark("ES256") { benchmark in + Benchmark( + "ES256", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in let key = ES256PrivateKey() let keyCollection = JWTKeyCollection() await keyCollection.addES256(key: key) @@ -12,7 +22,12 @@ let benchmarks = { } } - Benchmark("RSA") { benchmark in + Benchmark( + "RSA", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in let key = try Insecure.RSA.PrivateKey(pem: rsaPrivateKey) let keyCollection = JWTKeyCollection() await keyCollection.addRS256(key: key) @@ -21,7 +36,12 @@ let benchmarks = { } } - Benchmark("EdDSA") { benchmark in + Benchmark( + "EdDSA", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in let key = try EdDSA.PrivateKey() let keyCollection = JWTKeyCollection() await keyCollection.addEdDSA(key: key) diff --git a/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift b/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift index d00fa0c6..6218d3f9 100644 --- a/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift +++ b/Benchmarks/Benchmarks/TokenLifecycle/TokenLifecycle.swift @@ -2,8 +2,18 @@ import Benchmark import Foundation import JWTKit +let customThresholds = BenchmarkThresholds( + relative: [.p25: 15.0, .p50: 15.0, .p75: 15.0, .p90: 15.0, .p99: 15.0], + absolute: [:] +) + let benchmarks = { - Benchmark("ES256 Generated") { benchmark in + Benchmark( + "ES256 Generated", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in for _ in benchmark.scaledIterations { let key = ES256PrivateKey() let keyCollection = JWTKeyCollection() @@ -13,7 +23,12 @@ let benchmarks = { } } - Benchmark("ES256 PEM") { benchmark in + Benchmark( + "ES256 PEM", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in for _ in benchmark.scaledIterations { let key = try ES256PrivateKey(pem: ecdsaPrivateKey) let keyCollection = JWTKeyCollection() @@ -23,7 +38,12 @@ let benchmarks = { } } - Benchmark("RSA PEM") { benchmark in + Benchmark( + "RSA PEM", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in for _ in benchmark.scaledIterations { let key = try Insecure.RSA.PrivateKey(pem: rsaPrivateKey) let keyCollection = JWTKeyCollection() @@ -33,7 +53,12 @@ let benchmarks = { } } - Benchmark("EdDSA Generated") { benchmark in + Benchmark( + "EdDSA Generated", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in for _ in benchmark.scaledIterations { let key = try EdDSA.PrivateKey() let keyCollection = JWTKeyCollection() @@ -43,7 +68,12 @@ let benchmarks = { } } - Benchmark("EdDSA Coordinates") { benchmark in + Benchmark( + "EdDSA Coordinates", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in for _ in benchmark.scaledIterations { let key = try EdDSA.PrivateKey(x: eddsaPublicKeyBase64Url, d: eddsaPrivateKeyBase64Url, curve: .ed25519) let keyCollection = JWTKeyCollection() diff --git a/Benchmarks/Benchmarks/Verifying/Verifying.swift b/Benchmarks/Benchmarks/Verifying/Verifying.swift index 87a9fb83..408f3c05 100644 --- a/Benchmarks/Benchmarks/Verifying/Verifying.swift +++ b/Benchmarks/Benchmarks/Verifying/Verifying.swift @@ -2,8 +2,18 @@ import Benchmark import Foundation import JWTKit +let customThresholds = BenchmarkThresholds( + relative: [.p25: 15.0, .p50: 15.0, .p75: 15.0, .p90: 15.0, .p99: 15.0], + absolute: [:] +) + let benchmarks = { - Benchmark("ES256") { benchmark in + Benchmark( + "ES256", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in let pem = """ -----BEGIN PUBLIC KEY----- MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEEVs/o5+uQbTjL3chynL4wXgUg2R9 @@ -18,7 +28,12 @@ let benchmarks = { } } - Benchmark("RS256") { benchmark in + Benchmark( + "RS256", + configuration: .init( + thresholds: [.peakMemoryResident: customThresholds] + ) + ) { benchmark in let pem = """ -----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAu1SU1LfVLPHCozMxH2Mo