From 0a5b47629fde487b008a4458d02a2994867b66cd Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Fri, 13 Oct 2023 13:05:44 +0300 Subject: [PATCH 01/11] Add Benchmarks --- .../PrometheusBenchmarks/Benchmarks.swift | 97 +++++++++++++++ Benchmarks/Package.resolved | 113 ++++++++++++++++++ Benchmarks/Package.swift | 38 ++++++ 3 files changed, 248 insertions(+) create mode 100644 Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift create mode 100644 Benchmarks/Package.resolved create mode 100644 Benchmarks/Package.swift diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift new file mode 100644 index 0000000..10ac212 --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift @@ -0,0 +1,97 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +let benchmarks = { + Benchmark.defaultConfiguration.maxDuration = .seconds(5) + Benchmark.defaultConfiguration.scalingFactor = .kilo + + let registry = PrometheusCollectorRegistry() + + func metricsDimensions(_ idx: Int) -> [(String, String)] { + [ + ("job", "api_server_\(idx)"), + ("handler", "/api/handler_\(idx)"), + ("status_code", "200"), + ("version", "\(idx).0.0"), + ] + } + + Benchmark("1 - Metrics: Counter benchmark") { benchmark in + let ctr = registry.makeCounter(name: "counter_1", labels: metricsDimensions(1)) + benchmark.startMeasurement() + for _ in benchmark.scaledIterations { + blackHole(ctr.increment()) + } + } + + Benchmark("2 - Metrics: Gauge benchmark") { benchmark in + let gauge = registry.makeGauge(name: "gauge_1", labels: metricsDimensions(2)) + benchmark.startMeasurement() + for _ in benchmark.scaledIterations { + blackHole(gauge.increment()) + } + } + + Benchmark("3 - Metrics: Histogram benchmark") { benchmark in + let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: metricsDimensions(3), + buckets: [ + .milliseconds(100), + .milliseconds(250), + .milliseconds(500), + .seconds(1), + ]) + benchmark.startMeasurement() + for _ in benchmark.scaledIterations { + histogram.record(Duration.milliseconds(400)) + } + } + + Benchmark("4 - Metrics: export 5000 metrics", + configuration: .init(scalingFactor: .one)) { benchmark in + let metricsCount = 5000 + + let registryExport = PrometheusCollectorRegistry() + + var counterArray = [Counter]() + var gaugeArray = [Gauge]() + var buffer = [UInt8]() + + let counterExportSize = 620_000 + counterArray.reserveCapacity(metricsCount) + gaugeArray.reserveCapacity(metricsCount) + buffer.reserveCapacity(counterExportSize) + + for i in 0..<(metricsCount / 2) { + let counter = registryExport.makeCounter(name: "http_requests_total", labels: metricsDimensions(i)) + counter.increment() + counterArray.append(counter) + + let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: metricsDimensions(i)) + gauge.increment() + gaugeArray.append(gauge) + } + + benchmark.startMeasurement() + for _ in benchmark.scaledIterations { + blackHole(registryExport.emit(into: &buffer)) + + } + benchmark.stopMeasurement() + buffer.removeAll(keepingCapacity: true) + } +} diff --git a/Benchmarks/Package.resolved b/Benchmarks/Package.resolved new file mode 100644 index 0000000..eb89cb0 --- /dev/null +++ b/Benchmarks/Package.resolved @@ -0,0 +1,113 @@ +{ + "pins" : [ + { + "identity" : "package-benchmark", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-benchmark.git", + "state" : { + "revision" : "070dee37abe72c0a3d7e0feff5dca3effd66b541", + "version" : "1.11.2" + } + }, + { + "identity" : "package-datetime", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-datetime", + "state" : { + "revision" : "d1242188c9f48aad297e6ca9b717776f8660bc31", + "version" : "1.0.2" + } + }, + { + "identity" : "package-histogram", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-histogram", + "state" : { + "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", + "version" : "0.1.2" + } + }, + { + "identity" : "package-jemalloc", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/package-jemalloc", + "state" : { + "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", + "version" : "1.0.0" + } + }, + { + "identity" : "progress.swift", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/Progress.swift", + "state" : { + "revision" : "29dc5dc29d8408f42878b832c7aae38a35ff26ee", + "version" : "1.0.3" + } + }, + { + "identity" : "swift-argument-parser", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-argument-parser", + "state" : { + "revision" : "8f4d2753f0e4778c76d5f05ad16c74f707390531", + "version" : "1.2.3" + } + }, + { + "identity" : "swift-atomics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-atomics.git", + "state" : { + "revision" : "cd142fd2f64be2100422d658e7411e39489da985", + "version" : "1.2.0" + } + }, + { + "identity" : "swift-extras-json", + "kind" : "remoteSourceControl", + "location" : "https://github.com/swift-extras/swift-extras-json", + "state" : { + "revision" : "122b9454ef01bf89a4c190b8fd3717ddd0a2fbd0", + "version" : "0.6.0" + } + }, + { + "identity" : "swift-metrics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-metrics.git", + "state" : { + "revision" : "971ba26378ab69c43737ee7ba967a896cb74c0d1", + "version" : "2.4.1" + } + }, + { + "identity" : "swift-numerics", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-numerics", + "state" : { + "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", + "version" : "1.0.2" + } + }, + { + "identity" : "swift-system", + "kind" : "remoteSourceControl", + "location" : "https://github.com/apple/swift-system", + "state" : { + "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", + "version" : "1.2.1" + } + }, + { + "identity" : "texttable", + "kind" : "remoteSourceControl", + "location" : "https://github.com/ordo-one/TextTable", + "state" : { + "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", + "version" : "0.0.2" + } + } + ], + "version" : 2 +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift new file mode 100644 index 0000000..4cee4b4 --- /dev/null +++ b/Benchmarks/Package.swift @@ -0,0 +1,38 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import PackageDescription + +let package = Package( + name: "benchmarks", + platforms: [ + .macOS("13"), + ], + dependencies: [ + .package(path: "../"), + .package(url: "https://github.com/ordo-one/package-benchmark.git", from: "1.11.2"), + ], + targets: [ + .executableTarget( + name: "PrometheusBenchmarks", + dependencies: [ + .product(name: "Benchmark", package: "package-benchmark"), + .product(name: "BenchmarkPlugin", package: "package-benchmark"), + .product(name: "Prometheus", package: "swift-prometheus"), + ], + path: "Benchmarks/PrometheusBenchmarks" + ), + ] +) From 998da01543091dc0a96cf89f41ddceb2090ec009 Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Fri, 13 Oct 2023 18:22:12 +0300 Subject: [PATCH 02/11] Refactor and add docker scripts --- Benchmarks/.gitignore | 9 ++ .../PrometheusBenchmarks/Benchmarks.swift | 89 ++++++------------- .../PrometheusBenchmarks/Counter.swift | 31 +++++++ .../DurationHistogram.swift | 30 +++++++ .../PrometheusBenchmarks/Gauge.swift | 31 +++++++ .../PrometheusBenchmarks/RegistryEmit.swift | 43 +++++++++ Benchmarks/Package.swift | 8 +- .../PrometheusBenchmarks.Counter_#1.p90.json | 3 + .../PrometheusBenchmarks.Counter_#2.p90.json | 3 + ...theusBenchmarks.DurationHistogram.p90.json | 3 + .../5.9/PrometheusBenchmarks.Gauge.p90.json | 3 + ...marks.RegistryEmit_-_5000_metrics.p90.json | 3 + README.md | 14 +++ dev/update-benchmark-tresholds.sh | 41 +++++++++ docker/docker-compose.2204.57.yaml | 5 ++ docker/docker-compose.2204.58.yaml | 5 ++ docker/docker-compose.2204.59.yaml | 9 +- docker/docker-compose.2204.main.yaml | 3 + docker/docker-compose.yaml | 6 +- 19 files changed, 273 insertions(+), 66 deletions(-) create mode 100644 Benchmarks/.gitignore create mode 100644 Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift create mode 100644 Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift create mode 100644 Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift create mode 100644 Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift create mode 100644 Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json create mode 100644 Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json create mode 100644 Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json create mode 100644 Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json create mode 100644 Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json create mode 100755 dev/update-benchmark-tresholds.sh diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore new file mode 100644 index 0000000..785db78 --- /dev/null +++ b/Benchmarks/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +/.build +/Packages +xcuserdata/ +DerivedData/ +.swiftpm/configuration/registries.json +.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata +.netrc +.benchmarkBaselines/ diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift index 10ac212..29d9d0b 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift @@ -16,82 +16,49 @@ import Benchmark import Prometheus +let registry = PrometheusCollectorRegistry() + +public func makeLabels(_ idx: Int) -> [(String, String)] { + [ + ("job", "api_server_\(idx)"), + ("handler", "/api/handler_\(idx)"), + ("status_code", "200"), + ("version", "\(idx).0.0"), + ] +} + let benchmarks = { Benchmark.defaultConfiguration.maxDuration = .seconds(5) Benchmark.defaultConfiguration.scalingFactor = .kilo + // Benchmark.defaultConfiguration.metrics = [.wallClock, .throughput, .mallocCountTotal] + Benchmark.defaultConfiguration.metrics = [.mallocCountTotal] - let registry = PrometheusCollectorRegistry() - - func metricsDimensions(_ idx: Int) -> [(String, String)] { - [ - ("job", "api_server_\(idx)"), - ("handler", "/api/handler_\(idx)"), - ("status_code", "200"), - ("version", "\(idx).0.0"), - ] + Benchmark("Counter #1") { benchmark in + runCounterBench(benchmark.scaledIterations) } - Benchmark("1 - Metrics: Counter benchmark") { benchmark in - let ctr = registry.makeCounter(name: "counter_1", labels: metricsDimensions(1)) - benchmark.startMeasurement() + Benchmark("Counter #2") { benchmark, run in for _ in benchmark.scaledIterations { - blackHole(ctr.increment()) + run() } + } setup: { + setupCounterBench() } - Benchmark("2 - Metrics: Gauge benchmark") { benchmark in - let gauge = registry.makeGauge(name: "gauge_1", labels: metricsDimensions(2)) - benchmark.startMeasurement() - for _ in benchmark.scaledIterations { - blackHole(gauge.increment()) - } + Benchmark("Gauge") { benchmark in + runGaugeBench(benchmark.scaledIterations) } - Benchmark("3 - Metrics: Histogram benchmark") { benchmark in - let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: metricsDimensions(3), - buckets: [ - .milliseconds(100), - .milliseconds(250), - .milliseconds(500), - .seconds(1), - ]) - benchmark.startMeasurement() - for _ in benchmark.scaledIterations { - histogram.record(Duration.milliseconds(400)) - } + Benchmark("DurationHistogram") { benchmark in + runDurationHistogramBench(benchmark.scaledIterations) } - Benchmark("4 - Metrics: export 5000 metrics", - configuration: .init(scalingFactor: .one)) { benchmark in - let metricsCount = 5000 - - let registryExport = PrometheusCollectorRegistry() - - var counterArray = [Counter]() - var gaugeArray = [Gauge]() - var buffer = [UInt8]() - - let counterExportSize = 620_000 - counterArray.reserveCapacity(metricsCount) - gaugeArray.reserveCapacity(metricsCount) - buffer.reserveCapacity(counterExportSize) - - for i in 0..<(metricsCount / 2) { - let counter = registryExport.makeCounter(name: "http_requests_total", labels: metricsDimensions(i)) - counter.increment() - counterArray.append(counter) - - let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: metricsDimensions(i)) - gauge.increment() - gaugeArray.append(gauge) - } - - benchmark.startMeasurement() + Benchmark("RegistryEmit - 5000 metrics", + configuration: .init(scalingFactor: .one)) { benchmark, run in for _ in benchmark.scaledIterations { - blackHole(registryExport.emit(into: &buffer)) - + run() } - benchmark.stopMeasurement() - buffer.removeAll(keepingCapacity: true) + } setup: { + setupRegistryExport(numberOfMetrics: 5000) } } diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift new file mode 100644 index 0000000..a3f76a0 --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift @@ -0,0 +1,31 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runCounterBench(_ iterations: Range) { + let ctr = registry.makeCounter(name: "counter_1", labels: makeLabels(1)) + for _ in iterations { + blackHole(ctr.increment()) + } +} + +public func setupCounterBench() -> () -> Void { + let ctr = registry.makeCounter(name: "counter_2", labels: makeLabels(2)) + return { + blackHole(ctr.increment()) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift new file mode 100644 index 0000000..9e6c0f3 --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift @@ -0,0 +1,30 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runDurationHistogramBench(_ iterations: Range) { + let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: makeLabels(3), + buckets: [ + .milliseconds(100), + .milliseconds(250), + .milliseconds(500), + .seconds(1), + ]) + for _ in iterations { + blackHole(histogram.record(Duration.milliseconds(400))) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift new file mode 100644 index 0000000..2cf34eb --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift @@ -0,0 +1,31 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func runGaugeBench(_ iterations: Range) { + let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2)) + for _ in iterations { + blackHole(gauge.increment()) + } +} + +public func setupGaugeBench() -> () -> Void { + let gauge = registry.makeGauge(name: "gauge_1", labels: makeLabels(2)) + return { + blackHole(gauge.increment()) + } +} diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift new file mode 100644 index 0000000..8b92add --- /dev/null +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift @@ -0,0 +1,43 @@ +// swift-tools-version:5.7 +//===----------------------------------------------------------------------===// +// +// This source file is part of the SwiftPrometheus open source project +// +// Copyright (c) 2018-2023 SwiftPrometheus project authors +// Licensed under Apache License v2.0 +// +// See LICENSE.txt for license information +// See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors +// +// SPDX-License-Identifier: Apache-2.0 +// +//===----------------------------------------------------------------------===// + +import Benchmark +import Prometheus + +public func setupRegistryExport(numberOfMetrics: Int) -> () -> Void { + let registryExport = PrometheusCollectorRegistry() + + var counterArray = [Counter]() + var gaugeArray = [Gauge]() + var buffer = [UInt8]() + + let counterExportSize = 620_000 + counterArray.reserveCapacity(numberOfMetrics) + gaugeArray.reserveCapacity(numberOfMetrics) + buffer.reserveCapacity(counterExportSize) + + for i in 0..<(numberOfMetrics / 2) { + let counter = registryExport.makeCounter(name: "http_requests_total", labels: makeLabels(i)) + counter.increment() + counterArray.append(counter) + + let gauge = registryExport.makeGauge(name: "export_gauge_\(i)", labels: makeLabels(i)) + gauge.increment() + gaugeArray.append(gauge) + } + return { + blackHole(registryExport.emit(into: &buffer)) + } +} diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 4cee4b4..9674c1b 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -18,7 +18,7 @@ import PackageDescription let package = Package( name: "benchmarks", platforms: [ - .macOS("13"), + .macOS("14"), ], dependencies: [ .package(path: "../"), @@ -29,10 +29,12 @@ let package = Package( name: "PrometheusBenchmarks", dependencies: [ .product(name: "Benchmark", package: "package-benchmark"), - .product(name: "BenchmarkPlugin", package: "package-benchmark"), .product(name: "Prometheus", package: "swift-prometheus"), ], - path: "Benchmarks/PrometheusBenchmarks" + path: "Benchmarks/PrometheusBenchmarks", + plugins: [ + .plugin(name: "BenchmarkPlugin", package: "package-benchmark"), + ] ), ] ) diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/README.md b/README.md index 7aab78a..195418b 100644 --- a/README.md +++ b/README.md @@ -22,3 +22,17 @@ If you find a bug or have issues, please [create an issue](https://github.com/sw [Documentation]: https://swiftpackageindex.com/swift-server/swift-prometheus/documentation/prometheus [SSWG-Incubation]: https://www.swift.org/sswg/incubation-process.html + + +## Benchmarks + +Benchmarks for `swift-prometheus` are in a separate Swift Package in the `Benchmarks` subfolder of this repository. +They use the [`package-benchmark`](https://github.com/ordo-one/package-benchmark) plugin. +Benchmarks depends on the [`jemalloc`](https://jemalloc.net) memory allocation library, which is used by `package-benchmark` to capture memory allocation statistics. +An installation guide can be found in the [Getting Started article](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark/gettingstarted#Installing-Prerequisites-and-Platform-Support) of `package-benchmark`. +Afterwards you can run the benchmarks from CLI by going to the `Benchmarks` subfolder (e.g. `cd Benchmarks`) and invoking: +``` +swift package benchmark +``` + +For more information please refer to `swift package benchmark --help` or the [documentation of `package-benchmark`](https://swiftpackageindex.com/ordo-one/package-benchmark/documentation/benchmark). diff --git a/dev/update-benchmark-tresholds.sh b/dev/update-benchmark-tresholds.sh new file mode 100755 index 0000000..9a37298 --- /dev/null +++ b/dev/update-benchmark-tresholds.sh @@ -0,0 +1,41 @@ +#!/bin/bash +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftNIO open source project +## +## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## +##===----------------------------------------------------------------------===## +## +## This source file is part of the SwiftCertificates open source project +## +## Copyright (c) 2023 Apple Inc. and the SwiftCertificates project authors +## Licensed under Apache License v2.0 +## +## See LICENSE.txt for license information +## See CONTRIBUTORS.txt for the list of SwiftCertificates project authors +## +## SPDX-License-Identifier: Apache-2.0 +## +##===----------------------------------------------------------------------===## + +set -eu +set -o pipefail + +here="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +target_repo=${2-"$here/.."} + +for f in 57 58 59 510 -nightly; do + echo "swift$f" + + docker_file=$(if [[ "$f" == "-nightly" ]]; then f=main; fi && ls "$target_repo/docker/docker-compose."*"$f"*".yaml") + + docker-compose -f docker/docker-compose.yaml -f $docker_file run update-benchmark-baseline +done diff --git a/docker/docker-compose.2204.57.yaml b/docker/docker-compose.2204.57.yaml index fa96acd..5f1524f 100644 --- a/docker/docker-compose.2204.57.yaml +++ b/docker/docker-compose.2204.57.yaml @@ -17,5 +17,10 @@ services: environment: [] #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.7 + environment: + - SWIFT_VERSION=5.7 + shell: image: prometheus:22.04-5.7 diff --git a/docker/docker-compose.2204.58.yaml b/docker/docker-compose.2204.58.yaml index 1d27d8e..5d653b6 100644 --- a/docker/docker-compose.2204.58.yaml +++ b/docker/docker-compose.2204.58.yaml @@ -18,5 +18,10 @@ services: - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.8 + environment: + - SWIFT_VERSION=5.8 + shell: image: prometheus:22.04-5.8 diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index 1ae4771..e893b44 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -6,7 +6,8 @@ services: image: prometheus:22.04-5.9 build: args: - base_image: "swiftlang/swift:nightly-5.9-jammy" + ubuntu_version: "jammy" + swift_version: "5.9" documentation-check: image: prometheus:22.04-5.9 @@ -14,8 +15,14 @@ services: test: image: prometheus:22.04-5.9 environment: + - SWIFT_VERSION=main - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-5.9 + environment: + - SWIFT_VERSION=5.9 + shell: image: prometheus:22.04-5.9 diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 461eefc..707b242 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -17,5 +17,8 @@ services: - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread + update-benchmark-baseline: + image: prometheus:22.04-main + shell: image: prometheus:22.04-main diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index a53b7e5..5affd44 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -33,7 +33,11 @@ services: test: <<: *common depends_on: [runtime-setup] - command: /bin/bash -xcl "swift test -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" + command: /bin/bash -xcl "swift test -Xswiftc -warnings-as-errors --enable-test-discovery $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-} && cd Benchmarks && swift package benchmark baseline check --check-absolute-path Thresholds/$${SWIFT_VERSION-}/" + + update-benchmark-baseline: + <<: *common + command: /bin/bash -xcl "cd Benchmarks && swift package --disable-sandbox --scratch-path .build/$${SWIFT_VERSION-}/ --allow-writing-to-package-directory benchmark --format metricP90AbsoluteThresholds --path Thresholds/$${SWIFT_VERSION-}/" # util From 43cfe7b0905b126f60af28f3bdbc9ae2dac655d3 Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Mon, 16 Oct 2023 14:30:22 +0300 Subject: [PATCH 03/11] fix tresholds --- Benchmarks/Package.swift | 2 +- .../PrometheusBenchmarks.Counter_#1.p90.json | 3 +++ .../PrometheusBenchmarks.Counter_#2.p90.json | 3 +++ ...theusBenchmarks.DurationHistogram.p90.json | 3 +++ .../5.10/PrometheusBenchmarks.Gauge.p90.json | 3 +++ ...marks.RegistryEmit_-_5000_metrics.p90.json | 3 +++ .../PrometheusBenchmarks.Counter_#1.p90.json | 3 +++ .../PrometheusBenchmarks.Counter_#2.p90.json | 3 +++ ...theusBenchmarks.DurationHistogram.p90.json | 3 +++ .../5.7/PrometheusBenchmarks.Gauge.p90.json | 3 +++ ...marks.RegistryEmit_-_5000_metrics.p90.json | 3 +++ .../PrometheusBenchmarks.Counter_#1.p90.json | 3 +++ .../PrometheusBenchmarks.Counter_#2.p90.json | 3 +++ ...theusBenchmarks.DurationHistogram.p90.json | 3 +++ .../5.8/PrometheusBenchmarks.Gauge.p90.json | 3 +++ ...marks.RegistryEmit_-_5000_metrics.p90.json | 3 +++ docker/Dockerfile | 3 +++ docker/docker-compose.2204.510.yaml | 27 +++++++++++++++++++ docker/docker-compose.yaml | 4 +-- 19 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json create mode 100644 Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json create mode 100644 Benchmarks/Thresholds/5.10/PrometheusBenchmarks.DurationHistogram.p90.json create mode 100644 Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Gauge.p90.json create mode 100644 Benchmarks/Thresholds/5.10/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json create mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json create mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json create mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json create mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json create mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json create mode 100644 Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json create mode 100644 Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json create mode 100644 Benchmarks/Thresholds/5.8/PrometheusBenchmarks.DurationHistogram.p90.json create mode 100644 Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Gauge.p90.json create mode 100644 Benchmarks/Thresholds/5.8/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json create mode 100644 docker/docker-compose.2204.510.yaml diff --git a/Benchmarks/Package.swift b/Benchmarks/Package.swift index 9674c1b..e2bfec4 100644 --- a/Benchmarks/Package.swift +++ b/Benchmarks/Package.swift @@ -18,7 +18,7 @@ import PackageDescription let package = Package( name: "benchmarks", platforms: [ - .macOS("14"), + .macOS(.v13), ], dependencies: [ .package(path: "../"), diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile index 2d1e57d..834cb72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -16,6 +16,9 @@ ENV LANGUAGE en_US.UTF-8 RUN apt-get update && apt-get install -y wget RUN apt-get update && apt-get install -y lsof dnsutils netcat-openbsd net-tools libz-dev curl jq # used by integration tests +# install jemalloc for running allocation benchmarks +RUN apt-get update & apt-get install -y libjemalloc-dev + # ruby and jazzy for docs generation RUN apt-get update && apt-get install -y ruby ruby-dev libsqlite3-dev build-essential # jazzy no longer works on xenial as ruby is too old. diff --git a/docker/docker-compose.2204.510.yaml b/docker/docker-compose.2204.510.yaml new file mode 100644 index 0000000..8e23daf --- /dev/null +++ b/docker/docker-compose.2204.510.yaml @@ -0,0 +1,27 @@ +version: "3" + +services: + + runtime-setup: + image: prometheus:22.04-5.10 + build: + args: + base_image: "swiftlang/swift:nightly-5.10-jammy" + + documentation-check: + image: prometheus:22.04-5.10 + + test: + image: prometheus:22.04-5.10 + environment: + - SWIFT_VERSION=main + - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error + #- SANITIZER_ARG=--sanitize=thread + + update-benchmark-baseline: + image: prometheus:22.04-5.10 + environment: + - SWIFT_VERSION=5.10 + + shell: + image: prometheus:22.04-5.10 diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index 5affd44..c6e81ed 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -16,8 +16,8 @@ services: depends_on: [runtime-setup] volumes: - ~/.ssh:/root/.ssh - - ..:/code:z - working_dir: /code + - ..:/swift-prometheus:z + working_dir: /swift-prometheus cap_drop: - CAP_NET_RAW - CAP_NET_BIND_SERVICE From 3032be0d006b83fa943970d473f1b8fcb26a1a44 Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Mon, 16 Oct 2023 16:06:54 +0300 Subject: [PATCH 04/11] fix main thresholds --- .../Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json | 3 +++ .../Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json | 3 +++ .../main/PrometheusBenchmarks.DurationHistogram.p90.json | 3 +++ Benchmarks/Thresholds/main/PrometheusBenchmarks.Gauge.p90.json | 3 +++ .../PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json | 3 +++ docker/docker-compose.2204.59.yaml | 2 +- docker/docker-compose.2204.main.yaml | 2 ++ 7 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.DurationHistogram.p90.json create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.Gauge.p90.json create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.DurationHistogram.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.DurationHistogram.p90.json new file mode 100644 index 0000000..282dc64 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.DurationHistogram.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 2 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Gauge.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Gauge.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Gauge.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json new file mode 100644 index 0000000..9f9de44 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.RegistryEmit_-_5000_metrics.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 0 +} \ No newline at end of file diff --git a/docker/docker-compose.2204.59.yaml b/docker/docker-compose.2204.59.yaml index e893b44..488c255 100644 --- a/docker/docker-compose.2204.59.yaml +++ b/docker/docker-compose.2204.59.yaml @@ -15,7 +15,7 @@ services: test: image: prometheus:22.04-5.9 environment: - - SWIFT_VERSION=main + - SWIFT_VERSION=5.9 - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error #- SANITIZER_ARG=--sanitize=thread diff --git a/docker/docker-compose.2204.main.yaml b/docker/docker-compose.2204.main.yaml index 707b242..12e3474 100644 --- a/docker/docker-compose.2204.main.yaml +++ b/docker/docker-compose.2204.main.yaml @@ -19,6 +19,8 @@ services: update-benchmark-baseline: image: prometheus:22.04-main + environment: + - SWIFT_VERSION=main shell: image: prometheus:22.04-main From 9454c98a56d5e1929bc05fde3a3852dbd06b7dd1 Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:52:14 +0300 Subject: [PATCH 05/11] address feedback --- .../PrometheusBenchmarks/Benchmarks.swift | 5 ++--- .../DurationHistogram.swift | 17 ++++++++++------- ...enchmarks.Counter_-_increment_only.p90.json} | 0 ...arks.Counter_-_setup_and_increment.p90.json} | 0 ...enchmarks.Counter_-_increment_only.p90.json} | 0 ...arks.Counter_-_setup_and_increment.p90.json} | 0 ...rks.Counter_-_setup_and_increment_.p90.json} | 0 ...enchmarks.Counter_-_increment_only.p90.json} | 0 ...arks.Counter_-_setup_and_increment.p90.json} | 0 ...enchmarks.Counter_-_increment_only.p90.json} | 0 ...arks.Counter_-_setup_and_increment.p90.json} | 0 ...enchmarks.Counter_-_increment_only.p90.json} | 0 ...marks.Counter_-_setup_and_increment.p90.json | 3 +++ 13 files changed, 15 insertions(+), 10 deletions(-) rename Benchmarks/Thresholds/5.10/{PrometheusBenchmarks.Counter_#2.p90.json => PrometheusBenchmarks.Counter_-_increment_only.p90.json} (100%) rename Benchmarks/Thresholds/5.10/{PrometheusBenchmarks.Counter_#1.p90.json => PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json} (100%) rename Benchmarks/Thresholds/5.7/{PrometheusBenchmarks.Counter_#2.p90.json => PrometheusBenchmarks.Counter_-_increment_only.p90.json} (100%) rename Benchmarks/Thresholds/5.7/{PrometheusBenchmarks.Counter_#1.p90.json => PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json} (100%) rename Benchmarks/Thresholds/{5.8/PrometheusBenchmarks.Counter_#1.p90.json => 5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json} (100%) rename Benchmarks/Thresholds/5.8/{PrometheusBenchmarks.Counter_#2.p90.json => PrometheusBenchmarks.Counter_-_increment_only.p90.json} (100%) rename Benchmarks/Thresholds/{5.9/PrometheusBenchmarks.Counter_#1.p90.json => 5.8/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json} (100%) rename Benchmarks/Thresholds/5.9/{PrometheusBenchmarks.Counter_#2.p90.json => PrometheusBenchmarks.Counter_-_increment_only.p90.json} (100%) rename Benchmarks/Thresholds/{main/PrometheusBenchmarks.Counter_#1.p90.json => 5.9/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json} (100%) rename Benchmarks/Thresholds/main/{PrometheusBenchmarks.Counter_#2.p90.json => PrometheusBenchmarks.Counter_-_increment_only.p90.json} (100%) create mode 100644 Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift index 29d9d0b..4db7072 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift @@ -30,14 +30,13 @@ public func makeLabels(_ idx: Int) -> [(String, String)] { let benchmarks = { Benchmark.defaultConfiguration.maxDuration = .seconds(5) Benchmark.defaultConfiguration.scalingFactor = .kilo - // Benchmark.defaultConfiguration.metrics = [.wallClock, .throughput, .mallocCountTotal] Benchmark.defaultConfiguration.metrics = [.mallocCountTotal] - Benchmark("Counter #1") { benchmark in + Benchmark("Counter - setup and increment") { benchmark in runCounterBench(benchmark.scaledIterations) } - Benchmark("Counter #2") { benchmark, run in + Benchmark("Counter - increment only") { benchmark, run in for _ in benchmark.scaledIterations { run() } diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift index 9e6c0f3..3a571cc 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift @@ -17,13 +17,16 @@ import Benchmark import Prometheus public func runDurationHistogramBench(_ iterations: Range) { - let histogram = registry.makeDurationHistogram(name: "histogram_1", labels: makeLabels(3), - buckets: [ - .milliseconds(100), - .milliseconds(250), - .milliseconds(500), - .seconds(1), - ]) + let histogram = registry.makeDurationHistogram( + name: "histogram_1", + labels: makeLabels(3), + buckets: [ + .milliseconds(100), + .milliseconds(250), + .milliseconds(500), + .seconds(1), + ] + ) for _ in iterations { blackHole(histogram.record(Duration.milliseconds(400))) } diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_increment_only.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#2.p90.json rename to Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_increment_only.p90.json diff --git a/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_#1.p90.json rename to Benchmarks/Thresholds/5.10/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_increment_only.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#2.p90.json rename to Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_increment_only.p90.json diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_#1.p90.json rename to Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#1.p90.json rename to Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json diff --git a/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_-_increment_only.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_#2.p90.json rename to Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_-_increment_only.p90.json diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#1.p90.json rename to Benchmarks/Thresholds/5.8/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json diff --git a/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_-_increment_only.p90.json similarity index 100% rename from Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_#2.p90.json rename to Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_-_increment_only.p90.json diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json b/Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json similarity index 100% rename from Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#1.p90.json rename to Benchmarks/Thresholds/5.9/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_increment_only.p90.json similarity index 100% rename from Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_#2.p90.json rename to Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_increment_only.p90.json diff --git a/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json new file mode 100644 index 0000000..557fc88 --- /dev/null +++ b/Benchmarks/Thresholds/main/PrometheusBenchmarks.Counter_-_setup_and_increment.p90.json @@ -0,0 +1,3 @@ +{ + "mallocCountTotal" : 1 +} \ No newline at end of file From 293c5555930fe64bed21550506f78f83b33f4066 Mon Sep 17 00:00:00 2001 From: freef4ll <103502659+freef4ll@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:52:44 +0300 Subject: [PATCH 06/11] Delete Benchmarks/Package.resolved --- Benchmarks/Package.resolved | 113 ------------------------------------ 1 file changed, 113 deletions(-) delete mode 100644 Benchmarks/Package.resolved diff --git a/Benchmarks/Package.resolved b/Benchmarks/Package.resolved deleted file mode 100644 index eb89cb0..0000000 --- a/Benchmarks/Package.resolved +++ /dev/null @@ -1,113 +0,0 @@ -{ - "pins" : [ - { - "identity" : "package-benchmark", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-benchmark.git", - "state" : { - "revision" : "070dee37abe72c0a3d7e0feff5dca3effd66b541", - "version" : "1.11.2" - } - }, - { - "identity" : "package-datetime", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-datetime", - "state" : { - "revision" : "d1242188c9f48aad297e6ca9b717776f8660bc31", - "version" : "1.0.2" - } - }, - { - "identity" : "package-histogram", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-histogram", - "state" : { - "revision" : "a69fa24d7b70421870cafa86340ece900489e17e", - "version" : "0.1.2" - } - }, - { - "identity" : "package-jemalloc", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/package-jemalloc", - "state" : { - "revision" : "e8a5db026963f5bfeac842d9d3f2cc8cde323b49", - "version" : "1.0.0" - } - }, - { - "identity" : "progress.swift", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/Progress.swift", - "state" : { - "revision" : "29dc5dc29d8408f42878b832c7aae38a35ff26ee", - "version" : "1.0.3" - } - }, - { - "identity" : "swift-argument-parser", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-argument-parser", - "state" : { - "revision" : "8f4d2753f0e4778c76d5f05ad16c74f707390531", - "version" : "1.2.3" - } - }, - { - "identity" : "swift-atomics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-atomics.git", - "state" : { - "revision" : "cd142fd2f64be2100422d658e7411e39489da985", - "version" : "1.2.0" - } - }, - { - "identity" : "swift-extras-json", - "kind" : "remoteSourceControl", - "location" : "https://github.com/swift-extras/swift-extras-json", - "state" : { - "revision" : "122b9454ef01bf89a4c190b8fd3717ddd0a2fbd0", - "version" : "0.6.0" - } - }, - { - "identity" : "swift-metrics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-metrics.git", - "state" : { - "revision" : "971ba26378ab69c43737ee7ba967a896cb74c0d1", - "version" : "2.4.1" - } - }, - { - "identity" : "swift-numerics", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-numerics", - "state" : { - "revision" : "0a5bc04095a675662cf24757cc0640aa2204253b", - "version" : "1.0.2" - } - }, - { - "identity" : "swift-system", - "kind" : "remoteSourceControl", - "location" : "https://github.com/apple/swift-system", - "state" : { - "revision" : "025bcb1165deab2e20d4eaba79967ce73013f496", - "version" : "1.2.1" - } - }, - { - "identity" : "texttable", - "kind" : "remoteSourceControl", - "location" : "https://github.com/ordo-one/TextTable", - "state" : { - "revision" : "a27a07300cf4ae322e0079ca0a475c5583dd575f", - "version" : "0.0.2" - } - } - ], - "version" : 2 -} From fdeb454279f3e6fdce0fb98925e4453739b2e0fa Mon Sep 17 00:00:00 2001 From: freef4ll <103502659+freef4ll@users.noreply.github.com> Date: Wed, 25 Oct 2023 15:57:41 +0300 Subject: [PATCH 07/11] Delete Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json --- ...rometheusBenchmarks.Counter_-_setup_and_increment_.p90.json | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json diff --git a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json b/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json deleted file mode 100644 index 557fc88..0000000 --- a/Benchmarks/Thresholds/5.7/PrometheusBenchmarks.Counter_-_setup_and_increment_.p90.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "mallocCountTotal" : 1 -} \ No newline at end of file From e63303e3e178fc90076f861e82529be9ac9f4e5b Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:09:36 +0300 Subject: [PATCH 08/11] drop swift-version copy and paste on benchmark files --- Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift | 1 - Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift | 1 - .../Benchmarks/PrometheusBenchmarks/DurationHistogram.swift | 1 - Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift | 1 - Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift | 1 - 5 files changed, 5 deletions(-) diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift index 4db7072..7715438 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Benchmarks.swift @@ -1,4 +1,3 @@ -// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftPrometheus open source project diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift index a3f76a0..4e5493a 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Counter.swift @@ -1,4 +1,3 @@ -// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftPrometheus open source project diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift index 3a571cc..c6de28a 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/DurationHistogram.swift @@ -1,4 +1,3 @@ -// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftPrometheus open source project diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift index 2cf34eb..6c8be83 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/Gauge.swift @@ -1,4 +1,3 @@ -// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftPrometheus open source project diff --git a/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift index 8b92add..333872f 100644 --- a/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift +++ b/Benchmarks/Benchmarks/PrometheusBenchmarks/RegistryEmit.swift @@ -1,4 +1,3 @@ -// swift-tools-version:5.7 //===----------------------------------------------------------------------===// // // This source file is part of the SwiftPrometheus open source project From 7a8e280c01721a8ed856334f17f68d61e8e051bc Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:14:04 +0300 Subject: [PATCH 09/11] SwiftNIO -> SwiftPrometheus license fix up --- dev/update-benchmark-tresholds.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dev/update-benchmark-tresholds.sh b/dev/update-benchmark-tresholds.sh index 9a37298..49bf1ad 100755 --- a/dev/update-benchmark-tresholds.sh +++ b/dev/update-benchmark-tresholds.sh @@ -1,13 +1,13 @@ #!/bin/bash ##===----------------------------------------------------------------------===## ## -## This source file is part of the SwiftNIO open source project +## This source file is part of the SwiftPrometheus open source project ## -## Copyright (c) 2023 Apple Inc. and the SwiftNIO project authors +## Copyright (c) 2023 Apple Inc. and the SwiftPrometheus project authors ## Licensed under Apache License v2.0 ## ## See LICENSE.txt for license information -## See CONTRIBUTORS.txt for the list of SwiftNIO project authors +## See CONTRIBUTORS.txt for the list of SwiftPrometheus project authors ## ## SPDX-License-Identifier: Apache-2.0 ## From 3a7018836bfad634594f5530a129341967dd221b Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Thu, 26 Oct 2023 16:15:27 +0300 Subject: [PATCH 10/11] license fix up --- dev/update-benchmark-tresholds.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/update-benchmark-tresholds.sh b/dev/update-benchmark-tresholds.sh index 49bf1ad..d66bb1f 100755 --- a/dev/update-benchmark-tresholds.sh +++ b/dev/update-benchmark-tresholds.sh @@ -3,7 +3,7 @@ ## ## This source file is part of the SwiftPrometheus open source project ## -## Copyright (c) 2023 Apple Inc. and the SwiftPrometheus project authors +## Copyright (c) 2023 SwiftPrometheus project authors ## Licensed under Apache License v2.0 ## ## See LICENSE.txt for license information From a15bd27ce8d0da197f1655d03ad46cffc4b0bfd5 Mon Sep 17 00:00:00 2001 From: freefr4ll <103502659+freef4ll@users.noreply.github.com> Date: Fri, 10 Nov 2023 15:56:51 +0300 Subject: [PATCH 11/11] add .build to ignore --- Benchmarks/.gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/Benchmarks/.gitignore b/Benchmarks/.gitignore index 785db78..d3ff854 100644 --- a/Benchmarks/.gitignore +++ b/Benchmarks/.gitignore @@ -1,4 +1,5 @@ .DS_Store +.build /.build /Packages xcuserdata/