Skip to content

Commit 6a54100

Browse files
maxepmaxep
authored
Merge pull request #1977 from DataDog/feature/continuous-benchmarks
RUM-5551 Add Benchmark Tests Project
2 parents 088d191 + 092ae30 commit 6a54100

29 files changed

+1468
-96
lines changed

‎.gitlab-ci.yml

+19
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ stages:
55
- ui-test
66
- smoke-test
77
- e2e-test
8+
- benchmark-test
89
- dogfood
910
- release-build
1011
- release-publish
@@ -266,6 +267,24 @@ E2E Test (upload to s8s):
266267
- export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
267268
- make e2e-build-upload ARTIFACTS_PATH="artifacts/e2e"
268269

270+
# ┌────────────────────────────┐
271+
# │ Benchmark Test app upload: │
272+
# └────────────────────────────┘
273+
274+
Benchmark Test (upload to s8s):
275+
stage: benchmark-test
276+
rules:
277+
- if: '$CI_COMMIT_BRANCH == $DEVELOP_BRANCH'
278+
artifacts:
279+
paths:
280+
- artifacts
281+
expire_in: 2 weeks
282+
script:
283+
- ./tools/runner-setup.sh --xcode "$DEFAULT_XCODE" --datadog-ci
284+
- make clean
285+
- export DRY_RUN=${DRY_RUN:-0} # default to 0 if not specified
286+
- make benchmark-build-upload ARTIFACTS_PATH="artifacts/benchmark"
287+
269288
# ┌─────────────────┐
270289
# │ SDK dogfooding: │
271290
# └─────────────────┘

‎BenchmarkTests/BenchmarkTests.xcodeproj/.xcodesamplecode.plist

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
<?xml version="1.0" encoding="UTF-8"?> <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> <plist version="1.0"> <array/> </plist>

‎BenchmarkTests/BenchmarkTests.xcodeproj/project.pbxproj

+544
Large diffs are not rendered by default.

‎BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/contents.xcworkspacedata

+7
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎BenchmarkTests/BenchmarkTests.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist

+8
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
3+
<plist version="1.0">
4+
<dict>
5+
<key>IDEDidComputeMac32BitWarning</key>
6+
<true/>
7+
</dict>
8+
</plist>

‎BenchmarkTests/BenchmarkTests.xcodeproj/xcshareddata/xcschemes/Runner.xcscheme

+78
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<Scheme
3+
LastUpgradeVersion = "1540"
4+
version = "1.7">
5+
<BuildAction
6+
parallelizeBuildables = "YES"
7+
buildImplicitDependencies = "YES"
8+
buildArchitectures = "Automatic">
9+
<BuildActionEntries>
10+
<BuildActionEntry
11+
buildForTesting = "YES"
12+
buildForRunning = "YES"
13+
buildForProfiling = "YES"
14+
buildForArchiving = "YES"
15+
buildForAnalyzing = "YES">
16+
<BuildableReference
17+
BuildableIdentifier = "primary"
18+
BlueprintIdentifier = "D29F754C2C4AA07E00288638"
19+
BuildableName = "Runner.app"
20+
BlueprintName = "Runner"
21+
ReferencedContainer = "container:BenchmarkTests.xcodeproj">
22+
</BuildableReference>
23+
</BuildActionEntry>
24+
</BuildActionEntries>
25+
</BuildAction>
26+
<TestAction
27+
buildConfiguration = "Debug"
28+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
29+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
30+
shouldUseLaunchSchemeArgsEnv = "YES"
31+
shouldAutocreateTestPlan = "YES">
32+
</TestAction>
33+
<LaunchAction
34+
buildConfiguration = "Debug"
35+
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
36+
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
37+
launchStyle = "0"
38+
useCustomWorkingDirectory = "NO"
39+
ignoresPersistentStateOnLaunch = "NO"
40+
debugDocumentVersioning = "YES"
41+
debugServiceExtension = "internal"
42+
allowLocationSimulation = "YES">
43+
<BuildableProductRunnable
44+
runnableDebuggingMode = "0">
45+
<BuildableReference
46+
BuildableIdentifier = "primary"
47+
BlueprintIdentifier = "D29F754C2C4AA07E00288638"
48+
BuildableName = "Runner.app"
49+
BlueprintName = "Runner"
50+
ReferencedContainer = "container:BenchmarkTests.xcodeproj">
51+
</BuildableReference>
52+
</BuildableProductRunnable>
53+
</LaunchAction>
54+
<ProfileAction
55+
buildConfiguration = "Release"
56+
shouldUseLaunchSchemeArgsEnv = "YES"
57+
savedToolIdentifier = ""
58+
useCustomWorkingDirectory = "NO"
59+
debugDocumentVersioning = "YES">
60+
<BuildableProductRunnable
61+
runnableDebuggingMode = "0">
62+
<BuildableReference
63+
BuildableIdentifier = "primary"
64+
BlueprintIdentifier = "D29F754C2C4AA07E00288638"
65+
BuildableName = "Runner.app"
66+
BlueprintName = "Runner"
67+
ReferencedContainer = "container:BenchmarkTests.xcodeproj">
68+
</BuildableReference>
69+
</BuildableProductRunnable>
70+
</ProfileAction>
71+
<AnalyzeAction
72+
buildConfiguration = "Debug">
73+
</AnalyzeAction>
74+
<ArchiveAction
75+
buildConfiguration = "Release"
76+
revealArchiveInOrganizer = "YES">
77+
</ArchiveAction>
78+
</Scheme>

‎BenchmarkTests/Benchmarks/Package.swift

+57
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// swift-tools-version: 5.9
2+
// The swift-tools-version declares the minimum version of Swift required to build this package.
3+
4+
import PackageDescription
5+
import Foundation
6+
7+
let package = Package(
8+
name: "DatadogBenchmarks",
9+
products: [
10+
.library(
11+
name: "DatadogBenchmarks",
12+
targets: ["Benchmarks"]
13+
)
14+
]
15+
)
16+
17+
func addOpenTelemetryDependency(_ version: Version) {
18+
// The project must be open with the 'OTEL_SWIFT' env variable.
19+
// Please run 'make benchmark-tests-open' from the root directory.
20+
//
21+
// Note: Carthage will still try to resolve dependencies of Xcode projects in
22+
// sub directories, in this case the project will depend on the default
23+
// 'DataDog/opentelemetry-swift-packages' depedency.
24+
if ProcessInfo.processInfo.environment["OTEL_SWIFT"] != nil {
25+
package.dependencies = [
26+
.package(url: "https://github.com/open-telemetry/opentelemetry-swift", exact: version)
27+
]
28+
29+
package.targets = [
30+
.target(
31+
name: "Benchmarks",
32+
dependencies: [
33+
.product(name: "OpenTelemetryApi", package: "opentelemetry-swift"),
34+
.product(name: "OpenTelemetrySdk", package: "opentelemetry-swift")
35+
],
36+
swiftSettings: [.unsafeFlags(["-DOTEL_SWIFT"])]
37+
)
38+
]
39+
40+
} else {
41+
package.dependencies = [
42+
.package(url: "https://github.com/DataDog/opentelemetry-swift-packages", exact: version)
43+
]
44+
45+
package.targets = [
46+
.target(
47+
name: "Benchmarks",
48+
dependencies: [
49+
.product(name: "OpenTelemetryApi", package: "opentelemetry-swift-packages")
50+
],
51+
swiftSettings: [.unsafeFlags(["-DOTEL_API"])]
52+
)
53+
]
54+
}
55+
}
56+
57+
addOpenTelemetryDependency("1.6.0")

‎BenchmarkTests/Benchmarks/Sources/DatadogExporter.swift

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0.
3+
* This product includes software developed at Datadog (https://www.datadoghq.com/).
4+
* Copyright 2019-Present Datadog, Inc.
5+
*/
6+
7+
import Foundation
8+
9+
#if OTEL_API
10+
#error("Benchmarks depends on opentelemetry-swift. Please open the project with 'make benchmark-tests-open'.")
11+
#endif
12+
13+
#if OTEL_SWIFT
14+
import OpenTelemetrySdk
15+
16+
public final class DatadogExporter: SpanExporter, MetricExporter {
17+
private let session: URLSession
18+
19+
public convenience init() {
20+
let configuration: URLSessionConfiguration = .ephemeral
21+
configuration.urlCache = nil
22+
self.init(session: URLSession(configuration: configuration))
23+
}
24+
25+
public init(session: URLSession) {
26+
self.session = session
27+
}
28+
29+
public func export(spans: [SpanData]) -> SpanExporterResultCode {
30+
return .success
31+
}
32+
33+
public func export(metrics: [Metric], shouldCancel: (() -> Bool)?) -> MetricExporterResultCode {
34+
return .success
35+
}
36+
37+
public func flush() -> SpanExporterResultCode {
38+
return .success
39+
}
40+
41+
public func shutdown() {
42+
43+
}
44+
}
45+
46+
#endif

‎BenchmarkTests/Makefile

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
.PHONY: clean archive export upload
2+
3+
REPO_ROOT := ../
4+
include ../tools/utils/common.mk
5+
6+
BUILD_DIR := .build
7+
ARCHIVE_PATH := $(BUILD_DIR)/Runner.xcarchive
8+
IPA_PATH := $(ARTIFACTS_PATH)/Runner.ipa
9+
10+
clean:
11+
@$(ECHO_SUBTITLE2) "make clean"
12+
rm -rf "$(BUILD_DIR)"
13+
ifdef ARTIFACTS_PATH
14+
rm -rf "$(IPA_PATH)"
15+
endif
16+
17+
archive:
18+
@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
19+
@$(ECHO_SUBTITLE2) "make archive VERSION='$(VERSION)'"
20+
@xcrun agvtool new-version "$(VERSION)"
21+
set -eo pipefail; \
22+
OTEL_SWIFT=1 xcodebuild \
23+
-project BenchmarkTests.xcodeproj \
24+
-scheme Runner \
25+
-sdk iphoneos \
26+
-configuration Synthetics \
27+
-destination generic/platform=iOS \
28+
-archivePath $(ARCHIVE_PATH) \
29+
archive | xcbeautify
30+
@$(ECHO_SUCCESS) "Archive ready in '$(ARCHIVE_PATH)'"
31+
32+
export:
33+
@$(call require_param,ARTIFACTS_PATH)
34+
@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
35+
@$(ECHO_SUBTITLE2) "make export VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
36+
set -o pipefaill; \
37+
xcodebuild -exportArchive \
38+
-archivePath $(ARCHIVE_PATH) \
39+
-exportOptionsPlist exportOptions.plist \
40+
-exportPath $(BUILD_DIR) \
41+
| xcbeautify
42+
mkdir -p "$(ARTIFACTS_PATH)"
43+
cp -v "$(BUILD_DIR)/Runner.ipa" "$(IPA_PATH)"
44+
@$(ECHO_SUCCESS) "IPA exported to '$(IPA_PATH)'"
45+
46+
upload:
47+
@$(call require_param,ARTIFACTS_PATH)
48+
@$(call require_param,DATADOG_API_KEY)
49+
@$(call require_param,DATADOG_APP_KEY)
50+
@$(call require_param,S8S_APPLICATION_ID)
51+
@:$(eval VERSION ?= $(CURRENT_GIT_COMMIT_SHORT))
52+
@$(ECHO_SUBTITLE2) "make upload VERSION='$(VERSION)' ARTIFACTS_PATH='$(ARTIFACTS_PATH)'"
53+
datadog-ci synthetics upload-application \
54+
--mobileApp "$(IPA_PATH)" \
55+
--mobileApplicationId "${S8S_APPLICATION_ID}" \
56+
--versionName "$(VERSION)" \
57+
--latest
58+
59+
open:
60+
@$(ECHO_SUBTITLE2) "make open"
61+
@open --new --env OTEL_SWIFT BenchmarkTests.xcodeproj

‎BenchmarkTests/README.md

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
# Benchmark Tests
2+
3+
[Synthetics for Mobile](https://docs.datadoghq.com/mobile_app_testing/) runs Benchmark test scenarios to collect metrics of the SDK performances.
4+
5+
6+
## CI
7+
8+
CI continuously builds, signs, and uploads a runner application to Synthetics which runs predefined tests.
9+
10+
### Build
11+
12+
Before building the application, make sure the `BenchmarkTests/xcconfigs/Benchmark.local.xcconfig` configuration file is present and contains the `Mobile - Integration Org` client token, RUM application ID, and API Key. These values are sensitive and must be securely stored.
13+
14+
```ini
15+
CLIENT_TOKEN=
16+
RUM_APPLICATION_ID=
17+
API_KEY=
18+
```
19+
20+
### Sign
21+
22+
To sign the runner application, the certificate and provision profile defined in [Synthetics.xcconfig](xcconfigs/Synthetics.xcconfig) and in [exportOptions.plist](exportOptions.plist) needs to be installed on the build machine. The certificate and profile are sensitive files and must be securely stored. Make sure to update both files when updating the certificate and provisioning profile, otherwise signing fails.
23+
24+
> [!NOTE]
25+
> Certificate & Provisioning Profile are also available through the [App Store Connect API](https://developer.apple.com/documentation/appstoreconnectapi). But we don't have the tooling in place.
26+
27+
### Upload
28+
29+
The application version (build number) is set to the commit SHA of the current job, and the build is uploaded to Synthetics using the [datadog-ci](https://github.com/DataDog/datadog-ci) CLI. This step expects environment variables to authenticate with the `Mobile - Integration Org`:
30+
31+
```bash
32+
export DATADOG_API_KEY=
33+
export DATADOG_APP_KEY=
34+
export S8S_APPLICATION_ID=
35+
```
36+
37+
## Development
38+
39+
Each scenario is independent and can be considered as an app within the runner.
40+
41+
### Create a scenario
42+
43+
A scenario must comply with the [`Scenario`](Runner/Scenarios/Scenario.swift) protocol. Upon start, a scenario initializes the SDK, enables features, and returns a root view-controller.
44+
45+
Here is a simple example of a scenario using Logs:
46+
```swift
47+
import Foundation
48+
import UIKit
49+
50+
import DatadogCore
51+
import DatadogLogs
52+
53+
struct LogsScenario: Scenario {
54+
55+
func start(info: TestInfo) -> UIViewController {
56+
57+
Datadog.initialize(
58+
with: .benchmark(info: info), // SDK init with the benchmark configuration
59+
trackingConsent: .granted
60+
)
61+
62+
Logs.enable()
63+
64+
return LoggerViewController()
65+
}
66+
}
67+
```
68+
69+
Add the test to the [`SyntheticScenario`](Runner/Scenarios/Scenario.swift) enumeration so it can be selected, either manually or by setting the `BENCHMARK_SCENARIO` environment variable.
70+
71+
### Synthetics Configuration
72+
73+
Please refer to [Confluence page (internal)](https://datadoghq.atlassian.net/wiki/spaces/RUMP/pages/3981476482/Benchmarks+iOS)

0 commit comments

Comments
 (0)