Skip to content

Commit abc5768

Browse files
niklasberglundniklasberglund
committed
Change the way iOS end to end tests workflows are set up
1 parent 94c1da4 commit abc5768

15 files changed

+393
-123
lines changed

‎.github/actions/ios-end-to-end-tests/action.yml ‎.github/actions/build-ios-e2e-tests/action.yml

+18-22
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
name: 'iOS end to end tests action'
2-
description: 'Prepares and runs end to end tests on iOS device'
1+
name: 'Build iOS end to end tests action'
2+
description: 'Prepares and builds end to end tests on iOS device'
33
inputs:
44
ios_device_pin_code:
55
description: 'iOS Device Pin Code'
@@ -16,25 +16,19 @@ inputs:
1616
test_device_udid:
1717
description: 'Test Device UDID'
1818
required: true
19-
xcode_test_plan:
20-
description: 'Xcode Test Plan to run'
21-
required: true
2219
partner_api_token:
2320
description: 'Partner API Token'
2421
required: true
2522
test_name:
2623
description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
2724
required: false
25+
outputs_path:
26+
description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.'
27+
required: true
2828

2929
runs:
3030
using: 'composite'
3131
steps:
32-
- name: Make sure app is not installed
33-
run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
34-
shell: bash
35-
env:
36-
IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
37-
3832
- name: Configure Xcode project
3933
run: |
4034
for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
@@ -46,12 +40,21 @@ runs:
4640
sed -i "" \
4741
"/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
4842
UITests.xcconfig
43+
sed -i "" \
44+
"s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \
45+
UITests.xcconfig
4946
sed -i "" \
5047
"/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \
5148
UITests.xcconfig
5249
sed -i "" \
5350
"/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \
5451
UITests.xcconfig
52+
sed -i "" \
53+
"/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \
54+
UITests.xcconfig
55+
sed -i "" \
56+
"/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \
57+
UITests.xcconfig
5558
shell: bash
5659
working-directory: ios/Configurations
5760
env:
@@ -61,7 +64,7 @@ runs:
6164
NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
6265
PARTNER_API_TOKEN: ${{ inputs.partner_api_token }}
6366

64-
- name: Run end-to-end-tests
67+
- name: Build app and tests for testing
6568
run: |
6669
if [ -n "$TEST_NAME" ]; then
6770
TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
@@ -71,19 +74,12 @@ runs:
7174
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
7275
-project MullvadVPN.xcodeproj \
7376
-scheme MullvadVPNUITests \
74-
-testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \
75-
-resultBundlePath xcode-test-report \
77+
-testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \
7678
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
77-
clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report
79+
-derivedDataPath derived-data \
80+
clean build-for-testing 2>&1
7881
shell: bash
7982
working-directory: ios/
8083
env:
81-
XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
8284
TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
8385
TEST_NAME: ${{ inputs.test_name }}
84-
85-
- name: Uninstall app if still installed
86-
run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
87-
shell: bash
88-
env:
89-
IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}

‎.github/actions/run-ios-e2e-tests/action.yml

+74
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
name: 'Run iOS end to end tests action'
2+
description: 'Runs end to end tests on iOS device'
3+
inputs:
4+
test_name:
5+
description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
6+
required: false
7+
test_device_udid:
8+
description: 'Test Device UDID'
9+
required: true
10+
outputs_path:
11+
description: 'Path to where outputs are stored - both build outputs and outputs from running tests. This should be unique for each job run in order to avoid concurrency issues.'
12+
required: true
13+
14+
runs:
15+
using: 'composite'
16+
steps:
17+
# Set up a unique output directory
18+
- name: Set up outputs directory
19+
run: |
20+
if [ -n "$TEST_NAME" ]; then
21+
# Strip slashes to avoid creating subdirectories
22+
test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g')
23+
echo "Setting output directory tests-output-test-name-sanitized"
24+
echo "$test_name_sanitized"
25+
test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized"
26+
else
27+
echo "Setting output directory output"
28+
test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output"
29+
fi
30+
31+
echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV
32+
echo "TEST_NAME_SANITIZED=$test_name_sanitized" >> $GITHUB_ENV
33+
shell: bash
34+
env:
35+
TEST_NAME: ${{ inputs.test_name }}
36+
OUTPUTS_PATH: ${{ inputs.outputs_path }}
37+
38+
- name: Uninstall app
39+
run: ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
40+
shell: bash
41+
env:
42+
TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
43+
44+
- name: Run end-to-end-tests
45+
run: |
46+
if [ -n "$TEST_NAME" ]; then
47+
TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
48+
else
49+
TEST_NAME_ARGUMENT=""
50+
fi
51+
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
52+
-project MullvadVPN.xcodeproj \
53+
-scheme MullvadVPNUITests \
54+
-testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \
55+
-resultBundlePath ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report \
56+
-derivedDataPath derived-data \
57+
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
58+
test-without-building 2>&1 #| xcbeautify --report junit --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report
59+
shell: bash
60+
working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios
61+
env:
62+
TEST_NAME: ${{ inputs.test_name }}
63+
TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
64+
65+
- name: Store test report artifact
66+
if: always()
67+
uses: actions/upload-artifact@v4
68+
with:
69+
name: ${{ env.TEST_NAME_SANITIZED }}-test-results
70+
path: |
71+
${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report/junit.xml
72+
${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report.xcresult
73+
env:
74+
TEST_NAME: ${{ inputs.test_name }}

‎.github/workflows/ios-end-to-end-tests-api.yml

+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
name: iOS end-to-end API tests
3+
on:
4+
workflow_dispatch:
5+
jobs:
6+
reuse-e2e-workflow:
7+
uses: ./.github/workflows/ios-end-to-end-tests.yml
8+
with:
9+
arg_tests_json_key: "api-tests"

‎.github/workflows/ios-end-to-end-tests-merge-to-main.yml

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
---
2+
name: iOS end-to-end merge to main tests
3+
on:
4+
workflow_dispatch:
5+
pull_request:
6+
types:
7+
- closed
8+
branches:
9+
- main
10+
paths:
11+
- .github/workflows/ios-end-to-end-tests*.yml
12+
- ios/**
13+
jobs:
14+
reuse-e2e-workflow:
15+
uses: ./.github/workflows/ios-end-to-end-tests.yml
16+
with:
17+
arg_tests_json_key: "pr-merge-to-main"

‎.github/workflows/ios-end-to-end-tests-nightly.yml

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
---
2+
name: iOS end-to-end nightly tests
3+
on:
4+
workflow_dispatch:
5+
schedule:
6+
# At midnight every day.
7+
# Notifications for scheduled workflows are sent to the user who last modified the cron
8+
# syntax in the workflow file. If you update this you must have notifications for
9+
# Github Actions enabled, so these don't go unnoticed.
10+
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
11+
- cron: '0 0 * * *'
12+
jobs:
13+
reuse-e2e-workflow:
14+
uses: ./.github/workflows/ios-end-to-end-tests.yml
15+
with:
16+
arg_tests_json_key: "nightly"

‎.github/workflows/ios-end-to-end-tests-settings-migration.yml

+12-6
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
---
22
name: iOS settings migration tests
3+
concurrency:
4+
group: ios-end-to-end-tests
5+
cancel-in-progress: false
36
permissions:
47
contents: read
58
on:
@@ -11,6 +14,8 @@ on:
1114
# Github Actions enabled, so these don't go unnoticed.
1215
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
1316
- cron: '0 0 * * *'
17+
env:
18+
TEST_DEVICE_UDID: 00008130-0019181022F3803A
1419
jobs:
1520
test:
1621
name: Settings migration end to end tests
@@ -19,14 +24,15 @@ jobs:
1924
OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280
2025
steps:
2126
- name: Configure Rust
22-
uses: actions-rs/toolchain@v1
27+
uses: actions-rs/toolchain@v1.0.6
2328
with:
2429
toolchain: stable
2530
override: true
2631
target: aarch64-apple-ios
2732

2833
- name: Uninstall app
29-
run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
34+
timeout-minutes: 5
35+
run: ios-deploy --id $${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
3036

3137
- name: Checkout old repository version
3238
uses: actions/checkout@v4
@@ -40,7 +46,7 @@ jobs:
4046
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
4147
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
4248
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
43-
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
49+
test_device_udid: ${{ env.TEST_DEVICE_UDID }}
4450
xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
4551
partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
4652

@@ -62,7 +68,7 @@ jobs:
6268
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
6369
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
6470
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
65-
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
71+
test_device_udid: ${{ env.TEST_DEVICE_UDID }}
6672
partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
6773
xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
6874

@@ -86,7 +92,7 @@ jobs:
8692
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
8793
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
8894
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
89-
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
95+
test_device_udid: ${{ env.TEST_DEVICE_UDID }}
9096
partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
9197
xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
9298

@@ -108,7 +114,7 @@ jobs:
108114
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
109115
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
110116
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
111-
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
117+
test_device_udid: ${{ env.TEST_DEVICE_UDID }}
112118
partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
113119
xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
114120

0 commit comments

Comments
 (0)