Skip to content

Commit 7b646eb

Browse files
niklasberglundniklasberglund
committed
Add settings migration tests for iOS
1 parent 217fb88 commit 7b646eb

25 files changed

+805
-78
lines changed

‎.github/actions/ios-end-to-end-tests/action.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: 'iOS end to end tests action'
2+
description: 'Prepares and runs end to end tests on iOS device'
3+
inputs:
4+
ios_device_pin_code:
5+
description: 'iOS Device Pin Code'
6+
required: true
7+
test_device_identifier_uuid:
8+
description: 'Test Device Identifier UUID'
9+
required: true
10+
has_time_account_number:
11+
description: 'Has Time Account Number'
12+
required: true
13+
no_time_account_number:
14+
description: 'No Time Account Number'
15+
required: true
16+
test_device_udid:
17+
description: 'Test Device UDID'
18+
required: true
19+
xcode_test_plan:
20+
description: 'Xcode Test Plan to run'
21+
required: true
22+
23+
runs:
24+
using: 'composite'
25+
steps:
26+
- name: Configure Xcode project
27+
run: |
28+
for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
29+
sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
30+
sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
31+
sed -i "" \
32+
"/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
33+
UITests.xcconfig
34+
sed -i "" \
35+
"/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
36+
UITests.xcconfig
37+
echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
38+
echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
39+
shell: bash
40+
working-directory: ios/Configurations
41+
env:
42+
IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }}
43+
TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }}
44+
HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }}
45+
NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
46+
47+
- name: Run end-to-end-tests
48+
run: |
49+
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
50+
-project MullvadVPN.xcodeproj \
51+
-scheme MullvadVPNUITests \
52+
-testPlan $XCODE_TEST_PLAN \
53+
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
54+
clean test 2>&1 | xcbeautify --report junit --report-path test-report
55+
shell: bash
56+
working-directory: ios/
57+
env:
58+
XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
59+
TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}

‎.github/workflows/ios-end-to-end-tests-settings-migration.yml

+112
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
---
2+
name: iOS settings migration tests
3+
permissions:
4+
contents: read
5+
on:
6+
workflow_dispatch:
7+
schedule:
8+
# At midnight every day.
9+
# Notifications for scheduled workflows are sent to the user who last modified the cron
10+
# syntax in the workflow file. If you update this you must have notifications for
11+
# Github Actions enabled, so these don't go unnoticed.
12+
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
13+
- cron: '0 0 * * *'
14+
jobs:
15+
test:
16+
name: Settings migration end to end tests
17+
runs-on: [self-hosted, macOS, ios-test]
18+
env:
19+
OLD_APP_COMMIT_HASH: fbd13034f2a7dadaa9a404bea62345bd8d65165b
20+
steps:
21+
- name: Update Rust toolchain
22+
run: rustup update
23+
24+
- name: Uninstall app
25+
run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
26+
27+
- name: Checkout old repository version
28+
uses: actions/checkout@v4
29+
with:
30+
ref: ${{ env.OLD_APP_COMMIT_HASH }}
31+
32+
- name: Change DNS settings on old app version
33+
uses: ./.github/actions/ios-end-to-end-tests
34+
with:
35+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
36+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
37+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
38+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
39+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
40+
xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
41+
42+
- name: Store test report for changing DNS settings
43+
uses: actions/upload-artifact@v4
44+
if: always()
45+
with:
46+
name: test-report-change-dns-settings
47+
path: ios/test-report/junit.xml
48+
49+
- name: Checkout repository to get the current app version
50+
uses: actions/checkout@v4
51+
52+
- name: Verify DNS settings still changed on current app version
53+
uses: ./.github/actions/ios-end-to-end-tests
54+
if: always()
55+
with:
56+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
57+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
58+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
59+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
60+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
61+
xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
62+
63+
- name: Store test report for verifying DNS settings
64+
uses: actions/upload-artifact@v4
65+
if: always()
66+
with:
67+
name: test-report-verify-dns-settings
68+
path: ios/test-report/junit.xml
69+
70+
- name: Checkout old repository version
71+
uses: actions/checkout@v4
72+
with:
73+
ref: ${{ env.OLD_APP_COMMIT_HASH }}
74+
75+
- name: Change all other settings on old app version
76+
uses: ./.github/actions/ios-end-to-end-tests
77+
if: always()
78+
with:
79+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
80+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
81+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
82+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
83+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
84+
xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
85+
86+
- name: Store test report for changing all settings
87+
uses: actions/upload-artifact@v4
88+
if: always()
89+
with:
90+
name: test-report-change-all-other-settings
91+
path: ios/test-report/junit.xml
92+
93+
- name: Checkout repository to get the current app version
94+
uses: actions/checkout@v4
95+
96+
- name: Verify all other settings still changed on current app version
97+
uses: ./.github/actions/ios-end-to-end-tests
98+
if: always()
99+
with:
100+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
101+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
102+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
103+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
104+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
105+
xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
106+
107+
- name: Store test report for verifying all other settings
108+
uses: actions/upload-artifact@v4
109+
if: always()
110+
with:
111+
name: test-report-verify-all-other-settings
112+
path: ios/test-report/junit.xml

‎.github/workflows/ios-end-to-end-tests.yml

+28-34
Original file line numberDiff line numberDiff line change
@@ -14,53 +14,47 @@ on:
1414
- .github/workflows/ios-end-to-end-tests.yml
1515
- ios/**
1616
workflow_dispatch:
17+
schedule:
18+
# At midnight every day.
19+
# Notifications for scheduled workflows are sent to the user who last modified the cron
20+
# syntax in the workflow file. If you update this you must have notifications for
21+
# Github Actions enabled, so these don't go unnoticed.
22+
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
23+
- cron: '0 0 * * *'
1724
jobs:
1825
test:
1926
if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch'
2027
name: End to end tests
2128
runs-on: [self-hosted, macOS, ios-test]
22-
env:
23-
IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }}
24-
TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
25-
TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }}
26-
HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
27-
NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
2829
steps:
30+
- name: Update Rust toolchain
31+
run: rustup update
32+
2933
- name: Checkout repository
3034
uses: actions/checkout@v4
3135

32-
- name: Configure Rust
33-
run: rustup target add aarch64-apple-ios aarch64-apple-ios-sim
34-
35-
- name: Configure Xcode project
36+
- name: Select test plan to execute
3637
run: |
37-
for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
38-
sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
39-
sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
40-
sed -i "" \
41-
"/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
42-
UITests.xcconfig
43-
sed -i "" \
44-
"/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
45-
UITests.xcconfig
46-
echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
47-
echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
48-
working-directory: ios/Configurations
38+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
39+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV
40+
elif [[ "${{ github.event_name }}" == "schedule" ]]; then
41+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
42+
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
43+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
44+
fi
4945
50-
- name: Run end-to-end-tests
51-
run: |
52-
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
53-
-project MullvadVPN.xcodeproj \
54-
-scheme MullvadVPNUITests \
55-
-configuration Debug \
56-
-testPlan MullvadVPNUITestsSmoke \
57-
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
58-
-disableAutomaticPackageResolution \
59-
test 2>&1 | xcbeautify --report junit --report-path test-report
60-
working-directory: ios/
46+
- name: iOS end to end tests action
47+
uses: ./.github/actions/ios-end-to-end-tests
48+
with:
49+
xcode_test_plan: ${{ env.XCODE_TEST_PLAN }}
50+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
51+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
52+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
53+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
54+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
6155

6256
- name: Comment PR on test failure
63-
if: failure() && github.event_name != 'workflow_dispatch'
57+
if: failure() && github.event_name == 'pull_request'
6458
uses: actions/github-script@v7
6559
with:
6660
github-token: ${{secrets.GITHUB_TOKEN}}

0 commit comments

Comments
 (0)