Skip to content

Commit 98cd1b8

Browse files
niklasberglundniklasberglund
committed
Add settings migration tests for iOS
1 parent 0a45b98 commit 98cd1b8

24 files changed

+783
-95
lines changed

‎.github/actions/ios-end-to-end-tests/action.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
name: 'iOS end to end tests action'
2+
description: 'Prepares and runs end to end tests on iOS device'
3+
inputs:
4+
ios_device_pin_code:
5+
description: 'iOS Device Pin Code'
6+
required: true
7+
test_device_identifier_uuid:
8+
description: 'Test Device Identifier UUID'
9+
required: true
10+
has_time_account_number:
11+
description: 'Has Time Account Number'
12+
required: true
13+
no_time_account_number:
14+
description: 'No Time Account Number'
15+
required: true
16+
test_device_udid:
17+
description: 'Test Device UDID'
18+
required: true
19+
xcode_test_plan:
20+
description: 'Xcode Test Plan to run'
21+
required: true
22+
23+
runs:
24+
using: 'composite'
25+
steps:
26+
- name: Configure Xcode project
27+
run: |
28+
for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
29+
sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
30+
sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
31+
sed -i "" \
32+
"/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
33+
UITests.xcconfig
34+
sed -i "" \
35+
"/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
36+
UITests.xcconfig
37+
echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
38+
echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
39+
shell: bash
40+
working-directory: ios/Configurations
41+
env:
42+
IOS_DEVICE_PIN_CODE: ${{ inputs.ios_device_pin_code }}
43+
TEST_DEVICE_IDENTIFIER_UUID: ${{ inputs.test_device_identifier_uuid }}
44+
HAS_TIME_ACCOUNT_NUMBER: ${{ inputs.has_time_account_number }}
45+
NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
46+
47+
- name: Run end-to-end-tests
48+
run: |
49+
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
50+
-project MullvadVPN.xcodeproj \
51+
-scheme MullvadVPNUITests \
52+
-testPlan $XCODE_TEST_PLAN \
53+
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
54+
clean test 2>&1 | xcbeautify --report junit --report-path test-report
55+
shell: bash
56+
working-directory: ios/
57+
env:
58+
XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
59+
TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}

‎.github/workflows/ios-end-to-end-tests-settings-migration.yml

+109
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
---
2+
name: iOS settings migration tests
3+
permissions:
4+
contents: read
5+
on:
6+
workflow_dispatch:
7+
schedule:
8+
# At midnight every day.
9+
# Notifications for scheduled workflows are sent to the user who last modified the cron
10+
# syntax in the workflow file. If you update this you must have notifications for
11+
# Github Actions enabled, so these don't go unnoticed.
12+
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
13+
- cron: '0 0 * * *'
14+
jobs:
15+
test:
16+
name: Settings migration end to end tests
17+
runs-on: [self-hosted, macOS, ios-test]
18+
env:
19+
OLD_APP_COMMIT_HASH: fbd13034f2a7dadaa9a404bea62345bd8d65165b
20+
steps:
21+
- name: Uninstall app
22+
run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
23+
24+
- name: Checkout old repository version
25+
uses: actions/checkout@v4
26+
with:
27+
ref: ${{ env.OLD_APP_COMMIT_HASH }}
28+
29+
- name: Change DNS settings on old app version
30+
uses: ./.github/actions/ios-end-to-end-tests
31+
with:
32+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
33+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
34+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
35+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
36+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
37+
xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
38+
39+
- name: Store test report for changing DNS settings
40+
uses: actions/upload-artifact@v4
41+
if: always()
42+
with:
43+
name: test-report-change-dns-settings
44+
path: ios/test-report/junit.xml
45+
46+
- name: Checkout repository to get the current app version
47+
uses: actions/checkout@v4
48+
49+
- name: Verify DNS settings still changed on current app version
50+
uses: ./.github/actions/ios-end-to-end-tests
51+
if: always()
52+
with:
53+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
54+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
55+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
56+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
57+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
58+
xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
59+
60+
- name: Store test report for verifying DNS settings
61+
uses: actions/upload-artifact@v4
62+
if: always()
63+
with:
64+
name: test-report-verify-dns-settings
65+
path: ios/test-report/junit.xml
66+
67+
- name: Checkout old repository version
68+
uses: actions/checkout@v4
69+
with:
70+
ref: ${{ env.OLD_APP_COMMIT_HASH }}
71+
72+
- name: Change all other settings on old app version
73+
uses: ./.github/actions/ios-end-to-end-tests
74+
if: always()
75+
with:
76+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
77+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
78+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
79+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
80+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
81+
xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
82+
83+
- name: Store test report for changing all settings
84+
uses: actions/upload-artifact@v4
85+
if: always()
86+
with:
87+
name: test-report-change-all-other-settings
88+
path: ios/test-report/junit.xml
89+
90+
- name: Checkout repository to get the current app version
91+
uses: actions/checkout@v4
92+
93+
- name: Verify all other settings still changed on current app version
94+
uses: ./.github/actions/ios-end-to-end-tests
95+
if: always()
96+
with:
97+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
98+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
99+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
100+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
101+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
102+
xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'
103+
104+
- name: Store test report for verifying all other settings
105+
uses: actions/upload-artifact@v4
106+
if: always()
107+
with:
108+
name: test-report-verify-all-other-settings
109+
path: ios/test-report/junit.xml

‎.github/workflows/ios-end-to-end-tests.yml

+25-34
Original file line numberDiff line numberDiff line change
@@ -14,53 +14,44 @@ on:
1414
- .github/workflows/ios-end-to-end-tests.yml
1515
- ios/**
1616
workflow_dispatch:
17+
schedule:
18+
# At midnight every day.
19+
# Notifications for scheduled workflows are sent to the user who last modified the cron
20+
# syntax in the workflow file. If you update this you must have notifications for
21+
# Github Actions enabled, so these don't go unnoticed.
22+
# https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
23+
- cron: '0 0 * * *'
1724
jobs:
1825
test:
1926
if: github.event.pull_request.merged || github.event_name == 'workflow_dispatch'
2027
name: End to end tests
2128
runs-on: [self-hosted, macOS, ios-test]
22-
env:
23-
IOS_DEVICE_PIN_CODE: ${{ secrets.IOS_DEVICE_PIN_CODE }}
24-
TEST_DEVICE_IDENTIFIER_UUID: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
25-
TEST_DEVICE_UDID: ${{ secrets.IOS_TEST_DEVICE_UDID }}
26-
HAS_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
27-
NO_TIME_ACCOUNT_NUMBER: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
2829
steps:
2930
- name: Checkout repository
3031
uses: actions/checkout@v4
3132

32-
- name: Configure Rust
33-
run: rustup target add aarch64-apple-ios aarch64-apple-ios-sim
34-
35-
- name: Configure Xcode project
33+
- name: Select test plan to execute
3634
run: |
37-
for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
38-
sed -i "" "/^HAS_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
39-
sed -i "" "/^NO_TIME_ACCOUNT_NUMBER/d" UITests.xcconfig
40-
sed -i "" \
41-
"/IOS_DEVICE_PIN_CODE =/ s/= .*/= $IOS_DEVICE_PIN_CODE/" \
42-
UITests.xcconfig
43-
sed -i "" \
44-
"/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
45-
UITests.xcconfig
46-
echo -e "\nHAS_TIME_ACCOUNT_NUMBER = $HAS_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
47-
echo "NO_TIME_ACCOUNT_NUMBER = $NO_TIME_ACCOUNT_NUMBER" >> UITests.xcconfig
48-
working-directory: ios/Configurations
35+
if [[ "${{ github.event_name }}" == "pull_request" ]]; then
36+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsSmoke" >> $GITHUB_ENV
37+
elif [[ "${{ github.event_name }}" == "schedule" ]]; then
38+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
39+
elif [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
40+
echo "XCODE_TEST_PLAN=MullvadVPNUITestsAll" >> $GITHUB_ENV
41+
fi
4942
50-
- name: Run end-to-end-tests
51-
run: |
52-
set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
53-
-project MullvadVPN.xcodeproj \
54-
-scheme MullvadVPNUITests \
55-
-configuration Debug \
56-
-testPlan MullvadVPNUITestsSmoke \
57-
-destination "platform=iOS,id=$TEST_DEVICE_UDID" \
58-
-disableAutomaticPackageResolution \
59-
test 2>&1 | xcbeautify --report junit --report-path test-report
60-
working-directory: ios/
43+
- name: iOS end to end tests action
44+
uses: ./.github/actions/ios-end-to-end-tests
45+
with:
46+
xcode_test_plan: ${{ env.XCODE_TEST_PLAN }}
47+
ios_device_pin_code: ${{ secrets.IOS_DEVICE_PIN_CODE }}
48+
test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
49+
has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
50+
no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
51+
test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
6152

6253
- name: Comment PR on test failure
63-
if: failure() && github.event_name != 'workflow_dispatch'
54+
if: failure() && github.event_name == 'pull_request'
6455
uses: actions/github-script@v7
6556
with:
6657
github-token: ${{secrets.GITHUB_TOKEN}}

0 commit comments

Comments
 (0)