Change the way iOS end to end tests workflows are set up

Author: niklasberglund

.github/actions/ios-end-to-end-tests/action.yml .github/actions/build-ios-e2e-tests/action.yml

@@ -1,5 +1,5 @@
-name: 'iOS end to end tests action'
-description: 'Prepares and runs end to end tests on iOS device'
+name: 'Build iOS end to end tests action'
+description: 'Prepares and builds end to end tests on iOS device'
 inputs:
   ios_device_pin_code:
     description: 'iOS Device Pin Code'
@@ -16,25 +16,19 @@ inputs:
   test_device_udid:
     description: 'Test Device UDID'
     required: true
-  xcode_test_plan:
-    description: 'Xcode Test Plan to run'
-    required: true
   partner_api_token:
     description: 'Partner API Token'
     required: true
   test_name:
     description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
     required: false
+  outputs_path:
+    description: 'Path to store outputs. This should be unique for each job run in order to avoid concurrency issues.'
+    required: true
 
 runs:
   using: 'composite'
   steps:
-    - name: Make sure app is not installed
-      run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
-      shell: bash
-      env:
-        IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
-
     - name: Configure Xcode project
       run: |
         for file in *.xcconfig.template ; do cp $file ${file//.template/} ; done
@@ -46,12 +40,21 @@ runs:
           sed -i "" \
             "/TEST_DEVICE_IDENTIFIER_UUID =/ s/= .*/= $TEST_DEVICE_IDENTIFIER_UUID/" \
             UITests.xcconfig
+          sed -i "" \
+            "s#^// PARTNER_API_TOKEN =#PARTNER_API_TOKEN =#" \
+            UITests.xcconfig
           sed -i "" \
             "/PARTNER_API_TOKEN =/ s#= .*#= $PARTNER_API_TOKEN#" \
             UITests.xcconfig
           sed -i "" \
             "/ATTACH_APP_LOGS_ON_FAILURE =/ s#= .*#= 1#" \
             UITests.xcconfig
+          sed -i "" \
+            "/TEST_DEVICE_IS_IPAD =/ s#= .*#= 0#" \
+            UITests.xcconfig
+          sed -i "" \
+            "/UNINSTALL_APP_IN_TEST_SUITE_TEAR_DOWN =/ s#= .*#= 0#" \
+            UITests.xcconfig
       shell: bash
       working-directory: ios/Configurations
       env:
@@ -61,7 +64,7 @@ runs:
         NO_TIME_ACCOUNT_NUMBER: ${{ inputs.no_time_account_number }}
         PARTNER_API_TOKEN: ${{ inputs.partner_api_token }}
 
-    - name: Run end-to-end-tests
+    - name: Build app and tests for testing
       run: |
         if [ -n "$TEST_NAME" ]; then
           TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"
@@ -71,19 +74,12 @@ runs:
         set -o pipefail && env NSUnbufferedIO=YES xcodebuild \
           -project MullvadVPN.xcodeproj \
           -scheme MullvadVPNUITests \
-          -testPlan $XCODE_TEST_PLAN $TEST_NAME_ARGUMENT \
-          -resultBundlePath xcode-test-report \
+          -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT \
           -destination "platform=iOS,id=$TEST_DEVICE_UDID" \
-          clean test 2>&1 | xcbeautify --report junit --report-path junit-test-report
+          -derivedDataPath derived-data \
+          clean build-for-testing 2>&1
       shell: bash
       working-directory: ios/
       env:
-        XCODE_TEST_PLAN: ${{ inputs.xcode_test_plan }}
         TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
         TEST_NAME: ${{ inputs.test_name }}
-
-    - name: Uninstall app if still installed
-      run: ios-deploy --id $IOS_TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
-      shell: bash
-      env:
-        IOS_TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}

.github/actions/run-ios-e2e-tests/action.yml

@@ -0,0 +1,74 @@
+name: 'Run iOS end to end tests action'
+description: 'Runs end to end tests on iOS device'
+inputs:
+  test_name:
+    description: 'Test case/suite name. Will run all tests in the test plan if not provided.'
+    required: false
+  test_device_udid:
+    description: 'Test Device UDID'
+    required: true
+  outputs_path:
+    description: >
+      Path to where outputs are stored - both build outputs and outputs from running tests.
+      This should be unique for each job run in order to avoid concurrency issues.
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+    # Set up a unique output directory
+    - name: Set up outputs directory
+      run: |
+        if [ -n "$TEST_NAME" ]; then
+          # Strip slashes to avoid creating subdirectories
+          test_name_sanitized=$(printf "$TEST_NAME" | sed 's/\//_/g')
+          echo "Setting output directory tests-output-test-name-sanitized"
+          echo "$test_name_sanitized"
+          test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output-$test_name_sanitized"
+        else
+          echo "Setting output directory output"
+          test_output_directory="${{ env.OUTPUTS_PATH }}/tests-output"
+        fi
+
+        echo "TEST_OUTPUT_DIRECTORY=$test_output_directory" >> $GITHUB_ENV
+        echo "TEST_NAME_SANITIZED=$test_name_sanitized" >> $GITHUB_ENV
+      shell: bash
+      env:
+        TEST_NAME: ${{ inputs.test_name }}
+        OUTPUTS_PATH: ${{ inputs.outputs_path }}
+
+    - name: Uninstall app
+      run: ios-deploy --id $TEST_DEVICE_UDID --uninstall_only --bundle_id net.mullvad.MullvadVPN
+      shell: bash
+      env:
+        TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
+
+    - name: Run end-to-end-tests
+      run: >
+        if [ -n "$TEST_NAME" ]; then TEST_NAME_ARGUMENT=" -only-testing $TEST_NAME"; else TEST_NAME_ARGUMENT=""; fi
+
+        set -o pipefail && env NSUnbufferedIO=YES xcodebuild
+          -project MullvadVPN.xcodeproj
+          -scheme MullvadVPNUITests
+          -testPlan MullvadVPNUITestsAll $TEST_NAME_ARGUMENT
+          -resultBundlePath ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report
+          -derivedDataPath derived-data
+          -destination "platform=iOS,id=$TEST_DEVICE_UDID"
+          test-without-building 2>&1 | xcbeautify --report junit
+          --report-path ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report
+      shell: bash
+      working-directory: ${{ inputs.outputs_path }}/mullvadvpn-app/ios
+      env:
+        TEST_NAME: ${{ inputs.test_name }}
+        TEST_DEVICE_UDID: ${{ inputs.test_device_udid }}
+
+    - name: Store test report artifact
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: ${{ env.TEST_NAME_SANITIZED }}-test-results
+        path: |
+          ${{ env.TEST_OUTPUT_DIRECTORY }}/junit-test-report/junit.xml
+          ${{ env.TEST_OUTPUT_DIRECTORY }}/xcode-test-report.xcresult
+      env:
+        TEST_NAME: ${{ inputs.test_name }}

.github/workflows/ios-end-to-end-tests-api.yml

@@ -0,0 +1,9 @@
+---
+name: iOS end-to-end API tests
+on:
+  workflow_dispatch:
+jobs:
+  reuse-e2e-workflow:
+    uses: ./.github/workflows/ios-end-to-end-tests.yml
+    with:
+      arg_tests_json_key: "api-tests"

.github/workflows/ios-end-to-end-tests-merge-to-main.yml

@@ -0,0 +1,17 @@
+---
+name: iOS end-to-end merge to main tests
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+    paths:
+      - .github/workflows/ios-end-to-end-tests*.yml
+      - ios/**
+jobs:
+  reuse-e2e-workflow:
+    uses: ./.github/workflows/ios-end-to-end-tests.yml
+    with:
+      arg_tests_json_key: "pr-merge-to-main"

.github/workflows/ios-end-to-end-tests-nightly.yml

@@ -0,0 +1,16 @@
+---
+name: iOS end-to-end nightly tests
+on:
+  workflow_dispatch:
+  schedule:
+    # At midnight every day.
+    # Notifications for scheduled workflows are sent to the user who last modified the cron
+    # syntax in the workflow file. If you update this you must have notifications for
+    # Github Actions enabled, so these don't go unnoticed.
+    # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
+    - cron: '0 0 * * *'
+jobs:
+  reuse-e2e-workflow:
+    uses: ./.github/workflows/ios-end-to-end-tests.yml
+    with:
+      arg_tests_json_key: "nightly"

.github/workflows/ios-end-to-end-tests-settings-migration.yml

@@ -1,5 +1,8 @@
 ---
 name: iOS settings migration tests
+concurrency:
+  group: ios-end-to-end-tests
+  cancel-in-progress: false
 permissions:
   contents: read
 on:
@@ -11,6 +14,8 @@ on:
     # Github Actions enabled, so these don't go unnoticed.
     # https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/notifications-for-workflow-runs
     - cron: '0 0 * * *'
+env:
+  TEST_DEVICE_UDID: 00008130-0019181022F3803A
 jobs:
   test:
     name: Settings migration end to end tests
@@ -19,14 +24,15 @@ jobs:
       OLD_APP_COMMIT_HASH: 895b7d98825e678f5d7023d5ea3c9b7beee89280
     steps:
       - name: Configure Rust
-        uses: actions-rs/toolchain@v1
+        uses: actions-rs/toolchain@v1.0.6
         with:
           toolchain: stable
           override: true
           target: aarch64-apple-ios
 
       - name: Uninstall app
-        run: ios-deploy --id ${{ secrets.IOS_TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
+        timeout-minutes: 5
+        run: ios-deploy --id $${{ env.TEST_DEVICE_UDID }} --uninstall_only --bundle_id net.mullvad.MullvadVPN
 
       - name: Checkout old repository version
         uses: actions/checkout@v4
@@ -40,7 +46,7 @@ jobs:
           test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
           xcode_test_plan: 'MullvadVPNUITestsChangeDNSSettings'
           partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
 
@@ -62,7 +68,7 @@ jobs:
           test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
           partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsVerifyDNSSettingsChanged'
 
@@ -86,7 +92,7 @@ jobs:
           test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
           partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsChangeSettings'
 
@@ -108,7 +114,7 @@ jobs:
           test_device_identifier_uuid: ${{ secrets.IOS_TEST_DEVICE_IDENTIFIER_UUID }}
           has_time_account_number: ${{ secrets.IOS_HAS_TIME_ACCOUNT_NUMBER_PRODUCTION }}
           no_time_account_number: ${{ secrets.IOS_NO_TIME_ACCOUNT_NUMBER_PRODUCTION }}
-          test_device_udid: ${{ secrets.IOS_TEST_DEVICE_UDID }}
+          test_device_udid: ${{ env.TEST_DEVICE_UDID }}
           partner_api_token: ${{ secrets.STAGEMOLE_PARTNER_AUTH }}
           xcode_test_plan: 'MullvadVPNUITestsVerifySettingsChanged'