Use stagemole with e2e tests

Author: niklasberglund

.github/workflows/android-app.yml

@@ -37,21 +37,31 @@ on:
         type: boolean
         required: false
       mockapi_test_repeat:
-        description: Mockapi test repeat(self hosted)
+        description: Mockapi test repeat (self hosted)
         default: '1'
         required: true
         type: string
       e2e_test_repeat:
-        description: e2e test repeat(self hosted)
+        description: e2e test repeat (self hosted)
         default: '0'
         required: true
         type: string
+      e2e_tests_infra_flavor:
+        description: >
+          Infra environment to run e2e tests on (prod/stagemole).
+          If set to 'stagemole' test-related artefacts will be uploaded.
+        default: 'stagemole'
+        required: true
+        type: string
   # Build if main is updated to ensure up-to-date caches are available
   push:
     branches: [main]
 
 permissions: {}
 
+env:
+  TEST_OUTPUTS_PATH: '/sdcard/Download/test-outputs'
+
 jobs:
   prepare:
     name: Prepare
@@ -338,7 +348,9 @@ jobs:
 
       - name: Build stagemole app
         uses: burrunan/gradle-cache-action@v1
-        if: github.event.inputs.run_firebase_tests == 'true'
+        if: >
+          (github.event.inputs.e2e_test_repeat != '0' && github.event.inputs.e2e_tests_infra_flavor == 'stagemole') ||
+          github.event.inputs.run_firebase_tests == 'true'
         with:
           job-id: jdk17
           arguments: |
@@ -474,9 +486,18 @@ jobs:
           TEST_TYPE: ${{ matrix.test-type }}
           BILLING_FLAVOR: oss
           INFRA_FLAVOR: prod
+          TEST_DEVICE_OUTPUTS_DIR: ${{ env.TEST_OUTPUTS_PATH }}
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
 
+      - name: Pull test report
+        if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        shell: bash -ieo pipefail {0}
+        env:
+          TEST_DEVICE_OUTPUTS_DIR: ${{ env.TEST_OUTPUTS_PATH }}
+          REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
+        run: ./android/scripts/pull-test-output.sh --test-type ${{ matrix.test-type }}
+
       - name: Upload instrumentation report (${{ matrix.test-type }})
         uses: actions/upload-artifact@v4
         if: always() && matrix.test-repeat != 0
@@ -528,7 +549,7 @@ jobs:
 
       - name: Calculate timeout
         id: calculate-timeout
-        run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT
+        run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT
         shell: bash
 
       - name: Run instrumented test script
@@ -538,15 +559,38 @@ jobs:
         env:
           AUTO_FETCH_TEST_HELPER_APKS: true
           TEST_TYPE: e2e
-          BILLING_FLAVOR: oss
-          INFRA_FLAVOR: prod
-          VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }}
+          BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }}
+          INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }}
+          PARTNER_AUTH: |
+            ${{ github.event.inputs.e2e_tests_infra_flavor == 'stagemole' && secrets.STAGEMOLE_PARTNER_AUTH || '' }}
+          VALID_TEST_ACCOUNT_NUMBER: |
+            ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && secrets.ANDROID_PROD_TEST_ACCOUNT || '' }}
           INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000'
           ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }}
           ENABLE_ACCESS_TO_LOCAL_API_TESTS: true
+          TEST_DEVICE_OUTPUTS_DIR: ${{ env.TEST_OUTPUTS_PATH }}
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
 
+      - name: Pull test report
+        if: >
+          always() && matrix.test-repeat != 0 &&
+          github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        shell: bash -ieo pipefail {0}
+        env:
+          TEST_DEVICE_OUTPUTS_DIR: ${{ env.TEST_OUTPUTS_PATH }}
+          REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
+        run: ./android/scripts/pull-test-output.sh --test-type e2e
+
+      - name: Upload e2e instrumentation report
+        uses: actions/upload-artifact@v4
+        if: >
+          always() && matrix.test-repeat != 0 &&
+          github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
+        with:
+          name: e2e-instrumentation-report
+          path: ${{ steps.prepare-report-dir.outputs.report_dir }}
+
   firebase-tests:
     name: Run firebase tests
     if: github.event.inputs.run_firebase_tests == 'true'

android/scripts/pull-test-output.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+set -eu
+
+# Must match the path where e2e tests output their attachments
+TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs}"
+REPORT_DIR="${REPORT_DIR:-}"
+
+if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then
+    echo ""
+    echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set."
+    exit 1
+fi
+
+if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
+    echo ""
+    echo "Error: The variable REPORT_DIR must be set and the directory must exist."
+    exit 1
+fi
+
+echo "Collecting produced test attachments and logs..."
+adb pull "$TEST_DEVICE_OUTPUTS_DIR" "$REPORT_DIR"

android/scripts/run-instrumented-tests.sh

@@ -2,6 +2,20 @@
 
 set -eu
 
+
+cleanup() {
+    echo "### Trapped termination signal, clean up ###"
+    echo "### Store logcat and instrumentation log ###"
+    adb shell "mkdir -p $TEST_DEVICE_OUTPUTS_DIR"
+    adb shell "logcat -d > $TEST_DEVICE_OUTPUTS_DIR/logcat.txt"
+
+    if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then
+    rm -rf "$TEMP_DOWNLOAD_DIR"
+fi
+}
+
+trap cleanup SIGHUP EXIT
+
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 cd "$SCRIPT_DIR"
 
@@ -18,6 +32,7 @@ VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}"
 INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}"
 ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}"
 ENABLE_ACCESS_TO_LOCAL_API_TESTS="${ENABLE_ACCESS_TO_LOCAL_API_TESTS:-false}"
+TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs}" # Must match the path where e2e tests output their attachments
 REPORT_DIR="${REPORT_DIR:-}"
 
 while [[ "$#" -gt 0 ]]; do
@@ -72,6 +87,7 @@ if [[ -z ${BILLING_FLAVOR-} ]]; then
 fi
 
 echo "### Configuration ###"
+echo "Test device outputs dir: $TEST_DEVICE_OUTPUTS_DIR"
 echo "Report dir: $REPORT_DIR"
 echo "Test type: $TEST_TYPE"
 echo "Infra flavor: $INFRA_FLAVOR"
@@ -144,6 +160,12 @@ case "$TEST_TYPE" in
     ;;
 esac
 
+if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then
+    echo ""
+    echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set."
+    exit 1
+fi
+
 if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
     echo ""
     echo "Error: The variable REPORT_DIR must be set and the directory must exist."
@@ -153,17 +175,14 @@ fi
 GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS"
 
 INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt"
-LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt"
-LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots"
 DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
-DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments"
 
 echo ""
 echo "### Ensure clean report structure ###"
 rm -rf "${REPORT_DIR:?}/*"
 adb logcat --clear
 adb shell rm -rf "$DEVICE_SCREENSHOT_PATH"
-adb shell rm -rf "$DEVICE_TEST_ATTACHMENTS_PATH"
+adb shell rm -rf "$TEST_DEVICE_OUTPUTS_DIR"
 echo ""
 
 if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then
@@ -238,13 +257,5 @@ echo "### Checking logs for success message ###"
 if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then
     echo "Success, no failures!"
 else
-    echo "One or more tests failed, see logs for more details."
-    echo "Collecting report..."
-    adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots"
-    adb logcat -d > "$LOGCAT_FILE_PATH"
     exit 1
 fi
-
-if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then
-    rm -rf "$TEMP_DOWNLOAD_DIR"
-fi

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt

@@ -7,11 +7,13 @@ import java.io.IOException
 import org.junit.jupiter.api.fail
 
 object Attachment {
-    private const val DIRECTORY_NAME = "test-attachments"
+    const val DIRECTORY_NAME = "test-outputs"
+    val DIRECTORY_PATH = "${Environment.DIRECTORY_DOWNLOADS}/${DIRECTORY_NAME}"
+
     private val testAttachmentsDirectory =
         File(
             Environment.getExternalStorageDirectory(),
-            "${Environment.DIRECTORY_DOWNLOADS}/$DIRECTORY_NAME",
+            DIRECTORY_PATH,
         )
 
     fun saveAttachment(fileName: String, data: ByteArray) {

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt

@@ -55,6 +55,6 @@ class CaptureScreenRecordingsExtension : BeforeEachCallback, AfterEachCallback {
 
     companion object {
         val OUTPUT_DIRECTORY =
-            "${Environment.getExternalStorageDirectory().path}/Download/test-attachments/video"
+            "${Environment.getExternalStorageDirectory().path}/Download/${Attachment.DIRECTORY_NAME}/video"
     }
 }

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt

@@ -5,6 +5,7 @@ import android.content.ContentValues
 import android.graphics.Bitmap
 import android.os.Build
 import android.os.Environment
+import android.os.Environment.DIRECTORY_DOWNLOADS
 import android.os.Environment.DIRECTORY_PICTURES
 import android.provider.MediaStore
 import androidx.annotation.RequiresApi
@@ -16,6 +17,7 @@ import java.io.IOException
 import java.nio.file.Paths
 import java.time.OffsetDateTime
 import java.time.temporal.ChronoUnit
+import net.mullvad.mullvadvpn.test.common.misc.Attachment
 import org.junit.jupiter.api.extension.ExtensionContext
 import org.junit.jupiter.api.extension.TestWatcher
 
@@ -55,7 +57,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch
     ) {
         contentValues.apply {
             put(MediaStore.MediaColumns.DISPLAY_NAME, filename)
-            put(MediaStore.Images.Media.RELATIVE_PATH, "$DIRECTORY_PICTURES/$baseDir")
+            put(MediaStore.Images.Media.RELATIVE_PATH, "${Attachment.DIRECTORY_PATH}/$baseDir")
         }
 
         val uri =
@@ -78,7 +80,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch
     private fun Bitmap.writeToExternalStorage(baseDir: String, filename: String) {
         val screenshotBaseDirectory =
             Paths.get(
-                    Environment.getExternalStoragePublicDirectory(DIRECTORY_PICTURES).path,
+                    Environment.getExternalStoragePublicDirectory(Attachment.DIRECTORY_PATH).path,
                     baseDir,
                 )
                 .toFile()

android/test/e2e/README.md

@@ -58,4 +58,4 @@ docker run --rm --volumes-from gcloud-config -v ${PWD}:/android gcr.io/google.co
 ```
 
 ## Test artefacts
-Test artefacts are stored on the test device in `/sdcard/Download/test-attachments`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten.
+Test artefacts are stored on the test device in `/sdcard/Download/test-outputs`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten.

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt

@@ -17,6 +17,7 @@ import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL
 import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL
 import org.json.JSONArray
 import org.json.JSONObject
+import org.junit.jupiter.api.fail
 
 class SimpleMullvadHttpClient(context: Context) {
 
@@ -201,6 +202,10 @@ class SimpleMullvadHttpClient(context: Context) {
 
         private val onErrorResponse = { error: VolleyError ->
             if (error.networkResponse != null) {
+                if (error.networkResponse.statusCode == 429) {
+                    fail("Request failed with response status code 429: Too many requests")
+                }
+
                 Logger.e(
                     "Response returned error message: ${error.message} " +
                         "status code: ${error.networkResponse.statusCode}"