Fix for failing mockapi and e2e tests

Author: niklasberglund

.github/workflows/android-app.yml

@@ -416,10 +416,9 @@ jobs:
           - test-type: app
             path: android/app/build/outputs/apk
             test-repeat: 1
-          # Disabled (test-repeat='0') due to flakiness unless overridden by input.
           - test-type: mockapi
             path: android/test/mockapi/build/outputs/apk
-            test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 0 }}
+            test-repeat: ${{ github.event.inputs.mockapi_test_repeat || 1 }}
     steps:
       - name: Prepare report dir
         if: ${{ matrix.test-repeat != 0 }}
@@ -470,7 +469,8 @@ jobs:
 
   instrumented-e2e-tests:
     name: Run instrumented e2e tests
-    runs-on: [self-hosted, android-device]
+    # Temporary workaround for targeting the runner android-runner-v1
+    runs-on: [self-hosted, android-device, android-emulator]
     if: github.event_name == 'schedule' || github.event.inputs.run_e2e_tests == 'true'
     timeout-minutes: 30
     needs: [build-app, build-instrumented-tests]
@@ -507,6 +507,7 @@ jobs:
           INFRA_FLAVOR: prod
           VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }}
           INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000'
+          ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }}
           REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
         run: ./android/scripts/run-instrumented-tests.sh
 
@@ -521,6 +522,7 @@ jobs:
         clearPackageData=true,\
         runnerBuilder=de.mannodermaus.junit5.AndroidJUnit5Builder,\
         invalid_test_account_number=0000000000000000,\
+        enable_highly_rate_limited_tests=${{ github.event_name == 'schedule' && 'true' || 'false' }},\
         partner_auth=${{ secrets.STAGEMOLE_PARTNER_AUTH }}"
     strategy:
       fail-fast: false

android/gradle/libs.versions.toml

@@ -71,7 +71,7 @@ android-gradle-aapt = { module = "com.android.tools.build:aapt2" }
 android-volley = { module = "com.android.volley:volley", version.ref = "android-volley" }
 
 # AndroidX
-androidx-activity-Compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" }
+androidx-activity-compose = { module = "androidx.activity:activity-compose", version.ref = "androidx-activitycompose" }
 androidx-appcompat = { module = "androidx.appcompat:appcompat", version.ref = "androidx-appcompat" }
 androidx-coresplashscreen = { module = "androidx.core:core-splashscreen", version.ref = "androidx-coresplashscreen" }
 androidx-espresso = { module = "androidx.test.espresso:espresso-core", version.ref = "androidx-espresso" }

android/scripts/run-instrumented-tests.sh

@@ -16,6 +16,7 @@ TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/te
 PARTNER_AUTH="${PARTNER_AUTH:-}"
 VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}"
 INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}"
+ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}"
 REPORT_DIR="${REPORT_DIR:-}"
 
 while [[ "$#" -gt 0 ]]; do
@@ -130,6 +131,9 @@ case "$TEST_TYPE" in
         echo "Error: The variable PARTNER_AUTH or VALID_TEST_ACCOUNT_NUMBER must be set."
         exit 1
     fi
+    if [[ "${ENABLE_HIGHLY_RATE_LIMITED_TESTS}" == "false" ]]; then
+        OPTIONAL_TEST_ARGUMENTS+=" -e enable_highly_rate_limited_tests false"
+    fi
     USE_ORCHESTRATOR="true"
     PACKAGE_NAME="net.mullvad.mullvadvpn"
     if [[ "$INFRA_FLAVOR" =~ ^(devmole|stagemole)$ ]]; then

android/service/src/main/kotlin/net/mullvad/mullvadvpn/service/MullvadVpnService.kt

@@ -16,7 +16,6 @@ import kotlinx.coroutines.flow.filter
 import kotlinx.coroutines.flow.first
 import kotlinx.coroutines.launch
 import kotlinx.coroutines.runBlocking
-import net.mullvad.mullvadvpn.lib.common.constant.BuildTypes
 import net.mullvad.mullvadvpn.lib.common.constant.GRPC_SOCKET_FILE_NAMED_ARGUMENT
 import net.mullvad.mullvadvpn.lib.common.constant.KEY_CONNECT_ACTION
 import net.mullvad.mullvadvpn.lib.common.constant.KEY_DISCONNECT_ACTION
@@ -150,7 +149,7 @@ class MullvadVpnService : TalpidVpnService() {
 
     private fun startDaemon() {
         val apiEndpointConfiguration =
-            if (Build.TYPE == BuildTypes.DEBUG) {
+            if (BuildConfig.DEBUG) {
                 intentProvider.getLatestIntent()?.getApiEndpointConfigurationExtras()
                     ?: apiEndpointConfiguration
             } else {

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/extension/UiAutomatorExtensions.kt

@@ -24,13 +24,11 @@ fun UiDevice.findObjectWithTimeout(
 
     wait(Until.hasObject(selector), timeout)
 
-    return try {
-        findObject(selector)
-    } catch (e: NullPointerException) {
-        throw IllegalArgumentException(
-            "No matches for selector within timeout ($timeout): $selector"
-        )
-    }
+    val foundObject = findObject(selector)
+
+    require(foundObject != null) { "No matches for selector within timeout ($timeout): $selector" }
+
+    return foundObject
 }
 
 fun UiDevice.clickAgreeOnPrivacyDisclaimer() {

android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/interactor/AppInteractor.kt

@@ -52,15 +52,6 @@ class AppInteractor(
         ensureLoggedIn()
     }
 
-    fun launchAndCreateAccount() {
-        launch()
-        device.clickAgreeOnPrivacyDisclaimer()
-        device.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove()
-        waitForLoginPrompt()
-        attemptCreateAccount()
-        ensureAccountCreated()
-    }
-
     fun attemptLogin(accountNumber: String) {
         val loginObject =
             device.findObjectWithTimeout(By.clazz("android.widget.EditText")).apply {

android/test/e2e/build.gradle.kts

@@ -46,6 +46,7 @@ android {
                 put("clearPackageData", "true")
                 addOptionalPropertyAsArgument("valid_test_account_number")
                 addOptionalPropertyAsArgument("invalid_test_account_number")
+                addOptionalPropertyAsArgument("enable_highly_rate_limited_tests")
             }
     }
 
@@ -142,4 +143,12 @@ dependencies {
     implementation(libs.kotlin.stdlib)
 
     androidTestUtil(libs.androidx.test.orchestrator)
+
+    // Needed or else the app crashes when launched
+    implementation(Dependencies.junit5AndroidTestCompose)
+    implementation(libs.compose.material3)
+
+    // Need these for forcing later versions of dependencies
+    implementation(libs.compose.ui)
+    implementation(libs.androidx.activity.compose)
 }

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/ConnectionTest.kt

@@ -9,7 +9,6 @@ import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule
 import net.mullvad.mullvadvpn.test.e2e.misc.ConnCheckState
 import net.mullvad.mullvadvpn.test.e2e.misc.SimpleMullvadHttpClient
 import org.junit.jupiter.api.Assertions.assertEquals
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
 
@@ -35,7 +34,6 @@ class ConnectionTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
     }
 
     @Test
-    @Disabled("Disabled since the connection check isn't reliable in the stagemole infrastructure.")
     fun testConnectAndVerifyWithConnectionCheck() {
         // Given
         app.launchAndEnsureLoggedIn(accountTestRule.validAccountNumber)

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LoginTest.kt

@@ -5,8 +5,8 @@ import net.mullvad.mullvadvpn.test.common.constant.LOGIN_FAILURE_TIMEOUT
 import net.mullvad.mullvadvpn.test.common.extension.clickAgreeOnPrivacyDisclaimer
 import net.mullvad.mullvadvpn.test.common.extension.clickAllowOnNotificationPermissionPromptIfApiLevel33AndAbove
 import net.mullvad.mullvadvpn.test.common.extension.findObjectWithTimeout
+import net.mullvad.mullvadvpn.test.e2e.annotations.HighlyRateLimited
 import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule
-import org.junit.jupiter.api.Disabled
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.api.extension.RegisterExtension
 
@@ -27,7 +27,7 @@ class LoginTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
     }
 
     @Test
-    @Disabled("Disabled to avoid getting rate-limited.")
+    @HighlyRateLimited
     fun testLoginWithInvalidCredentials() {
         // Given
         val invalidDummyAccountNumber = accountTestRule.invalidAccountNumber

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LogoutTest.kt

@@ -23,17 +23,4 @@ class LogoutTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) {
         // Then
         assertNotNull(device.findObjectWithTimeout(By.text("Login")))
     }
-
-    @Test
-    fun testCreateAccountAndLogout() {
-        // Given
-        app.launchAndCreateAccount()
-
-        // When
-        app.clickAccountCog()
-        app.clickActionButtonByText("Log out")
-
-        // Then
-        assertNotNull(device.findObjectWithTimeout(By.text("Login")))
-    }
 }

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/annotations/HighlyRateLimited.kt

@@ -0,0 +1,38 @@
+package net.mullvad.mullvadvpn.test.e2e.annotations
+
+import androidx.test.platform.app.InstrumentationRegistry
+import net.mullvad.mullvadvpn.test.e2e.constant.ENABLE_HIGHLY_RATE_LIMITED
+import net.mullvad.mullvadvpn.test.e2e.extension.getRequiredArgument
+import org.junit.jupiter.api.extension.ConditionEvaluationResult
+import org.junit.jupiter.api.extension.ExecutionCondition
+import org.junit.jupiter.api.extension.ExtendWith
+import org.junit.jupiter.api.extension.ExtensionContext
+
+/**
+ * Annotation for tests making use of API endpoints/requests that are highly rate limited such as
+ * failed login requests.
+ */
+@Retention(AnnotationRetention.RUNTIME)
+@ExtendWith(HighlyRateLimited.ShouldRunWhenSeverelyAffectedByRateLimiting::class)
+annotation class HighlyRateLimited {
+    class ShouldRunWhenSeverelyAffectedByRateLimiting : ExecutionCondition {
+        override fun evaluateExecutionCondition(
+            context: ExtensionContext?
+        ): ConditionEvaluationResult {
+            val enableHighlyRateLimited =
+                InstrumentationRegistry.getArguments()
+                    .getRequiredArgument(ENABLE_HIGHLY_RATE_LIMITED)
+                    .toBoolean()
+
+            if (enableHighlyRateLimited) {
+                return ConditionEvaluationResult.enabled(
+                    "Running test highly affected by rate limiting."
+                )
+            } else {
+                return ConditionEvaluationResult.disabled(
+                    "Skipping test highly affected by rate limiting."
+                )
+            }
+        }
+    }
+}

android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/constant/Constants.kt

@@ -4,3 +4,4 @@ const val LOG_TAG = "mullvad-e2e"
 const val PARTNER_AUTH = "partner_auth"
 const val VALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "valid_test_account_number"
 const val INVALID_TEST_ACCOUNT_NUMBER_ARGUMENT_KEY = "invalid_test_account_number"
+const val ENABLE_HIGHLY_RATE_LIMITED = "enable_highly_rate_limited_tests"

android/test/mockapi/build.gradle.kts

@@ -88,4 +88,12 @@ dependencies {
     implementation(libs.mockkWebserver)
 
     androidTestUtil(libs.androidx.test.orchestrator)
+
+    // Needed or else the app crashes when launched
+    implementation(Dependencies.junit5AndroidTestCompose)
+    implementation(libs.compose.material3)
+
+    // Need these for forcing later versions of dependencies
+    implementation(libs.compose.ui)
+    implementation(libs.androidx.activity.compose)
 }