diff --git a/.github/workflows/android-app.yml b/.github/workflows/android-app.yml index 14c7bc0e5ac8..862e2e507d4c 100644 --- a/.github/workflows/android-app.yml +++ b/.github/workflows/android-app.yml @@ -37,15 +37,22 @@ on: type: boolean required: false mockapi_test_repeat: - description: Mockapi test repeat(self hosted) + description: Mockapi test repeat (self hosted) default: '1' required: true type: string e2e_test_repeat: - description: e2e test repeat(self hosted) + description: e2e test repeat (self hosted) default: '0' required: true type: string + e2e_tests_infra_flavor: + description: > + Infra environment to run e2e tests on (prod/stagemole). + If set to 'stagemole' test-related artefacts will be uploaded. + default: 'stagemole' + required: true + type: string # Build if main is updated to ensure up-to-date caches are available push: branches: [main] @@ -338,7 +345,9 @@ jobs: - name: Build stagemole app uses: burrunan/gradle-cache-action@v1 - if: github.event.inputs.run_firebase_tests == 'true' + if: > + (github.event.inputs.e2e_test_repeat != '0' && github.event.inputs.e2e_tests_infra_flavor == 'stagemole') || + github.event.inputs.run_firebase_tests == 'true' with: job-id: jdk17 arguments: | @@ -530,7 +539,7 @@ jobs: - name: Calculate timeout id: calculate-timeout - run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT + run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT shell: bash - name: Run instrumented test script @@ -540,15 +549,27 @@ jobs: env: AUTO_FETCH_TEST_HELPER_APKS: true TEST_TYPE: e2e - BILLING_FLAVOR: oss - INFRA_FLAVOR: prod - VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }} + BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }} + INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }} + PARTNER_AUTH: |- + ${{ github.event.inputs.e2e_tests_infra_flavor == 'stagemole' && secrets.STAGEMOLE_PARTNER_AUTH || '' }} + VALID_TEST_ACCOUNT_NUMBER: |- + ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && secrets.ANDROID_PROD_TEST_ACCOUNT || '' }} INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000' ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }} ENABLE_ACCESS_TO_LOCAL_API_TESTS: true REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }} run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }} + - name: Upload e2e instrumentation report + uses: actions/upload-artifact@v4 + if: > + always() && matrix.test-repeat != 0 && + github.event.inputs.e2e_tests_infra_flavor == 'stagemole' + with: + name: e2e-instrumentation-report + path: ${{ steps.prepare-report-dir.outputs.report_dir }} + firebase-tests: name: Run firebase tests if: github.event.inputs.run_firebase_tests == 'true' diff --git a/android/scripts/run-instrumented-tests.sh b/android/scripts/run-instrumented-tests.sh index 8d835ddb5ab7..88f1f2082b7a 100755 --- a/android/scripts/run-instrumented-tests.sh +++ b/android/scripts/run-instrumented-tests.sh @@ -156,6 +156,7 @@ INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt" LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt" LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots" DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE" +LOCAL_TEST_ATTACHMENTS_PATH="$REPORT_DIR/test-attachments" DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments" echo "" @@ -241,6 +242,7 @@ else echo "One or more tests failed, see logs for more details." echo "Collecting report..." adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots" + adb pull "$DEVICE_TEST_ATTACHMENTS_PATH" "$LOCAL_TEST_ATTACHMENTS_PATH" || echo "No test attachments" adb logcat -d > "$LOGCAT_FILE_PATH" exit 1 fi diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LeakTest.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LeakTest.kt index 46f1271257b8..63b3bd1ae1bc 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LeakTest.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/LeakTest.kt @@ -15,8 +15,10 @@ import net.mullvad.mullvadvpn.test.common.page.on import net.mullvad.mullvadvpn.test.common.rule.ForgetAllVpnAppsInSettingsTestRule import net.mullvad.mullvadvpn.test.e2e.annotations.HasDependencyOnLocalAPI import net.mullvad.mullvadvpn.test.e2e.misc.AccountTestRule -import net.mullvad.mullvadvpn.test.e2e.misc.LeakCheck +import net.mullvad.mullvadvpn.test.e2e.misc.NetworkTrafficChecker import net.mullvad.mullvadvpn.test.e2e.misc.NoTrafficToHostRule +import net.mullvad.mullvadvpn.test.e2e.misc.SomeTrafficToHostRule +import net.mullvad.mullvadvpn.test.e2e.misc.SomeTrafficToOtherHostsRule import net.mullvad.mullvadvpn.test.e2e.misc.TrafficGenerator import net.mullvad.mullvadvpn.test.e2e.router.packetCapture.PacketCapture import net.mullvad.mullvadvpn.test.e2e.router.packetCapture.PacketCaptureResult @@ -58,7 +60,7 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { @Test @HasDependencyOnLocalAPI - fun testNegativeLeak() = + fun testEnsureNoLeaksToSpecificHost() = runBlocking { app.launch() @@ -94,15 +96,20 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { val capturedStreams = captureResult.streams val capturedPcap = captureResult.pcap val timestamp = System.currentTimeMillis() - Attachment.saveAttachment("capture-testNegativeLeak-$timestamp.pcap", capturedPcap) + Attachment.saveAttachment( + "capture-${javaClass.enclosingMethod}-$timestamp.pcap", + capturedPcap, + ) - val leakRules = listOf(NoTrafficToHostRule(targetIpAddress)) - LeakCheck.assertNoLeaks(capturedStreams, leakRules) + NetworkTrafficChecker.checkTrafficStreamsAgainstRules( + capturedStreams, + NoTrafficToHostRule(targetIpAddress), + ) } @Test @HasDependencyOnLocalAPI - fun testShouldHaveNegativeLeak() = + fun testEnsureLeaksToSpecificHost() = runBlocking { app.launch() @@ -154,15 +161,21 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { val capturedStreams = captureResult.streams val capturedPcap = captureResult.pcap val timestamp = System.currentTimeMillis() - Attachment.saveAttachment("capture-testShouldHaveLeak-$timestamp.pcap", capturedPcap) + Attachment.saveAttachment( + "capture-${javaClass.enclosingMethod}-$timestamp.pcap", + capturedPcap, + ) - val leakRules = listOf(NoTrafficToHostRule(targetIpAddress)) - LeakCheck.assertLeaks(capturedStreams, leakRules) + NetworkTrafficChecker.checkTrafficStreamsAgainstRules( + capturedStreams, + SomeTrafficToHostRule(targetIpAddress), + SomeTrafficToOtherHostsRule(targetIpAddress), + ) } @Test @HasDependencyOnLocalAPI - fun testLeakWhenVpnSettingsChange() = + fun testEnsureNoLeaksToSpecificHostWhenSwitchingBetweenVariousVpnSettings() = runBlocking { app.launch() // Obfuscation and Post-Quantum are by default set to automatic. Explicitly set to off. @@ -208,12 +221,14 @@ class LeakTest : EndToEndTest(BuildConfig.FLAVOR_infrastructure) { val capturedPcap = captureResult.pcap val timestamp = System.currentTimeMillis() Attachment.saveAttachment( - "capture-testLeakWhenVpnSettingsChange-$timestamp.pcap", + "capture-${javaClass.enclosingMethod}-$timestamp.pcap", capturedPcap, ) - val leakRules = listOf(NoTrafficToHostRule(targetIpAddress)) - LeakCheck.assertLeaks(capturedStreams, leakRules) + NetworkTrafficChecker.checkTrafficStreamsAgainstRules( + capturedStreams, + NoTrafficToHostRule(targetIpAddress), + ) } private fun disableObfuscation() { diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/AccountTestRule.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/AccountTestRule.kt index ff6276a69f6d..1a414584e303 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/AccountTestRule.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/AccountTestRule.kt @@ -19,7 +19,7 @@ class AccountTestRule : BeforeEachCallback { override fun beforeEach(context: ExtensionContext) { InstrumentationRegistry.getArguments().also { bundle -> if (partnerAuth != null) { - validAccountNumber = client.createAccount() + validAccountNumber = client.createAccountUsingPartnerApi(partnerAuth) client.addTimeToAccountUsingPartnerAuth( accountNumber = validAccountNumber, daysToAdd = 1, diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/LeakCheck.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/LeakCheck.kt deleted file mode 100644 index 6770551f6542..000000000000 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/LeakCheck.kt +++ /dev/null @@ -1,33 +0,0 @@ -package net.mullvad.mullvadvpn.test.e2e.misc - -import net.mullvad.mullvadvpn.test.e2e.router.packetCapture.Stream -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assertions.assertTrue - -object LeakCheck { - fun assertNoLeaks(streams: List, rules: List) { - // Assert that there are streams to be analyzed. Stream objects are guaranteed to contain - // packets when initialized. - assertTrue(streams.isNotEmpty()) - - for (rule in rules) { - assertFalse(rule.isViolated(streams)) - } - } - - fun assertLeaks(streams: List, rules: List) { - for (rule in rules) { - assertTrue(rule.isViolated(streams)) - } - } -} - -interface LeakRule { - fun isViolated(streams: List): Boolean -} - -class NoTrafficToHostRule(private val host: String) : LeakRule { - override fun isViolated(streams: List): Boolean { - return streams.any { it.destinationHost.ipAddress == host } - } -} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/NetworkTrafficChecker.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/NetworkTrafficChecker.kt new file mode 100644 index 000000000000..373cd470261a --- /dev/null +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/NetworkTrafficChecker.kt @@ -0,0 +1,48 @@ +package net.mullvad.mullvadvpn.test.e2e.misc + +import net.mullvad.mullvadvpn.test.e2e.router.packetCapture.Stream +import org.junit.jupiter.api.Assertions.assertNotEquals +import org.junit.jupiter.api.Assertions.assertTrue + +object NetworkTrafficChecker { + fun checkTrafficStreamsAgainstRules(streams: List, vararg rules: TrafficRule) { + // Assert that there are streams to be analyzed. Stream objects are guaranteed to contain + // packets when initialized. + assertTrue(streams.isNotEmpty(), "List of streams is empty.") + + for (rule in rules) { + rule.assertTraffic(streams) + } + } +} + +interface TrafficRule { + fun assertTraffic(streams: List) +} + +class NoTrafficToHostRule(private val host: String) : TrafficRule { + override fun assertTraffic(streams: List) { + streams.forEach { assertNotEquals(host, it.destinationHost.ipAddress) } + } +} + +class SomeTrafficToHostRule(private val host: String) : TrafficRule { + override fun assertTraffic(streams: List) { + val hasAnyTrafficToSpecifiedHost = streams.any { it.destinationHost.ipAddress == host } + assertTrue( + hasAnyTrafficToSpecifiedHost, + "Expected some traffic to the specified host ($host)," + + "but all traffic had other destinations addresses.", + ) + } +} + +class SomeTrafficToOtherHostsRule(private val hostToExclude: String) : TrafficRule { + override fun assertTraffic(streams: List) { + val hasAnyTrafficToOtherHost = streams.any { it.destinationHost.ipAddress != hostToExclude } + assertTrue( + hasAnyTrafficToOtherHost, + "Expected some traffic to leak, but all traffic had destination address: $hostToExclude", + ) + } +} diff --git a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt index b5dcc1d64790..fa4dd8861379 100644 --- a/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt +++ b/android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt @@ -10,13 +10,13 @@ import com.android.volley.toolbox.JsonObjectRequest import com.android.volley.toolbox.RequestFuture import com.android.volley.toolbox.StringRequest import com.android.volley.toolbox.Volley -import net.mullvad.mullvadvpn.test.e2e.constant.ACCOUNT_URL import net.mullvad.mullvadvpn.test.e2e.constant.AUTH_URL import net.mullvad.mullvadvpn.test.e2e.constant.CONN_CHECK_URL import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL import org.json.JSONArray import org.json.JSONObject +import org.junit.jupiter.api.fail class SimpleMullvadHttpClient(context: Context) { @@ -40,9 +40,13 @@ class SimpleMullvadHttpClient(context: Context) { } } - fun createAccount(): String { - return sendSimpleSynchronousRequest(method = Request.Method.POST, url = ACCOUNT_URL)!! - .getString("number") + fun createAccountUsingPartnerApi(partnerAuth: String): String { + return sendSimpleSynchronousRequest( + method = Request.Method.POST, + url = PARTNER_ACCOUNT_URL, + authorizationHeader = "Basic $partnerAuth", + )!! + .getString("id") } fun addTimeToAccountUsingPartnerAuth( @@ -201,6 +205,10 @@ class SimpleMullvadHttpClient(context: Context) { private val onErrorResponse = { error: VolleyError -> if (error.networkResponse != null) { + if (error.networkResponse.statusCode == 429) { + fail("Request failed with response status code 429: Too many requests") + } + Logger.e( "Response returned error message: ${error.message} " + "status code: ${error.networkResponse.statusCode}"