Skip to content

Commit b5883b8

Browse files
niklasberglundniklasberglund
committedDec 18, 2024
Use stagemole with e2e tests
1 parent 15fc4f9 commit b5883b8

File tree

8 files changed

+124
-25
lines changed

8 files changed

+124
-25
lines changed
 

‎.github/workflows/android-app.yml

+48-7
Original file line numberDiff line numberDiff line change
@@ -37,15 +37,22 @@ on:
3737
type: boolean
3838
required: false
3939
mockapi_test_repeat:
40-
description: Mockapi test repeat(self hosted)
40+
description: Mockapi test repeat (self hosted)
4141
default: '1'
4242
required: true
4343
type: string
4444
e2e_test_repeat:
45-
description: e2e test repeat(self hosted)
45+
description: e2e test repeat (self hosted)
4646
default: '0'
4747
required: true
4848
type: string
49+
e2e_tests_infra_flavor:
50+
description: >
51+
Infra environment to run e2e tests on (prod/stagemole).
52+
If set to 'stagemole' test-related artefacts will be uploaded.
53+
default: 'stagemole'
54+
required: true
55+
type: string
4956
# Build if main is updated to ensure up-to-date caches are available
5057
push:
5158
branches: [main]
@@ -341,7 +348,9 @@ jobs:
341348

342349
- name: Build stagemole app
343350
uses: burrunan/gradle-cache-action@v1
344-
if: github.event.inputs.run_firebase_tests == 'true'
351+
if: >
352+
(github.event.inputs.e2e_test_repeat != '0' && github.event.inputs.e2e_tests_infra_flavor == 'stagemole') ||
353+
github.event.inputs.run_firebase_tests == 'true'
345354
with:
346355
job-id: jdk17
347356
arguments: assemblePlayStagemoleDebug
@@ -472,9 +481,18 @@ jobs:
472481
TEST_TYPE: ${{ matrix.test-type }}
473482
BILLING_FLAVOR: oss
474483
INFRA_FLAVOR: prod
484+
TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs'
475485
REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
476486
run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
477487

488+
- name: Pull test report
489+
if: always() && matrix.test-repeat != 0 && github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
490+
shell: bash -ieo pipefail {0}
491+
env:
492+
TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs'
493+
REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
494+
run: ./android/scripts/pull-test-output.sh --test-type ${{ matrix.test-type }}
495+
478496
- name: Upload instrumentation report (${{ matrix.test-type }})
479497
uses: actions/upload-artifact@v4
480498
if: always() && matrix.test-repeat != 0
@@ -526,7 +544,7 @@ jobs:
526544

527545
- name: Calculate timeout
528546
id: calculate-timeout
529-
run: echo "timeout=$(( ${{ matrix.test-repeat }} * 10 ))" >> $GITHUB_OUTPUT
547+
run: echo "timeout=$(( ${{ matrix.test-repeat }} * 15 ))" >> $GITHUB_OUTPUT
530548
shell: bash
531549

532550
- name: Run instrumented test script
@@ -536,15 +554,38 @@ jobs:
536554
env:
537555
AUTO_FETCH_TEST_HELPER_APKS: true
538556
TEST_TYPE: e2e
539-
BILLING_FLAVOR: oss
540-
INFRA_FLAVOR: prod
541-
VALID_TEST_ACCOUNT_NUMBER: ${{ secrets.ANDROID_PROD_TEST_ACCOUNT }}
557+
BILLING_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor == 'prod' && 'oss' || 'play' }}
558+
INFRA_FLAVOR: ${{ github.event.inputs.e2e_tests_infra_flavor }}
559+
PARTNER_AUTH: ${{ github.event.inputs.e2e_tests_infra_flavor == 'stagemole' && secrets.STAGEMOLE_PARTNER_AUTH || '' }}
560+
VALID_TEST_ACCOUNT_NUMBER: >
561+
github.event.inputs.e2e_tests_infra_flavor == 'prod' &&
562+
secrets.ANDROID_PROD_TEST_ACCOUNT || ''
542563
INVALID_TEST_ACCOUNT_NUMBER: '0000000000000000'
543564
ENABLE_HIGHLY_RATE_LIMITED_TESTS: ${{ github.event_name == 'schedule' && 'true' || 'false' }}
544565
ENABLE_ACCESS_TO_LOCAL_API_TESTS: true
566+
TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs'
545567
REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
546568
run: ./android/scripts/run-instrumented-tests-repeat.sh ${{ matrix.test-repeat }}
547569

570+
- name: Pull test report
571+
if: >
572+
always() && matrix.test-repeat != 0 &&
573+
github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
574+
shell: bash -ieo pipefail {0}
575+
env:
576+
TEST_DEVICE_OUTPUTS_DIR: '/sdcard/Download/test-outputs'
577+
REPORT_DIR: ${{ steps.prepare-report-dir.outputs.report_dir }}
578+
run: ./android/scripts/pull-test-output.sh --test-type e2e
579+
580+
- name: Upload e2e instrumentation report
581+
uses: actions/upload-artifact@v4
582+
if: >
583+
always() && matrix.test-repeat != 0 &&
584+
github.event.inputs.e2e_tests_infra_flavor == 'stagemole'
585+
with:
586+
name: e2e-instrumentation-report
587+
path: ${{ steps.prepare-report-dir.outputs.report_dir }}
588+
548589
firebase-tests:
549590
name: Run firebase tests
550591
if: github.event.inputs.run_firebase_tests == 'true'

‎android/scripts/pull-test-output.sh

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
#!/usr/bin/env bash
2+
3+
set -eu
4+
5+
TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs/attachments}" # Must match the path where e2e tests output their attachments
6+
REPORT_DIR="${REPORT_DIR:-}"
7+
8+
while [[ "$#" -gt 0 ]]; do
9+
case $1 in
10+
--test-type)
11+
if [[ -n "${2-}" && "$2" =~ ^(app|mockapi|e2e)$ ]]; then
12+
TEST_TYPE="$2"
13+
else
14+
echo "Error: Bad or missing test type. Must be one of: app, mockapi, e2e"
15+
exit 1
16+
fi
17+
shift 2
18+
;;
19+
*)
20+
echo "Unknown argument: $1"
21+
exit 1
22+
;;
23+
esac
24+
done
25+
26+
if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then
27+
echo ""
28+
echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set."
29+
exit 1
30+
fi
31+
32+
if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
33+
echo ""
34+
echo "Error: The variable REPORT_DIR must be set and the directory must exist."
35+
exit 1
36+
fi
37+
38+
echo "Collecting produced test attachments and logs..."
39+
adb pull "$TEST_DEVICE_OUTPUTS_DIR" "$REPORT_DIR"

‎android/scripts/run-instrumented-tests.sh

+23-12
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,20 @@
22

33
set -eu
44

5+
6+
cleanup() {
7+
echo "### Trapped termination signal, clean up ###"
8+
echo "### Store logcat and instrumentation log ###"
9+
adb shell "mkdir -p $TEST_DEVICE_OUTPUTS_DIR"
10+
adb shell "logcat -d > $TEST_DEVICE_OUTPUTS_DIR/logcat.txt"
11+
12+
if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then
13+
rm -rf "$TEMP_DOWNLOAD_DIR"
14+
fi
15+
}
16+
17+
trap cleanup SIGHUP EXIT
18+
519
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
620
cd "$SCRIPT_DIR"
721

@@ -18,6 +32,7 @@ VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}"
1832
INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}"
1933
ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}"
2034
ENABLE_ACCESS_TO_LOCAL_API_TESTS="${ENABLE_ACCESS_TO_LOCAL_API_TESTS:-false}"
35+
TEST_DEVICE_OUTPUTS_DIR="${TEST_DEVICE_OUTPUTS_DIR:-/sdcard/Download/test-outputs/attachments}" # Must match the path where e2e tests output their attachments
2136
REPORT_DIR="${REPORT_DIR:-}"
2237

2338
while [[ "$#" -gt 0 ]]; do
@@ -72,6 +87,7 @@ if [[ -z ${BILLING_FLAVOR-} ]]; then
7287
fi
7388

7489
echo "### Configuration ###"
90+
echo "Test device outputs dir: $TEST_DEVICE_OUTPUTS_DIR"
7591
echo "Report dir: $REPORT_DIR"
7692
echo "Test type: $TEST_TYPE"
7793
echo "Infra flavor: $INFRA_FLAVOR"
@@ -144,6 +160,12 @@ case "$TEST_TYPE" in
144160
;;
145161
esac
146162

163+
if [[ -z $TEST_DEVICE_OUTPUTS_DIR ]]; then
164+
echo ""
165+
echo "Error: The variable TEST_DEVICE_OUTPUTS_DIR must be set."
166+
exit 1
167+
fi
168+
147169
if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
148170
echo ""
149171
echo "Error: The variable REPORT_DIR must be set and the directory must exist."
@@ -153,17 +175,14 @@ fi
153175
GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS"
154176

155177
INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt"
156-
LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt"
157-
LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots"
158178
DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
159-
DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments"
160179

161180
echo ""
162181
echo "### Ensure clean report structure ###"
163182
rm -rf "${REPORT_DIR:?}/*"
164183
adb logcat --clear
165184
adb shell rm -rf "$DEVICE_SCREENSHOT_PATH"
166-
adb shell rm -rf "$DEVICE_TEST_ATTACHMENTS_PATH"
185+
adb shell rm -rf "$TEST_DEVICE_OUTPUTS_DIR"
167186
echo ""
168187

169188
if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then
@@ -238,13 +257,5 @@ echo "### Checking logs for success message ###"
238257
if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then
239258
echo "Success, no failures!"
240259
else
241-
echo "One or more tests failed, see logs for more details."
242-
echo "Collecting report..."
243-
adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots"
244-
adb logcat -d > "$LOGCAT_FILE_PATH"
245260
exit 1
246261
fi
247-
248-
if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then
249-
rm -rf "$TEMP_DOWNLOAD_DIR"
250-
fi

‎android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/Attachment.kt

+3-2
Original file line numberDiff line numberDiff line change
@@ -7,11 +7,12 @@ import java.io.IOException
77
import org.junit.jupiter.api.fail
88

99
object Attachment {
10-
private const val DIRECTORY_NAME = "test-attachments"
10+
val DIRECTORY_PATH = "${Environment.DIRECTORY_DOWNLOADS}/test-outputs/attachments"
11+
1112
private val testAttachmentsDirectory =
1213
File(
1314
Environment.getExternalStorageDirectory(),
14-
"${Environment.DIRECTORY_DOWNLOADS}/$DIRECTORY_NAME",
15+
DIRECTORY_PATH,
1516
)
1617

1718
fun saveAttachment(fileName: String, data: ByteArray) {

‎android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/misc/CaptureScreenRecordingsExtension.kt

+1-1
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,6 @@ class CaptureScreenRecordingsExtension : BeforeEachCallback, AfterEachCallback {
5555

5656
companion object {
5757
val OUTPUT_DIRECTORY =
58-
"${Environment.getExternalStorageDirectory().path}/Download/test-attachments/video"
58+
"${Environment.getExternalStorageDirectory().path}/Download/test-outputs/attachments/video"
5959
}
6060
}

‎android/test/common/src/main/kotlin/net/mullvad/mullvadvpn/test/common/rule/CaptureScreenshotOnFailedTestRule.kt

+4-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import android.content.ContentValues
55
import android.graphics.Bitmap
66
import android.os.Build
77
import android.os.Environment
8+
import android.os.Environment.DIRECTORY_DOWNLOADS
89
import android.os.Environment.DIRECTORY_PICTURES
910
import android.provider.MediaStore
1011
import androidx.annotation.RequiresApi
@@ -16,6 +17,7 @@ import java.io.IOException
1617
import java.nio.file.Paths
1718
import java.time.OffsetDateTime
1819
import java.time.temporal.ChronoUnit
20+
import net.mullvad.mullvadvpn.test.common.misc.Attachment
1921
import org.junit.jupiter.api.extension.ExtensionContext
2022
import org.junit.jupiter.api.extension.TestWatcher
2123

@@ -55,7 +57,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch
5557
) {
5658
contentValues.apply {
5759
put(MediaStore.MediaColumns.DISPLAY_NAME, filename)
58-
put(MediaStore.Images.Media.RELATIVE_PATH, "$DIRECTORY_PICTURES/$baseDir")
60+
put(MediaStore.Images.Media.RELATIVE_PATH, "${Attachment.DIRECTORY_PATH}/$baseDir")
5961
}
6062

6163
val uri =
@@ -78,7 +80,7 @@ class CaptureScreenshotOnFailedTestRule(private val testTag: String) : TestWatch
7880
private fun Bitmap.writeToExternalStorage(baseDir: String, filename: String) {
7981
val screenshotBaseDirectory =
8082
Paths.get(
81-
Environment.getExternalStoragePublicDirectory(DIRECTORY_PICTURES).path,
83+
Environment.getExternalStoragePublicDirectory(Attachment.DIRECTORY_PATH).path,
8284
baseDir,
8385
)
8486
.toFile()

‎android/test/e2e/README.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -58,4 +58,4 @@ docker run --rm --volumes-from gcloud-config -v ${PWD}:/android gcr.io/google.co
5858
```
5959

6060
## Test artefacts
61-
Test artefacts are stored on the test device in `/sdcard/Download/test-attachments`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten.
61+
Test artefacts are stored on the test device in `/sdcard/Download/test-outputs`. In CI this directory is cleared in between each test run, but note that when running tests locally the directory isn't cleared but already existing files are overwritten.

‎android/test/e2e/src/main/kotlin/net/mullvad/mullvadvpn/test/e2e/misc/SimpleMullvadHttpClient.kt

+5
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import net.mullvad.mullvadvpn.test.e2e.constant.DEVICE_LIST_URL
1717
import net.mullvad.mullvadvpn.test.e2e.constant.PARTNER_ACCOUNT_URL
1818
import org.json.JSONArray
1919
import org.json.JSONObject
20+
import org.junit.jupiter.api.fail
2021

2122
class SimpleMullvadHttpClient(context: Context) {
2223

@@ -201,6 +202,10 @@ class SimpleMullvadHttpClient(context: Context) {
201202

202203
private val onErrorResponse = { error: VolleyError ->
203204
if (error.networkResponse != null) {
205+
if (error.networkResponse.statusCode == 429) {
206+
fail("Request failed with response status code 429: Too many requests")
207+
}
208+
204209
Logger.e(
205210
"Response returned error message: ${error.message} " +
206211
"status code: ${error.networkResponse.statusCode}"

0 commit comments

Comments
 (0)
Failed to load comments.