run-instrumented-tests.sh

#!/usr/bin/env bash

set -eu

SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
cd "$SCRIPT_DIR"

AUTO_FETCH_TEST_HELPER_APKS=${AUTO_FETCH_TEST_HELPER_APKS:-"false"}

APK_BASE_DIR=${APK_BASE_DIR:-"$SCRIPT_DIR/.."}
LOG_SUCCESS_REGEX="OK \([1-9][0-9]* tests\)"

ORCHESTRATOR_URL=https://dl.google.com/android/maven2/androidx/test/orchestrator/1.5.0/orchestrator-1.5.0.apk
TEST_SERVICES_URL=https://dl.google.com/android/maven2/androidx/test/services/test-services/1.5.0/test-services-1.5.0.apk

PARTNER_AUTH="${PARTNER_AUTH:-}"
VALID_TEST_ACCOUNT_NUMBER="${VALID_TEST_ACCOUNT_NUMBER:-}"
INVALID_TEST_ACCOUNT_NUMBER="${INVALID_TEST_ACCOUNT_NUMBER:-}"
ENABLE_HIGHLY_RATE_LIMITED_TESTS="${ENABLE_HIGHLY_RATE_LIMITED_TESTS:-false}"
ENABLE_ACCESS_TO_LOCAL_API_TESTS="${ENABLE_ACCESS_TO_LOCAL_API_TESTS:-false}"
REPORT_DIR="${REPORT_DIR:-}"

while [[ "$#" -gt 0 ]]; do
    case $1 in
        --test-type)
            if [[ -n "${2-}" && "$2" =~ ^(app|mockapi|e2e)$ ]]; then
                TEST_TYPE="$2"
            else
                echo "Error: Bad or missing test type. Must be one of: app, mockapi, e2e"
                exit 1
            fi
            shift 2
            ;;
        --infra-flavor)
            if [[ -n "${2-}" && "$2" =~ ^(prod|stagemole)$ ]]; then
                INFRA_FLAVOR="$2"
            else
                echo "Error: Bad or missing infra flavor. Must be one of: prod, stagemole"
                exit 1
            fi
            shift 2
            ;;
        --billing-flavor)
            if [[ -n "${2-}" && "$2" =~ ^(oss|play)$ ]]; then
                BILLING_FLAVOR="$2"
            else
                echo "Error: Bad or missing billing flavor. Must be one of: oss, play"
                exit 1
            fi
            shift 2
            ;;
        *)
            echo "Unknown argument: $1"
            exit 1
            ;;
    esac
done

if [[ -z ${TEST_TYPE-} ]]; then
    echo "Error: Missing --test-type argument. Must be set to one of: app, e2e, mockapi"
    exit 1
fi

if [[ -z ${INFRA_FLAVOR-} ]]; then
    echo "Error: Missing --infra-flavor argument. Must be set to one of: prod, stagemole"
    exit 1
fi

if [[ -z ${BILLING_FLAVOR-} ]]; then
    echo "Error: Missing --billing-flavor argument. Must be set to one of: oss, play"
    exit 1
fi

echo "### Configuration ###"
echo "Report dir: $REPORT_DIR"
echo "Test type: $TEST_TYPE"
echo "Infra flavor: $INFRA_FLAVOR"
echo "Billing flavor: $BILLING_FLAVOR"

APK_PATH="$APK_BASE_DIR/app/build/outputs/apk/$BILLING_FLAVOR${INFRA_FLAVOR^}/debug/app-$BILLING_FLAVOR-$INFRA_FLAVOR-debug.apk"

case "$TEST_TYPE" in
    app)
    if [[ $BILLING_FLAVOR != "oss" || $INFRA_FLAVOR != "prod" ]]; then
        echo ""
        echo "Error: The 'app' test type only supports billing type 'oss' and infra type 'prod'."
        exit 1
    fi
    USE_ORCHESTRATOR="false"
    PACKAGE_NAME="net.mullvad.mullvadvpn"
    TEST_PACKAGE_NAME="net.mullvad.mullvadvpn.test"
    TEST_APK_PATH="$APK_BASE_DIR/app/build/outputs/apk/androidTest/$BILLING_FLAVOR${INFRA_FLAVOR^}/debug/app-$BILLING_FLAVOR-$INFRA_FLAVOR-debug-androidTest.apk"
    ;;
    mockapi)

    if [[ $BILLING_FLAVOR != "oss" || $INFRA_FLAVOR != "prod" ]]; then
        echo ""
        echo "Error: The 'mockapi' test type only supports billing type 'oss' and infra type 'prod'."
        exit 1
    fi
    USE_ORCHESTRATOR="true"
    PACKAGE_NAME="net.mullvad.mullvadvpn"
    TEST_PACKAGE_NAME="net.mullvad.mullvadvpn.test.mockapi"
    TEST_APK_PATH="$APK_BASE_DIR/test/mockapi/build/outputs/apk/$BILLING_FLAVOR/debug/mockapi-$BILLING_FLAVOR-debug.apk"
    ;;

    e2e)
    if [[ $BILLING_FLAVOR == "play" && $INFRA_FLAVOR != "stagemole" ]]; then
        echo ""
        echo "Error: The 'e2e' test type with billing flavor 'play' require infra flavor 'stagemole'."
        exit 1
    elif [[ $BILLING_FLAVOR == "oss" && $INFRA_FLAVOR != "prod" ]]; then
        echo ""
        echo "Error: The 'e2e' test type with billing flavor 'oss' require infra flavor 'prod'."
        exit 1
    fi
    OPTIONAL_TEST_ARGUMENTS=""
    if [[ -n ${INVALID_TEST_ACCOUNT_NUMBER-} ]]; then
        OPTIONAL_TEST_ARGUMENTS+=" -e invalid_test_account_number $INVALID_TEST_ACCOUNT_NUMBER"
    else
        echo "Error: The variable INVALID_TEST_ACCOUNT_NUMBER must be set."
        exit 1
    fi
    if [[ -n ${PARTNER_AUTH} ]]; then
        echo "Test account used for e2e test (provided/partner): partner"
        OPTIONAL_TEST_ARGUMENTS+=" -e partner_auth $PARTNER_AUTH"
    elif [[ -n ${VALID_TEST_ACCOUNT_NUMBER} ]]; then
        echo "Test account used for e2e test (provided/partner): provided"
        OPTIONAL_TEST_ARGUMENTS+=" -e valid_test_account_number $VALID_TEST_ACCOUNT_NUMBER"
    else
        echo ""
        echo "Error: The variable PARTNER_AUTH or VALID_TEST_ACCOUNT_NUMBER must be set."
        exit 1
    fi
    OPTIONAL_TEST_ARGUMENTS+=" -e enable_access_to_local_api_tests $ENABLE_ACCESS_TO_LOCAL_API_TESTS"
    OPTIONAL_TEST_ARGUMENTS+=" -e enable_highly_rate_limited_tests $ENABLE_HIGHLY_RATE_LIMITED_TESTS"
    USE_ORCHESTRATOR="true"
    PACKAGE_NAME="net.mullvad.mullvadvpn"
    if [[ "$INFRA_FLAVOR" =~ ^(devmole|stagemole)$ ]]; then
        PACKAGE_NAME+=".$INFRA_FLAVOR"
    fi
    TEST_PACKAGE_NAME="net.mullvad.mullvadvpn.test.e2e"
    TEST_APK_PATH="$APK_BASE_DIR/test/e2e/build/outputs/apk/$BILLING_FLAVOR${INFRA_FLAVOR^}/debug/e2e-$BILLING_FLAVOR-$INFRA_FLAVOR-debug.apk"
    ;;
esac

if [[ -z $REPORT_DIR || ! -d $REPORT_DIR ]]; then
    echo ""
    echo "Error: The variable REPORT_DIR must be set and the directory must exist."
    exit 1
fi

GRADLE_ENVIRONMENT_VARIABLES="TEST_E2E_ENABLEACCESSTOLOCALAPITESTS=$ENABLE_ACCESS_TO_LOCAL_API_TESTS"

INSTRUMENTATION_LOG_FILE_PATH="$REPORT_DIR/instrumentation-log.txt"
LOGCAT_FILE_PATH="$REPORT_DIR/logcat.txt"
LOCAL_SCREENSHOT_PATH="$REPORT_DIR/screenshots"
DEVICE_SCREENSHOT_PATH="/sdcard/Pictures/mullvad-$TEST_TYPE"
LOCAL_TEST_ATTACHMENTS_PATH="$REPORT_DIR/test-attachments"
DEVICE_TEST_ATTACHMENTS_PATH="/sdcard/Download/test-attachments"

echo ""
echo "### Ensure clean report structure ###"
rm -rf "${REPORT_DIR:?}/*"
adb logcat --clear
adb shell rm -rf "$DEVICE_SCREENSHOT_PATH"
adb shell rm -rf "$DEVICE_TEST_ATTACHMENTS_PATH"
echo ""

if [[ "${USE_ORCHESTRATOR-}" == "true" ]]; then
    if [[ "${AUTO_FETCH_TEST_HELPER_APKS-}" == "true" ]]; then
        echo "### Fetching orchestrator and test services apks ###"
        TEMP_DOWNLOAD_DIR=$(mktemp -d)
        ORCHESTRATOR_APK_PATH=$TEMP_DOWNLOAD_DIR/orchestrator.apk
        TEST_SERVICES_APK_PATH=$TEMP_DOWNLOAD_DIR/test-services.apk
        curl -sL "$ORCHESTRATOR_URL" -o "$ORCHESTRATOR_APK_PATH"
        curl -sL "$TEST_SERVICES_URL" -o "$TEST_SERVICES_APK_PATH"
        echo ""
    else
        if [[ -z ${ORCHESTRATOR_APK_PATH-} ]]; then
            echo "The variable ORCHESTRATOR_APK_PATH is not set."
            exit 1
        fi
        if [[ -z ${TEST_SERVICES_APK_PATH-} ]]; then
            echo "The variable TEST_SERVICES_APK_PATH is not set."
            exit 1
        fi
    fi
fi

echo "### Ensure that packages are not previously installed ###"
adb uninstall "$PACKAGE_NAME" || echo "App package not installed"
adb uninstall "$TEST_PACKAGE_NAME" || echo "Test package not installed"
adb uninstall androidx.test.services || echo "Test services package not installed"
adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not installed"
echo ""

echo "Starting instrumented tests of type: $TEST_TYPE"
echo ""

echo "### Install packages ###"
adb install -t "$APK_PATH"
adb install "$TEST_APK_PATH"
if [[ "$USE_ORCHESTRATOR" == "true" ]]; then
    echo "Using ORCHESTRATOR_APK_PATH: $ORCHESTRATOR_APK_PATH"
    adb install "$ORCHESTRATOR_APK_PATH"
    echo "Using TEST_SERVICES_APK_PATH: $TEST_SERVICES_APK_PATH"
    adb install "$TEST_SERVICES_APK_PATH"
fi
echo ""

echo "### Run instrumented test command ###"
if [[ "$USE_ORCHESTRATOR" == "true" ]]; then
    INSTRUMENTATION_COMMAND="\
    CLASSPATH=\$(pm path androidx.test.services) app_process / androidx.test.services.shellexecutor.ShellMain \
    am instrument -r -w \
    -e targetInstrumentation $TEST_PACKAGE_NAME/androidx.test.runner.AndroidJUnitRunner \
    -e clearPackageData true \
    -e runnerBuilder de.mannodermaus.junit5.AndroidJUnit5Builder \
    ${OPTIONAL_TEST_ARGUMENTS:-""} \
    androidx.test.orchestrator/androidx.test.orchestrator.AndroidTestOrchestrator"
else
    INSTRUMENTATION_COMMAND="\
    am instrument -w \
    -e runnerBuilder de.mannodermaus.junit5.AndroidJUnit5Builder \
    $TEST_PACKAGE_NAME/androidx.test.runner.AndroidJUnitRunner"
fi
adb shell "$GRADLE_ENVIRONMENT_VARIABLES $INSTRUMENTATION_COMMAND" | tee "$INSTRUMENTATION_LOG_FILE_PATH"
echo ""

echo "### Ensure that packages are uninstalled ###"
adb uninstall "$PACKAGE_NAME" || echo "App package not installed"
adb uninstall "$TEST_PACKAGE_NAME" || echo "Test package not installed"
adb uninstall androidx.test.services || echo "Test services package not installed"
adb uninstall androidx.test.orchestrator || echo "Test orchestrator package not installed"
echo ""

echo "### Checking logs for success message ###"
if grep -q -E "$LOG_SUCCESS_REGEX" "$INSTRUMENTATION_LOG_FILE_PATH"; then
    echo "Success, no failures!"
else
    echo "One or more tests failed, see logs for more details."
    echo "Collecting report..."
    adb pull "$DEVICE_SCREENSHOT_PATH" "$LOCAL_SCREENSHOT_PATH" || echo "No screenshots"
    adb pull "$DEVICE_TEST_ATTACHMENTS_PATH" "$LOCAL_TEST_ATTACHMENTS_PATH" || echo "No test attachments"
    adb logcat -d > "$LOGCAT_FILE_PATH"
    exit 1
fi

if [[ -n ${TEMP_DOWNLOAD_DIR-} ]]; then
    rm -rf "$TEMP_DOWNLOAD_DIR"
fi