Merge remote-tracking branch 'magnus/feature/use-standalone-binaries-in-tests'

Author: MarkusPettersson98

test/scripts/build.sh

@@ -7,6 +7,7 @@ set -eu
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 TEST_FRAMEWORK_ROOT="$SCRIPT_DIR/.."
+REPO_ROOT="$SCRIPT_DIR/../.."
 
 # Build
 build_linux() {
@@ -19,6 +20,10 @@ build_linux() {
     "$SCRIPT_DIR/build-runner.sh" linux
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/test-runner" "$TEST_FRAMEWORK_ROOT/dist/"
     cp "$TEST_FRAMEWORK_ROOT/target/x86_64-unknown-linux-gnu/release/connection-checker" "$TEST_FRAMEWORK_ROOT/dist/"
+
+    # Build mullvad-version
+    cargo build --manifest-path="$REPO_ROOT/Cargo.toml" --release --bin mullvad-version
+    cp "$REPO_ROOT/target/release/mullvad-version" "$TEST_FRAMEWORK_ROOT/dist/"
 }
 
 build_linux

test/scripts/container-run.sh

@@ -10,7 +10,7 @@ if [ ! -d "$PACKAGE_DIR" ]; then
   echo "$PACKAGE_DIR does not exist. It is needed to build the test bundle, so please go ahead and create the directory and re-run this script."
 fi
 
-SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" > /dev/null && pwd )"
 REPO_DIR="$SCRIPT_DIR/../.."
 cd "$SCRIPT_DIR"
 

test/scripts/test-utils.sh

@@ -2,6 +2,11 @@
 
 set -eu
 
+function executable_not_found_in_dist_error {
+    1>&2 echo "Executable \"$1\" not found in specified dist dir. Exiting."
+    exit 1
+}
+
 # Returns the directory of the test-utils.sh script
 function get_test_utls_dir {
     local script_path="${BASH_SOURCE[0]}"
@@ -24,7 +29,14 @@ LATEST_STABLE_RELEASE=$(jq -r '[.[] | select(.prerelease==false)] | .[0].tag_nam
 function get_current_version {
     local app_dir
     app_dir="$(get_test_utls_dir)/../.."
-    cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+    if [ -n "${TEST_DIST_DIR+x}" ]; then
+        if [ ! -x "${TEST_DIST_DIR%/}/mullvad-version" ]; then
+            executable_not_found_in_dist_error mullvad-version
+        fi
+        "${TEST_DIST_DIR%/}/mullvad-version"
+    else
+        cargo run -q --manifest-path="$app_dir/Cargo.toml" --bin mullvad-version
+    fi
 }
 
 CURRENT_VERSION=$(get_current_version)
@@ -224,12 +236,20 @@ function run_tests_for_os {
         exit 1
     fi
 
-    echo "**********************************"
-    echo "* Building test runner"
-    echo "**********************************"
-
-    nice_time build_test_runner "$vm"
+    if [ -n "${TEST_DIST_DIR+x}" ]; then
+        if [ ! -x "${TEST_DIST_DIR%/}/test-runner" ]; then
+            executable_not_found_in_dist_error test-runner
+        fi
 
+        echo "**********************************"
+        echo "* Using test-runner in $TEST_DIST_DIR"
+        echo "**********************************"
+    else
+        echo "**********************************"
+        echo "* Building test runner"
+        echo "**********************************"
+        nice_time build_test_runner "$vm"
+    fi
 
     echo "**********************************"
     echo "* Running tests"
@@ -256,15 +276,26 @@ function run_tests_for_os {
     test_dir=$(get_test_utls_dir)/..
     read -ra test_filters_arg <<<"${TEST_FILTERS:-}" # Split the string by words into an array
     pushd "$test_dir"
-        if ! RUST_LOG_STYLE=always cargo run --bin test-manager \
-            run-tests \
+        if [ -n "${TEST_DIST_DIR+x}" ]; then
+            if [ ! -x "${TEST_DIST_DIR%/}/test-manager" ]; then
+                executable_not_found_in_dist_error test-manager
+            fi
+            test_manager="${TEST_DIST_DIR%/}/test-manager"
+            runner_dir_flag=("--runner-dir" "$TEST_DIST_DIR")
+        else
+            test_manager="cargo run --bin test-manager"
+            runner_dir_flag=()
+        fi
+
+        if ! RUST_LOG_STYLE=always $test_manager run-tests \
             --account "${ACCOUNT_TOKEN:?Error: ACCOUNT_TOKEN not set}" \
             --app-package "${APP_PACKAGE:?Error: APP_PACKAGE not set}" \
             "${upgrade_package_arg[@]}" \
             "${test_report_arg[@]}" \
             --package-dir "${package_dir}" \
             --vm "$vm" \
             "${test_filters_arg[@]}" \
+            "${runner_dir_flag[@]}" \
             2>&1 | sed -r "s/${ACCOUNT_TOKEN}/\{ACCOUNT_TOKEN\}/g"; then
             echo "Test run failed"
             exit 1
@@ -279,7 +310,7 @@ function build_current_version {
     app_dir="$(get_test_utls_dir)/../.."
     local app_filename
     # TODO: TEST_OS must be set to local OS manually, should be set automatically
-    app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}") 
+    app_filename=$(get_app_filename "$CURRENT_VERSION" "${TEST_OS:?Error: TEST_OS not set}")
     local package_dir
     package_dir=$(get_package_dir)
     local app_package="$package_dir"/"$app_filename"
@@ -310,4 +341,4 @@ function build_current_version {
     else
         echo "GUI e2e executable for current version already exists at $gui_test_bin, skipping build"
     fi
-}
+}

test/test-by-version.sh

@@ -11,6 +11,7 @@ usage() {
     echo "Optional environment variables:"
     echo "  - APP_VERSION: The version of the app to test (defaults to the latest stable release)"
     echo "  - APP_PACKAGE_TO_UPGRADE_FROM: The package version to upgrade from (defaults to none)"
+    echo "  - TEST_DIST_DIR: Relative path to a directory with prebuilt binaries as produced by scripts/build.sh."
     echo "  - TEST_FILTERS: specifies which tests to run (defaults to all)"
     echo "  - TEST_REPORT : path to save the test results in a structured format"
 }