Run tests in the order given

Author: Serock3

test/test-manager/src/main.rs

@@ -13,6 +13,7 @@ use std::{net::SocketAddr, path::PathBuf};
 
 use anyhow::{Context, Result};
 use clap::Parser;
+use tests::{config::TEST_CONFIG, get_filtered_tests};
 use vm::provision;
 
 use crate::tests::config::OpenVPNCertificate;
@@ -118,7 +119,8 @@ enum Commands {
         #[arg(long)]
         openvpn_certificate: Option<PathBuf>,
 
-        /// Only run tests matching substrings
+        /// Names of tests to run. The order given will be respected. If not set, all tests will be
+        /// run.
         test_filters: Vec<String>,
 
         /// Print results live
@@ -294,6 +296,28 @@ async fn main() -> Result<()> {
                 .await
                 .context("Failed to run provisioning for VM")?;
 
+            TEST_CONFIG.init(tests::config::TestConfig::new(
+                account,
+                artifacts_dir,
+                manifest
+                    .app_package_path
+                    .file_name()
+                    .unwrap()
+                    .to_string_lossy()
+                    .into_owned(),
+                manifest
+                    .app_package_to_upgrade_from_path
+                    .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
+                manifest
+                    .gui_package_path
+                    .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
+                mullvad_host,
+                vm::network::bridge()?,
+                test_rpc::meta::Os::from(vm_config.os_type),
+                openvpn_certificate,
+            ));
+            let tests = get_filtered_tests(&test_filters)?;
+
             // For convenience, spawn a SOCKS5 server that is reachable for tests that need it
             let socks = socks_server::spawn(SocketAddr::new(
                 crate::vm::network::NON_TUN_GATEWAY.into(),
@@ -303,35 +327,9 @@ async fn main() -> Result<()> {
 
             let skip_wait = vm_config.provisioner != config::Provisioner::Noop;
 
-            let result = run_tests::run(
-                tests::config::TestConfig::new(
-                    account,
-                    artifacts_dir,
-                    manifest
-                        .app_package_path
-                        .file_name()
-                        .unwrap()
-                        .to_string_lossy()
-                        .into_owned(),
-                    manifest
-                        .app_package_to_upgrade_from_path
-                        .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
-                    manifest
-                        .gui_package_path
-                        .map(|path| path.file_name().unwrap().to_string_lossy().into_owned()),
-                    mullvad_host,
-                    vm::network::bridge()?,
-                    test_rpc::meta::Os::from(vm_config.os_type),
-                    openvpn_certificate,
-                ),
-                &*instance,
-                &test_filters,
-                skip_wait,
-                !verbose,
-                summary_logger,
-            )
-            .await
-            .context("Tests failed");
+            let result = run_tests::run(&*instance, tests, skip_wait, !verbose, summary_logger)
+                .await
+                .context("Tests failed");
 
             if display {
                 instance.wait().await;

test/test-manager/src/run_tests.rs

@@ -2,7 +2,7 @@ use crate::{
     logging::{Logger, Panic, TestOutput, TestResult},
     mullvad_daemon::{self, MullvadClientArgument, RpcClientProvider},
     summary::SummaryLogger,
-    tests::{self, config::TEST_CONFIG, get_tests, TestContext},
+    tests::{self, config::TEST_CONFIG, TestContext, TestMetadata},
     vm,
 };
 use anyhow::{Context, Result};
@@ -101,15 +101,13 @@ impl TestHandler<'_> {
 }
 
 pub async fn run(
-    config: tests::config::TestConfig,
     instance: &dyn vm::VmInstance,
-    test_filters: &[String],
+    tests: Vec<&TestMetadata>,
     skip_wait: bool,
     print_failed_tests_only: bool,
     summary_logger: Option<SummaryLogger>,
 ) -> Result<TestResult> {
     log::trace!("Setting test constants");
-    TEST_CONFIG.init(config);
 
     let pty_path = instance.get_pty();
 
@@ -141,22 +139,6 @@ pub async fn run(
         logger: Logger::get_or_init(),
     };
 
-    let mut tests = get_tests();
-
-    tests.retain(|test| test.should_run_on_os(TEST_CONFIG.os));
-
-    if !test_filters.is_empty() {
-        tests.retain(|test| {
-            for command in test_filters {
-                let command = command.to_lowercase();
-                if test.command.to_lowercase().contains(&command) {
-                    return true;
-                }
-            }
-            false
-        });
-    }
-
     if TEST_CONFIG.app_package_to_upgrade_from_filename.is_some() {
         test_handler
             .run_test(

test/test-manager/src/tests/mod.rs

@@ -78,6 +78,26 @@ pub fn get_tests() -> Vec<&'static TestMetadata> {
     tests
 }
 
+pub fn get_filtered_tests(specified_tests: &[String]) -> Result<Vec<&TestMetadata>, anyhow::Error> {
+    let mut tests = get_tests();
+    tests.retain(|test| test.should_run_on_os(TEST_CONFIG.os));
+    if !specified_tests.is_empty() {
+        specified_tests
+            .iter()
+            .map(|f| {
+                tests
+                    .iter()
+                    .find(|t| t.command.eq_ignore_ascii_case(f))
+                    .cloned()
+                    .ok_or(anyhow::anyhow!("Test '{f}' not found"))
+            })
+            .collect()
+    } else {
+        // Keep all tests
+        Ok(tests)
+    }
+}
+
 /// Make sure the daemon is installed and logged in and restore settings to the defaults.
 pub async fn prepare_daemon(
     rpc: &ServiceClient,