Run tests in the order given

Author: Serock3

test/test-manager/src/main.rs

@@ -13,6 +13,7 @@ use std::{net::SocketAddr, path::PathBuf};
 
 use anyhow::{Context, Result};
 use clap::Parser;
+use tests::get_filtered_tests;
 use vm::provision;
 
 use crate::tests::config::OpenVPNCertificate;
@@ -118,7 +119,8 @@ enum Commands {
         #[arg(long)]
         openvpn_certificate: Option<PathBuf>,
 
-        /// Only run tests matching substrings
+        /// Names of tests to run. The order given will be respected. If not set, all tests will be
+        /// run.
         test_filters: Vec<String>,
 
         /// Print results live
@@ -253,6 +255,7 @@ async fn main() -> Result<()> {
                 (false, true) => config::Display::Vnc,
                 (true, true) => unreachable!("invalid combination"),
             };
+            let tests = get_filtered_tests(&test_filters)?;
 
             let mullvad_host = config.get_host();
             log::debug!("Mullvad host: {mullvad_host}");
@@ -325,7 +328,7 @@ async fn main() -> Result<()> {
                     openvpn_certificate,
                 ),
                 &*instance,
-                &test_filters,
+                tests,
                 skip_wait,
                 !verbose,
                 summary_logger,

test/test-manager/src/run_tests.rs

@@ -2,7 +2,7 @@ use crate::{
     logging::{Logger, Panic, TestOutput, TestResult},
     mullvad_daemon::{self, MullvadClientArgument, RpcClientProvider},
     summary::SummaryLogger,
-    tests::{self, config::TEST_CONFIG, get_tests, TestContext},
+    tests::{self, config::TEST_CONFIG, TestContext, TestMetadata},
     vm,
 };
 use anyhow::{Context, Result};
@@ -103,7 +103,7 @@ impl TestHandler<'_> {
 pub async fn run(
     config: tests::config::TestConfig,
     instance: &dyn vm::VmInstance,
-    test_filters: &[String],
+    tests: Vec<&TestMetadata>,
     skip_wait: bool,
     print_failed_tests_only: bool,
     summary_logger: Option<SummaryLogger>,
@@ -141,22 +141,6 @@ pub async fn run(
         logger: Logger::get_or_init(),
     };
 
-    let mut tests = get_tests();
-
-    tests.retain(|test| test.should_run_on_os(TEST_CONFIG.os));
-
-    if !test_filters.is_empty() {
-        tests.retain(|test| {
-            for command in test_filters {
-                let command = command.to_lowercase();
-                if test.command.to_lowercase().contains(&command) {
-                    return true;
-                }
-            }
-            false
-        });
-    }
-
     if TEST_CONFIG.app_package_to_upgrade_from_filename.is_some() {
         test_handler
             .run_test(

test/test-manager/src/tests/mod.rs

@@ -78,6 +78,26 @@ pub fn get_tests() -> Vec<&'static TestMetadata> {
     tests
 }
 
+pub fn get_filtered_tests(specified_tests: &[String]) -> Result<Vec<&TestMetadata>, anyhow::Error> {
+    let mut tests = get_tests();
+    tests.retain(|test| test.should_run_on_os(TEST_CONFIG.os));
+    if !specified_tests.is_empty() {
+        specified_tests
+            .iter()
+            .map(|f| {
+                tests
+                    .iter()
+                    .find(|t| t.command.eq_ignore_ascii_case(f))
+                    .cloned()
+                    .ok_or(anyhow::anyhow!("Test '{f}' not found"))
+            })
+            .collect()
+    } else {
+        // Keep all tests
+        Ok(tests)
+    }
+}
+
 /// Make sure the daemon is installed and logged in and restore settings to the defaults.
 pub async fn prepare_daemon(
     rpc: &ServiceClient,