options: add --randomize option

Shuffle testing suites tests list when --randomize|-x option is defined.

Reviewed-by: Petr Vorel <pvorel@suse.cz>
Reviewed-by: Cyril Hrubis <chrubis@suse.cz>
Signed-off-by: Andrea Cervesato <andrea.cervesato@suse.com>

Author: acerv

libkirk/main.py

@@ -335,6 +335,7 @@ async def session_run() -> None:
                 restore=restore_dir,
                 suite_iterate=args.suite_iterate,
                 skip_tests=skip_tests,
+                randomize=args.randomize,
             )
         except asyncio.CancelledError:
             await session.stop()
@@ -468,6 +469,11 @@ def run(cmd_args: list = None) -> None:
         "-p",
         action="store_true",
         help="Force parallelization execution of all tests")
+    parser.add_argument(
+        "--randomize",
+        "-x",
+        action="store_true",
+        help="Force parallelization execution of all tests")
 
     # session arguments
     parser.add_argument(

libkirk/session.py

@@ -8,6 +8,7 @@
 import os
 import re
 import copy
+import random
 import logging
 import asyncio
 import libkirk
@@ -374,6 +375,8 @@ async def run(self, **kwargs: dict) -> None:
         :type restore: str
         :param suite_iterate: execute all suites multiple times
         :type suite_iterate: int
+        :param randomize: randomize all tests if True
+        :type randomize: bool
         """
         command = kwargs.get("command", None)
         suites = kwargs.get("suites", None)
@@ -382,6 +385,7 @@ async def run(self, **kwargs: dict) -> None:
         report_path = kwargs.get("report_path", None)
         restore = kwargs.get("restore", None)
         suite_iterate = kwargs.get("suite_iterate", 1)
+        randomize = kwargs.get("randomize", False)
 
         async with self._run_lock:
             await libkirk.events.fire("session_started", self._tmpdir.abspath)
@@ -404,6 +408,10 @@ async def run(self, **kwargs: dict) -> None:
                     suites_obj = self._apply_iterate(
                         suites_obj, suite_iterate)
 
+                    if randomize:
+                        for suite in suites_obj:
+                            random.shuffle(suite.tests)
+
                     await self._scheduler.schedule(suites_obj)
             except KirkException as err:
                 if not self._stop:

libkirk/tests/test_main.py

@@ -381,3 +381,31 @@ def test_suite_iterate(self, tmpdir):
         assert excinfo.value.code == libkirk.main.RC_OK
 
         self.read_report(temp, 8)
+
+    def test_randomize(self, tmpdir):
+        """
+        Test --randomize option.
+        """
+        num_of_suites = 10
+
+        temp = tmpdir.mkdir("temp")
+        cmd_args = [
+            "--tmp-dir", str(temp),
+            "--framework", "dummy",
+            "--run-suite",
+        ]
+        cmd_args.extend(["suite01"] * num_of_suites)
+        cmd_args.append("--randomize")
+
+        with pytest.raises(SystemExit) as excinfo:
+            libkirk.main.run(cmd_args=cmd_args)
+
+        assert excinfo.value.code == libkirk.main.RC_OK
+
+        report_d = self.read_report(temp, 2 * num_of_suites)
+
+        tests_names = []
+        for test in report_d["results"]:
+            tests_names.append(test["test_fqn"])
+
+        assert ["test01", "test02"] * num_of_suites != tests_names

libkirk/tests/test_session.py

@@ -139,3 +139,27 @@ async def test_run_suite_iterate(self, tmpdir, session, iterate, expect):
         with open(report, "r", encoding="utf-8") as report_file:
             report_data = json.loads(report_file.read())
             assert len(report_data["results"]) == expect
+
+    async def test_run_randomize(self, tmpdir, session):
+        """
+        Test run method when executing shuffled tests.
+        """
+        num_of_suites = 5
+
+        report = str(tmpdir / "report.json")
+        await session.run(
+            suites=["suite01"] * num_of_suites,
+            randomize=True,
+            report_path=report)
+
+        report_data = None
+        with open(report, "r", encoding="utf-8") as report_file:
+            report_data = json.loads(report_file.read())
+
+        assert len(report_data["results"]) == 2 * num_of_suites
+
+        tests_names = []
+        for test in report_data["results"]:
+            tests_names.append(test["test_fqn"])
+
+        assert ["test01", "test02"] * num_of_suites != tests_names