options: add --suite-iterate option

Add --suite-iterate|-I option to execute testing suites multiple times.
The --skip-file option was using -I so we replaced it with -K. For
continuity we also changed --skip-file into -k.

Reviewed-by: Petr Vorel <[email protected]>
Signed-off-by: Andrea Cervesato <[email protected]>

Author: acerv

libkirk/data.py

@@ -37,6 +37,16 @@ def name(self):
         """
         return self._name
 
+    @name.setter
+    def name(self, value: str):
+        """
+        Set the suite name.
+        """
+        if not value:
+            raise ValueError("empty suite name")
+
+        self._name = value
+
     @property
     def tests(self):
         """

libkirk/main.py

@@ -130,6 +130,25 @@ def _env_config(value: str) -> dict:
     return config
 
 
+def _iterate_config(value: str) -> int:
+    """
+    Return the iterate value.
+    """
+    if not value:
+        return 1
+
+    ret = 1
+    try:
+        ret = int(value)
+    except TypeError as err:
+        raise argparse.ArgumentTypeError("Invalid number") from err
+
+    if ret <= 1:
+        return 1
+
+    return ret
+
+
 def _discover_sut(path: str) -> None:
     """
     Discover new SUT implementations.
@@ -315,6 +334,7 @@ async def session_run() -> None:
                 pattern=run_pattern,
                 report_path=args.json_report,
                 restore=restore_dir,
+                suite_iterate=args.suite_iterate,
             )
         except asyncio.CancelledError:
             await session.stop()
@@ -396,12 +416,12 @@ def run(cmd_args: list = None) -> None:
         help="List of key=value environment values separated by ':'")
     parser.add_argument(
         "--skip-tests",
-        "-i",
+        "-k",
         type=str,
         help="Skip specific tests")
     parser.add_argument(
         "--skip-file",
-        "-I",
+        "-K",
         type=str,
         help="Skip specific tests using a skip file (newline separated item)")
     parser.add_argument(
@@ -416,6 +436,12 @@ def run(cmd_args: list = None) -> None:
         type=int,
         default=3600,
         help="Timeout before stopping a single execution")
+    parser.add_argument(
+        "--suite-iterate",
+        "-I",
+        type=_iterate_config,
+        default=1,
+        help="Number of times to repeat testing suites")
 
     # tests execution arguments
     parser.add_argument(

libkirk/session.py

@@ -7,6 +7,7 @@
 """
 import os
 import re
+import copy
 import logging
 import asyncio
 import libkirk
@@ -208,6 +209,23 @@ async def _get_suites_objects(self, names: list) -> list:
 
         return suites_obj
 
+    @staticmethod
+    def _apply_iterate(suites_obj: list, suite_iterate: int) -> list:
+        """
+        Return testing suites after applying iterate parameters.
+        """
+        if suite_iterate <= 1:
+            return suites_obj
+
+        suites_list = []
+        for suite in suites_obj:
+            for i in range(0, suite_iterate):
+                obj = copy.deepcopy(suite)
+                obj.name = f"{suite.name}[{i}]"
+                suites_list.append(obj)
+
+        return suites_list
+
     async def _read_suites(
             self,
             suites: list,
@@ -319,15 +337,7 @@ async def stop(self) -> None:
             await libkirk.events.fire("session_stopped")
             self._stop = False
 
-    # consider changing the arguments handling if new ones must be added
-    # pylint: disable=too-many-positional-arguments
-    async def run(
-            self,
-            command: str = None,
-            suites: list = None,
-            pattern: str = None,
-            report_path: str = None,
-            restore: str = None) -> None:
+    async def run(self, **kwargs: dict) -> None:
         """
         Run a new session and store results inside a JSON file.
         :param command: single command to run before suites
@@ -340,7 +350,16 @@ async def run(
         :type report_path: str
         :param restore: temporary directory generated by a previous session
         :type restore: str
+        :param suite_iterate: execute all suites multiple times
+        :type suite_iterate: int
         """
+        command = kwargs.get("command", None)
+        suites = kwargs.get("suites", None)
+        pattern = kwargs.get("pattern", None)
+        report_path = kwargs.get("report_path", None)
+        restore = kwargs.get("restore", None)
+        suite_iterate = kwargs.get("suite_iterate", 1)
+
         async with self._run_lock:
             await libkirk.events.fire("session_started", self._tmpdir.abspath)
 
@@ -358,6 +377,10 @@ async def run(
                 if suites:
                     suites_obj = await self._read_suites(
                         suites, pattern, restore)
+
+                    suites_obj = self._apply_iterate(
+                        suites_obj, suite_iterate)
+
                     await self._scheduler.schedule(suites_obj)
             except KirkException as err:
                 if not self._stop:

libkirk/tests/test_main.py

@@ -67,7 +67,7 @@ def test_run_command(self, tmpdir):
 
         assert excinfo.value.code == libkirk.main.RC_OK
 
-    @pytest.mark.skipif(sys.version_info < (3,8), reason="requires python3.8+")
+    @pytest.mark.skipif(sys.version_info < (3, 8), reason="requires python3.8+")
     def test_run_command_timeout(self, tmpdir):
         """
         Test --run-command option with timeout.
@@ -362,3 +362,22 @@ def test_env(self, tmpdir):
 
         report_d = self.read_report(temp, 1)
         assert report_d["results"][0]["test"]["log"] == "ciao"
+
+    def test_suite_iterate(self, tmpdir):
+        """
+        Test --suite-iterate option.
+        """
+        temp = tmpdir.mkdir("temp")
+        cmd_args = [
+            "--tmp-dir", str(temp),
+            "--framework", "dummy",
+            "--run-suite", "suite01",
+            "--suite-iterate", "4",
+        ]
+
+        with pytest.raises(SystemExit) as excinfo:
+            libkirk.main.run(cmd_args=cmd_args)
+
+        assert excinfo.value.code == libkirk.main.RC_OK
+
+        self.read_report(temp, 8)

libkirk/tests/test_session.py

@@ -125,3 +125,25 @@ async def test_run_skip_tests(self, tmpdir, sut, dummy_framework):
         with open(report, "r") as report_file:
             report_data = json.loads(report_file.read())
             assert len(report_data["results"]) == 0
+
+    @pytest.mark.parametrize(
+        "iterate,expect",
+        [
+            (0, 4),
+            (1, 4),
+            (3, 12),
+        ]
+    )
+    async def test_run_suite_iterate(self, tmpdir, session, iterate, expect):
+        """
+        Test run method when executing a testing suite multiple times.
+        """
+        report = str(tmpdir / "report.json")
+        await session.run(
+            suites=["suite01", "suite02"],
+            suite_iterate=iterate,
+            report_path=report)
+
+        with open(report, "r", encoding="utf-8") as report_file:
+            report_data = json.loads(report_file.read())
+            assert len(report_data["results"]) == expect