From 5a928086c6d3aedda5f5de696f265c24db05f2e7 Mon Sep 17 00:00:00 2001 From: raiden00pl Date: Tue, 14 Apr 2026 20:28:34 +0200 Subject: [PATCH] fix multi-session results placed in separate timestamp directories All sessions from a manifest run now share a single timestamped result directory instead of each session creating its own The final directory layout looks like this: result// report.xml report/ result_summary.txt result_summary.html session-a/ report.xml report.html session.config.txt ... session-b/ report.xml report.html session.config.txt --- Documentation/multi-session.rst | 25 ++++++++++++++++++++++--- src/ntfc/multi.py | 18 ++++++++++++------ src/ntfc/pytest/mypytest.py | 13 +++++++++---- tests/test_multi.py | 20 ++++++++++++++++---- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/Documentation/multi-session.rst b/Documentation/multi-session.rst index 5278880..f12a485 100644 --- a/Documentation/multi-session.rst +++ b/Documentation/multi-session.rst @@ -101,8 +101,9 @@ fails the entire run is aborted immediately. Phase 2: Test ------------- -After all builds succeed, test sessions are executed. Each session gets its -own sub-directory under a single timestamped result directory +After all builds succeed, a single timestamped result directory is created +(e.g. ``result/2026-04-14_18-30-00/``). Each session writes its results +into a sub-directory named after the session (``result///``). In **sequential mode** (default) sessions run in manifest order. With @@ -121,13 +122,31 @@ Phase 3: Report --------------- Individual session JUnit XML reports are merged into a single -``report.xml`` at the master result directory. Testsuite names and +``report.xml`` in the shared result directory. Testsuite names and testcase classnames are prefixed with the session name (``::``), so results from different sessions never collide. A unified HTML summary is generated from the merged report. +The final directory layout looks like this:: + + result// + report.xml # merged JUnit XML + report/ + result_summary.txt # aggregated summary + result_summary.html + session-a/ + report.xml # session-a JUnit XML + report.html + session.config.txt + ... + session-b/ + report.xml + report.html + session.config.txt + ... + Resource Tags ============= diff --git a/src/ntfc/multi.py b/src/ntfc/multi.py index 00f86f8..69f68ad 100644 --- a/src/ntfc/multi.py +++ b/src/ntfc/multi.py @@ -263,10 +263,15 @@ def run(self) -> int: if built_configs is None: return 1 + # Create shared session directory for all sessions + log_manager = LogManager(self._logcfg) + log_manager.cleanup() + self._session_dir = log_manager.new_session_dir() + # Phase 2: Run all test sessions results = self._phase_test(built_configs) - # Phase 3: Merge reports + # Phase 3: Merge reports into the shared session directory self._phase_report(results) # Print final summary @@ -519,7 +524,11 @@ def _run_session( pt = MyPytest(conf, exitonfail, self._verbose, modules=modules) logger.info(f"[Multi] Running session '{session.name}'") - result: Dict[str, Any] = {"logcfg": self._logcfg} + session_result_dir = os.path.join(self._session_dir, session.name) + result: Dict[str, Any] = { + "logcfg": self._logcfg, + "result_dir": session_result_dir, + } exit_code = pt.runner(session.testpath, result) # read result_dir from the instance, not global pytest module @@ -621,11 +630,8 @@ def _phase_report(self, results: List["SessionResult"]) -> None: """ logger.info("[Multi] Phase 3: Merging reports") - log_manager = LogManager(self._logcfg) - merge_dir = log_manager.new_session_dir() - reporter = Reporter() - self._merge_session_reports(merge_dir, results, reporter) + self._merge_session_reports(self._session_dir, results, reporter) @staticmethod def _copy_testcase( diff --git a/src/ntfc/pytest/mypytest.py b/src/ntfc/pytest/mypytest.py index 63839cb..6910d95 100644 --- a/src/ntfc/pytest/mypytest.py +++ b/src/ntfc/pytest/mypytest.py @@ -338,10 +338,15 @@ def runner( opt.append(testpath) if not nologs: # pragma: no cover - # create result directory via LogManager - log_manager = LogManager(result.get("logcfg")) - log_manager.cleanup() - pytest.result_dir = log_manager.new_session_dir() + if "result_dir" in result: + # Use pre-created result directory (multi-session mode) + pytest.result_dir = result["result_dir"] + os.makedirs(pytest.result_dir, exist_ok=True) + else: + # create result directory via LogManager + log_manager = LogManager(result.get("logcfg")) + log_manager.cleanup() + pytest.result_dir = log_manager.new_session_dir() self.result_dir = pytest.result_dir self._write_session_config(pytest.result_dir) diff --git a/tests/test_multi.py b/tests/test_multi.py index 48479cc..e62b02b 100644 --- a/tests/test_multi.py +++ b/tests/test_multi.py @@ -394,10 +394,14 @@ def test_run_all_pass(tmp_path): ) runner = _make_runner(mc) + mock_log_mgr = MagicMock() + mock_log_mgr.new_session_dir.return_value = tmp + with ( patch.object(runner, "_phase_build") as mock_build, patch.object(runner, "_phase_test") as mock_test, patch.object(runner, "_phase_report"), + patch("ntfc.multi.LogManager", return_value=mock_log_mgr), ): mock_build.return_value = {"s1": {}, "s2": {}} mock_test.return_value = [ @@ -417,10 +421,14 @@ def test_run_session_fails(tmp_path): ) runner = _make_runner(mc) + mock_log_mgr = MagicMock() + mock_log_mgr.new_session_dir.return_value = tmp + with ( patch.object(runner, "_phase_build") as mock_build, patch.object(runner, "_phase_test") as mock_test, patch.object(runner, "_phase_report"), + patch("ntfc.multi.LogManager", return_value=mock_log_mgr), ): mock_build.return_value = {"s1": {}} mock_test.return_value = [ @@ -751,6 +759,7 @@ def test_run_session_calls_mypytest(tmp_path): ], ) runner = MultiSessionRunner(mc, rebuild=False) + runner._session_dir = tmp mock_pt = MagicMock() mock_pt.runner.return_value = 0 @@ -763,6 +772,11 @@ def test_run_session_calls_mypytest(tmp_path): assert result.result_dir == "/tmp/fake_result" mock_pt.runner.assert_called_once() + # Verify result_dir was passed to runner + call_args = mock_pt.runner.call_args + result_dict = call_args[0][1] + assert result_dict["result_dir"] == os.path.join(tmp, "s1") + def test_run_session_sets_fail_event(tmp_path): tmp = str(tmp_path) @@ -773,6 +787,7 @@ def test_run_session_sets_fail_event(tmp_path): sessions=[SessionConfig(name="s1", confpath=confpath, testpath=tmp)], ) runner = MultiSessionRunner(mc, rebuild=False) + runner._session_dir = tmp fail_event = threading.Event() @@ -847,14 +862,11 @@ def test_phase_report(tmp_path): sessions=[SessionConfig(name="s1", confpath=confpath, testpath=tmp)], ) runner = MultiSessionRunner(mc, rebuild=False) + runner._session_dir = tmp results = [SessionResult("s1", 0, tmp)] - mock_log_mgr = MagicMock() - mock_log_mgr.new_session_dir.return_value = tmp - with ( - patch("ntfc.multi.LogManager", return_value=mock_log_mgr), patch("ntfc.multi.Reporter") as mock_rep_cls, patch.object( MultiSessionRunner, "_merge_session_reports"