Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 22 additions & 3 deletions Documentation/multi-session.rst
Original file line number Diff line number Diff line change
Expand Up @@ -101,8 +101,9 @@ fails the entire run is aborted immediately.
Phase 2: Test
-------------

After all builds succeed, test sessions are executed. Each session gets its
own sub-directory under a single timestamped result directory
After all builds succeed, a single timestamped result directory is created
(e.g. ``result/2026-04-14_18-30-00/``). Each session writes its results
into a sub-directory named after the session
(``result/<timestamp>/<session-name>/``).

In **sequential mode** (default) sessions run in manifest order. With
Expand All @@ -121,13 +122,31 @@ Phase 3: Report
---------------

Individual session JUnit XML reports are merged into a single
``report.xml`` at the master result directory. Testsuite names and
``report.xml`` in the shared result directory. Testsuite names and
testcase classnames are prefixed with the session name
(``<session>::<original>``), so results from different sessions never
collide.

A unified HTML summary is generated from the merged report.

The final directory layout looks like this::

result/<timestamp>/
report.xml # merged JUnit XML
report/
result_summary.txt # aggregated summary
result_summary.html
session-a/
report.xml # session-a JUnit XML
report.html
session.config.txt
...
session-b/
report.xml
report.html
session.config.txt
...

Resource Tags
=============

Expand Down
18 changes: 12 additions & 6 deletions src/ntfc/multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -263,10 +263,15 @@ def run(self) -> int:
if built_configs is None:
return 1

# Create shared session directory for all sessions
log_manager = LogManager(self._logcfg)
log_manager.cleanup()
self._session_dir = log_manager.new_session_dir()

# Phase 2: Run all test sessions
results = self._phase_test(built_configs)

# Phase 3: Merge reports
# Phase 3: Merge reports into the shared session directory
self._phase_report(results)

# Print final summary
Expand Down Expand Up @@ -519,7 +524,11 @@ def _run_session(
pt = MyPytest(conf, exitonfail, self._verbose, modules=modules)

logger.info(f"[Multi] Running session '{session.name}'")
result: Dict[str, Any] = {"logcfg": self._logcfg}
session_result_dir = os.path.join(self._session_dir, session.name)
result: Dict[str, Any] = {
"logcfg": self._logcfg,
"result_dir": session_result_dir,
}
exit_code = pt.runner(session.testpath, result)

# read result_dir from the instance, not global pytest module
Expand Down Expand Up @@ -621,11 +630,8 @@ def _phase_report(self, results: List["SessionResult"]) -> None:
"""
logger.info("[Multi] Phase 3: Merging reports")

log_manager = LogManager(self._logcfg)
merge_dir = log_manager.new_session_dir()

reporter = Reporter()
self._merge_session_reports(merge_dir, results, reporter)
self._merge_session_reports(self._session_dir, results, reporter)

@staticmethod
def _copy_testcase(
Expand Down
13 changes: 9 additions & 4 deletions src/ntfc/pytest/mypytest.py
Original file line number Diff line number Diff line change
Expand Up @@ -338,10 +338,15 @@ def runner(
opt.append(testpath)

if not nologs: # pragma: no cover
# create result directory via LogManager
log_manager = LogManager(result.get("logcfg"))
log_manager.cleanup()
pytest.result_dir = log_manager.new_session_dir()
if "result_dir" in result:
# Use pre-created result directory (multi-session mode)
pytest.result_dir = result["result_dir"]
os.makedirs(pytest.result_dir, exist_ok=True)
else:
# create result directory via LogManager
log_manager = LogManager(result.get("logcfg"))
log_manager.cleanup()
pytest.result_dir = log_manager.new_session_dir()
self.result_dir = pytest.result_dir
self._write_session_config(pytest.result_dir)

Expand Down
20 changes: 16 additions & 4 deletions tests/test_multi.py
Original file line number Diff line number Diff line change
Expand Up @@ -394,10 +394,14 @@ def test_run_all_pass(tmp_path):
)
runner = _make_runner(mc)

mock_log_mgr = MagicMock()
mock_log_mgr.new_session_dir.return_value = tmp

with (
patch.object(runner, "_phase_build") as mock_build,
patch.object(runner, "_phase_test") as mock_test,
patch.object(runner, "_phase_report"),
patch("ntfc.multi.LogManager", return_value=mock_log_mgr),
):
mock_build.return_value = {"s1": {}, "s2": {}}
mock_test.return_value = [
Expand All @@ -417,10 +421,14 @@ def test_run_session_fails(tmp_path):
)
runner = _make_runner(mc)

mock_log_mgr = MagicMock()
mock_log_mgr.new_session_dir.return_value = tmp

with (
patch.object(runner, "_phase_build") as mock_build,
patch.object(runner, "_phase_test") as mock_test,
patch.object(runner, "_phase_report"),
patch("ntfc.multi.LogManager", return_value=mock_log_mgr),
):
mock_build.return_value = {"s1": {}}
mock_test.return_value = [
Expand Down Expand Up @@ -751,6 +759,7 @@ def test_run_session_calls_mypytest(tmp_path):
],
)
runner = MultiSessionRunner(mc, rebuild=False)
runner._session_dir = tmp

mock_pt = MagicMock()
mock_pt.runner.return_value = 0
Expand All @@ -763,6 +772,11 @@ def test_run_session_calls_mypytest(tmp_path):
assert result.result_dir == "/tmp/fake_result"
mock_pt.runner.assert_called_once()

# Verify result_dir was passed to runner
call_args = mock_pt.runner.call_args
result_dict = call_args[0][1]
assert result_dict["result_dir"] == os.path.join(tmp, "s1")


def test_run_session_sets_fail_event(tmp_path):
tmp = str(tmp_path)
Expand All @@ -773,6 +787,7 @@ def test_run_session_sets_fail_event(tmp_path):
sessions=[SessionConfig(name="s1", confpath=confpath, testpath=tmp)],
)
runner = MultiSessionRunner(mc, rebuild=False)
runner._session_dir = tmp

fail_event = threading.Event()

Expand Down Expand Up @@ -847,14 +862,11 @@ def test_phase_report(tmp_path):
sessions=[SessionConfig(name="s1", confpath=confpath, testpath=tmp)],
)
runner = MultiSessionRunner(mc, rebuild=False)
runner._session_dir = tmp

results = [SessionResult("s1", 0, tmp)]

mock_log_mgr = MagicMock()
mock_log_mgr.new_session_dir.return_value = tmp

with (
patch("ntfc.multi.LogManager", return_value=mock_log_mgr),
patch("ntfc.multi.Reporter") as mock_rep_cls,
patch.object(
MultiSessionRunner, "_merge_session_reports"
Expand Down