From 476750cda2a7f588e1e5476448945f10789afff4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 19 Jun 2026 16:53:58 +0200 Subject: [PATCH 1/4] Unify MTP terminal reporter to handle 1..N assemblies The terminal test reporter was single-assembly (in-process MTP host). The dotnet test orchestrator runs N child assemblies, so the SDK had to hard-fork the reporter. This converts the reporter core to a multi-assembly model so a single shared source can render both cases: - Replace the single TestProgressState + _assembly/_targetFramework/_architecture fields with a ConcurrentDictionary keyed by a caller-provided execution id (in-process passes one fixed id). - Route TestCompleted/TestInProgress/TestDiscovered/AssemblyRunStarted/ AssemblyRunCompleted/ArtifactAdded per execution id. - Aggregate run/discovery summaries across all assemblies; keep the N=1 (in-process) output byte-identical via an assemblies.Count == 1 branch that still appends the per-assembly link to the verdict line. - Expand TestRunArtifact and ArtifactAdded with assembly/tfm/arch/executionId. - Add TotalTests aggregate and surface isHelp/isRetry on TestExecutionStarted. - Adapt the in-process caller (TerminalOutputDevice) to the new API with a single fixed InProcessExecutionId. The SDK-orchestrator-only surface (handshake-failure recap, instanceId-based retry counting, build errors, exit-code-in-summary) is intentionally deferred to the SDK plug-in PR, where it can be driven and verified against real multi-assembly runs. Verified: platform builds clean on net8.0/net9.0/netstandard2.0; full Microsoft.Testing.Platform.UnitTests suite green (1197, 0 failed), including a new N>1 test; the standalone TerminalReporterContract consumer still compiles in isolation. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TerminalTestReporter.Formatting.cs | 16 +- .../TerminalTestReporter.Lifecycle.cs | 58 ++--- .../TerminalTestReporter.Messaging.cs | 4 +- .../Terminal/TerminalTestReporter.Summary.cs | 53 ++-- .../TerminalTestReporter.TestCompletion.cs | 7 +- .../Terminal/TerminalTestReporter.cs | 29 +-- .../Terminal/TestProgressState.cs | 4 + .../OutputDevice/Terminal/TestRunArtifact.cs | 2 +- .../TerminalOutputDevice.DataConsumption.cs | 23 +- .../TerminalOutputDevice.Initialization.cs | 9 +- .../TerminalOutputDevice.SessionLifecycle.cs | 8 +- .../Terminal/TerminalTestReporterTests.cs | 234 +++++++++++------- 12 files changed, 271 insertions(+), 176 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Formatting.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Formatting.cs index a38ac24d92..78e4bad8a3 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Formatting.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Formatting.cs @@ -123,27 +123,27 @@ private static void FormatStandardAndErrorOutput(ITerminal terminal, string? sta terminal.ResetColor(); } - private void AppendAssemblyLinkTargetFrameworkAndArchitecture(ITerminal terminal) + private static void AppendAssemblyLinkTargetFrameworkAndArchitecture(ITerminal terminal, TestProgressState assemblyRun) { - terminal.AppendLink(_assembly, lineNumber: null); - if (_targetFramework == null && _architecture == null) + terminal.AppendLink(assemblyRun.Assembly, lineNumber: null); + if (assemblyRun.TargetFramework == null && assemblyRun.Architecture == null) { return; } terminal.Append(" ("); - if (_targetFramework != null) + if (assemblyRun.TargetFramework != null) { - terminal.Append(_targetFramework); - if (_architecture != null) + terminal.Append(assemblyRun.TargetFramework); + if (assemblyRun.Architecture != null) { terminal.Append('|'); } } - if (_architecture != null) + if (assemblyRun.Architecture != null) { - terminal.Append(_architecture); + terminal.Append(assemblyRun.Architecture); } terminal.Append(')'); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs index 7a40f70122..9340f70f3a 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs @@ -8,57 +8,57 @@ namespace Microsoft.Testing.Platform.OutputDevice.Terminal; [UnsupportedOSPlatform("browser")] internal sealed partial class TerminalTestReporter { - public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, bool isDiscovery) + private bool _isHelp; + private bool _isRetry; + + public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, bool isDiscovery, bool isHelp, bool isRetry) { _isDiscovery = isDiscovery; + _isHelp = isHelp; + _isRetry = isRetry; _testExecutionStartTime = testStartTime; _terminalWithProgress.StartShowingProgress(workerCount); } - public void AssemblyRunStarted() - => GetOrAddAssemblyRun(); - - private TestProgressState GetOrAddAssemblyRun() - { - if (_testProgressState is not null) - { - return _testProgressState; - } + public void AssemblyRunStarted(string assembly, string? targetFramework, string? architecture, string executionId, string instanceId) + => GetOrAddAssemblyRun(assembly, targetFramework, architecture, executionId); - lock (_lock) + private TestProgressState GetOrAddAssemblyRun(string assembly, string? targetFramework, string? architecture, string executionId) + => _assemblies.GetOrAdd(executionId, _ => { - if (_testProgressState is not null) + lock (_lock) { - return _testProgressState; + IStopwatch sw = CreateStopwatch(); + var assemblyRun = new TestProgressState(Interlocked.Increment(ref _counter), assembly, targetFramework, architecture, sw, _isDiscovery); + int slotIndex = _terminalWithProgress.AddWorker(assemblyRun); + assemblyRun.SlotIndex = slotIndex; + return assemblyRun; } + }); - IStopwatch sw = CreateStopwatch(); - var assemblyRun = new TestProgressState(Interlocked.Increment(ref _counter), _assembly, _targetFramework, _architecture, sw, _isDiscovery); - int slotIndex = _terminalWithProgress.AddWorker(assemblyRun); - assemblyRun.SlotIndex = slotIndex; - _testProgressState = assemblyRun; - return assemblyRun; + internal void AssemblyRunCompleted(string executionId) + { + if (!_assemblies.TryGetValue(executionId, out TestProgressState? assemblyRun)) + { + return; } - } - internal void AssemblyRunCompleted() - { - TestProgressState assemblyRun = GetOrAddAssemblyRun(); assemblyRun.Stopwatch.Stop(); - _terminalWithProgress.RemoveWorker(assemblyRun.SlotIndex); } - public void TestExecutionCompleted(DateTimeOffset endTime) + public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode) { _testExecutionEndTime = endTime; _terminalWithProgress.StopShowingProgress(); - _terminalWithProgress.WriteToTerminal(_isDiscovery ? AppendTestDiscoverySummary : AppendTestRunSummary); + if (!_isHelp) + { + _terminalWithProgress.WriteToTerminal(_isDiscovery ? AppendTestDiscoverySummary : AppendTestRunSummary); + } - // This is relevant for HotReload scenarios. We want the next test sessions to start - // on a new TestProgressState - _testProgressState = null; + // This is relevant for HotReload scenarios. We want the next test sessions to start fresh. + _assemblies.Clear(); _testExecutionStartTime = null; _testExecutionEndTime = null; diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Messaging.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Messaging.cs index 1174ef975b..64461e536f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Messaging.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Messaging.cs @@ -44,15 +44,15 @@ private void WriteMessage(string text, TerminalColor? color = null, int? padding }); public void TestInProgress( + string executionId, string testNodeUid, string displayName) { - if (_testProgressState is null) + if (!_assemblies.TryGetValue(executionId, out TestProgressState? asm)) { throw ApplicationStateGuard.Unreachable(); } - TestProgressState asm = _testProgressState; if (_options.ShowActiveTests) { asm.TestNodeResultsState ??= new(Interlocked.Increment(ref _counter)); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs index d0ede47c0c..74d8a70fb3 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Summary.cs @@ -40,9 +40,12 @@ private void AppendTestRunSummary(ITerminal terminal) terminal.AppendLine(); - int totalTests = _testProgressState?.TotalTests ?? 0; - int totalFailedTests = _testProgressState?.FailedTests ?? 0; - int totalSkippedTests = _testProgressState?.SkippedTests ?? 0; + List assemblies = [.. _assemblies.Values.OrderBy(static a => a.Id)]; + + int totalTests = assemblies.Sum(static a => a.TotalTests); + int totalFailedTests = assemblies.Sum(static a => a.FailedTests); + int totalSkippedTests = assemblies.Sum(static a => a.SkippedTests); + int totalPassedTests = assemblies.Sum(static a => a.PassedTests); // DESIGN: `allTestsWereSkipped` is intentionally treated as a failed run. Skipped tests don't count as // "ran", so an all-skipped (or zero-test) run is reported in red as "Zero tests ran". This is the strict @@ -60,17 +63,23 @@ private void AppendTestRunSummary(ITerminal terminal) terminal.Append(' '); terminal.Append(TestRunSummaryHelper.GetVerdictText(totalTests, totalFailedTests, totalSkippedTests, WasCancelled, _options.MinimumExpectedTests)); - terminal.SetColor(TerminalColor.DarkGray); - terminal.Append(" - "); - terminal.ResetColor(); - AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal); + // For a single assembly (the in-process host) the verdict is followed by the assembly link, exactly as + // before. For multiple assemblies (the dotnet test orchestrator) the per-assembly identity is rendered in + // the progress area, so we keep the run-level verdict line link-free. + if (assemblies.Count == 1) + { + terminal.SetColor(TerminalColor.DarkGray); + terminal.Append(" - "); + terminal.ResetColor(); + AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assemblies[0]); + } terminal.AppendLine(); - int total = _testProgressState?.TotalTests ?? 0; - int failed = _testProgressState?.FailedTests ?? 0; - int passed = _testProgressState?.PassedTests ?? 0; - int skipped = _testProgressState?.SkippedTests ?? 0; + int total = totalTests; + int failed = totalFailedTests; + int passed = totalPassedTests; + int skipped = totalSkippedTests; TimeSpan runDuration = _testExecutionStartTime != null && _testExecutionEndTime != null ? (_testExecutionEndTime - _testExecutionStartTime).Value : TimeSpan.Zero; bool colorizeFailed = failed > 0; @@ -126,14 +135,13 @@ private void AppendTestRunSummary(ITerminal terminal) terminal.AppendLine(); } - internal void TestDiscovered(string displayName) + internal void TestDiscovered(string executionId, string displayName) { - if (_testProgressState is null) + if (!_assemblies.TryGetValue(executionId, out TestProgressState? asm)) { throw ApplicationStateGuard.Unreachable(); } - TestProgressState asm = _testProgressState; asm.DiscoveredTests++; if (_isDiscovery) @@ -150,13 +158,13 @@ internal void TestDiscovered(string displayName) public void AppendTestDiscoverySummary(ITerminal terminal) { - TestProgressState? assembly = _testProgressState; + List assemblies = [.. _assemblies.Values.OrderBy(static a => a.Id)]; terminal.AppendLine(); - int totalTests = assembly?.TotalTests ?? 0; + int totalTests = assemblies.Sum(static a => a.TotalTests); bool runFailed = WasCancelled || totalTests < 1; - if (assembly is not null) + foreach (TestProgressState assembly in assemblies) { foreach (string displayName in assembly.DiscoveredTestDisplayNames) { @@ -170,10 +178,13 @@ public void AppendTestDiscoverySummary(ITerminal terminal) terminal.SetColor(runFailed ? TerminalColor.DarkRed : TerminalColor.DarkGreen); terminal.Append(string.Format(CultureInfo.CurrentCulture, TerminalResources.TestDiscoverySummarySingular, totalTests)); - terminal.SetColor(TerminalColor.DarkGray); - terminal.Append(" - "); - terminal.ResetColor(); - AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal); + if (assemblies.Count == 1) + { + terminal.SetColor(TerminalColor.DarkGray); + terminal.Append(" - "); + terminal.ResetColor(); + AppendAssemblyLinkTargetFrameworkAndArchitecture(terminal, assemblies[0]); + } terminal.ResetColor(); terminal.AppendLine(); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.TestCompletion.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.TestCompletion.cs index 8ac8e4d883..d6bf228ab4 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.TestCompletion.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.TestCompletion.cs @@ -9,6 +9,7 @@ namespace Microsoft.Testing.Platform.OutputDevice.Terminal; internal sealed partial class TerminalTestReporter { internal void TestCompleted( + string executionId, string testNodeUid, string displayName, TestOutcome outcome, @@ -23,6 +24,7 @@ internal void TestCompleted( { FlatException[] flatExceptions = ExceptionFlattener.Flatten(errorMessage, exception); TestCompleted( + executionId, testNodeUid, displayName, outcome, @@ -36,6 +38,7 @@ internal void TestCompleted( } private void TestCompleted( + string executionId, string testNodeUid, string displayName, TestOutcome outcome, @@ -47,13 +50,11 @@ private void TestCompleted( string? standardOutput, string? errorOutput) { - if (_testProgressState is null) + if (!_assemblies.TryGetValue(executionId, out TestProgressState? asm)) { throw ApplicationStateGuard.Unreachable(); } - TestProgressState asm = _testProgressState; - if (_options.ShowActiveTests) { asm.TestNodeResultsState?.RemoveRunningTestNode(testNodeUid); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs index d41bed0f2c..ef190de684 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs @@ -36,15 +36,6 @@ internal event EventHandler OnProgressStopUpdate remove => _terminalWithProgress.OnProgressStopUpdate -= value; } - private readonly string _assembly; - private readonly string? _targetFramework; - private readonly string? _architecture; - - /// - /// Returns whether external cancellation (e.g. Ctrl+C) has been requested. Injected as a delegate - /// rather than taking a concrete platform service so the reporter can be shared outside of - /// Microsoft.Testing.Platform (e.g. by the dotnet test multi-assembly orchestrator). - /// private readonly Func _isCancellationRequested; private readonly List _artifacts = []; @@ -61,13 +52,21 @@ internal event EventHandler OnProgressStopUpdate private readonly uint? _originalConsoleMode; - private TestProgressState? _testProgressState; + /// + /// Per-assembly run state, keyed by the caller-provided execution id. The in-process Microsoft.Testing.Platform + /// host registers a single assembly; the dotnet test orchestrator registers one entry per child test + /// assembly. Progress rendering already supports N workers (slots), so this only generalizes the bookkeeping. + /// + private readonly ConcurrentDictionary _assemblies = new(); private bool _isDiscovery; private DateTimeOffset? _testExecutionStartTime; private DateTimeOffset? _testExecutionEndTime; + /// Total number of tests across all registered assemblies. + public int TotalTests => _assemblies.Values.Sum(a => a.TotalTests); + private bool WasCancelled { get => field || _isCancellationRequested(); @@ -82,16 +81,10 @@ private bool WasCancelled /// Initializes a new instance of the class with custom terminal and manual refresh for testing. /// public TerminalTestReporter( - string assembly, - string? targetFramework, - string? architecture, IConsole console, Func isCancellationRequested, TerminalTestReporterOptions options) { - _assembly = assembly; - _targetFramework = targetFramework; - _architecture = architecture; _isCancellationRequested = isCancellationRequested; _options = options; @@ -168,8 +161,8 @@ public void Dispose() NativeMethods.RestoreConsoleMode(_originalConsoleMode); } - public void ArtifactAdded(bool outOfProcess, string? testName, string path) - => _artifacts.Add(new TestRunArtifact(outOfProcess, testName, path)); + public void ArtifactAdded(bool outOfProcess, string? assembly, string? targetFramework, string? architecture, string? executionId, string? testName, string path) + => _artifacts.Add(new TestRunArtifact(outOfProcess, assembly, targetFramework, architecture, executionId, testName, path)); /// /// Let the user know that cancellation was triggered. diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs index 1837ba34b3..4b80f16aaa 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs @@ -15,10 +15,14 @@ public TestProgressState(long id, string assembly, string? targetFramework, stri TargetFramework = targetFramework; Architecture = architecture; Stopwatch = stopwatch; + Assembly = assembly; AssemblyName = Path.GetFileName(assembly); IsDiscovery = isDiscovery; } + /// The assembly path or display name as provided by the caller (used for the summary link). + public string Assembly { get; } + public string AssemblyName { get; } public string? TargetFramework { get; } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestRunArtifact.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestRunArtifact.cs index 6d9134c5c6..62538d2cad 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestRunArtifact.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestRunArtifact.cs @@ -9,4 +9,4 @@ namespace Microsoft.Testing.Platform.OutputDevice.Terminal; /// An artifact / attachment that was reported during run. /// [Embedded] -internal sealed record TestRunArtifact(bool OutOfProcess, string? TestName, string Path); +internal sealed record TestRunArtifact(bool OutOfProcess, string? Assembly, string? TargetFramework, string? Architecture, string? ExecutionId, string? TestName, string Path); diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs index 1261e9bef2..28df0262de 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs @@ -141,7 +141,11 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case FileArtifactProperty fa: _terminalTestReporter.ArtifactAdded( outOfProcess: _processRole != TestProcessRole.TestHost, - testNodeStateChanged.TestNode.DisplayName, + assembly: _assemblyName, + targetFramework: _targetFramework, + architecture: _shortArchitecture, + executionId: InProcessExecutionId, + testName: testNodeStateChanged.TestNode.DisplayName, fa.FileInfo.FullName); break; } @@ -155,12 +159,14 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo { case InProgressTestNodeStateProperty: _terminalTestReporter.TestInProgress( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName); break; case ErrorTestNodeStateProperty errorState: _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, TestOutcome.Error, @@ -176,6 +182,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case FailedTestNodeStateProperty failedState: _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, TestOutcome.Fail, @@ -191,6 +198,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case TimeoutTestNodeStateProperty timeoutState: _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, TestOutcome.Timeout, @@ -208,6 +216,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case CancelledTestNodeStateProperty cancelledState: #pragma warning restore CS0618, MTP0001 // Type or member is obsolete _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, TestOutcome.Canceled, @@ -223,6 +232,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case PassedTestNodeStateProperty: _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, outcome: TestOutcome.Passed, @@ -238,6 +248,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case SkippedTestNodeStateProperty skippedState: _terminalTestReporter.TestCompleted( + InProcessExecutionId, testNodeStateChanged.TestNode.Uid.Value, testNodeStateChanged.TestNode.DisplayName, TestOutcome.Skipped, @@ -252,7 +263,7 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo break; case DiscoveredTestNodeStateProperty: - _terminalTestReporter.TestDiscovered(testNodeStateChanged.TestNode.DisplayName); + _terminalTestReporter.TestDiscovered(InProcessExecutionId, testNodeStateChanged.TestNode.DisplayName); break; } @@ -262,6 +273,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo { _terminalTestReporter.ArtifactAdded( outOfProcess: _processRole != TestProcessRole.TestHost, + assembly: _assemblyName, + targetFramework: _targetFramework, + architecture: _shortArchitecture, + executionId: InProcessExecutionId, testName: null, artifact.FileInfo.FullName); } @@ -271,6 +286,10 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo { _terminalTestReporter.ArtifactAdded( outOfProcess: _processRole != TestProcessRole.TestHost, + assembly: _assemblyName, + targetFramework: _targetFramework, + architecture: _shortArchitecture, + executionId: InProcessExecutionId, testName: null, artifact.FileInfo.FullName); } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.Initialization.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.Initialization.cs index d3705eec91..58a21c63f6 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.Initialization.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.Initialization.cs @@ -10,6 +10,13 @@ namespace Microsoft.Testing.Platform.OutputDevice; internal sealed partial class TerminalOutputDevice { + /// + /// Execution id used for the single assembly handled by an in-process Microsoft.Testing.Platform host. The + /// terminal reporter is now multi-assembly (one entry per dotnet test child assembly); in-process there + /// is exactly one, so a constant id is used. + /// + private const string InProcessExecutionId = "in-process"; + public async Task InitializeAsync() { if (_fileLoggerInformation is not null) @@ -151,7 +158,7 @@ await _policiesService.RegisterOnAbortCallbackAsync( : !_testHostControllerInfo.IsCurrentProcessTestHostController; // This is single exe run, don't show all the details of assemblies and their summaries. - _terminalTestReporter = new TerminalTestReporter(_assemblyName, _targetFramework, _shortArchitecture, _console, () => _testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested, new() + _terminalTestReporter = new TerminalTestReporter(_console, () => _testApplicationCancellationTokenSource.CancellationToken.IsCancellationRequested, new() { ShowPassedTests = showPassed, MinimumExpectedTests = PlatformCommandLineProvider.GetMinimumExpectedTests(_commandLineOptions), diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.SessionLifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.SessionLifecycle.cs index 3d87906e03..823d11e523 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.SessionLifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.SessionLifecycle.cs @@ -86,8 +86,8 @@ public async Task DisplayBeforeSessionStartAsync(CancellationToken cancellationT // Start test execution here, rather than in ShowBanner, because then we know // if we are a testHost controller or not, and if we should show progress bar. - _terminalTestReporter.TestExecutionStarted(_clock.UtcNow, workerCount: 1, isDiscovery: _isListTests); - _terminalTestReporter.AssemblyRunStarted(); + _terminalTestReporter.TestExecutionStarted(_clock.UtcNow, workerCount: 1, isDiscovery: _isListTests, isHelp: false, isRetry: false); + _terminalTestReporter.AssemblyRunStarted(_assemblyName, _targetFramework, _shortArchitecture, InProcessExecutionId, InProcessExecutionId); if (_logger is not null && _logger.IsEnabled(LogLevel.Trace)) { await _logger.LogTraceAsync("DisplayBeforeSessionStartAsync").ConfigureAwait(false); @@ -150,8 +150,8 @@ private async Task DisplayAfterSessionEndRunInternalAsync() { if (_processRole == TestProcessRole.TestHost) { - _terminalTestReporter.AssemblyRunCompleted(); - _terminalTestReporter.TestExecutionCompleted(_clock.UtcNow); + _terminalTestReporter.AssemblyRunCompleted(InProcessExecutionId); + _terminalTestReporter.TestExecutionCompleted(_clock.UtcNow, exitCode: null); } else { diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs index fde73719a8..4316ff455f 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs @@ -105,7 +105,7 @@ public void NonAnsiTerminal_OutputFormattingIsCorrect() string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, @@ -117,29 +117,29 @@ public void NonAnsiTerminal_OutputFormattingIsCorrect() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); string standardOutput = "Hello!"; string errorOutput = "Oh no!"; - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary - terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); - terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); - terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.ArtifactAdded(outOfProcess: true, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact1.txt"); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact2.txt"); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -201,7 +201,7 @@ public void SimpleAnsiTerminal_OutputFormattingIsCorrect() string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, @@ -213,29 +213,29 @@ public void SimpleAnsiTerminal_OutputFormattingIsCorrect() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); string standardOutput = "Hello!"; string errorOutput = "Oh no!"; - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary - terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); - terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); - terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.ArtifactAdded(outOfProcess: true, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact1.txt"); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact2.txt"); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -297,7 +297,7 @@ public void AnsiTerminal_OutputFormattingIsCorrect() string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, // Like if we autodetect that we are in ANSI capable terminal. @@ -308,31 +308,31 @@ public void AnsiTerminal_OutputFormattingIsCorrect() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; string folderLink = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work/" : "mnt/work/"; string folderLinkNoSlash = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:/work" : "mnt/work"; - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); string standardOutput = "Hello!"; string errorOutput = "Oh no!"; - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); // timed out + canceled + failed should all report as failed in summary - terminalReporter.TestCompleted(testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "TimedoutTest1", "TimedoutTest1", TestOutcome.Timeout, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "CanceledTest1", "CanceledTest1", TestOutcome.Canceled, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: "Tests failed", exception: new StackTraceException(@$" at FailingTest() in {folder}codefile.cs:line 10"), expected: "ABC", actual: "DEF", standardOutput, errorOutput); - terminalReporter.ArtifactAdded(outOfProcess: true, testName: null, @$"{folder}artifact1.txt"); - terminalReporter.ArtifactAdded(outOfProcess: false, testName: null, @$"{folder}artifact2.txt"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.ArtifactAdded(outOfProcess: true, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact1.txt"); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly: assembly, targetFramework: targetFramework, architecture: architecture, executionId: "0", testName: null, @$"{folder}artifact2.txt"); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -395,7 +395,7 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() var stringBuilderConsole = new StringBuilderConsole(); var stopwatchFactory = new StopwatchFactory(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, // Like if we autodetect that we are in ANSI capable terminal. @@ -417,30 +417,30 @@ public void AnsiTerminal_OutputProgressFrameIsCorrect() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); string standardOutput = "Hello!"; string errorOutput = "Oh no!"; // Note: Add 1ms to make the order of the progress frame deterministic. // Otherwise all tests that run for 1m31s could show in any order. - terminalReporter.TestInProgress(testNodeUid: "PassedTest1", displayName: "PassedTest1"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "PassedTest1", displayName: "PassedTest1"); stopwatchFactory.AddTime(TimeSpan.FromMilliseconds(1)); - terminalReporter.TestInProgress(testNodeUid: "SkippedTest1", displayName: "SkippedTest1"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "SkippedTest1", displayName: "SkippedTest1"); stopwatchFactory.AddTime(TimeSpan.FromMilliseconds(1)); - terminalReporter.TestInProgress(testNodeUid: "InProgressTest1", displayName: "InProgressTest1"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "InProgressTest1", displayName: "InProgressTest1"); stopwatchFactory.AddTime(TimeSpan.FromMinutes(1)); - terminalReporter.TestInProgress(testNodeUid: "InProgressTest2", displayName: "InProgressTest2"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "InProgressTest2", displayName: "InProgressTest2"); stopwatchFactory.AddTime(TimeSpan.FromSeconds(30)); - terminalReporter.TestInProgress(testNodeUid: "InProgressTest3", displayName: "InProgressTest3"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "InProgressTest3", displayName: "InProgressTest3"); stopwatchFactory.AddTime(TimeSpan.FromSeconds(1)); - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); - terminalReporter.TestCompleted(testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "SkippedTest1", "SkippedTest1", TestOutcome.Skipped, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput, errorOutput); string output = stringBuilderConsole.Output; @@ -538,7 +538,7 @@ public void NonAnsiTerminal_ShowOutputNone_DoesNotShowOutput() string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.NoAnsi, @@ -549,16 +549,16 @@ public void NonAnsiTerminal_ShowOutputNone_DoesNotShowOutput() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: "Hello!", errorOutput: "Oh no!"); - terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), + terminalReporter.TestCompleted("0", testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), informativeMessage: null, errorMessage: "Tests failed", exception: null, expected: null, actual: null, standardOutput: "Hello!", errorOutput: "Oh no!"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -576,7 +576,7 @@ public void NonAnsiTerminal_ShowOutputFailed_ShowsOutputOnlyForFailedTests() string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.NoAnsi, @@ -587,16 +587,16 @@ public void NonAnsiTerminal_ShowOutputFailed_ShowsOutputOnlyForFailedTests() DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); - terminalReporter.TestCompleted(testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + terminalReporter.TestCompleted("0", testNodeUid: "PassedTest1", "PassedTest1", TestOutcome.Passed, TimeSpan.FromSeconds(1), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: "passed-stdout", errorOutput: "passed-stderr"); - terminalReporter.TestCompleted(testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), + terminalReporter.TestCompleted("0", testNodeUid: "FailedTest1", "FailedTest1", TestOutcome.Fail, TimeSpan.FromSeconds(1), informativeMessage: null, errorMessage: "Tests failed", exception: null, expected: null, actual: null, standardOutput: "failed-stdout", errorOutput: "failed-stderr"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -829,7 +829,7 @@ public void TestDisplayNames_WithControlCharacters_AreNormalized(char controlCha string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.NoAnsi, @@ -838,17 +838,17 @@ public void TestDisplayNames_WithControlCharacters_AreNormalized(char controlCha DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); // Test display name with the specific control character string testDisplayName = $"Test{controlChar}Name"; - terminalReporter.TestCompleted(testNodeUid: "Test1", testDisplayName, TestOutcome.Passed, TimeSpan.FromSeconds(1), + terminalReporter.TestCompleted("0", testNodeUid: "Test1", testDisplayName, TestOutcome.Passed, TimeSpan.FromSeconds(1), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -909,7 +909,7 @@ public void TestDiscovery_WithControlCharacters_AreNormalized(char controlChar, string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.NoAnsi, @@ -918,16 +918,16 @@ public void TestDiscovery_WithControlCharacters_AreNormalized(char controlChar, DateTimeOffset startTime = DateTimeOffset.MinValue; DateTimeOffset endTime = DateTimeOffset.MaxValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: true); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: true, isHelp: false, isRetry: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); // Test discovery with the specific control character string testDisplayName = $"Test{controlChar}Name"; - terminalReporter.TestDiscovered(testDisplayName); + terminalReporter.TestDiscovered("0", testDisplayName); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -1110,8 +1110,10 @@ public void TerminalTestReporter_WhenInDiscoveryMode_ShouldIncrementDiscoveredTe { // Arrange string assembly = "test.dll"; + string targetFramework = "net8.0"; + string architecture = "x64"; var stringBuilderConsole = new StringBuilderConsole(); - var terminalReporter = new TerminalTestReporter(assembly, "net8.0", "x64", stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => false, AnsiMode = AnsiMode.NoAnsi, @@ -1122,12 +1124,12 @@ public void TerminalTestReporter_WhenInDiscoveryMode_ShouldIncrementDiscoveredTe DateTimeOffset endTime = DateTimeOffset.MaxValue; // Act - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: true); - terminalReporter.AssemblyRunStarted(); - terminalReporter.TestDiscovered("TestMethod1"); - terminalReporter.TestDiscovered("TestMethod2"); - terminalReporter.AssemblyRunCompleted(); - terminalReporter.TestExecutionCompleted(endTime); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: true, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); + terminalReporter.TestDiscovered("0", "TestMethod1"); + terminalReporter.TestDiscovered("0", "TestMethod2"); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); string output = stringBuilderConsole.Output; @@ -1135,6 +1137,64 @@ public void TerminalTestReporter_WhenInDiscoveryMode_ShouldIncrementDiscoveredTe Assert.IsTrue(output.Contains('2') || output.Contains("TestMethod1"), "Output should contain information about discovered tests"); } + [TestMethod] + public void TerminalTestReporter_WhenMultipleAssemblies_AggregatesCountsAndOmitsAssemblyLinkOnVerdict() + { + // Arrange — two assemblies (the dotnet test orchestrator case), each registered under its own + // execution id. The reporter must aggregate the per-assembly counts into a single run summary and, + // unlike the single-assembly (in-process) case, must NOT append a per-assembly link to the verdict + // line because the per-assembly identity is rendered in the progress area instead. + string firstAssembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\first.dll" : "/mnt/work/first.dll"; + string secondAssembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\second.dll" : "/mnt/work/second.dll"; + + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions + { + ShowPassedTests = () => false, + AnsiMode = AnsiMode.NoAnsi, + ShowProgress = () => false, + }); + + DateTimeOffset startTime = DateTimeOffset.MinValue; + DateTimeOffset endTime = DateTimeOffset.MaxValue; + + // Act + terminalReporter.TestExecutionStarted(startTime, workerCount: 2, isDiscovery: false, isHelp: false, isRetry: false); + + terminalReporter.AssemblyRunStarted(firstAssembly, "net8.0", "x64", "exec-1", "exec-1"); + terminalReporter.AssemblyRunStarted(secondAssembly, "net9.0", "arm64", "exec-2", "exec-2"); + + terminalReporter.TestCompleted("exec-1", testNodeUid: "A1", "A1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + terminalReporter.TestCompleted("exec-1", testNodeUid: "A2", "A2", TestOutcome.Fail, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: "boom", exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + terminalReporter.TestCompleted("exec-2", testNodeUid: "B1", "B1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + terminalReporter.TestCompleted("exec-2", testNodeUid: "B2", "B2", TestOutcome.Skipped, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + + terminalReporter.AssemblyRunCompleted("exec-1"); + terminalReporter.AssemblyRunCompleted("exec-2"); + + // TotalTests aggregates across both assemblies (captured before TestExecutionCompleted clears state). + Assert.AreEqual(4, terminalReporter.TotalTests); + + terminalReporter.TestExecutionCompleted(endTime, exitCode: null); + + string output = stringBuilderConsole.Output; + + // Assert — counts are aggregated across both assemblies. + Assert.Contains(" total: 4", output); + Assert.Contains(" failed: 1", output); + Assert.Contains(" succeeded: 2", output); + Assert.Contains(" skipped: 1", output); + + // The verdict line must be link-free for multiple assemblies: unlike the single-assembly case the + // assembly path is never appended to the "Test run summary:" line. + Assert.DoesNotContain(firstAssembly, output); + Assert.DoesNotContain(secondAssembly, output); + } + [TestMethod] public void SimpleTerminal_UsesWindowWidthNotBufferWidth() { @@ -1185,7 +1245,7 @@ public void AnsiTerminal_ProgressFrame_OnlyUpdatesDuration_WhenProgressVersionUn var stringBuilderConsole = new StringBuilderConsole(); var stopwatchFactory = new StopwatchFactory(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.ForceAnsi, @@ -1206,8 +1266,8 @@ public void AnsiTerminal_ProgressFrame_OnlyUpdatesDuration_WhenProgressVersionUn terminalReporter.OnProgressStartUpdate += (sender, args) => renderGate.WaitOne(); terminalReporter.OnProgressStopUpdate += (sender, args) => renderDone.Set(); - terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, 1, isDiscovery: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); // Pick a starting elapsed value whose rendered form ("1s") has the same length as the value // we will use for the second tick ("2s"). The duration-only path only fires when the rendered @@ -1266,7 +1326,7 @@ public void AnsiTerminal_ProgressFrame_UseWindowWidthForCursorPositioning_WhenBu // Console with BufferWidth=4096 but WindowWidth=120, mimicking the bug scenario. var stringBuilderConsole = new StringBuilderConsoleWithCustomWidths(bufferWidth: 4096, windowWidth: 120); var stopwatchFactory = new StopwatchFactory(); - var terminalReporter = new TerminalTestReporter(assembly, targetFramework, architecture, stringBuilderConsole, static () => false, new TerminalTestReporterOptions + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions { ShowPassedTests = () => true, AnsiMode = AnsiMode.ForceAnsi, @@ -1284,13 +1344,13 @@ public void AnsiTerminal_ProgressFrame_UseWindowWidthForCursorPositioning_WhenBu terminalReporter.OnProgressStopUpdate += (sender, args) => stopHandle.Set(); DateTimeOffset startTime = DateTimeOffset.MinValue; - terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false); - terminalReporter.AssemblyRunStarted(); + terminalReporter.TestExecutionStarted(startTime, 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, targetFramework, architecture, "0", "0"); - terminalReporter.TestInProgress(testNodeUid: "Test1", displayName: "Test1"); + terminalReporter.TestInProgress(executionId: "0", testNodeUid: "Test1", displayName: "Test1"); stopwatchFactory.AddTime(TimeSpan.FromMinutes(1) + TimeSpan.FromSeconds(31)); - terminalReporter.TestCompleted(testNodeUid: "Test1", "Test1", TestOutcome.Passed, TimeSpan.FromSeconds(10), + terminalReporter.TestCompleted("0", testNodeUid: "Test1", "Test1", TestOutcome.Passed, TimeSpan.FromSeconds(10), informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); string output = stringBuilderConsole.Output; From 4a966e2fd9dfe3f939246292949b0f3f0f5f4b56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 19 Jun 2026 17:12:38 +0200 Subject: [PATCH 2/4] Address review: fix GetOrAdd races, dead field, doc/indent warnings - GetOrAddAssemblyRun: replace ConcurrentDictionary.GetOrAdd (whose value factory could run multiple times under contention and orphan worker slots / bump _counter) with an explicit lock + TryGetValue + add, guaranteeing exactly one worker per executionId. - Remove the unused _isRetry field/assignment (IDE0052); keep the isRetry parameter for SDK API parity. - Prefix the Assembly and TotalTests doc summaries with 'Gets' (SA1623). - Normalize the TestCompleted argument indentation in the Failed/Timeout/ Cancelled cases (IDE0055). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TerminalTestReporter.Lifecycle.cs | 27 ++++--- .../Terminal/TerminalTestReporter.cs | 2 +- .../Terminal/TestProgressState.cs | 2 +- .../TerminalOutputDevice.DataConsumption.cs | 72 +++++++++---------- 4 files changed, 55 insertions(+), 48 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs index 9340f70f3a..27ea666066 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs @@ -9,13 +9,11 @@ namespace Microsoft.Testing.Platform.OutputDevice.Terminal; internal sealed partial class TerminalTestReporter { private bool _isHelp; - private bool _isRetry; public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, bool isDiscovery, bool isHelp, bool isRetry) { _isDiscovery = isDiscovery; _isHelp = isHelp; - _isRetry = isRetry; _testExecutionStartTime = testStartTime; _terminalWithProgress.StartShowingProgress(workerCount); } @@ -24,17 +22,26 @@ public void AssemblyRunStarted(string assembly, string? targetFramework, string? => GetOrAddAssemblyRun(assembly, targetFramework, architecture, executionId); private TestProgressState GetOrAddAssemblyRun(string assembly, string? targetFramework, string? architecture, string executionId) - => _assemblies.GetOrAdd(executionId, _ => + { + // NOTE: we intentionally do not use ConcurrentDictionary.GetOrAdd with a value factory here. GetOrAdd may + // invoke the factory more than once under contention and discard all but one result; that would allocate a + // worker slot (and bump _counter) for every losing race, leaving orphaned workers in the progress UI. Doing + // the get-or-add under the same lock that guards AddWorker guarantees exactly one worker per executionId. + lock (_lock) { - lock (_lock) + if (_assemblies.TryGetValue(executionId, out TestProgressState? existing)) { - IStopwatch sw = CreateStopwatch(); - var assemblyRun = new TestProgressState(Interlocked.Increment(ref _counter), assembly, targetFramework, architecture, sw, _isDiscovery); - int slotIndex = _terminalWithProgress.AddWorker(assemblyRun); - assemblyRun.SlotIndex = slotIndex; - return assemblyRun; + return existing; } - }); + + IStopwatch sw = CreateStopwatch(); + var assemblyRun = new TestProgressState(Interlocked.Increment(ref _counter), assembly, targetFramework, architecture, sw, _isDiscovery); + int slotIndex = _terminalWithProgress.AddWorker(assemblyRun); + assemblyRun.SlotIndex = slotIndex; + _assemblies[executionId] = assemblyRun; + return assemblyRun; + } + } internal void AssemblyRunCompleted(string executionId) { diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs index ef190de684..1aa73dd99f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs @@ -64,7 +64,7 @@ internal event EventHandler OnProgressStopUpdate private DateTimeOffset? _testExecutionEndTime; - /// Total number of tests across all registered assemblies. + /// Gets the total number of tests across all registered assemblies. public int TotalTests => _assemblies.Values.Sum(a => a.TotalTests); private bool WasCancelled diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs index 4b80f16aaa..b032ba1ffa 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TestProgressState.cs @@ -20,7 +20,7 @@ public TestProgressState(long id, string assembly, string? targetFramework, stri IsDiscovery = isDiscovery; } - /// The assembly path or display name as provided by the caller (used for the summary link). + /// Gets the assembly path or display name as provided by the caller (used for the summary link). public string Assembly { get; } public string AssemblyName { get; } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs index 28df0262de..1afab8c444 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/TerminalOutputDevice.DataConsumption.cs @@ -182,52 +182,52 @@ public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationTo case FailedTestNodeStateProperty failedState: _terminalTestReporter.TestCompleted( - InProcessExecutionId, - testNodeStateChanged.TestNode.Uid.Value, - testNodeStateChanged.TestNode.DisplayName, - TestOutcome.Fail, - duration, - null, - failedState.Explanation, - failedState.Exception, - expected: failedState.Exception?.Data["assert.expected"] as string, - actual: failedState.Exception?.Data["assert.actual"] as string, - standardOutput, - standardError); + InProcessExecutionId, + testNodeStateChanged.TestNode.Uid.Value, + testNodeStateChanged.TestNode.DisplayName, + TestOutcome.Fail, + duration, + null, + failedState.Explanation, + failedState.Exception, + expected: failedState.Exception?.Data["assert.expected"] as string, + actual: failedState.Exception?.Data["assert.actual"] as string, + standardOutput, + standardError); break; case TimeoutTestNodeStateProperty timeoutState: _terminalTestReporter.TestCompleted( - InProcessExecutionId, - testNodeStateChanged.TestNode.Uid.Value, - testNodeStateChanged.TestNode.DisplayName, - TestOutcome.Timeout, - duration, - null, - timeoutState.Explanation, - timeoutState.Exception, - expected: null, - actual: null, - standardOutput, - standardError); + InProcessExecutionId, + testNodeStateChanged.TestNode.Uid.Value, + testNodeStateChanged.TestNode.DisplayName, + TestOutcome.Timeout, + duration, + null, + timeoutState.Explanation, + timeoutState.Exception, + expected: null, + actual: null, + standardOutput, + standardError); break; #pragma warning disable CS0618, MTP0001 // Type or member is obsolete case CancelledTestNodeStateProperty cancelledState: #pragma warning restore CS0618, MTP0001 // Type or member is obsolete _terminalTestReporter.TestCompleted( - InProcessExecutionId, - testNodeStateChanged.TestNode.Uid.Value, - testNodeStateChanged.TestNode.DisplayName, - TestOutcome.Canceled, - duration, - null, - cancelledState.Explanation, - cancelledState.Exception, - expected: null, - actual: null, - standardOutput, - standardError); + InProcessExecutionId, + testNodeStateChanged.TestNode.Uid.Value, + testNodeStateChanged.TestNode.DisplayName, + TestOutcome.Canceled, + duration, + null, + cancelledState.Explanation, + cancelledState.Exception, + expected: null, + actual: null, + standardOutput, + standardError); break; case PassedTestNodeStateProperty: From ca497afadcc2c2de9b9e5b67c216e0f651ba8aeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 19 Jun 2026 17:24:08 +0200 Subject: [PATCH 3/4] Address expert review: DCL fast path, static lambda, instanceId comment - GetOrAddAssemblyRun: add a lock-free TryGetValue fast path (double-checked locking) so repeat AssemblyRunStarted calls don't take the lock, matching the reviewer's suggested pattern. - TotalTests: make the Sum lambda static to avoid a per-call delegate allocation on the SDK orchestrator's progress-tick path. - Document instanceId on AssemblyRunStarted as reserved for the SDK orchestrator retry-counting follow-up (unused by the in-process host path). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TerminalTestReporter.Lifecycle.cs | 25 ++++++++++++------- .../Terminal/TerminalTestReporter.cs | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs index 27ea666066..350678961f 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs @@ -19,27 +19,34 @@ public void TestExecutionStarted(DateTimeOffset testStartTime, int workerCount, } public void AssemblyRunStarted(string assembly, string? targetFramework, string? architecture, string executionId, string instanceId) + // instanceId: reserved for the SDK orchestrator's instanceId-based retry-counting logic (follow-up PR). + // It is not used by the in-process host path, which registers a single assembly per run. => GetOrAddAssemblyRun(assembly, targetFramework, architecture, executionId); private TestProgressState GetOrAddAssemblyRun(string assembly, string? targetFramework, string? architecture, string executionId) { // NOTE: we intentionally do not use ConcurrentDictionary.GetOrAdd with a value factory here. GetOrAdd may // invoke the factory more than once under contention and discard all but one result; that would allocate a - // worker slot (and bump _counter) for every losing race, leaving orphaned workers in the progress UI. Doing - // the get-or-add under the same lock that guards AddWorker guarantees exactly one worker per executionId. + // worker slot (and bump _counter) for every losing race, leaking a fixed-size progress slot that is never + // RemoveWorker'd. Double-checked locking around AddWorker guarantees exactly one worker per executionId. + if (_assemblies.TryGetValue(executionId, out TestProgressState? result)) + { + return result; + } + lock (_lock) { - if (_assemblies.TryGetValue(executionId, out TestProgressState? existing)) + if (_assemblies.TryGetValue(executionId, out result)) { - return existing; + return result; } IStopwatch sw = CreateStopwatch(); - var assemblyRun = new TestProgressState(Interlocked.Increment(ref _counter), assembly, targetFramework, architecture, sw, _isDiscovery); - int slotIndex = _terminalWithProgress.AddWorker(assemblyRun); - assemblyRun.SlotIndex = slotIndex; - _assemblies[executionId] = assemblyRun; - return assemblyRun; + result = new TestProgressState(Interlocked.Increment(ref _counter), assembly, targetFramework, architecture, sw, _isDiscovery); + int slotIndex = _terminalWithProgress.AddWorker(result); + result.SlotIndex = slotIndex; + _assemblies[executionId] = result; + return result; } } diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs index 1aa73dd99f..9a039a1892 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.cs @@ -65,7 +65,7 @@ internal event EventHandler OnProgressStopUpdate private DateTimeOffset? _testExecutionEndTime; /// Gets the total number of tests across all registered assemblies. - public int TotalTests => _assemblies.Values.Sum(a => a.TotalTests); + public int TotalTests => _assemblies.Values.Sum(static a => a.TotalTests); private bool WasCancelled { From 0e11d1ea6982a08989c0b16dc9725206ea700b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Amaury=20Lev=C3=A9?= Date: Fri, 19 Jun 2026 17:34:56 +0200 Subject: [PATCH 4/4] Address review: reset all per-run state on TestExecutionCompleted TestExecutionCompleted cleared the per-assembly runs but left _artifacts and WasCancelled intact (pre-existing, but contradicts the documented HotReload 'start fresh' intent on a method this PR reworks): a later session would re-print the previous session's artifacts or stay stuck in the aborted state. Now also clear _artifacts and reset WasCancelled after the summary has consumed them. Adds a regression test exercising two sessions on the same reporter. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../TerminalTestReporter.Lifecycle.cs | 7 ++- .../Terminal/TerminalTestReporterTests.cs | 47 +++++++++++++++++++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs index 350678961f..5068f8b578 100644 --- a/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs +++ b/src/Platform/Microsoft.Testing.Platform/OutputDevice/Terminal/TerminalTestReporter.Lifecycle.cs @@ -71,8 +71,13 @@ public void TestExecutionCompleted(DateTimeOffset endTime, int? exitCode) _terminalWithProgress.WriteToTerminal(_isDiscovery ? AppendTestDiscoverySummary : AppendTestRunSummary); } - // This is relevant for HotReload scenarios. We want the next test sessions to start fresh. + // This is relevant for HotReload scenarios. We want the next test sessions to start fresh, so we reset all + // per-run state here (after the summary above has consumed it): the per-assembly runs, the collected + // artifacts, and the cancellation flag. Otherwise a later session would re-print the previous session's + // artifacts or stay stuck in the aborted state. _assemblies.Clear(); + _artifacts.Clear(); + WasCancelled = false; _testExecutionStartTime = null; _testExecutionEndTime = null; diff --git a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs index 4316ff455f..c9086b12dd 100644 --- a/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs +++ b/test/UnitTests/Microsoft.Testing.Platform.UnitTests/OutputDevice/Terminal/TerminalTestReporterTests.cs @@ -1195,6 +1195,53 @@ public void TerminalTestReporter_WhenMultipleAssemblies_AggregatesCountsAndOmits Assert.DoesNotContain(secondAssembly, output); } + [TestMethod] + public void TerminalTestReporter_WhenReusedAcrossSessions_DoesNotLeakArtifactsOrCancelledState() + { + // Reproduces the HotReload reuse case: the same reporter instance runs multiple sessions. After a session + // completes, the per-run state (artifacts, cancellation) must be reset so a later session neither re-prints + // the previous session's artifacts nor stays stuck in the aborted state. + string assembly = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\assembly.dll" : "/mnt/work/assembly.dll"; + string folder = RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? @"C:\work\" : "/mnt/work/"; + string firstSessionArtifact = $"{folder}first-session-artifact.txt"; + + var stringBuilderConsole = new StringBuilderConsole(); + var terminalReporter = new TerminalTestReporter(stringBuilderConsole, static () => false, new TerminalTestReporterOptions + { + ShowPassedTests = () => false, + AnsiMode = AnsiMode.NoAnsi, + ShowProgress = () => false, + }); + + // First session: produces an artifact and is cancelled. + terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, "net8.0", "x64", "0", "0"); + terminalReporter.TestCompleted("0", testNodeUid: "T1", "T1", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + terminalReporter.ArtifactAdded(outOfProcess: false, assembly: assembly, targetFramework: "net8.0", architecture: "x64", executionId: "0", testName: null, firstSessionArtifact); + terminalReporter.StartCancelling(); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: null); + + // Second session on the SAME reporter: no artifacts, not cancelled. + terminalReporter.TestExecutionStarted(DateTimeOffset.MinValue, workerCount: 1, isDiscovery: false, isHelp: false, isRetry: false); + terminalReporter.AssemblyRunStarted(assembly, "net8.0", "x64", "0", "0"); + int outputLengthBeforeSecondSummary = stringBuilderConsole.Output.Length; + terminalReporter.TestCompleted("0", testNodeUid: "T2", "T2", TestOutcome.Passed, TimeSpan.FromSeconds(1), + informativeMessage: null, errorMessage: null, exception: null, expected: null, actual: null, standardOutput: null, errorOutput: null); + terminalReporter.AssemblyRunCompleted("0"); + terminalReporter.TestExecutionCompleted(DateTimeOffset.MaxValue, exitCode: null); + + string secondSessionOutput = stringBuilderConsole.Output.Substring(outputLengthBeforeSecondSummary); + + // The first session's artifact must not be re-printed in the second session's summary. + Assert.DoesNotContain(firstSessionArtifact, secondSessionOutput); + + // The second session is a clean pass, so its summary must not be marked as failed/aborted. + Assert.DoesNotContain(TerminalResources.Aborted, secondSessionOutput); + Assert.Contains(" failed: 0", secondSessionOutput); + } + [TestMethod] public void SimpleTerminal_UsesWindowWidthNotBufferWidth() {