diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLivePublishingModels.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLivePublishingModels.cs index b939985146..5ef5e02a08 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLivePublishingModels.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsLivePublishingModels.cs @@ -15,6 +15,19 @@ internal static class AzureDevOpsLivePublishingConstants public const int MaxRunNameLength = 256; public const string NotExecutedTestOutcome = "NotExecuted"; public const string PassedTestOutcome = "Passed"; + + /// Maximum size (in bytes) of a single attachment uploaded to Azure DevOps. Files larger than this are skipped. + public const long MaxAttachmentSizeBytes = 16L * 1024 * 1024; + + /// Maximum size (in bytes) of an inline (string-based) attachment such as stdout/stderr. Content beyond this is truncated. + public const int MaxInlineAttachmentBytes = 256 * 1024; +} + +internal static class AzureDevOpsAttachmentTypes +{ + public const string CodeCoverage = "CodeCoverage"; + public const string ConsoleLog = "ConsoleLog"; + public const string GeneralAttachment = "GeneralAttachment"; } internal sealed record AzureDevOpsPublishConfiguration( @@ -37,6 +50,44 @@ internal sealed record AzureDevOpsTestCaseResult( [property: JsonPropertyName("startedDate")] DateTimeOffset? StartedDate, [property: JsonPropertyName("completedDate")] DateTimeOffset? CompletedDate); +/// A test case result bundled with optional attachments to upload after the result is published. +internal sealed record AzureDevOpsTestCaseResultWithAttachments( + AzureDevOpsTestCaseResult Result, + IReadOnlyList Attachments); + +/// +/// Describes an attachment to upload to Azure DevOps (either to a test result or to the test run). +/// The payload can come from a file on disk () or from inline string content (). +/// Exactly one of or is non-null. +/// +internal sealed class AzureDevOpsTestResultAttachment +{ + private AzureDevOpsTestResultAttachment(string fileName, string attachmentType, string? comment, string? filePath, string? inlineContent) + { + FileName = fileName; + AttachmentType = attachmentType; + Comment = comment; + FilePath = filePath; + InlineContent = inlineContent; + } + + public string FileName { get; } + + public string AttachmentType { get; } + + public string? Comment { get; } + + public string? FilePath { get; } + + public string? InlineContent { get; } + + public static AzureDevOpsTestResultAttachment FromFile(string filePath, string attachmentType, string? comment = null) + => new(Path.GetFileName(filePath), attachmentType, comment, filePath, inlineContent: null); + + public static AzureDevOpsTestResultAttachment FromString(string content, string fileName, string attachmentType, string? comment = null) + => new(fileName, attachmentType, comment, filePath: null, inlineContent: content); +} + internal sealed record AzureDevOpsTestResultsPublisherOptions( int BatchSize, TimeSpan FlushInterval, diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsClient.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsClient.cs index 32b6e55a98..16e8e744f4 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsClient.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsClient.cs @@ -4,6 +4,7 @@ using System.Net; using System.Net.Http.Headers; using System.Net.Sockets; +using System.Security; using System.Text.Json; using System.Text.Json.Serialization; @@ -18,8 +19,10 @@ internal sealed class AzureDevOpsTestResultsClient : IAzureDevOpsTestResultsClie private const int MaxAttempts = 3; private const int BaseDelayMilliseconds = 500; private static readonly TimeSpan AttemptTimeout = TimeSpan.FromSeconds(15); - private static readonly TimeSpan HttpClientTimeout = TimeSpan.FromSeconds(30); private static readonly TimeSpan RequestTimeout = TimeSpan.FromSeconds(60); + + // Attachments can be up to 16 MB and base64 encoding bloats them ~4/3x. Allow generously longer timeouts. + private static readonly TimeSpan AttachmentRequestTimeout = TimeSpan.FromMinutes(5); private static readonly HttpMethod PatchMethod = new("PATCH"); private static readonly JsonSerializerOptions JsonSerializerOptions = new() { @@ -58,7 +61,9 @@ public async Task CreateTestRunAsync(AzureDevOpsPublishConfiguration config : throw new InvalidOperationException(AzureDevOpsResources.AzureDevOpsLivePublishingInvalidResponse); } - public Task PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken) + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", Justification = "Response types are internal, fixed, and controlled by this extension.")] + [UnconditionalSuppressMessage("Aot", "IL3050", Justification = "Response types are internal, fixed, and controlled by this extension.")] + public async Task?> PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken) { using HttpRequestMessage request = CreateRequest( HttpMethod.Post, @@ -66,7 +71,74 @@ public Task PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuratio configuration.AccessToken, results); - return SendAsync(request, cancellationToken); + // Transport/HTTP failures throw from SendCoreAsync — caller retries. + using var requestTimeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + requestTimeoutSource.CancelAfter(RequestTimeout); + using HttpResponseMessage response = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken, AttemptTimeout).ConfigureAwait(false); + string payload = await ReadAsStringAsync(response.Content, requestTimeoutSource.Token).ConfigureAwait(false); + + // From this point on the AzDO server has accepted the results. Failing to parse the response + // must not cause the caller to retry the publish (that would duplicate result rows). + try + { + PublishTestResultsResponse? parsed = JsonSerializer.Deserialize(payload, JsonSerializerOptions); + if (parsed?.Value is null || parsed.Value.Length != results.Count) + { + return null; + } + + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + if (parsed.Value[i].Id <= 0 + || !string.Equals(parsed.Value[i].AutomatedTestName, results[i].AutomatedTestName, StringComparison.Ordinal)) + { + return null; + } + + ids[i] = parsed.Value[i].Id; + } + + return ids; + } + catch (JsonException) + { + return null; + } + } + + public async Task UploadTestResultAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, int testCaseResultId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken) + { + AttachmentRequest? payload = TryBuildAttachmentRequest(attachment); + if (payload is null) + { + return; + } + + using HttpRequestMessage request = CreateRequest( + HttpMethod.Post, + BuildResultAttachmentsUri(configuration.CollectionUri, configuration.Project, runId, testCaseResultId), + configuration.AccessToken, + payload); + + await SendAsync(request, cancellationToken, AttachmentRequestTimeout).ConfigureAwait(false); + } + + public async Task UploadTestRunAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken) + { + AttachmentRequest? payload = TryBuildAttachmentRequest(attachment); + if (payload is null) + { + return; + } + + using HttpRequestMessage request = CreateRequest( + HttpMethod.Post, + BuildRunAttachmentsUri(configuration.CollectionUri, configuration.Project, runId), + configuration.AccessToken, + payload); + + await SendAsync(request, cancellationToken, AttachmentRequestTimeout).ConfigureAwait(false); } public Task UpdateTestRunStateAsync(AzureDevOpsPublishConfiguration configuration, int runId, string state, CancellationToken cancellationToken) @@ -89,7 +161,7 @@ private static HttpClient CreateHttpClient() return new HttpClient(handler, disposeHandler: false) { - Timeout = HttpClientTimeout, + Timeout = Timeout.InfiniteTimeSpan, }; } @@ -102,6 +174,66 @@ private static Uri BuildRunUri(string collectionUri, string project, int runId) private static Uri BuildResultsUri(string collectionUri, string project, int runId) => new(new Uri(collectionUri, UriKind.Absolute), $"{Uri.EscapeDataString(project)}/_apis/test/runs/{runId}/results?api-version={ApiVersion}"); + private static Uri BuildResultAttachmentsUri(string collectionUri, string project, int runId, int testCaseResultId) + => new(new Uri(collectionUri, UriKind.Absolute), $"{Uri.EscapeDataString(project)}/_apis/test/runs/{runId}/results/{testCaseResultId}/attachments?api-version={ApiVersion}"); + + private static Uri BuildRunAttachmentsUri(string collectionUri, string project, int runId) + => new(new Uri(collectionUri, UriKind.Absolute), $"{Uri.EscapeDataString(project)}/_apis/test/runs/{runId}/attachments?api-version={ApiVersion}"); + + private static AttachmentRequest? TryBuildAttachmentRequest(AzureDevOpsTestResultAttachment attachment) + { + byte[]? bytes; + if (attachment.FilePath is { Length: > 0 } filePath) + { + FileInfo fileInfo; + try + { + fileInfo = new FileInfo(filePath); + if (!fileInfo.Exists) + { + return null; + } + + if (fileInfo.Length > AzureDevOpsLivePublishingConstants.MaxAttachmentSizeBytes) + { + return null; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SecurityException or PathTooLongException) + { + return null; + } + + try + { + bytes = File.ReadAllBytes(filePath); + if (bytes.Length > AzureDevOpsLivePublishingConstants.MaxAttachmentSizeBytes) + { + return null; + } + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SecurityException or PathTooLongException) + { + return null; + } + } + else if (attachment.InlineContent is { } inline) + { + // Inline content (stdout/stderr) is already truncated by the publisher to MaxInlineAttachmentBytes. + bytes = Encoding.UTF8.GetBytes(inline); + } + else + { + return null; + } + + return new AttachmentRequest( + Convert.ToBase64String(bytes), + attachment.FileName, + attachment.Comment, + attachment.AttachmentType); + } + [UnconditionalSuppressMessage("ReflectionAnalysis", "IL2026", Justification = "Payload types are internal, fixed, and controlled by this extension.")] [UnconditionalSuppressMessage("Aot", "IL3050", Justification = "Payload types are internal, fixed, and controlled by this extension.")] private static HttpRequestMessage CreateRequest(HttpMethod method, Uri uri, string accessToken, TPayload payload) @@ -126,7 +258,7 @@ private async Task SendAsync(HttpRequestMessage request, C using var requestTimeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); requestTimeoutSource.CancelAfter(RequestTimeout); - using HttpResponseMessage response = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken).ConfigureAwait(false); + using HttpResponseMessage response = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken, AttemptTimeout).ConfigureAwait(false); string payload = await ReadAsStringAsync(response.Content, requestTimeoutSource.Token).ConfigureAwait(false); TResponse? deserialized = JsonSerializer.Deserialize(payload, JsonSerializerOptions); return deserialized ?? throw new InvalidOperationException(AzureDevOpsResources.AzureDevOpsLivePublishingInvalidResponse); @@ -136,10 +268,17 @@ private async Task SendAsync(HttpRequestMessage request, CancellationToken cance { using var requestTimeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); requestTimeoutSource.CancelAfter(RequestTimeout); - using HttpResponseMessage ignoredResponse = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken).ConfigureAwait(false); + using HttpResponseMessage ignoredResponse = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken, AttemptTimeout).ConfigureAwait(false); } - private async Task SendCoreAsync(HttpRequestMessage request, CancellationToken requestCancellationToken, CancellationToken userCancellationToken) + private async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken, TimeSpan requestTimeout) + { + using var requestTimeoutSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + requestTimeoutSource.CancelAfter(requestTimeout); + using HttpResponseMessage ignoredResponse = await SendCoreAsync(request, requestTimeoutSource.Token, cancellationToken, requestTimeout).ConfigureAwait(false); + } + + private async Task SendCoreAsync(HttpRequestMessage request, CancellationToken requestCancellationToken, CancellationToken userCancellationToken, TimeSpan attemptTimeout) { Exception? lastException = null; @@ -149,7 +288,7 @@ private async Task SendCoreAsync(HttpRequestMessage request { using HttpRequestMessage currentRequest = await CloneAsync(request, requestCancellationToken).ConfigureAwait(false); using var attemptTimeoutSource = CancellationTokenSource.CreateLinkedTokenSource(requestCancellationToken); - attemptTimeoutSource.CancelAfter(AttemptTimeout); + attemptTimeoutSource.CancelAfter(attemptTimeout); try { @@ -283,4 +422,18 @@ private sealed record BuildReference([property: JsonPropertyName("id")] int Id); private sealed record CreateTestRunResponse([property: JsonPropertyName("id")] int Id); private sealed record UpdateTestRunStateRequest([property: JsonPropertyName("state")] string State); + + private sealed record PublishTestResultsResponse( + [property: JsonPropertyName("count")] int Count, + [property: JsonPropertyName("value")] PublishedTestResult[]? Value); + + private sealed record PublishedTestResult( + [property: JsonPropertyName("id")] int Id, + [property: JsonPropertyName("automatedTestName")] string? AutomatedTestName); + + private sealed record AttachmentRequest( + [property: JsonPropertyName("stream")] string Stream, + [property: JsonPropertyName("fileName")] string FileName, + [property: JsonPropertyName("comment")] string? Comment, + [property: JsonPropertyName("attachmentType")] string AttachmentType); } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.cs index dc0e170595..25b4aed643 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsTestResultsPublisher.cs @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT license. See LICENSE file in the project root for full license information. +using System.Security; + using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; using Microsoft.Testing.Extensions.Reporting; using Microsoft.Testing.Platform; @@ -30,8 +32,9 @@ internal sealed class AzureDevOpsTestResultsPublisher : IDataConsumer, ITestSess private readonly ILogger _logger; private readonly AzureDevOpsTestResultsPublisherOptions _options; // Mutate only while holding _flushSemaphore. - private readonly Stack _retryResults = new(); - private readonly ConcurrentQueue _pendingResults = new(); + private readonly Stack _retryResults = new(); + private readonly ConcurrentQueue _pendingResults = new(); + private readonly ConcurrentQueue _pendingRunAttachments = new(); private readonly SemaphoreSlim _flushSemaphore = new(1, 1); private AzureDevOpsPublishConfiguration? _publishConfiguration; @@ -85,7 +88,7 @@ internal AzureDevOpsTestResultsPublisher( _lastFlushTime = clock.UtcNow; } - public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage), typeof(SessionFileArtifact)]; public string Uid => nameof(AzureDevOpsTestResultsPublisher); @@ -165,22 +168,31 @@ public async Task ConsumeAsync(IDataProducer dataProducer, IData value, Cancella { cancellationToken.ThrowIfCancellationRequested(); - if (_publishConfiguration is null || _runIdCoordinator is null || _coordinatedRun is null || CurrentRunId is null || value is not TestNodeUpdateMessage testNodeUpdateMessage) + if (_publishConfiguration is null || _runIdCoordinator is null || _coordinatedRun is null || CurrentRunId is null) { return; } try { - AzureDevOpsTestCaseResult? testCaseResult = CreateTestCaseResult(testNodeUpdateMessage.TestNode, _publishConfiguration.AutomatedTestStorage); - if (testCaseResult is null) + switch (value) { - return; - } + case TestNodeUpdateMessage testNodeUpdateMessage: + AzureDevOpsTestCaseResultWithAttachments? testCaseResult = CreateTestCaseResult(testNodeUpdateMessage.TestNode, _publishConfiguration.AutomatedTestStorage); + if (testCaseResult is null) + { + return; + } - await _runIdCoordinator.RenewLeaseAsync(_coordinatedRun, cancellationToken).ConfigureAwait(false); - _pendingResults.Enqueue(testCaseResult); - await FlushPendingResultsAsync(force: false, cancellationToken).ConfigureAwait(false); + await _runIdCoordinator.RenewLeaseAsync(_coordinatedRun, cancellationToken).ConfigureAwait(false); + _pendingResults.Enqueue(testCaseResult); + await FlushPendingResultsAsync(force: false, cancellationToken).ConfigureAwait(false); + break; + + case SessionFileArtifact sessionFileArtifact when TryCreateRunAttachment(sessionFileArtifact) is { } runAttachment: + _pendingRunAttachments.Enqueue(runAttachment); + break; + } } catch (Exception ex) when (ex is not OperationCanceledException) { @@ -231,6 +243,19 @@ public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionCon _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingPublishResultsFailed} {ex.Message}"); } + try + { + await UploadPendingRunAttachmentsAsync(testSessionContext.CancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Best-effort: don't fail finalization if the session was canceled mid-upload. + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingRunAttachmentFailed} {ex.Message}"); + } + // Azure DevOps test runs use "Aborted" specifically for cancellation or session-level // infrastructure failures. Individual failing tests should still mark the run as // "Completed" — only treat process exit codes other than Success/AtLeastOneTestFailed as @@ -260,7 +285,7 @@ await _runIdCoordinator.FinalizeRunAsync( } } - internal static AzureDevOpsTestCaseResult? CreateTestCaseResult(TestNode testNode, string automatedTestStorage) + internal static AzureDevOpsTestCaseResultWithAttachments? CreateTestCaseResult(TestNode testNode, string automatedTestStorage) { TestNodeStateProperty? state = testNode.Properties.SingleOrDefault(); if (state is null or DiscoveredTestNodeStateProperty or InProgressTestNodeStateProperty) @@ -271,7 +296,7 @@ await _runIdCoordinator.FinalizeRunAsync( TimingProperty? timing = testNode.Properties.SingleOrDefault(); string automatedTestName = testNode.Uid.Value; - return state switch + AzureDevOpsTestCaseResult? result = state switch { PassedTestNodeStateProperty passed => CreateResult(testNode.DisplayName, automatedTestName, automatedTestStorage, AzureDevOpsLivePublishingConstants.PassedTestOutcome, passed.Explanation, null, timing), FailedTestNodeStateProperty failed => CreateResult(testNode.DisplayName, automatedTestName, automatedTestStorage, AzureDevOpsLivePublishingConstants.FailedTestOutcome, failed.Exception?.Message ?? failed.Explanation, failed.Exception?.StackTrace, timing), @@ -283,6 +308,216 @@ await _runIdCoordinator.FinalizeRunAsync( #pragma warning restore CS0618, MTP0001 // Type or member is obsolete _ => null, }; + + if (result is null) + { + return null; + } + + // Only attach artifacts for non-passing outcomes to avoid uploading large dumps/logs for every + // passing test. Users who want pass-time artifacts can use the pipeline-level + // `--report-azdo-upload-artifacts files` option instead. + bool isFailure = state is FailedTestNodeStateProperty or ErrorTestNodeStateProperty or TimeoutTestNodeStateProperty +#pragma warning disable CS0618, MTP0001 // Type or member is obsolete + or CancelledTestNodeStateProperty +#pragma warning restore CS0618, MTP0001 // Type or member is obsolete + ; + + IReadOnlyList attachments = isFailure + ? BuildAttachmentsFromTestNode(testNode) + : []; + + return new AzureDevOpsTestCaseResultWithAttachments(result, attachments); + } + + private static IReadOnlyList BuildAttachmentsFromTestNode(TestNode testNode) + { + List? attachments = null; + + foreach (FileArtifactProperty fileArtifact in testNode.Properties.OfType()) + { + string? fullPath; + try + { + fullPath = fileArtifact.FileInfo.FullName; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SecurityException or PathTooLongException) + { + continue; + } + + attachments ??= []; + attachments.Add(AzureDevOpsTestResultAttachment.FromFile( + fullPath, + AzureDevOpsAttachmentTypes.GeneralAttachment, + comment: fileArtifact.Description ?? fileArtifact.DisplayName)); + } + + StandardOutputProperty? stdout = testNode.Properties.OfType().FirstOrDefault(); + if (stdout is not null && !RoslynString.IsNullOrEmpty(stdout.StandardOutput)) + { + attachments ??= []; + attachments.Add(AzureDevOpsTestResultAttachment.FromString( + TruncateInline(stdout.StandardOutput), + "stdout.log", + AzureDevOpsAttachmentTypes.ConsoleLog)); + } + + StandardErrorProperty? stderr = testNode.Properties.OfType().FirstOrDefault(); + if (stderr is not null && !RoslynString.IsNullOrEmpty(stderr.StandardError)) + { + attachments ??= []; + attachments.Add(AzureDevOpsTestResultAttachment.FromString( + TruncateInline(stderr.StandardError), + "stderr.log", + AzureDevOpsAttachmentTypes.GeneralAttachment)); + } + + return (IReadOnlyList?)attachments ?? []; + } + + private static string TruncateInline(string content) + { + int maxBytes = AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes; + if (Encoding.UTF8.GetByteCount(content) <= maxBytes) + { + return content; + } + + // UTF-8 can use up to 4 bytes per char; reserve room for the truncation marker. + const string Marker = "\n...[truncated]"; + int markerBytes = Encoding.UTF8.GetByteCount(Marker); + int budget = maxBytes - markerBytes; + if (budget <= 0) + { + return Marker; + } + + int byteCount = 0; + int charCount = 0; + while (charCount < content.Length) + { + int charBytes = GetUtf8ByteCount(content, charCount, out int charsConsumed); + if (byteCount + charBytes > budget) + { + break; + } + + byteCount += charBytes; + charCount += charsConsumed; + } + + if (charCount > 0 && char.IsHighSurrogate(content[charCount - 1])) + { + charCount--; + } + + return content[..charCount] + Marker; + } + + private static int GetUtf8ByteCount(string content, int index, out int charsConsumed) + { + char ch = content[index]; + charsConsumed = 1; + if (ch < 0x80) + { + return 1; + } + + if (ch < 0x800) + { + return 2; + } + + if (char.IsHighSurrogate(ch) && index + 1 < content.Length && char.IsLowSurrogate(content[index + 1])) + { + charsConsumed = 2; + return 4; + } + + return 3; + } + + private static AzureDevOpsTestResultAttachment? TryCreateRunAttachment(SessionFileArtifact sessionFileArtifact) + { + FileInfo fileInfo = sessionFileArtifact.FileInfo; + string name; + string fullName; + try + { + name = fileInfo.Name; + fullName = fileInfo.FullName; + } + catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or SecurityException or PathTooLongException) + { + return null; + } + + if (RoslynString.IsNullOrEmpty(name)) + { + return null; + } + + bool isCoverage = name.EndsWith(".coverage", StringComparison.OrdinalIgnoreCase) + || name.EndsWith(".cobertura.xml", StringComparison.OrdinalIgnoreCase) + || name.EndsWith(".opencover.xml", StringComparison.OrdinalIgnoreCase); + + return !isCoverage + ? null + : AzureDevOpsTestResultAttachment.FromFile( + fullName, + AzureDevOpsAttachmentTypes.CodeCoverage, + comment: sessionFileArtifact.Description ?? sessionFileArtifact.DisplayName); + } + + private async Task UploadPendingRunAttachmentsAsync(CancellationToken cancellationToken) + { + if (_publishConfiguration is null || CurrentRunId is null) + { + return; + } + + while (_pendingRunAttachments.TryDequeue(out AzureDevOpsTestResultAttachment? attachment)) + { + try + { + await _client.UploadTestRunAttachmentAsync(_publishConfiguration, CurrentRunId.Value, attachment, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Cancellation aborts the drain; the attachment is lost. The only caller is session + // finishing, where cancellation means the test host is tearing down anyway. + throw; + } + catch (Exception ex) + { + _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingRunAttachmentFailed} {ex.Message}"); + } + } + } + + private async Task UploadResultAttachmentsAsync(int testCaseResultId, IReadOnlyList attachments, CancellationToken cancellationToken) + { + if (_publishConfiguration is null || CurrentRunId is null || attachments.Count == 0) + { + return; + } + + foreach (AzureDevOpsTestResultAttachment attachment in attachments) + { + try + { + await _client.UploadTestResultAttachmentAsync(_publishConfiguration, CurrentRunId.Value, testCaseResultId, attachment, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingResultAttachmentFailed} {ex.Message}"); + } + } } private async Task BackgroundFlushLoopAsync(CancellationToken cancellationToken) @@ -330,13 +565,13 @@ private async Task FlushPendingResultsAsync(bool force, CancellationToken cancel return; } - List batch = []; + List batch = []; while (batch.Count < _options.BatchSize && _retryResults.Count > 0) { batch.Add(_retryResults.Pop()); } - while (batch.Count < _options.BatchSize && _pendingResults.TryDequeue(out AzureDevOpsTestCaseResult? result)) + while (batch.Count < _options.BatchSize && _pendingResults.TryDequeue(out AzureDevOpsTestCaseResultWithAttachments? result)) { batch.Add(result); } @@ -346,6 +581,7 @@ private async Task FlushPendingResultsAsync(bool force, CancellationToken cancel return; } + IReadOnlyList? resultIds; try { if (_coordinatedRun is not null && _runIdCoordinator is not null) @@ -353,12 +589,19 @@ private async Task FlushPendingResultsAsync(bool force, CancellationToken cancel await _runIdCoordinator.RenewLeaseAsync(_coordinatedRun, cancellationToken).ConfigureAwait(false); } - await _client.PublishTestResultsAsync(_publishConfiguration, CurrentRunId.Value, batch, cancellationToken).ConfigureAwait(false); + var resultsOnly = new AzureDevOpsTestCaseResult[batch.Count]; + for (int i = 0; i < batch.Count; i++) + { + resultsOnly[i] = batch[i].Result; + } + + resultIds = await _client.PublishTestResultsAsync(_publishConfiguration, CurrentRunId.Value, resultsOnly, cancellationToken).ConfigureAwait(false); _lastFlushTime = _clock.UtcNow; } catch (Exception ex) when (ex is not OperationCanceledException) { - // Push results in reverse so Pop retries them in the original batch order. + // Transport/HTTP failure — AzDO may not have accepted the batch, so it's safe to + // requeue and retry. Push results in reverse so Pop retries them in batch order. for (int i = batch.Count - 1; i >= 0; i--) { _retryResults.Push(batch[i]); @@ -369,6 +612,45 @@ private async Task FlushPendingResultsAsync(bool force, CancellationToken cancel _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingPublishResultsFailed} {ex.Message}"); return; } + + // POST succeeded. If we couldn't parse the response we cannot upload result-level + // attachments for this batch, but we MUST NOT republish (that would create duplicate + // result rows in AzDO). Continue with the next batch. + if (resultIds is null) + { + if (BatchHasAttachments(batch)) + { + _logger.LogWarning(AzureDevOpsResources.AzureDevOpsLivePublishingResultIdParseFailedWarning); + } + + continue; + } + + for (int i = 0; i < batch.Count; i++) + { + if (batch[i].Attachments.Count == 0) + { + continue; + } + + try + { + if (_coordinatedRun is not null && _runIdCoordinator is not null) + { + await _runIdCoordinator.RenewLeaseAsync(_coordinatedRun, cancellationToken).ConfigureAwait(false); + } + + await UploadResultAttachmentsAsync(resultIds[i], batch[i].Attachments, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + _logger.LogWarning($"{AzureDevOpsResources.AzureDevOpsLivePublishingResultAttachmentFailed} {ex.Message}"); + } + } } } finally @@ -377,6 +659,19 @@ private async Task FlushPendingResultsAsync(bool force, CancellationToken cancel } } + private static bool BatchHasAttachments(IReadOnlyList batch) + { + for (int i = 0; i < batch.Count; i++) + { + if (batch[i].Attachments.Count > 0) + { + return true; + } + } + + return false; + } + private bool TryCreatePublishConfiguration(out AzureDevOpsPublishConfiguration? publishConfiguration, out string? warning) { publishConfiguration = null; diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsTestResultsClient.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsTestResultsClient.cs index e15edd0077..92798884b4 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsTestResultsClient.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsTestResultsClient.cs @@ -7,7 +7,24 @@ internal interface IAzureDevOpsTestResultsClient { Task CreateTestRunAsync(AzureDevOpsPublishConfiguration configuration, CancellationToken cancellationToken); - Task PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken); + /// + /// Publishes a batch of test case results to Azure DevOps and returns the IDs assigned to each result + /// in the same order as the input. Returns if the request succeeded (HTTP 2xx) + /// but the response could not be parsed or did not match the submitted batch; in that case the caller + /// MUST NOT retry the publish (the results were already accepted) but cannot upload result-level + /// attachments. Throws on transport/HTTP failures, which the caller may retry. + /// + Task?> PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken); + + /// + /// Uploads an attachment to a specific test case result within a test run. + /// + Task UploadTestResultAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, int testCaseResultId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken); + + /// + /// Uploads an attachment to the test run itself (e.g. code coverage files). + /// + Task UploadTestRunAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken); Task UpdateTestRunStateAsync(AzureDevOpsPublishConfiguration configuration, int runId, string state, CancellationToken cancellationToken); } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PACKAGE.md b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PACKAGE.md index a7f24e3859..200151d501 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PACKAGE.md +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PACKAGE.md @@ -17,6 +17,8 @@ This package extends Microsoft.Testing.Platform with: - **Azure DevOps reporting**: emits individual CI errors/warnings for each test failure via the Azure DevOps logging commands - **Configurable severity**: supports `--report-azdo-severity` (`error` or `warning`) - **CI auto-detection**: detects Azure DevOps environments through the `TF_BUILD` variable +- **Live publishing**: streams test results to the Azure DevOps Tests tab while the run is still in progress (`--publish-azdo-test-results`) +- **Automatic attachments**: for failed tests, attaches stdout/stderr and any `FileArtifactProperty` data (dumps, screenshots) to the matching result in the Tests tab; uploads `*.coverage`, `*.cobertura.xml`, and `*.opencover.xml` artifacts as run-level attachments Enable Azure DevOps reporting with the `--report-azdo` command line option. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx index 4591cdc65c..2e0b42ef7c 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx @@ -166,6 +166,15 @@ Azure DevOps REST API request failed after all retries. + + Azure DevOps live publishing failed to upload a test result attachment. + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. {Locked="azdo-runid.<buildId>.json"} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf index 8b9a0f3564..a07f190f01 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf @@ -77,6 +77,21 @@ Žádost na rozhraní REST API Azure DevOps se po všech opakovaných pokusech nezdařila. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Živé publikování Azure DevOps nalezlo soubor azdo-runid.<buildId>.json pro jiný projekt Azure DevOps. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf index 7ae7ba2a12..5efc5cd22e 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf @@ -77,6 +77,21 @@ Die Azure DevOps REST API-Anforderung ist nach allen Wiederholungen fehlgeschlagen. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Die Azure DevOps Liveveröffentlichung hat eine azdo-runid.<buildId>.json-Datei für ein anderes Azure DevOps-Projekt gefunden. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf index cdac8a93fc..c1309e3335 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf @@ -77,6 +77,21 @@ La solicitud a la API de REST de Azure DevOps falló después de todos los reintentos. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. La publicación en directo de Azure DevOps encontró un archivo azdo-runid.<buildId>.json para otro proyecto de Azure DevOps. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf index bedded65fa..aea15e7d7b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf @@ -77,6 +77,21 @@ Nous n’avons pas pu effectué la requête d’API REST Azure DevOps après toutes les nouvelles tentatives. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. La publication en direct d’Azure DevOps a trouvé un fichier azdo-runid.<buildId>.json pour un autre projet Azure DevOps. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf index 580a3c3b57..5186f4c7f3 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf @@ -77,6 +77,21 @@ La richiesta dell'API REST di Azure DevOps non è riuscita dopo tutti i tentativi. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. La pubblicazione live di Azure DevOps ha trovato un file azdo-runid.<buildId>.json per un progetto Azure DevOps diverso. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf index a3e9f89478..bc917cf348 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf @@ -77,6 +77,21 @@ Azure DevOps REST API 要求は、すべての再試行後に失敗しました。 + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps ライブ公開で、別の Azure DevOps プロジェクトの azdo-runid.<buildId>.json ファイルが見つかりました。 diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf index 9c12544342..e4764e256f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf @@ -77,6 +77,21 @@ 모든 재시도 후 Azure DevOps REST API 요청이 실패했습니다. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps 라이브 게시에서 다른 Azure DevOps 프로젝트에 대한 azdo-runid.<buildId>.json 파일을 찾았습니다. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf index c8767f4934..236ae9c8a2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf @@ -77,6 +77,21 @@ Żądanie interfejsu API REST usługi Azure DevOps nie powiodło się po wszystkich ponownych próbach. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Publikowanie na żywo w usłudze Azure DevOps znalazło plik azdo-runid.<buildId>.json dla innego projektu usługi Azure DevOps. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf index e2369ceae3..22e7e521cc 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf @@ -77,6 +77,21 @@ A solicitação da API REST do Azure DevOps falhou após todas as tentativas. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. A publicação ao vivo do Azure DevOps encontrou um arquivo azdo-runid.<buildId>.json para um projeto do Azure DevOps diferente. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf index 72ffd7618a..c8d9abc8a2 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf @@ -77,6 +77,21 @@ Azure DevOps: не удалось выполнить запрос REST API после повторных попыток. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps: при публикации в реальном времени обнаружен файл azdo-runid.<buildId>.json для другого проекта Azure DevOps. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf index 737ffdb75e..3103836c28 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf @@ -77,6 +77,21 @@ Azure DevOps REST API isteği tüm yeniden denemelerden sonra başarısız oldu. + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps canlı yayımlama, farklı bir Azure DevOps projesi için azdo-runid.<buildId>.json dosyası buldu. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf index 9dbab55293..177d4d865d 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf @@ -77,6 +77,21 @@ Azure DevOps REST API 请求在所有重试后均失败。 + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps 实时发布发现了属于其他 Azure DevOps 项目的 azdo-runid.<buildId>.json 文件。 diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf index 3a996fa122..e9b0fe8ccd 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf @@ -77,6 +77,21 @@ 在所有重試後，Azure DevOps REST API 要求失敗。 + + Azure DevOps live publishing failed to upload a test result attachment. + Azure DevOps live publishing failed to upload a test result attachment. + + + + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + Azure DevOps live publishing accepted a batch of test results but could not parse the assigned result IDs; result-level attachments for this batch will be skipped. + + + + Azure DevOps live publishing failed to upload a test run attachment. + Azure DevOps live publishing failed to upload a test run attachment. + + Azure DevOps live publishing found an azdo-runid.<buildId>.json file for a different Azure DevOps project. Azure DevOps 即時發佈找到不同 Azure DevOps 專案的 azdo-runid.<buildId>.json 檔案。 diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsLivePublishingTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsLivePublishingTests.cs index c6fbe3620f..b5b67d3060 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsLivePublishingTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsLivePublishingTests.cs @@ -4,6 +4,7 @@ using System.Diagnostics; using System.Net; using System.Net.Http; +using System.Text; using System.Text.Json; using Microsoft.Testing.Extensions.AzureDevOpsReport; @@ -25,6 +26,8 @@ namespace Microsoft.Testing.Extensions.UnitTests; [TestClass] public sealed class AzureDevOpsLivePublishingTests { + private const string TruncationMarker = "\n...[truncated]"; + private readonly List _directoriesToDelete = []; [TestCleanup] @@ -73,7 +76,13 @@ public async Task ConsumeAsync_FlushesWhenBatchSizeIsReached() client.PublishTestResultsAsyncFunc = (_, _, results, _) => { publishedBatches.Add(results.ToArray()); - return Task.CompletedTask; + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = i + 1; + } + + return Task.FromResult?>(ids); }; await StartPublisherAsync(publisher); @@ -95,7 +104,13 @@ public async Task ConsumeAsync_FlushesWhenFlushIntervalElapsed() client.PublishTestResultsAsyncFunc = (_, _, results, _) => { publishedBatches.Add(results.ToArray()); - return Task.CompletedTask; + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = i + 1; + } + + return Task.FromResult?>(ids); }; await StartPublisherAsync(publisher); @@ -119,7 +134,13 @@ public async Task OnTestSessionFinishingAsync_FlushesRemainingResults() client.PublishTestResultsAsyncFunc = (_, _, results, _) => { publishedBatches.Add(results.ToArray()); - return Task.CompletedTask; + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = i + 1; + } + + return Task.FromResult?>(ids); }; await StartPublisherAsync(publisher); @@ -202,12 +223,12 @@ public void CreateTestCaseResult_MapsMtpStatesToAzdoResults() { DateTimeOffset startTime = new(2025, 1, 1, 12, 0, 0, TimeSpan.Zero); TimingProperty timing = new(new TimingInfo(startTime, startTime.AddSeconds(2), TimeSpan.FromSeconds(2))); - AzureDevOpsTestCaseResult? passed = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("passed", new PassedTestNodeStateProperty(), startTime, timing), "tests.dll"); - AzureDevOpsTestCaseResult? failed = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("failed", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), startTime, timing), "tests.dll"); - AzureDevOpsTestCaseResult? skipped = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("skipped", new SkippedTestNodeStateProperty("skip"), startTime, timing), "tests.dll"); - AzureDevOpsTestCaseResult? timeout = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("timeout", new TimeoutTestNodeStateProperty("too slow"), startTime, timing), "tests.dll"); + AzureDevOpsTestCaseResult? passed = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("passed", new PassedTestNodeStateProperty(), startTime, timing), "tests.dll")?.Result; + AzureDevOpsTestCaseResult? failed = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("failed", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), startTime, timing), "tests.dll")?.Result; + AzureDevOpsTestCaseResult? skipped = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("skipped", new SkippedTestNodeStateProperty("skip"), startTime, timing), "tests.dll")?.Result; + AzureDevOpsTestCaseResult? timeout = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("timeout", new TimeoutTestNodeStateProperty("too slow"), startTime, timing), "tests.dll")?.Result; #pragma warning disable CS0618, MTP0001 // Type or member is obsolete - AzureDevOpsTestCaseResult? cancelled = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("cancelled", new CancelledTestNodeStateProperty("stopped"), startTime, timing), "tests.dll"); + AzureDevOpsTestCaseResult? cancelled = AzureDevOpsTestResultsPublisher.CreateTestCaseResult(CreateNode("cancelled", new CancelledTestNodeStateProperty("stopped"), startTime, timing), "tests.dll")?.Result; #pragma warning restore CS0618, MTP0001 // Type or member is obsolete Assert.AreEqual(AzureDevOpsLivePublishingConstants.PassedTestOutcome, passed?.Outcome); @@ -316,7 +337,7 @@ public async Task ConsumeAsync_PublishFailureLogsWarningAndDoesNotThrow() using TestDirectory directory = CreateTestDirectory(); AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out CollectingLogger logger); client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(103); - client.PublishTestResultsAsyncFunc = (_, _, _, _) => Task.FromException(new JsonException("publish failed")); + client.PublishTestResultsAsyncFunc = (_, _, _, _) => Task.FromException?>(new JsonException("publish failed")); await StartPublisherAsync(publisher); TestNodeUpdateMessage message = CreateMessage(CreateNode("test-1", new PassedTestNodeStateProperty(), clock.UtcNow)); @@ -339,9 +360,18 @@ public async Task ConsumeAsync_PublishFailureRetriesBatchOnFinalFlush() { publishAttempts++; publishedBatches.Add(results.ToArray()); - return publishAttempts == 1 - ? Task.FromException(new JsonException("publish failed")) - : Task.CompletedTask; + if (publishAttempts == 1) + { + return Task.FromException?>(new JsonException("publish failed")); + } + + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = i + 1; + } + + return Task.FromResult?>(ids); }; await StartPublisherAsync(publisher); @@ -355,6 +385,278 @@ public async Task ConsumeAsync_PublishFailureRetriesBatchOnFinalFlush() Assert.Contains(AzureDevOpsResources.AzureDevOpsLivePublishingPublishResultsFailed, string.Join(Environment.NewLine, logger.Logs)); } + [TestMethod] + public async Task ConsumeAsync_UploadsAttachmentsForFailedTests() + { + using TestDirectory directory = CreateTestDirectory(); + string dumpPath = Path.Combine(directory.Path, "dump.txt"); + File.WriteAllText(dumpPath, "dump content"); + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(200); + client.PublishTestResultsAsyncFunc = (_, _, results, _) => + { + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = 1_000 + i; + } + + return Task.FromResult?>(ids); + }; + + TestNode node = CreateNode("failed-test", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), clock.UtcNow); + node.Properties.Add(new FileArtifactProperty(new FileInfo(dumpPath), "dump", "crash dump")); + node.Properties.Add(new StandardOutputProperty("stdout content")); + node.Properties.Add(new StandardErrorProperty("stderr content")); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + + Assert.HasCount(3, client.UploadTestResultAttachmentCalls); + Assert.AreEqual(1_000, client.UploadTestResultAttachmentCalls[0].TestCaseResultId); + Assert.AreEqual(1_000, client.UploadTestResultAttachmentCalls[1].TestCaseResultId); + Assert.AreEqual(1_000, client.UploadTestResultAttachmentCalls[2].TestCaseResultId); + Assert.AreEqual("dump.txt", client.UploadTestResultAttachmentCalls[0].Attachment.FileName); + Assert.AreEqual(AzureDevOpsAttachmentTypes.GeneralAttachment, client.UploadTestResultAttachmentCalls[0].Attachment.AttachmentType); + Assert.AreEqual("stdout.log", client.UploadTestResultAttachmentCalls[1].Attachment.FileName); + Assert.AreEqual(AzureDevOpsAttachmentTypes.ConsoleLog, client.UploadTestResultAttachmentCalls[1].Attachment.AttachmentType); + Assert.AreEqual("stderr.log", client.UploadTestResultAttachmentCalls[2].Attachment.FileName); + Assert.AreEqual(AzureDevOpsAttachmentTypes.GeneralAttachment, client.UploadTestResultAttachmentCalls[2].Attachment.AttachmentType); + } + + [TestMethod] + public async Task ConsumeAsync_DoesNotUploadAttachmentsForPassedTests() + { + using TestDirectory directory = CreateTestDirectory(); + string dumpPath = Path.Combine(directory.Path, "passing-dump.txt"); + File.WriteAllText(dumpPath, "should not be uploaded"); + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(201); + + TestNode node = CreateNode("passing-test", new PassedTestNodeStateProperty(), clock.UtcNow); + node.Properties.Add(new FileArtifactProperty(new FileInfo(dumpPath), "dump")); + node.Properties.Add(new StandardOutputProperty("stdout content")); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + + Assert.HasCount(0, client.UploadTestResultAttachmentCalls); + } + + [TestMethod] + public async Task ConsumeAsync_AttachmentUploadFailureLogsWarningAndDoesNotRetryPublish() + { + using TestDirectory directory = CreateTestDirectory(); + string dumpPath = Path.Combine(directory.Path, "dump.txt"); + File.WriteAllText(dumpPath, "dump content"); + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out CollectingLogger logger); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(202); + + int publishCalls = 0; + client.PublishTestResultsAsyncFunc = (_, _, results, _) => + { + publishCalls++; + return Task.FromResult?>(Enumerable.Range(1, results.Count).ToArray()); + }; + client.UploadTestResultAttachmentAsyncFunc = (_, _, _, _, _) => throw new HttpRequestException("simulated upload failure"); + + TestNode node = CreateNode("failed-test", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), clock.UtcNow); + node.Properties.Add(new FileArtifactProperty(new FileInfo(dumpPath), "dump")); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + await publisher.OnTestSessionFinishingAsync(new Microsoft.Testing.Platform.Services.TestSessionContext(CancellationToken.None)); + + Assert.AreEqual(1, publishCalls); + Assert.HasCount(1, client.UploadTestResultAttachmentCalls); + Assert.Contains(AzureDevOpsResources.AzureDevOpsLivePublishingResultAttachmentFailed, string.Join(Environment.NewLine, logger.Logs)); + } + + [TestMethod] + public async Task ConsumeAsync_PublishReturnsNullSkipsAttachmentsAndDoesNotRetry() + { + using TestDirectory directory = CreateTestDirectory(); + string dumpPath = Path.Combine(directory.Path, "dump.txt"); + File.WriteAllText(dumpPath, "dump content"); + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out CollectingLogger logger); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(203); + + int publishCalls = 0; + client.PublishTestResultsAsyncFunc = (_, _, _, _) => + { + publishCalls++; + return Task.FromResult?>(null); + }; + + TestNode node = CreateNode("failed-test", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), clock.UtcNow); + node.Properties.Add(new FileArtifactProperty(new FileInfo(dumpPath), "dump")); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + await publisher.OnTestSessionFinishingAsync(new Microsoft.Testing.Platform.Services.TestSessionContext(CancellationToken.None)); + + Assert.AreEqual(1, publishCalls); + Assert.HasCount(0, client.UploadTestResultAttachmentCalls); + Assert.Contains(AzureDevOpsResources.AzureDevOpsLivePublishingResultIdParseFailedWarning, string.Join(Environment.NewLine, logger.Logs)); + } + + [TestMethod] + public async Task ConsumeAsync_SkipsOversizedFileAttachment() + { + using TestDirectory directory = CreateTestDirectory(); + string smallPath = Path.Combine(directory.Path, "small.txt"); + File.WriteAllText(smallPath, "small content"); + string bigPath = Path.Combine(directory.Path, "big.bin"); + using (FileStream fs = File.OpenWrite(bigPath)) + { + fs.SetLength(AzureDevOpsLivePublishingConstants.MaxAttachmentSizeBytes + 1); + } + + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(204); + // The publisher still queues the oversized attachment; the client side TryBuildAttachmentRequest + // drops it. In this fake we just record the call regardless — the contract is exercised end-to-end + // when running against the real client. For the unit test we only assert what the publisher sends. + client.UploadTestResultAttachmentAsyncFunc = (_, _, _, attachment, _) => Task.CompletedTask; + + TestNode node = CreateNode("failed-test", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), clock.UtcNow); + node.Properties.Add(new FileArtifactProperty(new FileInfo(smallPath), "small")); + node.Properties.Add(new FileArtifactProperty(new FileInfo(bigPath), "big")); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + + // The publisher passes both attachments through; the client's TryBuildAttachmentRequest is what + // skips oversized ones. We sanity-check that the publisher does forward both names so the client + // gets a chance to filter. + Assert.HasCount(2, client.UploadTestResultAttachmentCalls); + Assert.Contains(c => c.Attachment.FileName == "small.txt", client.UploadTestResultAttachmentCalls); + Assert.Contains(c => c.Attachment.FileName == "big.bin", client.UploadTestResultAttachmentCalls); + } + + [TestMethod] + public async Task ConsumeAsync_DoesNotTruncateStdoutAtInlineByteLimit() + { + string stdout = new('x', AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes); + + AzureDevOpsTestResultAttachment uploaded = await UploadStdoutAttachmentAsync(stdout); + + Assert.AreEqual(stdout, uploaded.InlineContent); + Assert.AreEqual(AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes, Encoding.UTF8.GetByteCount(uploaded.InlineContent!)); + } + + [TestMethod] + public async Task ConsumeAsync_TruncatesLargeStdoutInline() + { + string oversizedStdout = new('x', AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes + 10_000); + + AzureDevOpsTestResultAttachment uploaded = await UploadStdoutAttachmentAsync(oversizedStdout); + + Assert.IsNotNull(uploaded.InlineContent); + Assert.IsLessThanOrEqualTo(AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes, Encoding.UTF8.GetByteCount(uploaded.InlineContent!)); + Assert.EndsWith(TruncationMarker, uploaded.InlineContent!, StringComparison.Ordinal); + } + + [DataRow("€")] + [DataRow("😀")] + [TestMethod] + public async Task ConsumeAsync_TruncatesUnicodeStdoutWithinInlineByteLimit(string textElement) + { + int repeatCount = (AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes / Encoding.UTF8.GetByteCount(textElement)) + 10_000; + string oversizedStdout = string.Concat(Enumerable.Repeat(textElement, repeatCount)); + + AzureDevOpsTestResultAttachment uploaded = await UploadStdoutAttachmentAsync(oversizedStdout); + + Assert.IsNotNull(uploaded.InlineContent); + Assert.IsLessThanOrEqualTo(AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes, Encoding.UTF8.GetByteCount(uploaded.InlineContent!)); + Assert.EndsWith(TruncationMarker, uploaded.InlineContent!, StringComparison.Ordinal); + + string retainedContent = uploaded.InlineContent![..^TruncationMarker.Length]; + Assert.IsFalse(retainedContent.Length > 0 && char.IsHighSurrogate(retainedContent[^1])); + } + + [TestMethod] + public async Task ConsumeAsync_TruncatesStdoutWithoutSplittingSurrogatePairAtBoundary() + { + string emoji = "😀"; + int markerBytes = Encoding.UTF8.GetByteCount(TruncationMarker); + int budget = AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes - markerBytes; + string prefix = new('x', budget - Encoding.UTF8.GetByteCount(emoji)); + string oversizedStdout = prefix + emoji + new string('x', markerBytes + 1); + + AzureDevOpsTestResultAttachment uploaded = await UploadStdoutAttachmentAsync(oversizedStdout); + + Assert.IsNotNull(uploaded.InlineContent); + Assert.IsLessThanOrEqualTo(AzureDevOpsLivePublishingConstants.MaxInlineAttachmentBytes, Encoding.UTF8.GetByteCount(uploaded.InlineContent!)); + Assert.AreEqual(prefix + emoji + TruncationMarker, uploaded.InlineContent); + } + + [TestMethod] + public async Task OnTestSessionFinishingAsync_UploadsCoverageSessionFileArtifacts() + { + using TestDirectory directory = CreateTestDirectory(); + string coveragePath = Path.Combine(directory.Path, "results.cobertura.xml"); + File.WriteAllText(coveragePath, ""); + string opencoverPath = Path.Combine(directory.Path, "results.opencover.xml"); + File.WriteAllText(opencoverPath, ""); + string binaryCoveragePath = Path.Combine(directory.Path, "results.coverage"); + File.WriteAllBytes(binaryCoveragePath, [0, 1, 2, 3]); + + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(10, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out _, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(206); + + await StartPublisherAsync(publisher); + SessionUid sessionUid = new(Guid.NewGuid().ToString()); + await publisher.ConsumeAsync(Mock.Of(), new SessionFileArtifact(sessionUid, new FileInfo(coveragePath), "cobertura", "coverage report"), CancellationToken.None); + await publisher.ConsumeAsync(Mock.Of(), new SessionFileArtifact(sessionUid, new FileInfo(opencoverPath), "opencover"), CancellationToken.None); + await publisher.ConsumeAsync(Mock.Of(), new SessionFileArtifact(sessionUid, new FileInfo(binaryCoveragePath), "vs-coverage"), CancellationToken.None); + await publisher.OnTestSessionFinishingAsync(new Microsoft.Testing.Platform.Services.TestSessionContext(CancellationToken.None)); + + Assert.HasCount(3, client.UploadTestRunAttachmentCalls); + Assert.IsTrue(client.UploadTestRunAttachmentCalls.All(c => c.Attachment.AttachmentType == AzureDevOpsAttachmentTypes.CodeCoverage)); + Assert.Contains(c => c.Attachment.FileName == "results.cobertura.xml", client.UploadTestRunAttachmentCalls); + Assert.Contains(c => c.Attachment.FileName == "results.opencover.xml", client.UploadTestRunAttachmentCalls); + Assert.Contains(c => c.Attachment.FileName == "results.coverage", client.UploadTestRunAttachmentCalls); + } + + [TestMethod] + public async Task OnTestSessionFinishingAsync_SkipsNonCoverageSessionFileArtifacts() + { + using TestDirectory directory = CreateTestDirectory(); + string logPath = Path.Combine(directory.Path, "run.log"); + File.WriteAllText(logPath, "some log"); + + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(10, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out _, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(207); + + await StartPublisherAsync(publisher); + SessionUid sessionUid = new(Guid.NewGuid().ToString()); + await publisher.ConsumeAsync(Mock.Of(), new SessionFileArtifact(sessionUid, new FileInfo(logPath), "log"), CancellationToken.None); + await publisher.OnTestSessionFinishingAsync(new Microsoft.Testing.Platform.Services.TestSessionContext(CancellationToken.None)); + + Assert.HasCount(0, client.UploadTestRunAttachmentCalls); + } + + [TestMethod] + public async Task OnTestSessionFinishingAsync_CoverageUploadFailureLogsWarning() + { + using TestDirectory directory = CreateTestDirectory(); + string coveragePath = Path.Combine(directory.Path, "results.cobertura.xml"); + File.WriteAllText(coveragePath, ""); + + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(10, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out _, out CollectingLogger logger); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(208); + client.UploadTestRunAttachmentAsyncFunc = (_, _, _, _) => throw new HttpRequestException("simulated upload failure"); + + await StartPublisherAsync(publisher); + SessionUid sessionUid = new(Guid.NewGuid().ToString()); + await publisher.ConsumeAsync(Mock.Of(), new SessionFileArtifact(sessionUid, new FileInfo(coveragePath), "cobertura"), CancellationToken.None); + await publisher.OnTestSessionFinishingAsync(new Microsoft.Testing.Platform.Services.TestSessionContext(CancellationToken.None)); + + Assert.HasCount(1, client.UploadTestRunAttachmentCalls); + Assert.Contains(AzureDevOpsResources.AzureDevOpsLivePublishingRunAttachmentFailed, string.Join(Environment.NewLine, logger.Logs)); + } + [TestMethod] public async Task RunIdCoordinator_CreateAndReadFlowSharesRunIdAcrossProcesses() { @@ -555,6 +857,22 @@ public async Task RunIdCoordinator_AcquireRunAsync_JoinerGivesUpWhenOwnerLeaseEx Assert.IsTrue(coordinatedRun.IsOwner); } + private async Task UploadStdoutAttachmentAsync(string stdout) + { + using TestDirectory directory = CreateTestDirectory(); + AzureDevOpsTestResultsPublisher publisher = CreatePublisher(directory.Path, options: new(1, TimeSpan.FromMinutes(1), 4, TimeSpan.FromMilliseconds(1)), out FakeAzureDevOpsTestResultsClient client, out FakeClock clock, out _); + client.CreateTestRunAsyncFunc = (_, _) => Task.FromResult(205); + + TestNode node = CreateNode("failed-test", new FailedTestNodeStateProperty(new InvalidOperationException("boom")), clock.UtcNow); + node.Properties.Add(new StandardOutputProperty(stdout)); + + await StartPublisherAsync(publisher); + await publisher.ConsumeAsync(Mock.Of(), CreateMessage(node), CancellationToken.None); + + Assert.HasCount(1, client.UploadTestResultAttachmentCalls); + return client.UploadTestResultAttachmentCalls[0].Attachment; + } + private static async Task StartPublisherAsync(AzureDevOpsTestResultsPublisher publisher) { Assert.IsTrue(await publisher.IsEnabledAsync()); @@ -676,14 +994,32 @@ private sealed class FakeAzureDevOpsTestResultsClient : IAzureDevOpsTestResultsC { public Func> CreateTestRunAsyncFunc { get; set; } = (_, _) => Task.FromResult(0); - public Func, CancellationToken, Task> PublishTestResultsAsyncFunc { get; set; } = (_, _, _, _) => Task.CompletedTask; + public Func, CancellationToken, Task?>> PublishTestResultsAsyncFunc { get; set; } = + (_, _, results, _) => + { + int[] ids = new int[results.Count]; + for (int i = 0; i < results.Count; i++) + { + ids[i] = i + 1; + } + + return Task.FromResult?>(ids); + }; + + public Func UploadTestResultAttachmentAsyncFunc { get; set; } = (_, _, _, _, _) => Task.CompletedTask; + + public Func UploadTestRunAttachmentAsyncFunc { get; set; } = (_, _, _, _) => Task.CompletedTask; public List<(AzureDevOpsPublishConfiguration Configuration, int RunId, string State)> UpdateTestRunStateCalls { get; } = []; + public List<(int RunId, int TestCaseResultId, AzureDevOpsTestResultAttachment Attachment)> UploadTestResultAttachmentCalls { get; } = []; + + public List<(int RunId, AzureDevOpsTestResultAttachment Attachment)> UploadTestRunAttachmentCalls { get; } = []; + public Task CreateTestRunAsync(AzureDevOpsPublishConfiguration configuration, CancellationToken cancellationToken) => CreateTestRunAsyncFunc(configuration, cancellationToken); - public Task PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken) + public Task?> PublishTestResultsAsync(AzureDevOpsPublishConfiguration configuration, int runId, IReadOnlyList results, CancellationToken cancellationToken) => PublishTestResultsAsyncFunc(configuration, runId, results, cancellationToken); public Task UpdateTestRunStateAsync(AzureDevOpsPublishConfiguration configuration, int runId, string state, CancellationToken cancellationToken) @@ -691,6 +1027,18 @@ public Task UpdateTestRunStateAsync(AzureDevOpsPublishConfiguration configuratio UpdateTestRunStateCalls.Add((configuration, runId, state)); return Task.CompletedTask; } + + public Task UploadTestResultAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, int testCaseResultId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken) + { + UploadTestResultAttachmentCalls.Add((runId, testCaseResultId, attachment)); + return UploadTestResultAttachmentAsyncFunc(configuration, runId, testCaseResultId, attachment, cancellationToken); + } + + public Task UploadTestRunAttachmentAsync(AzureDevOpsPublishConfiguration configuration, int runId, AzureDevOpsTestResultAttachment attachment, CancellationToken cancellationToken) + { + UploadTestRunAttachmentCalls.Add((runId, attachment)); + return UploadTestRunAttachmentAsyncFunc(configuration, runId, attachment, cancellationToken); + } } private sealed class FakeTask(Action? delayCallback = null) : ITask