diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs index 6d7051d4c5..6adbcab906 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs @@ -10,6 +10,9 @@ internal static class AzureDevOpsCommandLineOptions public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history"; public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file"; public const string AzureDevOpsReportSeverity = "report-azdo-severity"; + public const string AzureDevOpsSlowTestHistory = "report-azdo-slow-test-history"; + public const string AzureDevOpsSlowTestHistoryMinSample = "report-azdo-slow-test-history-min-sample"; + public const string AzureDevOpsSlowTestHistoryMultiplier = "report-azdo-slow-test-history-multiplier"; public const string AzureDevOpsStackFrameFilter = "report-azdo-stackframe-filter"; public const string AzureDevOpsSummary = "report-azdo-summary"; public const string AzureDevOpsUploadArtifactExclude = "report-azdo-upload-artifact-exclude"; @@ -22,4 +25,8 @@ internal static class AzureDevOpsCommandLineOptions public const string AzureDevOpsUploadArtifactsModeTagsOnly = "tags-only"; public const string PublishAzureDevOpsRunNameOptionName = "publish-azdo-run-name"; public const string PublishAzureDevOpsTestResultsOptionName = "publish-azdo-test-results"; + + public const int SlowTestHistoryDefaultMinSample = 10; + public const double SlowTestHistoryDefaultMultiplier = 3.0; + public const int SlowTestStaticThresholdSeconds = 60; } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs index a50e79cd6c..81cd8a5d1a 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs @@ -37,6 +37,16 @@ internal sealed class AzureDevOpsCommandLineProvider : CommandLineOptionsProvide MaxStackFrameFilterPatterns, StackFrameFilterMatchTimeoutMs); + private static readonly string SlowTestHistoryMinSampleOptionDescriptionFormatted = string.Format( + CultureInfo.InvariantCulture, + AzureDevOpsResources.SlowTestHistoryMinSampleOptionDescription, + AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMinSample); + + private static readonly string SlowTestHistoryMultiplierOptionDescriptionFormatted = string.Format( + CultureInfo.InvariantCulture, + AzureDevOpsResources.SlowTestHistoryMultiplierOptionDescription, + AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMultiplier); + public AzureDevOpsCommandLineProvider() : base( nameof(AzureDevOpsCommandLineProvider), @@ -49,6 +59,9 @@ public AzureDevOpsCommandLineProvider() new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory, AzureDevOpsResources.SlowTestHistoryOptionDescription, ArgumentArity.ExactlyOne, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample, SlowTestHistoryMinSampleOptionDescriptionFormatted, ArgumentArity.ExactlyOne, false), + new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier, SlowTestHistoryMultiplierOptionDescriptionFormatted, ArgumentArity.ExactlyOne, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter, StackFrameFilterOptionDescriptionFormatted, ArgumentArity.OneOrMore, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSummary, AzureDevOpsResources.SummaryOptionDescription, ArgumentArity.ZeroOrOne, false), new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsUploadArtifactExclude, AzureDevOpsResources.UploadArtifactExcludeOptionDescription, ArgumentArity.ZeroOrMore, false), @@ -65,6 +78,9 @@ public override Task ValidateOptionArgumentsAsync(CommandLineO => commandOption.Name switch { AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments), + AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory => ValidateSlowTestHistoryArgumentsAsync(arguments), + AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample => ValidateSlowTestHistoryMinSampleArgumentsAsync(arguments), + AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier => ValidateSlowTestHistoryMultiplierArgumentsAsync(arguments), AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity when !SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase) => ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0])), AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter => ValidateStackFrameFilterArgumentsAsync(arguments), @@ -96,6 +112,10 @@ public override Task ValidateCommandLineOptionsAsync(ICommandL { errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps; } + else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryRequiresAzureDevOps; + } else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter)) { errorMessage = AzureDevOpsResources.AzureDevOpsStackFrameFilterRequiresAzureDevOps; @@ -111,6 +131,22 @@ public override Task ValidateCommandLineOptionsAsync(ICommandL errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory; } + // The slow-test-history sub-options depend on '--report-azdo-slow-test-history' regardless of whether + // '--report-azdo' itself is set, so these checks live outside the '--report-azdo' branch above. + if (errorMessage is null + && commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample) + && !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryMinSampleRequiresSlowTestHistory; + } + + if (errorMessage is null + && commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier) + && !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory)) + { + errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryMultiplierRequiresSlowTestHistory; + } + if (errorMessage is null && HasArtifactUploadConfiguration(commandLineOptions) && IsArtifactUploadDisabled(commandLineOptions)) { errorMessage = AzureDevOpsResources.ArtifactUploadOptionsRequireUploadArtifacts; @@ -175,6 +211,24 @@ private static Task ValidateFlakyHistoryArgumentsAsync(string[ ? ValidationResult.ValidTask : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0])); + private static Task ValidateSlowTestHistoryArgumentsAsync(string[] arguments) + => int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days) + && days is >= 1 and <= 90 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryDays, arguments[0])); + + private static Task ValidateSlowTestHistoryMinSampleArgumentsAsync(string[] arguments) + => int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int minimum) + && minimum >= 1 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryMinSample, arguments[0])); + + private static Task ValidateSlowTestHistoryMultiplierArgumentsAsync(string[] arguments) + => double.TryParse(arguments[0], NumberStyles.Float, CultureInfo.InvariantCulture, out double multiplier) + && multiplier is > 0 and <= 10_000 + ? ValidationResult.ValidTask + : ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryMultiplier, arguments[0])); + private static Task ValidateStackFrameFilterArgumentsAsync(string[] arguments) { if (arguments.Length > MaxStackFrameFilterPatterns) diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs index afd1d62ccb..e59c4896ce 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs @@ -44,6 +44,17 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder) serviceProvider.GetTestApplicationModuleInfo(), serviceProvider.GetLoggerFactory())); + var compositeSlowTestReporter = + new CompositeExtensionFactory(serviceProvider => + new AzureDevOpsSlowTestReporter( + serviceProvider.GetCommandLineOptions(), + serviceProvider.GetEnvironment(), + serviceProvider.GetOutputDevice(), + serviceProvider.GetTask(), + serviceProvider.GetClock(), + serviceProvider.GetLoggerFactory(), + historyService ??= CreateHistoryService(serviceProvider))); + var compositeLogGroupReporter = new CompositeExtensionFactory(serviceProvider => new AzureDevOpsLogGroupReporter( @@ -81,12 +92,14 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder) }); builder.TestHost.AddDataConsumer(compositeArtifactUploader); builder.TestHost.AddDataConsumer(compositeSummaryReporter); + builder.TestHost.AddDataConsumer(compositeSlowTestReporter); builder.TestHost.AddDataConsumer(compositeTestResultsPublisher); builder.TestHost.AddDataConsumer(compositeLogGroupReporter); builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider => historyService ??= CreateHistoryService(serviceProvider)); builder.TestHost.AddTestSessionLifetimeHandler(compositeArtifactUploader); builder.TestHost.AddTestSessionLifetimeHandler(compositeSummaryReporter); + builder.TestHost.AddTestSessionLifetimeHandler(compositeSlowTestReporter); builder.TestHost.AddTestSessionLifetimeHandler(compositeTestResultsPublisher); // Registered last so its OnTestSessionFinishingAsync (the closing ##[endgroup]) runs after diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs index ef33aade64..4cba121c1f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs @@ -261,7 +261,7 @@ private static async Task ParseResultsAsync(HttpResp : new AzureDevOpsTestResultsPage( [.. payload.Value .Where(static result => !RoslynString.IsNullOrWhiteSpace(result.AutomatedTestName) && !RoslynString.IsNullOrWhiteSpace(result.Outcome)) - .Select(static result => new AzureDevOpsTestResult(result.AutomatedTestName!, result.Outcome!))], + .Select(static result => new AzureDevOpsTestResult(result.AutomatedTestName!, result.Outcome!, result.DurationInMs))], continuationToken); } @@ -357,6 +357,9 @@ internal sealed class AzureDevOpsResultResponse [JsonPropertyName("outcome")] public string? Outcome { get; set; } + + [JsonPropertyName("durationInMs")] + public double? DurationInMs { get; set; } } internal sealed class AzureDevOpsResultsResponse @@ -367,15 +370,18 @@ internal sealed class AzureDevOpsResultsResponse internal sealed class AzureDevOpsTestResult { - public AzureDevOpsTestResult(string automatedTestName, string outcome) + public AzureDevOpsTestResult(string automatedTestName, string outcome, double? durationMilliseconds = null) { AutomatedTestName = automatedTestName; Outcome = outcome; + DurationMilliseconds = durationMilliseconds; } public string AutomatedTestName { get; } public string Outcome { get; } + + public double? DurationMilliseconds { get; } } internal sealed class AzureDevOpsTestResultsPage diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs index 0e5e82e272..b7a2b22984 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryService.cs @@ -23,8 +23,13 @@ internal sealed class AzureDevOpsHistoryService : ITestSessionLifetimeHandler, I // NOTE: Bound per-run paging so a single large run cannot keep session startup busy indefinitely. private const int MaxResultPagesPerRun = 50; + // NOTE: Bound the number of duration samples retained per test so the slow-test history feature + // cannot grow memory without limit on a heavily-run test; p95/p99 are stable well below this cap. + private const int MaxDurationSamplesPerTest = 1000; + private static readonly TimeSpan HistoryLoadBudget = TimeSpan.FromSeconds(30); private static readonly IReadOnlyDictionary EmptyStatsByTest = new Dictionary(StringComparer.OrdinalIgnoreCase); + private static readonly IReadOnlyDictionary EmptyDurationStatsByTest = new Dictionary(StringComparer.OrdinalIgnoreCase); private readonly ICommandLineOptions _commandLineOptions; private readonly IEnvironment _environment; @@ -33,7 +38,9 @@ internal sealed class AzureDevOpsHistoryService : ITestSessionLifetimeHandler, I private readonly ITask _task; private readonly ILogger _logger; private int _historyWindowInDays; + private bool _collectDurations; private IReadOnlyDictionary _statsByTest = EmptyStatsByTest; + private IReadOnlyDictionary _durationStatsByTest = EmptyDurationStatsByTest; public AzureDevOpsHistoryService( ICommandLineOptions commandLineOptions, @@ -63,16 +70,19 @@ public AzureDevOpsHistoryService( public Task IsEnabledAsync() => Task.FromResult(_commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) - && _commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory)); + && (_commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory) + || _commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory))); public async Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) { if (!_commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) - || !TryGetHistoryWindowInDays(out int historyWindowInDays)) + || !TryGetHistoryConfiguration(out int historyWindowInDays, out bool collectDurations)) { return; } + _collectDurations = collectDurations; + if (!AzureDevOpsConstants.IsRunningInAzureDevOps(_environment)) { return; @@ -152,6 +162,18 @@ public bool IsLikelyFlaky(string testName, double threshold) && stats.TotalCount > 0 && stats.FailureRate >= threshold; + public bool TryGetDurationStats(string testName, out DurationHistoryStats stats) + { + if (RoslynString.IsNullOrWhiteSpace(testName)) + { + stats = default; + return false; + } + + // NOTE: Callers must only query stats after test-session start completes; the published snapshot is empty until then. + return Volatile.Read(ref _durationStatsByTest).TryGetValue(testName, out stats); + } + private async Task LoadHistoryAsync(AzureDevOpsHistoryQuery query, int historyWindowInDays, CancellationToken cancellationToken) { IReadOnlyList runs = await _historyClient.GetRunsAsync(query, MaxRunsToInspect + 1, cancellationToken).ConfigureAwait(false); @@ -162,18 +184,24 @@ private async Task LoadHistoryAsync(AzureDevOpsHistoryQuery query, int historyWi } var counts = new Dictionary(StringComparer.OrdinalIgnoreCase); +#pragma warning disable IDE0028 // Collection initialization can be simplified - the comparer cannot be passed via a collection expression. + Dictionary>? durations = _collectDurations + ? new Dictionary>(StringComparer.OrdinalIgnoreCase) + : null; +#pragma warning restore IDE0028 foreach (AzureDevOpsTestRun run in runs) { - await AggregateRunResultsAsync(query, run, counts, cancellationToken).ConfigureAwait(false); + await AggregateRunResultsAsync(query, run, counts, durations, cancellationToken).ConfigureAwait(false); } - PublishHistoryStats(historyWindowInDays, counts); + PublishHistoryStats(historyWindowInDays, counts, durations); } private async Task AggregateRunResultsAsync( AzureDevOpsHistoryQuery query, AzureDevOpsTestRun run, Dictionary counts, + Dictionary>? durations, CancellationToken cancellationToken) { string? continuationToken = null; @@ -197,6 +225,20 @@ private async Task AggregateRunResultsAsync( "Failed" => (currentCount.PassCount, currentCount.FailCount + 1), _ => currentCount, }; + + if (durations is not null && result.DurationMilliseconds is double durationMs && durationMs > 0) + { + if (!durations.TryGetValue(result.AutomatedTestName, out List? samples)) + { + samples = []; + durations[result.AutomatedTestName] = samples; + } + + if (samples.Count < MaxDurationSamplesPerTest) + { + samples.Add(durationMs); + } + } } if (page.Results.Count == 0) @@ -229,7 +271,7 @@ private async Task AggregateRunResultsAsync( _logger.LogWarning(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.FlakyHistoryResultsPagingStoppedWarning, run.Url, MaxResultPagesPerRun)); } - private void PublishHistoryStats(int historyWindowInDays, Dictionary counts) + private void PublishHistoryStats(int historyWindowInDays, Dictionary counts, Dictionary>? durations) { Volatile.Write(ref _historyWindowInDays, historyWindowInDays); IReadOnlyDictionary publishedStats = counts.ToDictionary( @@ -239,6 +281,23 @@ private void PublishHistoryStats(int historyWindowInDays, Dictionary(durations.Count, StringComparer.OrdinalIgnoreCase); + foreach (KeyValuePair> entry in durations) + { + if (DurationHistoryStats.TryCreate(entry.Value, out DurationHistoryStats stats)) + { + publishedDurationStats[entry.Key] = stats; + } + } + + Volatile.Write(ref _durationStatsByTest, publishedDurationStats); } private void ResetHistoryState() @@ -247,14 +306,42 @@ private void ResetHistoryState() // NOTE: Volatile.Write on a reference atomically publishes the empty snapshot when history loading is skipped or fails. Volatile.Write(ref _statsByTest, EmptyStatsByTest); + Volatile.Write(ref _durationStatsByTest, EmptyDurationStatsByTest); } - private bool TryGetHistoryWindowInDays(out int historyWindowInDays) + private bool TryGetHistoryConfiguration(out int historyWindowInDays, out bool collectDurations) { historyWindowInDays = 0; - return _commandLineOptions.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, out string[]? arguments) + collectDurations = false; + bool anyEnabled = false; + + // Flaky-history and slow-test-history share a single Azure DevOps query to avoid paying for the fetch + // twice. When both are enabled with different windows we deliberately query the *maximum* of the two + // windows so neither feature is starved of data; the narrower feature simply ignores the extra days. + // This means the flaky "in last {N}d" annotation reflects the effective (maximum) window, not necessarily + // the value passed to --report-azdo-flaky-history when the slow-test window is larger. + if (TryGetWindowInDays(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, out int flakyWindow)) + { + anyEnabled = true; + historyWindowInDays = Math.Max(historyWindowInDays, flakyWindow); + } + + if (TryGetWindowInDays(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory, out int slowWindow)) + { + anyEnabled = true; + collectDurations = true; + historyWindowInDays = Math.Max(historyWindowInDays, slowWindow); + } + + return anyEnabled; + } + + private bool TryGetWindowInDays(string optionName, out int windowInDays) + { + windowInDays = 0; + return _commandLineOptions.TryGetOptionArgumentList(optionName, out string[]? arguments) && arguments is [string value] - && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out historyWindowInDays); + && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out windowInDays); } private bool TryCreateQuery(int historyWindowInDays, [NotNullWhen(true)] out AzureDevOpsHistoryQuery? query) diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs index ede01491c1..0471c3dae8 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsReporter.cs @@ -21,7 +21,6 @@ internal sealed class AzureDevOpsReporter : { internal const double KnownFlakyFailureRateThreshold = 0.25; private const string DeterministicBuildRoot = "/_/"; - private const string FullyQualifiedNamePropertyKey = "vstest.TestCase.FullyQualifiedName"; private const int MinSamplesForRegressionAnnotation = 5; private const string QuarantineBuildTagLine = "##vso[build.addbuildtag]has-quarantined-test-failure"; private const string WarningSeverity = "warning"; @@ -434,10 +433,7 @@ private Regex[] LoadUserStackFrameFilters() } private static string GetTestName(TestNode testNode) - => testNode.Properties - .OfType() - .FirstOrDefault(static property => property.Key == FullyQualifiedNamePropertyKey)?.Value - ?? testNode.DisplayName; + => TestNodeIdentity.GetTestName(testNode); /// /// Formats the reporter message so the test name lands on its own line. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs new file mode 100644 index 0000000000..b10b0c1353 --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestReporter.cs @@ -0,0 +1,314 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.AzureDevOpsReport.Resources; +using Microsoft.Testing.Extensions.Reporting; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Extensions.TestHost; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +/// +/// Surfaces tests that are still running past a per-test threshold as durable scrollback lines, lowering the +/// threshold for tests with a known-short historical runtime and decorating each emission with the historical +/// p95/p99 fetched from Azure DevOps test history. +/// +/// +/// This is a self-contained emitter for the Azure DevOps host. Once the platform-level IProgressEnricher +/// hook (issue #9139) ships, the surfacing/backoff logic should migrate onto it and this type should only +/// supply the history-driven threshold and decoration. +/// +internal sealed class AzureDevOpsSlowTestReporter : IDataConsumer, ITestSessionLifetimeHandler, IOutputDeviceDataProducer +{ + private const string AzureDevOpsTfBuildVariableName = "TF_BUILD"; + private static readonly TimeSpan ScanInterval = TimeSpan.FromSeconds(1); + + private readonly ICommandLineOptions _commandLineOptions; + private readonly IEnvironment _environment; + private readonly IOutputDevice _outputDevice; + private readonly ITask _task; + private readonly IClock _clock; + private readonly ILogger _logger; + private readonly IAzureDevOpsHistoryService _historyService; + private readonly ConcurrentDictionary _inProgress = new(StringComparer.Ordinal); + private readonly bool _isEnabled; + private readonly TimeSpan _staticThreshold; + + private double _multiplier; + private volatile int _minimumSampleCount; + private volatile bool _active; + private CancellationTokenSource? _loopCancellationTokenSource; + private Task? _loopTask; + + public AzureDevOpsSlowTestReporter( + ICommandLineOptions commandLineOptions, + IEnvironment environment, + IOutputDevice outputDevice, + ITask task, + IClock clock, + ILoggerFactory loggerFactory, + IAzureDevOpsHistoryService historyService) + { + _commandLineOptions = commandLineOptions; + _environment = environment; + _outputDevice = outputDevice; + _task = task; + _clock = clock; + _logger = loggerFactory.CreateLogger(); + _historyService = historyService; + _staticThreshold = TimeSpan.FromSeconds(AzureDevOpsCommandLineOptions.SlowTestStaticThresholdSeconds); + _isEnabled = commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsOptionName) + && commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory); + } + + public Type[] DataTypesConsumed { get; } = [typeof(TestNodeUpdateMessage)]; + + public string Uid => nameof(AzureDevOpsSlowTestReporter); + + public string Version => ExtensionVersion.DefaultSemVer; + + public string DisplayName => AzureDevOpsResources.DisplayName; + + public string Description => AzureDevOpsResources.Description; + + public Task IsEnabledAsync() => Task.FromResult(_isEnabled); + + public async Task OnTestSessionStartingAsync(ITestSessionContext testSessionContext) + { + try + { + testSessionContext.CancellationToken.ThrowIfCancellationRequested(); + + _active = false; + _inProgress.Clear(); + + if (!_isEnabled) + { + return; + } + + if (!string.Equals(_environment.GetEnvironmentVariable(AzureDevOpsTfBuildVariableName), "true", StringComparison.OrdinalIgnoreCase)) + { + // Outside Azure DevOps the feature truly no-ops: we only leave a low-noise trace and never + // surface an output-device line, so local/dev runs that happen to pass the option stay quiet. + if (_logger.IsEnabled(LogLevel.Trace)) + { + _logger.LogTrace(AzureDevOpsResources.SlowTestHistoryRequiresTfBuildWarning); + } + + return; + } + + // 'double' cannot be marked 'volatile', so publish the multiplier through Volatile.Write; the + // remaining fields use the 'volatile' modifier. Writing _active = true last (below) acts as the + // release fence that publishes all three to the test-data-producer threads in ConsumeAsync. + Volatile.Write(ref _multiplier, GetMultiplier()); + _minimumSampleCount = GetMinimumSampleCount(); + + _loopCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(testSessionContext.CancellationToken); + _active = true; + _loopTask = _task.RunLongRunning(() => ScanLoopAsync(_loopCancellationTokenSource.Token), nameof(AzureDevOpsSlowTestReporter), _loopCancellationTokenSource.Token); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionStartingAsync), ex); + } + } + + public Task ConsumeAsync(IDataProducer dataProducer, IData value, CancellationToken cancellationToken) + { + try + { + cancellationToken.ThrowIfCancellationRequested(); + + if (!_active || value is not TestNodeUpdateMessage update) + { + return Task.CompletedTask; + } + + string uid = update.TestNode.Uid; + TestNodeStateProperty? state = update.TestNode.Properties.SingleOrDefault(); + if (state is InProgressTestNodeStateProperty) + { + string testName = TestNodeIdentity.GetTestName(update.TestNode); + TimeSpan threshold = ResolveThreshold(testName); + _inProgress[uid] = new InProgressTest(testName, _clock.UtcNow, threshold); + } + else if (state is not null) + { + // Any non-in-progress state (passed/failed/skipped/error/timeout/cancelled) is terminal for surfacing. + _inProgress.TryRemove(uid, out _); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception ex) + { + LogUnexpectedException(nameof(ConsumeAsync), ex); + } + + return Task.CompletedTask; + } + + public async Task OnTestSessionFinishingAsync(ITestSessionContext testSessionContext) + { + _active = false; + + CancellationTokenSource? loopCancellationTokenSource = _loopCancellationTokenSource; + if (loopCancellationTokenSource is not null) + { +#pragma warning disable VSTHRD103 // CancelAsync is unavailable on all target frameworks. + loopCancellationTokenSource.Cancel(); +#pragma warning restore VSTHRD103 + } + + if (_loopTask is not null) + { + try + { + await _loopTask.ConfigureAwait(false); + } + catch (OperationCanceledException) + { + // Expected during normal shutdown: cancelling _loopCancellationTokenSource above unblocks the + // scan loop, which surfaces as a cancellation here. Nothing to do — swallow and finish teardown. + } + catch (Exception ex) + { + LogUnexpectedException(nameof(OnTestSessionFinishingAsync), ex); + } + } + + loopCancellationTokenSource?.Dispose(); + _loopCancellationTokenSource = null; + _loopTask = null; + _inProgress.Clear(); + } + + private async Task ScanLoopAsync(CancellationToken cancellationToken) + { + while (!cancellationToken.IsCancellationRequested) + { + try + { + await _task.Delay(ScanInterval, cancellationToken).ConfigureAwait(false); + await ScanOnceAsync(_clock.UtcNow, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException) + { + return; + } + } + } + + // Internal for unit testing: performs a single surfacing pass at the given 'now' so tests can drive + // the emission/backoff logic deterministically without relying on the timer-driven loop. + internal async Task ScanOnceAsync(DateTimeOffset now, CancellationToken cancellationToken) + { + foreach (KeyValuePair entry in _inProgress) + { + InProgressTest test = entry.Value; + TimeSpan elapsed = now - test.StartTime; + if (elapsed < test.NextEmitThreshold) + { + continue; + } + + // Exponential backoff so a genuinely stuck test does not spam the log: T, 2T, 4T, ... + // Clamp at TimeSpan.MaxValue so a very long-running test cannot overflow Ticks * 2 into a + // negative value (which would make the backoff fire on every scan). + long currentTicks = test.NextEmitThreshold.Ticks; + test.NextEmitThreshold = currentTicks > TimeSpan.MaxValue.Ticks / 2 + ? TimeSpan.MaxValue + : TimeSpan.FromTicks(currentTicks * 2); + + try + { + await EmitSlowTestAsync(test, elapsed, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OperationCanceledException) + { + LogUnexpectedException(nameof(ScanOnceAsync), ex); + } + } + } + + private async Task EmitSlowTestAsync(InProgressTest test, TimeSpan elapsed, CancellationToken cancellationToken) + { + string elapsedText = AzureDevOpsSlowTestThresholds.FormatDuration(elapsed.TotalMilliseconds); + string line = string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.SlowTestStillRunning, elapsedText, test.TestName); + + if (_historyService.TryGetDurationStats(test.TestName, out DurationHistoryStats stats) + && AzureDevOpsSlowTestThresholds.HasUsableHistory(stats, hasStats: true, _minimumSampleCount)) + { + string decoration = string.Format( + CultureInfo.InvariantCulture, + AzureDevOpsResources.SlowTestHistoryDecoration, + AzureDevOpsSlowTestThresholds.FormatDuration(stats.P95Milliseconds), + AzureDevOpsSlowTestThresholds.FormatDuration(stats.P99Milliseconds), + stats.SampleCount.ToString(CultureInfo.InvariantCulture)); + line = $"{line} {decoration}"; + } + + await _outputDevice.DisplayAsync(this, new FormattedTextOutputDeviceData(line), cancellationToken).ConfigureAwait(false); + } + + private TimeSpan ResolveThreshold(string testName) + { + bool hasStats = _historyService.TryGetDurationStats(testName, out DurationHistoryStats stats); + return AzureDevOpsSlowTestThresholds.ComputeThreshold(_staticThreshold, stats, hasStats, Volatile.Read(ref _multiplier), _minimumSampleCount); + } + + private double GetMultiplier() + => _commandLineOptions.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier, out string[]? arguments) + && arguments is [string value] + && double.TryParse(value, NumberStyles.Float, CultureInfo.InvariantCulture, out double multiplier) + && multiplier > 0 + ? multiplier + : AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMultiplier; + + private int GetMinimumSampleCount() + => _commandLineOptions.TryGetOptionArgumentList(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample, out string[]? arguments) + && arguments is [string value] + && int.TryParse(value, NumberStyles.Integer, CultureInfo.InvariantCulture, out int minimum) + && minimum >= 1 + ? minimum + : AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMinSample; + + private void LogUnexpectedException(string callbackName, Exception ex) + { + if (_logger.IsEnabled(LogLevel.Warning)) + { + _logger.LogWarning($"Unexpected exception in {callbackName}: {ex}"); + } + } + + private sealed class InProgressTest + { + public InProgressTest(string testName, DateTimeOffset startTime, TimeSpan threshold) + { + TestName = testName; + StartTime = startTime; + NextEmitThreshold = threshold; + } + + public string TestName { get; } + + public DateTimeOffset StartTime { get; } + + public TimeSpan NextEmitThreshold { get; set; } + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestThresholds.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestThresholds.cs new file mode 100644 index 0000000000..a17a8a411c --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsSlowTestThresholds.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +/// +/// Pure decision logic for the history-driven slow-test threshold feature. Kept free of any +/// platform dependency so it can be exercised directly by unit tests. +/// +internal static class AzureDevOpsSlowTestThresholds +{ + /// + /// Returns the effective slow-test threshold for a test. When sufficient history is available the + /// threshold is lowered to p99 * multiplier, but never raised above the static default. + /// + /// The static fallback threshold (e.g. 60s). + /// The historical duration statistics, when available. + /// Whether holds usable history. + /// The multiplier applied to the historical p99. + /// The minimum number of samples before history is trusted. + public static TimeSpan ComputeThreshold(TimeSpan staticThreshold, in DurationHistoryStats stats, bool hasStats, double multiplier, int minimumSampleCount) + { + if (!hasStats || stats.SampleCount < minimumSampleCount || stats.P99Milliseconds <= 0 || multiplier <= 0) + { + return staticThreshold; + } + + double historyThresholdMs = stats.P99Milliseconds * multiplier; + + // The effective threshold is min(static, history), so any history value that is not strictly below the + // static threshold falls back to the static one without a TimeSpan conversion. Short-circuiting here also + // guards against TimeSpan.FromMilliseconds throwing: it covers NaN, +Infinity, and any large *finite* + // value that would exceed TimeSpan.MaxValue.TotalMilliseconds and overflow during conversion. + return double.IsNaN(historyThresholdMs) || historyThresholdMs <= 0 || historyThresholdMs >= staticThreshold.TotalMilliseconds + ? staticThreshold + : TimeSpan.FromMilliseconds(historyThresholdMs); + } + + /// + /// Indicates whether the history decoration should be emitted for the given statistics. + /// + public static bool HasUsableHistory(in DurationHistoryStats stats, bool hasStats, int minimumSampleCount) + => hasStats && stats.SampleCount >= minimumSampleCount && stats.P99Milliseconds > 0; + + /// + /// Formats a duration (in milliseconds) into a compact human-readable string such as 2s, 2.5s, or 500ms. + /// + public static string FormatDuration(double milliseconds) + { + if (milliseconds < 0) + { + milliseconds = 0; + } + + return milliseconds >= 60_000 + ? TrimTrailingZero(milliseconds / 60_000.0) + "m" + : milliseconds >= 1_000 + ? TrimTrailingZero(milliseconds / 1_000.0) + "s" + : ((long)Math.Round(milliseconds)).ToString(CultureInfo.InvariantCulture) + "ms"; + } + + private static string TrimTrailingZero(double value) + { + // One decimal place is enough resolution for a heartbeat line; drop a trailing ".0". + double rounded = Math.Round(value, 1, MidpointRounding.AwayFromZero); + long whole = (long)Math.Round(rounded, 0, MidpointRounding.AwayFromZero); + return Math.Abs(rounded - whole) < 1e-9 + ? whole.ToString(CultureInfo.InvariantCulture) + : rounded.ToString("0.0", CultureInfo.InvariantCulture); + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/DurationHistoryStats.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/DurationHistoryStats.cs new file mode 100644 index 0000000000..eb76be7e3b --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/DurationHistoryStats.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +/// +/// Immutable per-test historical duration statistics derived from Azure DevOps test result history. +/// +internal readonly struct DurationHistoryStats +{ + public DurationHistoryStats(double p95Milliseconds, double p99Milliseconds, int sampleCount) + { + P95Milliseconds = p95Milliseconds; + P99Milliseconds = p99Milliseconds; + SampleCount = sampleCount; + } + + public double P95Milliseconds { get; } + + public double P99Milliseconds { get; } + + public int SampleCount { get; } + + /// + /// Builds the statistics from a collection of per-run durations (in milliseconds). + /// + /// The per-run durations. Non-positive values are ignored. + /// The computed statistics when at least one positive sample is present. + /// when at least one positive sample is present; otherwise . + public static bool TryCreate(IReadOnlyList durationsMilliseconds, out DurationHistoryStats stats) + { + var samples = new List(durationsMilliseconds.Count); + foreach (double duration in durationsMilliseconds) + { + if (duration > 0) + { + samples.Add(duration); + } + } + + if (samples.Count == 0) + { + stats = default; + return false; + } + + samples.Sort(); + stats = new DurationHistoryStats( + PercentileCalculator.Compute(samples, 95), + PercentileCalculator.Compute(samples, 99), + samples.Count); + return true; + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs index c87575d9f6..eed490f791 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/IAzureDevOpsHistoryService.cs @@ -10,4 +10,6 @@ internal interface IAzureDevOpsHistoryService bool TryGetStats(string testName, out FlakyStats stats); bool IsLikelyFlaky(string testName, double threshold); + + bool TryGetDurationStats(string testName, out DurationHistoryStats stats); } diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PercentileCalculator.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PercentileCalculator.cs new file mode 100644 index 0000000000..4ac4d74d0e --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/PercentileCalculator.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +/// +/// Computes percentiles from a set of samples using the nearest-rank method. +/// Azure DevOps Analytics does not expose percentile metrics directly, so the slow-test +/// history feature fetches raw per-run durations and computes p95/p99 client-side. +/// +internal static class PercentileCalculator +{ + /// + /// Computes the percentile value using the nearest-rank method. + /// + /// Samples sorted in ascending order. Must not be empty. + /// The percentile to compute, in the inclusive range (0, 100]. + /// The sample at the nearest rank for the requested percentile. + public static double Compute(IReadOnlyList sortedSamples, double percentile) + { + if (sortedSamples.Count == 0) + { + throw new ArgumentException("At least one sample is required.", nameof(sortedSamples)); + } + + if (percentile is <= 0 or > 100) + { + throw new ArgumentOutOfRangeException(nameof(percentile), percentile, "Percentile must be in the range (0, 100]."); + } + + // Nearest-rank: rank = ceil(p/100 * n), 1-based; index = rank - 1, clamped to the array bounds. + int rank = (int)Math.Ceiling(percentile / 100.0 * sortedSamples.Count); + int index = Math.Min(sortedSamples.Count - 1, Math.Max(0, rank - 1)); + return sortedSamples[index]; + } +} diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx index d3c20a64ce..9e21aae3e9 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/AzureDevOpsResources.resx @@ -352,4 +352,52 @@ Upload test result files and/or add build tags to Azure DevOps. Options are: off (default), tags-only, files, and all. {Locked="off"}{Locked="tags-only"}{Locked="files"}{Locked="all"} + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + + + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf index 6620341d6a..b10da00025 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.cs.xlf @@ -117,6 +117,21 @@ --report-azdo-severity vyžaduje, aby byla povolena možnost --report-azdo {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled --report-azdo-stackframe-filter vyžaduje povolení --report-azdo @@ -202,6 +217,21 @@ Neplatná možnost {0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Neplatný regulární výraz --report-azdo-stackframe-filter {0}: {1} @@ -272,6 +302,36 @@ Závažnost, která se má použít pro hlášenou událost. Možnosti: error (výchozí) a warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Další vzory regulárních výrazů (porovnávané s plně kvalifikovanou předponou typu každého bloku zásobníku), které se mají přeskočit při hledání místa volání uživatele k anotaci. Opakovatelné; až tolik vzorů: {0}. Kompilováno s časovým limitem shody {1} ms. Doplňkové k vestavěným předponám implementace kontrolních výrazů MSTest v rozšíření. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf index 47e25b8e5c..216a053da5 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.de.xlf @@ -117,6 +117,21 @@ Für „--report-azdo-severity“ muss „--report-azdo“ aktiviert sein. {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled Für „--report-azdo-stackframe-filter“ muss „--report-azdo“ aktiviert sein @@ -202,6 +217,21 @@ Ungültige Option „{0}“. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Ungültiger „--report-azdo-stackframe-filter“ RegEx „{0}“: {1} @@ -272,6 +302,36 @@ Schweregrad, der für das gemeldete Ereignis verwendet werden soll. Optionen sind: error (Standard) und warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Zusätzliche RegEx-Muster (abgeglichen mit dem vollqualifizierten Typpräfix jedes Stapelrahmens), die übersprungen werden sollten, wenn nach der Aufrufwebsite des Benutzers gesucht wird, die kommentiert werden soll. Wiederholbar; bis hin zu {0} Mustern. Kompiliert mit einem {1} ms-Übereinstimmungstimeout. Additiv zu den integrierten MSTest-Assertionsimplementierungspräfixen der Erweiterung. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf index ce0f94dd73..def1b7213f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.es.xlf @@ -117,6 +117,21 @@ "--report-azdo-severity" requiere que se habilite "--report-azdo" {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled "--report-azdo-stackframe-filter" requiere que se habilite "--report-azdo" @@ -202,6 +217,21 @@ Opción no válida {0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Expresión regular no válida de "--report-azdo-stackframe-filter" "{0}": {1} @@ -272,6 +302,36 @@ Gravedad que se va a usar para el evento notificado. Las opciones son: error (valor predeterminado) y warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Patrones de expresiones regulares adicionales (que coinciden con el prefijo del tipo completamente calificado de cada marco de pila) que se deben omitir al buscar el punto de llamada del usuario que se va a anotar. Repetible; hasta {0} patrones. Compilado con un {1}tiempo de espera de coincidencia ms. Adición a los prefijos de implementación de aserción de MSTest integrados de la extensión. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf index d853bceffd..7d33561a32 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.fr.xlf @@ -117,6 +117,21 @@ « --report-azdo-severity » nécessite l’activation de « --report-azdo » {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled « --report-azdo-stackframe-filter » nécessite l’activation de « --report-azdo » @@ -202,6 +217,21 @@ Option {0} non valide. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Expression régulière « {0} » non valide pour « --report-azdo-stackframe-filter » : {1} @@ -272,6 +302,36 @@ Gravité à utiliser pour l’événement signalé. Les options sont : error (par défaut) et warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Modèles d’expression régulière supplémentaires (mis en correspondance avec le préfixe complet du type de chaque frame de pile) à ignorer lors de la recherche du site d’appel de l’utilisateur à annoter. Répétable, jusqu’à {0} modèles. Compilé avec un délai d’expiration de correspondance de {1} ms. S’ajoute aux préfixes d’implémentation d’assertion MSTest intégrés de l’extension. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf index f33258fcf0..f5f83bfdef 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.it.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' richiede l'autenticazione di '--report-azdo' {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled "--report-azdo-stackframe-filter" richiede che "--report-azdo" sia abilitato @@ -202,6 +217,21 @@ Opzione non valida {0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Espressione regolare non valida per "--report-azdo-stackframe-filter" "{0}": {1} @@ -272,6 +302,36 @@ Gravità da usare per l'evento segnalato. Le opzioni sono: error (impostazione predefinita) e warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Altri modelli di espressioni regolari (confrontati con il prefisso di tipo completo di ogni stack frame) da ignorare quando si cerca il sito di chiamata dell'utente da annotare. Ripetibile; fino a {0} criteri. Compilato con un timeout di corrispondenza di {1} ms. Aggiuntivo ai prefissi di implementazione delle asserzioni MSTest integrati nell'estensione. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf index 4906a1ef6a..4490ab4acf 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ja.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' では、'--report-azdo' を有効にする必要があります {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter' では、'--report-azdo' を有効にする必要があります @@ -202,6 +217,21 @@ オプション {0} が無効です。 {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} 無効 '--report-azdo-stackframe-filter' regex '{0}': {1} @@ -272,6 +302,36 @@ 報告されたイベントに使用する重大度。オプションは、error (既定) と warning です。 {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. 注釈を付けるユーザーの呼び出しサイトを探す際にスキップすべき追加の正規表現パターン (各スタック フレームの完全修飾型プレフィックスに一致するもの)。繰り返し指定可能。最大 {0} 個のパターンです。{1} ミリ秒の一致タイムアウトでコンパイルされました。拡張機能に組み込まれている MSTest のアサーション実装プレフィックスに追加されます。 diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf index 47ee030429..820b464aaa 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ko.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity'를 사용하려면 '--report-azdo'를 사용 설정 해야합니다. {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter'를 사용하려면 '--report-azdo' 활성화 필요 @@ -202,6 +217,21 @@ {0} 옵션이 잘못되었습니다. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} 잘못된 '--report-azdo-stackframe-filter' 정규식 '{0}': {1} @@ -272,6 +302,36 @@ 보고된 이벤트에 사용할 심각도입니다. 옵션은 error(기본값)와 warning, 두 가지입니다. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. 주석을 달 사용자의 호출 위치를 찾을 때 건너뛰어야 하는 추가 정규식 패턴(각 스택 프레임의 정규화된 형식 접두사와 일치)입니다. 반복 가능하며 최대 {0}개 패턴을 지정할 수 있습니다. {1}ms 일치 시간 제한으로 컴파일됩니다. 확장에 기본 제공되는 MSTest 어설션 구현 접두사에 추가됩니다. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf index 081222c1b6..8786286458 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pl.xlf @@ -117,6 +117,21 @@ Element „--report-azdo-severity” wymaga włączenia polecenia „--report-azdo” {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled Opcja „--report-azdo-stackframe-filter” wymaga włączenia opcji „--report-azdo” @@ -202,6 +217,21 @@ Nieprawidłowa opcja {0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Nieprawidłowy wyrażenie regularne „--report-azdo-stackframe-filter” „{0}”: {1} @@ -272,6 +302,36 @@ Ważność do użycia dla zgłoszonego zdarzenia. Dostępne opcje to: error (domyślny) i warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Dodatkowe wzorce wyrażeń regularnych (dopasowywane do w pełni kwalifikowanego prefiksu typu każdej ramki stosu), które należy pominąć podczas wyszukiwania miejsca wywołania użytkownika do adnotacji. Powtarzalne; maksymalnie {0} wzorów. Kompilowane z limitem czasu dopasowania wynoszącym {1} ms. Dodawane do wbudowanych prefiksów implementacji asercji MSTest w rozszerzeniu. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf index aaf4310fee..c54eee9ef4 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.pt-BR.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' requer que '--report-azdo' esteja habilitado {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter' exige que '--report-azdo' esteja habilitado @@ -202,6 +217,21 @@ Opção {0} inválida. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Regex '--report-azdo-stackframe-filter' inválida '{0}': {1} @@ -272,6 +302,36 @@ A gravidade que será usada para o evento relatado. As opções são: error (padrão) e warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Padrões de regex adicionais (correspondentes ao prefixo de tipo totalmente qualificado de cada quadro da pilha) que devem ser ignorados ao procurar o ponto de chamada do usuário para anotação. Repetível; até {0} padrões. Compilado com tempo limite de correspondência de {1} ms. Adicionais aos prefixos internos de implementação de asserção do MSTest da extensão. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf index 2891a49255..3d70ab5d7b 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.ru.xlf @@ -117,6 +117,21 @@ "--report-azdo-severity" требует включения "--report-azdo" {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled "--report-azdo-stackframe-filter" требует включения "--report-azdo" @@ -202,6 +217,21 @@ Недопустимый параметр {0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Недопустимое регулярное выражение "--report-azdo-stackframe-filter" "{0}": {1} @@ -272,6 +302,36 @@ Уровень серьезности, используемый для события из отчета. Параметры: error (по умолчанию) и warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Дополнительные шаблоны регулярных выражений (сопоставляемые с полным префиксом типа каждого кадра стека), которые следует пропускать при поиске места вызова пользователя для аннотирования. Повторяемые; до {0} шаблонов. Скомпилировано с таймаутом совпадения {1} мс. Дополнение к встроенным префиксам реализации утверждений MSTest в данном расширении. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf index bbbfe62932..dd11936cab 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.tr.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' öğesi '--report-azdo' öğesinin etkinleştirilmesini gerektirir {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter' öğesi '--report-azdo' öğesinin etkinleştirilmesini gerektirir @@ -202,6 +217,21 @@ Geçersiz seçenek{0}. {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} Geçersiz '--report-azdo-stackframe-filter' normal ifade '{0}': {1} @@ -272,6 +302,36 @@ Raporlanan olay için kullanılacak önem derecesi. Seçenekler şunlardır: error (varsayılan) ve warning. {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. Kullanıcının açıklama eklenecek çağrı noktasını ararken atlanması gereken, her yığın çerçevesinin tam nitelikli tür ön ekiyle eşleşen ek normal ifade desenleri. Tekrarlanabilir; en fazla {0} desen. {1} ms eşleşme zaman aşımı ile derlenir. Uzantının yerleşik MSTest onaylama uygulama ön eklerine eklenir. diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf index 54101a00e0..7316a9d81f 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hans.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' 要求启用 '--report-azdo' {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter' 需要启用 '--report-azdo' @@ -202,6 +217,21 @@ 选项 {0} 无效。 {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} '--report-azdo-stackframe-filter' 正则表达式 '{0}' 无效: {1} @@ -272,6 +302,36 @@ 要用于所报告事件的严重性。选项为: error (默认)和 warning。 {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. 查找要批注的用户调用站点时应跳过的其他正则表达式模式(与每个堆栈帧的完全限定类型前缀匹配)。可重复；最多 {0} 个模式。已编译，匹配超时为 {1} 毫秒。追加到扩展的内置 MSTest 断言实现前缀。 diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf index fbcec24885..9683859d21 100644 --- a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/Resources/xlf/AzureDevOpsResources.zh-Hant.xlf @@ -117,6 +117,21 @@ '--report-azdo-severity' 需要啟用 '--report-azdo' {Locked="--report-azdo-severity"}{Locked="--report-azdo"} + + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-min-sample"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + '--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled + {Locked="--report-azdo-slow-test-history-multiplier"}{Locked="--report-azdo-slow-test-history"} + + + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + '--report-azdo-slow-test-history' requires '--report-azdo' to be enabled + {Locked="--report-azdo-slow-test-history"}{Locked="--report-azdo"} + '--report-azdo-stackframe-filter' requires '--report-azdo' to be enabled '--report-azdo-stackframe-filter' 需要啟用 '--report-azdo' to be enabled @@ -202,6 +217,21 @@ 無效的選項 {0}。 {0} is the invalid option value. + + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + Invalid value '{0}' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + Invalid value '{0}' for '--report-azdo-slow-test-history-min-sample'. Provide an integer greater than or equal to 1. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-min-sample"} + + + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + Invalid value '{0}' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000. + {0} is the invalid value. {Locked="--report-azdo-slow-test-history-multiplier"} + Invalid '--report-azdo-stackframe-filter' regex '{0}': {1} 無效的 '--report-azdo-stackframe-filter' regex '{0}': {1} @@ -272,6 +302,36 @@ 要用於所報告事件的嚴重性。選項為: error (預設) warning。 {Locked="error"}{Locked="warning"} + + (historical p95 = {0}, p99 = {1}, samples = {2}) + (historical p95 = {0}, p99 = {1}, samples = {2}) + {0} is the p95 duration. {1} is the p99 duration. {2} is the sample count. + + + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default minimum sample count. {Locked="--report-azdo-slow-test-history"} + + + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to {0}. Requires '--report-azdo-slow-test-history'. + {0} is the default multiplier. {Locked="--report-azdo-slow-test-history"} + + + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + {Locked="--report-azdo"} + + + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + Azure DevOps slow-test history was requested, but TF_BUILD is not set to 'true'; skipping slow-test surfacing. + {Locked="TF_BUILD"}{Locked="true"} + + + [slow] still running after {0}: {1} + [slow] still running after {0}: {1} + {0} is the elapsed duration. {1} is the test name. {Locked="[slow]"} + Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to {0} patterns. Compiled with a {1}ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. 其他 regex 模式 (與每個堆疊框架相符的完整類型前置詞)，而在尋找使用者的呼叫網站進行標註時，應該略過這些模式。可重複; 最多 {0} 個模式。已使用 {1}毫秒比對逾時進行編譯。加至延伸模組內建 MSTest 判斷提示實作前置詞的項目。 diff --git a/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs new file mode 100644 index 0000000000..4eb5d4c76e --- /dev/null +++ b/src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/TestNodeIdentity.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Platform.Extensions.Messages; + +namespace Microsoft.Testing.Extensions.AzureDevOpsReport; + +internal static class TestNodeIdentity +{ + private const string FullyQualifiedNamePropertyKey = "vstest.TestCase.FullyQualifiedName"; + + /// + /// Resolves the stable test name used to match a against Azure DevOps history + /// (which keys results by AutomatedTestName). Falls back to the display name when the + /// fully-qualified name property is unavailable. + /// + public static string GetTestName(TestNode testNode) + => testNode.Properties + .OfType() + .FirstOrDefault(static property => property.Key == FullyQualifiedNamePropertyKey)?.Value + ?? testNode.DisplayName; +} diff --git a/test/Directory.Build.targets b/test/Directory.Build.targets index 4efa79864c..71829b4ab8 100644 --- a/test/Directory.Build.targets +++ b/test/Directory.Build.targets @@ -21,6 +21,11 @@ output device when TF_BUILD is not 'true', unlike the silent report-azdo. Gate it on $(TF_BUILD) so local 'dotnet test' runs stay quiet and only CI pipelines opt into the AzDO output. --> $(TestingPlatformCommandLineArguments) --report-azdo-summary + + $(TestingPlatformCommandLineArguments) --report-azdo-slow-test-history 30 diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AzureDevOpsCommandLineTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AzureDevOpsCommandLineTests.cs index e6dfdbb7b0..52ae1f89ff 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AzureDevOpsCommandLineTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/AzureDevOpsCommandLineTests.cs @@ -32,6 +32,61 @@ public async Task AzureDevOps_WhenSeverityValueIsInvalid_ShouldFail(string tfm) testHostResult.AssertOutputContains("Option '--report-azdo-severity' has invalid arguments: Invalid option invalid."); } + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task AzureDevOps_WhenSlowTestHistoryValueIsInvalid_ShouldFail(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-azdo --report-azdo-slow-test-history 0", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("Option '--report-azdo-slow-test-history' has invalid arguments: Invalid value '0' for '--report-azdo-slow-test-history'. Provide an integer between 1 and 90."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task AzureDevOps_WhenSlowTestHistoryMultiplierIsInvalid_ShouldFail(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-azdo --report-azdo-slow-test-history 30 --report-azdo-slow-test-history-multiplier 0", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("Option '--report-azdo-slow-test-history-multiplier' has invalid arguments: Invalid value '0' for '--report-azdo-slow-test-history-multiplier'. Provide a number greater than 0 and at most 10000."); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task AzureDevOps_WhenSlowTestHistoryUsedWithoutReportAzdo_ShouldFail(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-azdo-slow-test-history 30", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-azdo-slow-test-history' requires '--report-azdo' to be enabled"); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task AzureDevOps_WhenSlowTestHistoryMinSampleUsedWithoutSlowTestHistory_ShouldFail(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-azdo --report-azdo-slow-test-history-min-sample 5", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-azdo-slow-test-history-min-sample' requires '--report-azdo-slow-test-history' to be enabled"); + } + + [DynamicData(nameof(TargetFrameworks.AllForDynamicData), typeof(TargetFrameworks))] + [TestMethod] + public async Task AzureDevOps_WhenSlowTestHistoryMultiplierUsedWithoutSlowTestHistoryAndWithoutReportAzdo_ShouldFail(string tfm) + { + var testHost = TestInfrastructure.TestHost.LocateFrom(AssetFixture.TargetAssetPath, AssetName, tfm); + TestHostResult testHostResult = await testHost.ExecuteAsync("--report-azdo-slow-test-history-multiplier 3", cancellationToken: TestContext.CancellationToken); + + testHostResult.AssertExitCodeIs(ExitCode.InvalidCommandLine); + testHostResult.AssertOutputContains("'--report-azdo-slow-test-history-multiplier' requires '--report-azdo-slow-test-history' to be enabled"); + } + public sealed class TestAssetFixture() : TestAssetFixtureBase() { private const string Sources = """ diff --git a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs index d40173b990..4a92230154 100644 --- a/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs +++ b/test/IntegrationTests/Microsoft.Testing.Platform.Acceptance.IntegrationTests/HelpInfoAllExtensionsTests.cs @@ -137,6 +137,12 @@ Demote failures with an Azure DevOps flaky history of at least 25% in the select Path to a text file that lists quarantined test fully qualified names or glob patterns. Matching failures are reported as warnings. --report-azdo-severity Severity to use for the reported event. Options are: error (default) and warning. + --report-azdo-slow-test-history + Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + --report-azdo-slow-test-history-min-sample + Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to 10. Requires '--report-azdo-slow-test-history'. + --report-azdo-slow-test-history-multiplier + Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to 3. Requires '--report-azdo-slow-test-history'. --report-azdo-stackframe-filter Additional regex patterns (matched against the fully-qualified type prefix of each stack frame) that should be skipped when looking for the user's call site to annotate. Repeatable; up to 16 patterns. Compiled with a 500ms match timeout. Additive to the extension's built-in MSTest assertion-implementation prefixes. --report-azdo-summary @@ -410,6 +416,18 @@ This option takes precedence over the deprecated --no-progress flag. Arity: 1 Hidden: False Description: Severity to use for the reported event. Options are: error (default) and warning. + --report-azdo-slow-test-history + Arity: 1 + Hidden: False + Description: Query Azure DevOps test result history for the past N days (1-90) and lower the per-test 'still running' threshold for tests with a known-short historical runtime. Requires '--report-azdo'. + --report-azdo-slow-test-history-min-sample + Arity: 1 + Hidden: False + Description: Minimum number of historical samples required before a test's history is used to adjust its slow-test threshold or decorate emitted lines. Defaults to 10. Requires '--report-azdo-slow-test-history'. + --report-azdo-slow-test-history-multiplier + Arity: 1 + Hidden: False + Description: Multiplier applied to a test's historical p99 duration to derive its slow-test threshold (threshold = min(static default, p99 * multiplier)). Defaults to 3. Requires '--report-azdo-slow-test-history'. --report-azdo-stackframe-filter Arity: 1..N Hidden: False diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryServiceTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryServiceTests.cs index a367332ff9..c6b96a80c6 100644 --- a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryServiceTests.cs +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsHistoryServiceTests.cs @@ -75,6 +75,61 @@ public async Task HistoryService_AggregatesStatsFromSinglePageAsync() Assert.AreEqual(0, stableStats.FailCount); } + [TestMethod] + public async Task HistoryService_ComputesDurationStatsWhenSlowTestHistoryEnabledAsync() + { + Mock historyClientMock = new(); + historyClientMock + .Setup(x => x.GetRunsAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync([new AzureDevOpsTestRun("https://example/_apis/test/Runs/1")]); + + List results = []; + for (int i = 1; i <= 100; i++) + { + results.Add(new AzureDevOpsTestResult("Namespace.Tests.Slow", "Passed", i * 100.0)); + } + + results.Add(new AzureDevOpsTestResult("Namespace.Tests.NoDuration", "Passed")); + results.Add(new AzureDevOpsTestResult("Namespace.Tests.NonPositive", "Passed", 0)); + + historyClientMock + .Setup(x => x.GetResultsAsync(It.IsAny(), "https://example/_apis/test/Runs/1", 0, 1000, null, It.IsAny())) + .ReturnsAsync(new AzureDevOpsTestResultsPage(results, continuationToken: null)); + + ICommandLineOptions options = CreateCommandLineOptions((AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory, ["30"])); + AzureDevOpsHistoryService historyService = CreateHistoryService(historyClientMock, options); + await historyService.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + Assert.IsTrue(historyService.TryGetDurationStats("Namespace.Tests.Slow", out DurationHistoryStats stats)); + Assert.AreEqual(100, stats.SampleCount); + + // Nearest-rank over 100ms..10000ms (step 100ms): p95 -> rank 95 -> 9500ms, p99 -> rank 99 -> 9900ms. + Assert.AreEqual(9500.0, stats.P95Milliseconds); + Assert.AreEqual(9900.0, stats.P99Milliseconds); + + // Tests with no positive duration samples are not published. + Assert.IsFalse(historyService.TryGetDurationStats("Namespace.Tests.NoDuration", out _)); + Assert.IsFalse(historyService.TryGetDurationStats("Namespace.Tests.NonPositive", out _)); + } + + [TestMethod] + public async Task HistoryService_DoesNotComputeDurationStatsWhenSlowTestHistoryDisabledAsync() + { + Mock historyClientMock = new(); + historyClientMock + .Setup(x => x.GetRunsAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync([new AzureDevOpsTestRun("https://example/_apis/test/Runs/1")]); + historyClientMock + .Setup(x => x.GetResultsAsync(It.IsAny(), "https://example/_apis/test/Runs/1", 0, 1000, null, It.IsAny())) + .ReturnsAsync(new AzureDevOpsTestResultsPage([new AzureDevOpsTestResult("Namespace.Tests.Slow", "Passed", 1234.0)], continuationToken: null)); + + // Default options enable only flaky-history, so durations must not be collected. + AzureDevOpsHistoryService historyService = CreateHistoryService(historyClientMock); + await historyService.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + Assert.IsFalse(historyService.TryGetDurationStats("Namespace.Tests.Slow", out _)); + } + [TestMethod] public async Task HistoryService_AggregatesStatsAcrossPagesAsync() { diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestHistoryTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestHistoryTests.cs new file mode 100644 index 0000000000..8a13204901 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestHistoryTests.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Testing.Extensions.AzureDevOpsReport; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class AzureDevOpsSlowTestHistoryTests +{ + [TestMethod] + public void Percentile_NearestRank_ReturnsExpectedSamples() + { + double[] samples = [10, 20, 30, 40, 50, 60, 70, 80, 90, 100]; + + Assert.AreEqual(100.0, PercentileCalculator.Compute(samples, 100)); + Assert.AreEqual(100.0, PercentileCalculator.Compute(samples, 99)); + Assert.AreEqual(100.0, PercentileCalculator.Compute(samples, 95)); + Assert.AreEqual(50.0, PercentileCalculator.Compute(samples, 50)); + Assert.AreEqual(10.0, PercentileCalculator.Compute(samples, 1)); + } + + [TestMethod] + public void Percentile_SingleSample_ReturnsThatSample() + => Assert.AreEqual(42.0, PercentileCalculator.Compute([42], 95)); + + [TestMethod] + public void Percentile_EmptySamples_Throws() + => Assert.ThrowsExactly(() => + { + _ = PercentileCalculator.Compute([], 95); + }); + + [TestMethod] + [DataRow(0.0)] + [DataRow(-1.0)] + [DataRow(100.1)] + public void Percentile_OutOfRange_Throws(double percentile) + => Assert.ThrowsExactly(() => + { + _ = PercentileCalculator.Compute([1, 2, 3], percentile); + }); + + [TestMethod] + public void DurationHistoryStats_TryCreate_IgnoresNonPositiveSamples() + { + Assert.IsTrue(DurationHistoryStats.TryCreate([0, -5, 100, 200, 300], out DurationHistoryStats stats)); + Assert.AreEqual(3, stats.SampleCount); + Assert.AreEqual(300.0, stats.P99Milliseconds); + } + + [TestMethod] + public void DurationHistoryStats_TryCreate_WithNoPositiveSamples_ReturnsFalse() + => Assert.IsFalse(DurationHistoryStats.TryCreate([0, -1], out _)); + + [TestMethod] + public void ComputeThreshold_WithoutHistory_UsesStaticThreshold() + { + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), default, hasStats: false, multiplier: 3.0, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(60), threshold); + } + + [TestMethod] + public void ComputeThreshold_BelowMinimumSampleCount_UsesStaticThreshold() + { + var stats = new DurationHistoryStats(2000, 3000, sampleCount: 5); + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), stats, hasStats: true, multiplier: 3.0, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(60), threshold); + } + + [TestMethod] + public void ComputeThreshold_WithShortHistory_LowersThreshold() + { + // p99 = 3s, multiplier 3 => 9s, which is below the 60s static default. + var stats = new DurationHistoryStats(2000, 3000, sampleCount: 120); + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), stats, hasStats: true, multiplier: 3.0, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(9), threshold); + } + + [TestMethod] + public void ComputeThreshold_WhenHistoryExceedsStatic_KeepsStaticThreshold() + { + // p99 = 30s, multiplier 3 => 90s, which is above the 60s static default. + var stats = new DurationHistoryStats(25000, 30000, sampleCount: 120); + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), stats, hasStats: true, multiplier: 3.0, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(60), threshold); + } + + [TestMethod] + public void ComputeThreshold_WhenMultiplierOverflowsToInfinity_UsesStaticThreshold() + { + // A pathological multiplier would overflow p99 * multiplier to +Infinity, which would throw inside + // TimeSpan.FromMilliseconds; the guard must fall back to the static threshold instead. + var stats = new DurationHistoryStats(25000, 30000, sampleCount: 120); + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), stats, hasStats: true, multiplier: double.MaxValue, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(60), threshold); + } + + [TestMethod] + public void ComputeThreshold_WhenHistoryExceedsTimeSpanMax_UsesStaticThreshold() + { + // A large *finite* history threshold (neither NaN nor Infinity) can still exceed + // TimeSpan.MaxValue.TotalMilliseconds and throw inside TimeSpan.FromMilliseconds. Because the result is + // never below the static threshold, the method must short-circuit to the static one without converting. + double hugeButFinite = TimeSpan.MaxValue.TotalMilliseconds * 2; + var stats = new DurationHistoryStats(hugeButFinite, hugeButFinite, sampleCount: 120); + TimeSpan threshold = AzureDevOpsSlowTestThresholds.ComputeThreshold( + TimeSpan.FromSeconds(60), stats, hasStats: true, multiplier: 1.0, minimumSampleCount: 10); + + Assert.AreEqual(TimeSpan.FromSeconds(60), threshold); + } + + [TestMethod] + public void HasUsableHistory_RespectsMinimumSampleCount() + { + var stats = new DurationHistoryStats(2000, 3000, sampleCount: 9); + Assert.IsFalse(AzureDevOpsSlowTestThresholds.HasUsableHistory(stats, hasStats: true, minimumSampleCount: 10)); + Assert.IsTrue(AzureDevOpsSlowTestThresholds.HasUsableHistory(new DurationHistoryStats(2000, 3000, 10), hasStats: true, minimumSampleCount: 10)); + Assert.IsFalse(AzureDevOpsSlowTestThresholds.HasUsableHistory(stats, hasStats: false, minimumSampleCount: 1)); + } + + [TestMethod] + [DataRow(2000.0, "2s")] + [DataRow(2500.0, "2.5s")] + [DataRow(500.0, "500ms")] + [DataRow(120000.0, "2m")] + [DataRow(90000.0, "1.5m")] + [DataRow(-10.0, "0ms")] + public void FormatDuration_ProducesCompactStrings(double milliseconds, string expected) + => Assert.AreEqual(expected, AzureDevOpsSlowTestThresholds.FormatDuration(milliseconds)); +} diff --git a/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestReporterTests.cs b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestReporterTests.cs new file mode 100644 index 0000000000..67e489f358 --- /dev/null +++ b/test/UnitTests/Microsoft.Testing.Extensions.UnitTests/AzureDevOpsSlowTestReporterTests.cs @@ -0,0 +1,280 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Diagnostics.CodeAnalysis; + +using Microsoft.Testing.Extensions.AzureDevOpsReport; +using Microsoft.Testing.Extensions.Reporting; +using Microsoft.Testing.Platform.CommandLine; +using Microsoft.Testing.Platform.Extensions.Messages; +using Microsoft.Testing.Platform.Extensions.OutputDevice; +using Microsoft.Testing.Platform.Helpers; +using Microsoft.Testing.Platform.Logging; +using Microsoft.Testing.Platform.OutputDevice; +using Microsoft.Testing.Platform.Services; +using Microsoft.Testing.Platform.TestHost; + +using Moq; + +namespace Microsoft.Testing.Extensions.UnitTests; + +[TestClass] +public sealed class AzureDevOpsSlowTestReporterTests +{ + private static readonly DateTimeOffset Start = new(2025, 05, 16, 12, 00, 00, TimeSpan.Zero); + + [TestMethod] + public async Task OnTestSessionStarting_WhenNotInAzureDevOps_NoOpsAndIgnoresUpdatesAsync() + { + CapturingOutputDevice outputDevice = new(); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: false); + + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(120), CancellationToken.None).ConfigureAwait(false); + + // No output device emission and nothing tracked when running outside Azure DevOps. + Assert.IsEmpty(outputDevice.Lines); + } + + [TestMethod] + public async Task ScanOnce_AfterThreshold_EmitsSingleSlowTestLineAsync() + { + CapturingOutputDevice outputDevice = new(); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + // Default static threshold is 60s; nothing emitted before it. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(30), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(90), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + Assert.Contains("[slow] still running after", outputDevice.Lines[0]); + Assert.Contains("Ns.T1", outputDevice.Lines[0]); + + // No history for this test, so no decoration suffix. + Assert.DoesNotContain("historical", outputDevice.Lines[0]); + } + + [TestMethod] + public async Task ScanOnce_WithHistory_LowersThresholdAndDecoratesLineAsync() + { + CapturingOutputDevice outputDevice = new(); + FakeHistoryService history = new(); + history.Add("Ns.Fast", new DurationHistoryStats(2000, 3000, sampleCount: 120)); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: true, history: history); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.Fast", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + // p99 = 3s, multiplier 3 => 9s threshold (below the 60s static default). + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(8), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(10), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + Assert.Contains("(historical p95 = 2s, p99 = 3s, samples = 120)", outputDevice.Lines[0]); + } + + [TestMethod] + public async Task ScanOnce_UsesExponentialBackoffAsync() + { + CapturingOutputDevice outputDevice = new(); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + // First emit at >= 60s; threshold then doubles to 120s. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(90), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + + // Still below 120s -> no new emission. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(110), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(1, outputDevice.Lines); + + // Past 120s -> second emission. + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(130), CancellationToken.None).ConfigureAwait(false); + Assert.HasCount(2, outputDevice.Lines); + } + + [TestMethod] + public async Task ConsumeAsync_TerminalState_StopsTrackingAsync() + { + CapturingOutputDevice outputDevice = new(); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new PassedTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(120), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + } + + [TestMethod] + public async Task OnTestSessionFinishing_DrainsLoopAndClearsTrackingAsync() + { + CapturingOutputDevice outputDevice = new(); + AzureDevOpsSlowTestReporter reporter = CreateReporter(outputDevice, tfBuild: true); + await reporter.OnTestSessionStartingAsync(new TestSessionContextStub()).ConfigureAwait(false); + await reporter.ConsumeAsync(null!, CreateMessage("u1", "Ns.T1", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + + await reporter.OnTestSessionFinishingAsync(new TestSessionContextStub()).ConfigureAwait(false); + + // After finishing, the reporter is inactive: further updates are ignored and nothing is tracked. + await reporter.ConsumeAsync(null!, CreateMessage("u2", "Ns.T2", new InProgressTestNodeStateProperty()), CancellationToken.None).ConfigureAwait(false); + await reporter.ScanOnceAsync(Start + TimeSpan.FromSeconds(120), CancellationToken.None).ConfigureAwait(false); + Assert.IsEmpty(outputDevice.Lines); + } + + private static AzureDevOpsSlowTestReporter CreateReporter(CapturingOutputDevice outputDevice, bool tfBuild, FakeHistoryService? history = null) + { + Dictionary options = new(StringComparer.OrdinalIgnoreCase) + { + [AzureDevOpsCommandLineOptions.AzureDevOpsOptionName] = [], + [AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory] = ["30"], + }; + + Mock environmentMock = new(); + environmentMock.Setup(x => x.GetEnvironmentVariable("TF_BUILD")).Returns(tfBuild ? "true" : null); + + return new AzureDevOpsSlowTestReporter( + new FakeCommandLineOptions(options), + environmentMock.Object, + outputDevice, + new NonRunningTask(), + new FixedClock(Start), + new StubLoggerFactory(), + history ?? new FakeHistoryService()); + } + + private static TestNodeUpdateMessage CreateMessage(string uid, string fullyQualifiedName, TestNodeStateProperty state) + { + PropertyBag propertyBag = new(); + propertyBag.Add(state); + propertyBag.Add(new SerializableKeyValuePairStringProperty("vstest.TestCase.FullyQualifiedName", fullyQualifiedName)); + + return new TestNodeUpdateMessage(new SessionUid("session"), new TestNode + { + Uid = uid, + DisplayName = fullyQualifiedName, + Properties = propertyBag, + }); + } + + private sealed class FakeHistoryService : IAzureDevOpsHistoryService + { + private readonly Dictionary _durationStats = []; + + public int HistoryWindowInDays => 30; + + public void Add(string testName, DurationHistoryStats stats) + => _durationStats[testName] = stats; + + public bool TryGetStats(string testName, out FlakyStats stats) + { + stats = default; + return false; + } + + public bool IsLikelyFlaky(string testName, double threshold) + => false; + + public bool TryGetDurationStats(string testName, out DurationHistoryStats stats) + => _durationStats.TryGetValue(testName, out stats); + } + + private sealed class FakeCommandLineOptions(IReadOnlyDictionary options) : ICommandLineOptions + { + private readonly IReadOnlyDictionary _options = options; + + public bool IsOptionSet(string optionName) + => _options.ContainsKey(optionName); + + public bool TryGetOptionArgumentList(string optionName, [NotNullWhen(true)] out string[]? arguments) + { + if (_options.TryGetValue(optionName, out string[]? values)) + { + arguments = values; + return true; + } + + arguments = null; + return false; + } + } + + private sealed class CapturingOutputDevice : IOutputDevice + { + public List Lines { get; } = []; + + public Task DisplayAsync(IOutputDeviceDataProducer producer, IOutputDeviceData data, CancellationToken cancellationToken) + { + Lines.Add(((TextOutputDeviceData)data).Text); + return Task.CompletedTask; + } + } + + private sealed class FixedClock(DateTimeOffset now) : IClock + { + public DateTimeOffset UtcNow { get; } = now; + } + + // Does not actually run the background scan loop, so OnTestSessionStartingAsync returns immediately + // and tests can drive ScanOnceAsync deterministically. + private sealed class NonRunningTask : ITask + { + public Task Delay(int millisecondDelay) + => Task.CompletedTask; + + public Task Delay(TimeSpan timeSpan, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task Run(Action action) + { + action(); + return Task.CompletedTask; + } + + public Task Run(Func function, CancellationToken cancellationToken) + => function(); + + public Task Run(Func?> function, CancellationToken cancellationToken) + => function()!; + + public Task RunLongRunning(Func action, string name, CancellationToken cancellationToken) + => Task.CompletedTask; + + public Task WhenAll(params Task[] tasks) + => Task.WhenAll(tasks); + } + + private sealed class TestSessionContextStub : ITestSessionContext + { + public SessionUid SessionUid { get; } = new("session"); + + public CancellationToken CancellationToken { get; } = CancellationToken.None; + } + + private sealed class StubLoggerFactory : ILoggerFactory + { + public ILogger CreateLogger(string categoryName) + => new NullLogger(); + + private sealed class NullLogger : ILogger + { + public bool IsEnabled(LogLevel logLevel) + => false; + + public void Log(LogLevel logLevel, TState state, Exception? exception, Func formatter) + { + } + + public Task LogAsync(LogLevel logLevel, TState state, Exception? exception, Func formatter) + => Task.CompletedTask; + } + } +}