Skip to content

Add Azure DevOps history-driven slow-test threshold enricher (#9140) #9182

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Evangelink merged 5 commits into from
Jun 17, 2026
Merged

Add Azure DevOps history-driven slow-test threshold enricher (#9140) #9182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
da39a83
Add Azure DevOps history-driven slow-test threshold enricher (#9140)
Evangelink Jun 16, 2026
8e96477
Address PR review feedback on slow-test history enricher
Evangelink Jun 16, 2026
37e8d2f
Add reporter unit tests, dependent-option validation, and CI dogfoodi…
Evangelink Jun 16, 2026
8551de2
Merge remote-tracking branch 'origin/main' into evangelink/issue-9140…
Evangelink Jun 16, 2026
fb67002
Address review: guard threshold overflow, clamp backoff, document sha…
Evangelink Jun 16, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineOptions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ internal static class AzureDevOpsCommandLineOptions
public const string AzureDevOpsFlakyHistory = "report-azdo-flaky-history";
public const string AzureDevOpsQuarantineFile = "report-azdo-quarantine-file";
public const string AzureDevOpsReportSeverity = "report-azdo-severity";
public const string AzureDevOpsSlowTestHistory = "report-azdo-slow-test-history";
public const string AzureDevOpsSlowTestHistoryMinSample = "report-azdo-slow-test-history-min-sample";
public const string AzureDevOpsSlowTestHistoryMultiplier = "report-azdo-slow-test-history-multiplier";
public const string AzureDevOpsStackFrameFilter = "report-azdo-stackframe-filter";
public const string AzureDevOpsSummary = "report-azdo-summary";
public const string AzureDevOpsUploadArtifactExclude = "report-azdo-upload-artifact-exclude";
Expand All @@ -22,4 +25,8 @@ internal static class AzureDevOpsCommandLineOptions
public const string AzureDevOpsUploadArtifactsModeTagsOnly = "tags-only";
public const string PublishAzureDevOpsRunNameOptionName = "publish-azdo-run-name";
public const string PublishAzureDevOpsTestResultsOptionName = "publish-azdo-test-results";

public const int SlowTestHistoryDefaultMinSample = 10;
public const double SlowTestHistoryDefaultMultiplier = 3.0;
public const int SlowTestStaticThresholdSeconds = 60;
}
54 changes: 54 additions & 0 deletions ...Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsCommandLineProvider.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,16 @@ internal sealed class AzureDevOpsCommandLineProvider : CommandLineOptionsProvide
MaxStackFrameFilterPatterns,
StackFrameFilterMatchTimeoutMs);

private static readonly string SlowTestHistoryMinSampleOptionDescriptionFormatted = string.Format(
CultureInfo.InvariantCulture,
AzureDevOpsResources.SlowTestHistoryMinSampleOptionDescription,
AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMinSample);

private static readonly string SlowTestHistoryMultiplierOptionDescriptionFormatted = string.Format(
CultureInfo.InvariantCulture,
AzureDevOpsResources.SlowTestHistoryMultiplierOptionDescription,
AzureDevOpsCommandLineOptions.SlowTestHistoryDefaultMultiplier);

public AzureDevOpsCommandLineProvider()
: base(
nameof(AzureDevOpsCommandLineProvider),
Expand All @@ -49,6 +59,9 @@ public AzureDevOpsCommandLineProvider()
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory, AzureDevOpsResources.FlakyHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsQuarantineFile, AzureDevOpsResources.QuarantineFileOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity, AzureDevOpsResources.SeverityOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory, AzureDevOpsResources.SlowTestHistoryOptionDescription, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample, SlowTestHistoryMinSampleOptionDescriptionFormatted, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier, SlowTestHistoryMultiplierOptionDescriptionFormatted, ArgumentArity.ExactlyOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter, StackFrameFilterOptionDescriptionFormatted, ArgumentArity.OneOrMore, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsSummary, AzureDevOpsResources.SummaryOptionDescription, ArgumentArity.ZeroOrOne, false),
new CommandLineOption(AzureDevOpsCommandLineOptions.AzureDevOpsUploadArtifactExclude, AzureDevOpsResources.UploadArtifactExcludeOptionDescription, ArgumentArity.ZeroOrMore, false),
Expand All @@ -65,6 +78,9 @@ public override Task<ValidationResult> ValidateOptionArgumentsAsync(CommandLineO
=> commandOption.Name switch
{
AzureDevOpsCommandLineOptions.AzureDevOpsFlakyHistory => ValidateFlakyHistoryArgumentsAsync(arguments),
AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory => ValidateSlowTestHistoryArgumentsAsync(arguments),
AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample => ValidateSlowTestHistoryMinSampleArgumentsAsync(arguments),
AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier => ValidateSlowTestHistoryMultiplierArgumentsAsync(arguments),
AzureDevOpsCommandLineOptions.AzureDevOpsReportSeverity when !SeverityOptions.Contains(arguments[0], StringComparer.OrdinalIgnoreCase)
=> ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSeverity, arguments[0])),
AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter => ValidateStackFrameFilterArgumentsAsync(arguments),
Expand Down Expand Up @@ -96,6 +112,10 @@ public override Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandL
{
errorMessage = AzureDevOpsResources.AzureDevOpsReportSeverityRequiresAzureDevOps;
}
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory))
{
errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryRequiresAzureDevOps;
}
else if (commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsStackFrameFilter))
{
errorMessage = AzureDevOpsResources.AzureDevOpsStackFrameFilterRequiresAzureDevOps;
Expand All @@ -111,6 +131,22 @@ public override Task<ValidationResult> ValidateCommandLineOptionsAsync(ICommandL
errorMessage = AzureDevOpsResources.AzureDevOpsDemoteKnownFlakyRequiresFlakyHistory;
}

// The slow-test-history sub-options depend on '--report-azdo-slow-test-history' regardless of whether
// '--report-azdo' itself is set, so these checks live outside the '--report-azdo' branch above.
if (errorMessage is null
&& commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMinSample)
&& !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory))
{
errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryMinSampleRequiresSlowTestHistory;
}

if (errorMessage is null
&& commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistoryMultiplier)
&& !commandLineOptions.IsOptionSet(AzureDevOpsCommandLineOptions.AzureDevOpsSlowTestHistory))
{
errorMessage = AzureDevOpsResources.AzureDevOpsSlowTestHistoryMultiplierRequiresSlowTestHistory;
}

if (errorMessage is null && HasArtifactUploadConfiguration(commandLineOptions) && IsArtifactUploadDisabled(commandLineOptions))
{
errorMessage = AzureDevOpsResources.ArtifactUploadOptionsRequireUploadArtifacts;
Expand Down Expand Up @@ -175,6 +211,24 @@ private static Task<ValidationResult> ValidateFlakyHistoryArgumentsAsync(string[
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidFlakyHistoryDays, arguments[0]));

private static Task<ValidationResult> ValidateSlowTestHistoryArgumentsAsync(string[] arguments)
=> int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int days)
&& days is >= 1 and <= 90
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryDays, arguments[0]));

private static Task<ValidationResult> ValidateSlowTestHistoryMinSampleArgumentsAsync(string[] arguments)
=> int.TryParse(arguments[0], NumberStyles.Integer, CultureInfo.InvariantCulture, out int minimum)
&& minimum >= 1
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryMinSample, arguments[0]));

private static Task<ValidationResult> ValidateSlowTestHistoryMultiplierArgumentsAsync(string[] arguments)
=> double.TryParse(arguments[0], NumberStyles.Float, CultureInfo.InvariantCulture, out double multiplier)
&& multiplier is > 0 and <= 10_000
? ValidationResult.ValidTask
: ValidationResult.InvalidTask(string.Format(CultureInfo.InvariantCulture, AzureDevOpsResources.InvalidSlowTestHistoryMultiplier, arguments[0]));

Comment thread
Evangelink marked this conversation as resolved.
private static Task<ValidationResult> ValidateStackFrameFilterArgumentsAsync(string[] arguments)
{
if (arguments.Length > MaxStackFrameFilterPatterns)
Expand Down
13 changes: 13 additions & 0 deletions src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsExtensions.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,17 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
serviceProvider.GetTestApplicationModuleInfo(),
serviceProvider.GetLoggerFactory()));

var compositeSlowTestReporter =
new CompositeExtensionFactory<AzureDevOpsSlowTestReporter>(serviceProvider =>
new AzureDevOpsSlowTestReporter(
serviceProvider.GetCommandLineOptions(),
serviceProvider.GetEnvironment(),
serviceProvider.GetOutputDevice(),
serviceProvider.GetTask(),
serviceProvider.GetClock(),
serviceProvider.GetLoggerFactory(),
historyService ??= CreateHistoryService(serviceProvider)));

var compositeLogGroupReporter =
new CompositeExtensionFactory<AzureDevOpsLogGroupReporter>(serviceProvider =>
new AzureDevOpsLogGroupReporter(
Expand Down Expand Up @@ -81,12 +92,14 @@ public static void AddAzureDevOpsProvider(this ITestApplicationBuilder builder)
});
builder.TestHost.AddDataConsumer(compositeArtifactUploader);
builder.TestHost.AddDataConsumer(compositeSummaryReporter);
builder.TestHost.AddDataConsumer(compositeSlowTestReporter);
builder.TestHost.AddDataConsumer(compositeTestResultsPublisher);
builder.TestHost.AddDataConsumer(compositeLogGroupReporter);
builder.TestHost.AddTestSessionLifetimeHandler(serviceProvider =>
historyService ??= CreateHistoryService(serviceProvider));
builder.TestHost.AddTestSessionLifetimeHandler(compositeArtifactUploader);
builder.TestHost.AddTestSessionLifetimeHandler(compositeSummaryReporter);
builder.TestHost.AddTestSessionLifetimeHandler(compositeSlowTestReporter);
builder.TestHost.AddTestSessionLifetimeHandler(compositeTestResultsPublisher);

// Registered last so its OnTestSessionFinishingAsync (the closing ##[endgroup]) runs after
Expand Down
10 changes: 8 additions & 2 deletions src/Platform/Microsoft.Testing.Extensions.AzureDevOpsReport/AzureDevOpsHistoryClient.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -261,7 +261,7 @@ private static async Task<AzureDevOpsTestResultsPage> ParseResultsAsync(HttpResp
: new AzureDevOpsTestResultsPage(
[.. payload.Value
.Where(static result => !RoslynString.IsNullOrWhiteSpace(result.AutomatedTestName) && !RoslynString.IsNullOrWhiteSpace(result.Outcome))
.Select(static result => new AzureDevOpsTestResult(result.AutomatedTestName!, result.Outcome!))],
.Select(static result => new AzureDevOpsTestResult(result.AutomatedTestName!, result.Outcome!, result.DurationInMs))],
continuationToken);
}

Expand Down Expand Up @@ -357,6 +357,9 @@ internal sealed class AzureDevOpsResultResponse

[JsonPropertyName("outcome")]
public string? Outcome { get; set; }

[JsonPropertyName("durationInMs")]
public double? DurationInMs { get; set; }
}

internal sealed class AzureDevOpsResultsResponse
Expand All @@ -367,15 +370,18 @@ internal sealed class AzureDevOpsResultsResponse

internal sealed class AzureDevOpsTestResult
{
public AzureDevOpsTestResult(string automatedTestName, string outcome)
public AzureDevOpsTestResult(string automatedTestName, string outcome, double? durationMilliseconds = null)
{
AutomatedTestName = automatedTestName;
Outcome = outcome;
DurationMilliseconds = durationMilliseconds;
}

public string AutomatedTestName { get; }

public string Outcome { get; }

public double? DurationMilliseconds { get; }
}

internal sealed class AzureDevOpsTestResultsPage
Expand Down
Loading
Loading