Skip to content

Commit 6fbf213

Browse files
rolfbjarnemandel-macaque
rolfbjarne
authored and
mandel-macaque
committed
[xharness] Improve parsing of NUnitV2 xml reports to fix an issue with failures in parameterized tests. (#9259)
We weren't properly reporting all failures when there were multiple failures in parameterized tests, because we'd incorrectly skip too many xml nodes when we were looking for the next test failure.
1 parent 6110664 commit 6fbf213

4 files changed

Lines changed: 106 additions & 11 deletions

File tree

‎tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Microsoft.DotNet.XHarness.iOS.Shared.Tests.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
<None Remove="Samples\TouchUnitSample.xml" />
1919
<None Remove="Samples\xUnitSample.xml" />
2020
<None Remove="Samples\NUnitV2SampleFailure.xml" />
21+
<None Remove="Samples\TestCaseFailures.xml" />
2122
</ItemGroup>
2223

2324
<ItemGroup>
@@ -52,6 +53,7 @@
5253
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
5354
</EmbeddedResource>
5455
<EmbeddedResource Include="Samples\NUnitV2SampleFailure.xml" />
56+
<EmbeddedResource Include="Samples\TestCaseFailures.xml" />
5557
</ItemGroup>
5658

5759
<ItemGroup>

‎tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/Samples/TestCaseFailures.xml‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
<?xml version="1.0" encoding="utf-8" standalone="no"?>
2+
<test-results name="mtouch.dll" total="440" errors="4" failures="19" not-run="6" inconclusive="0" ignored="6" skipped="0" invalid="0" date="2020-07-31" time="03:17:46">
3+
<environment nunit-version="3.6.0.0" clr-version="4.0.30319.42000" os-version="Unix 19.5.0.0" platform="Unix" cwd="..." machine-name="..." user="..." user-domain="..." />
4+
<culture-info current-culture="en-US" current-uiculture="en-US" />
5+
<test-suite type="Assembly" name="mtouch.dll" executed="True" result="Failure" success="False" time="3393.268" asserts="7560">
6+
<properties>
7+
<property name="_PID" value="72474" />
8+
<property name="_APPDOMAIN" value="domain-" />
9+
</properties>
10+
<failure>
11+
<message><![CDATA[One or more child tests had errors]]></message>
12+
<stack-trace />
13+
</failure>
14+
<results>
15+
<test-suite type="SetUpFixture" name="[default namespace]" executed="True" result="Failure" success="False" time="3393.255" asserts="7560">
16+
<failure>
17+
<message><![CDATA[One or more child tests had errors]]></message>
18+
<stack-trace />
19+
</failure>
20+
<results>
21+
<test-suite type="TestSuite" name="Xamarin" executed="True" result="Failure" success="False" time="3393.250" asserts="7560">
22+
<failure>
23+
<message><![CDATA[One or more child tests had errors]]></message>
24+
<stack-trace />
25+
</failure>
26+
<results>
27+
<test-suite type="TestFixture" name="MTouch" executed="True" result="Failure" success="False" time="2720.409" asserts="576">
28+
<failure>
29+
<message><![CDATA[One or more child tests had errors]]></message>
30+
<stack-trace />
31+
</failure>
32+
<results>
33+
<test-suite type="ParameterizedTest" name="FastDev_LinkAll" executed="True" result="Failure" success="False" time="16.633" asserts="2">
34+
<failure>
35+
<message><![CDATA[One or more child tests had errors]]></message>
36+
<stack-trace />
37+
</failure>
38+
<results>
39+
<test-case name="Xamarin.MTouch.FastDev_LinkAll(iOS)" executed="True" result="Failure" success="False" time="8.162" asserts="1">
40+
<failure>
41+
<message>message</message>
42+
<stack-trace>stacktrace</stack-trace>
43+
</failure>
44+
</test-case>
45+
</results>
46+
</test-suite>
47+
<test-case name="Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode" executed="True" result="Failure" success="False" time="0.443" asserts="0">
48+
<failure>
49+
<message>message</message>
50+
<stack-trace>stacktrace</stack-trace>
51+
</failure>
52+
</test-case>
53+
</results>
54+
</test-suite>
55+
</results>
56+
</test-suite>
57+
</results>
58+
</test-suite>
59+
</results>
60+
</test-suite>
61+
</test-results>
62+

‎tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared.Tests/XmlResultParserTests.cs‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -361,9 +361,31 @@ public void NUnitV2GenerateTestReport ()
361361
message</div>
362362
</li>
363363
<li>
364-
ErrorTest2: SingleLineErrorMessage</li>
365-
<li>
366364
NUnit.Tests.Assemblies.MockTestFixture.FailingTest: Intentional failure</li>
365+
<li>
366+
NUnit.Tests.Assemblies.MockTestFixture.TestWithException: System.ApplicationException : Intentional Exception</li>
367+
</ul>
368+
</div>
369+
";
370+
Assert.AreEqual (expectedOutput, writer.ToString (), "Output");
371+
}
372+
373+
374+
[Test]
375+
public void NUnitV2GenerateTestReportWithTestCaseFailures ()
376+
{
377+
using var writer = new StringWriter ();
378+
using var stream = GetType ().Assembly.GetManifestResourceStream ("Microsoft.DotNet.XHarness.iOS.Shared.Tests.Samples.TestCaseFailures.xml");
379+
using var reader = new StreamReader (stream);
380+
resultParser.GenerateTestReport (writer, reader, XmlResultJargon.NUnitV2);
381+
Console.WriteLine (writer.ToString ());
382+
var expectedOutput =
383+
@"<div style='padding-left: 15px;'>
384+
<ul>
385+
<li>
386+
Xamarin.MTouch.FastDev_LinkAll(iOS): message</li>
387+
<li>
388+
Xamarin.MTouch.RebuildWhenReferenceSymbolsInCode: message</li>
367389
</ul>
368390
</div>
369391
";

‎tests/xharness/Microsoft.DotNet.XHarness.iOS.Shared/XmlResultParser.cs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -518,7 +518,14 @@ static void GenerateNUnitV2TestReport (TextWriter writer, XmlReader reader)
518518
void write_failure ()
519519
{
520520
var name = reader ["name"];
521-
var message = reader.ReadToDescendant ("message") ? reader.ReadElementContentAsString () : null;
521+
string? message = null;
522+
var depth = reader.Depth;
523+
if (reader.ReadToDescendant ("message")) {
524+
message = reader.ReadElementContentAsString ();
525+
// ReadToParent
526+
while (depth > reader.Depth && reader.Read ()) {
527+
}
528+
}
522529
var message_block = message?.IndexOf ('\n') >= 0;
523530
writer.WriteLine ("<li>");
524531
writer.Write (name.AsHtml ());
@@ -544,16 +551,18 @@ void write_failure ()
544551
if (result == "Error")
545552
write_failure ();
546553

547-
if (!reader.ReadToDescendant ("test-case"))
548-
continue;
554+
var depth = reader.Depth;
549555

550-
do {
551-
result = reader ["result"];
552-
if (result != "Error" && result != "Failure")
553-
continue;
556+
while (reader.Read ()) {
557+
if (reader.NodeType != XmlNodeType.Element || reader.Name != "test-case")
558+
continue;
554559

555-
write_failure ();
556-
} while (reader.ReadToNextSibling ("test-case"));
560+
result = reader ["result"];
561+
if (result == "Error" || result == "Failure")
562+
write_failure ();
563+
if (reader.Depth < depth)
564+
break;
565+
}
557566
}
558567

559568
writer.WriteLine ("</ul>");

0 commit comments

Comments
 (0)