Skip to content

Commit 4627556

Browse files
rolfbjarnerolfbjarne
committed
[DRAFT] [devops/tests] Create a html report for the Windows tests
This is very much in progress, no need to review.
1 parent 677bf86 commit 4627556

19 files changed

+402
-90
lines changed

‎scripts/create-html-report/create-html-report.cs

Lines changed: 177 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/*
2+
# Expected files:
3+
# Expected files:
4+
#
5+
# $Env:BUILD_SOURCESDIRECTORY/xamarin-macios/jenkins-results/windows-remote-dotnet-tests.trx"
6+
# $(Build.SourcesDirectory)/xamarin-macios/jenkins-results/windows-dotnet-tests.trx"
7+
# $Env:BUILD_SOURCESDIRECTORY/xamarin-macios/jenkins-results/windows/bgen-tests/results.trx
8+
#
9+
# $Env:BUILD_SOURCESDIRECTORY\xamarin-macios\jenkins-results\windows-remote-logs.zip
10+
#
11+
*/
12+
13+
using System.IO;
14+
using System.Text;
15+
using System.Xml;
16+
17+
public class Program {
18+
static string GetOutcomeColor (string outcome)
19+
{
20+
switch (outcome.ToLower ()) {
21+
case "passed":
22+
case "completed":
23+
return "green";
24+
default:
25+
return "red";
26+
}
27+
}
28+
29+
static string FormatHtml (string text)
30+
{
31+
text = text.Replace ("\r", "");
32+
text = text.Replace ("&", "&");
33+
text = text.Replace ("<", "&lt;");
34+
text = text.Replace (">", "&gt;");
35+
text = text.Replace (" ", "&nbsp;&nbsp;");
36+
text = text.Replace (" &nbsp;", "&nbsp;&nbsp;");
37+
text = text.Replace ("&nbsp; ", "&nbsp;&nbsp;");
38+
text = text.Replace ("\n", "<br />\n");
39+
return text;
40+
}
41+
42+
static string GetSourcesDirectory ()
43+
{
44+
var pwd = Environment.CurrentDirectory!;
45+
var dir = pwd;
46+
while (true) {
47+
if (Directory.Exists (Path.Combine (dir, ".git")))
48+
return dir;
49+
var parentDir = Path.GetDirectoryName (dir);
50+
if (string.IsNullOrEmpty (parentDir) || parentDir == dir || parentDir.Length <= 2)
51+
throw new Exception ($"Unable to find a .git subdirectory in any directory up the directory hierarchy from {pwd}");
52+
dir = parentDir;
53+
}
54+
throw new Exception ($"Unable to find a .git subdirectory in any directory up the directory hierarchy from {pwd}");
55+
}
56+
57+
public static int Main (string [] args)
58+
{
59+
var sourcesDirectory = GetSourcesDirectory ();
60+
var allTestsSucceeded = true;
61+
var outputDirectory = Path.Combine (sourcesDirectory, "jenkins-results");
62+
var indexFile = Path.Combine (outputDirectory, "index.html");
63+
var summaryFile = Path.Combine (sourcesDirectory, "tests", "TestSummary.md");
64+
65+
var trxFiles = new [] {
66+
new { Name = "Remote .NET tests", TestResults = Path.Combine (outputDirectory, "windows-remote-dotnet-tests.trx") },
67+
new { Name = "Local .NET tests", TestResults = Path.Combine (outputDirectory, "windows-dotnet-tests.trx") },
68+
new { Name = "Generator tests", TestResults = Path.Combine (outputDirectory, "windows", "bgen-tests", "results.trx") },
69+
};
70+
71+
var extraFiles = new []{
72+
Path.Combine(outputDirectory, "windows-remote-logs.zip"),
73+
};
74+
75+
var indexContents = new StringBuilder ();
76+
var summaryContents = new StringBuilder ();
77+
78+
indexContents.AppendLine ($"<!DOCTYPE html>");
79+
indexContents.AppendLine ($"<html>");
80+
indexContents.AppendLine ($" <head>");
81+
indexContents.AppendLine ($" <meta charset=\"utf-8\"/>");
82+
indexContents.AppendLine ($" <title>Test results</title>");
83+
indexContents.AppendLine ($" <style>");
84+
indexContents.AppendLine ($" .pdiv {{");
85+
indexContents.AppendLine ($" display: table;");
86+
indexContents.AppendLine ($" padding-top: 10px;");
87+
indexContents.AppendLine ($" }}");
88+
indexContents.AppendLine ($" </style>");
89+
indexContents.AppendLine ($" </head>");
90+
indexContents.AppendLine ($" <body>");
91+
indexContents.AppendLine ($" <h1>Test results</h1>");
92+
foreach (var trx in trxFiles) {
93+
var name = trx.Name;
94+
var path = trx.TestResults;
95+
string? outcome;
96+
var messageLines = new List<string> ();
97+
98+
try {
99+
var xml = new XmlDocument ();
100+
xml.Load (path);
101+
outcome = xml.SelectSingleNode ("/*[local-name() = 'TestRun']/*[local-name() = 'ResultSummary']")?.Attributes? ["outcome"]?.Value;
102+
if (outcome is null) {
103+
outcome = $"Could not find outcome in trx file {path}";
104+
} else {
105+
var failedTests = xml.SelectNodes ("/*[local-name() = 'TestRun']/*[local-name() = 'Results']/*[local-name() = 'UnitTestResult'][@outcome != 'Passed']")?.Cast<XmlNode> ();
106+
if (failedTests?.Any () == true) {
107+
messageLines.Add (" <ul>");
108+
foreach (var node in failedTests) {
109+
var testName = node.Attributes? ["testName"]?.Value ?? "<unknown test name>";
110+
var testOutcome = node.Attributes? ["outcome"]?.Value ?? "<unknown test outcome>";
111+
var testMessage = node.SelectSingleNode ("*[local-name() = 'Output']/*[local-name() = 'ErrorInfo']/*[local-name() = 'Message']")?.InnerText;
112+
if (string.IsNullOrEmpty (testMessage)) {
113+
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)</li>");
114+
} else {
115+
messageLines.Add ($" <li>{testName} (<span style='color: {GetOutcomeColor (testOutcome)}'>{testOutcome}</span>)</li>");
116+
messageLines.Add ($" <div class='pdiv' style='margin-left: 20px;'>");
117+
messageLines.Add (FormatHtml (testMessage));
118+
messageLines.Add ($" </div>");
119+
}
120+
}
121+
messageLines.Add (" </ul>");
122+
allTestsSucceeded = false;
123+
} else if (outcome != "Completed" && outcome != "Passed") {
124+
messageLines.Add ($" Failed to find any test failures in the trx file {path}");
125+
}
126+
}
127+
var htmlPath = Path.ChangeExtension (path, "html");
128+
if (File.Exists (htmlPath)) {
129+
var relativeHtmlPath = Path.GetRelativePath (outputDirectory, htmlPath);
130+
messageLines.Add ($"Html results: <a href='{relativeHtmlPath}'>{Path.GetFileName (relativeHtmlPath)}</a>");
131+
}
132+
} catch (Exception e) {
133+
outcome = "Failed to parse test results";
134+
messageLines.Add ($"<div>{FormatHtml (e.ToString ())}</div>");
135+
allTestsSucceeded = false;
136+
}
137+
138+
indexContents.AppendLine ($" <div class='pdiv'><span>{name} (</span><span style='color: {GetOutcomeColor (outcome)}'>{outcome}</span><span>)</span></div>");
139+
if (messageLines.Any ()) {
140+
indexContents.AppendLine (" <div class='pdiv' style='margin-left: 20px;'>");
141+
foreach (var line in messageLines)
142+
indexContents.AppendLine ($" {line}");
143+
indexContents.AppendLine (" </div>");
144+
}
145+
}
146+
var existingExtraFiles = extraFiles.Where (File.Exists).ToList ();
147+
if (existingExtraFiles.Any ()) {
148+
indexContents.AppendLine ($" <div class='pdiv'>Extra files:</div>");
149+
indexContents.AppendLine ($" <ul>");
150+
foreach (var ef in existingExtraFiles) {
151+
var relative = Path.GetRelativePath (outputDirectory, ef);
152+
indexContents.AppendLine ($" <li><a href='{relative}'>{Path.GetFileName (ef)}</a></li>");
153+
}
154+
indexContents.AppendLine ($" </ul>");
155+
}
156+
indexContents.AppendLine ($" </body>");
157+
indexContents.AppendLine ($"</html>");
158+
159+
if (allTestsSucceeded) {
160+
summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
161+
} else {
162+
summaryContents.AppendLine ($"# :tada: All {trxFiles.Length} tests passed :tada:");
163+
}
164+
165+
166+
Directory.CreateDirectory (outputDirectory);
167+
File.WriteAllText (indexFile, indexContents.ToString ());
168+
File.WriteAllText (summaryFile, summaryContents.ToString ());
169+
170+
Console.WriteLine ($"Created {indexFile} successfully.");
171+
Console.WriteLine (indexContents);
172+
Console.WriteLine ($"Created {summaryFile} successfully.");
173+
Console.WriteLine (summaryContents);
174+
Console.WriteLine ($"All tests succeeded: {allTestsSucceeded}");
175+
return allTestsSucceeded ? 0 : 1;
176+
}
177+
}

‎scripts/create-html-report/create-html-report.csproj

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<Project Sdk="Microsoft.NET.Sdk">
2+
<PropertyGroup>
3+
<OutputType>Exe</OutputType>
4+
<TargetFramework>net$(BundledNETCoreAppTargetFrameworkVersion)</TargetFramework>
5+
<ImplicitUsings>enable</ImplicitUsings>
6+
<Nullable>enable</Nullable>
7+
</PropertyGroup>
8+
</Project>

‎tests/.gitignore

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,5 @@ logs
3333
*.generated.cs
3434
x86
3535
*.7z
36+
TestSummary.md
37+

‎tools/devops/automation/scripts/TestConfiguration.Tests.ps1

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ Describe 'Get-TestConfiguration' {
1313
"label": "cecil",
1414
"splitByPlatforms": "false",
1515
"testPrefix": "test-prefix_",
16+
"testStage": "simulator",
1617
},
1718
{
1819
"label": "dotnettests",
@@ -66,6 +67,7 @@ Describe 'Get-TestConfiguration' {
6667
"cecil": {
6768
"LABEL": "cecil",
6869
"TESTS_LABELS": "extra-test-labels,run-cecil-tests",
70+
"TEST_STAGE": "simulator",
6971
"LABEL_WITH_PLATFORM": "cecil",
7072
"STATUS_CONTEXT": "status-context - cecil",
7173
"TEST_PREFIX": "test-prefix_cecil",
@@ -74,6 +76,7 @@ Describe 'Get-TestConfiguration' {
7476
"dotnettests_iOS": {
7577
"LABEL": "dotnettests",
7678
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
79+
"TEST_STAGE": "test-prefix_",
7780
"LABEL_WITH_PLATFORM": "dotnettests_iOS",
7881
"STATUS_CONTEXT": "status-context - dotnettests - iOS",
7982
"TEST_PREFIX": "test-prefix_dotnettests_iOS",
@@ -83,6 +86,7 @@ Describe 'Get-TestConfiguration' {
8386
"dotnettests_macOS": {
8487
"LABEL": "dotnettests",
8588
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
89+
"TEST_STAGE": "test-prefix_",
8690
"LABEL_WITH_PLATFORM": "dotnettests_macOS",
8791
"STATUS_CONTEXT": "status-context - dotnettests - macOS",
8892
"TEST_PREFIX": "test-prefix_dotnettests_macOS",
@@ -92,6 +96,7 @@ Describe 'Get-TestConfiguration' {
9296
"dotnettests_MacCatalyst": {
9397
"LABEL": "dotnettests",
9498
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
99+
"TEST_STAGE": "test-prefix_",
95100
"LABEL_WITH_PLATFORM": "dotnettests_MacCatalyst",
96101
"STATUS_CONTEXT": "status-context - dotnettests - MacCatalyst",
97102
"TEST_PREFIX": "test-prefix_dotnettests_MacCatalyst",
@@ -101,6 +106,7 @@ Describe 'Get-TestConfiguration' {
101106
"dotnettests_tvOS": {
102107
"LABEL": "dotnettests",
103108
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
109+
"TEST_STAGE": "test-prefix_",
104110
"LABEL_WITH_PLATFORM": "dotnettests_tvOS",
105111
"STATUS_CONTEXT": "status-context - dotnettests - tvOS",
106112
"TEST_PREFIX": "test-prefix_dotnettests_tvOS",
@@ -110,6 +116,7 @@ Describe 'Get-TestConfiguration' {
110116
"dotnettests_Multiple": {
111117
"LABEL": "dotnettests",
112118
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
119+
"TEST_STAGE": "test-prefix_",
113120
"LABEL_WITH_PLATFORM": "dotnettests_Multiple",
114121
"STATUS_CONTEXT": "status-context - dotnettests - Multiple",
115122
"TEST_PREFIX": "test-prefix_dotnettests_Multiple",
@@ -119,6 +126,7 @@ Describe 'Get-TestConfiguration' {
119126
"monotouchtest_ios": {
120127
"LABEL": "monotouchtest",
121128
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
129+
"TEST_STAGE": "test-prefix_",
122130
"LABEL_WITH_PLATFORM": "monotouchtest_iOS",
123131
"STATUS_CONTEXT": "status-context - monotouchtest - iOS",
124132
"TEST_PREFIX": "test-prefix_monotouchtest_ios",
@@ -128,6 +136,7 @@ Describe 'Get-TestConfiguration' {
128136
"monotouchtest_macos": {
129137
"LABEL": "monotouchtest",
130138
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
139+
"TEST_STAGE": "test-prefix_",
131140
"LABEL_WITH_PLATFORM": "monotouchtest_macOS",
132141
"STATUS_CONTEXT": "status-context - monotouchtest - macOS",
133142
"TEST_PREFIX": "test-prefix_monotouchtest_macos",
@@ -137,6 +146,7 @@ Describe 'Get-TestConfiguration' {
137146
"monotouchtest_maccatalyst": {
138147
"LABEL": "monotouchtest",
139148
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
149+
"TEST_STAGE": "test-prefix_",
140150
"LABEL_WITH_PLATFORM": "monotouchtest_MacCatalyst",
141151
"STATUS_CONTEXT": "status-context - monotouchtest - MacCatalyst",
142152
"TEST_PREFIX": "test-prefix_monotouchtest_maccatalyst",
@@ -146,6 +156,7 @@ Describe 'Get-TestConfiguration' {
146156
"monotouchtest_tvos": {
147157
"LABEL": "monotouchtest",
148158
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
159+
"TEST_STAGE": "test-prefix_",
149160
"LABEL_WITH_PLATFORM": "monotouchtest_tvOS",
150161
"STATUS_CONTEXT": "status-context - monotouchtest - tvOS",
151162
"TEST_PREFIX": "test-prefix_monotouchtest_tvos",
@@ -172,6 +183,7 @@ Describe 'Get-TestConfiguration' {
172183
"cecil": {
173184
"LABEL": "cecil",
174185
"TESTS_LABELS": "extra-test-labels,run-cecil-tests",
186+
"TEST_STAGE": "simulator",
175187
"LABEL_WITH_PLATFORM": "cecil",
176188
"STATUS_CONTEXT": "status-context - cecil",
177189
"TEST_PREFIX": "test-prefix_cecil",
@@ -180,6 +192,7 @@ Describe 'Get-TestConfiguration' {
180192
"dotnettests_iOS": {
181193
"LABEL": "dotnettests",
182194
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
195+
"TEST_STAGE": "test-prefix_",
183196
"LABEL_WITH_PLATFORM": "dotnettests_iOS",
184197
"STATUS_CONTEXT": "status-context - dotnettests - iOS",
185198
"TEST_PREFIX": "test-prefix_dotnettests_iOS",
@@ -189,6 +202,7 @@ Describe 'Get-TestConfiguration' {
189202
"monotouchtest_ios": {
190203
"LABEL": "monotouchtest",
191204
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
205+
"TEST_STAGE": "test-prefix_",
192206
"LABEL_WITH_PLATFORM": "monotouchtest_iOS",
193207
"STATUS_CONTEXT": "status-context - monotouchtest - iOS",
194208
"TEST_PREFIX": "test-prefix_monotouchtest_ios",
@@ -200,7 +214,7 @@ Describe 'Get-TestConfiguration' {
200214

201215
}
202216

203-
It 'suceeds when no dotnet platforms enabled' {
217+
It 'succeeds when no dotnet platforms enabled' {
204218
$EnabledPlatforms = ""
205219

206220
$config = Get-TestConfiguration `
@@ -215,6 +229,7 @@ Describe 'Get-TestConfiguration' {
215229
"cecil": {
216230
"LABEL": "cecil",
217231
"TESTS_LABELS": "extra-test-labels,run-cecil-tests",
232+
"TEST_STAGE": "simulator",
218233
"LABEL_WITH_PLATFORM": "cecil",
219234
"STATUS_CONTEXT": "status-context - cecil",
220235
"TEST_PREFIX": "test-prefix_cecil",
@@ -240,6 +255,7 @@ Describe 'Get-TestConfiguration' {
240255
"cecil": {
241256
"LABEL": "cecil",
242257
"TESTS_LABELS": "extra-test-labels,run-cecil-tests",
258+
"TEST_STAGE": "simulator",
243259
"LABEL_WITH_PLATFORM": "cecil",
244260
"STATUS_CONTEXT": "status-context - cecil",
245261
"TEST_PREFIX": "test-prefix_cecil",
@@ -248,6 +264,7 @@ Describe 'Get-TestConfiguration' {
248264
"dotnettests_iOS": {
249265
"LABEL": "dotnettests",
250266
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
267+
"TEST_STAGE": "test-prefix_",
251268
"LABEL_WITH_PLATFORM": "dotnettests_iOS",
252269
"STATUS_CONTEXT": "status-context - dotnettests - iOS",
253270
"TEST_PREFIX": "test-prefix_dotnettests_iOS",
@@ -257,6 +274,7 @@ Describe 'Get-TestConfiguration' {
257274
"dotnettests_macOS": {
258275
"LABEL": "dotnettests",
259276
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
277+
"TEST_STAGE": "test-prefix_",
260278
"LABEL_WITH_PLATFORM": "dotnettests_macOS",
261279
"STATUS_CONTEXT": "status-context - dotnettests - macOS",
262280
"TEST_PREFIX": "test-prefix_dotnettests_macOS",
@@ -266,6 +284,7 @@ Describe 'Get-TestConfiguration' {
266284
"dotnettests_MacCatalyst": {
267285
"LABEL": "dotnettests",
268286
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
287+
"TEST_STAGE": "test-prefix_",
269288
"LABEL_WITH_PLATFORM": "dotnettests_MacCatalyst",
270289
"STATUS_CONTEXT": "status-context - dotnettests - MacCatalyst",
271290
"TEST_PREFIX": "test-prefix_dotnettests_MacCatalyst",
@@ -275,6 +294,7 @@ Describe 'Get-TestConfiguration' {
275294
"dotnettests_Multiple": {
276295
"LABEL": "dotnettests",
277296
"TESTS_LABELS": "extra-test-labels,run-dotnettests-tests",
297+
"TEST_STAGE": "test-prefix_",
278298
"LABEL_WITH_PLATFORM": "dotnettests_Multiple",
279299
"STATUS_CONTEXT": "status-context - dotnettests - Multiple",
280300
"TEST_PREFIX": "test-prefix_dotnettests_Multiple",
@@ -284,6 +304,7 @@ Describe 'Get-TestConfiguration' {
284304
"monotouchtest_ios": {
285305
"LABEL": "monotouchtest",
286306
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
307+
"TEST_STAGE": "test-prefix_",
287308
"LABEL_WITH_PLATFORM": "monotouchtest_iOS",
288309
"STATUS_CONTEXT": "status-context - monotouchtest - iOS",
289310
"TEST_PREFIX": "test-prefix_monotouchtest_ios",
@@ -293,6 +314,7 @@ Describe 'Get-TestConfiguration' {
293314
"monotouchtest_macos": {
294315
"LABEL": "monotouchtest",
295316
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
317+
"TEST_STAGE": "test-prefix_",
296318
"LABEL_WITH_PLATFORM": "monotouchtest_macOS",
297319
"STATUS_CONTEXT": "status-context - monotouchtest - macOS",
298320
"TEST_PREFIX": "test-prefix_monotouchtest_macos",
@@ -302,6 +324,7 @@ Describe 'Get-TestConfiguration' {
302324
"monotouchtest_maccatalyst": {
303325
"LABEL": "monotouchtest",
304326
"TESTS_LABELS": "extra-test-labels,run-monotouchtest-tests",
327+
"TEST_STAGE": "test-prefix_",
305328
"LABEL_WITH_PLATFORM": "monotouchtest_MacCatalyst",
306329
"STATUS_CONTEXT": "status-context - monotouchtest - MacCatalyst",
307330
"TEST_PREFIX": "test-prefix_monotouchtest_maccatalyst",

0 commit comments

Comments
 (0)