From 23de9a94de39615c134d7438f8c574a7c735038c Mon Sep 17 00:00:00 2001 From: George Adams Date: Mon, 16 Nov 2026 22:10:50 +0000 Subject: [PATCH 1/3] json2junit: preserve repeated test runs Allow completed test names to run again within a package, as happens with internal and external tests and go test -count. Preserve each result and assign unique JUnit names while retaining errors for overlapping duplicate runs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- json2junit/json2junit.go | 40 +++++---- json2junit/json2junit_test.go | 82 +++++++++++++++++++ .../repeatednames/repeatednames.xml | 9 ++ .../testdata/inputs/good/repeatednames.jsonl | 10 +++ 4 files changed, 126 insertions(+), 15 deletions(-) create mode 100644 json2junit/testdata/TestConverter/repeatednames/repeatednames.xml create mode 100644 json2junit/testdata/inputs/good/repeatednames.jsonl diff --git a/json2junit/json2junit.go b/json2junit/json2junit.go index 5a2c741a..3bdd7b0d 100644 --- a/json2junit/json2junit.go +++ b/json2junit/json2junit.go @@ -82,6 +82,10 @@ func ConvertFileWithOptions(out, in string, opts *Options) error { // size chunks that don't have to align with the JSON lines. // The JUnit output is not written immediately, but chunked into test // suites which are written to w as soon as they are complete. +// +// Repeated test names are supported when their runs do not overlap, as can +// happen with tests in a package and its external test package, or with -count. +// Later runs receive a numeric suffix in their JUnit names. type Converter struct { b []byte // input buffer suites map[string]*junitTestSuite @@ -171,17 +175,18 @@ type jsonEntry struct { // https://llg.cubic.org/docs/junit/. type junitTestSuite struct { - XMLName xml.Name `xml:"testsuite"` - Name string `xml:"name,attr"` - Tests int `xml:"tests,attr"` - Failures int `xml:"failures,attr"` - Skipped int `xml:"skipped,attr"` - Time float64 `xml:"time,attr"` - Timestamp string `xml:"timestamp,attr"` - SystemOut *systemOut `xml:",omitempty"` - TestCases []*junitTestCase // Empty until written to XML. Will be filled in with testCases. + XMLName xml.Name `xml:"testsuite"` + Name string `xml:"name,attr"` + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Skipped int `xml:"skipped,attr"` + Time float64 `xml:"time,attr"` + Timestamp string `xml:"timestamp,attr"` + SystemOut *systemOut `xml:",omitempty"` + TestCases []*junitTestCase - testCases map[string]*junitTestCase `xml:"-"` // Map from test name to *junitTestCase. + testCases map[string]*junitTestCase `xml:"-"` // Active test cases, keyed by Go test name. + testNames map[string]bool `xml:"-"` // JUnit names already used in this suite. } type systemOut struct { @@ -221,6 +226,7 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { Name: suiteName, Timestamp: entry.Time.Format(time.RFC3339), testCases: make(map[string]*junitTestCase), + testNames: make(map[string]bool), } case "run": // start a new test case. @@ -236,10 +242,17 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { if c.opts.IncludePackageInTestName { name = entry.Package + "." + entry.Test } - suite.testCases[entry.Test] = &junitTestCase{ + baseName := name + for n := 2; suite.testNames[name]; n++ { + name = fmt.Sprintf("%s [%d]", baseName, n) + } + suite.testNames[name] = true + testCase := &junitTestCase{ Name: name, Classname: entry.Package, } + suite.testCases[entry.Test] = testCase + suite.TestCases = append(suite.TestCases, testCase) case "output": // append output to the current test case or suite. suite, ok := c.suites[entry.Package] @@ -305,6 +318,7 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { // Clear systemOut, it's already in the event. // In case of success, we don't care about the output. testCase.systemOut = nil + delete(suite.testCases, entry.Test) case "pause", "cont": // Ignore. default: @@ -316,10 +330,6 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { // writeXMLTestSuite writes a single test suite to the writer. // Test cases are sorted by name. func (c *Converter) writeXMLTestSuite(suite *junitTestSuite) error { - suite.TestCases = make([]*junitTestCase, 0, len(suite.testCases)) - for _, testCase := range suite.testCases { - suite.TestCases = append(suite.TestCases, testCase) - } slices.SortFunc(suite.TestCases, func(a, b *junitTestCase) int { return cmp.Compare(a.Name, b.Name) }) diff --git a/json2junit/json2junit_test.go b/json2junit/json2junit_test.go index a84a02ae..9bbf6726 100644 --- a/json2junit/json2junit_test.go +++ b/json2junit/json2junit_test.go @@ -4,6 +4,8 @@ package json2junit import ( + "bytes" + "encoding/xml" "io" "os" "path/filepath" @@ -13,6 +15,86 @@ import ( "github.com/microsoft/go-infra/goldentest" ) +func TestConverterRepeatedTests(t *testing.T) { + const input = `{"Action":"start","Package":"crypto/rand"} +{"Action":"run","Package":"crypto/rand","Test":"TestAllocations"} +{"Action":"output","Package":"crypto/rand","Test":"TestAllocations","Output":"first run\n"} +{"Action":"fail","Package":"crypto/rand","Test":"TestAllocations","Elapsed":1} +{"Action":"run","Package":"crypto/rand","Test":"TestAllocations"} +{"Action":"output","Package":"crypto/rand","Test":"TestAllocations","Output":"second run\n"} +{"Action":"skip","Package":"crypto/rand","Test":"TestAllocations","Elapsed":2} +{"Action":"run","Package":"crypto/rand","Test":"TestAllocations"} +{"Action":"output","Package":"crypto/rand","Test":"TestAllocations","Output":"third run\n"} +{"Action":"pass","Package":"crypto/rand","Test":"TestAllocations","Elapsed":3} +{"Action":"fail","Package":"crypto/rand","Elapsed":6} +` + for _, includePackage := range []bool{false, true} { + name := "without_package" + prefix := "" + if includePackage { + name = "with_package" + prefix = "crypto/rand." + } + t.Run(name, func(t *testing.T) { + var out bytes.Buffer + if err := ConvertWithOptions(&out, strings.NewReader(input), &Options{ + IncludePackageInTestName: includePackage, + }); err != nil { + t.Fatal(err) + } + type testResult struct { + Content string `xml:",cdata"` + } + var result struct { + Suites []struct { + Tests int `xml:"tests,attr"` + Failures int `xml:"failures,attr"` + Skipped int `xml:"skipped,attr"` + TestCases []struct { + Name string `xml:"name,attr"` + Classname string `xml:"classname,attr"` + Time float64 `xml:"time,attr"` + Failure *testResult `xml:"failure"` + Skipped *testResult `xml:"skipped"` + } `xml:"testcase"` + } `xml:"testsuite"` + } + if err := xml.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatal(err) + } + if len(result.Suites) != 1 { + t.Fatalf("got %d suites, want 1", len(result.Suites)) + } + suite := result.Suites[0] + if suite.Tests != 3 || suite.Failures != 1 || suite.Skipped != 1 { + t.Errorf("got tests=%d failures=%d skipped=%d, want 3, 1, 1", suite.Tests, suite.Failures, suite.Skipped) + } + if len(suite.TestCases) != 3 { + t.Fatalf("got %d test cases, want 3", len(suite.TestCases)) + } + for i, wantName := range []string{"TestAllocations", "TestAllocations [2]", "TestAllocations [3]"} { + got := suite.TestCases[i] + if got.Name != prefix+wantName || got.Classname != "crypto/rand" || got.Time != float64(i+1) { + t.Errorf("test case %d: got name=%q classname=%q time=%v", i, got.Name, got.Classname, got.Time) + } + if i == 2 { + if got.Failure != nil || got.Skipped != nil { + t.Errorf("passing run has failure=%+v skipped=%+v", got.Failure, got.Skipped) + } + continue + } + gotResult, otherResult, wantOutput := got.Failure, got.Skipped, "first run" + if i == 1 { + gotResult, otherResult, wantOutput = got.Skipped, got.Failure, "second run" + } + if gotResult == nil || gotResult.Content != wantOutput || otherResult != nil { + t.Errorf("test case %d: got result=%+v other=%+v, want output %q", i, gotResult, otherResult, wantOutput) + } + } + }) + } +} + func TestConverter(t *testing.T) { dir := filepath.Join("testdata", "inputs", "good") files, err := os.ReadDir(dir) diff --git a/json2junit/testdata/TestConverter/repeatednames/repeatednames.xml b/json2junit/testdata/TestConverter/repeatednames/repeatednames.xml new file mode 100644 index 00000000..87518023 --- /dev/null +++ b/json2junit/testdata/TestConverter/repeatednames/repeatednames.xml @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/json2junit/testdata/inputs/good/repeatednames.jsonl b/json2junit/testdata/inputs/good/repeatednames.jsonl new file mode 100644 index 00000000..61614f25 --- /dev/null +++ b/json2junit/testdata/inputs/good/repeatednames.jsonl @@ -0,0 +1,10 @@ +{"Time":"2026-11-16T21:57:36Z","Action":"start","Package":"example"} +{"Action":"run","Package":"example","Test":"TestRepeated [2]"} +{"Action":"pass","Package":"example","Test":"TestRepeated [2]","Elapsed":1} +{"Action":"run","Package":"example","Test":"TestRepeated"} +{"Action":"pass","Package":"example","Test":"TestRepeated","Elapsed":2} +{"Action":"run","Package":"example","Test":"TestRepeated"} +{"Action":"pass","Package":"example","Test":"TestRepeated","Elapsed":3} +{"Action":"run","Package":"example","Test":"TestRepeated [2]"} +{"Action":"pass","Package":"example","Test":"TestRepeated [2]","Elapsed":4} +{"Action":"pass","Package":"example","Elapsed":10} From a8c7f2f1462ac8b8bdf51f9e185cfac8ca85309c Mon Sep 17 00:00:00 2001 From: George Adams Date: Mon, 16 Nov 2026 22:17:03 +0000 Subject: [PATCH 2/3] json2junit: resume suffix searches for repeated tests Remember the next suffix for each base JUnit name instead of rescanning all previous runs. Keep used-name checks for collisions with literal test names, and cover collisions and repeated-run scaling with tests and benchmarks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- json2junit/json2junit.go | 19 +++++---- json2junit/json2junit_test.go | 79 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/json2junit/json2junit.go b/json2junit/json2junit.go index 3bdd7b0d..fabf5624 100644 --- a/json2junit/json2junit.go +++ b/json2junit/json2junit.go @@ -185,8 +185,9 @@ type junitTestSuite struct { SystemOut *systemOut `xml:",omitempty"` TestCases []*junitTestCase - testCases map[string]*junitTestCase `xml:"-"` // Active test cases, keyed by Go test name. - testNames map[string]bool `xml:"-"` // JUnit names already used in this suite. + testCases map[string]*junitTestCase `xml:"-"` // Active test cases, keyed by Go test name. + testNames map[string]bool `xml:"-"` // JUnit names already used in this suite. + nextTestSuffix map[string]int `xml:"-"` // Next suffix to try for each base JUnit name. } type systemOut struct { @@ -223,10 +224,11 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { suiteName += " [attempt " + c.opts.JobAttempt + "]" } c.suites[entry.Package] = &junitTestSuite{ - Name: suiteName, - Timestamp: entry.Time.Format(time.RFC3339), - testCases: make(map[string]*junitTestCase), - testNames: make(map[string]bool), + Name: suiteName, + Timestamp: entry.Time.Format(time.RFC3339), + testCases: make(map[string]*junitTestCase), + testNames: make(map[string]bool), + nextTestSuffix: make(map[string]int), } case "run": // start a new test case. @@ -243,9 +245,12 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { name = entry.Package + "." + entry.Test } baseName := name - for n := 2; suite.testNames[name]; n++ { + n := max(2, suite.nextTestSuffix[baseName]) + for suite.testNames[name] { name = fmt.Sprintf("%s [%d]", baseName, n) + n++ } + suite.nextTestSuffix[baseName] = n suite.testNames[name] = true testCase := &junitTestCase{ Name: name, diff --git a/json2junit/json2junit_test.go b/json2junit/json2junit_test.go index 9bbf6726..3398839e 100644 --- a/json2junit/json2junit_test.go +++ b/json2junit/json2junit_test.go @@ -6,15 +6,94 @@ package json2junit import ( "bytes" "encoding/xml" + "fmt" "io" "os" "path/filepath" + "slices" "strings" "testing" "github.com/microsoft/go-infra/goldentest" ) +func TestConverterRepeatedNameCollisions(t *testing.T) { + const input = `{"Action":"start","Package":"example"} +{"Action":"run","Package":"example","Test":"TestRepeated [2]"} +{"Action":"pass","Package":"example","Test":"TestRepeated [2]"} +{"Action":"run","Package":"example","Test":"TestRepeated"} +{"Action":"pass","Package":"example","Test":"TestRepeated"} +{"Action":"run","Package":"example","Test":"TestRepeated"} +{"Action":"pass","Package":"example","Test":"TestRepeated"} +{"Action":"run","Package":"example","Test":"TestRepeated [4]"} +{"Action":"pass","Package":"example","Test":"TestRepeated [4]"} +{"Action":"run","Package":"example","Test":"TestRepeated"} +{"Action":"pass","Package":"example","Test":"TestRepeated"} +{"Action":"run","Package":"example","Test":"TestRepeated [3]"} +{"Action":"pass","Package":"example","Test":"TestRepeated [3]"} +{"Action":"pass","Package":"example"} +` + for _, includePackage := range []bool{false, true} { + t.Run(fmt.Sprintf("include_package=%t", includePackage), func(t *testing.T) { + var out bytes.Buffer + if err := ConvertWithOptions(&out, strings.NewReader(input), &Options{ + IncludePackageInTestName: includePackage, + }); err != nil { + t.Fatal(err) + } + var result struct { + Suite struct { + Tests int `xml:"tests,attr"` + TestCases []struct { + Name string `xml:"name,attr"` + } `xml:"testcase"` + } `xml:"testsuite"` + } + if err := xml.Unmarshal(out.Bytes(), &result); err != nil { + t.Fatal(err) + } + want := []string{ + "TestRepeated", "TestRepeated [2]", "TestRepeated [3]", + "TestRepeated [3] [2]", "TestRepeated [4]", "TestRepeated [5]", + } + if includePackage { + for i := range want { + want[i] = "example." + want[i] + } + } + var got []string + for _, testCase := range result.Suite.TestCases { + got = append(got, testCase.Name) + } + if result.Suite.Tests != len(want) || !slices.Equal(got, want) { + t.Errorf("got tests=%d names=%q, want tests=%d names=%q", result.Suite.Tests, got, len(want), want) + } + }) + } +} + +func BenchmarkConverterRepeatedTests(b *testing.B) { + for _, runs := range []int{1000, 4000} { + b.Run(fmt.Sprint(runs), func(b *testing.B) { + var input strings.Builder + input.WriteString("{\"Action\":\"start\",\"Package\":\"example\"}\n") + for range runs { + input.WriteString("{\"Action\":\"run\",\"Package\":\"example\",\"Test\":\"TestRepeated\"}\n") + input.WriteString("{\"Action\":\"pass\",\"Package\":\"example\",\"Test\":\"TestRepeated\"}\n") + } + input.WriteString("{\"Action\":\"pass\",\"Package\":\"example\"}\n") + data := input.String() + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if err := Convert(io.Discard, strings.NewReader(data)); err != nil { + b.Fatal(err) + } + } + }) + } +} + func TestConverterRepeatedTests(t *testing.T) { const input = `{"Action":"start","Package":"crypto/rand"} {"Action":"run","Package":"crypto/rand","Test":"TestAllocations"} From a362ac872c7e9ba452a183921a2a6699e28737f3 Mon Sep 17 00:00:00 2001 From: George Adams Date: Mon, 16 Nov 2026 23:36:53 +0000 Subject: [PATCH 3/3] json2junit: use zero-sized set values Use struct{} for the used-name set, which only needs membership checks. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- json2junit/json2junit.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/json2junit/json2junit.go b/json2junit/json2junit.go index fabf5624..104c7613 100644 --- a/json2junit/json2junit.go +++ b/json2junit/json2junit.go @@ -186,7 +186,7 @@ type junitTestSuite struct { TestCases []*junitTestCase testCases map[string]*junitTestCase `xml:"-"` // Active test cases, keyed by Go test name. - testNames map[string]bool `xml:"-"` // JUnit names already used in this suite. + testNames map[string]struct{} `xml:"-"` // JUnit names already used in this suite. nextTestSuffix map[string]int `xml:"-"` // Next suffix to try for each base JUnit name. } @@ -227,7 +227,7 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { Name: suiteName, Timestamp: entry.Time.Format(time.RFC3339), testCases: make(map[string]*junitTestCase), - testNames: make(map[string]bool), + testNames: make(map[string]struct{}), nextTestSuffix: make(map[string]int), } case "run": @@ -246,12 +246,15 @@ func (c *Converter) processJSONEntry(entry jsonEntry) error { } baseName := name n := max(2, suite.nextTestSuffix[baseName]) - for suite.testNames[name] { + for { + if _, used := suite.testNames[name]; !used { + break + } name = fmt.Sprintf("%s [%d]", baseName, n) n++ } suite.nextTestSuffix[baseName] = n - suite.testNames[name] = true + suite.testNames[name] = struct{}{} testCase := &junitTestCase{ Name: name, Classname: entry.Package,