Increase granularity of JUnit reports. (#509)

Signed-off-by: Marcin Owsiany <[email protected]>

pkg/report/report.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"sync"
 	"time"
 )
 
@@ -67,7 +68,7 @@ type Testcase struct {
 	end time.Time
 }
 
-// TestSuite is a collection of Testcase and is a summary of those details.
+// Testsuite is a collection of Testcase and is a summary of those details.
 type Testsuite struct {
 	// Tests is the number of Testcases in the collection.
 	Tests int `xml:"tests,attr" json:"tests"`
@@ -82,8 +83,11 @@ type Testsuite struct {
 	Name string `xml:"name,attr" json:"name"`
 	// Properties which are specific to this suite.
 	Properties *Properties `xml:"properties" json:"properties,omitempty"`
-	// Testcase is a collection of test cases.
-	Testcase []*Testcase `xml:"testcase" json:"testcase,omitempty"`
+	// Testcases is a collection of test cases.
+	Testcases []*Testcase `xml:"testcase" json:"testcase,omitempty"`
+	// SubSuites is a collection of child test suites.
+	SubSuites []*Testsuite `xml:"testsuite" json:"testsuite,omitempty"`
+	lock      sync.Mutex
 }
 
 // Testsuites is a collection of Testsuite and defines the rollup summary of all stats.
@@ -148,7 +152,7 @@ func (ts *Testsuite) AddTestcase(testcase *Testcase) {
 	testcase.Time = fmt.Sprintf("%.3f", elapsed.Seconds())
 	testcase.Classname = filepath.Base(ts.Name)
 
-	ts.Testcase = append(ts.Testcase, testcase)
+	ts.Testcases = append(ts.Testcases, testcase)
 	ts.Tests++
 	if testcase.Failure != nil {
 		ts.Failures++
@@ -168,6 +172,37 @@ func (ts *Testsuite) AddProperty(property Property) {
 	ts.Properties.Property = append(ts.Properties.Property, property)
 }
 
+// NewSubSuite creates a new child suite and returns it.
+func (ts *Testsuite) NewSubSuite(name string) *Testsuite {
+	s := NewSuite(name)
+	ts.lock.Lock()
+	defer ts.lock.Unlock()
+	ts.SubSuites = append(ts.SubSuites, s)
+	return s
+}
+
+// summarize sets counters and elapsed time attributes based on children.
+// Returns the timestamp of the end of latest child.
+func (ts *Testsuite) summarize() time.Time {
+	end := ts.Timestamp
+	for _, subSuite := range ts.SubSuites {
+		subSuiteEnd := subSuite.summarize()
+		if subSuiteEnd.After(end) {
+			end = subSuiteEnd
+		}
+		ts.Tests += subSuite.Tests
+		ts.Failures += subSuite.Failures
+	}
+	for _, testcase := range ts.Testcases {
+		if testcase.end.After(end) {
+			end = testcase.end
+		}
+	}
+	elapsed := end.Sub(ts.Timestamp)
+	ts.Time = fmt.Sprintf("%.3f", elapsed.Seconds())
+	return end
+}
+
 // AddTestSuite is a convenience method to add a testsuite to the collection in testsuites
 func (ts *Testsuites) AddTestSuite(testsuite *Testsuite) {
 	// testsuite is added prior to stat availability, stat management in the close of the testsuites
@@ -194,24 +229,14 @@ func (ts *Testsuites) Close() {
 
 	// async work makes this necessary (stats for each testsuite)
 	for _, testsuite := range ts.Testsuite {
-		elapsed = latestEnd(testsuite.Timestamp, testsuite.Testcase).Sub(testsuite.Timestamp)
-		testsuite.Time = fmt.Sprintf("%.3f", elapsed.Seconds())
+		testsuite.summarize()
 
 		ts.Tests += testsuite.Tests
 		ts.Failures += testsuite.Failures
 	}
 }
 
 // latestEnd provides the time of the latest end out of the collection of testcases
-func latestEnd(start time.Time, testcases []*Testcase) time.Time {
-	end := start
-	for _, testcase := range testcases {
-		if testcase.end.After(end) {
-			end = testcase.end
-		}
-	}
-	return end
-}
 
 // Report prints a report for TestSuites to the directory.  ftype == json | xml
 func (ts *Testsuites) Report(dir, name string, ftype Type) error {

pkg/report/report_test.go

@@ -36,9 +36,19 @@ AssertionError`,
 		Failures: 1,
 		Time:     "",
 		Name:     "github.com/kubebuilder/kuttl/pkg/version",
-		Testcase: []*Testcase{
+		Testcases: []*Testcase{
 			tcase,
 		},
+		SubSuites: []*Testsuite{
+			{
+				Tests:    7,
+				Failures: 1,
+				Name:     "sub-test-suite",
+				Testcases: []*Testcase{
+					tcase,
+				},
+			},
+		},
 	}
 
 	suites := Testsuites{

pkg/report/testdata/report.json.golden

@@ -29,6 +29,27 @@
              "message": "test failure"
            }
          }
+       ],
+       "testsuite": [
+         {
+           "tests": 7,
+           "failures": 1,
+           "timestamp": "0001-01-01T00:00:00Z",
+           "time": "",
+           "name": "sub-test-suite",
+           "testcase": [
+             {
+               "classname": "pkg1.test.test_things",
+               "name": "test_params_func:2",
+               "timestamp": "0001-01-01T00:00:00Z",
+               "time": "",
+               "failure": {
+                 "text": "Traceback (most recent call last):\n  File \"nose2/plugins/loader/parameters.py\", line 162, in func\n    return obj(*argSet)\n  File \"nose2/tests/functional/support/scenario/tests_in_package/pkg1/test/test_things.py\", line 64, in test_params_func\n    assert a == 1\nAssertionError",
+                 "message": "test failure"
+               }
+             }
+           ]
+         }
        ]
      }
    ]

pkg/report/testdata/report.xml.golden

@@ -6,5 +6,10 @@
      <testcase classname="pkg1.test.test_things" name="test_params_func:2" timestamp="0001-01-01T00:00:00Z" time="" assertions="0">
        <failure message="test failure" type="">Traceback (most recent call last):&#xA;  File &#34;nose2/plugins/loader/parameters.py&#34;, line 162, in func&#xA;    return obj(*argSet)&#xA;  File &#34;nose2/tests/functional/support/scenario/tests_in_package/pkg1/test/test_things.py&#34;, line 64, in test_params_func&#xA;    assert a == 1&#xA;AssertionError</failure>
      </testcase>
+     <testsuite tests="7" failures="1" timestamp="0001-01-01T00:00:00Z" time="" name="sub-test-suite">
+       <testcase classname="pkg1.test.test_things" name="test_params_func:2" timestamp="0001-01-01T00:00:00Z" time="" assertions="0">
+         <failure message="test failure" type="">Traceback (most recent call last):&#xA;  File &#34;nose2/plugins/loader/parameters.py&#34;, line 162, in func&#xA;    return obj(*argSet)&#xA;  File &#34;nose2/tests/functional/support/scenario/tests_in_package/pkg1/test/test_things.py&#34;, line 64, in test_params_func&#xA;    assert a == 1&#xA;AssertionError</failure>
+       </testcase>
+     </testsuite>
    </testsuite>
  </testsuites>

pkg/test/case.go

@@ -314,12 +314,14 @@ func shortString(obj *corev1.ObjectReference) string {
 }
 
 // Run runs a test case including all of its steps.
-func (t *Case) Run(test *testing.T, tc *report.Testcase) {
+func (t *Case) Run(test *testing.T, ts *report.Testsuite) {
+	setupReport := report.NewCase("setup")
 	ns := t.determineNamespace()
 
 	cl, err := t.Client(false)
 	if err != nil {
-		tc.Failure = report.NewFailure(err.Error(), nil)
+		setupReport.Failure = report.NewFailure(err.Error(), nil)
+		ts.AddTestcase(setupReport)
 		test.Fatal(err)
 	}
 
@@ -332,7 +334,8 @@ func (t *Case) Run(test *testing.T, tc *report.Testcase) {
 
 		cl, err := newClient(testStep.Kubeconfig)(false)
 		if err != nil {
-			tc.Failure = report.NewFailure(err.Error(), nil)
+			setupReport.Failure = report.NewFailure(err.Error(), nil)
+			ts.AddTestcase(setupReport)
 			test.Fatal(err)
 		}
 
@@ -341,12 +344,15 @@ func (t *Case) Run(test *testing.T, tc *report.Testcase) {
 
 	for _, c := range clients {
 		if err := t.CreateNamespace(test, c, ns); err != nil {
-			tc.Failure = report.NewFailure(err.Error(), nil)
+			setupReport.Failure = report.NewFailure(err.Error(), nil)
+			ts.AddTestcase(setupReport)
 			test.Fatal(err)
 		}
 	}
+	ts.AddTestcase(setupReport)
 
 	for _, testStep := range t.Steps {
+		tc := report.NewCase("step " + testStep.String())
 		testStep.Client = t.Client
 		if testStep.Kubeconfig != "" {
 			testStep.Client = newClient(testStep.Kubeconfig)
@@ -359,14 +365,18 @@ func (t *Case) Run(test *testing.T, tc *report.Testcase) {
 		tc.Assertions += len(testStep.Asserts)
 		tc.Assertions += len(testStep.Errors)
 
-		if errs := testStep.Run(test, ns.Name); len(errs) > 0 {
+		errs := testStep.Run(test, ns.Name)
+		if len(errs) > 0 {
 			caseErr := fmt.Errorf("failed in step %s", testStep.String())
 			tc.Failure = report.NewFailure(caseErr.Error(), errs)
 
 			test.Error(caseErr)
 			for _, err := range errs {
 				test.Error(err)
 			}
+		}
+		ts.AddTestcase(tc)
+		if len(errs) > 0 {
 			break
 		}
 	}

pkg/test/case_integration_test.go

@@ -111,5 +111,5 @@ func TestMultiClusterCase(t *testing.T) {
 		},
 	}
 
-	c.Run(t, &report.Testcase{})
+	c.Run(t, &report.Testsuite{})
 }

pkg/test/harness.go

@@ -379,7 +379,7 @@ func (h *Harness) RunTests() {
 
 	h.T.Run("harness", func(t *testing.T) {
 		for testDir, tests := range realTestSuite {
-			suite := h.report.NewSuite(testDir)
+			suiteReport := h.report.NewSuite(testDir)
 			for _, test := range tests {
 				test := test
 
@@ -397,9 +397,8 @@ func (h *Harness) RunTests() {
 						t.Fatal(err)
 					}
 
-					tc := report.NewCase(test.Name)
-					test.Run(t, tc)
-					suite.AddTestcase(tc)
+					testReport := suiteReport.NewSubSuite(test.Name)
+					test.Run(t, testReport)
 				})
 			}
 		}