Show output of failed tests

Author: bitfield

README.md

@@ -58,7 +58,7 @@ Some more advanced ways to use `gotestdox`:
 
 ## Exit status
 
-If there are any test failures, `gotestdox` will report exit status 1.
+If there are any test failures, `gotestdox` will print the output messages from the offending test and report status 1 on exit.
 
 ## Colour
 
@@ -94,7 +94,8 @@ github.com/octocat/mymodule/api:
  ✔ NewServer returns a correctly configured server (0.00s)
 
 github.com/octocat/mymodule/util:
- ✔ LeftPad adds the correct number of leading spaces (0.00s)
+ x LeftPad adds the correct number of leading spaces (0.00s)
+    util_test.go:133: want "  dummy", got " dummy"
  ```
 
 ## Multi-word function names

gotestdox.go

@@ -14,6 +14,43 @@ import (
 	"github.com/mattn/go-isatty"
 )
 
+const Usage = `gotestdox is a command-line tool for turning Go test names into readable sentences.
+
+Usage:
+
+	gotestdox [ARGS]
+
+This will run 'go test -json [ARGS]' in the current directory and format the results in a readable
+way. You can use any arguments that 'go test -json' accepts, including a list of packages, for
+example.
+
+If the standard input is not an interactive terminal, gotestdox will assume you want to pipe JSON
+data into it. For example:
+
+	go test -json |gotestdox
+
+See https://github.com/bitfield/gotestdox for more information.`
+
+// Main runs the command-line interface for gotestdox. The exit status for the
+// binary is 0 if the tests passed, or 1 if the tests failed, or there was some
+// error.
+func Main() int {
+	if len(os.Args) > 1 && os.Args[1] == "-h" {
+		fmt.Println(Usage)
+		return 0
+	}
+	td := NewTestDoxer()
+	if isatty.IsTerminal(os.Stdin.Fd()) {
+		td.ExecGoTest(os.Args[1:])
+	} else {
+		td.Filter()
+	}
+	if !td.OK {
+		return 1
+	}
+	return 0
+}
+
 // TestDoxer holds the state and config associated with a particular invocation
 // of 'go test'.
 type TestDoxer struct {
@@ -72,6 +109,7 @@ func (td *TestDoxer) ExecGoTest(userArgs []string) {
 func (td *TestDoxer) Filter() {
 	td.OK = true
 	results := map[string][]Event{}
+	outputs := map[string][]string{}
 	scanner := bufio.NewScanner(td.Stdin)
 	for scanner.Scan() {
 		event, err := ParseJSON(scanner.Text())
@@ -80,27 +118,46 @@ func (td *TestDoxer) Filter() {
 			fmt.Fprintln(td.Stderr, err)
 			return
 		}
-		if event.Action == "fail" {
-			td.OK = false
-		}
-		if event.IsPackageResult() {
+		switch {
+		case event.IsPackageResult():
 			fmt.Fprintf(td.Stdout, "%s:\n", event.Package)
 			tests := results[event.Package]
 			sort.Slice(tests, func(i, j int) bool {
 				return tests[i].Sentence < tests[j].Sentence
 			})
 			for _, r := range tests {
 				fmt.Fprintln(td.Stdout, r.String())
+				if r.Action == "fail" {
+					for _, line := range outputs[r.Test] {
+						fmt.Fprint(td.Stdout, line)
+					}
+				}
 			}
 			fmt.Fprintln(td.Stdout)
-		}
-		if event.Relevant() {
+		case event.IsOutput():
+			outputs[event.Test] = append(outputs[event.Test], event.Output)
+		case event.IsTestResult():
 			event.Sentence = Prettify(event.Test)
 			results[event.Package] = append(results[event.Package], event)
+			if event.Action == "fail" {
+				td.OK = false
+			}
 		}
 	}
 }
 
+// ParseJSON takes a string representing a single JSON test record as emitted
+// by 'go test -json', and attempts to parse it into an [Event], returning any
+// parsing error encountered.
+func ParseJSON(line string) (Event, error) {
+	event := Event{}
+	err := json.Unmarshal([]byte(line), &event)
+	if err != nil {
+		return Event{}, fmt.Errorf("parsing JSON: %w\ninput: %s", err, line)
+	}
+	return event, nil
+}
+
 // Event represents a Go test event as recorded by the 'go test -json' command.
 // It does not attempt to unmarshal all the data, only those fields it needs to
 // know about. It is based on the (unexported) 'event' struct used by Go's
@@ -110,6 +167,7 @@ type Event struct {
 	Package  string
 	Test     string
 	Sentence string
+	Output   string
 	Elapsed  float64
 }
 
@@ -132,11 +190,11 @@ func (e Event) String() string {
 	return fmt.Sprintf(" %s %s (%.2fs)", status, e.Sentence, e.Elapsed)
 }
 
-// Relevant determines whether or not the test event is one that we are
+// IsTestResult determines whether or not the test event is one that we are
 // interested in (namely, a pass or fail event on a test). Events on non-tests
 // (for example, examples) are ignored, and all events on tests other than pass
 // or fail events (for example, run or pause events) are also ignored.
-func (e Event) Relevant() bool {
+func (e Event) IsTestResult() bool {
 	// Events on non-tests are irrelevant
 	if !strings.HasPrefix(e.Test, "Test") {
 		return false
@@ -160,30 +218,18 @@ func (e Event) IsPackageResult() bool {
 	return false
 }
 
-// ParseJSON takes a string representing a single JSON test record as emitted
-// by 'go test -json', and attempts to parse it into an [Event], returning any
-// parsing error encountered.
-func ParseJSON(line string) (Event, error) {
-	event := Event{}
-	err := json.Unmarshal([]byte(line), &event)
-	if err != nil {
-		return Event{}, fmt.Errorf("parsing JSON: %w\ninput: %s", err, line)
+// IsOutput determines whether or not the event is a test output (for example
+// from [testing.T.Error]), excluding status messages automatically generated
+// by 'go test' such as "--- FAIL: ..." or "=== RUN / PAUSE / CONT".
+func (e Event) IsOutput() bool {
+	if e.Action != "output" {
+		return false
 	}
-	return event, nil
-}
-
-// Main runs the command-line interface for gotestdox. The exit status for the
-// binary is 0 if the tests passed, or 1 if the tests failed, or there was some
-// error.
-func Main() int {
-	td := NewTestDoxer()
-	if isatty.IsTerminal(os.Stdin.Fd()) {
-		td.ExecGoTest(os.Args[1:])
-	} else {
-		td.Filter()
+	if strings.HasPrefix(e.Output, "---") {
+		return false
 	}
-	if !td.OK {
-		return 1
+	if strings.HasPrefix(e.Output, "===") {
+		return false
 	}
-	return 0
+	return true
 }

gotestdox_test.go

@@ -83,7 +83,7 @@ func TestRelevantIsTrueForTestPassOrFailEvents(t *testing.T) {
 		},
 	}
 	for _, event := range tcs {
-		relevant := event.Relevant()
+		relevant := event.IsTestResult()
 		if !relevant {
 			t.Errorf("false for relevant event %q on %q", event.Action, event.Test)
 		}
@@ -115,7 +115,7 @@ func TestRelevantIsFalseForNonTestPassFailEvents(t *testing.T) {
 		},
 	}
 	for _, event := range tcs {
-		relevant := event.Relevant()
+		relevant := event.IsTestResult()
 		if relevant {
 			t.Errorf("true for irrelevant event %q on %q", event.Action, event.Test)
 		}
@@ -213,33 +213,33 @@ func ExampleEvent_String() {
 	// ✔ It works (0.00s)
 }
 
-func ExampleEvent_Relevant_true() {
+func ExampleEvent_IsTestResult_true() {
 	event := gotestdox.Event{
 		Action: "pass",
 		Test:   "TestItWorks",
 	}
-	fmt.Println(event.Relevant())
+	fmt.Println(event.IsTestResult())
 	// Output:
 	// true
 }
 
-func ExampleEvent_Relevant_false() {
+func ExampleEvent_IsTestResult_false() {
 	event := gotestdox.Event{
 		Action: "fail",
 		Test:   "ExampleIsIrrelevant",
 	}
-	fmt.Println(event.Relevant())
+	fmt.Println(event.IsTestResult())
 	// Output:
 	// false
 }
 
 func ExampleParseJSON() {
-	input := `{"Action":"pass","Package":"demo","Test":"TestItWorks","Elapsed":0.2}`
+	input := `{"Action":"pass","Package":"demo","Test":"TestItWorks","Output":"","Elapsed":0.2}`
 	event, err := gotestdox.ParseJSON(input)
 	if err != nil {
 		log.Fatal(err)
 	}
 	fmt.Printf("%#v\n", event)
 	// Output:
-	// gotestdox.Event{Action:"pass", Package:"demo", Test:"TestItWorks", Sentence:"", Elapsed:0.2}
+	// gotestdox.Event{Action:"pass", Package:"demo", Test:"TestItWorks", Sentence:"", Output:"", Elapsed:0.2}
 }

testdata/script/help_requested.txtar

@@ -0,0 +1,2 @@
+exec gotestdox -h
+stdout 'Usage'

testdata/script/tests_fail.txtar

@@ -6,6 +6,7 @@ cmp stdout golden.txt
 {"Action":"run","Package":"dummy","Test":"TestDummy"}
 {"Action":"output","Package":"dummy","Test":"TestDummy","Output":"=== RUN   TestDummy\n"}
 {"Action":"output","Package":"dummy","Test":"TestDummy","Output":"--- FAIL: TestDummy (0.00s)\n"}
+{"Action":"output","Package":"dummy","Test":"TestDummy","Output":"    dummy_test.go:23: oh no\n"}
 {"Action":"fail","Package":"dummy","Test":"TestDummy"}
 {"Action":"output","Package":"dummy","Output":"FAIL\n"}
 {"Action":"output","Package":"dummy","Output":"exit status 1\n"}
@@ -14,4 +15,5 @@ cmp stdout golden.txt
 -- golden.txt --
 dummy:
  x Dummy (0.00s)
+    dummy_test.go:23: oh no