[WiP] Contributing / Custom parser guidelines

README.md

@@ -98,3 +98,7 @@ test-results publish --force other-results.xml
 ## Using the CLI on a local machine
 
 Latest CLI binaries are available [here](https://github.com/semaphoreci/test-results/releases/latest).
+
+## Contributing
+
+Bug reports and pull requests are welcome. If your test runner is not supported, see documentation pages on writing [a new parser](docs/custom-parsers.md)

cmd/combine.go

@@ -1,21 +1,5 @@
 package cmd
 
-/*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 import (
 	"github.com/semaphoreci/test-results/pkg/cli"
 	"github.com/semaphoreci/test-results/pkg/logger"

cmd/compile.go

@@ -1,21 +1,5 @@
 package cmd
 
-/*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 import (
 	"encoding/json"
 	"io/ioutil"

cmd/gen-pipeline-report.go

@@ -1,21 +1,5 @@
 package cmd
 
-/*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 import (
 	"encoding/json"
 	"io/ioutil"

cmd/publish.go

@@ -1,21 +1,5 @@
 package cmd
 
-/*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 import (
 	"encoding/json"
 	"fmt"

cmd/root.go

@@ -1,21 +1,5 @@
 package cmd
 
-/*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
 import (
 	"fmt"
 	"os"

docs/custom-parsers.md

@@ -0,0 +1,33 @@
+# Custom parsers
+
+Generic parser parses JUnit XML documents according to [JUnit XML Schema](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd). In some situations, it might be necessary to write a  custom parser. This document will guide you through the process.
+
+> **Note:** This guide assumes you have a basic understanding of [golang](https://go.dev/).
+
+## Creating a new parser
+
+Every parser provides an implementation of the [`Parser`](https://pkg.go.dev/github.com/semaphoreci/[email protected]/pkg/parser#Parser) interface, in particular:
+
+- `GetName() string` - returns the name of the parser that can then be used in the CLI as a `--parser` option
+
+- `IsApplicable(path string) bool` - should return true if the parser is applicable to the given file at `path`
+
+- `Parse(path string) parser.TestResults` - parses the file at `path` and returns a `parser.TestResults` struct. This struct has a well-defined format and can be serialized to JSON.
+
+[Generic parser implementation](https://github.com/semaphoreci/test-results/blob/master/pkg/parsers/generic.go) is a good starting template for creating a custom parser.
+
+After the parser is implemented, it has to be added to the list of [available parsers](https://github.com/semaphoreci/test-results/blob/master/pkg/parsers/parsers.go#L10).
+
+## Good parser qualities
+
+- The parser is as generic as possible.
+
+    Currently, custom parsers need to be compiled, thus merged to the main repository. Each test runner should have at most one parser.
+
+- Parsing is idempotent.
+
+    If you parse the same file twice, the results should be the same. It's is highly crucial for test IDs. Please refer to [`ID generation guide`](id-generation.md) for more details.
+
+- It's tested.
+
+    If the parser is lacking tests, it will most likely be rejected.

docs/id-generation.md

@@ -1,23 +1,21 @@
-## ID generation
+# ID generation
 
-This PR introduces `id` generator for tests results, test suites, and tests.
+Entity `id`'s are being generated in form of UUID formatted strings.
 
-`id`'s are being generated in form of UUID strings.
-
-To generate consistent `id`'s between builds following method is implemented for all parsers:
+To generate consistent `id`'s following namespacing method is used for all parsers:
 
 - ID generation for `Test results`(top-level element)
 
   1. If the element has an ID, generate UUID based on that ID
   2. If the element doesn't have an ID - generate UUID based on the `name` attribute
-  3. If the element has a framework name - generate UUID based on the `name` attribute and `framework`
+  3. If the element has a framework name - generate UUID based on the `name` and `framework` attributes
   4. Otherwise, generate uuid based on an empty string `""`
 
-- ID generation for `Suites`
+- ID generation for test `Suites`
 
   The same rules apply as for `Test results` however every `Suite ID` is namespaced by `Test results` ID
 
-- ID generation for `Tests`
+- ID generation for `Test` cases
 
   The same rules apply as for `Test results` however every `Test ID` is namespaced by `Suite` ID and `Test classname` if present.
   If a test is failed/errored the state is also added as namespace, as failed/errored cases can happen simultaneously in the same suite.

main.go

@@ -1,5 +1,5 @@
 /*
-Copyright © 2021 NAME HERE <EMAIL ADDRESS>
+Copyright © 2021 Rendered Text
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
+
 package main
 
 import "github.com/semaphoreci/test-results/cmd"

pkg/parser/parser.go

@@ -1,8 +1,11 @@
 package parser
 
-// Parser ...
+// Parser interface defines the methods that a parser must implement.
 type Parser interface {
-	Parse(string) TestResults
-	IsApplicable(string) bool
+	// Parse return a TestResults struct containing the results of the parsing file at path
+	Parse(path string) TestResults
+	// IsApplicable returns true if the parser is applicable to the file at path
+	IsApplicable(path string) bool
+	// GetName returns the name of the parser
 	GetName() string
 }

pkg/parser/types.go

@@ -15,31 +15,22 @@ type Properties map[string]string
 type State string
 
 const (
-	// StatePassed indicates that test was successful
-	StatePassed State = "passed"
-	// StateError indicates that test errored due to unexpected behaviour when running test i.e. exception
-	StateError State = "error"
-	// StateFailed indicates that test failed due to invalid test result
-	StateFailed State = "failed"
-	// StateSkipped indicates that test was skipped
-	StateSkipped State = "skipped"
-	// StateDisabled indicates that test was disabled
-	StateDisabled State = "disabled"
+	StatePassed   State = "passed"   // test was successful
+	StateError    State = "error"    // test errored due to unexpected behaviour when running test i.e. exception
+	StateFailed   State = "failed"   // test failed due to invalid test result
+	StateSkipped  State = "skipped"  // test was skipped
+	StateDisabled State = "disabled" // test was disabled
 )
 
 // Status stores information about parsing results
 type Status string
 
 const (
-	// StatusSuccess indicates that parsing was successful
-	StatusSuccess Status = "success"
-
-	// StatusError indicates that parsing failed due to error
-	StatusError Status = "error"
+	StatusSuccess Status = "success" // parsing was successful
+	StatusError   Status = "error"   // parsing failed due to error
 )
 
-// Result ...
-// [TODO] Better name is required...
+// Result is a collection of test results
 type Result struct {
 	TestResults []TestResults `json:"testResults"`
 }
@@ -101,16 +92,16 @@ func (me *Result) hasTestResults(testResults TestResults) (int, bool) {
 	return -1, false
 }
 
-// TestResults ...
+// TestResults represents well defined group of test suites and.
 type TestResults struct {
-	ID            string  `json:"id"`
-	Name          string  `json:"name"`
-	Framework     string  `json:"framework"`
-	IsDisabled    bool    `json:"isDisabled"`
-	Suites        []Suite `json:"suites"`
-	Summary       Summary `json:"summary"`
-	Status        Status  `json:"status"`
-	StatusMessage string  `json:"statusMessage"`
+	ID            string  `json:"id"`            // deterministic identifiers are required for test analytics, please follow https://github.com/semaphoreci/test-results/blob/master/docs/id-generation.md
+	Name          string  `json:"name"`          //
+	Framework     string  `json:"framework"`     // parsers use this field to determine if they're applicable
+	IsDisabled    bool    `json:"isDisabled"`    //
+	Suites        []Suite `json:"suites"`        //
+	Summary       Summary `json:"summary"`       //
+	Status        Status  `json:"status"`        // combined with StatusMessage can be used to provide insights into parser failures
+	StatusMessage string  `json:"statusMessage"` //
 }
 
 // NewTestResults ...
@@ -425,15 +416,15 @@ func NewError() Error {
 	return Error{}
 }
 
-// Summary ...
+// Summary contains group metrics
 type Summary struct {
-	Total    int           `json:"total"`
-	Passed   int           `json:"passed"`
-	Skipped  int           `json:"skipped"`
-	Error    int           `json:"error"`
-	Failed   int           `json:"failed"`
-	Disabled int           `json:"disabled"`
-	Duration time.Duration `json:"duration"`
+	Total    int           `json:"total"`    // Total tests in group
+	Passed   int           `json:"passed"`   // Passed tests in group
+	Skipped  int           `json:"skipped"`  // Skipped tests in group
+	Error    int           `json:"error"`    // Errored tests in group
+	Failed   int           `json:"failed"`   // Failed tests in group
+	Disabled int           `json:"disabled"` // Disabled tests in group
+	Duration time.Duration `json:"duration"` // Total duration of the group
 }
 
 // UUID ...

pkg/parsers/generic.go

@@ -8,27 +8,28 @@ import (
 	"github.com/semaphoreci/test-results/pkg/parser"
 )
 
-// Generic ...
+// Generic parser struct
 type Generic struct {
 }
 
-// NewGeneric ...
+// NewGeneric returns a new generic parser
 func NewGeneric() Generic {
 	return Generic{}
 }
 
-// IsApplicable ...
+// IsApplicable returns true if this parser is applicable to file at given path
+// always true for the generic parser
 func (me Generic) IsApplicable(path string) bool {
 	logger.Debug("Checking applicability of %s parser", me.GetName())
 	return true
 }
 
-// GetName ...
+// GetName returns a name of the parser
 func (me Generic) GetName() string {
 	return "generic"
 }
 
-// Parse ...
+// Parse parses file at path and returns a well-defined TestResults struct
 func (me Generic) Parse(path string) parser.TestResults {
 	results := parser.NewTestResults()
 

pkg/parsers/golang.go

@@ -8,21 +8,22 @@ import (
 	"github.com/semaphoreci/test-results/pkg/parser"
 )
 
-// GoLang ...
+// GoLang parser struct
 type GoLang struct {
 }
 
-// NewGoLang ...
+// NewGoLang returns a new golang parser
 func NewGoLang() GoLang {
 	return GoLang{}
 }
 
-// GetName ...
+// GetName returns a name of the parser
 func (me GoLang) GetName() string {
 	return "golang"
 }
 
-// IsApplicable ...
+// IsApplicable returns true if this parser is applicable to file at given path
+// Checks for the presence of `go.version`` property on <testsuite> element.
 func (me GoLang) IsApplicable(path string) bool {
 	xmlElement, err := LoadXML(path)
 	logger.Debug("Checking applicability of %s parser", me.GetName())
@@ -65,7 +66,7 @@ func hasProperty(testsuiteElement parser.XMLElement, property string) bool {
 	return false
 }
 
-// Parse ...
+// Parse parses file at path and returns a well-defined TestResults struct
 func (me GoLang) Parse(path string) parser.TestResults {
 	results := parser.NewTestResults()