diff --git a/README.md b/README.md index f1c00a7..61a612f 100644 --- a/README.md +++ b/README.md @@ -98,3 +98,7 @@ test-results publish --force other-results.xml ## Using the CLI on a local machine Latest CLI binaries are available [here](https://github.com/semaphoreci/test-results/releases/latest). + +## Contributing + +Bug reports and pull requests are welcome. If your test runner is not supported, see documentation pages on writing [a new parser](docs/custom-parsers.md) diff --git a/cmd/combine.go b/cmd/combine.go index 730c027..d30f3fb 100644 --- a/cmd/combine.go +++ b/cmd/combine.go @@ -1,21 +1,5 @@ package cmd -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - import ( "github.com/semaphoreci/test-results/pkg/cli" "github.com/semaphoreci/test-results/pkg/logger" diff --git a/cmd/compile.go b/cmd/compile.go index 37f6593..225e941 100644 --- a/cmd/compile.go +++ b/cmd/compile.go @@ -1,21 +1,5 @@ package cmd -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - import ( "encoding/json" "io/ioutil" diff --git a/cmd/gen-pipeline-report.go b/cmd/gen-pipeline-report.go index ffb3c04..625cd3e 100644 --- a/cmd/gen-pipeline-report.go +++ b/cmd/gen-pipeline-report.go @@ -1,21 +1,5 @@ package cmd -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - import ( "encoding/json" "io/ioutil" diff --git a/cmd/publish.go b/cmd/publish.go index daa7cea..f424415 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -1,21 +1,5 @@ package cmd -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - import ( "encoding/json" "fmt" diff --git a/cmd/root.go b/cmd/root.go index 5ed4e54..4f8d40c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -1,21 +1,5 @@ package cmd -/* -Copyright © 2021 NAME HERE - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - import ( "fmt" "os" diff --git a/docs/custom-parsers.md b/docs/custom-parsers.md new file mode 100644 index 0000000..0db6f92 --- /dev/null +++ b/docs/custom-parsers.md @@ -0,0 +1,33 @@ +# Custom parsers + +Generic parser parses JUnit XML documents according to [JUnit XML Schema](https://github.com/windyroad/JUnit-Schema/blob/master/JUnit.xsd). In some situations, it might be necessary to write a custom parser. This document will guide you through the process. + +> **Note:** This guide assumes you have a basic understanding of [golang](https://go.dev/). + +## Creating a new parser + +Every parser provides an implementation of the [`Parser`](https://pkg.go.dev/github.com/semaphoreci/test-results@v0.4.13/pkg/parser#Parser) interface, in particular: + +- `GetName() string` - returns the name of the parser that can then be used in the CLI as a `--parser` option + +- `IsApplicable(path string) bool` - should return true if the parser is applicable to the given file at `path` + +- `Parse(path string) parser.TestResults` - parses the file at `path` and returns a `parser.TestResults` struct. This struct has a well-defined format and can be serialized to JSON. + +[Generic parser implementation](https://github.com/semaphoreci/test-results/blob/master/pkg/parsers/generic.go) is a good starting template for creating a custom parser. + +After the parser is implemented, it has to be added to the list of [available parsers](https://github.com/semaphoreci/test-results/blob/master/pkg/parsers/parsers.go#L10). + +## Good parser qualities + +- The parser is as generic as possible. + + Currently, custom parsers need to be compiled, thus merged to the main repository. Each test runner should have at most one parser. + +- Parsing is idempotent. + + If you parse the same file twice, the results should be the same. It's is highly crucial for test IDs. Please refer to [`ID generation guide`](id-generation.md) for more details. + +- It's tested. + + If the parser is lacking tests, it will most likely be rejected. diff --git a/docs/id-generation.md b/docs/id-generation.md index 8a86d2c..fac1706 100644 --- a/docs/id-generation.md +++ b/docs/id-generation.md @@ -1,23 +1,21 @@ -## ID generation +# ID generation -This PR introduces `id` generator for tests results, test suites, and tests. +Entity `id`'s are being generated in form of UUID formatted strings. -`id`'s are being generated in form of UUID strings. - -To generate consistent `id`'s between builds following method is implemented for all parsers: +To generate consistent `id`'s following namespacing method is used for all parsers: - ID generation for `Test results`(top-level element) 1. If the element has an ID, generate UUID based on that ID 2. If the element doesn't have an ID - generate UUID based on the `name` attribute - 3. If the element has a framework name - generate UUID based on the `name` attribute and `framework` + 3. If the element has a framework name - generate UUID based on the `name` and `framework` attributes 4. Otherwise, generate uuid based on an empty string `""` -- ID generation for `Suites` +- ID generation for test `Suites` The same rules apply as for `Test results` however every `Suite ID` is namespaced by `Test results` ID -- ID generation for `Tests` +- ID generation for `Test` cases The same rules apply as for `Test results` however every `Test ID` is namespaced by `Suite` ID and `Test classname` if present. If a test is failed/errored the state is also added as namespace, as failed/errored cases can happen simultaneously in the same suite. diff --git a/main.go b/main.go index 1f7c35b..616b035 100644 --- a/main.go +++ b/main.go @@ -1,5 +1,5 @@ /* -Copyright © 2021 NAME HERE +Copyright © 2021 Rendered Text Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ + package main import "github.com/semaphoreci/test-results/cmd" diff --git a/pkg/parser/parser.go b/pkg/parser/parser.go index 0c578f1..a415430 100644 --- a/pkg/parser/parser.go +++ b/pkg/parser/parser.go @@ -1,8 +1,11 @@ package parser -// Parser ... +// Parser interface defines the methods that a parser must implement. type Parser interface { - Parse(string) TestResults - IsApplicable(string) bool + // Parse return a TestResults struct containing the results of the parsing file at path + Parse(path string) TestResults + // IsApplicable returns true if the parser is applicable to the file at path + IsApplicable(path string) bool + // GetName returns the name of the parser GetName() string } diff --git a/pkg/parser/types.go b/pkg/parser/types.go index f72b132..32a9429 100644 --- a/pkg/parser/types.go +++ b/pkg/parser/types.go @@ -15,31 +15,22 @@ type Properties map[string]string type State string const ( - // StatePassed indicates that test was successful - StatePassed State = "passed" - // StateError indicates that test errored due to unexpected behaviour when running test i.e. exception - StateError State = "error" - // StateFailed indicates that test failed due to invalid test result - StateFailed State = "failed" - // StateSkipped indicates that test was skipped - StateSkipped State = "skipped" - // StateDisabled indicates that test was disabled - StateDisabled State = "disabled" + StatePassed State = "passed" // test was successful + StateError State = "error" // test errored due to unexpected behaviour when running test i.e. exception + StateFailed State = "failed" // test failed due to invalid test result + StateSkipped State = "skipped" // test was skipped + StateDisabled State = "disabled" // test was disabled ) // Status stores information about parsing results type Status string const ( - // StatusSuccess indicates that parsing was successful - StatusSuccess Status = "success" - - // StatusError indicates that parsing failed due to error - StatusError Status = "error" + StatusSuccess Status = "success" // parsing was successful + StatusError Status = "error" // parsing failed due to error ) -// Result ... -// [TODO] Better name is required... +// Result is a collection of test results type Result struct { TestResults []TestResults `json:"testResults"` } @@ -101,16 +92,16 @@ func (me *Result) hasTestResults(testResults TestResults) (int, bool) { return -1, false } -// TestResults ... +// TestResults represents well defined group of test suites and. type TestResults struct { - ID string `json:"id"` - Name string `json:"name"` - Framework string `json:"framework"` - IsDisabled bool `json:"isDisabled"` - Suites []Suite `json:"suites"` - Summary Summary `json:"summary"` - Status Status `json:"status"` - StatusMessage string `json:"statusMessage"` + ID string `json:"id"` // deterministic identifiers are required for test analytics, please follow https://github.com/semaphoreci/test-results/blob/master/docs/id-generation.md + Name string `json:"name"` // + Framework string `json:"framework"` // parsers use this field to determine if they're applicable + IsDisabled bool `json:"isDisabled"` // + Suites []Suite `json:"suites"` // + Summary Summary `json:"summary"` // + Status Status `json:"status"` // combined with StatusMessage can be used to provide insights into parser failures + StatusMessage string `json:"statusMessage"` // } // NewTestResults ... @@ -425,15 +416,15 @@ func NewError() Error { return Error{} } -// Summary ... +// Summary contains group metrics type Summary struct { - Total int `json:"total"` - Passed int `json:"passed"` - Skipped int `json:"skipped"` - Error int `json:"error"` - Failed int `json:"failed"` - Disabled int `json:"disabled"` - Duration time.Duration `json:"duration"` + Total int `json:"total"` // Total tests in group + Passed int `json:"passed"` // Passed tests in group + Skipped int `json:"skipped"` // Skipped tests in group + Error int `json:"error"` // Errored tests in group + Failed int `json:"failed"` // Failed tests in group + Disabled int `json:"disabled"` // Disabled tests in group + Duration time.Duration `json:"duration"` // Total duration of the group } // UUID ... diff --git a/pkg/parsers/generic.go b/pkg/parsers/generic.go index 8c341e2..48a92af 100644 --- a/pkg/parsers/generic.go +++ b/pkg/parsers/generic.go @@ -8,27 +8,28 @@ import ( "github.com/semaphoreci/test-results/pkg/parser" ) -// Generic ... +// Generic parser struct type Generic struct { } -// NewGeneric ... +// NewGeneric returns a new generic parser func NewGeneric() Generic { return Generic{} } -// IsApplicable ... +// IsApplicable returns true if this parser is applicable to file at given path +// always true for the generic parser func (me Generic) IsApplicable(path string) bool { logger.Debug("Checking applicability of %s parser", me.GetName()) return true } -// GetName ... +// GetName returns a name of the parser func (me Generic) GetName() string { return "generic" } -// Parse ... +// Parse parses file at path and returns a well-defined TestResults struct func (me Generic) Parse(path string) parser.TestResults { results := parser.NewTestResults() diff --git a/pkg/parsers/golang.go b/pkg/parsers/golang.go index f195938..f60e0de 100644 --- a/pkg/parsers/golang.go +++ b/pkg/parsers/golang.go @@ -8,21 +8,22 @@ import ( "github.com/semaphoreci/test-results/pkg/parser" ) -// GoLang ... +// GoLang parser struct type GoLang struct { } -// NewGoLang ... +// NewGoLang returns a new golang parser func NewGoLang() GoLang { return GoLang{} } -// GetName ... +// GetName returns a name of the parser func (me GoLang) GetName() string { return "golang" } -// IsApplicable ... +// IsApplicable returns true if this parser is applicable to file at given path +// Checks for the presence of `go.version`` property on element. func (me GoLang) IsApplicable(path string) bool { xmlElement, err := LoadXML(path) logger.Debug("Checking applicability of %s parser", me.GetName()) @@ -65,7 +66,7 @@ func hasProperty(testsuiteElement parser.XMLElement, property string) bool { return false } -// Parse ... +// Parse parses file at path and returns a well-defined TestResults struct func (me GoLang) Parse(path string) parser.TestResults { results := parser.NewTestResults()