From 635e766ed566aae97764c721252b0a8831c5a225 Mon Sep 17 00:00:00 2001 From: Nova Solomon Date: Mon, 12 Sep 2022 13:48:14 +0200 Subject: [PATCH] Add support for setting custom xml attributes With the newly added 'annotate' decorator, it is now possible to add custom xml attributes to both test cases and test suites. This makes it possible to mark tests with IDs, which can then be used to automatically process the test results and integrate it with third party test management tools. --- README.md | 29 +++++++++++++++++ tests/testsuite.py | 76 +++++++++++++++++++++++++++++++++++++++++++++ xmlrunner/result.py | 25 +++++++++++++++ xmlrunner/runner.py | 24 ++++++++++++++ 4 files changed, 154 insertions(+) diff --git a/README.md b/README.md index b9d3102..b0b541d 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,35 @@ if __name__ == '__main__': failfast=False, buffer=False, catchbreak=False) ```` +### Custom XML attributes + +In order to add custom xml attributes to the xml report, use the `annotate` decorator. +It takes a key and a value, or for more than one attribute, a dictionary. +Keys must be strings; values must be strings, integers, floats or booleans. + +````python +import unittest +import xmlrunner +from xmlrunner.runner import annotate + +@annotate("TestsuiteId", 1234567890) +class DemonstrateAnnotations(unittest.TestCase): + + @annotate({ + "Tester": "Nova Solomon", + "TestTicketId": 13, + "UsesUnittestModule": True}) + def test_annotation(self): + pass + +if __name__ == '__main__': + unittest.main( + testRunner=xmlrunner.XMLTestRunner(output='test-reports'), + # these make sure that some options that are not applicable + # remain hidden from the help menu. + failfast=False, buffer=False, catchbreak=False) +```` + ### Doctest support The XMLTestRunner can also be used to report on docstrings style tests. diff --git a/tests/testsuite.py b/tests/testsuite.py index 76db902..0ebd087 100755 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -14,6 +14,7 @@ from xmlrunner.result import _DuplicateWriter from xmlrunner.result import _XMLTestResult from xmlrunner.result import resolve_filename +from xmlrunner.runner import annotate import doctest import tests.doctest_example from io import StringIO, BytesIO @@ -196,6 +197,47 @@ class DecoratedUnitTest(unittest.TestCase): def test_pass(self): pass + @annotate("type", "testsuite") + class AnnotatedUnitTest(unittest.TestCase): + + @annotate("two", "arguments") + def test_two_arguments(self): + pass + + @annotate("int", 1) + def test_int_annotation(self): + pass + + @annotate("float", 1.0) + def test_float_annotation(self): + 1 / 0 + + @annotate("bool", True) + def test_bool_annotation(self): + pass + + @annotate({"type": "dict"}) + def test_dict_annotation(self): + pass + + @annotate({"str": "test", "int": 1, "float": 1.0, "bool": True}) + def test_dict_annotation_types(self): + pass + + @annotate() + def test_empty_annotation(self): + pass + + @unittest.expectedFailure + @annotate("should", "fail") + def test_annotated_failure(self): + self.assertTrue(False) + + @unittest.skip("Testing annotated skip") + @annotate("should", "skip") + def test_annotated_skip(self): + pass + class DummyErrorInCallTest(unittest.TestCase): def __call__(self, result): @@ -809,6 +851,40 @@ def test_xmlrunner_hold_traceback(self): countAfterTest = sys.getrefcount(self.DummyRefCountTest.dummy) self.assertEqual(countBeforeTest, countAfterTest) + def test_annotations(self): + suite = unittest.TestSuite() + test_methods = ( + "test_two_arguments", + "test_int_annotation", + "test_float_annotation", + "test_bool_annotation", + "test_dict_annotation", + "test_dict_annotation_types", + "test_empty_annotation", + "test_annotated_failure", + "test_annotated_skip" + ) + for method in test_methods: + suite.addTest(self.AnnotatedUnitTest(method)) + outdir = BytesIO() + + self._test_xmlrunner(suite, outdir=outdir) + + output = outdir.getvalue() + xml_attributes = ( + b'type="testsuite"', + b'two="arguments"', + b'int="1"', + b'float="1.0"', + b'bool="True"', + b'type="dict"', + b'str="test"', + b'should="fail"', + b'should="skip"' + ) + for attr in xml_attributes: + self.assertIn(attr, output) + class StderrXMLTestRunner(xmlrunner.XMLTestRunner): """ XMLTestRunner that outputs to sys.stderr that might be replaced diff --git a/xmlrunner/result.py b/xmlrunner/result.py index 96fc675..8aabd00 100644 --- a/xmlrunner/result.py +++ b/xmlrunner/result.py @@ -169,6 +169,8 @@ def __init__(self, test_result, test_method, outcome=SUCCESS, err=None, subTest= self.filename = filename self.lineno = lineno self.doc = doc + self.annotations = test_result._annotations + self.suite_annotations = test_result._suite_annotations def id(self): return self.test_id @@ -332,6 +334,7 @@ def addSuccess(self, test): Called when a test executes successfully. """ self._save_output_data() + self._copy_annotations(test) self._prepare_callback( self.infoclass(self, test), self.successes, 'ok', '.' ) @@ -342,6 +345,7 @@ def addFailure(self, test, err): Called when a test method fails. """ self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass( self, test, self.infoclass.FAILURE, err) self.failures.append(( @@ -356,6 +360,7 @@ def addError(self, test, err): Called when a test method raises an error. """ self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass( self, test, self.infoclass.ERROR, err) self.errors.append(( @@ -384,6 +389,7 @@ def addSubTest(self, testcase, test, err): errorList = self.errors self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass( self, testcase, errorValue, err, subTest=test) @@ -398,6 +404,7 @@ def addSkip(self, test, reason): Called when a test method was skipped. """ self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass( self, test, self.infoclass.SKIP, reason) testinfo.test_exception_name = 'skip' @@ -410,6 +417,7 @@ def addExpectedFailure(self, test, err): Missing in xmlrunner, copy-pasted from xmlrunner addError. """ self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass(self, test, self.infoclass.SKIP, err) testinfo.test_exception_name = 'XFAIL' @@ -424,6 +432,7 @@ def addUnexpectedSuccess(self, test): Missing in xmlrunner, copy-pasted from xmlrunner addSuccess. """ self._save_output_data() + self._copy_annotations(test) testinfo = self.infoclass(self, test) # do not set outcome here because it will need exception testinfo.outcome = self.infoclass.ERROR @@ -449,6 +458,11 @@ def printErrorList(self, flavour, errors): self.stream.writeln('%s' % test_info.get_error_info()) self.stream.flush() + def _copy_annotations(self, test): + test_method = getattr(test, test._testMethodName) + self._annotations = getattr(test_method, "_annotations", None) + self._suite_annotations = getattr(test, "_annotations", None) + def _get_info_by_testcase(self): """ Organizes test results by TestCase module. This information is @@ -512,6 +526,13 @@ def _report_testsuite(suite_name, tests, xml_document, parentElement, skips = filter(lambda e: e.outcome == _TestInfo.SKIP, tests) testsuite.setAttribute('skipped', str(len(list(skips)))) + # indexing is necessary since each test info instance + # carries the annotations of its test suite + annotations = tests[0].suite_annotations + if annotations: + for attr, value in annotations.items(): + testsuite.setAttribute(attr, str(value)) + _XMLTestResult._report_testsuite_properties( testsuite, xml_document, properties) @@ -575,6 +596,10 @@ def _report_testcase(test_result, xml_testsuite, xml_document): if test_result.lineno is not None: testcase.setAttribute('line', str(test_result.lineno)) + if test_result.annotations: + for attr, value in test_result.annotations.items(): + testcase.setAttribute(attr, str(value)) + if test_result.doc is not None: comment = str(test_result.doc) # The use of '--' is forbidden in XML comments diff --git a/xmlrunner/runner.py b/xmlrunner/runner.py index 6b6588b..503eef3 100644 --- a/xmlrunner/runner.py +++ b/xmlrunner/runner.py @@ -190,3 +190,27 @@ def runTests(self): finally: if output_file is not None: output_file.close() + +def annotate(*args): + """ + Add further information as xml attributes to a test cases or a test suite + """ + def decorator(test_item): # data as dict + data = {} + if len(args) == 1 and isinstance(args[0], dict): + for key, value in args[0].items(): + if not (isinstance(key, str) or isinstance(value, (str, int, float, bool))): + raise TypeError("add_info takes (key: str, value: (str, int, float)) or (dict(key:str, value, (str, int, float, bool)))") + data = args[0] + elif ( + len(args) == 2 and + isinstance(args[0], str) and + isinstance(args[1], (str, int, float, bool))): + data = {args[0]: args[1]} + elif not args: + return test_item + else: + raise TypeError("add_info takes (key: str, value: (str, int, float)) or (dict(key:str, value, (str, int, float, bool)))") + test_item._annotations = data + return test_item + return decorator