From 1bbc4bc59707eefd8e887e0ed95d08e8aabea67a Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 7 Sep 2021 16:07:52 -0300 Subject: [PATCH 01/48] First prototype of native pytest plugin for launch based tests Signed-off-by: Ivan Santiago Paunovic --- launch/launch/launch_service.py | 4 + .../launch_testing/pytest/__init__.py | 21 ++ .../launch_testing/pytest/fixture.py | 88 ++++++ .../launch_testing/pytest/legacy/__init__.py | 0 .../pytest/{ => legacy}/hooks.py | 6 +- .../pytest/{ => legacy}/hookspecs.py | 0 .../launch_testing/pytest/plugin.py | 258 ++++++++++++++++++ launch_testing/package.xml | 3 +- launch_testing/setup.py | 5 +- .../new_hooks/test_function_scope.py | 33 +++ .../new_hooks/test_module_scope.py | 33 +++ 11 files changed, 446 insertions(+), 5 deletions(-) create mode 100644 launch_testing/launch_testing/pytest/fixture.py create mode 100644 launch_testing/launch_testing/pytest/legacy/__init__.py rename launch_testing/launch_testing/pytest/{ => legacy}/hooks.py (97%) rename launch_testing/launch_testing/pytest/{ => legacy}/hookspecs.py (100%) create mode 100644 launch_testing/launch_testing/pytest/plugin.py create mode 100644 launch_testing/test/launch_testing/new_hooks/test_function_scope.py create mode 100644 launch_testing/test/launch_testing/new_hooks/test_module_scope.py diff --git a/launch/launch/launch_service.py b/launch/launch/launch_service.py index 182b50124..ccc38d03b 100644 --- a/launch/launch/launch_service.py +++ b/launch/launch/launch_service.py @@ -413,3 +413,7 @@ def shutdown(self, force_sync=False) -> Optional[Coroutine]: def context(self): """Getter for context.""" return self.__context + + @property + def event_loop(self): + return self.__loop_from_run_thread diff --git a/launch_testing/launch_testing/pytest/__init__.py b/launch_testing/launch_testing/pytest/__init__.py index e69de29bb..f99014d81 100644 --- a/launch_testing/launch_testing/pytest/__init__.py +++ b/launch_testing/launch_testing/pytest/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import legacy +from .fixture import fixture + +__all__ = [ + 'fixture', + 'legacy', +] diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py new file mode 100644 index 000000000..380572068 --- /dev/null +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -0,0 +1,88 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import inspect + +import launch +import pytest + +try: + from _pytest.scope import Scope + + def scope_gt(scope1, scope2): + return Scope(scope1) > Scope(scope2) +except ImportError: + from _pytest.fixtures import scopemismatch as scope_gt + + +def get_launch_service_fixture(scope='function'): + + @pytest.fixture(scope=scope) + def launch_service(): + ls = launch.LaunchService() + yield ls + assert ls._is_idle(), ( + 'launch service must be shut down before fixture tear down' + ) + launch_service._launch_testing_overridable_fixture = True + return launch_service + + +def get_event_loop_fixture(scope='function'): + + @pytest.fixture(scope=scope) + def event_loop(): + loop = asyncio.get_event_loop_policy().new_event_loop() + yield loop + loop.close() + event_loop._launch_testing_overridable_fixture = True + event_loop._launch_testing_fixture_scope = scope + return event_loop + + +def fixture(*args, **kwargs): + """ + Decorate launch_test fixtures. + + For documentation on the supported arguments, see + https://docs.pytest.org/en/latest/reference/reference.html#pytest-fixture. + """ + # Automagically override the event_loop and launch_testing fixtures + # with a fixture of the correct scope. + # This is not done if the user explicitly provided this fixture, + # they might get an ScopeError if not correctly defined. + scope = kwargs.get('scope', 'function') + if scope != 'function': + frame = inspect.stack()[1] + mod = inspect.getmodule(frame[0]) + mod_locals = vars(mod) + for name, getter in ( + ('launch_service', get_launch_service_fixture), + ('event_loop', get_event_loop_fixture) + ): + if name in mod_locals: + obj = mod_locals[name] + if ( + getattr(obj, '_launch_testing_overridable_fixture', False) and + scope_gt(scope, obj._launch_testing_fixture_scope) + ): + mod_locals[name] = getter(scope) + else: + mod_locals[name] = getter(scope) + + def decorator(fixture_function): + fixture_function._launch_pytest_fixture = True + return pytest.fixture(fixture_function, *args, **kwargs) + return decorator diff --git a/launch_testing/launch_testing/pytest/legacy/__init__.py b/launch_testing/launch_testing/pytest/legacy/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/launch_testing/launch_testing/pytest/hooks.py b/launch_testing/launch_testing/pytest/legacy/hooks.py similarity index 97% rename from launch_testing/launch_testing/pytest/hooks.py rename to launch_testing/launch_testing/pytest/legacy/hooks.py index 4e55f5d2d..f364f2572 100644 --- a/launch_testing/launch_testing/pytest/hooks.py +++ b/launch_testing/launch_testing/pytest/legacy/hooks.py @@ -19,8 +19,8 @@ import pytest -from ..loader import LoadTestsFromPythonModule -from ..test_runner import LaunchTestRunner +from ...loader import LoadTestsFromPythonModule +from ...test_runner import LaunchTestRunner class LaunchTestFailure(Exception): @@ -189,7 +189,7 @@ def pytest_launch_collect_makemodule(path, parent, entrypoint): def pytest_addhooks(pluginmanager): - import launch_testing.pytest.hookspecs as hookspecs + import launch_testing.pytest.legacy.hookspecs as hookspecs pluginmanager.add_hookspecs(hookspecs) diff --git a/launch_testing/launch_testing/pytest/hookspecs.py b/launch_testing/launch_testing/pytest/legacy/hookspecs.py similarity index 100% rename from launch_testing/launch_testing/pytest/hookspecs.py rename to launch_testing/launch_testing/pytest/legacy/hookspecs.py diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py new file mode 100644 index 000000000..7409082d5 --- /dev/null +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -0,0 +1,258 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import functools +import inspect +import itertools +import warnings + +import launch +from launch.launch_service import LaunchService +import launch_testing +import pytest + + +""" +launch_testing native pytest based implementation. +""" + + +try: + from _pytest.python import transfer_markers +except ImportError: # Pytest 4.1.0 removes the transfer_marker api (#104) + + def transfer_markers(*args, **kwargs): # noqa + """Noop when over pytest 4.1.0""" + pass + + +def pytest_configure(config): + """Inject launch_testing marker documentation.""" + config.addinivalue_line( + 'markers', + 'launch_testing: ' + 'mark the test as a launch test, it will be ' + 'run using the specified launch_pad.', + ) + +# Adapted from https://github.com/pytest-dev/pytest-asyncio, +# see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. +@pytest.mark.tryfirst +def pytest_pycollect_makeitem(collector, name, obj): + """Collect coroutine based launch tests.""" + if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj): + item = pytest.Function.from_parent(collector, name=name) + + # Due to how pytest test collection works, module-level pytestmarks + # are applied after the collection step. Since this is the collection + # step, we look ourselves. + transfer_markers(obj, item.cls, item.module) + item = pytest.Function.from_parent(collector, name=name) # To reload keywords. + + if 'launch_testing' in item.keywords: + return list(collector._genfunctions(name, obj)) + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_fixture_setup(fixturedef, request): + """Set up launch service for all launch_pytest fixtures.""" + if getattr(fixturedef.func, '_launch_pytest_fixture', False): + eprefix = f"When running launch_pytest fixture '{fixturedef.func.__name__}':" + ls = request.getfixturevalue('launch_service') + event_loop = request.getfixturevalue('event_loop') + # get the result of the launch fixture, we take advantage of other wrappers this way + outcome = yield + ret = outcome.get_result() + wrong_ret_type_error = ( + f'{eprefix} return value must be either a launch description ' + 'or a launch description, locals pair' + ) + ld = ret + if isinstance(ret, tuple): + assert len(ret) == 2, wrong_ret_type_error + ld, _ = ret + assert isinstance(ld, launch.LaunchDescription), wrong_ret_type_error + ls.include_launch_description(ld) + run_async_task = event_loop.create_task(ls.run_async( + # TODO(ivanpauno): maybe this could be configurable (?) + shutdown_when_idle=True + )) + ready = get_ready_to_test_action(ld) + asyncio.set_event_loop(event_loop) + event = asyncio.Event() + ready._add_callback(lambda: event.set()) + + def finalize(): + ls.shutdown() + rc = event_loop.run_until_complete(run_async_task) + assert rc == 0, f"{eprefix} launch service failed when finishing, return code '{rc}'" + fixturedef.addfinalizer(finalize) + run_until_complete(event_loop, event.wait()) + return + yield + + +# TODO(ivanpauno): Deduplicate with launch_testing +def iterate_ready_to_test_actions(entities): + """Search recursively LaunchDescription entities for all ReadyToTest actions.""" + for entity in entities: + if isinstance(entity, launch_testing.actions.ReadyToTest): + yield entity + yield from iterate_ready_to_test_actions( + entity.describe_sub_entities() + ) + for conditional_sub_entity in entity.describe_conditional_sub_entities(): + yield from iterate_ready_to_test_actions( + conditional_sub_entity[1] + ) + + +def get_ready_to_test_action(launch_description): + """Extract the ready to test action from the launch description.""" + gen = (e for e in iterate_ready_to_test_actions(launch_description.entities)) + try: + ready_action = next(gen) + except StopIteration: # No ReadyToTest action found + raise RuntimeError( + 'launch_pytest fixtures must return a LaunchDescription ' + 'containing a ReadyToTest action' + ) + try: + next(gen) + except StopIteration: # Only one ReadeToTest action must be found + return ready_action + raise RuntimeError( + 'launch_pytest fixtures must return a LaunchDescription ' + 'containing only one ReadyToTest action' + ) + + +def pytest_runtest_setup(item): + """Inject fixtures in launch_testing marked tests.""" + marker = item.get_closest_marker('launch_testing') + if marker is not None: + # inject the corrent launch_testing fixture here + if 'fixture' not in marker.kwargs: + raise RuntimeError( + 'from keyword argument is required in a pytest.mark.launch_testing() decorator: \n' + f'{get_error_context_from_obj(item.obj)}' + ) + fixturename = marker.kwargs['fixture'].__name__ + if fixturename in item.fixturenames: + item.fixturenames.remove(fixturename) + item.fixturenames.insert(0, fixturename) + if 'launch_service' in item.fixturenames: + item.fixturenames.remove('launch_service') + item.fixturenames.insert(0, 'launch_service') + + +def get_error_context_from_obj(obj): + """Return formatted information of the object location.""" + try: + fspath = inspect.getsourcefile(obj) + except TypeError: + return 'location information of the object not found' + try: + lines, lineno = inspect.getsourcelines(obj) + except IOError: + return f'file {fspath}: source code not available' + error_msg = f'file {fspath}, line {lineno}' + for line in lines: + line = line.rstrip() + error_msg += f'\n {line}' + if line.lstrip().startswith('def'): + break + return error_msg + + +@pytest.hookimpl(hookwrapper=True, tryfirst=True) +def pytest_pyfunc_call(pyfuncitem): + """Run launch_testing test coroutines and functions in an event loop.""" + marker = pyfuncitem.get_closest_marker('launch_testing') + if marker is not None: + args = {} + func = pyfuncitem.obj + spec = inspect.getfullargspec(func) + fixturename = marker.kwargs['fixture'].__name__ + ld_extra_args_pair = pyfuncitem.funcargs[fixturename] + extra_args = {} + if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: + extra_args = ld_extra_args_pair[1] + ls = pyfuncitem.funcargs['launch_service'] + for name, value in extra_args.items(): + if name in itertools.chain(spec.args, spec.kwonlyargs): + args[name] = value + if inspect.iscoroutinefunction(func): + pyfuncitem.obj = wrap_coroutine(func, args, ls) + else: + pyfuncitem.obj = wrap_func(func, args, ls) + yield + + +def wrap_coroutine(func, args, ls): + """Return a sync wrapper around an async function to be executed in the event loop.""" + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + coro = func(**kwargs) + loop = ls.event_loop + task = asyncio.ensure_future(coro, loop=loop) + run_until_complete(loop, task) + + return inner + + +def wrap_func(func, args, ls): + """Return a wrapper that runs the test in a separate thread while driving the event loop.""" + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + loop = ls.event_loop + future = loop.run_in_executor(None, functools.partial(func, **kwargs)) + run_until_complete(loop, future) + return inner + + +def update_arguments(kwargs, extra_kwargs): + """Update kwargs with extra_kwargs, print a warning if a key overlaps.""" + overlapping_keys = kwargs.keys() & extra_kwargs.keys() + if overlapping_keys: + warnings.warn( + 'Argument(s) returned in launch_testing fixture has the same name than a pytest ' + 'fixture. Use different names to avoid confusing errors.') + + kwargs.update(extra_kwargs) + + +def run_until_complete(loop, future_like): + """Similar to `asyncio.EventLoop.run_until_complete`, but it consumes all exceptions.""" + try: + loop.run_until_complete(future_like) + except BaseException: + if future_like.done() and not future_like.cancelled(): + future_like.exception() + raise + + +@pytest.fixture +def launch_service(): + """Create an instance of the launch service for each test case.""" + ls = LaunchService() + yield ls + assert ls._is_idle(), ( + 'launch service must be shut down before fixture tear down' + ) diff --git a/launch_testing/package.xml b/launch_testing/package.xml index 4c414847c..f184b7f62 100644 --- a/launch_testing/package.xml +++ b/launch_testing/package.xml @@ -15,12 +15,13 @@ ament_index_python launch osrf_pycommon + python3-pytest + python3-pytest-asyncio ament_copyright ament_flake8 ament_pep257 launch - python3-pytest ament_python diff --git a/launch_testing/setup.py b/launch_testing/setup.py index 14e72b01a..e6cb247b2 100644 --- a/launch_testing/setup.py +++ b/launch_testing/setup.py @@ -16,7 +16,10 @@ ], entry_points={ 'console_scripts': ['launch_test=launch_testing.launch_test:main'], - 'pytest11': ['launch = launch_testing.pytest.hooks'], + 'pytest11': [ + 'launch_testing_legacy = launch_testing.pytest.legacy.hooks', + 'launch_testing = launch_testing.pytest.plugin' + ], }, install_requires=['setuptools'], zip_safe=True, diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py new file mode 100644 index 000000000..d17c11a0c --- /dev/null +++ b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py @@ -0,0 +1,33 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import launch +import launch_testing + +import pytest + + +@launch_testing.pytest.fixture() +def ld(): + return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) + + +@pytest.mark.launch_testing(fixture=ld) +async def test_case_1(): + assert True + + +@pytest.mark.launch_testing(fixture=ld) +def test_case_2(): + assert True diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py new file mode 100644 index 000000000..9ad461627 --- /dev/null +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -0,0 +1,33 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import launch +import launch_testing + +import pytest + + +@launch_testing.pytest.fixture(scope='module') +def ld(): + return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) + + +@pytest.mark.launch_testing(fixture=ld) +async def test_case_1(): + assert True + + +@pytest.mark.launch_testing(fixture=ld) +def test_case_2(): + assert True From 7406962f3bb263a3b5ebf2a28ae5349544930976 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 8 Sep 2021 10:34:45 -0300 Subject: [PATCH 02/48] fix typo Signed-off-by: Ivan Santiago Paunovic Co-authored-by: Michel Hidalgo --- launch_testing/launch_testing/pytest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 7409082d5..cab884ee9 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -131,7 +131,7 @@ def get_ready_to_test_action(launch_description): ) try: next(gen) - except StopIteration: # Only one ReadeToTest action must be found + except StopIteration: # Only one ReadyToTest action must be found return ready_action raise RuntimeError( 'launch_pytest fixtures must return a LaunchDescription ' From ce81a007a749d399df14d343841a39d6d8e653fd Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 8 Sep 2021 10:36:08 -0300 Subject: [PATCH 03/48] typo Signed-off-by: Ivan Santiago Paunovic Co-authored-by: Michel Hidalgo --- launch_testing/launch_testing/pytest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index cab884ee9..e4e53b232 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -143,7 +143,7 @@ def pytest_runtest_setup(item): """Inject fixtures in launch_testing marked tests.""" marker = item.get_closest_marker('launch_testing') if marker is not None: - # inject the corrent launch_testing fixture here + # inject the correct launch_testing fixture here if 'fixture' not in marker.kwargs: raise RuntimeError( 'from keyword argument is required in a pytest.mark.launch_testing() decorator: \n' From 716e367d501f83e65992229aa3026e1a615048f9 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 8 Sep 2021 11:20:04 -0300 Subject: [PATCH 04/48] Shutdown tests and more stuff Signed-off-by: Ivan Santiago Paunovic --- launch/launch/launch_service.py | 7 + .../launch_testing/pytest/fixture.py | 2 +- .../launch_testing/pytest/plugin.py | 144 ++++++++++++------ .../new_hooks/test_function_scope.py | 6 +- .../new_hooks/test_module_scope.py | 28 +++- 5 files changed, 130 insertions(+), 57 deletions(-) diff --git a/launch/launch/launch_service.py b/launch/launch/launch_service.py index ccc38d03b..fcfd8c3a0 100644 --- a/launch/launch/launch_service.py +++ b/launch/launch/launch_service.py @@ -86,6 +86,7 @@ def __init__( # it being set to None by run() as it exits. self.__loop_from_run_thread_lock = threading.RLock() self.__loop_from_run_thread = None + self.__this_task = None # Used to indicate when shutdown() has been called. self.__shutting_down = False @@ -183,6 +184,7 @@ def _prepare_run_loop(self): except AttributeError: this_task = asyncio.Task.current_task(this_loop) + self.__this_task = this_task # Setup custom signal handlers for SIGINT, SIGTERM and maybe SIGQUIT. sigint_received = False @@ -417,3 +419,8 @@ def context(self): @property def event_loop(self): return self.__loop_from_run_thread + + @property + def task(self): + """Return asyncio task associated with this launch service.""" + return self.__this_task diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index 380572068..eb2d898a0 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -30,7 +30,7 @@ def scope_gt(scope1, scope2): def get_launch_service_fixture(scope='function'): @pytest.fixture(scope=scope) - def launch_service(): + def launch_service(event_loop): ls = launch.LaunchService() yield ls assert ls._is_idle(), ( diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index e4e53b232..5cfb6d2b7 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -47,22 +47,13 @@ def pytest_configure(config): 'run using the specified launch_pad.', ) -# Adapted from https://github.com/pytest-dev/pytest-asyncio, -# see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. -@pytest.mark.tryfirst -def pytest_pycollect_makeitem(collector, name, obj): - """Collect coroutine based launch tests.""" - if collector.funcnamefilter(name) and inspect.iscoroutinefunction(obj): - item = pytest.Function.from_parent(collector, name=name) - # Due to how pytest test collection works, module-level pytestmarks - # are applied after the collection step. Since this is the collection - # step, we look ourselves. - transfer_markers(obj, item.cls, item.module) - item = pytest.Function.from_parent(collector, name=name) # To reload keywords. - - if 'launch_testing' in item.keywords: - return list(collector._genfunctions(name, obj)) +def finalize_launch_service(ls, eprefix=''): + ls.shutdown() + loop = ls.event_loop + if loop is not None: + rc = loop.run_until_complete(ls.task) + assert rc == 0, f"{eprefix} launch service failed when finishing, return code '{rc}'" @pytest.hookimpl(hookwrapper=True, tryfirst=True) @@ -94,12 +85,13 @@ def pytest_fixture_setup(fixturedef, request): event = asyncio.Event() ready._add_callback(lambda: event.set()) - def finalize(): - ls.shutdown() - rc = event_loop.run_until_complete(run_async_task) - assert rc == 0, f"{eprefix} launch service failed when finishing, return code '{rc}'" - fixturedef.addfinalizer(finalize) + fixturedef.addfinalizer(functools.partial(finalize_launch_service, ls=ls, eprefix=eprefix)) run_until_complete(event_loop, event.wait()) + # this is guaranteed by the current run_async() implementation, let's check it just in case + # it changes in the future + assert ls.event_loop is event_loop + assert ls.context.asyncio_loop is event_loop + assert ls.task is run_async_task return yield @@ -139,23 +131,48 @@ def get_ready_to_test_action(launch_description): ) -def pytest_runtest_setup(item): - """Inject fixtures in launch_testing marked tests.""" - marker = item.get_closest_marker('launch_testing') - if marker is not None: - # inject the correct launch_testing fixture here - if 'fixture' not in marker.kwargs: - raise RuntimeError( - 'from keyword argument is required in a pytest.mark.launch_testing() decorator: \n' - f'{get_error_context_from_obj(item.obj)}' - ) - fixturename = marker.kwargs['fixture'].__name__ - if fixturename in item.fixturenames: - item.fixturenames.remove(fixturename) - item.fixturenames.insert(0, fixturename) - if 'launch_service' in item.fixturenames: - item.fixturenames.remove('launch_service') - item.fixturenames.insert(0, 'launch_service') +def is_valid_test_item(obj): + """Return true if obj is a valid launch test item.""" + return inspect.iscoroutinefunction(obj) or inspect.isfunction(obj) + +# Adapted from https://github.com/pytest-dev/pytest-asyncio, +# see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. +@pytest.mark.tryfirst +def pytest_pycollect_makeitem(collector, name, obj): + """Collect coroutine based launch tests.""" + if collector.funcnamefilter(name) and is_valid_test_item(obj): + item = pytest.Function.from_parent(collector, name=name) + + # Due to how pytest test collection works, module-level pytestmarks + # are applied after the collection step. Since this is the collection + # step, we look ourselves. + transfer_markers(obj, item.cls, item.module) + item = pytest.Function.from_parent(collector, name=name) # To reload keywords. + + marker = item.get_closest_marker('launch_testing') + if marker is not None: + # inject the correct launch_testing fixture here + if 'fixture' not in marker.kwargs: + warnings.warn( + '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' + f'decorator: \n{get_error_context_from_obj(item.obj)}' + ) + return None + fixturename = marker.kwargs['fixture'].__name__ + # injects the needed fixtures in all items here + # injecting the fixture here makes sure that pytest reorder tests correctly + items = list(collector._genfunctions(name, obj)) + for item in items: + if fixturename in item.fixturenames: + item.fixturenames.remove(fixturename) + item.fixturenames.insert(0, fixturename) + if 'launch_service' in item.fixturenames: + item.fixturenames.remove('launch_service') + item.fixturenames.insert(0, 'launch_service') + if 'event_loop' in item.fixturenames: + item.fixturenames.remove('event_loop') + item.fixturenames.insert(0, 'event_loop') + return items def get_error_context_from_obj(obj): @@ -177,6 +194,25 @@ def get_error_context_from_obj(obj): return error_msg +@pytest.mark.trylast +def pytest_collection_modifyitems(session, config, items): + """Reorder tests, so shutdown tests happen after the corresponding fixture teardown.""" + + def cmp(left, right): + leftm = left.get_closest_marker('launch_testing') + rightm = right.get_closest_marker('launch_testing') + if None in (leftm, rightm): + return 0 + if leftm.kwargs['fixture'] is not rightm.kwargs['fixture']: + return 0 + left_is_shutdown = int(leftm.kwargs.get('shutdown', False)) + right_is_shutdown = int(rightm.kwargs.get('shutdown', False)) + return left_is_shutdown - right_is_shutdown + + # python sort is guaranteed to be stable + items.sort(key=functools.cmp_to_key(cmp)) + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_pyfunc_call(pyfuncitem): """Run launch_testing test coroutines and functions in an event loop.""" @@ -186,44 +222,56 @@ def pytest_pyfunc_call(pyfuncitem): func = pyfuncitem.obj spec = inspect.getfullargspec(func) fixturename = marker.kwargs['fixture'].__name__ + shutdown_test = marker.kwargs.get('shutdown', False) ld_extra_args_pair = pyfuncitem.funcargs[fixturename] extra_args = {} if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: extra_args = ld_extra_args_pair[1] + event_loop = pyfuncitem.funcargs['event_loop'] ls = pyfuncitem.funcargs['launch_service'] for name, value in extra_args.items(): if name in itertools.chain(spec.args, spec.kwonlyargs): args[name] = value + before = None + if shutdown_test: + def before(): + finalize_launch_service( + ls, + eprefix=( + 'Failed to finalize launch service while running test' + f' "{pyfuncitem.obj.__name__}"')) if inspect.iscoroutinefunction(func): - pyfuncitem.obj = wrap_coroutine(func, args, ls) + pyfuncitem.obj = wrap_coroutine(func, args, event_loop, before) else: - pyfuncitem.obj = wrap_func(func, args, ls) + pyfuncitem.obj = wrap_func(func, args, event_loop, before) yield -def wrap_coroutine(func, args, ls): +def wrap_coroutine(func, args, event_loop, before): """Return a sync wrapper around an async function to be executed in the event loop.""" @functools.wraps(func) def inner(**kwargs): + if before is not None: + before() update_arguments(kwargs, args) coro = func(**kwargs) - loop = ls.event_loop - task = asyncio.ensure_future(coro, loop=loop) - run_until_complete(loop, task) + task = asyncio.ensure_future(coro, loop=event_loop) + run_until_complete(event_loop, task) return inner -def wrap_func(func, args, ls): +def wrap_func(func, args, event_loop, before): """Return a wrapper that runs the test in a separate thread while driving the event loop.""" @functools.wraps(func) def inner(**kwargs): + if before is not None: + before() update_arguments(kwargs, args) - loop = ls.event_loop - future = loop.run_in_executor(None, functools.partial(func, **kwargs)) - run_until_complete(loop, future) + future = event_loop.run_in_executor(None, functools.partial(func, **kwargs)) + run_until_complete(event_loop, future) return inner @@ -249,7 +297,7 @@ def run_until_complete(loop, future_like): @pytest.fixture -def launch_service(): +def launch_service(event_loop): """Create an instance of the launch service for each test case.""" ls = LaunchService() yield ls diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py index d17c11a0c..e031f4a4e 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py @@ -19,15 +19,15 @@ @launch_testing.pytest.fixture() -def ld(): +def launch_description(): return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) -@pytest.mark.launch_testing(fixture=ld) +@pytest.mark.launch_testing(fixture=launch_description) async def test_case_1(): assert True -@pytest.mark.launch_testing(fixture=ld) +@pytest.mark.launch_testing(fixture=launch_description) def test_case_2(): assert True diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index 9ad461627..9a624b24a 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -18,16 +18,34 @@ import pytest +@pytest.fixture(scope='module') +def order(): + return [] + + @launch_testing.pytest.fixture(scope='module') -def ld(): +def launch_description(): return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) -@pytest.mark.launch_testing(fixture=ld) -async def test_case_1(): +@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +async def test_after_shutdown(order, launch_service): + order.append('test_after_shutdown') + assert launch_service._is_idle() + assert launch_service.event_loop is None + + +@pytest.mark.launch_testing(fixture=launch_description) +async def test_case_1(order): + order.append('test_case_1') assert True -@pytest.mark.launch_testing(fixture=ld) -def test_case_2(): +@pytest.mark.launch_testing(fixture=launch_description) +def test_case_2(order): + order.append('test_case_2') assert True + + +def test_order(order): + assert order == ['test_case_1', 'test_case_2', 'test_after_shutdown'] From 72f0545326ae5a89b6ff7e2c8ece12d9616445f1 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 8 Sep 2021 18:16:24 -0300 Subject: [PATCH 05/48] Support using generators and asyncgens for tests with a shutdown step... might be a problem in function scoped launch tests right now Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 149 +++++++++++++++--- .../new_hooks/test_module_scope.py | 14 ++ 2 files changed, 139 insertions(+), 24 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 5cfb6d2b7..551186a60 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -18,9 +18,13 @@ import itertools import warnings +from _pytest.outcomes import fail +from _pytest.outcomes import skip + import launch from launch.launch_service import LaunchService import launch_testing + import pytest @@ -133,7 +137,36 @@ def get_ready_to_test_action(launch_description): def is_valid_test_item(obj): """Return true if obj is a valid launch test item.""" - return inspect.iscoroutinefunction(obj) or inspect.isfunction(obj) + return ( + inspect.iscoroutinefunction(obj) or inspect.isfunction(obj) + or inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) + ) + + +def need_shutdown_test_item(obj): + """Return true if we also need to generate a shutdown test item for this object.""" + return inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) + + +def generate_test_items(collector, name, obj, fixturename, is_shutdown): + """Return list of test items for the corresponding object and injects the needed fixtures.""" + items = list(collector._genfunctions(name, obj)) + for item in items: + if fixturename in item.fixturenames: + item.fixturenames.remove(fixturename) + item.fixturenames.insert(0, fixturename) + if 'launch_service' in item.fixturenames: + item.fixturenames.remove('launch_service') + item.fixturenames.insert(0, 'launch_service') + if 'event_loop' in item.fixturenames: + item.fixturenames.remove('event_loop') + item.fixturenames.insert(0, 'event_loop') + item._launch_testing_is_shutdown = is_shutdown + if is_shutdown: + # rename the items, to differentiate them from the normal test stage + item.name = f'{name}[shutdown_test]' + return items + # Adapted from https://github.com/pytest-dev/pytest-asyncio, # see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @@ -159,19 +192,13 @@ def pytest_pycollect_makeitem(collector, name, obj): ) return None fixturename = marker.kwargs['fixture'].__name__ - # injects the needed fixtures in all items here - # injecting the fixture here makes sure that pytest reorder tests correctly - items = list(collector._genfunctions(name, obj)) - for item in items: - if fixturename in item.fixturenames: - item.fixturenames.remove(fixturename) - item.fixturenames.insert(0, fixturename) - if 'launch_service' in item.fixturenames: - item.fixturenames.remove('launch_service') - item.fixturenames.insert(0, 'launch_service') - if 'event_loop' in item.fixturenames: - item.fixturenames.remove('event_loop') - item.fixturenames.insert(0, 'event_loop') + is_shutdown = marker.kwargs.get('shutdown', False) + items = generate_test_items(collector, name, obj, fixturename, is_shutdown) + if need_shutdown_test_item(obj): + shutdown_items = generate_test_items(collector, name, obj, fixturename, True) + for item, shutdown_item in zip(items, shutdown_items): + item._launch_testing_shutdown_item = shutdown_item + items.extend(shutdown_items) return items @@ -205,8 +232,8 @@ def cmp(left, right): return 0 if leftm.kwargs['fixture'] is not rightm.kwargs['fixture']: return 0 - left_is_shutdown = int(leftm.kwargs.get('shutdown', False)) - right_is_shutdown = int(rightm.kwargs.get('shutdown', False)) + left_is_shutdown = int(left._launch_testing_is_shutdown) + right_is_shutdown = int(right._launch_testing_is_shutdown) return left_is_shutdown - right_is_shutdown # python sort is guaranteed to be stable @@ -222,7 +249,7 @@ def pytest_pyfunc_call(pyfuncitem): func = pyfuncitem.obj spec = inspect.getfullargspec(func) fixturename = marker.kwargs['fixture'].__name__ - shutdown_test = marker.kwargs.get('shutdown', False) + shutdown_test = pyfuncitem._launch_testing_is_shutdown ld_extra_args_pair = pyfuncitem.funcargs[fixturename] extra_args = {} if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: @@ -233,16 +260,26 @@ def pytest_pyfunc_call(pyfuncitem): if name in itertools.chain(spec.args, spec.kwonlyargs): args[name] = value before = None + shutdown_func = functools.partial( + finalize_launch_service, + ls, + eprefix=( + 'Failed to finalize launch service while running test' + f' "{pyfuncitem.obj.__name__}"')) if shutdown_test: - def before(): - finalize_launch_service( - ls, - eprefix=( - 'Failed to finalize launch service while running test' - f' "{pyfuncitem.obj.__name__}"')) + before = shutdown_func + if inspect.iscoroutinefunction(func): pyfuncitem.obj = wrap_coroutine(func, args, event_loop, before) - else: + elif inspect.isgeneratorfunction(func): + pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( + wrap_generator(func, args, event_loop, shutdown_func) + ) + elif inspect.isasyncgenfunction(func): + pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( + wrap_asyncgen(func, args, event_loop, shutdown_func) + ) + elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): pyfuncitem.obj = wrap_func(func, args, event_loop, before) yield @@ -275,6 +312,70 @@ def inner(**kwargs): return inner +def wrap_generator(func, args, event_loop, shutdown_func): + """Return wrappers for the normal test and the teardown test for a generator function.""" + + @functools.wraps(func) + def shutdown(**kwargs): + gen = getattr(shutdown, 'gen', None) + if gen is None: + skip('shutdown test skipped because the test failed before') + shutdown_func() + try: + next(gen) + except StopIteration: + return + fail( + 'launch tests using a generator function must stop iteration after yielding once', + pytrace=False + ) + shutdown._launch_testing_wrapped = True + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + gen = func(**kwargs) + future = event_loop.run_in_executor(None, lambda: next(gen)) + run_until_complete(event_loop, future) + shutdown.gen = gen + + return inner, shutdown + + +def wrap_asyncgen(func, args, event_loop, shutdown_func): + """Return wrappers for the normal test and the teardown test for an async gen function.""" + + @functools.wraps(func) + def shutdown(**kwargs): + agen = getattr(shutdown, 'agen', None) + if agen is None: + skip('shutdown test skipped because the test failed before') + shutdown_func() + # event_loop = kwargs['event_loop'] + try: + coro = agen.__anext__() + task = asyncio.ensure_future(coro, loop=event_loop) + run_until_complete(event_loop, task) + except StopAsyncIteration: + return + fail( + 'launch tests using an async gen function must stop iteration after yielding once', + pytrace=False + ) + shutdown._launch_testing_wrapped = True + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + agen = func(**kwargs) + shutdown.agen = agen + coro = agen.__anext__() + task = asyncio.ensure_future(coro, loop=event_loop) + run_until_complete(event_loop, task) + + return inner, shutdown + + def update_arguments(kwargs, extra_kwargs): """Update kwargs with extra_kwargs, print a warning if a key overlaps.""" overlapping_keys = kwargs.keys() & extra_kwargs.keys() diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index 9a624b24a..6b89286c6 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -47,5 +47,19 @@ def test_case_2(order): assert True +@pytest.mark.launch_testing(fixture=launch_description) +def test_case_3(): + assert True + yield + assert True + + +@pytest.mark.launch_testing(fixture=launch_description) +async def test_case_4(): + assert True + yield + assert True + + def test_order(order): assert order == ['test_case_1', 'test_case_2', 'test_after_shutdown'] From 1a318dfb047c45484d6ca2f662687d8673806bf9 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 9 Sep 2021 16:03:43 -0300 Subject: [PATCH 06/48] Fixes in how to handle shutdown tests, including considerations for function scoped launch test fixtures Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/fixture.py | 33 ++-- .../launch_testing/pytest/plugin.py | 158 ++++++++++++------ .../launch_testing/util/__init__.py | 5 +- .../new_hooks/test_function_scope.py | 55 +++++- .../new_hooks/test_module_scope.py | 32 +++- 5 files changed, 207 insertions(+), 76 deletions(-) diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index eb2d898a0..321db15b2 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -27,28 +27,41 @@ def scope_gt(scope1, scope2): from _pytest.fixtures import scopemismatch as scope_gt -def get_launch_service_fixture(scope='function'): +def finalize_launch_service(launch_service, eprefix=''): + launch_service.shutdown(force_sync=True) + loop = launch_service.event_loop + if loop is not None and not loop.is_closed(): + rc = loop.run_until_complete(launch_service.task) + assert rc == 0, f"{eprefix} launch service failed when finishing, return code '{rc}'" + + +def get_launch_service_fixture(*, scope='function', overridable=True): + """Return a launch service fixture.""" @pytest.fixture(scope=scope) def launch_service(event_loop): + """Create an instance of the launch service for each test case.""" ls = launch.LaunchService() yield ls - assert ls._is_idle(), ( - 'launch service must be shut down before fixture tear down' - ) - launch_service._launch_testing_overridable_fixture = True + finalize_launch_service(ls, eprefix='When tearing down launch_service fixture') + if overridable: + launch_service._launch_testing_overridable_fixture = True + launch_service._launch_testing_fixture_scope = scope return launch_service -def get_event_loop_fixture(scope='function'): +def get_event_loop_fixture(*, scope='function', overridable=True): + """Return an event loop fixture.""" @pytest.fixture(scope=scope) def event_loop(): + """Create an event loop instance for each test case.""" loop = asyncio.get_event_loop_policy().new_event_loop() yield loop loop.close() - event_loop._launch_testing_overridable_fixture = True - event_loop._launch_testing_fixture_scope = scope + if overridable: + event_loop._launch_testing_overridable_fixture = True + event_loop._launch_testing_fixture_scope = scope return event_loop @@ -78,9 +91,9 @@ def fixture(*args, **kwargs): getattr(obj, '_launch_testing_overridable_fixture', False) and scope_gt(scope, obj._launch_testing_fixture_scope) ): - mod_locals[name] = getter(scope) + mod_locals[name] = getter(scope=scope) else: - mod_locals[name] = getter(scope) + mod_locals[name] = getter(scope=scope) def decorator(fixture_function): fixture_function._launch_pytest_fixture = True diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 551186a60..cec94cbf0 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -22,11 +22,12 @@ from _pytest.outcomes import skip import launch -from launch.launch_service import LaunchService import launch_testing import pytest +from .fixture import finalize_launch_service +from .fixture import get_launch_service_fixture """ launch_testing native pytest based implementation. @@ -52,14 +53,6 @@ def pytest_configure(config): ) -def finalize_launch_service(ls, eprefix=''): - ls.shutdown() - loop = ls.event_loop - if loop is not None: - rc = loop.run_until_complete(ls.task) - assert rc == 0, f"{eprefix} launch service failed when finishing, return code '{rc}'" - - @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_fixture_setup(fixturedef, request): """Set up launch service for all launch_pytest fixtures.""" @@ -89,7 +82,7 @@ def pytest_fixture_setup(fixturedef, request): event = asyncio.Event() ready._add_callback(lambda: event.set()) - fixturedef.addfinalizer(functools.partial(finalize_launch_service, ls=ls, eprefix=eprefix)) + fixturedef.addfinalizer(functools.partial(finalize_launch_service, ls, eprefix=eprefix)) run_until_complete(event_loop, event.wait()) # this is guaranteed by the current run_async() implementation, let's check it just in case # it changes in the future @@ -191,10 +184,14 @@ def pytest_pycollect_makeitem(collector, name, obj): f'decorator: \n{get_error_context_from_obj(item.obj)}' ) return None - fixturename = marker.kwargs['fixture'].__name__ + fixture = marker.kwargs['fixture'] + fixturename = fixture.__name__ + scope = fixture._pytestfixturefunction.scope is_shutdown = marker.kwargs.get('shutdown', False) items = generate_test_items(collector, name, obj, fixturename, is_shutdown) - if need_shutdown_test_item(obj): + if need_shutdown_test_item(obj) and scope != 'function': + # for function scope we only need one shutdown item + # if not we're going to use two event loops!!! shutdown_items = generate_test_items(collector, name, obj, fixturename, True) for item, shutdown_item in zip(items, shutdown_items): item._launch_testing_shutdown_item = shutdown_item @@ -230,7 +227,10 @@ def cmp(left, right): rightm = right.get_closest_marker('launch_testing') if None in (leftm, rightm): return 0 - if leftm.kwargs['fixture'] is not rightm.kwargs['fixture']: + fixture = leftm.kwargs['fixture'] + if fixture is not rightm.kwargs['fixture']: + return 0 + if fixture._pytestfixturefunction.scope == 'function': return 0 left_is_shutdown = int(left._launch_testing_is_shutdown) right_is_shutdown = int(right._launch_testing_is_shutdown) @@ -238,6 +238,14 @@ def cmp(left, right): # python sort is guaranteed to be stable items.sort(key=functools.cmp_to_key(cmp)) + # This approach doesn't work, because we cannot guarantee when fixture + # finalizers are going to be run. + # remove the launch_testing fixture from shutdown tests + # so launch service shutdown happens right before... by magic!!! + for item in items: + marker = item.get_closest_marker('launch_testing') + if marker is not None and item._launch_testing_is_shutdown: + item.fixturenames.remove(marker.kwargs['fixture'].__name__) @pytest.hookimpl(hookwrapper=True, tryfirst=True) @@ -248,49 +256,51 @@ def pytest_pyfunc_call(pyfuncitem): args = {} func = pyfuncitem.obj spec = inspect.getfullargspec(func) - fixturename = marker.kwargs['fixture'].__name__ shutdown_test = pyfuncitem._launch_testing_is_shutdown - ld_extra_args_pair = pyfuncitem.funcargs[fixturename] extra_args = {} - if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: - extra_args = ld_extra_args_pair[1] + fixture = marker.kwargs['fixture'] + scope = fixture._pytestfixturefunction.scope + if not shutdown_test: + fixturename = marker.kwargs['fixture'].__name__ + ld_extra_args_pair = pyfuncitem.funcargs[fixturename] + if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: + extra_args = ld_extra_args_pair[1] event_loop = pyfuncitem.funcargs['event_loop'] ls = pyfuncitem.funcargs['launch_service'] + on_shutdown = functools.partial( + finalize_launch_service, ls, eprefix=f'When running test {func.__name__}') + before_test = on_shutdown if shutdown_test else None for name, value in extra_args.items(): if name in itertools.chain(spec.args, spec.kwonlyargs): args[name] = value - before = None - shutdown_func = functools.partial( - finalize_launch_service, - ls, - eprefix=( - 'Failed to finalize launch service while running test' - f' "{pyfuncitem.obj.__name__}"')) - if shutdown_test: - before = shutdown_func - if inspect.iscoroutinefunction(func): - pyfuncitem.obj = wrap_coroutine(func, args, event_loop, before) + pyfuncitem.obj = wrap_coroutine(func, args, event_loop, before_test) elif inspect.isgeneratorfunction(func): - pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( - wrap_generator(func, args, event_loop, shutdown_func) - ) + if scope != 'function': + pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( + wrap_generator(func, args, event_loop, on_shutdown) + ) + else: + pyfuncitem.obj = wrap_generator_fscope(func, args, event_loop, on_shutdown) elif inspect.isasyncgenfunction(func): - pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( - wrap_asyncgen(func, args, event_loop, shutdown_func) - ) + if scope != 'function': + pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( + wrap_asyncgen(func, args, event_loop, on_shutdown) + ) + else: + pyfuncitem.obj = wrap_asyncgen_fscope(func, args, event_loop, on_shutdown) elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): - pyfuncitem.obj = wrap_func(func, args, event_loop, before) + pyfuncitem.obj = wrap_func(func, args, event_loop, before_test) yield -def wrap_coroutine(func, args, event_loop, before): +def wrap_coroutine(func, args, event_loop, before_test): """Return a sync wrapper around an async function to be executed in the event loop.""" @functools.wraps(func) def inner(**kwargs): - if before is not None: - before() + if before_test is not None: + before_test() update_arguments(kwargs, args) coro = func(**kwargs) task = asyncio.ensure_future(coro, loop=event_loop) @@ -299,20 +309,20 @@ def inner(**kwargs): return inner -def wrap_func(func, args, event_loop, before): +def wrap_func(func, args, event_loop, before_test): """Return a wrapper that runs the test in a separate thread while driving the event loop.""" @functools.wraps(func) def inner(**kwargs): - if before is not None: - before() + if before_test is not None: + before_test() update_arguments(kwargs, args) future = event_loop.run_in_executor(None, functools.partial(func, **kwargs)) run_until_complete(event_loop, future) return inner -def wrap_generator(func, args, event_loop, shutdown_func): +def wrap_generator(func, args, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for a generator function.""" @functools.wraps(func) @@ -320,7 +330,7 @@ def shutdown(**kwargs): gen = getattr(shutdown, 'gen', None) if gen is None: skip('shutdown test skipped because the test failed before') - shutdown_func() + on_shutdown() try: next(gen) except StopIteration: @@ -342,7 +352,29 @@ def inner(**kwargs): return inner, shutdown -def wrap_asyncgen(func, args, event_loop, shutdown_func): +def wrap_generator_fscope(func, args, event_loop, on_shutdown): + """Return wrappers for the normal test and the teardown test for a generator function.""" + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + gen = func(**kwargs) + future = event_loop.run_in_executor(None, lambda: next(gen)) + run_until_complete(event_loop, future) + on_shutdown() + try: + next(gen) + except StopIteration: + return + fail( + 'launch tests using a generator function must stop iteration after yielding once', + pytrace=False + ) + + return inner + + +def wrap_asyncgen(func, args, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for an async gen function.""" @functools.wraps(func) @@ -350,8 +382,7 @@ def shutdown(**kwargs): agen = getattr(shutdown, 'agen', None) if agen is None: skip('shutdown test skipped because the test failed before') - shutdown_func() - # event_loop = kwargs['event_loop'] + on_shutdown() try: coro = agen.__anext__() task = asyncio.ensure_future(coro, loop=event_loop) @@ -376,6 +407,31 @@ def inner(**kwargs): return inner, shutdown +def wrap_asyncgen_fscope(func, args, event_loop, on_shutdown): + """Return wrappers for the normal test and the teardown test for an async gen function.""" + + @functools.wraps(func) + def inner(**kwargs): + update_arguments(kwargs, args) + agen = func(**kwargs) + coro = agen.__anext__() + task = asyncio.ensure_future(coro, loop=event_loop) + run_until_complete(event_loop, task) + on_shutdown() + try: + coro = agen.__anext__() + task = asyncio.ensure_future(coro, loop=event_loop) + run_until_complete(event_loop, task) + except StopAsyncIteration: + return + fail( + 'launch tests using an async gen function must stop iteration after yielding once', + pytrace=False + ) + + return inner + + def update_arguments(kwargs, extra_kwargs): """Update kwargs with extra_kwargs, print a warning if a key overlaps.""" overlapping_keys = kwargs.keys() & extra_kwargs.keys() @@ -397,11 +453,5 @@ def run_until_complete(loop, future_like): raise -@pytest.fixture -def launch_service(event_loop): - """Create an instance of the launch service for each test case.""" - ls = LaunchService() - yield ls - assert ls._is_idle(), ( - 'launch service must be shut down before fixture tear down' - ) +"""Launch service fixture.""" +launch_service = get_launch_service_fixture(overridable=False) diff --git a/launch_testing/launch_testing/util/__init__.py b/launch_testing/launch_testing/util/__init__.py index fb9b83b05..aadc5e841 100644 --- a/launch_testing/launch_testing/util/__init__.py +++ b/launch_testing/launch_testing/util/__init__.py @@ -30,9 +30,12 @@ def KeepAliveProc(): another process to keep the launch service alive while the tests are running. """ script = """ +import signal +import time + try: while True: - pass + time.sleep(1) except KeyboardInterrupt: pass """ diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py index e031f4a4e..0619b92c8 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py @@ -18,16 +18,65 @@ import pytest +@pytest.fixture(scope='module') +def order(): + return [] + + @launch_testing.pytest.fixture() def launch_description(): - return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) + return launch.LaunchDescription([ + launch_testing.util.KeepAliveProc(), + launch_testing.actions.ReadyToTest(), + ]) + + +# TODO(ivanpauno) +# We cannot get variables from the dictionary returned by launch_description +# because we're removing the fixture before running the tests. +# Maybe we can delete this feature, and use generators/asyncgens. +# We can always get the variables returned by the launch_testing fixture there. +@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +async def test_after_shutdown(order, launch_service): + order.append('test_after_shutdown') + assert launch_service._is_idle() + assert launch_service.event_loop is None @pytest.mark.launch_testing(fixture=launch_description) -async def test_case_1(): +async def test_case_1(order): + order.append('test_case_1') assert True @pytest.mark.launch_testing(fixture=launch_description) -def test_case_2(): +def test_case_2(order): + order.append('test_case_2') assert True + + +@pytest.mark.launch_testing(fixture=launch_description) +def test_case_3(order, launch_service): + order.append('test_case_3') + yield + # assert launch_service._finalized + order.append('test_case_3[shutdown]') + + +@pytest.mark.launch_testing(fixture=launch_description) +async def test_case_4(order): + order.append('test_case_4') + yield + order.append('test_case_4[shutdown]') + + +def test_order(order): + assert order == [ + 'test_after_shutdown', + 'test_case_1', + 'test_case_2', + 'test_case_3', + 'test_case_3[shutdown]', + 'test_case_4', + 'test_case_4[shutdown]', + ] diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index 6b89286c6..c0c487045 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -25,9 +25,16 @@ def order(): @launch_testing.pytest.fixture(scope='module') def launch_description(): - return launch.LaunchDescription([launch_testing.actions.ReadyToTest()]) + return launch.LaunchDescription([ + launch_testing.util.KeepAliveProc(), + launch_testing.actions.ReadyToTest(), + ]) +# TODO(ivanpauno) +# We cannot get variables from the dictionary returned by launch_description +# because we're removing the fixture before running the tests. +# Maybe we can delete this feature, and use generators/asyncgens. @pytest.mark.launch_testing(fixture=launch_description, shutdown=True) async def test_after_shutdown(order, launch_service): order.append('test_after_shutdown') @@ -48,18 +55,27 @@ def test_case_2(order): @pytest.mark.launch_testing(fixture=launch_description) -def test_case_3(): - assert True +def test_case_3(order, launch_service): + order.append('test_case_3') yield - assert True + # assert launch_service._finalized + order.append('test_case_3[shutdown]') @pytest.mark.launch_testing(fixture=launch_description) -async def test_case_4(): - assert True +async def test_case_4(order): + order.append('test_case_4') yield - assert True + order.append('test_case_4[shutdown]') def test_order(order): - assert order == ['test_case_1', 'test_case_2', 'test_after_shutdown'] + assert order == [ + 'test_case_1', + 'test_case_2', + 'test_case_3', + 'test_case_4', + 'test_after_shutdown', + 'test_case_3[shutdown]', + 'test_case_4[shutdown]', + ] From 523c32fa7a4df22fb833179d272e41efc97848c5 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 10 Sep 2021 17:52:28 -0300 Subject: [PATCH 07/48] Add missing docstring Signed-off-by: Ivan Santiago Paunovic --- launch/launch/launch_service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/launch/launch/launch_service.py b/launch/launch/launch_service.py index fcfd8c3a0..d22579d94 100644 --- a/launch/launch/launch_service.py +++ b/launch/launch/launch_service.py @@ -418,6 +418,7 @@ def context(self): @property def event_loop(self): + """Getter for the event loop being used in the thread running the launch service.""" return self.__loop_from_run_thread @property From cb99b92a0045d6b67f97152455c00844b9b48538 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 10 Sep 2021 18:00:20 -0300 Subject: [PATCH 08/48] reorder lines Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index cec94cbf0..df3322ed6 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -399,10 +399,10 @@ def shutdown(**kwargs): def inner(**kwargs): update_arguments(kwargs, args) agen = func(**kwargs) - shutdown.agen = agen coro = agen.__anext__() task = asyncio.ensure_future(coro, loop=event_loop) run_until_complete(event_loop, task) + shutdown.agen = agen return inner, shutdown From f7ffdfed82126bd124c456d16366f94ed0e7737f Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 13 Sep 2021 18:11:03 -0300 Subject: [PATCH 09/48] Use nonlocal variable instead of inserting an attribute to a function object Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index df3322ed6..752a6c885 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -324,10 +324,11 @@ def inner(**kwargs): def wrap_generator(func, args, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for a generator function.""" + gen = None @functools.wraps(func) def shutdown(**kwargs): - gen = getattr(shutdown, 'gen', None) + nonlocal gen if gen is None: skip('shutdown test skipped because the test failed before') on_shutdown() @@ -343,11 +344,12 @@ def shutdown(**kwargs): @functools.wraps(func) def inner(**kwargs): + nonlocal gen update_arguments(kwargs, args) - gen = func(**kwargs) - future = event_loop.run_in_executor(None, lambda: next(gen)) + local_gen = func(**kwargs) + future = event_loop.run_in_executor(None, lambda: next(local_gen)) run_until_complete(event_loop, future) - shutdown.gen = gen + gen = local_gen return inner, shutdown @@ -376,10 +378,11 @@ def inner(**kwargs): def wrap_asyncgen(func, args, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for an async gen function.""" + agen = None @functools.wraps(func) def shutdown(**kwargs): - agen = getattr(shutdown, 'agen', None) + nonlocal agen if agen is None: skip('shutdown test skipped because the test failed before') on_shutdown() @@ -397,12 +400,13 @@ def shutdown(**kwargs): @functools.wraps(func) def inner(**kwargs): + nonlocal agen update_arguments(kwargs, args) - agen = func(**kwargs) - coro = agen.__anext__() + local_agen = func(**kwargs) + coro = local_agen.__anext__() task = asyncio.ensure_future(coro, loop=event_loop) run_until_complete(event_loop, task) - shutdown.agen = agen + agen = local_agen return inner, shutdown From be8ac46b310d8b51aa89d8d20d9ae60edc7f8d63 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 22 Sep 2021 16:58:45 -0300 Subject: [PATCH 10/48] Refactor Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 331 ++++++++++-------- .../new_hooks/test_module_scope.py | 77 ++-- 2 files changed, 240 insertions(+), 168 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 752a6c885..527946603 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -16,6 +16,7 @@ import functools import inspect import itertools +from collections.abc import Sequence import warnings from _pytest.outcomes import fail @@ -43,6 +44,11 @@ def transfer_markers(*args, **kwargs): # noqa pass +class LaunchTestWarning(pytest.PytestWarning): + """Raised in this plugin to warn users.""" + pass + + def pytest_configure(config): """Inject launch_testing marker documentation.""" config.addinivalue_line( @@ -53,6 +59,41 @@ def pytest_configure(config): ) +# TODO(ivanpauno): Deduplicate with launch_testing +def iterate_ready_to_test_actions(entities): + """Search recursively LaunchDescription entities for all ReadyToTest actions.""" + for entity in entities: + if isinstance(entity, launch_testing.actions.ReadyToTest): + yield entity + yield from iterate_ready_to_test_actions( + entity.describe_sub_entities() + ) + for conditional_sub_entity in entity.describe_conditional_sub_entities(): + yield from iterate_ready_to_test_actions( + conditional_sub_entity[1] + ) + + +def get_ready_to_test_action(launch_description): + """Extract the ready to test action from the launch description.""" + gen = (e for e in iterate_ready_to_test_actions(launch_description.entities)) + try: + ready_action = next(gen) + except StopIteration: # No ReadyToTest action found + raise RuntimeError( + 'launch_pytest fixtures must return a LaunchDescription ' + 'containing a ReadyToTest action' + ) + try: + next(gen) + except StopIteration: # Only one ReadyToTest action must be found + return ready_action + raise RuntimeError( + 'launch_pytest fixtures must return a LaunchDescription ' + 'containing only one ReadyToTest action' + ) + + @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_fixture_setup(fixturedef, request): """Set up launch service for all launch_pytest fixtures.""" @@ -65,12 +106,12 @@ def pytest_fixture_setup(fixturedef, request): ret = outcome.get_result() wrong_ret_type_error = ( f'{eprefix} return value must be either a launch description ' - 'or a launch description, locals pair' + 'or a sequence which first item is a launch description' ) ld = ret - if isinstance(ret, tuple): - assert len(ret) == 2, wrong_ret_type_error - ld, _ = ret + if isinstance(ret, Sequence): + assert len(ret) > 0, wrong_ret_type_error + ld = ret[0] assert isinstance(ld, launch.LaunchDescription), wrong_ret_type_error ls.include_launch_description(ld) run_async_task = event_loop.create_task(ls.run_async( @@ -93,39 +134,49 @@ def pytest_fixture_setup(fixturedef, request): yield -# TODO(ivanpauno): Deduplicate with launch_testing -def iterate_ready_to_test_actions(entities): - """Search recursively LaunchDescription entities for all ReadyToTest actions.""" - for entity in entities: - if isinstance(entity, launch_testing.actions.ReadyToTest): - yield entity - yield from iterate_ready_to_test_actions( - entity.describe_sub_entities() - ) - for conditional_sub_entity in entity.describe_conditional_sub_entities(): - yield from iterate_ready_to_test_actions( - conditional_sub_entity[1] - ) +def is_launch_test(item): + """Return `True` if the item is a launch test.""" + mark = item.get_closest_marker('launch_testing') + return mark is not None -def get_ready_to_test_action(launch_description): - """Extract the ready to test action from the launch description.""" - gen = (e for e in iterate_ready_to_test_actions(launch_description.entities)) - try: - ready_action = next(gen) - except StopIteration: # No ReadyToTest action found - raise RuntimeError( - 'launch_pytest fixtures must return a LaunchDescription ' - 'containing a ReadyToTest action' - ) +def is_launch_test_mark_valid(item): + """ + Return `True` if the item is a launch test. + + If not, a warning and a skip mark will be added to the item. + """ + kwargs = item.get_closest_marker('launch_testing').kwargs + ret = 'fixture' in kwargs and kwargs['fixture'] is not None + if not ret: + msg = ( + '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' + f'decorator') + item.warn(LaunchTestWarning(msg)) + item.add_marker(pytest.mark.skip(msg)) + return ret + + +def has_shutdown_kwarg(item): + """Return `True` if the launch test shutdown kwarg is true.""" + return item.get_closest_marker('launch_testing').kwargs.get('shutdown', False) + + +def get_launch_test_fixture(item): + """Return the launch test fixture name, `None` if this isn't a launch test.""" + mark = item.get_closest_marker('launch_testing') + if mark is None: + return None try: - next(gen) - except StopIteration: # Only one ReadyToTest action must be found - return ready_action - raise RuntimeError( - 'launch_pytest fixtures must return a LaunchDescription ' - 'containing only one ReadyToTest action' - ) + return mark.kwargs['fixture'] + except KeyError: + return None + + +def get_launch_test_fixturename(item): + """Return the launch test fixture name, `None` if this isn't a launch test.""" + fixture = get_launch_test_fixture(item) + return None if fixture is None else fixture.__name__ def is_valid_test_item(obj): @@ -141,23 +192,23 @@ def need_shutdown_test_item(obj): return inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) -def generate_test_items(collector, name, obj, fixturename, is_shutdown): +def generate_test_items(collector, name, obj, fixturename, is_shutdown, needs_renaming): """Return list of test items for the corresponding object and injects the needed fixtures.""" + # Inject all needed fixtures. + # We use the `usefixtures` pytest mark instead of injecting them in fixturenames + # directly, because the second option doesn't handle parameterized launch fixtures + # correctly. + # We also need to inject the mark before calling _genfunctions(), + # so it actually returns many items for parameterized launch fixtures. + fixtures_to_inject = (fixturename, 'launch_service', 'event_loop') + pytest.mark.usefixtures(*fixtures_to_inject)(obj) items = list(collector._genfunctions(name, obj)) for item in items: - if fixturename in item.fixturenames: - item.fixturenames.remove(fixturename) - item.fixturenames.insert(0, fixturename) - if 'launch_service' in item.fixturenames: - item.fixturenames.remove('launch_service') - item.fixturenames.insert(0, 'launch_service') - if 'event_loop' in item.fixturenames: - item.fixturenames.remove('event_loop') - item.fixturenames.insert(0, 'event_loop') + # Mark shutdown tests correctly item._launch_testing_is_shutdown = is_shutdown - if is_shutdown: + if is_shutdown and needs_renaming: # rename the items, to differentiate them from the normal test stage - item.name = f'{name}[shutdown_test]' + item.name = f'{item.name}[shutdown_test]' return items @@ -175,24 +226,19 @@ def pytest_pycollect_makeitem(collector, name, obj): transfer_markers(obj, item.cls, item.module) item = pytest.Function.from_parent(collector, name=name) # To reload keywords. - marker = item.get_closest_marker('launch_testing') - if marker is not None: - # inject the correct launch_testing fixture here - if 'fixture' not in marker.kwargs: - warnings.warn( - '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' - f'decorator: \n{get_error_context_from_obj(item.obj)}' - ) - return None - fixture = marker.kwargs['fixture'] + if is_launch_test(item): + if not is_launch_test_mark_valid(item): + # return an item with a warning that's going to be skipped + return [item] + fixture = get_launch_test_fixture(item) fixturename = fixture.__name__ scope = fixture._pytestfixturefunction.scope - is_shutdown = marker.kwargs.get('shutdown', False) - items = generate_test_items(collector, name, obj, fixturename, is_shutdown) + is_shutdown = has_shutdown_kwarg(item) + items = generate_test_items(collector, name, obj, fixturename, is_shutdown, False) if need_shutdown_test_item(obj) and scope != 'function': # for function scope we only need one shutdown item # if not we're going to use two event loops!!! - shutdown_items = generate_test_items(collector, name, obj, fixturename, True) + shutdown_items = generate_test_items(collector, name, obj, fixturename, True, True) for item, shutdown_item in zip(items, shutdown_items): item._launch_testing_shutdown_item = shutdown_item items.extend(shutdown_items) @@ -218,90 +264,119 @@ def get_error_context_from_obj(obj): return error_msg +def is_shutdown_test(item): + """Return `True` if the item is a launch test.""" + return getattr(item, '_launch_testing_is_shutdown', False) + + +def is_same_launch_test_fixture(left_item, right_item): + """Return `True` if both items are using the same fixture with the same parameters.""" + lfn = get_launch_test_fixture(left_item) + rfn = get_launch_test_fixture(right_item) + if None in (lfn, rfn): + return False + if lfn != rfn: + return False + if lfn._pytestfixturefunction.scope == 'function': + return False + name = lfn.__name__ + def get_fixture_params(item): + if getattr(item, 'callspec', None) is None: + return None + return item.callspec.params.get(name, None) + return get_fixture_params(left_item) == get_fixture_params(right_item) + + @pytest.mark.trylast def pytest_collection_modifyitems(session, config, items): - """Reorder tests, so shutdown tests happen after the corresponding fixture teardown.""" - - def cmp(left, right): - leftm = left.get_closest_marker('launch_testing') - rightm = right.get_closest_marker('launch_testing') - if None in (leftm, rightm): - return 0 - fixture = leftm.kwargs['fixture'] - if fixture is not rightm.kwargs['fixture']: - return 0 - if fixture._pytestfixturefunction.scope == 'function': - return 0 - left_is_shutdown = int(left._launch_testing_is_shutdown) - right_is_shutdown = int(right._launch_testing_is_shutdown) - return left_is_shutdown - right_is_shutdown - - # python sort is guaranteed to be stable - items.sort(key=functools.cmp_to_key(cmp)) - # This approach doesn't work, because we cannot guarantee when fixture - # finalizers are going to be run. - # remove the launch_testing fixture from shutdown tests - # so launch service shutdown happens right before... by magic!!! + """Move shutdown tests after normal tests.""" + + for i, item in enumerate(items): + # This algo has worst case Nitems * Nlaunchtestitems time complexity. + # We could probably find something better, but it's not that easy because: + # - Tests using the same launch fixture might not be already grouped, + # i.e. there might be a test not using the fixture in the middle. + # TODO(ivanpauno): Check if that's not actually guaranteed. + # If that's the case, we can easily modify this to have Nlaunchtestitems complexity. + # - There's no strict partial order relation we can use. + if not is_launch_test(item): + continue + new_position = None + for j, other in enumerate(items[i+1:]): + if ( + is_same_launch_test_fixture(item, other) + and is_shutdown_test(item) and not is_shutdown_test(other) + ): + new_position = i + 1 + j + if new_position is not None: + items.insert(new_position, items.pop(i)) for item in items: - marker = item.get_closest_marker('launch_testing') - if marker is not None and item._launch_testing_is_shutdown: - item.fixturenames.remove(marker.kwargs['fixture'].__name__) + if not is_shutdown_test(item): + continue + fixturename = get_launch_test_fixturename(item) + if fixturename in item.fixturenames: + item.fixturenames.remove(fixturename) @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_pyfunc_call(pyfuncitem): """Run launch_testing test coroutines and functions in an event loop.""" - marker = pyfuncitem.get_closest_marker('launch_testing') - if marker is not None: + if is_launch_test(pyfuncitem): args = {} func = pyfuncitem.obj spec = inspect.getfullargspec(func) - shutdown_test = pyfuncitem._launch_testing_is_shutdown - extra_args = {} - fixture = marker.kwargs['fixture'] + shutdown_test = is_shutdown_test(pyfuncitem) + fixture = get_launch_test_fixture(pyfuncitem) scope = fixture._pytestfixturefunction.scope - if not shutdown_test: - fixturename = marker.kwargs['fixture'].__name__ - ld_extra_args_pair = pyfuncitem.funcargs[fixturename] - if isinstance(ld_extra_args_pair, tuple) and len(ld_extra_args_pair) == 2: - extra_args = ld_extra_args_pair[1] event_loop = pyfuncitem.funcargs['event_loop'] ls = pyfuncitem.funcargs['launch_service'] on_shutdown = functools.partial( finalize_launch_service, ls, eprefix=f'When running test {func.__name__}') before_test = on_shutdown if shutdown_test else None - for name, value in extra_args.items(): - if name in itertools.chain(spec.args, spec.kwonlyargs): - args[name] = value if inspect.iscoroutinefunction(func): - pyfuncitem.obj = wrap_coroutine(func, args, event_loop, before_test) + pyfuncitem.obj = wrap_coroutine(func, event_loop, before_test) elif inspect.isgeneratorfunction(func): if scope != 'function': - pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( - wrap_generator(func, args, event_loop, on_shutdown) + shutdown_item = pyfuncitem._launch_testing_shutdown_item + pyfuncitem.obj, shutdown_item.obj = ( + wrap_generator(func, event_loop, on_shutdown) ) + shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( + shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) else: - pyfuncitem.obj = wrap_generator_fscope(func, args, event_loop, on_shutdown) + pyfuncitem.obj = wrap_generator_fscope(func, event_loop, on_shutdown) elif inspect.isasyncgenfunction(func): if scope != 'function': - pyfuncitem.obj, pyfuncitem._launch_testing_shutdown_item.obj = ( - wrap_asyncgen(func, args, event_loop, on_shutdown) + shutdown_item = pyfuncitem._launch_testing_shutdown_item + pyfuncitem.obj, shutdown_item.obj = ( + wrap_asyncgen(func, event_loop, on_shutdown) ) + shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( + shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) else: - pyfuncitem.obj = wrap_asyncgen_fscope(func, args, event_loop, on_shutdown) + pyfuncitem.obj = wrap_asyncgen_fscope(func, event_loop, on_shutdown) elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): - pyfuncitem.obj = wrap_func(func, args, event_loop, before_test) + pyfuncitem.obj = wrap_func(func, event_loop, before_test) yield -def wrap_coroutine(func, args, event_loop, before_test): +def run_until_complete(loop, future_like): + """Similar to `asyncio.EventLoop.run_until_complete`, but it consumes all exceptions.""" + try: + loop.run_until_complete(future_like) + except BaseException: + if future_like.done() and not future_like.cancelled(): + future_like.exception() + raise + + +def wrap_coroutine(func, event_loop, before_test): """Return a sync wrapper around an async function to be executed in the event loop.""" @functools.wraps(func) def inner(**kwargs): if before_test is not None: before_test() - update_arguments(kwargs, args) coro = func(**kwargs) task = asyncio.ensure_future(coro, loop=event_loop) run_until_complete(event_loop, task) @@ -309,25 +384,23 @@ def inner(**kwargs): return inner -def wrap_func(func, args, event_loop, before_test): +def wrap_func(func, event_loop, before_test): """Return a wrapper that runs the test in a separate thread while driving the event loop.""" @functools.wraps(func) def inner(**kwargs): if before_test is not None: before_test() - update_arguments(kwargs, args) future = event_loop.run_in_executor(None, functools.partial(func, **kwargs)) run_until_complete(event_loop, future) return inner -def wrap_generator(func, args, event_loop, on_shutdown): +def wrap_generator(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for a generator function.""" gen = None - @functools.wraps(func) - def shutdown(**kwargs): + def shutdown(): nonlocal gen if gen is None: skip('shutdown test skipped because the test failed before') @@ -341,11 +414,11 @@ def shutdown(**kwargs): pytrace=False ) shutdown._launch_testing_wrapped = True + shutdown.__name__ = f'{func.__name__}[shutdown]' @functools.wraps(func) def inner(**kwargs): nonlocal gen - update_arguments(kwargs, args) local_gen = func(**kwargs) future = event_loop.run_in_executor(None, lambda: next(local_gen)) run_until_complete(event_loop, future) @@ -354,12 +427,11 @@ def inner(**kwargs): return inner, shutdown -def wrap_generator_fscope(func, args, event_loop, on_shutdown): +def wrap_generator_fscope(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for a generator function.""" @functools.wraps(func) def inner(**kwargs): - update_arguments(kwargs, args) gen = func(**kwargs) future = event_loop.run_in_executor(None, lambda: next(gen)) run_until_complete(event_loop, future) @@ -376,11 +448,10 @@ def inner(**kwargs): return inner -def wrap_asyncgen(func, args, event_loop, on_shutdown): +def wrap_asyncgen(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for an async gen function.""" agen = None - @functools.wraps(func) def shutdown(**kwargs): nonlocal agen if agen is None: @@ -396,12 +467,12 @@ def shutdown(**kwargs): 'launch tests using an async gen function must stop iteration after yielding once', pytrace=False ) + shutdown.__name__ = f'{func.__name__}[shutdown]' shutdown._launch_testing_wrapped = True @functools.wraps(func) def inner(**kwargs): nonlocal agen - update_arguments(kwargs, args) local_agen = func(**kwargs) coro = local_agen.__anext__() task = asyncio.ensure_future(coro, loop=event_loop) @@ -411,12 +482,11 @@ def inner(**kwargs): return inner, shutdown -def wrap_asyncgen_fscope(func, args, event_loop, on_shutdown): +def wrap_asyncgen_fscope(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for an async gen function.""" @functools.wraps(func) def inner(**kwargs): - update_arguments(kwargs, args) agen = func(**kwargs) coro = agen.__anext__() task = asyncio.ensure_future(coro, loop=event_loop) @@ -436,26 +506,5 @@ def inner(**kwargs): return inner -def update_arguments(kwargs, extra_kwargs): - """Update kwargs with extra_kwargs, print a warning if a key overlaps.""" - overlapping_keys = kwargs.keys() & extra_kwargs.keys() - if overlapping_keys: - warnings.warn( - 'Argument(s) returned in launch_testing fixture has the same name than a pytest ' - 'fixture. Use different names to avoid confusing errors.') - - kwargs.update(extra_kwargs) - - -def run_until_complete(loop, future_like): - """Similar to `asyncio.EventLoop.run_until_complete`, but it consumes all exceptions.""" - try: - loop.run_until_complete(future_like) - except BaseException: - if future_like.done() and not future_like.cancelled(): - future_like.exception() - raise - - """Launch service fixture.""" launch_service = get_launch_service_fixture(overridable=False) diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index c0c487045..71e98b235 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -20,62 +20,85 @@ @pytest.fixture(scope='module') def order(): - return [] + print('once') + yield [] + print('end') -@launch_testing.pytest.fixture(scope='module') -def launch_description(): +@launch_testing.pytest.fixture(scope='module', params=['asd', 'bsd']) +def launch_description(request): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), launch_testing.actions.ReadyToTest(), - ]) + ]), request.param # TODO(ivanpauno) # We cannot get variables from the dictionary returned by launch_description # because we're removing the fixture before running the tests. # Maybe we can delete this feature, and use generators/asyncgens. -@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) -async def test_after_shutdown(order, launch_service): - order.append('test_after_shutdown') - assert launch_service._is_idle() - assert launch_service.event_loop is None +# @pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +# def test_after_shutdown(order, launch_service, launch_description): +# param = launch_description[1] +# order.append(f'test_after_shutdown[{param}]') +# assert launch_service._is_idle() +# assert launch_service.event_loop is None @pytest.mark.launch_testing(fixture=launch_description) -async def test_case_1(order): - order.append('test_case_1') +async def test_case_1(order, launch_description): + param = launch_description[1] + order.append(f'test_case_1[{param}]') assert True @pytest.mark.launch_testing(fixture=launch_description) -def test_case_2(order): - order.append('test_case_2') +def test_case_2(order, launch_description): + param = launch_description[1] + order.append(f'test_case_2[{param}]') assert True @pytest.mark.launch_testing(fixture=launch_description) -def test_case_3(order, launch_service): - order.append('test_case_3') +def test_case_3(order, launch_service, launch_description): + param = launch_description[1] + order.append(f'test_case_3[{param}]') yield - # assert launch_service._finalized - order.append('test_case_3[shutdown]') + assert launch_service._is_idle() + assert launch_service.event_loop is None + order.append(f'test_case_3[{param}][shutdown]') @pytest.mark.launch_testing(fixture=launch_description) -async def test_case_4(order): - order.append('test_case_4') +async def test_case_4(order, launch_service, launch_description): + param = launch_description[1] + order.append(f'test_case_4[{param}]') yield - order.append('test_case_4[shutdown]') + assert launch_service._is_idle() + assert launch_service.event_loop is None + + order.append(f'test_case_4[{param}][shutdown]') + + +# @pytest.mark.launch_testing +# def test_should_be_skipped(): +# assert True def test_order(order): assert order == [ - 'test_case_1', - 'test_case_2', - 'test_case_3', - 'test_case_4', - 'test_after_shutdown', - 'test_case_3[shutdown]', - 'test_case_4[shutdown]', + 'test_case_1[asd]', + 'test_case_2[asd]', + 'test_case_3[asd]', + 'test_case_4[asd]', + # 'test_after_shutdown[asd]', + 'test_case_3[asd][shutdown]', + 'test_case_4[asd][shutdown]', + 'test_case_1[bsd]', + 'test_case_2[bsd]', + 'test_case_3[bsd]', + 'test_case_4[bsd]', + # 'test_after_shutdown[bsd]', + 'test_case_3[bsd][shutdown]', + 'test_case_4[bsd][shutdown]', ] From f730face668869ec30f48ab0bf06d7a00fd353eb Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 24 Sep 2021 13:26:17 -0300 Subject: [PATCH 11/48] Append ready to test action to launch description if doesn't exist Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 527946603..18961f55c 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -80,10 +80,9 @@ def get_ready_to_test_action(launch_description): try: ready_action = next(gen) except StopIteration: # No ReadyToTest action found - raise RuntimeError( - 'launch_pytest fixtures must return a LaunchDescription ' - 'containing a ReadyToTest action' - ) + ready_action = launch_testing.actions.ReadyToTest() + launch_description.append(ready_action) + return ready_action try: next(gen) except StopIteration: # Only one ReadyToTest action must be found From 303779714d813c582ec77afd1716c84082dab93c Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 24 Sep 2021 19:21:23 -0300 Subject: [PATCH 12/48] Support pure shutdown tests again Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 20 ++++++++++++------- .../new_hooks/test_module_scope.py | 16 +++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 18961f55c..c7cd8decf 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -290,7 +290,15 @@ def get_fixture_params(item): def pytest_collection_modifyitems(session, config, items): """Move shutdown tests after normal tests.""" - for i, item in enumerate(items): + def enumerate_reversed(sequence): + # reversed(enumerate(sequence)), doesn't work + # here a little generator for that + n = len(sequence) - 1 + for elem in sequence[::-1]: + yield n, elem + n -= 1 + + for i, item in enumerate_reversed(items): # This algo has worst case Nitems * Nlaunchtestitems time complexity. # We could probably find something better, but it's not that easy because: # - Tests using the same launch fixture might not be already grouped, @@ -298,6 +306,8 @@ def pytest_collection_modifyitems(session, config, items): # TODO(ivanpauno): Check if that's not actually guaranteed. # If that's the case, we can easily modify this to have Nlaunchtestitems complexity. # - There's no strict partial order relation we can use. + # + # iterate in reverse order, so the order of shutdown item is stable if not is_launch_test(item): continue new_position = None @@ -307,14 +317,10 @@ def pytest_collection_modifyitems(session, config, items): and is_shutdown_test(item) and not is_shutdown_test(other) ): new_position = i + 1 + j + print(f'moving {item.name} after {other.name}: {i} {i+j+1}') if new_position is not None: items.insert(new_position, items.pop(i)) - for item in items: - if not is_shutdown_test(item): - continue - fixturename = get_launch_test_fixturename(item) - if fixturename in item.fixturenames: - item.fixturenames.remove(fixturename) + print([item.name for item in items]) @pytest.hookimpl(hookwrapper=True, tryfirst=True) diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index 71e98b235..254276f07 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -37,12 +37,12 @@ def launch_description(request): # We cannot get variables from the dictionary returned by launch_description # because we're removing the fixture before running the tests. # Maybe we can delete this feature, and use generators/asyncgens. -# @pytest.mark.launch_testing(fixture=launch_description, shutdown=True) -# def test_after_shutdown(order, launch_service, launch_description): -# param = launch_description[1] -# order.append(f'test_after_shutdown[{param}]') -# assert launch_service._is_idle() -# assert launch_service.event_loop is None +@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +def test_after_shutdown(order, launch_service, launch_description): + param = launch_description[1] + order.append(f'test_after_shutdown[{param}]') + assert launch_service._is_idle() + assert launch_service.event_loop is None @pytest.mark.launch_testing(fixture=launch_description) @@ -91,14 +91,14 @@ def test_order(order): 'test_case_2[asd]', 'test_case_3[asd]', 'test_case_4[asd]', - # 'test_after_shutdown[asd]', + 'test_after_shutdown[asd]', 'test_case_3[asd][shutdown]', 'test_case_4[asd][shutdown]', 'test_case_1[bsd]', 'test_case_2[bsd]', 'test_case_3[bsd]', 'test_case_4[bsd]', - # 'test_after_shutdown[bsd]', + 'test_after_shutdown[bsd]', 'test_case_3[bsd][shutdown]', 'test_case_4[bsd][shutdown]', ] From 2f4a3db77803b2b0018639aab2930beb525a8ad2 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 24 Sep 2021 19:23:22 -0300 Subject: [PATCH 13/48] linters Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index c7cd8decf..8bba26fe6 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -13,11 +13,9 @@ # limitations under the License. import asyncio +from collections.abc import Sequence import functools import inspect -import itertools -from collections.abc import Sequence -import warnings from _pytest.outcomes import fail from _pytest.outcomes import skip @@ -46,6 +44,7 @@ def transfer_markers(*args, **kwargs): # noqa class LaunchTestWarning(pytest.PytestWarning): """Raised in this plugin to warn users.""" + pass @@ -279,6 +278,7 @@ def is_same_launch_test_fixture(left_item, right_item): if lfn._pytestfixturefunction.scope == 'function': return False name = lfn.__name__ + def get_fixture_params(item): if getattr(item, 'callspec', None) is None: return None @@ -327,9 +327,7 @@ def enumerate_reversed(sequence): def pytest_pyfunc_call(pyfuncitem): """Run launch_testing test coroutines and functions in an event loop.""" if is_launch_test(pyfuncitem): - args = {} func = pyfuncitem.obj - spec = inspect.getfullargspec(func) shutdown_test = is_shutdown_test(pyfuncitem) fixture = get_launch_test_fixture(pyfuncitem) scope = fixture._pytestfixturefunction.scope From eefbcd49687b96b7753d89d9db1b4f762cfd82ad Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 28 Sep 2021 16:32:18 -0300 Subject: [PATCH 14/48] Fix issue when decorator does not have any arguments Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/fixture.py | 6 ++++-- .../test/launch_testing/new_hooks/test_function_scope.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index 321db15b2..103e96550 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -65,7 +65,7 @@ def event_loop(): return event_loop -def fixture(*args, **kwargs): +def fixture(decorated = None, *args, **kwargs): """ Decorate launch_test fixtures. @@ -98,4 +98,6 @@ def fixture(*args, **kwargs): def decorator(fixture_function): fixture_function._launch_pytest_fixture = True return pytest.fixture(fixture_function, *args, **kwargs) - return decorator + if decorated is None: + return decorator + return decorator(decorated) diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py index 0619b92c8..5976187ad 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py @@ -23,7 +23,7 @@ def order(): return [] -@launch_testing.pytest.fixture() +@launch_testing.pytest.fixture def launch_description(): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), From 14148a87c05e894382028737aedc708b98e9ad9f Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 28 Sep 2021 16:43:12 -0300 Subject: [PATCH 15/48] Generators or async generators post shotdown items are not allowed Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 78 +++++++++++-------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 8bba26fe6..377eb52d4 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -326,40 +326,50 @@ def enumerate_reversed(sequence): @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_pyfunc_call(pyfuncitem): """Run launch_testing test coroutines and functions in an event loop.""" - if is_launch_test(pyfuncitem): - func = pyfuncitem.obj - shutdown_test = is_shutdown_test(pyfuncitem) - fixture = get_launch_test_fixture(pyfuncitem) - scope = fixture._pytestfixturefunction.scope - event_loop = pyfuncitem.funcargs['event_loop'] - ls = pyfuncitem.funcargs['launch_service'] - on_shutdown = functools.partial( - finalize_launch_service, ls, eprefix=f'When running test {func.__name__}') - before_test = on_shutdown if shutdown_test else None - if inspect.iscoroutinefunction(func): - pyfuncitem.obj = wrap_coroutine(func, event_loop, before_test) - elif inspect.isgeneratorfunction(func): - if scope != 'function': - shutdown_item = pyfuncitem._launch_testing_shutdown_item - pyfuncitem.obj, shutdown_item.obj = ( - wrap_generator(func, event_loop, on_shutdown) - ) - shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( - shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) - else: - pyfuncitem.obj = wrap_generator_fscope(func, event_loop, on_shutdown) - elif inspect.isasyncgenfunction(func): - if scope != 'function': - shutdown_item = pyfuncitem._launch_testing_shutdown_item - pyfuncitem.obj, shutdown_item.obj = ( - wrap_asyncgen(func, event_loop, on_shutdown) - ) - shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( - shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) - else: - pyfuncitem.obj = wrap_asyncgen_fscope(func, event_loop, on_shutdown) - elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): - pyfuncitem.obj = wrap_func(func, event_loop, before_test) + if not is_launch_test(pyfuncitem): + yield + return + + func = pyfuncitem.obj + if has_shutdown_kwarg(pyfuncitem) and need_shutdown_test_item(func): + skip( + 'generator or asyncgenerator based launch test items cannot be marked with' + ' shutdown=True' + ) + yield + return + shutdown_test = is_shutdown_test(pyfuncitem) + fixture = get_launch_test_fixture(pyfuncitem) + scope = fixture._pytestfixturefunction.scope + event_loop = pyfuncitem.funcargs['event_loop'] + ls = pyfuncitem.funcargs['launch_service'] + on_shutdown = functools.partial( + finalize_launch_service, ls, eprefix=f'When running test {func.__name__}') + before_test = on_shutdown if shutdown_test else None + if inspect.iscoroutinefunction(func): + pyfuncitem.obj = wrap_coroutine(func, event_loop, before_test) + elif inspect.isgeneratorfunction(func): + if scope != 'function': + shutdown_item = pyfuncitem._launch_testing_shutdown_item + pyfuncitem.obj, shutdown_item.obj = ( + wrap_generator(func, event_loop, on_shutdown) + ) + shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( + shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) + else: + pyfuncitem.obj = wrap_generator_fscope(func, event_loop, on_shutdown) + elif inspect.isasyncgenfunction(func): + if scope != 'function': + shutdown_item = pyfuncitem._launch_testing_shutdown_item + pyfuncitem.obj, shutdown_item.obj = ( + wrap_asyncgen(func, event_loop, on_shutdown) + ) + shutdown_item._fixtureinfo = shutdown_item.session._fixturemanager.getfixtureinfo( + shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) + else: + pyfuncitem.obj = wrap_asyncgen_fscope(func, event_loop, on_shutdown) + elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): + pyfuncitem.obj = wrap_func(func, event_loop, before_test) yield From b47fc15f36720007bbd9f0832b2c89d8b663c4e5 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 28 Sep 2021 19:17:03 -0300 Subject: [PATCH 16/48] Add auto_shutdown, shutdown_when_idle options Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/fixture.py | 27 ++++++++++++++++--- .../launch_testing/pytest/plugin.py | 13 ++++++--- 2 files changed, 32 insertions(+), 8 deletions(-) diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index 103e96550..691568deb 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -27,8 +27,9 @@ def scope_gt(scope1, scope2): from _pytest.fixtures import scopemismatch as scope_gt -def finalize_launch_service(launch_service, eprefix=''): - launch_service.shutdown(force_sync=True) +def finalize_launch_service(launch_service, eprefix='', auto_shutdown=True): + if auto_shutdown: + launch_service.shutdown(force_sync=True) loop = launch_service.event_loop if loop is not None and not loop.is_closed(): rc = loop.run_until_complete(launch_service.task) @@ -65,12 +66,26 @@ def event_loop(): return event_loop -def fixture(decorated = None, *args, **kwargs): +def fixture( + decorated = None, + *args, + shutdown_when_idle = True, + auto_shutdown = True, + **kwargs +): """ Decorate launch_test fixtures. - For documentation on the supported arguments, see + See also https://docs.pytest.org/en/latest/reference/reference.html#pytest-fixture. + + :param decorated: object to be decorated. + :param \*args: extra posicional arguments to be passed to pytest.fixture(). + :param shutdown_when_idle: when true, the launch service will shutdown when idle. + :param auto_shutdown: when true, the launch service will be shutdown automatically + after all pre-shutdown tests get run. If false, shutdown needs to be signaled in a + different way or the launch fixture should be self terminating. + :param \**kwargs: extra keyword arguments to be passed to pytest.fixture(). """ # Automagically override the event_loop and launch_testing fixtures # with a fixture of the correct scope. @@ -97,6 +112,10 @@ def fixture(decorated = None, *args, **kwargs): def decorator(fixture_function): fixture_function._launch_pytest_fixture = True + fixture_function._launch_pytest_fixture_options = { + 'shutdown_when_idle': shutdown_when_idle, + 'auto_shutdown': auto_shutdown, + } return pytest.fixture(fixture_function, *args, **kwargs) if decorated is None: return decorator diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 377eb52d4..42fdfa73c 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -96,6 +96,7 @@ def get_ready_to_test_action(launch_description): def pytest_fixture_setup(fixturedef, request): """Set up launch service for all launch_pytest fixtures.""" if getattr(fixturedef.func, '_launch_pytest_fixture', False): + options = fixturedef.func._launch_pytest_fixture_options eprefix = f"When running launch_pytest fixture '{fixturedef.func.__name__}':" ls = request.getfixturevalue('launch_service') event_loop = request.getfixturevalue('event_loop') @@ -114,14 +115,14 @@ def pytest_fixture_setup(fixturedef, request): ls.include_launch_description(ld) run_async_task = event_loop.create_task(ls.run_async( # TODO(ivanpauno): maybe this could be configurable (?) - shutdown_when_idle=True + shutdown_when_idle=options['shutdown_when_idle'] )) ready = get_ready_to_test_action(ld) asyncio.set_event_loop(event_loop) event = asyncio.Event() ready._add_callback(lambda: event.set()) - - fixturedef.addfinalizer(functools.partial(finalize_launch_service, ls, eprefix=eprefix)) + fixturedef.addfinalizer(functools.partial( + finalize_launch_service, ls, eprefix=eprefix, auto_shutdown=options['auto_shutdown'])) run_until_complete(event_loop, event.wait()) # this is guaranteed by the current run_async() implementation, let's check it just in case # it changes in the future @@ -343,8 +344,12 @@ def pytest_pyfunc_call(pyfuncitem): scope = fixture._pytestfixturefunction.scope event_loop = pyfuncitem.funcargs['event_loop'] ls = pyfuncitem.funcargs['launch_service'] + auto_shutdown = fixture._launch_pytest_fixture_options['auto_shutdown'] on_shutdown = functools.partial( - finalize_launch_service, ls, eprefix=f'When running test {func.__name__}') + finalize_launch_service, + ls, + eprefix=f'When running test {func.__name__}', + auto_shutdown=auto_shutdown) before_test = on_shutdown if shutdown_test else None if inspect.iscoroutinefunction(func): pyfuncitem.obj = wrap_coroutine(func, event_loop, before_test) From 5720715fe72aa209b5b53d902d07c8a4a373ea6b Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 16:42:16 -0300 Subject: [PATCH 17/48] Simple tools to interact with processes, needs more work Signed-off-by: Ivan Santiago Paunovic --- launch/launch/actions/execute_local.py | 131 ++++++++++++------ launch/launch/actions/execute_process.py | 2 + launch/launch/launch_context.py | 14 +- .../launch_testing/pytest/fixture.py | 16 ++- .../launch_testing/pytest/plugin.py | 6 +- .../launch_testing/tools/pytest_process.py | 117 ++++++++++++++++ .../examples/pytest_hello_world.py | 58 ++++++++ 7 files changed, 297 insertions(+), 47 deletions(-) create mode 100644 launch_testing/launch_testing/tools/pytest_process.py create mode 100644 launch_testing/test/launch_testing/examples/pytest_hello_world.py diff --git a/launch/launch/actions/execute_local.py b/launch/launch/actions/execute_local.py index 676717cfb..6a025b202 100644 --- a/launch/launch/actions/execute_local.py +++ b/launch/launch/actions/execute_local.py @@ -16,6 +16,7 @@ import asyncio import io +import logging import os import platform import signal @@ -91,6 +92,7 @@ def __init__( emulate_tty: bool = False, output: Text = 'log', output_format: Text = '[{this.process_description.final_name}] {line}', + cached_output: bool = False, log_cmd: bool = False, on_exit: Optional[Union[ SomeActionsType, @@ -176,6 +178,8 @@ def __init__( :param: log_cmd if True, prints the final cmd before executing the process, which is useful for debugging when substitutions are involved. + :param: cached_output if `True`, both stdout and stderr will be cached. + Use get_stdout() and get_stderr() to read the buffered output. :param: on_exit list of actions to execute upon process exit. :param: respawn if 'True', relaunch the process that abnormally died. Defaults to 'False'. @@ -191,6 +195,7 @@ def __init__( self.__output_format = output_format self.__log_cmd = log_cmd + self.__cached_output = cached_output self.__on_exit = on_exit self.__respawn = respawn self.__respawn_delay = respawn_delay @@ -329,59 +334,32 @@ def __on_process_stdin( cast(ProcessStdin, event) return None - def __on_process_stdout( - self, event: ProcessIO + def __on_process_output( + self, event: ProcessIO, buffer: io.TextIOBase, logger: logging.Logger ) -> Optional[SomeActionsType]: to_write = event.text.decode(errors='replace') - if self.__stdout_buffer.closed: - # __stdout_buffer was probably closed by __flush_buffers on shutdown. Output without + if buffer.closed: + # buffer was probably closed by __flush_buffers on shutdown. Output without # buffering. - self.__stdout_logger.info( + buffer.info( self.__output_format.format(line=to_write, this=self) ) else: - self.__stdout_buffer.write(to_write) - self.__stdout_buffer.seek(0) + buffer.write(to_write) + buffer.seek(0) last_line = None - for line in self.__stdout_buffer: + for line in buffer: if line.endswith(os.linesep): - self.__stdout_logger.info( + logger.info( self.__output_format.format(line=line[:-len(os.linesep)], this=self) ) else: last_line = line break - self.__stdout_buffer.seek(0) - self.__stdout_buffer.truncate(0) + buffer.seek(0) + buffer.truncate(0) if last_line is not None: - self.__stdout_buffer.write(last_line) - - def __on_process_stderr( - self, event: ProcessIO - ) -> Optional[SomeActionsType]: - to_write = event.text.decode(errors='replace') - if self.__stderr_buffer.closed: - # __stderr buffer was probably closed by __flush_buffers on shutdown. Output without - # buffering. - self.__stderr_logger.info( - self.__output_format.format(line=to_write, this=self) - ) - else: - self.__stderr_buffer.write(to_write) - self.__stderr_buffer.seek(0) - last_line = None - for line in self.__stderr_buffer: - if line.endswith(os.linesep): - self.__stderr_logger.info( - self.__output_format.format(line=line[:-len(os.linesep)], this=self) - ) - else: - last_line = line - break - self.__stderr_buffer.seek(0) - self.__stderr_buffer.truncate(0) - if last_line is not None: - self.__stderr_buffer.write(last_line) + buffer.write(last_line) def __flush_buffers(self, event, context): line = self.__stdout_buffer.getvalue() @@ -407,6 +385,35 @@ def __flush_buffers(self, event, context): self.__stderr_buffer.seek(0) self.__stderr_buffer.truncate(0) + def __on_process_output_cached( + self, event: ProcessIO, buffer, logger + ) -> Optional[SomeActionsType]: + to_write = event.text.decode(errors='replace') + last_cursor = buffer.tell() + self.__stdout_buffer.seek(0, 2) # go to end of buffer + buffer.write(to_write) + buffer.seek(last_cursor) + new_cursor = last_cursor + for line in buffer: + if not line.endswith(os.linesep): + break + new_cursor = buffer.tell() + logger.info( + self.__output_format.format(line=line[:-len(os.linesep)], this=self) + ) + buffer.seek(new_cursor) + + def __flush_cached_buffers(self, event, context): + for line in self.__stdout_buffer: + self.__stdout_buffer.info( + self.__output_format.format(line=line, this=self) + ) + + for line in self.__stderr_buffer: + self.__stderr_logger.info( + self.__output_format.format(line=line, this=self) + ) + def __on_shutdown(self, event: Event, context: LaunchContext) -> Optional[SomeActionsType]: due_to_sigint = cast(Shutdown, event).due_to_sigint return self._shutdown_process( @@ -614,6 +621,13 @@ def execute(self, context: LaunchContext) -> Optional[List[LaunchDescriptionEnti # If shutdown starts before execution can start, don't start execution. return None + if self.__cached_output: + on_output_method = self.__on_process_output_cached + flush_buffers_method = self.__flush_cached_buffers + else: + on_output_method = self.__on_process_output + flush_buffers_method = self.__flush_buffers + event_handlers = [ EventHandler( matcher=lambda event: is_a_subclass(event, ShutdownProcess), @@ -626,8 +640,10 @@ def execute(self, context: LaunchContext) -> Optional[List[LaunchDescriptionEnti OnProcessIO( target_action=self, on_stdin=self.__on_process_stdin, - on_stdout=self.__on_process_stdout, - on_stderr=self.__on_process_stderr + on_stdout=lambda event: on_output_method( + event, self.__stdout_buffer, self.__stdout_logger), + on_stderr=lambda event: on_output_method( + event, self.__stderr_buffer, self.__stderr_logger), ), OnShutdown( on_shutdown=self.__on_shutdown, @@ -638,7 +654,7 @@ def execute(self, context: LaunchContext) -> Optional[List[LaunchDescriptionEnti ), OnProcessExit( target_action=self, - on_exit=self.__flush_buffers, + on_exit=flush_buffers_method, ), ] for event_handler in event_handlers: @@ -660,3 +676,34 @@ def execute(self, context: LaunchContext) -> Optional[List[LaunchDescriptionEnti def get_asyncio_future(self) -> Optional[asyncio.Future]: """Return an asyncio Future, used to let the launch system know when we're done.""" return self.__completed_future + + def get_stdout(self): + """ + Get cached stdout. + + :raises RuntimeError: if cached_output is false. + """ + if not self.__cached_output: + raise RuntimeError( + 'cached output must be true to be able to get stdout,' + f" proc '{self.__process_description.name}'") + return self.__stdout_buffer.getvalue() + + def get_stderr(self): + """ + Get cached stdout. + + :raises RuntimeError: if cached_output is false. + """ + if not self.__cached_output: + raise RuntimeError( + 'cached output must be true to be able to get stderr, proc' + f" '{self.__process_description.name}'") + return self.__stderr_buffer.getvalue() + + @property + def return_code(self): + """Get the process return code, None if it hasn't finished.""" + if self._subprocess_transport is None: + return None + return self._subprocess_transport.get_returncode() diff --git a/launch/launch/actions/execute_process.py b/launch/launch/actions/execute_process.py index 3acef66c5..1b373a41f 100644 --- a/launch/launch/actions/execute_process.py +++ b/launch/launch/actions/execute_process.py @@ -228,6 +228,8 @@ def __init__( :param: log_cmd if True, prints the final cmd before executing the process, which is useful for debugging when substitutions are involved. + :param: cached_output if `True`, both stdout and stderr will be cached. + Use get_stdout() and get_stderr() to read the buffered output. :param: on_exit list of actions to execute upon process exit. :param: respawn if 'True', relaunch the process that abnormally died. Defaults to 'False'. diff --git a/launch/launch/launch_context.py b/launch/launch/launch_context.py index 1cb44a02d..5264cb94b 100644 --- a/launch/launch/launch_context.py +++ b/launch/launch/launch_context.py @@ -174,9 +174,17 @@ def would_handle_event(self, event: Event) -> bool: """Check whether an event would be handled or not.""" return any(handler.matches(event) for handler in self._event_handlers) - def register_event_handler(self, event_handler: BaseEventHandler) -> None: - """Register a event handler.""" - self._event_handlers.appendleft(event_handler) + def register_event_handler(self, event_handler: BaseEventHandler, append = False) -> None: + """ + Register a event handler. + + :param append: if 'true', the new event handler will be executed after the previously + registered ones. If not, it will prepend the old handlers. + """ + if append: + self._event_handlers.append(event_handler) + else: + self._event_handlers.appendleft(event_handler) def unregister_event_handler(self, event_handler: BaseEventHandler) -> None: """Unregister an event handler.""" diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index 691568deb..e07072529 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -51,6 +51,19 @@ def launch_service(event_loop): return launch_service +def get_launch_context_fixture(*, scope='function', overridable=True): + """Return a launch service fixture.""" + + @pytest.fixture(scope=scope) + def launch_context(launch_service): + """Create an instance of the launch service for each test case.""" + return launch_service.context + if overridable: + launch_context._launch_testing_overridable_fixture = True + launch_context._launch_testing_fixture_scope = scope + return launch_context + + def get_event_loop_fixture(*, scope='function', overridable=True): """Return an event loop fixture.""" @@ -98,7 +111,8 @@ def fixture( mod_locals = vars(mod) for name, getter in ( ('launch_service', get_launch_service_fixture), - ('event_loop', get_event_loop_fixture) + ('event_loop', get_event_loop_fixture), + ('launch_context', get_launch_context_fixture), ): if name in mod_locals: obj = mod_locals[name] diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 42fdfa73c..9f8348d6c 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -27,6 +27,7 @@ from .fixture import finalize_launch_service from .fixture import get_launch_service_fixture +from .fixture import get_launch_context_fixture """ launch_testing native pytest based implementation. @@ -524,5 +525,8 @@ def inner(**kwargs): return inner -"""Launch service fixture.""" launch_service = get_launch_service_fixture(overridable=False) +"""Launch service fixture.""" + +launch_context = get_launch_context_fixture(overridable=False) +"""Launch context fixture.""" diff --git a/launch_testing/launch_testing/tools/pytest_process.py b/launch_testing/launch_testing/tools/pytest_process.py new file mode 100644 index 000000000..efe648f84 --- /dev/null +++ b/launch_testing/launch_testing/tools/pytest_process.py @@ -0,0 +1,117 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import contextlib +import threading +import time + +import launch +from launch import event_handlers + + +@contextlib.contextmanager +def register_event_handler(context, event_handler): + # Code to acquire resource, e.g.: + try: + yield context.register_event_handler(event_handler, append=True) + finally: + context.unregister_event_handler(event_handler) + +def _get_on_process_start(execute_process_action, pyevent): + event_handlers.OnProcessStart( + target_action=execute_process_action, on_start=lambda _1, _2: pyevent.set()) + +async def _wait_for_event( + launch_context, execute_process_action, get_launch_event_handler, timeout=None +): + pyevent = asyncio.Event() + event_handler = get_launch_event_handler(execute_process_action, pyevent) + with register_event_handler(launch_context, event_handler): + await asyncio.wait_for(pyevent.wait(), timeout) + +async def _wait_for_event_with_condition( + launch_context, execute_process_action, get_launch_event_handler, condition, timeout=None +): + pyevent = asyncio.Event() + event_handler = get_launch_event_handler(execute_process_action, pyevent) + cond_value = condition() + with register_event_handler(launch_context, event_handler): + start = time.time() + now = start + while not cond_value and (timeout is None or now < start + timeout): + await asyncio.wait_for(pyevent.wait(), start - now + timeout) + pyevent.clear() + cond_value = condition() + now = time.time() + return cond_value + +def _wait_for_event_sync( + launch_context, execute_process_action, get_launch_event_handler, timeout=None +): + pyevent = threading.Event() + event_handler = get_launch_event_handler(execute_process_action, pyevent) + with register_event_handler(launch_context, event_handler): + pyevent.wait(timeout) + +def _wait_for_event_with_condition_sync( + launch_context, execute_process_action, get_launch_event_handler, condition, timeout=None +): + pyevent = threading.Event() + event_handler = get_launch_event_handler(execute_process_action, pyevent) + cond_value = condition() + with register_event_handler(launch_context, event_handler): + start = time.time() + now = start + while not cond_value and (timeout is None or now < start + timeout): + pyevent.wait(start - now + timeout) + pyevent.clear() + cond_value = condition() + now = time.time() + return cond_value + +def _get_stdout_event_handler(action, pyevent): + return event_handlers.OnProcessIO( + target_action=action, on_stdout=lambda _1: pyevent.set()) + +async def wait_for_output( + launch_context, execute_process_action, validate_output, timeout=None +): + def condition(): + try: + return validate_output(execute_process_action.get_stdout()) + except AssertionError: + return False + success = await _wait_for_event_with_condition( + launch_context, execute_process_action, _get_stdout_event_handler, condition, timeout) + if not success: + # Validate the output again, this time not catching assertion errors. + # This allows the user to use asserts directly, errors will be nicely rendeded by pytest. + return validate_output(execute_process_action.get_stdout()) + + +def wait_for_output_sync( + launch_context, execute_process_action, validate_output, timeout=None +): + def condition(): + try: + return validate_output(execute_process_action.get_stdout()) + except AssertionError: + return False + success = _wait_for_event_with_condition_sync( + launch_context, execute_process_action, _get_stdout_event_handler, condition, timeout) + if not success: + # Validate the output again, this time not catching assertion errors. + # This allows the user to use asserts directly, errors will be nicely rendeded by pytest. + return validate_output(execute_process_action.get_stdout()) in (None, True) diff --git a/launch_testing/test/launch_testing/examples/pytest_hello_world.py b/launch_testing/test/launch_testing/examples/pytest_hello_world.py new file mode 100644 index 000000000..97e3e0389 --- /dev/null +++ b/launch_testing/test/launch_testing/examples/pytest_hello_world.py @@ -0,0 +1,58 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import launch +import launch.actions +import launch_testing.actions +import launch_testing.markers +from launch_testing.tools import pytest_process as tools +import pytest + + +@pytest.fixture +def hello_world_proc(): + # Launch a process to test + return launch.actions.ExecuteProcess( + cmd=['echo', 'hello_world'], + shell=True, + cached_output=True, + ) + +# This function specifies the processes to be run for our test. +@launch_testing.pytest.fixture +def launch_description(hello_world_proc): + """Launch a simple process to print 'hello_world'.""" + return launch.LaunchDescription([ + hello_world_proc, + # Tell launch when to start the test + # If no ReadyToTest action is added, one will be appended automatically. + launch_testing.actions.ReadyToTest() + ]) + + +@pytest.mark.launch_testing(fixture=launch_description) +def test_read_stdout(hello_world_proc, launch_context): + """Check if 'hello_world' was found in the stdout.""" + def validate_output(output): + # this function can use assertions to validate the output or return a boolean. + # pytest generates easier to understand failures when assertions are used. + assert output == 'hello_world\n', 'process never printed hello_world' + assert tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=5) + def validate_output(output): + return output == 'this will never happen' + assert not tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=0.1) + yield + # this is executed after launch service shutdown + assert hello_world_proc.return_code == 0 From 1debb55f3a5ee7325f231ec4954a0747a08e90dd Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:02:56 -0300 Subject: [PATCH 18/48] Apply reviewer suggestion Signed-off-by: Ivan Santiago Paunovic Co-authored-by: Michel Hidalgo --- launch_testing/launch_testing/pytest/fixture.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index e07072529..8117b61f1 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -93,7 +93,7 @@ def fixture( https://docs.pytest.org/en/latest/reference/reference.html#pytest-fixture. :param decorated: object to be decorated. - :param \*args: extra posicional arguments to be passed to pytest.fixture(). + :param \*args: extra positional arguments to be passed to `pytest.fixture()` :param shutdown_when_idle: when true, the launch service will shutdown when idle. :param auto_shutdown: when true, the launch service will be shutdown automatically after all pre-shutdown tests get run. If false, shutdown needs to be signaled in a From 4a667e48cdc7fa56a09d9b20167064a635d5f413 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:09:00 -0300 Subject: [PATCH 19/48] Correct overindented code Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 9f8348d6c..f83a14cda 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -150,8 +150,8 @@ def is_launch_test_mark_valid(item): ret = 'fixture' in kwargs and kwargs['fixture'] is not None if not ret: msg = ( - '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' - f'decorator') + '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' + f'decorator') item.warn(LaunchTestWarning(msg)) item.add_marker(pytest.mark.skip(msg)) return ret From 00c6691de1ebe7f9a4b5bff559779f0c68bdfd71 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:10:39 -0300 Subject: [PATCH 20/48] Drop debug prints Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index f83a14cda..54d6b7459 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -319,10 +319,8 @@ def enumerate_reversed(sequence): and is_shutdown_test(item) and not is_shutdown_test(other) ): new_position = i + 1 + j - print(f'moving {item.name} after {other.name}: {i} {i+j+1}') if new_position is not None: items.insert(new_position, items.pop(i)) - print([item.name for item in items]) @pytest.hookimpl(hookwrapper=True, tryfirst=True) From 66931b5437f1b9dfeb694b76ad32ec279b8e6361 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:16:35 -0300 Subject: [PATCH 21/48] Delete unused helper function Signed-off-by: Ivan Santiago Paunovic --- .../launch_testing/pytest/plugin.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 54d6b7459..b9dd319e1 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -245,25 +245,6 @@ def pytest_pycollect_makeitem(collector, name, obj): return items -def get_error_context_from_obj(obj): - """Return formatted information of the object location.""" - try: - fspath = inspect.getsourcefile(obj) - except TypeError: - return 'location information of the object not found' - try: - lines, lineno = inspect.getsourcelines(obj) - except IOError: - return f'file {fspath}: source code not available' - error_msg = f'file {fspath}, line {lineno}' - for line in lines: - line = line.rstrip() - error_msg += f'\n {line}' - if line.lstrip().startswith('def'): - break - return error_msg - - def is_shutdown_test(item): """Return `True` if the item is a launch test.""" return getattr(item, '_launch_testing_is_shutdown', False) From dc2529c2f203054f983af1e0a1724778114139aa Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:18:11 -0300 Subject: [PATCH 22/48] Delete outdated comments Signed-off-by: Ivan Santiago Paunovic --- .../test/launch_testing/new_hooks/test_function_scope.py | 5 ----- .../test/launch_testing/new_hooks/test_module_scope.py | 4 ---- 2 files changed, 9 deletions(-) diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py index 5976187ad..38bcde9b2 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_function_scope.py @@ -31,11 +31,6 @@ def launch_description(): ]) -# TODO(ivanpauno) -# We cannot get variables from the dictionary returned by launch_description -# because we're removing the fixture before running the tests. -# Maybe we can delete this feature, and use generators/asyncgens. -# We can always get the variables returned by the launch_testing fixture there. @pytest.mark.launch_testing(fixture=launch_description, shutdown=True) async def test_after_shutdown(order, launch_service): order.append('test_after_shutdown') diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py index 254276f07..e7d92a894 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_testing/test/launch_testing/new_hooks/test_module_scope.py @@ -33,10 +33,6 @@ def launch_description(request): ]), request.param -# TODO(ivanpauno) -# We cannot get variables from the dictionary returned by launch_description -# because we're removing the fixture before running the tests. -# Maybe we can delete this feature, and use generators/asyncgens. @pytest.mark.launch_testing(fixture=launch_description, shutdown=True) def test_after_shutdown(order, launch_service, launch_description): param = launch_description[1] From c73b28d98d610c98d0f7cb96e778d32e840cf9a2 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 17:20:05 -0300 Subject: [PATCH 23/48] != -> is not Signed-off-by: Ivan Santiago Paunovic --- launch_testing/launch_testing/pytest/plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index b9dd319e1..80d056a2d 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -256,7 +256,7 @@ def is_same_launch_test_fixture(left_item, right_item): rfn = get_launch_test_fixture(right_item) if None in (lfn, rfn): return False - if lfn != rfn: + if lfn is not rfn: return False if lfn._pytestfixturefunction.scope == 'function': return False From a0a14e6b79b922e5feb7997b672b4320e4768011 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Thu, 14 Oct 2021 18:20:34 -0300 Subject: [PATCH 24/48] Please linters Signed-off-by: Ivan Santiago Paunovic --- launch/launch/launch_context.py | 4 ++-- launch_testing/launch_testing/pytest/fixture.py | 11 +++++------ launch_testing/launch_testing/pytest/plugin.py | 2 +- launch_testing/launch_testing/tools/pytest_process.py | 8 +++++++- .../launch_testing/examples/pytest_hello_world.py | 4 +++- 5 files changed, 18 insertions(+), 11 deletions(-) diff --git a/launch/launch/launch_context.py b/launch/launch/launch_context.py index 5264cb94b..f06cd8110 100644 --- a/launch/launch/launch_context.py +++ b/launch/launch/launch_context.py @@ -174,10 +174,10 @@ def would_handle_event(self, event: Event) -> bool: """Check whether an event would be handled or not.""" return any(handler.matches(event) for handler in self._event_handlers) - def register_event_handler(self, event_handler: BaseEventHandler, append = False) -> None: + def register_event_handler(self, event_handler: BaseEventHandler, append=False) -> None: """ Register a event handler. - + :param append: if 'true', the new event handler will be executed after the previously registered ones. If not, it will prepend the old handlers. """ diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_testing/launch_testing/pytest/fixture.py index 8117b61f1..5365fa974 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_testing/launch_testing/pytest/fixture.py @@ -80,17 +80,16 @@ def event_loop(): def fixture( - decorated = None, + decorated=None, *args, - shutdown_when_idle = True, - auto_shutdown = True, + shutdown_when_idle=True, + auto_shutdown=True, **kwargs ): - """ + r""" Decorate launch_test fixtures. - See also - https://docs.pytest.org/en/latest/reference/reference.html#pytest-fixture. + See also https://docs.pytest.org/en/latest/reference/reference.html#pytest-fixture. :param decorated: object to be decorated. :param \*args: extra positional arguments to be passed to `pytest.fixture()` diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_testing/launch_testing/pytest/plugin.py index 80d056a2d..17cde2ffd 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_testing/launch_testing/pytest/plugin.py @@ -26,8 +26,8 @@ import pytest from .fixture import finalize_launch_service -from .fixture import get_launch_service_fixture from .fixture import get_launch_context_fixture +from .fixture import get_launch_service_fixture """ launch_testing native pytest based implementation. diff --git a/launch_testing/launch_testing/tools/pytest_process.py b/launch_testing/launch_testing/tools/pytest_process.py index efe648f84..fb50e71c6 100644 --- a/launch_testing/launch_testing/tools/pytest_process.py +++ b/launch_testing/launch_testing/tools/pytest_process.py @@ -17,7 +17,6 @@ import threading import time -import launch from launch import event_handlers @@ -29,10 +28,12 @@ def register_event_handler(context, event_handler): finally: context.unregister_event_handler(event_handler) + def _get_on_process_start(execute_process_action, pyevent): event_handlers.OnProcessStart( target_action=execute_process_action, on_start=lambda _1, _2: pyevent.set()) + async def _wait_for_event( launch_context, execute_process_action, get_launch_event_handler, timeout=None ): @@ -41,6 +42,7 @@ async def _wait_for_event( with register_event_handler(launch_context, event_handler): await asyncio.wait_for(pyevent.wait(), timeout) + async def _wait_for_event_with_condition( launch_context, execute_process_action, get_launch_event_handler, condition, timeout=None ): @@ -57,6 +59,7 @@ async def _wait_for_event_with_condition( now = time.time() return cond_value + def _wait_for_event_sync( launch_context, execute_process_action, get_launch_event_handler, timeout=None ): @@ -65,6 +68,7 @@ def _wait_for_event_sync( with register_event_handler(launch_context, event_handler): pyevent.wait(timeout) + def _wait_for_event_with_condition_sync( launch_context, execute_process_action, get_launch_event_handler, condition, timeout=None ): @@ -81,10 +85,12 @@ def _wait_for_event_with_condition_sync( now = time.time() return cond_value + def _get_stdout_event_handler(action, pyevent): return event_handlers.OnProcessIO( target_action=action, on_stdout=lambda _1: pyevent.set()) + async def wait_for_output( launch_context, execute_process_action, validate_output, timeout=None ): diff --git a/launch_testing/test/launch_testing/examples/pytest_hello_world.py b/launch_testing/test/launch_testing/examples/pytest_hello_world.py index 97e3e0389..57c29cab1 100644 --- a/launch_testing/test/launch_testing/examples/pytest_hello_world.py +++ b/launch_testing/test/launch_testing/examples/pytest_hello_world.py @@ -50,9 +50,11 @@ def validate_output(output): # pytest generates easier to understand failures when assertions are used. assert output == 'hello_world\n', 'process never printed hello_world' assert tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=5) + def validate_output(output): return output == 'this will never happen' - assert not tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=0.1) + assert not tools.wait_for_output_sync( + launch_context, hello_world_proc, validate_output, timeout=0.1) yield # this is executed after launch service shutdown assert hello_world_proc.return_code == 0 From 4819cd6cbf012dadca6d375c13724c530ee59962 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 18 Oct 2021 18:11:25 -0300 Subject: [PATCH 25/48] Split launch_testing from new launch_pytest package Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/__init__.py | 21 +++++++++ .../launch_pytest}/fixture.py | 18 ++++---- .../launch_pytest}/plugin.py | 34 +++++++------- launch_pytest/launch_pytest/tools/__init__.py | 19 ++++++++ .../launch_pytest/tools/process.py | 0 launch_pytest/package.xml | 28 ++++++++++++ .../resource/launch_pytest | 0 launch_pytest/setup.cfg | 4 ++ launch_pytest/setup.py | 44 +++++++++++++++++++ .../test}/examples/pytest_hello_world.py | 18 ++++---- .../test}/test_function_scope.py | 13 +++--- .../test}/test_module_scope.py | 18 +++----- .../launch_testing/pytest/__init__.py | 21 --------- .../pytest/{legacy => }/hooks.py | 6 +-- .../pytest/{legacy => }/hookspecs.py | 0 launch_testing/setup.py | 3 +- 16 files changed, 170 insertions(+), 77 deletions(-) create mode 100644 launch_pytest/launch_pytest/__init__.py rename {launch_testing/launch_testing/pytest => launch_pytest/launch_pytest}/fixture.py (87%) rename {launch_testing/launch_testing/pytest => launch_pytest/launch_pytest}/plugin.py (94%) create mode 100644 launch_pytest/launch_pytest/tools/__init__.py rename launch_testing/launch_testing/tools/pytest_process.py => launch_pytest/launch_pytest/tools/process.py (100%) create mode 100644 launch_pytest/package.xml rename launch_testing/launch_testing/pytest/legacy/__init__.py => launch_pytest/resource/launch_pytest (100%) create mode 100644 launch_pytest/setup.cfg create mode 100644 launch_pytest/setup.py rename {launch_testing/test/launch_testing => launch_pytest/test}/examples/pytest_hello_world.py (83%) rename {launch_testing/test/launch_testing/new_hooks => launch_pytest/test}/test_function_scope.py (84%) rename {launch_testing/test/launch_testing/new_hooks => launch_pytest/test}/test_module_scope.py (84%) rename launch_testing/launch_testing/pytest/{legacy => }/hooks.py (97%) rename launch_testing/launch_testing/pytest/{legacy => }/hookspecs.py (100%) diff --git a/launch_pytest/launch_pytest/__init__.py b/launch_pytest/launch_pytest/__init__.py new file mode 100644 index 000000000..04441b9b1 --- /dev/null +++ b/launch_pytest/launch_pytest/__init__.py @@ -0,0 +1,21 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import tools +from .fixture import fixture + +__all__ = [ + 'fixture', + 'tools', +] diff --git a/launch_testing/launch_testing/pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py similarity index 87% rename from launch_testing/launch_testing/pytest/fixture.py rename to launch_pytest/launch_pytest/fixture.py index 5365fa974..4c2a4d4d0 100644 --- a/launch_testing/launch_testing/pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -46,8 +46,8 @@ def launch_service(event_loop): yield ls finalize_launch_service(ls, eprefix='When tearing down launch_service fixture') if overridable: - launch_service._launch_testing_overridable_fixture = True - launch_service._launch_testing_fixture_scope = scope + launch_service._launch_pytest_overridable_fixture = True + launch_service._launch_pytest_fixture_scope = scope return launch_service @@ -59,8 +59,8 @@ def launch_context(launch_service): """Create an instance of the launch service for each test case.""" return launch_service.context if overridable: - launch_context._launch_testing_overridable_fixture = True - launch_context._launch_testing_fixture_scope = scope + launch_context._launch_pytest_overridable_fixture = True + launch_context._launch_pytest_fixture_scope = scope return launch_context @@ -74,8 +74,8 @@ def event_loop(): yield loop loop.close() if overridable: - event_loop._launch_testing_overridable_fixture = True - event_loop._launch_testing_fixture_scope = scope + event_loop._launch_pytest_overridable_fixture = True + event_loop._launch_pytest_fixture_scope = scope return event_loop @@ -99,7 +99,7 @@ def fixture( different way or the launch fixture should be self terminating. :param \**kwargs: extra keyword arguments to be passed to pytest.fixture(). """ - # Automagically override the event_loop and launch_testing fixtures + # Automagically override the event_loop, launch_service and launch_context fixtures # with a fixture of the correct scope. # This is not done if the user explicitly provided this fixture, # they might get an ScopeError if not correctly defined. @@ -116,8 +116,8 @@ def fixture( if name in mod_locals: obj = mod_locals[name] if ( - getattr(obj, '_launch_testing_overridable_fixture', False) and - scope_gt(scope, obj._launch_testing_fixture_scope) + getattr(obj, '_launch_pytest_overridable_fixture', False) and + scope_gt(scope, obj._launch_pytest_fixture_scope) ): mod_locals[name] = getter(scope=scope) else: diff --git a/launch_testing/launch_testing/pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py similarity index 94% rename from launch_testing/launch_testing/pytest/plugin.py rename to launch_pytest/launch_pytest/plugin.py index 17cde2ffd..69d56b677 100644 --- a/launch_testing/launch_testing/pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -30,7 +30,7 @@ from .fixture import get_launch_service_fixture """ -launch_testing native pytest based implementation. +Native pytest plugin for launch based tests. """ @@ -50,10 +50,10 @@ class LaunchTestWarning(pytest.PytestWarning): def pytest_configure(config): - """Inject launch_testing marker documentation.""" + """Inject launch marker documentation.""" config.addinivalue_line( 'markers', - 'launch_testing: ' + 'launch: ' 'mark the test as a launch test, it will be ' 'run using the specified launch_pad.', ) @@ -136,7 +136,7 @@ def pytest_fixture_setup(fixturedef, request): def is_launch_test(item): """Return `True` if the item is a launch test.""" - mark = item.get_closest_marker('launch_testing') + mark = item.get_closest_marker('launch') return mark is not None @@ -146,11 +146,11 @@ def is_launch_test_mark_valid(item): If not, a warning and a skip mark will be added to the item. """ - kwargs = item.get_closest_marker('launch_testing').kwargs + kwargs = item.get_closest_marker('launch').kwargs ret = 'fixture' in kwargs and kwargs['fixture'] is not None if not ret: msg = ( - '"fixture" keyword argument is required in a pytest.mark.launch_testing() ' + '"fixture" keyword argument is required in a pytest.mark.launch() ' f'decorator') item.warn(LaunchTestWarning(msg)) item.add_marker(pytest.mark.skip(msg)) @@ -159,12 +159,12 @@ def is_launch_test_mark_valid(item): def has_shutdown_kwarg(item): """Return `True` if the launch test shutdown kwarg is true.""" - return item.get_closest_marker('launch_testing').kwargs.get('shutdown', False) + return item.get_closest_marker('launch').kwargs.get('shutdown', False) def get_launch_test_fixture(item): """Return the launch test fixture name, `None` if this isn't a launch test.""" - mark = item.get_closest_marker('launch_testing') + mark = item.get_closest_marker('launch') if mark is None: return None try: @@ -205,7 +205,7 @@ def generate_test_items(collector, name, obj, fixturename, is_shutdown, needs_re items = list(collector._genfunctions(name, obj)) for item in items: # Mark shutdown tests correctly - item._launch_testing_is_shutdown = is_shutdown + item._launch_pytest_is_shutdown = is_shutdown if is_shutdown and needs_renaming: # rename the items, to differentiate them from the normal test stage item.name = f'{item.name}[shutdown_test]' @@ -240,14 +240,14 @@ def pytest_pycollect_makeitem(collector, name, obj): # if not we're going to use two event loops!!! shutdown_items = generate_test_items(collector, name, obj, fixturename, True, True) for item, shutdown_item in zip(items, shutdown_items): - item._launch_testing_shutdown_item = shutdown_item + item._launch_pytest_shutdown_item = shutdown_item items.extend(shutdown_items) return items def is_shutdown_test(item): """Return `True` if the item is a launch test.""" - return getattr(item, '_launch_testing_is_shutdown', False) + return getattr(item, '_launch_pytest_is_shutdown', False) def is_same_launch_test_fixture(left_item, right_item): @@ -306,7 +306,7 @@ def enumerate_reversed(sequence): @pytest.hookimpl(hookwrapper=True, tryfirst=True) def pytest_pyfunc_call(pyfuncitem): - """Run launch_testing test coroutines and functions in an event loop.""" + """Run launch_pytest test coroutines and functions in an event loop.""" if not is_launch_test(pyfuncitem): yield return @@ -335,7 +335,7 @@ def pytest_pyfunc_call(pyfuncitem): pyfuncitem.obj = wrap_coroutine(func, event_loop, before_test) elif inspect.isgeneratorfunction(func): if scope != 'function': - shutdown_item = pyfuncitem._launch_testing_shutdown_item + shutdown_item = pyfuncitem._launch_pytest_shutdown_item pyfuncitem.obj, shutdown_item.obj = ( wrap_generator(func, event_loop, on_shutdown) ) @@ -345,7 +345,7 @@ def pytest_pyfunc_call(pyfuncitem): pyfuncitem.obj = wrap_generator_fscope(func, event_loop, on_shutdown) elif inspect.isasyncgenfunction(func): if scope != 'function': - shutdown_item = pyfuncitem._launch_testing_shutdown_item + shutdown_item = pyfuncitem._launch_pytest_shutdown_item pyfuncitem.obj, shutdown_item.obj = ( wrap_asyncgen(func, event_loop, on_shutdown) ) @@ -353,7 +353,7 @@ def pytest_pyfunc_call(pyfuncitem): shutdown_item, shutdown_item.obj, shutdown_item.cls, funcargs=True) else: pyfuncitem.obj = wrap_asyncgen_fscope(func, event_loop, on_shutdown) - elif not getattr(pyfuncitem.obj, '_launch_testing_wrapped', False): + elif not getattr(pyfuncitem.obj, '_launch_pytest_wrapped', False): pyfuncitem.obj = wrap_func(func, event_loop, before_test) yield @@ -411,7 +411,7 @@ def shutdown(): 'launch tests using a generator function must stop iteration after yielding once', pytrace=False ) - shutdown._launch_testing_wrapped = True + shutdown._launch_pytest_wrapped = True shutdown.__name__ = f'{func.__name__}[shutdown]' @functools.wraps(func) @@ -466,7 +466,7 @@ def shutdown(**kwargs): pytrace=False ) shutdown.__name__ = f'{func.__name__}[shutdown]' - shutdown._launch_testing_wrapped = True + shutdown._launch_pytest_wrapped = True @functools.wraps(func) def inner(**kwargs): diff --git a/launch_pytest/launch_pytest/tools/__init__.py b/launch_pytest/launch_pytest/tools/__init__.py new file mode 100644 index 000000000..a995dc8dd --- /dev/null +++ b/launch_pytest/launch_pytest/tools/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from . import process + +__all__ = [ + 'process', +] diff --git a/launch_testing/launch_testing/tools/pytest_process.py b/launch_pytest/launch_pytest/tools/process.py similarity index 100% rename from launch_testing/launch_testing/tools/pytest_process.py rename to launch_pytest/launch_pytest/tools/process.py diff --git a/launch_pytest/package.xml b/launch_pytest/package.xml new file mode 100644 index 000000000..c1f686955 --- /dev/null +++ b/launch_pytest/package.xml @@ -0,0 +1,28 @@ + + + + launch_pytest + 0.19.0 + A package to create tests which involve launch files and multiple processes. + William Woodall + Michel Hidalgo + Apache License 2.0 + + Ivan Paunovic + + ament_index_python + launch + launch_testing + osrf_pycommon + python3-pytest + python3-pytest-asyncio + + ament_copyright + ament_flake8 + ament_pep257 + launch + + + ament_python + + diff --git a/launch_testing/launch_testing/pytest/legacy/__init__.py b/launch_pytest/resource/launch_pytest similarity index 100% rename from launch_testing/launch_testing/pytest/legacy/__init__.py rename to launch_pytest/resource/launch_pytest diff --git a/launch_pytest/setup.cfg b/launch_pytest/setup.cfg new file mode 100644 index 000000000..2d7cc1c8c --- /dev/null +++ b/launch_pytest/setup.cfg @@ -0,0 +1,4 @@ +[coverage:run] +# This will let coverage find files with 0% coverage (not hit by tests at all) +source = . +omit = setup.py diff --git a/launch_pytest/setup.py b/launch_pytest/setup.py new file mode 100644 index 000000000..431c4fd5f --- /dev/null +++ b/launch_pytest/setup.py @@ -0,0 +1,44 @@ +import glob + +from setuptools import find_packages +from setuptools import setup + + +package_name = 'launch_pytest' + +setup( + name=package_name, + version='0.19.0', + packages=find_packages(exclude=['test']), + data_files=[ + ('share/ament_index/resource_index/packages', [f'resource/{package_name}']), + (f'lib/{package_name}', glob.glob('example_processes/**')), + (f'share/{package_name}', ['package.xml']), + (f'share/{package_name}/examples', glob.glob(f'test/{package_name}/examples/[!_]**')), + ], + entry_points={ + 'pytest11': [ + 'launch_pytest = launch_pytest.plugin' + ], + }, + install_requires=['setuptools'], + zip_safe=True, + author='Ivan Paunovic', + author_email='ivanpauno@ekumenlabs.com', + maintainer='William Woodall, Michel Hidalgo', + maintainer_email='william@osrfoundation.org, michel@ekumenlabs.com', + url='https://github.com/ros2/launch', + download_url='https://github.com/ros2/launch/releases', + keywords=['ROS'], + classifiers=[ + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python', + 'Topic :: Software Development', + ], + description='Create tests which involve launch files and multiple processes.', + long_description=('A package to create tests which involve' + ' launch files and multiple processes.'), + license='Apache License, Version 2.0', + tests_require=['pytest'], +) diff --git a/launch_testing/test/launch_testing/examples/pytest_hello_world.py b/launch_pytest/test/examples/pytest_hello_world.py similarity index 83% rename from launch_testing/test/launch_testing/examples/pytest_hello_world.py rename to launch_pytest/test/examples/pytest_hello_world.py index 57c29cab1..fe63d8db1 100644 --- a/launch_testing/test/launch_testing/examples/pytest_hello_world.py +++ b/launch_pytest/test/examples/pytest_hello_world.py @@ -14,10 +14,12 @@ import launch -import launch.actions -import launch_testing.actions -import launch_testing.markers -from launch_testing.tools import pytest_process as tools + +import launch_testing + +import launch_pytest +from launch_pytest.tools import process as process_tools + import pytest @@ -31,7 +33,7 @@ def hello_world_proc(): ) # This function specifies the processes to be run for our test. -@launch_testing.pytest.fixture +@launch_pytest.fixture def launch_description(hello_world_proc): """Launch a simple process to print 'hello_world'.""" return launch.LaunchDescription([ @@ -42,18 +44,18 @@ def launch_description(hello_world_proc): ]) -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) def test_read_stdout(hello_world_proc, launch_context): """Check if 'hello_world' was found in the stdout.""" def validate_output(output): # this function can use assertions to validate the output or return a boolean. # pytest generates easier to understand failures when assertions are used. assert output == 'hello_world\n', 'process never printed hello_world' - assert tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=5) + assert process_tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=5) def validate_output(output): return output == 'this will never happen' - assert not tools.wait_for_output_sync( + assert not process_tools.wait_for_output_sync( launch_context, hello_world_proc, validate_output, timeout=0.1) yield # this is executed after launch service shutdown diff --git a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py b/launch_pytest/test/test_function_scope.py similarity index 84% rename from launch_testing/test/launch_testing/new_hooks/test_function_scope.py rename to launch_pytest/test/test_function_scope.py index 38bcde9b2..c74fe5fa3 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_function_scope.py +++ b/launch_pytest/test/test_function_scope.py @@ -14,6 +14,7 @@ import launch import launch_testing +import launch_pytest import pytest @@ -23,7 +24,7 @@ def order(): return [] -@launch_testing.pytest.fixture +@launch_pytest.fixture def launch_description(): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), @@ -31,26 +32,26 @@ def launch_description(): ]) -@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +@pytest.mark.launch(fixture=launch_description, shutdown=True) async def test_after_shutdown(order, launch_service): order.append('test_after_shutdown') assert launch_service._is_idle() assert launch_service.event_loop is None -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) async def test_case_1(order): order.append('test_case_1') assert True -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) def test_case_2(order): order.append('test_case_2') assert True -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) def test_case_3(order, launch_service): order.append('test_case_3') yield @@ -58,7 +59,7 @@ def test_case_3(order, launch_service): order.append('test_case_3[shutdown]') -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) async def test_case_4(order): order.append('test_case_4') yield diff --git a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py b/launch_pytest/test/test_module_scope.py similarity index 84% rename from launch_testing/test/launch_testing/new_hooks/test_module_scope.py rename to launch_pytest/test/test_module_scope.py index e7d92a894..de17115e7 100644 --- a/launch_testing/test/launch_testing/new_hooks/test_module_scope.py +++ b/launch_pytest/test/test_module_scope.py @@ -13,6 +13,7 @@ # limitations under the License. import launch +import launch_pytest import launch_testing import pytest @@ -25,7 +26,7 @@ def order(): print('end') -@launch_testing.pytest.fixture(scope='module', params=['asd', 'bsd']) +@launch_pytest.fixture(scope='module', params=['asd', 'bsd']) def launch_description(request): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), @@ -33,7 +34,7 @@ def launch_description(request): ]), request.param -@pytest.mark.launch_testing(fixture=launch_description, shutdown=True) +@pytest.mark.launch(fixture=launch_description, shutdown=True) def test_after_shutdown(order, launch_service, launch_description): param = launch_description[1] order.append(f'test_after_shutdown[{param}]') @@ -41,21 +42,21 @@ def test_after_shutdown(order, launch_service, launch_description): assert launch_service.event_loop is None -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) async def test_case_1(order, launch_description): param = launch_description[1] order.append(f'test_case_1[{param}]') assert True -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) def test_case_2(order, launch_description): param = launch_description[1] order.append(f'test_case_2[{param}]') assert True -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) def test_case_3(order, launch_service, launch_description): param = launch_description[1] order.append(f'test_case_3[{param}]') @@ -65,7 +66,7 @@ def test_case_3(order, launch_service, launch_description): order.append(f'test_case_3[{param}][shutdown]') -@pytest.mark.launch_testing(fixture=launch_description) +@pytest.mark.launch(fixture=launch_description) async def test_case_4(order, launch_service, launch_description): param = launch_description[1] order.append(f'test_case_4[{param}]') @@ -76,11 +77,6 @@ async def test_case_4(order, launch_service, launch_description): order.append(f'test_case_4[{param}][shutdown]') -# @pytest.mark.launch_testing -# def test_should_be_skipped(): -# assert True - - def test_order(order): assert order == [ 'test_case_1[asd]', diff --git a/launch_testing/launch_testing/pytest/__init__.py b/launch_testing/launch_testing/pytest/__init__.py index f99014d81..e69de29bb 100644 --- a/launch_testing/launch_testing/pytest/__init__.py +++ b/launch_testing/launch_testing/pytest/__init__.py @@ -1,21 +0,0 @@ -# Copyright 2021 Open Source Robotics Foundation, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from . import legacy -from .fixture import fixture - -__all__ = [ - 'fixture', - 'legacy', -] diff --git a/launch_testing/launch_testing/pytest/legacy/hooks.py b/launch_testing/launch_testing/pytest/hooks.py similarity index 97% rename from launch_testing/launch_testing/pytest/legacy/hooks.py rename to launch_testing/launch_testing/pytest/hooks.py index f364f2572..5b4cebaf7 100644 --- a/launch_testing/launch_testing/pytest/legacy/hooks.py +++ b/launch_testing/launch_testing/pytest/hooks.py @@ -19,8 +19,8 @@ import pytest -from ...loader import LoadTestsFromPythonModule -from ...test_runner import LaunchTestRunner +from ..loader import LoadTestsFromPythonModule +from ..test_runner import LaunchTestRunner class LaunchTestFailure(Exception): @@ -189,7 +189,7 @@ def pytest_launch_collect_makemodule(path, parent, entrypoint): def pytest_addhooks(pluginmanager): - import launch_testing.pytest.legacy.hookspecs as hookspecs + from launch_testing.pytest import hookspecs pluginmanager.add_hookspecs(hookspecs) diff --git a/launch_testing/launch_testing/pytest/legacy/hookspecs.py b/launch_testing/launch_testing/pytest/hookspecs.py similarity index 100% rename from launch_testing/launch_testing/pytest/legacy/hookspecs.py rename to launch_testing/launch_testing/pytest/hookspecs.py diff --git a/launch_testing/setup.py b/launch_testing/setup.py index e6cb247b2..c6ccaff3d 100644 --- a/launch_testing/setup.py +++ b/launch_testing/setup.py @@ -17,8 +17,7 @@ entry_points={ 'console_scripts': ['launch_test=launch_testing.launch_test:main'], 'pytest11': [ - 'launch_testing_legacy = launch_testing.pytest.legacy.hooks', - 'launch_testing = launch_testing.pytest.plugin' + 'launch_testing = launch_testing.pytest.hooks', ], }, install_requires=['setuptools'], From 256a46169454659d4cce5096c149706237ec8767 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 18 Oct 2021 18:15:04 -0300 Subject: [PATCH 26/48] Minimize diff with master Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/setup.py | 4 +--- launch_testing/package.xml | 1 - launch_testing/setup.py | 4 +--- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/launch_pytest/setup.py b/launch_pytest/setup.py index 431c4fd5f..f198c763a 100644 --- a/launch_pytest/setup.py +++ b/launch_pytest/setup.py @@ -17,9 +17,7 @@ (f'share/{package_name}/examples', glob.glob(f'test/{package_name}/examples/[!_]**')), ], entry_points={ - 'pytest11': [ - 'launch_pytest = launch_pytest.plugin' - ], + 'pytest11': ['launch_pytest = launch_pytest.plugin'], }, install_requires=['setuptools'], zip_safe=True, diff --git a/launch_testing/package.xml b/launch_testing/package.xml index f184b7f62..bd4d70828 100644 --- a/launch_testing/package.xml +++ b/launch_testing/package.xml @@ -16,7 +16,6 @@ launch osrf_pycommon python3-pytest - python3-pytest-asyncio ament_copyright ament_flake8 diff --git a/launch_testing/setup.py b/launch_testing/setup.py index c6ccaff3d..b55254998 100644 --- a/launch_testing/setup.py +++ b/launch_testing/setup.py @@ -16,9 +16,7 @@ ], entry_points={ 'console_scripts': ['launch_test=launch_testing.launch_test:main'], - 'pytest11': [ - 'launch_testing = launch_testing.pytest.hooks', - ], + 'pytest11': ['launch_testing = launch_testing.pytest.hooks'], }, install_requires=['setuptools'], zip_safe=True, From 27910ae93b0cdcd0b0261c4fc0912e94c9a3b66b Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 19 Oct 2021 14:42:14 -0300 Subject: [PATCH 27/48] Reexport convenient utilities from launch_testing Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/__init__.py | 2 ++ .../launch_pytest/actions/__init__.py | 19 +++++++++++++++++++ launch_pytest/launch_pytest/plugin.py | 6 +++--- .../test/examples/pytest_hello_world.py | 4 +--- launch_pytest/test/test_function_scope.py | 2 +- launch_pytest/test/test_module_scope.py | 2 +- 6 files changed, 27 insertions(+), 8 deletions(-) create mode 100644 launch_pytest/launch_pytest/actions/__init__.py diff --git a/launch_pytest/launch_pytest/__init__.py b/launch_pytest/launch_pytest/__init__.py index 04441b9b1..4199e9d9a 100644 --- a/launch_pytest/launch_pytest/__init__.py +++ b/launch_pytest/launch_pytest/__init__.py @@ -12,10 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +from . import actions from . import tools from .fixture import fixture __all__ = [ + 'actions', 'fixture', 'tools', ] diff --git a/launch_pytest/launch_pytest/actions/__init__.py b/launch_pytest/launch_pytest/actions/__init__.py new file mode 100644 index 000000000..8e6260b2d --- /dev/null +++ b/launch_pytest/launch_pytest/actions/__init__.py @@ -0,0 +1,19 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from launch_testing.actions import ReadyToTest + +__all__ = [ + 'ReadyToTest', +] \ No newline at end of file diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 69d56b677..b9847d0b3 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -21,10 +21,10 @@ from _pytest.outcomes import skip import launch -import launch_testing import pytest +from .actions import ReadyToTest from .fixture import finalize_launch_service from .fixture import get_launch_context_fixture from .fixture import get_launch_service_fixture @@ -63,7 +63,7 @@ def pytest_configure(config): def iterate_ready_to_test_actions(entities): """Search recursively LaunchDescription entities for all ReadyToTest actions.""" for entity in entities: - if isinstance(entity, launch_testing.actions.ReadyToTest): + if isinstance(entity, ReadyToTest): yield entity yield from iterate_ready_to_test_actions( entity.describe_sub_entities() @@ -80,7 +80,7 @@ def get_ready_to_test_action(launch_description): try: ready_action = next(gen) except StopIteration: # No ReadyToTest action found - ready_action = launch_testing.actions.ReadyToTest() + ready_action = ReadyToTest() launch_description.append(ready_action) return ready_action try: diff --git a/launch_pytest/test/examples/pytest_hello_world.py b/launch_pytest/test/examples/pytest_hello_world.py index fe63d8db1..ab29d0964 100644 --- a/launch_pytest/test/examples/pytest_hello_world.py +++ b/launch_pytest/test/examples/pytest_hello_world.py @@ -15,8 +15,6 @@ import launch -import launch_testing - import launch_pytest from launch_pytest.tools import process as process_tools @@ -40,7 +38,7 @@ def launch_description(hello_world_proc): hello_world_proc, # Tell launch when to start the test # If no ReadyToTest action is added, one will be appended automatically. - launch_testing.actions.ReadyToTest() + launch_pytest.actions.ReadyToTest() ]) diff --git a/launch_pytest/test/test_function_scope.py b/launch_pytest/test/test_function_scope.py index c74fe5fa3..8fba31a65 100644 --- a/launch_pytest/test/test_function_scope.py +++ b/launch_pytest/test/test_function_scope.py @@ -28,7 +28,7 @@ def order(): def launch_description(): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), - launch_testing.actions.ReadyToTest(), + launch_pytest.actions.ReadyToTest(), ]) diff --git a/launch_pytest/test/test_module_scope.py b/launch_pytest/test/test_module_scope.py index de17115e7..dd4180c6e 100644 --- a/launch_pytest/test/test_module_scope.py +++ b/launch_pytest/test/test_module_scope.py @@ -30,7 +30,7 @@ def order(): def launch_description(request): return launch.LaunchDescription([ launch_testing.util.KeepAliveProc(), - launch_testing.actions.ReadyToTest(), + launch_pytest.actions.ReadyToTest(), ]), request.param From 339eb995424285768acb154b6c326dbd06b52860 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 19 Oct 2021 16:35:46 -0300 Subject: [PATCH 28/48] Add another example Signed-off-by: Ivan Santiago Paunovic --- .../test/examples/check_node_msgs.py | 78 +++++++++++++++++++ .../test/examples/executables/listener.py | 47 +++++++++++ 2 files changed, 125 insertions(+) create mode 100644 launch_pytest/test/examples/check_node_msgs.py create mode 100644 launch_pytest/test/examples/executables/listener.py diff --git a/launch_pytest/test/examples/check_node_msgs.py b/launch_pytest/test/examples/check_node_msgs.py new file mode 100644 index 000000000..c071dea91 --- /dev/null +++ b/launch_pytest/test/examples/check_node_msgs.py @@ -0,0 +1,78 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +import sys +from threading import Event +from threading import Thread + +import launch +import launch_pytest +import launch_ros + +from std_msgs.msg import String + +import rclpy +from rclpy.node import Node + +import pytest + + +@launch_pytest.fixture +def generate_test_description(): + path_to_test = Path(__file__).parent + + return launch.LaunchDescription([ + launch_ros.actions.Node( + executable=sys.executable, + arguments=[str(path_to_test / 'executables' / 'talker.py')], + additional_env={'PYTHONUNBUFFERED': '1'}, + name='demo_node_1', + ), + ]) + + +@pytest.mark.launch +def test_check_if_msgs_published(): + rclpy.init() + try: + node = MakeTestNode('test_node') + node.start_subscriber() + msgs_received_flag = node.msg_event_object.wait(timeout=5.0) + assert msgs_received_flag, 'Did not receive msgs !' + finally: + rclpy.shutdown() + + +class MakeTestNode(Node): + + def __init__(self, name='test_node'): + super().__init__(name) + self.msg_event_object = Event() + + def start_subscriber(self): + # Create a subscriber + self.subscription = self.create_subscription( + String, + 'chatter', + self.subscriber_callback, + 10 + ) + + # Add a spin thread + self.ros_spin_thread = Thread(target=lambda node: rclpy.spin(node), args=(self,)) + self.ros_spin_thread.start() + + def subscriber_callback(self, data): + self.msg_event_object.set() diff --git a/launch_pytest/test/examples/executables/listener.py b/launch_pytest/test/examples/executables/listener.py new file mode 100644 index 000000000..861ae743d --- /dev/null +++ b/launch_pytest/test/examples/executables/listener.py @@ -0,0 +1,47 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import rclpy +from rclpy.node import Node + +from std_msgs.msg import String + + +class Listener(Node): + + def __init__(self): + super().__init__('listener') + self.subscription = self.create_subscription( + String, 'chatter', self.callback, 10 + ) + + def callback(self, msg): + self.get_logger().info('I heard: [%s]' % msg.data) + + +def main(args=None): + rclpy.init(args=args) + + node = Listener() + try: + rclpy.spin(node) + except KeyboardInterrupt: + pass + finally: + node.destroy_node() + rclpy.shutdown() + + +if __name__ == '__main__': + main() From c517c666ffd742aa142d2b614f5765fbf12400df Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 19 Oct 2021 18:05:10 -0300 Subject: [PATCH 29/48] Complete process tools, add tests, fix bugs Signed-off-by: Ivan Santiago Paunovic --- launch/launch/actions/execute_local.py | 4 +- launch_pytest/launch_pytest/plugin.py | 2 +- launch_pytest/launch_pytest/tools/__init__.py | 16 ++ launch_pytest/launch_pytest/tools/process.py | 149 +++++++++++++----- launch_pytest/test/tools/test_process.py | 67 ++++++++ 5 files changed, 199 insertions(+), 39 deletions(-) create mode 100644 launch_pytest/test/tools/test_process.py diff --git a/launch/launch/actions/execute_local.py b/launch/launch/actions/execute_local.py index 6a025b202..8993cffe1 100644 --- a/launch/launch/actions/execute_local.py +++ b/launch/launch/actions/execute_local.py @@ -390,7 +390,7 @@ def __on_process_output_cached( ) -> Optional[SomeActionsType]: to_write = event.text.decode(errors='replace') last_cursor = buffer.tell() - self.__stdout_buffer.seek(0, 2) # go to end of buffer + buffer.seek(0, 2) # go to end of buffer buffer.write(to_write) buffer.seek(last_cursor) new_cursor = last_cursor @@ -405,7 +405,7 @@ def __on_process_output_cached( def __flush_cached_buffers(self, event, context): for line in self.__stdout_buffer: - self.__stdout_buffer.info( + self.__stdout_logger.info( self.__output_format.format(line=line, this=self) ) diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index b9847d0b3..59c5d0c67 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -81,7 +81,7 @@ def get_ready_to_test_action(launch_description): ready_action = next(gen) except StopIteration: # No ReadyToTest action found ready_action = ReadyToTest() - launch_description.append(ready_action) + launch_description.entities.append(ready_action) return ready_action try: next(gen) diff --git a/launch_pytest/launch_pytest/tools/__init__.py b/launch_pytest/launch_pytest/tools/__init__.py index a995dc8dd..14279b69b 100644 --- a/launch_pytest/launch_pytest/tools/__init__.py +++ b/launch_pytest/launch_pytest/tools/__init__.py @@ -13,7 +13,23 @@ # limitations under the License. from . import process +from .process import wait_for_exit +from .process import wait_for_exit_sync +from .process import wait_for_output +from .process import wait_for_output_sync +from .process import wait_for_start +from .process import wait_for_start_sync +from .process import wait_for_stderr +from .process import wait_for_stderr_sync __all__ = [ 'process', + 'wait_for_exit', + 'wait_for_exit_sync', + 'wait_for_output', + 'wait_for_output_sync', + 'wait_for_start', + 'wait_for_start_sync', + 'wait_for_stderr', + 'wait_for_stderr_sync', ] diff --git a/launch_pytest/launch_pytest/tools/process.py b/launch_pytest/launch_pytest/tools/process.py index fb50e71c6..4479984db 100644 --- a/launch_pytest/launch_pytest/tools/process.py +++ b/launch_pytest/launch_pytest/tools/process.py @@ -29,18 +29,17 @@ def register_event_handler(context, event_handler): context.unregister_event_handler(event_handler) -def _get_on_process_start(execute_process_action, pyevent): - event_handlers.OnProcessStart( - target_action=execute_process_action, on_start=lambda _1, _2: pyevent.set()) - - async def _wait_for_event( launch_context, execute_process_action, get_launch_event_handler, timeout=None ): pyevent = asyncio.Event() event_handler = get_launch_event_handler(execute_process_action, pyevent) with register_event_handler(launch_context, event_handler): - await asyncio.wait_for(pyevent.wait(), timeout) + try: + await asyncio.wait_for(pyevent.wait(), timeout) + except asyncio.TimeoutError: + return False + return True async def _wait_for_event_with_condition( @@ -48,16 +47,29 @@ async def _wait_for_event_with_condition( ): pyevent = asyncio.Event() event_handler = get_launch_event_handler(execute_process_action, pyevent) - cond_value = condition() + cond_value = False + try: + cond_value = condition() + except AssertionError: + pass with register_event_handler(launch_context, event_handler): start = time.time() now = start while not cond_value and (timeout is None or now < start + timeout): - await asyncio.wait_for(pyevent.wait(), start - now + timeout) + try: + await asyncio.wait_for(pyevent.wait(), start - now + timeout) + except asyncio.TimeoutError: + break pyevent.clear() - cond_value = condition() + try: + cond_value = condition() + except AssertionError: + pass now = time.time() - return cond_value + # Call condition() again, if before it returned False. + # If assertions were being used and the condition is still not satisfied it should raise here, + # pytest renders assertion errors nicely. + return condition() if not cond_value else cond_value def _wait_for_event_sync( @@ -66,7 +78,7 @@ def _wait_for_event_sync( pyevent = threading.Event() event_handler = get_launch_event_handler(execute_process_action, pyevent) with register_event_handler(launch_context, event_handler): - pyevent.wait(timeout) + return pyevent.wait(timeout) def _wait_for_event_with_condition_sync( @@ -74,16 +86,26 @@ def _wait_for_event_with_condition_sync( ): pyevent = threading.Event() event_handler = get_launch_event_handler(execute_process_action, pyevent) - cond_value = condition() + cond_value = False + try: + cond_value = condition() + except AssertionError: + pass # Allow asserts in the condition closures with register_event_handler(launch_context, event_handler): start = time.time() now = start while not cond_value and (timeout is None or now < start + timeout): pyevent.wait(start - now + timeout) pyevent.clear() - cond_value = condition() + try: + cond_value = condition() + except AssertionError: + pass now = time.time() - return cond_value + # Call condition() again, if before it returned False. + # If assertions were being used and the condition is still not satisfied it should raise here, + # pytest renders assertion errors nicely. + return condition() if not cond_value else cond_value def _get_stdout_event_handler(action, pyevent): @@ -94,30 +116,85 @@ def _get_stdout_event_handler(action, pyevent): async def wait_for_output( launch_context, execute_process_action, validate_output, timeout=None ): - def condition(): - try: - return validate_output(execute_process_action.get_stdout()) - except AssertionError: - return False - success = await _wait_for_event_with_condition( - launch_context, execute_process_action, _get_stdout_event_handler, condition, timeout) - if not success: - # Validate the output again, this time not catching assertion errors. - # This allows the user to use asserts directly, errors will be nicely rendeded by pytest. - return validate_output(execute_process_action.get_stdout()) + return await _wait_for_event_with_condition( + launch_context, + execute_process_action, + _get_stdout_event_handler, + lambda: validate_output(execute_process_action.get_stdout()), + timeout) def wait_for_output_sync( launch_context, execute_process_action, validate_output, timeout=None ): - def condition(): - try: - return validate_output(execute_process_action.get_stdout()) - except AssertionError: - return False - success = _wait_for_event_with_condition_sync( - launch_context, execute_process_action, _get_stdout_event_handler, condition, timeout) - if not success: - # Validate the output again, this time not catching assertion errors. - # This allows the user to use asserts directly, errors will be nicely rendeded by pytest. - return validate_output(execute_process_action.get_stdout()) in (None, True) + return _wait_for_event_with_condition_sync( + launch_context, + execute_process_action, + _get_stdout_event_handler, + lambda: validate_output(execute_process_action.get_stdout()), + timeout) + + +def _get_stderr_event_handler(action, pyevent): + return event_handlers.OnProcessIO( + target_action=action, on_stderr=lambda _1: pyevent.set()) + + +async def wait_for_stderr( + launch_context, execute_process_action, validate_output, timeout=None +): + return await _wait_for_event_with_condition( + launch_context, + execute_process_action, + _get_stderr_event_handler, + lambda: validate_output(execute_process_action.get_stderr()), + timeout) + + +def wait_for_stderr_sync( + launch_context, execute_process_action, validate_output, timeout=None +): + return _wait_for_event_with_condition_sync( + launch_context, + execute_process_action, + _get_stderr_event_handler, + lambda: validate_output(execute_process_action.get_stderr()), + timeout) + + +def _get_on_process_start_event_handler(execute_process_action, pyevent): + return event_handlers.OnProcessStart( + target_action=execute_process_action, on_start=lambda _1, _2: pyevent.set()) + + +async def wait_for_start( + launch_context, execute_process_action, timeout=None +): + return await _wait_for_event( + launch_context, execute_process_action, _get_on_process_start_event_handler, timeout) + + +def wait_for_start_sync( + launch_context, execute_process_action, timeout=None +): + return _wait_for_event_sync( + launch_context, execute_process_action, _get_on_process_start_event_handler, timeout) + + +def _get_on_process_exit_event_handler(execute_process_action, pyevent): + return event_handlers.OnProcessExit( + target_action=execute_process_action, on_exit=lambda _1, _2: pyevent.set()) + + +async def wait_for_exit( + launch_context, execute_process_action, timeout=None +): + return await _wait_for_event( + launch_context, execute_process_action, _get_on_process_exit_event_handler, timeout) + + +def wait_for_exit_sync( + launch_context, execute_process_action, timeout=None +): + return _wait_for_event_sync( + launch_context, execute_process_action, _get_on_process_exit_event_handler, timeout) diff --git a/launch_pytest/test/tools/test_process.py b/launch_pytest/test/tools/test_process.py new file mode 100644 index 000000000..464a7cfb8 --- /dev/null +++ b/launch_pytest/test/tools/test_process.py @@ -0,0 +1,67 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import launch +import launch_pytest +from launch_pytest import tools + +import pytest + +PYTHON_SCRIPT = \ +""" +import sys +import time + +print('hello') +print('world', file=sys.stderr) +time.sleep(5) +""" + +@pytest.fixture +def dut(): + return launch.actions.ExecuteProcess( + cmd=['python3', '-c', PYTHON_SCRIPT], + cached_output=True, + output='screen' + ) + +@launch_pytest.fixture +def launch_description(dut): + return launch.LaunchDescription([ + dut, + ]) + + +@pytest.mark.launch(fixture=launch_description) +async def test_async_process_tools(dut, launch_context): + await tools.wait_for_start(launch_context, dut, timeout=10) + def check_output(output): assert output == 'hello\n' + await tools.wait_for_output( + launch_context, dut, check_output, timeout=10) + def check_stderr(err): assert err == 'world\n' + await tools.wait_for_stderr( + launch_context, dut, check_stderr, timeout=10) + await tools.wait_for_exit(launch_context, dut, timeout=10) + + +@pytest.mark.launch(fixture=launch_description) +def test_sync_process_tools(dut, launch_context): + tools.wait_for_start_sync(launch_context, dut, timeout=10) + def check_output(output): assert output == 'hello\n' + tools.wait_for_output_sync( + launch_context, dut, check_output, timeout=10) + def check_stderr(err): assert err == 'world\n' + tools.wait_for_stderr_sync( + launch_context, dut, check_stderr, timeout=10) + tools.wait_for_exit_sync(launch_context, dut, timeout=10) From 495c98ebc713d47dd8edb62770b88aedf3e03bad Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 19 Oct 2021 18:38:28 -0300 Subject: [PATCH 30/48] Avoid python3-pytest-asyncio dependency Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 12 ++++++++++++ launch_pytest/launch_pytest/plugin.py | 4 ++++ launch_pytest/package.xml | 1 - 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index 4c2a4d4d0..ffd4cb6d6 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -67,12 +67,24 @@ def launch_context(launch_service): def get_event_loop_fixture(*, scope='function', overridable=True): """Return an event loop fixture.""" + # Adapted from https://github.com/pytest-dev/pytest-asyncio, + # see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @pytest.fixture(scope=scope) def event_loop(): """Create an event loop instance for each test case.""" loop = asyncio.get_event_loop_policy().new_event_loop() + policy = asyncio.get_event_loop_policy() + try: + old_loop = policy.get_event_loop() + if old_loop is not loop: + old_loop.close() + except RuntimeError: + # Swallow this, since it's probably bad event loop hygiene. + pass + policy.set_event_loop(loop) yield loop loop.close() + asyncio.set_event_loop_policy(None) if overridable: event_loop._launch_pytest_overridable_fixture = True event_loop._launch_pytest_fixture_scope = scope diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 59c5d0c67..62f7dc416 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -26,6 +26,7 @@ from .actions import ReadyToTest from .fixture import finalize_launch_service +from .fixture import get_event_loop_fixture from .fixture import get_launch_context_fixture from .fixture import get_launch_service_fixture @@ -509,3 +510,6 @@ def inner(**kwargs): launch_context = get_launch_context_fixture(overridable=False) """Launch context fixture.""" + +event_loop = get_event_loop_fixture(overridable=False) +"""Event loop fixture.""" diff --git a/launch_pytest/package.xml b/launch_pytest/package.xml index c1f686955..c324380e1 100644 --- a/launch_pytest/package.xml +++ b/launch_pytest/package.xml @@ -15,7 +15,6 @@ launch_testing osrf_pycommon python3-pytest - python3-pytest-asyncio ament_copyright ament_flake8 From b4d993556faf9045f614b85525571db6979f40a4 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 25 Oct 2021 17:58:23 -0300 Subject: [PATCH 31/48] Add linters, test some of the plugin corner cases Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/test/conftest.py | 15 ++ .../examples/check_node_msgs.py | 4 +- .../examples/executables/listener.py | 0 .../examples/pytest_hello_world.py | 4 +- .../test_function_scope.py | 2 +- .../{ => launch_pytest}/test_module_scope.py | 0 .../test/launch_pytest/test_plugin.py | 180 ++++++++++++++++++ .../{ => launch_pytest}/tools/test_process.py | 7 +- launch_pytest/test/test_copyright.py | 23 +++ launch_pytest/test/test_flake8.py | 25 +++ launch_pytest/test/test_pep257.py | 23 +++ 11 files changed, 277 insertions(+), 6 deletions(-) create mode 100644 launch_pytest/test/conftest.py rename launch_pytest/test/{ => launch_pytest}/examples/check_node_msgs.py (100%) rename launch_pytest/test/{ => launch_pytest}/examples/executables/listener.py (100%) rename launch_pytest/test/{ => launch_pytest}/examples/pytest_hello_world.py (94%) rename launch_pytest/test/{ => launch_pytest}/test_function_scope.py (100%) rename launch_pytest/test/{ => launch_pytest}/test_module_scope.py (100%) create mode 100644 launch_pytest/test/launch_pytest/test_plugin.py rename launch_pytest/test/{ => launch_pytest}/tools/test_process.py (98%) create mode 100644 launch_pytest/test/test_copyright.py create mode 100644 launch_pytest/test/test_flake8.py create mode 100644 launch_pytest/test/test_pep257.py diff --git a/launch_pytest/test/conftest.py b/launch_pytest/test/conftest.py new file mode 100644 index 000000000..f9178550d --- /dev/null +++ b/launch_pytest/test/conftest.py @@ -0,0 +1,15 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +pytest_plugins = ['pytester'] diff --git a/launch_pytest/test/examples/check_node_msgs.py b/launch_pytest/test/launch_pytest/examples/check_node_msgs.py similarity index 100% rename from launch_pytest/test/examples/check_node_msgs.py rename to launch_pytest/test/launch_pytest/examples/check_node_msgs.py index c071dea91..bae89e689 100644 --- a/launch_pytest/test/examples/check_node_msgs.py +++ b/launch_pytest/test/launch_pytest/examples/check_node_msgs.py @@ -21,12 +21,12 @@ import launch_pytest import launch_ros -from std_msgs.msg import String +import pytest import rclpy from rclpy.node import Node -import pytest +from std_msgs.msg import String @launch_pytest.fixture diff --git a/launch_pytest/test/examples/executables/listener.py b/launch_pytest/test/launch_pytest/examples/executables/listener.py similarity index 100% rename from launch_pytest/test/examples/executables/listener.py rename to launch_pytest/test/launch_pytest/examples/executables/listener.py diff --git a/launch_pytest/test/examples/pytest_hello_world.py b/launch_pytest/test/launch_pytest/examples/pytest_hello_world.py similarity index 94% rename from launch_pytest/test/examples/pytest_hello_world.py rename to launch_pytest/test/launch_pytest/examples/pytest_hello_world.py index ab29d0964..89246a232 100644 --- a/launch_pytest/test/examples/pytest_hello_world.py +++ b/launch_pytest/test/launch_pytest/examples/pytest_hello_world.py @@ -30,6 +30,7 @@ def hello_world_proc(): cached_output=True, ) + # This function specifies the processes to be run for our test. @launch_pytest.fixture def launch_description(hello_world_proc): @@ -49,7 +50,8 @@ def validate_output(output): # this function can use assertions to validate the output or return a boolean. # pytest generates easier to understand failures when assertions are used. assert output == 'hello_world\n', 'process never printed hello_world' - assert process_tools.wait_for_output_sync(launch_context, hello_world_proc, validate_output, timeout=5) + assert process_tools.wait_for_output_sync( + launch_context, hello_world_proc, validate_output, timeout=5) def validate_output(output): return output == 'this will never happen' diff --git a/launch_pytest/test/test_function_scope.py b/launch_pytest/test/launch_pytest/test_function_scope.py similarity index 100% rename from launch_pytest/test/test_function_scope.py rename to launch_pytest/test/launch_pytest/test_function_scope.py index 8fba31a65..3e3519b4d 100644 --- a/launch_pytest/test/test_function_scope.py +++ b/launch_pytest/test/launch_pytest/test_function_scope.py @@ -13,8 +13,8 @@ # limitations under the License. import launch -import launch_testing import launch_pytest +import launch_testing import pytest diff --git a/launch_pytest/test/test_module_scope.py b/launch_pytest/test/launch_pytest/test_module_scope.py similarity index 100% rename from launch_pytest/test/test_module_scope.py rename to launch_pytest/test/launch_pytest/test_module_scope.py diff --git a/launch_pytest/test/launch_pytest/test_plugin.py b/launch_pytest/test/launch_pytest/test_plugin.py new file mode 100644 index 000000000..03ab3e68b --- /dev/null +++ b/launch_pytest/test/launch_pytest/test_plugin.py @@ -0,0 +1,180 @@ +# Copyright 2021 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +def test_launch_fixture_is_not_a_launch_description(pytester): + pytester.makepyfile("""\ +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return object() + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(errors=1) + result.stdout.re_match_lines(['.*must be either a launch description.*']) + + +def test_launch_fixture_is_not_a_sequence_starting_with_a_ld(pytester): + pytester.makepyfile("""\ +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return [object, 'asd'] + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(errors=1) + result.stdout.re_match_lines(['.*must be either a launch description.*']) + + +def test_multiple_ready_to_test_actions(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest(), launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(errors=1) + result.stdout.re_match_lines(['.*only one ReadyToTest action.*']) + + +def test_generator_yields_twice(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert True + yield + assert True + yield +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1) + result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) + + +def test_generator_yields_twice_module_scope(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture(scope='module') +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert True + yield + assert True + yield +""") + + result = pytester.runpytest() + + result.assert_outcomes(passed=1, failed=1) + result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) + + +def test_asyncgenerator_yields_twice(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +async def test_case(): + assert True + yield + assert True + yield +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1) + result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) + + +def test_asyncgenerator_yields_twice_module_scope(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture(scope='module') +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +async def test_case(): + assert True + yield + assert True + yield +""") + + result = pytester.runpytest() + + result.assert_outcomes(passed=1, failed=1) + result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) diff --git a/launch_pytest/test/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py similarity index 98% rename from launch_pytest/test/tools/test_process.py rename to launch_pytest/test/launch_pytest/tools/test_process.py index 464a7cfb8..396ac285d 100644 --- a/launch_pytest/test/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -18,8 +18,7 @@ import pytest -PYTHON_SCRIPT = \ -""" +PYTHON_SCRIPT = """\ import sys import time @@ -28,6 +27,7 @@ time.sleep(5) """ + @pytest.fixture def dut(): return launch.actions.ExecuteProcess( @@ -36,6 +36,7 @@ def dut(): output='screen' ) + @launch_pytest.fixture def launch_description(dut): return launch.LaunchDescription([ @@ -49,6 +50,7 @@ async def test_async_process_tools(dut, launch_context): def check_output(output): assert output == 'hello\n' await tools.wait_for_output( launch_context, dut, check_output, timeout=10) + def check_stderr(err): assert err == 'world\n' await tools.wait_for_stderr( launch_context, dut, check_stderr, timeout=10) @@ -61,6 +63,7 @@ def test_sync_process_tools(dut, launch_context): def check_output(output): assert output == 'hello\n' tools.wait_for_output_sync( launch_context, dut, check_output, timeout=10) + def check_stderr(err): assert err == 'world\n' tools.wait_for_stderr_sync( launch_context, dut, check_stderr, timeout=10) diff --git a/launch_pytest/test/test_copyright.py b/launch_pytest/test/test_copyright.py new file mode 100644 index 000000000..cf0fae31f --- /dev/null +++ b/launch_pytest/test/test_copyright.py @@ -0,0 +1,23 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_copyright.main import main +import pytest + + +@pytest.mark.copyright +@pytest.mark.linter +def test_copyright(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found errors' diff --git a/launch_pytest/test/test_flake8.py b/launch_pytest/test/test_flake8.py new file mode 100644 index 000000000..27ee1078f --- /dev/null +++ b/launch_pytest/test/test_flake8.py @@ -0,0 +1,25 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_flake8.main import main_with_errors +import pytest + + +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + rc, errors = main_with_errors(argv=[]) + assert rc == 0, \ + 'Found %d code style errors / warnings:\n' % len(errors) + \ + '\n'.join(errors) diff --git a/launch_pytest/test/test_pep257.py b/launch_pytest/test/test_pep257.py new file mode 100644 index 000000000..3aeb4d348 --- /dev/null +++ b/launch_pytest/test/test_pep257.py @@ -0,0 +1,23 @@ +# Copyright 2015 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pep257.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.pep257 +def test_pep257(): + rc = main(argv=[]) + assert rc == 0, 'Found code style errors / warnings' From 7de829255ddaa99e04607ccbb329ef37c4d65226 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 25 Oct 2021 18:25:18 -0300 Subject: [PATCH 32/48] Test more corner cases Signed-off-by: Ivan Santiago Paunovic --- .../test/launch_pytest/test_plugin.py | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/launch_pytest/test/launch_pytest/test_plugin.py b/launch_pytest/test/launch_pytest/test_plugin.py index 03ab3e68b..d98dc4366 100644 --- a/launch_pytest/test/launch_pytest/test_plugin.py +++ b/launch_pytest/test/launch_pytest/test_plugin.py @@ -178,3 +178,124 @@ async def test_case(): result.assert_outcomes(passed=1, failed=1) result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) + + +def test_fixture_kwarg_is_mandatory(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch +def test_case(): + pass +""") + + result = pytester.runpytest() + + result.assert_outcomes(skipped=1) + result.stdout.re_match_lines(['.*"fixture" keyword argument is required .*']) + + +def test_generator_shutdown_kwarg_true(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description, shutdown=True) +def test_case(): + assert True + yield + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1) + result.stdout.re_match_lines(['.*generator or async generator.* shutdown=True.*']) + + +def test_async_generator_shutdown_kwarg_true(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description, shutdown=True) +async def test_case(): + assert True + yield + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1) + result.stdout.re_match_lines(['.*generator or async generator.* shutdown=True.*']) + + +def test_generator_with_pre_shutdown_failure(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture(scope='module') +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +def test_case(): + assert False + yield + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1, skipped=1) + + +def test_async_generator_with_pre_shutdown_failure(pytester): + pytester.makepyfile("""\ +from launch import LaunchDescription +import launch_pytest +import pytest + +@launch_pytest.fixture(scope='module') +def launch_description(): + return LaunchDescription( + [launch_pytest.actions.ReadyToTest()] + ) + +@pytest.mark.launch(fixture=launch_description) +async def test_case(): + assert False + yield + assert True +""") + + result = pytester.runpytest() + + result.assert_outcomes(failed=1, skipped=1) From eef84632b03c3a7823e9adf6dd24b5ddbb8202c6 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 26 Oct 2021 10:28:08 -0300 Subject: [PATCH 33/48] fail instead of skipping Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/plugin.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 62f7dc416..8787cd546 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -314,10 +314,10 @@ def pytest_pyfunc_call(pyfuncitem): func = pyfuncitem.obj if has_shutdown_kwarg(pyfuncitem) and need_shutdown_test_item(func): - skip( - 'generator or asyncgenerator based launch test items cannot be marked with' - ' shutdown=True' - ) + error_msg = ( + 'generator or async generator based launch test items cannot be marked with' + ' shutdown=True') + fail(error_msg) yield return shutdown_test = is_shutdown_test(pyfuncitem) From ef691c95ae49c652b17d93c4f46e0f1e427990df Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 26 Oct 2021 11:14:10 -0300 Subject: [PATCH 34/48] Address peer review comments Signed-off-by: Ivan Santiago Paunovic --- launch/launch/actions/execute_local.py | 2 +- .../launch_pytest/actions/__init__.py | 2 +- launch_pytest/launch_pytest/fixture.py | 1 - launch_pytest/launch_pytest/plugin.py | 28 ++++++++----------- 4 files changed, 14 insertions(+), 19 deletions(-) diff --git a/launch/launch/actions/execute_local.py b/launch/launch/actions/execute_local.py index 8993cffe1..02b91d9a0 100644 --- a/launch/launch/actions/execute_local.py +++ b/launch/launch/actions/execute_local.py @@ -390,7 +390,7 @@ def __on_process_output_cached( ) -> Optional[SomeActionsType]: to_write = event.text.decode(errors='replace') last_cursor = buffer.tell() - buffer.seek(0, 2) # go to end of buffer + buffer.seek(0, os.SEEK_END) # go to end of buffer buffer.write(to_write) buffer.seek(last_cursor) new_cursor = last_cursor diff --git a/launch_pytest/launch_pytest/actions/__init__.py b/launch_pytest/launch_pytest/actions/__init__.py index 8e6260b2d..507c0ef2b 100644 --- a/launch_pytest/launch_pytest/actions/__init__.py +++ b/launch_pytest/launch_pytest/actions/__init__.py @@ -16,4 +16,4 @@ __all__ = [ 'ReadyToTest', -] \ No newline at end of file +] diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index ffd4cb6d6..6d8e89548 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -66,7 +66,6 @@ def launch_context(launch_service): def get_event_loop_fixture(*, scope='function', overridable=True): """Return an event loop fixture.""" - # Adapted from https://github.com/pytest-dev/pytest-asyncio, # see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @pytest.fixture(scope=scope) diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 8787cd546..815198d90 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -116,7 +116,6 @@ def pytest_fixture_setup(fixturedef, request): assert isinstance(ld, launch.LaunchDescription), wrong_ret_type_error ls.include_launch_description(ld) run_async_task = event_loop.create_task(ls.run_async( - # TODO(ivanpauno): maybe this could be configurable (?) shutdown_when_idle=options['shutdown_when_idle'] )) ready = get_ready_to_test_action(ld) @@ -148,14 +147,7 @@ def is_launch_test_mark_valid(item): If not, a warning and a skip mark will be added to the item. """ kwargs = item.get_closest_marker('launch').kwargs - ret = 'fixture' in kwargs and kwargs['fixture'] is not None - if not ret: - msg = ( - '"fixture" keyword argument is required in a pytest.mark.launch() ' - f'decorator') - item.warn(LaunchTestWarning(msg)) - item.add_marker(pytest.mark.skip(msg)) - return ret + return 'fixture' in kwargs and kwargs['fixture'] is not None def has_shutdown_kwarg(item): @@ -168,10 +160,7 @@ def get_launch_test_fixture(item): mark = item.get_closest_marker('launch') if mark is None: return None - try: - return mark.kwargs['fixture'] - except KeyError: - return None + return mark.kwargs.get('fixture') def get_launch_test_fixturename(item): @@ -193,7 +182,7 @@ def need_shutdown_test_item(obj): return inspect.isgeneratorfunction(obj) or inspect.isasyncgenfunction(obj) -def generate_test_items(collector, name, obj, fixturename, is_shutdown, needs_renaming): +def generate_test_items(collector, name, obj, fixturename, *, is_shutdown, needs_renaming): """Return list of test items for the corresponding object and injects the needed fixtures.""" # Inject all needed fixtures. # We use the `usefixtures` pytest mark instead of injecting them in fixturenames @@ -230,16 +219,23 @@ def pytest_pycollect_makeitem(collector, name, obj): if is_launch_test(item): if not is_launch_test_mark_valid(item): # return an item with a warning that's going to be skipped + msg = ( + '"fixture" keyword argument is required in a pytest.mark.launch() ' + f'decorator') + item.warn(LaunchTestWarning(msg)) + item.add_marker(pytest.mark.skip(msg)) return [item] fixture = get_launch_test_fixture(item) fixturename = fixture.__name__ scope = fixture._pytestfixturefunction.scope is_shutdown = has_shutdown_kwarg(item) - items = generate_test_items(collector, name, obj, fixturename, is_shutdown, False) + items = generate_test_items( + collector, name, obj, fixturename, is_shutdown=is_shutdown, needs_renaming=False) if need_shutdown_test_item(obj) and scope != 'function': # for function scope we only need one shutdown item # if not we're going to use two event loops!!! - shutdown_items = generate_test_items(collector, name, obj, fixturename, True, True) + shutdown_items = generate_test_items( + collector, name, obj, fixturename, is_shutdown=True, needs_renaming=True) for item, shutdown_item in zip(items, shutdown_items): item._launch_pytest_shutdown_item = shutdown_item items.extend(shutdown_items) From c01d9141f5ed5e6e63565ba6727ddea352efbbd6 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 26 Oct 2021 11:30:22 -0300 Subject: [PATCH 35/48] fix bug Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/test/launch_pytest/test_plugin.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/launch_pytest/test/launch_pytest/test_plugin.py b/launch_pytest/test/launch_pytest/test_plugin.py index d98dc4366..ea4555a2e 100644 --- a/launch_pytest/test/launch_pytest/test_plugin.py +++ b/launch_pytest/test/launch_pytest/test_plugin.py @@ -180,7 +180,7 @@ async def test_case(): result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) -def test_fixture_kwarg_is_mandatory(pytester): +def test_fixture_kwarg_is_mandatory(LineMatcher, pytester): pytester.makepyfile("""\ from launch import LaunchDescription import launch_pytest @@ -200,7 +200,10 @@ def test_case(): result = pytester.runpytest() result.assert_outcomes(skipped=1) - result.stdout.re_match_lines(['.*"fixture" keyword argument is required .*']) + # Warnings can appear in both stdout or stderr, depending on pytest capture mode. + # Test for a match in any of both. + LineMatcher(result.outlines + result.errlines).re_match_lines( + ['.*"fixture" keyword argument is required .*']) def test_generator_shutdown_kwarg_true(pytester): From 3277df5eb28ef0e247563586f73418065219a1f1 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 26 Oct 2021 11:54:29 -0300 Subject: [PATCH 36/48] fix build issue Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/setup.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/launch_pytest/setup.py b/launch_pytest/setup.py index f198c763a..f49dc58dc 100644 --- a/launch_pytest/setup.py +++ b/launch_pytest/setup.py @@ -1,4 +1,5 @@ import glob +from pathlib import Path from setuptools import find_packages from setuptools import setup @@ -14,7 +15,15 @@ ('share/ament_index/resource_index/packages', [f'resource/{package_name}']), (f'lib/{package_name}', glob.glob('example_processes/**')), (f'share/{package_name}', ['package.xml']), - (f'share/{package_name}/examples', glob.glob(f'test/{package_name}/examples/[!_]**')), + ( + f'share/{package_name}/examples', + [x for x in glob.glob( + f'test/{package_name}/examples/[!_]*') if Path(x).is_file()] + ), + ( + f'share/{package_name}/examples/executables', + glob.glob(f'test/{package_name}/examples/executables/[!_]*') + ), ], entry_points={ 'pytest11': ['launch_pytest = launch_pytest.plugin'], From 510d6d2aa3db0f3bdca3aadbfa314a41abcaeb6d Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Tue, 26 Oct 2021 12:04:39 -0300 Subject: [PATCH 37/48] please linters Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 2 -- launch_pytest/launch_pytest/plugin.py | 5 ----- 2 files changed, 7 deletions(-) diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index 6d8e89548..cd74e2d7d 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -38,7 +38,6 @@ def finalize_launch_service(launch_service, eprefix='', auto_shutdown=True): def get_launch_service_fixture(*, scope='function', overridable=True): """Return a launch service fixture.""" - @pytest.fixture(scope=scope) def launch_service(event_loop): """Create an instance of the launch service for each test case.""" @@ -53,7 +52,6 @@ def launch_service(event_loop): def get_launch_context_fixture(*, scope='function', overridable=True): """Return a launch service fixture.""" - @pytest.fixture(scope=scope) def launch_context(launch_service): """Create an instance of the launch service for each test case.""" diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 815198d90..10e7099c9 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -269,7 +269,6 @@ def get_fixture_params(item): @pytest.mark.trylast def pytest_collection_modifyitems(session, config, items): """Move shutdown tests after normal tests.""" - def enumerate_reversed(sequence): # reversed(enumerate(sequence)), doesn't work # here a little generator for that @@ -367,7 +366,6 @@ def run_until_complete(loop, future_like): def wrap_coroutine(func, event_loop, before_test): """Return a sync wrapper around an async function to be executed in the event loop.""" - @functools.wraps(func) def inner(**kwargs): if before_test is not None: @@ -381,7 +379,6 @@ def inner(**kwargs): def wrap_func(func, event_loop, before_test): """Return a wrapper that runs the test in a separate thread while driving the event loop.""" - @functools.wraps(func) def inner(**kwargs): if before_test is not None: @@ -424,7 +421,6 @@ def inner(**kwargs): def wrap_generator_fscope(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for a generator function.""" - @functools.wraps(func) def inner(**kwargs): gen = func(**kwargs) @@ -479,7 +475,6 @@ def inner(**kwargs): def wrap_asyncgen_fscope(func, event_loop, on_shutdown): """Return wrappers for the normal test and the teardown test for an async gen function.""" - @functools.wraps(func) def inner(**kwargs): agen = func(**kwargs) From 5a593e71e6b3e4309ae08a4e17e65c851aa27008 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 27 Oct 2021 12:39:19 -0300 Subject: [PATCH 38/48] Add a readme Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/README.md | 152 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 152 insertions(+) create mode 100644 launch_pytest/README.md diff --git a/launch_pytest/README.md b/launch_pytest/README.md new file mode 100644 index 000000000..54ee950b8 --- /dev/null +++ b/launch_pytest/README.md @@ -0,0 +1,152 @@ +# launch_pytest + +This tool is a framework for launch integration testing. For example: + + * The exit codes of all processes are available to the tests. + * Tests can check that all processes shut down normally, or with specific exit codes. + * Tests can fail when a process dies unexpectedly. + * The stdout and stderr of all processes are available to the tests. + * The command-line used to launch the processes are available to the tests. + * Some tests run concurrently with the launch and can interact with the running processes. + +## Differences with launch_testing + +launch_testing is an standalone testing tool, which lacks many features: + * It's impossible to filter test cases by name and run only some. + * It's impossible to mark a test as skipped or xfail. + * The error reporting of the tool was custom, and the output wasn't as nice as the output + generated by other testing frameworks as unittest and pytest. + +launch_pytest is a really simple pytest plugin leveraging pytest fixtures to manage a launch service lifetime easily. + +## Quick start example + +Start with the [`pytest_hello_world.py`](test/launch_pytest/examples/pytest_hello_world.py) example. + +Run the example by doing: + +```sh +python3 -m pytest test/launch_pytest/examples/pytest_hello_world.py +``` + +The `launch_pytest` plugin will launch the nodes found in the `launch_descripture` fixture, run the tests from the `test_read_stdout()` class, shut down the launched nodes, and then run the statements after the `yield` statement in `test_read_stdout()`. + +#### launch_pytest fixtures + +```python +@launch_pytest.fixture +def launch_description(hello_world_proc): + """Launch a simple process to print 'hello_world'.""" + return launch.LaunchDescription([ + hello_world_proc, + # Tell launch when to start the test + # If no ReadyToTest action is added, one will be appended automatically. + launch_pytest.actions.ReadyToTest() + ]) +``` + +A `@launch_pytest.fixture` function should return a `launch.LaunchDescription` object, or a sequence of objects which first item is a `launch.LaunchDescription`. +This launch description will be used in all tests with a mark `@pytest.mark.launch(fixture=)`, in this case `=launch_description`. + +The launch description can include a `ReadyToTest` action to signal to the test framework that it's safe to start the active tests. +If one isn't included, a `ReadyToTest` action will be appended at the end. + +launch_pytest fixtures can have `module`, `class` or `function` scope, the default is `function`. +For example: + +```python +@launch_pytest.fixture(scope=my_scope) +def my_fixture(): + return LaunchDescription(...) + +@pytest.mark.launch(fixture=my_fixture) +def test_case_1(): + pass + +@pytest.mark.launch(fixture=my_fixture) +def test_case_2(): + pass +``` + +If `my_scope=function`, the following happens: + +- A launch service using the `LaunchDescription` returned by `my_fixture()` is started. +- `test_case_1()` is run. +- The launch service is shutdown. +- Another launch service using the `LaunchDescription` returned by `my_fixture()` is started, `my_fixture()` is called again. +- `test_case_2()` is run. +- The launch service is shutdown. + +Whereas when `my_scope=module`, `test case_2()` will run immediately after `test case_1()`, concurrently with the same launch service. + +It's not recommended to mix fixtures with `module` scope with fixtures of `class`/`function` scope in the same file. +It's not recommended to use fixtures with scope larger than `module`. +A test shouldn't depend on more than one `launch_pytest` fixture. +Neither of the three things above automatically generates an error in the current `launch_pytest` implementation, but future versions might. + +#### Active Tests and shutdwon tests + +Test cases marked with `@pytest.mark.launch` will be run concurrently with the launch service or after launch shutdown, depending on the object being marked and the mark arguements. + +- functions: Functions marked with `@pytest.mark.launch` will run concurrently with the launch service, except when `shutdown=True` is passed as an argument to the decorator. + +```python +@pytest.mark.launch(fixture=my_ld_fixture) +def normal_test_case(): + pass + +@pytest.mark.launch(fixture=my_ld_fixture, shutdown=True) +def shutdown_test_case(): + pass +``` + +- coroutine functions: The same rules as normal functions apply. + Coroutines will be run in the same event loop as the launch description, whereas normal functions run concurrently in another thread. + +```python +@pytest.mark.launch(fixture=my_ld_fixture) +async def normal_test_case(): + pass + +@pytest.mark.launch(fixture=my_ld_fixture, shutdown=True) +async def shutdown_test_case(): + pass +``` + +- generators: The first time the generator is called it runs concurrently with the launch service. + The generator will be resumed after the launch service is shutdown. + i.e. This allows to write a test that has a step that runs concurrently with the service and one + that runs after shutdown easily. + The yielded value is ignored. + If the generator doesn't stop iteration after being resumed for a second time, the test will fail. + Passing a `shutdown` argument to the decorator is not allowed in this case. + +```python +@pytest.mark.launch(fixture=my_ld_fixture) +def normal_test_case(): + assert True + yield + assert True +``` + +- async generators: The same rules as for generators apply here as well. + The only difference between the two is that async generator will run in the same event loop as the launch service, whereas a generator will run concurrently in another thread. + +```python +@pytest.mark.launch(fixture=my_ld_fixture) +async def normal_test_case(): + assert True + yield + assert True +``` + +## Fixtures + +The `launch_pytest` plugin will provide the following fixtures. + +- launch_service: The launch service being used to run the tests. + It will have the same scope as the launch_pytest fixture with wider scope in the module. +- launch_context: The launch context being used to run the tests. + It will have the same scope as the launch_pytest fixture with wider scope in the module. +- event_loop: The event loop being used to run the launch service and to run async tests. + It will have the same scope as the launch_pytest fixture with wider scope in the module. From 8de34a021c001cc2993dd0af0fe3c8401406896a Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 27 Oct 2021 12:43:00 -0300 Subject: [PATCH 39/48] fix linters Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/plugin.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 10e7099c9..60c60ffb9 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -40,7 +40,7 @@ except ImportError: # Pytest 4.1.0 removes the transfer_marker api (#104) def transfer_markers(*args, **kwargs): # noqa - """Noop when over pytest 4.1.0""" + # Noop when over pytest 4.1.0 pass @@ -221,7 +221,7 @@ def pytest_pycollect_makeitem(collector, name, obj): # return an item with a warning that's going to be skipped msg = ( '"fixture" keyword argument is required in a pytest.mark.launch() ' - f'decorator') + 'decorator') item.warn(LaunchTestWarning(msg)) item.add_marker(pytest.mark.skip(msg)) return [item] From 0bb5147a723f82aa990fe9ce9433a17dfe6fb23b Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 27 Oct 2021 15:38:48 -0300 Subject: [PATCH 40/48] Address peer review comments Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/launch_pytest/README.md b/launch_pytest/README.md index 54ee950b8..d94687843 100644 --- a/launch_pytest/README.md +++ b/launch_pytest/README.md @@ -15,7 +15,7 @@ launch_testing is an standalone testing tool, which lacks many features: * It's impossible to filter test cases by name and run only some. * It's impossible to mark a test as skipped or xfail. * The error reporting of the tool was custom, and the output wasn't as nice as the output - generated by other testing frameworks as unittest and pytest. + generated by other testing frameworks such as unittest and pytest. launch_pytest is a really simple pytest plugin leveraging pytest fixtures to manage a launch service lifetime easily. @@ -29,7 +29,7 @@ Run the example by doing: python3 -m pytest test/launch_pytest/examples/pytest_hello_world.py ``` -The `launch_pytest` plugin will launch the nodes found in the `launch_descripture` fixture, run the tests from the `test_read_stdout()` class, shut down the launched nodes, and then run the statements after the `yield` statement in `test_read_stdout()`. +The `launch_pytest` plugin will launch the nodes found in the `launch_description` fixture, run the tests from the `test_read_stdout()` class, shut down the launched nodes, and then run the statements after the `yield` statement in `test_read_stdout()`. #### launch_pytest fixtures @@ -45,13 +45,14 @@ def launch_description(hello_world_proc): ]) ``` -A `@launch_pytest.fixture` function should return a `launch.LaunchDescription` object, or a sequence of objects which first item is a `launch.LaunchDescription`. +A `@launch_pytest.fixture` function should return a `launch.LaunchDescription` object, or a sequence of objects whose first item is a `launch.LaunchDescription`. This launch description will be used in all tests with a mark `@pytest.mark.launch(fixture=)`, in this case `=launch_description`. The launch description can include a `ReadyToTest` action to signal to the test framework that it's safe to start the active tests. If one isn't included, a `ReadyToTest` action will be appended at the end. -launch_pytest fixtures can have `module`, `class` or `function` scope, the default is `function`. +`launch_pytest` fixtures can have `module`, `class` or `function` scope. +The default is `function`. For example: ```python From 1b0096fdd608691bb15b709bdcb2fbefeeb6e29f Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 27 Oct 2021 15:48:59 -0300 Subject: [PATCH 41/48] Improve code attribution Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 7 +++++-- launch_pytest/launch_pytest/plugin.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index cd74e2d7d..103214633 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -64,8 +64,11 @@ def launch_context(launch_service): def get_event_loop_fixture(*, scope='function', overridable=True): """Return an event loop fixture.""" - # Adapted from https://github.com/pytest-dev/pytest-asyncio, - # see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. + # Adapted from: + # https://github.com/pytest-dev/pytest-asyncio/blob/f21e0da345f877755b89ff87b6dcea70815b4497/pytest_asyncio/plugin.py#L224-L229 + # https://github.com/pytest-dev/pytest-asyncio/blob/f21e0da345f877755b89ff87b6dcea70815b4497/pytest_asyncio/plugin.py#L93-L101 + # https://github.com/pytest-dev/pytest-asyncio/blob/f21e0da345f877755b89ff87b6dcea70815b4497/pytest_asyncio/plugin.py#L84. + # See their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @pytest.fixture(scope=scope) def event_loop(): """Create an event loop instance for each test case.""" diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index 60c60ffb9..c52f3e7ab 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -202,8 +202,9 @@ def generate_test_items(collector, name, obj, fixturename, *, is_shutdown, needs return items -# Adapted from https://github.com/pytest-dev/pytest-asyncio, -# see their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. +# Part of this function was adapted from +# https://github.com/pytest-dev/pytest-asyncio/blob/f21e0da345f877755b89ff87b6dcea70815b4497/pytest_asyncio/plugin.py#L37-L50. +# See their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @pytest.mark.tryfirst def pytest_pycollect_makeitem(collector, name, obj): """Collect coroutine based launch tests.""" From 5bd6f323c6fb481a5d3fde474b88f0cc9c5865f8 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Wed, 27 Oct 2021 17:02:35 -0300 Subject: [PATCH 42/48] nit Signed-off-by: Ivan Santiago Paunovic Co-authored-by: Jacob Perron --- launch_pytest/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch_pytest/README.md b/launch_pytest/README.md index d94687843..538f5d4fe 100644 --- a/launch_pytest/README.md +++ b/launch_pytest/README.md @@ -1,6 +1,6 @@ # launch_pytest -This tool is a framework for launch integration testing. For example: +This is a framework for launch integration testing. For example: * The exit codes of all processes are available to the tests. * Tests can check that all processes shut down normally, or with specific exit codes. From af0c896768f534563127188bc1ca6acf7c06c34f Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 29 Oct 2021 10:46:29 -0300 Subject: [PATCH 43/48] logs to debug on windows without a VM Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 7 +++++++ launch_pytest/test/launch_pytest/tools/test_process.py | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index 103214633..178ab0e5a 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -72,19 +72,26 @@ def get_event_loop_fixture(*, scope='function', overridable=True): @pytest.fixture(scope=scope) def event_loop(): """Create an event loop instance for each test case.""" + import sys + import traceback loop = asyncio.get_event_loop_policy().new_event_loop() policy = asyncio.get_event_loop_policy() try: old_loop = policy.get_event_loop() if old_loop is not loop: old_loop.close() + print('bye event loop 2', file=sys.stderr) + traceback.print_stack() except RuntimeError: # Swallow this, since it's probably bad event loop hygiene. pass policy.set_event_loop(loop) + print('new event loop', file=sys.stderr) yield loop loop.close() asyncio.set_event_loop_policy(None) + print('bye event loop 1', file=sys.stderr) + traceback.print_stack() if overridable: event_loop._launch_pytest_overridable_fixture = True event_loop._launch_pytest_fixture_scope = scope diff --git a/launch_pytest/test/launch_pytest/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py index 396ac285d..fd2113629 100644 --- a/launch_pytest/test/launch_pytest/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -27,6 +27,7 @@ time.sleep(5) """ +import sys @pytest.fixture def dut(): @@ -39,6 +40,7 @@ def dut(): @launch_pytest.fixture def launch_description(dut): + print('new launch description', file=sys.stderr) return launch.LaunchDescription([ dut, ]) @@ -46,6 +48,7 @@ def launch_description(dut): @pytest.mark.launch(fixture=launch_description) async def test_async_process_tools(dut, launch_context): + print('test async', file=sys.stderr) await tools.wait_for_start(launch_context, dut, timeout=10) def check_output(output): assert output == 'hello\n' await tools.wait_for_output( @@ -59,6 +62,7 @@ def check_stderr(err): assert err == 'world\n' @pytest.mark.launch(fixture=launch_description) def test_sync_process_tools(dut, launch_context): + print('test sync', file=sys.stderr) tools.wait_for_start_sync(launch_context, dut, timeout=10) def check_output(output): assert output == 'hello\n' tools.wait_for_output_sync( From ee4743972b51fd0815af3f5dadb66411a4a7f3d8 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 29 Oct 2021 13:36:47 -0300 Subject: [PATCH 44/48] Revert "logs to debug on windows without a VM" This reverts commit af0c896768f534563127188bc1ca6acf7c06c34f. Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 7 ------- launch_pytest/test/launch_pytest/tools/test_process.py | 4 ---- 2 files changed, 11 deletions(-) diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index 178ab0e5a..103214633 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -72,26 +72,19 @@ def get_event_loop_fixture(*, scope='function', overridable=True): @pytest.fixture(scope=scope) def event_loop(): """Create an event loop instance for each test case.""" - import sys - import traceback loop = asyncio.get_event_loop_policy().new_event_loop() policy = asyncio.get_event_loop_policy() try: old_loop = policy.get_event_loop() if old_loop is not loop: old_loop.close() - print('bye event loop 2', file=sys.stderr) - traceback.print_stack() except RuntimeError: # Swallow this, since it's probably bad event loop hygiene. pass policy.set_event_loop(loop) - print('new event loop', file=sys.stderr) yield loop loop.close() asyncio.set_event_loop_policy(None) - print('bye event loop 1', file=sys.stderr) - traceback.print_stack() if overridable: event_loop._launch_pytest_overridable_fixture = True event_loop._launch_pytest_fixture_scope = scope diff --git a/launch_pytest/test/launch_pytest/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py index fd2113629..396ac285d 100644 --- a/launch_pytest/test/launch_pytest/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -27,7 +27,6 @@ time.sleep(5) """ -import sys @pytest.fixture def dut(): @@ -40,7 +39,6 @@ def dut(): @launch_pytest.fixture def launch_description(dut): - print('new launch description', file=sys.stderr) return launch.LaunchDescription([ dut, ]) @@ -48,7 +46,6 @@ def launch_description(dut): @pytest.mark.launch(fixture=launch_description) async def test_async_process_tools(dut, launch_context): - print('test async', file=sys.stderr) await tools.wait_for_start(launch_context, dut, timeout=10) def check_output(output): assert output == 'hello\n' await tools.wait_for_output( @@ -62,7 +59,6 @@ def check_stderr(err): assert err == 'world\n' @pytest.mark.launch(fixture=launch_description) def test_sync_process_tools(dut, launch_context): - print('test sync', file=sys.stderr) tools.wait_for_start_sync(launch_context, dut, timeout=10) def check_output(output): assert output == 'hello\n' tools.wait_for_output_sync( From 6c4f4d8377a3f40b563a1e216c646bbaa4f2fe09 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 29 Oct 2021 13:48:55 -0300 Subject: [PATCH 45/48] fix windows (hopefully) Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/test/launch_pytest/tools/test_process.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/launch_pytest/test/launch_pytest/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py index 396ac285d..958fb38e4 100644 --- a/launch_pytest/test/launch_pytest/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -12,6 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import sys + import launch import launch_pytest from launch_pytest import tools @@ -31,7 +33,7 @@ @pytest.fixture def dut(): return launch.actions.ExecuteProcess( - cmd=['python3', '-c', PYTHON_SCRIPT], + cmd=[sys.executable, '-c', PYTHON_SCRIPT], cached_output=True, output='screen' ) From 0279cf7f53f2f1268deb5da965211a7eeb4c7f03 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 29 Oct 2021 15:46:23 -0300 Subject: [PATCH 46/48] fix windows Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/test/launch_pytest/tools/test_process.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/launch_pytest/test/launch_pytest/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py index 958fb38e4..94f087d5e 100644 --- a/launch_pytest/test/launch_pytest/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os import sys import launch @@ -49,11 +50,11 @@ def launch_description(dut): @pytest.mark.launch(fixture=launch_description) async def test_async_process_tools(dut, launch_context): await tools.wait_for_start(launch_context, dut, timeout=10) - def check_output(output): assert output == 'hello\n' + def check_output(output): assert output == f'hello{os.linesep}' await tools.wait_for_output( launch_context, dut, check_output, timeout=10) - def check_stderr(err): assert err == 'world\n' + def check_stderr(err): assert err == f'world{os.linesep}' await tools.wait_for_stderr( launch_context, dut, check_stderr, timeout=10) await tools.wait_for_exit(launch_context, dut, timeout=10) @@ -62,11 +63,11 @@ def check_stderr(err): assert err == 'world\n' @pytest.mark.launch(fixture=launch_description) def test_sync_process_tools(dut, launch_context): tools.wait_for_start_sync(launch_context, dut, timeout=10) - def check_output(output): assert output == 'hello\n' + def check_output(output): assert output == f'hello{os.linesep}' tools.wait_for_output_sync( launch_context, dut, check_output, timeout=10) - def check_stderr(err): assert err == 'world\n' + def check_stderr(err): assert err == f'world{os.linesep}' tools.wait_for_stderr_sync( launch_context, dut, check_stderr, timeout=10) tools.wait_for_exit_sync(launch_context, dut, timeout=10) From a9300bf71bf807b5eccaa350dbc94aa0ec3aa255 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Fri, 29 Oct 2021 17:08:43 -0300 Subject: [PATCH 47/48] compatibility with apt provided pytest Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/launch_pytest/fixture.py | 2 +- launch_pytest/launch_pytest/plugin.py | 12 ++- launch_pytest/pytest.ini | 2 + .../test/launch_pytest/test_plugin.py | 91 +++++++++++-------- 4 files changed, 64 insertions(+), 43 deletions(-) create mode 100644 launch_pytest/pytest.ini diff --git a/launch_pytest/launch_pytest/fixture.py b/launch_pytest/launch_pytest/fixture.py index 103214633..aec03c6ba 100644 --- a/launch_pytest/launch_pytest/fixture.py +++ b/launch_pytest/launch_pytest/fixture.py @@ -141,7 +141,7 @@ def decorator(fixture_function): 'shutdown_when_idle': shutdown_when_idle, 'auto_shutdown': auto_shutdown, } - return pytest.fixture(fixture_function, *args, **kwargs) + return pytest.fixture(*args, **kwargs)(fixture_function) if decorated is None: return decorator return decorator(decorated) diff --git a/launch_pytest/launch_pytest/plugin.py b/launch_pytest/launch_pytest/plugin.py index c52f3e7ab..57b7a5294 100644 --- a/launch_pytest/launch_pytest/plugin.py +++ b/launch_pytest/launch_pytest/plugin.py @@ -202,6 +202,14 @@ def generate_test_items(collector, name, obj, fixturename, *, is_shutdown, needs return items +def from_parent(cls, *args, **kwargs): + # Compatibility with old pytest. + # from_parent() was added in pytest 5.4 + if hasattr(cls, 'from_parent'): + return cls.from_parent(*args, **kwargs) + return cls(*args, **kwargs) + + # Part of this function was adapted from # https://github.com/pytest-dev/pytest-asyncio/blob/f21e0da345f877755b89ff87b6dcea70815b4497/pytest_asyncio/plugin.py#L37-L50. # See their license https://github.com/pytest-dev/pytest-asyncio/blob/master/LICENSE. @@ -209,13 +217,13 @@ def generate_test_items(collector, name, obj, fixturename, *, is_shutdown, needs def pytest_pycollect_makeitem(collector, name, obj): """Collect coroutine based launch tests.""" if collector.funcnamefilter(name) and is_valid_test_item(obj): - item = pytest.Function.from_parent(collector, name=name) + item = from_parent(pytest.Function, parent=collector, name=name) # Due to how pytest test collection works, module-level pytestmarks # are applied after the collection step. Since this is the collection # step, we look ourselves. transfer_markers(obj, item.cls, item.module) - item = pytest.Function.from_parent(collector, name=name) # To reload keywords. + item = from_parent(pytest.Function, parent=collector, name=name) # To reload keywords. if is_launch_test(item): if not is_launch_test_mark_valid(item): diff --git a/launch_pytest/pytest.ini b/launch_pytest/pytest.ini new file mode 100644 index 000000000..fe55d2ed6 --- /dev/null +++ b/launch_pytest/pytest.ini @@ -0,0 +1,2 @@ +[pytest] +junit_family=xunit2 diff --git a/launch_pytest/test/launch_pytest/test_plugin.py b/launch_pytest/test/launch_pytest/test_plugin.py index ea4555a2e..deb9fbd12 100644 --- a/launch_pytest/test/launch_pytest/test_plugin.py +++ b/launch_pytest/test/launch_pytest/test_plugin.py @@ -13,8 +13,8 @@ # limitations under the License. -def test_launch_fixture_is_not_a_launch_description(pytester): - pytester.makepyfile("""\ +def test_launch_fixture_is_not_a_launch_description(testdir): + testdir.makepyfile("""\ import launch_pytest import pytest @@ -27,14 +27,17 @@ def test_case(): assert True """) - result = pytester.runpytest() - - result.assert_outcomes(errors=1) + result = testdir.runpytest() + try: + result.assert_outcomes(errors=1) + except TypeError: + # Compatibility to pytest older than 6.0 + result.assert_outcomes(error=1) result.stdout.re_match_lines(['.*must be either a launch description.*']) -def test_launch_fixture_is_not_a_sequence_starting_with_a_ld(pytester): - pytester.makepyfile("""\ +def test_launch_fixture_is_not_a_sequence_starting_with_a_ld(testdir): + testdir.makepyfile("""\ import launch_pytest import pytest @@ -47,14 +50,18 @@ def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() - result.assert_outcomes(errors=1) + try: + result.assert_outcomes(errors=1) + except TypeError: + # Compatibility to pytest older than 6.0 + result.assert_outcomes(error=1) result.stdout.re_match_lines(['.*must be either a launch description.*']) -def test_multiple_ready_to_test_actions(pytester): - pytester.makepyfile("""\ +def test_multiple_ready_to_test_actions(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -70,14 +77,18 @@ def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() - result.assert_outcomes(errors=1) + try: + result.assert_outcomes(errors=1) + except TypeError: + # Compatibility to pytest older than 6.0 + result.assert_outcomes(error=1) result.stdout.re_match_lines(['.*only one ReadyToTest action.*']) -def test_generator_yields_twice(pytester): - pytester.makepyfile("""\ +def test_generator_yields_twice(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -96,14 +107,14 @@ def test_case(): yield """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1) result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) -def test_generator_yields_twice_module_scope(pytester): - pytester.makepyfile("""\ +def test_generator_yields_twice_module_scope(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -122,14 +133,14 @@ def test_case(): yield """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(passed=1, failed=1) result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) -def test_asyncgenerator_yields_twice(pytester): - pytester.makepyfile("""\ +def test_asyncgenerator_yields_twice(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -148,14 +159,14 @@ async def test_case(): yield """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1) result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) -def test_asyncgenerator_yields_twice_module_scope(pytester): - pytester.makepyfile("""\ +def test_asyncgenerator_yields_twice_module_scope(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -174,14 +185,14 @@ async def test_case(): yield """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(passed=1, failed=1) result.stdout.re_match_lines(['.*must stop iteration after yielding once.*']) -def test_fixture_kwarg_is_mandatory(LineMatcher, pytester): - pytester.makepyfile("""\ +def test_fixture_kwarg_is_mandatory(LineMatcher, testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -197,7 +208,7 @@ def test_case(): pass """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(skipped=1) # Warnings can appear in both stdout or stderr, depending on pytest capture mode. @@ -206,8 +217,8 @@ def test_case(): ['.*"fixture" keyword argument is required .*']) -def test_generator_shutdown_kwarg_true(pytester): - pytester.makepyfile("""\ +def test_generator_shutdown_kwarg_true(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -225,14 +236,14 @@ def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1) result.stdout.re_match_lines(['.*generator or async generator.* shutdown=True.*']) -def test_async_generator_shutdown_kwarg_true(pytester): - pytester.makepyfile("""\ +def test_async_generator_shutdown_kwarg_true(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -250,14 +261,14 @@ async def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1) result.stdout.re_match_lines(['.*generator or async generator.* shutdown=True.*']) -def test_generator_with_pre_shutdown_failure(pytester): - pytester.makepyfile("""\ +def test_generator_with_pre_shutdown_failure(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -275,13 +286,13 @@ def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1, skipped=1) -def test_async_generator_with_pre_shutdown_failure(pytester): - pytester.makepyfile("""\ +def test_async_generator_with_pre_shutdown_failure(testdir): + testdir.makepyfile("""\ from launch import LaunchDescription import launch_pytest import pytest @@ -299,6 +310,6 @@ async def test_case(): assert True """) - result = pytester.runpytest() + result = testdir.runpytest() result.assert_outcomes(failed=1, skipped=1) From cf66f4b8cc5515c00cb5cd0a411c92edaa093647 Mon Sep 17 00:00:00 2001 From: Ivan Santiago Paunovic Date: Mon, 1 Nov 2021 15:01:31 -0300 Subject: [PATCH 48/48] address peer review comment Signed-off-by: Ivan Santiago Paunovic --- launch_pytest/test/launch_pytest/tools/test_process.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/launch_pytest/test/launch_pytest/tools/test_process.py b/launch_pytest/test/launch_pytest/tools/test_process.py index 94f087d5e..f3281dd91 100644 --- a/launch_pytest/test/launch_pytest/tools/test_process.py +++ b/launch_pytest/test/launch_pytest/tools/test_process.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import sys import launch @@ -50,11 +49,11 @@ def launch_description(dut): @pytest.mark.launch(fixture=launch_description) async def test_async_process_tools(dut, launch_context): await tools.wait_for_start(launch_context, dut, timeout=10) - def check_output(output): assert output == f'hello{os.linesep}' + def check_output(output): assert output.splitlines() == ['hello'] await tools.wait_for_output( launch_context, dut, check_output, timeout=10) - def check_stderr(err): assert err == f'world{os.linesep}' + def check_stderr(err): assert err.splitlines() == ['world'] await tools.wait_for_stderr( launch_context, dut, check_stderr, timeout=10) await tools.wait_for_exit(launch_context, dut, timeout=10) @@ -63,11 +62,11 @@ def check_stderr(err): assert err == f'world{os.linesep}' @pytest.mark.launch(fixture=launch_description) def test_sync_process_tools(dut, launch_context): tools.wait_for_start_sync(launch_context, dut, timeout=10) - def check_output(output): assert output == f'hello{os.linesep}' + def check_output(output): assert output.splitlines() == ['hello'] tools.wait_for_output_sync( launch_context, dut, check_output, timeout=10) - def check_stderr(err): assert err == f'world{os.linesep}' + def check_stderr(err): assert err.splitlines() == ['world'] tools.wait_for_stderr_sync( launch_context, dut, check_stderr, timeout=10) tools.wait_for_exit_sync(launch_context, dut, timeout=10)