From bffcd04ca4357f3d382cb5b3d69c6956f9ae9bcb Mon Sep 17 00:00:00 2001 From: Matt Wozniski Date: Fri, 2 Aug 2024 20:11:31 -0400 Subject: [PATCH] tests: Skip snapshot tests if Textual is outdated Have our test suite check the version of Textual and pytest-textual-snapshot that it's running with, and automatically disable the Textual snapshot tests if we can determine statically that they will go on to fail. This is a better experience for 3rd party package maintainers and simplifies our handling of Python versions that Textual has already dropped support for. This doesn't negatively affect our CI, since we keep our dependencies live at head in CI. Signed-off-by: Matt Wozniski --- requirements-test.txt | 1 + setup.py | 1 + tests/conftest.py | 53 ++++++++++++++++++++++++++++++-- tests/unit/test_tree_reporter.py | 23 +++----------- tests/unit/test_tui_reporter.py | 23 +++----------- 5 files changed, 62 insertions(+), 39 deletions(-) diff --git a/requirements-test.txt b/requirements-test.txt index 2829f4dad1..1030ae6375 100644 --- a/requirements-test.txt +++ b/requirements-test.txt @@ -8,3 +8,4 @@ setuptools; python_version >= '3.12' pkgconfig pytest-textual-snapshot textual >= 0.43, != 0.65.2, != 0.66 +packaging diff --git a/setup.py b/setup.py index 25425c0cc4..005ac84489 100644 --- a/setup.py +++ b/setup.py @@ -120,6 +120,7 @@ def build_js_files(self): "setuptools; python_version >= '3.12'", "pytest-textual-snapshot", "textual >= 0.43, != 0.65.2, != 0.66", + "packaging", ] benchmark_requires = [ diff --git a/tests/conftest.py b/tests/conftest.py index 4c379cadba..dffca914f4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2,6 +2,12 @@ import sys import pytest +from packaging import version + +SNAPSHOT_MINIMUM_VERSIONS = { + "textual": "0.73", + "pytest-textual-snapshot": "1.0", +} @pytest.fixture @@ -13,7 +19,48 @@ def free_port(): return port_number -if sys.version_info < (3, 8): - # Ignore unused Textual snapshots on Python 3.7 - def pytest_configure(config): +def _snapshot_skip_reason(): + if sys.version_info < (3, 8): + # Every version available for 3.7 is too old + return f"snapshot tests require textual>={SNAPSHOT_MINIMUM_VERSIONS['textual']}" + + from importlib import metadata # Added in 3.8 + + for lib, min_ver in SNAPSHOT_MINIMUM_VERSIONS.items(): + try: + ver = version.parse(metadata.version(lib)) + except ImportError: + return f"snapshot tests require {lib} but it is not installed" + + if ver < version.parse(min_ver): + return f"snapshot tests require {lib}>={min_ver} but {ver} is installed" + + return None + + +def pytest_configure(config): + if config.option.update_snapshots: + from importlib import metadata # Added in 3.8 + + for lib, min_ver in SNAPSHOT_MINIMUM_VERSIONS.items(): + ver = version.parse(metadata.version(lib)) + if ver != version.parse(min_ver): + pytest.exit( + f"snapshots must be generated with {lib}=={min_ver}" + f" or SNAPSHOT_MINIMUM_VERSIONS must be updated to {ver}" + f" in {__file__}" + ) + return + + reason = _snapshot_skip_reason() + if reason: + config.issue_config_time_warning(UserWarning(reason), stacklevel=2) config.option.warn_unused_snapshots = True + + +def pytest_collection_modifyitems(config, items): + reason = _snapshot_skip_reason() + if reason: + for item in items: + if "snap_compare" in item.fixturenames: + item.add_marker(pytest.mark.skip(reason=reason)) diff --git a/tests/unit/test_tree_reporter.py b/tests/unit/test_tree_reporter.py index 1b69a949b5..66a8a0c6ee 100644 --- a/tests/unit/test_tree_reporter.py +++ b/tests/unit/test_tree_reporter.py @@ -1,4 +1,3 @@ -import sys from dataclasses import dataclass from textwrap import dedent from typing import Any @@ -1534,15 +1533,6 @@ def test_render_runs_the_app(self): @pytest.fixture def compare(monkeypatch, tmp_path, snap_compare): - # The snapshots we've generated using current versions of Textual aren't - # expected to match anymore on Python 3.7, as Textual dropped support for - # Python 3.7 in the 0.44 release. However, we'd still like to run our - # snapshot tests on Python 3.7, to confirm that no unexpected exceptions - # occur and that the app doesn't crash. So, allow `snap_compare()` to drive - # the application, but always return `True` on Python 3.7 as long as no - # exception was raised. - succeed_even_if_mismatched = sys.version_info < (3, 8) - def compare_impl( allocations: Iterator[AllocationRecord], press: Iterable[str] = (), @@ -1561,14 +1551,11 @@ def compare_impl( with monkeypatch.context() as app_patch: app_patch.setitem(globals(), app_global, app) tmp_main.write_text(f"from {__name__} import {app_global} as app") - return ( - snap_compare( - str(tmp_main), - press=press, - terminal_size=terminal_size, - run_before=run_before, - ) - or succeed_even_if_mismatched + return snap_compare( + str(tmp_main), + press=press, + terminal_size=terminal_size, + run_before=run_before, ) yield compare_impl diff --git a/tests/unit/test_tui_reporter.py b/tests/unit/test_tui_reporter.py index 985f369fd9..3e7e1496c8 100644 --- a/tests/unit/test_tui_reporter.py +++ b/tests/unit/test_tui_reporter.py @@ -1,6 +1,5 @@ import asyncio import datetime -import sys from io import StringIO from typing import Awaitable from typing import Callable @@ -103,15 +102,6 @@ def get_current_snapshot( def compare(monkeypatch, tmp_path, snap_compare): monkeypatch.setattr(memray.reporters.tui, "datetime", FakeDatetime) - # The snapshots we've generated using current versions of Textual aren't - # expected to match anymore on Python 3.7, as Textual dropped support for - # Python 3.7 in the 0.44 release. However, we'd still like to run our - # snapshot tests on Python 3.7, to confirm that no unexpected exceptions - # occur and that the app doesn't crash. So, allow `snap_compare()` to drive - # the application, but always return `True` on Python 3.7 as long as no - # exception was raised. - succeed_even_if_mismatched = sys.version_info < (3, 8) - def compare_impl( cmdline_override: Optional[str] = None, press: Iterable[str] = (), @@ -138,14 +128,11 @@ async def run_before_wrapper(pilot) -> None: with monkeypatch.context() as app_patch: app_patch.setitem(globals(), app_global, app) tmp_main.write_text(f"from {__name__} import {app_global} as app") - return ( - snap_compare( - str(tmp_main), - press=press, - terminal_size=terminal_size, - run_before=run_before_wrapper, - ) - or succeed_even_if_mismatched + return snap_compare( + str(tmp_main), + press=press, + terminal_size=terminal_size, + run_before=run_before_wrapper, ) yield compare_impl