From 59630cce20f3ef5382b4d7af99285e5251867d49 Mon Sep 17 00:00:00 2001 From: Bram Stoeller Date: Thu, 6 Oct 2022 14:34:49 +0200 Subject: [PATCH 1/3] Unit tests for 'modules' utils Signed-off-by: Bram Stoeller --- src/power_grid_model_io/utils/modules.py | 71 +++++--------- tests/unit/filters/test_auto_id.py | 3 + tests/unit/filters/test_modules.py | 117 +++++++++++++++++++++++ 3 files changed, 146 insertions(+), 45 deletions(-) create mode 100644 tests/unit/filters/test_auto_id.py create mode 100644 tests/unit/filters/test_modules.py diff --git a/src/power_grid_model_io/utils/modules.py b/src/power_grid_model_io/utils/modules.py index b0b4f779..3fed8def 100644 --- a/src/power_grid_model_io/utils/modules.py +++ b/src/power_grid_model_io/utils/modules.py @@ -8,7 +8,10 @@ import sys from importlib import import_module from pathlib import Path -from typing import Callable, List, Optional +from types import ModuleType +from typing import Callable + +MAIN_PACKAGE = "power-grid-model-io" # extra_key -> module -> pip_package DEPENDENCIES = { @@ -16,18 +19,13 @@ } -def running_from_conda_env() -> bool: +def running_from_conda() -> bool: """ Check if the conda is used """ - return Path(sys.prefix, "conda-meta").exists() - - -def module_loaded(module: str) -> bool: - """ - Check if the module is already loaded - """ - return hasattr(sys, module) + # If conda is used, we expect a directory called conda-meta in the root dir of the environment + env_dir = Path(sys.prefix) + return (env_dir / "conda-meta").exists() def module_installed(module: str) -> bool: @@ -37,61 +35,44 @@ def module_installed(module: str) -> bool: return importlib.util.find_spec(module) is not None -def import_optional_module(module: str, extra: str): +def import_optional_module(module: str, extra: str) -> ModuleType: """ Check if the required module is installed and load it """ - assert_dependencies(extra=extra, modules=[module]) - if module_loaded(module): - return getattr(sys, module) + assert_optional_module_installed(module=module, extra=extra) return importlib.import_module(module) -def assert_dependencies(extra: str, modules: Optional[List[str]] = None): +def assert_optional_module_installed(module: str, extra): """ - Check if the required module is installed, or raise a human readable errormessage with instructions if it doesn't. + Check if the required module is installed, or raise a human readable error message with instructions if it doesn't. """ + # Check if the module is installed + if module_installed(module): + return + # Get the dependencies for the given extra try: dependencies = DEPENDENCIES[extra] except KeyError as ex: raise KeyError(f"Extra requirements '{extra}' is not defined.") from ex - # Get, or validate the modules - if modules is not None: - for module in modules: - if module not in dependencies: - raise KeyError(f"Module '{module} is not included in the extra requirements '{extra}'") - else: - modules = list(dependencies.keys()) - - # Check which modules are missing - missing = [module for module in modules if not module_loaded(module) and not module_installed(module)] - if not missing: - return - - # Define the main module name - module_name = __name__.split(".", maxsplit=1)[0] + # Check if the module is part of the extra requirement + if module not in dependencies: + raise KeyError(f"Module '{module}' is not included in the extra requirements '{extra}'") # Atempt to guess the package manager - if running_from_conda_env(): + if running_from_conda(): cmd = "conda install " elif module_installed("pip"): cmd = "pip install " else: cmd = "" - # Are we missing just one module, or multiple? - if len(missing) == 1: - msg = ( - f"Missing optional module: `{missing[0]}`. Install it with `{cmd}{module_name}[{extra}]` " - f"or `{cmd}{dependencies[missing[0]]}`" - ) - else: - msg = ( - f"Missing optional modules: {', '.join(missing)}. Install them with `{cmd}{module_name}[{extra}]` " - f"or `{cmd}{' '.join(dependencies[m] for m in missing)}`" - ) + msg = ( + f"Missing optional module: `{module}`. Install it with `{cmd}{MAIN_PACKAGE}[{extra}]` " + f"or `{cmd}{dependencies[module]}`" + ) # Raise the exception raise ModuleNotFoundError(msg) @@ -107,9 +88,9 @@ def get_function(fn_name: str) -> Callable: try: module = import_module(module_path) except ModuleNotFoundError as ex: - raise AttributeError(f"Function: {fn_name} does not exist") from ex + raise AttributeError(f"Module '{module_path}' does not exist (tried to resolve function '{fn_name}')!") from ex try: fn_ptr = getattr(module, function_name) except AttributeError as ex: - raise AttributeError(f"Function: {function_name} does not exist in {module_path}") from ex + raise AttributeError(f"Function '{function_name}' does not exist in module '{module_path}'!") from ex return fn_ptr diff --git a/tests/unit/filters/test_auto_id.py b/tests/unit/filters/test_auto_id.py new file mode 100644 index 00000000..7f6cad5b --- /dev/null +++ b/tests/unit/filters/test_auto_id.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project +# +# SPDX-License-Identifier: MPL-2.0 diff --git a/tests/unit/filters/test_modules.py b/tests/unit/filters/test_modules.py new file mode 100644 index 00000000..ccc8f518 --- /dev/null +++ b/tests/unit/filters/test_modules.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project +# +# SPDX-License-Identifier: MPL-2.0 + +from unittest.mock import MagicMock, patch + +from pytest import mark, raises + +from power_grid_model_io.utils.modules import ( + DEPENDENCIES, + assert_optional_module_installed, + get_function, + import_optional_module, + module_installed, + running_from_conda, +) + + +@mark.parametrize("conda_exists", [True, False]) +@patch("power_grid_model_io.utils.modules.Path.exists") +def test_running_from_conda(exists_mock: MagicMock, conda_exists: bool): + exists_mock.return_value = conda_exists + assert running_from_conda() == conda_exists + + +def test_module_installed(): + assert module_installed("power_grid_model_io") + assert not module_installed("non_existing_module") + + +@patch("importlib.import_module") +@patch("power_grid_model_io.utils.modules.assert_optional_module_installed") +def test_import_optional_module(assert_mock: MagicMock, import_mock: MagicMock): + module = import_optional_module(module="module", extra="extra") + assert_mock.assert_called_once_with(module="module", extra="extra") + import_mock.assert_called_once_with("module") + assert module == import_mock.return_value + + +@patch("power_grid_model_io.utils.modules.module_installed") +def test_assert_optional_module_installed__ok(module_installed_mock: MagicMock): + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + module_installed_mock.return_value = True + assert_optional_module_installed(module="dummy_module", extra="dummy_extra") + + +def test_assert_optional_module_installed__unknown_extra(): + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + with raises(KeyError, match="unknown_extra"): + assert_optional_module_installed(module="dummy_module", extra="unknown_extra") + + +def test_assert_optional_module_installed__unknown_module(): + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + with raises(KeyError, match="unknown_module.*dummy_extra"): + assert_optional_module_installed(module="unknown_module", extra="dummy_extra") + + +@patch("power_grid_model_io.utils.modules.running_from_conda") +def test_assert_optional_module_installed__conda(conda_mock: MagicMock): + conda_mock.return_value = True + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + with raises( + ModuleNotFoundError, match=r"Missing.*dummy_module.*`conda install power-grid-model-io\[dummy_extra\]`" + ): + assert_optional_module_installed(module="dummy_module", extra="dummy_extra") + + +@patch("power_grid_model_io.utils.modules.running_from_conda") +@patch("power_grid_model_io.utils.modules.module_installed") +def test_assert_optional_module_installed__pip(module_mock: MagicMock, conda_mock: MagicMock): + module_mock.side_effect = [False, True] # dummy_module: False, pip: True + conda_mock.return_value = False + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + with raises(ModuleNotFoundError, match=r"Missing.*dummy_module.*`pip install power-grid-model-io\[dummy_extra\]`"): + assert_optional_module_installed(module="dummy_module", extra="dummy_extra") + + +@patch("power_grid_model_io.utils.modules.running_from_conda") +@patch("power_grid_model_io.utils.modules.module_installed") +def test_assert_optional_module_installed__unkown_package_manager(module_mock: MagicMock, conda_mock: MagicMock): + module_mock.side_effect = [False, False] # dummy_module: False, pip: False + conda_mock.return_value = False + DEPENDENCIES["dummy_extra"] = {"dummy_module": "dummy_package"} + with raises(ModuleNotFoundError, match=r"Missing.*dummy_module.*`power-grid-model-io\[dummy_extra\]`"): + assert_optional_module_installed(module="dummy_module", extra="dummy_extra") + + +def test_get_function__builtins(): + assert get_function("min") == min + + +def test_get_function__native(): + assert get_function("pytest.mark") == mark + + +def test_get_function__custom(): + from power_grid_model_io.filters import multiply + + assert get_function("power_grid_model_io.filters.multiply") == multiply + + +def test_get_function__module_doesnt_exist(): + with raises(AttributeError, match=r"Module 'a\.b' does not exist \(tried to resolve function 'a\.b\.c'\)!"): + assert get_function("a.b.c") + + +def test_get_function__function_doesnt_exist(): + with raises( + AttributeError, match="Function 'unknown_function' does not exist in module 'power_grid_model_io.filters'!" + ): + assert get_function("power_grid_model_io.filters.unknown_function") + + +def test_get_function__builtin_doesnt_exist(): + with raises(AttributeError, match="Function 'mean' does not exist in module 'builtins'!"): + assert get_function("mean") From fddb759211b227b613d9d7306ccb73813ad2c41b Mon Sep 17 00:00:00 2001 From: Bram Stoeller Date: Thu, 6 Oct 2022 14:53:14 +0200 Subject: [PATCH 2/3] Unit tests for AutoId util Signed-off-by: Bram Stoeller --- tests/unit/filters/test_auto_id.py | 48 ++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/tests/unit/filters/test_auto_id.py b/tests/unit/filters/test_auto_id.py index 7f6cad5b..b1e7e83c 100644 --- a/tests/unit/filters/test_auto_id.py +++ b/tests/unit/filters/test_auto_id.py @@ -1,3 +1,51 @@ # SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project # # SPDX-License-Identifier: MPL-2.0 +from pytest import raises + +from power_grid_model_io.utils.auto_id import AutoID + + +def test_auto_id__without_items(): + auto_id = AutoID() + assert auto_id() == 0 + assert auto_id() == 1 + assert auto_id() == 2 + assert auto_id[0] == 0 + assert auto_id[1] == 1 + assert auto_id[2] == 2 + with raises(IndexError): + _ = auto_id[3] + + +def test_auto_id__with_hashable_items(): + auto_id = AutoID() + assert auto_id(item="Alpha") == 0 + assert auto_id(item="Bravo") == 1 + assert auto_id(item="Alpha") == 0 # because key "Alpha" already existed + assert auto_id[0] == "Alpha" + assert auto_id[1] == "Bravo" + with raises(IndexError): + _ = auto_id[2] + + +def test_auto_id__with_non_hashable_items(): + auto_id = AutoID() + assert auto_id(item={"name": "Alpha"}, key="Alpha") == 0 + assert auto_id(item={"name": "Bravo"}, key="Bravo") == 1 + assert auto_id(item={"name": "Alpha"}, key="Alpha") == 0 # because key "Alpha" already existed + assert auto_id[0] == {"name": "Alpha"} + assert auto_id[1] == {"name": "Bravo"} + with raises(IndexError): + _ = auto_id[2] + + +def test_auto_id__with_clashing_keys(): + auto_id = AutoID() + assert auto_id(item={"name": "Alpha"}, key="Alpha") == 0 + assert auto_id(item={"name": "Bravo"}, key="Bravo") == 1 + assert auto_id(item={"name": "Charly"}, key="Alpha") == 0 # because key "Alpha" already existed + assert auto_id[0] == {"name": "Charly"} # Note that the item was overwritten silently + assert auto_id[1] == {"name": "Bravo"} + with raises(IndexError): + _ = auto_id[2] From 2d7c9f7d8d1d100b38fdf9fe36e03c7bddd5c79e Mon Sep 17 00:00:00 2001 From: Bram Stoeller Date: Tue, 11 Oct 2022 10:01:42 +0200 Subject: [PATCH 3/3] Move utils unit tests Signed-off-by: Bram Stoeller --- tests/unit/utils/__init__.py | 3 +++ tests/unit/{filters => utils}/test_auto_id.py | 0 tests/unit/{filters => utils}/test_modules.py | 0 3 files changed, 3 insertions(+) create mode 100644 tests/unit/utils/__init__.py rename tests/unit/{filters => utils}/test_auto_id.py (100%) rename tests/unit/{filters => utils}/test_modules.py (100%) diff --git a/tests/unit/utils/__init__.py b/tests/unit/utils/__init__.py new file mode 100644 index 00000000..7f6cad5b --- /dev/null +++ b/tests/unit/utils/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project +# +# SPDX-License-Identifier: MPL-2.0 diff --git a/tests/unit/filters/test_auto_id.py b/tests/unit/utils/test_auto_id.py similarity index 100% rename from tests/unit/filters/test_auto_id.py rename to tests/unit/utils/test_auto_id.py diff --git a/tests/unit/filters/test_modules.py b/tests/unit/utils/test_modules.py similarity index 100% rename from tests/unit/filters/test_modules.py rename to tests/unit/utils/test_modules.py