Merge pull request #71 from Cumulocity-IoT/feature/get-as-tuples

Added as_tuples parameter to select and get_all functions.

Author: chsou

CHANGELOG.md

@@ -1,5 +1,8 @@
 # Changelog
 
+* Added `as_tuples` parameter to `select` and `get_all` functions in all inventory API as well
+  as Events and Alarms API. This parameter can be used to directly extract specific values from
+  the results instead of parsing the JSON. 
 * Added `reload` function to inventory object classes
 * Added `delete_tree` function to inventory object classes, implicitly using `cascade` or
   `forceCascade` parameters depending on the use case.

c8y_api/model/_base.py

@@ -13,6 +13,48 @@
 from c8y_api.model._util import _DateUtil, _StringUtil, _QueryUtil
 
 
+def get_by_path(dictionary: dict, path: str, default: Any = None) -> Any:
+    """Select a nested value from a dictionary by path-like expression
+    (dot notation).
+
+    Args:
+        dictionary (dict):  the dictionary to extract values from
+        path (str):  a path-like expressions
+        default (Any):  default value to return if the path expression
+            doesn't match a value in the dictionary.
+
+    Return:
+        The extracted value or the specified default.
+    """
+    def _get_by_path(current: dict, segments: list[str]) -> Any:
+        if not segments:
+            return current
+        if segments[0] in current:
+            return _get_by_path(current[segments[0]], segments[1:])
+        return default
+
+    return _get_by_path(dictionary, path.split('.'))
+
+
+def get_all_by_path(dictionary: dict, paths: list[str] | dict[str, Any]) -> tuple:
+    """Select nested values from a dictionary by path-like expressions
+    (dot notation).
+
+    Args:
+        dictionary (dict):  the dictionary to extract values from
+        paths: (list or dict):  a set of path-like expressions; use
+            a dictionary to define default values for each
+
+    Return:
+        The extracted values (or defaults it specified) as tuple. The
+        number of elements in the tuple matches the length of the paths
+        argument.
+    """
+    if isinstance(paths, dict):
+        return tuple(get_by_path(dictionary, p, d) for p, d in paths.items())
+    return tuple(get_by_path(dictionary, p) for p in paths)
+
+
 class _DictWrapper(MutableMapping, dict):
 
     def __init__(self, dictionary: dict, on_update=None):
@@ -704,7 +746,8 @@ def _iterate(self, base_query: str, page_number: int | None, limit: int | None,
             for result in results:
                 if limit and num_results >= limit:
                     return
-                result.c8y = self.c8y  # inject c8y connection into instance
+                if hasattr(result, 'c8y'):
+                    result.c8y = self.c8y  # inject c8y connection into instance
                 yield result
                 num_results = num_results + 1
             # when a specific page was specified we don't read more pages

c8y_api/model/alarms.py

@@ -3,10 +3,10 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import List, Generator
+from typing import List, Generator, Any
 
 from c8y_api._base_api import CumulocityRestApi
-from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject
+from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject, get_all_by_path
 from c8y_api.model._parser import ComplexObjectParser
 from c8y_api.model._util import _DateUtil
 
@@ -255,6 +255,7 @@ def select(self,
                reverse: bool = False, limit: int = None,
                with_source_assets: bool = None, with_source_devices: bool = None,
                page_size: int = 1000, page_number: int = None,
+               as_tuples: list[str] | dict[str, Any] = None,
                **kwargs) -> Generator[Alarm]:
         """Query the database for alarms and iterate over the results.
 
@@ -306,6 +307,10 @@ def select(self,
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (list[str] or dict[str, Any]):  Don't parse Alarms, but
+                extract the values at certain JSON paths as tuples; If the
+                path is not defined in a result, None is used; Specify a
+                dictionary to define proper default values for each path.
 
         Returns:
             Generator of Alarm objects
@@ -324,7 +329,11 @@ def select(self,
             min_age=min_age, max_age=max_age,
             reverse=reverse, page_size=page_size,
             **kwargs)
-        return super()._iterate(base_query, page_number, limit, Alarm.from_json)
+        return super()._iterate(
+            base_query,
+            page_number,
+            limit,
+            Alarm.from_json if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
 
     def get_all(
             self,
@@ -341,6 +350,7 @@ def get_all(
             with_source_assets: bool = None, with_source_devices: bool = None,
             reverse: bool = False, limit: int = None,
             page_size: int = 1000, page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> List[Alarm]:
         """Query the database for alarms and return the results as list.
 
@@ -365,6 +375,7 @@ def get_all(
             min_age=min_age, max_age=max_age, reverse=reverse,
             with_source_devices=with_source_devices, with_source_assets=with_source_assets,
             limit=limit, page_size=page_size, page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs))
 
     def count(

c8y_api/model/events.py

@@ -6,10 +6,10 @@
 from __future__ import annotations
 
 from datetime import datetime, timedelta
-from typing import Generator, List, BinaryIO
+from typing import Generator, List, BinaryIO, Any
 
 from c8y_api._base_api import CumulocityRestApi
-from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject
+from c8y_api.model._base import CumulocityResource, SimpleObject, ComplexObject, get_all_by_path
 from c8y_api.model._parser import ComplexObjectParser
 from c8y_api.model._util import _DateUtil
 
@@ -264,6 +264,7 @@ def select(self,
                reverse: bool = False, limit: int = None,
                with_source_assets: bool = None, with_source_devices: bool = None,
                page_size: int = 1000, page_number: int = None,
+               as_tuples: list[str] | dict[str, Any] = None,
                **kwargs) -> Generator[Event]:
         """Query the database for events and iterate over the results.
 
@@ -314,6 +315,10 @@ def select(self,
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (list[str] or dict[str, Any]):  Don't parse Events, but
+                extract the values at certain JSON paths as tuples; If the
+                path is not defined in a result, None is used; Specify a
+                dictionary to define proper default values for each path.
 
         Returns:
             Generator for Event objects
@@ -335,7 +340,11 @@ def select(self,
             with_source_devices=with_source_devices, with_source_assets=with_source_assets,
             page_size=page_size,
             **kwargs)
-        return super()._iterate(base_query, page_number, limit, Event.from_json)
+        return super()._iterate(
+            base_query,
+            page_number,
+            limit,
+            Event.from_json if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
 
     def get_all(
             self,
@@ -352,6 +361,7 @@ def get_all(
             reverse: bool = False, limit: int = None,
             with_source_assets: bool = None, with_source_devices: bool = None,
             page_size: int = 1000, page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> List[Event]:
         """Query the database for events and return the results as list.
 
@@ -377,6 +387,7 @@ def get_all(
             reverse=reverse,
             with_source_devices=with_source_devices, with_source_assets=with_source_assets,
             limit=limit, page_size=page_size, page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs
         ))
 

c8y_api/model/inventory.py

@@ -5,7 +5,7 @@
 
 from typing import Any, Generator, List
 
-from c8y_api.model._base import CumulocityResource
+from c8y_api.model._base import CumulocityResource, get_all_by_path
 from c8y_api.model._util import _QueryUtil
 from c8y_api.model.managedobjects import ManagedObjectUtil, ManagedObject, Device, Availability, DeviceGroup
 
@@ -61,6 +61,7 @@ def get_all(
             reverse: bool = None,
             limit: int = None,
             page_size: int = 1000,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> List[ManagedObject]:
         """ Query the database for managed objects and return the results
         as list.
@@ -93,6 +94,7 @@ def get_all(
             reverse=reverse,
             limit=limit,
             page_size=page_size,
+            as_tuples=as_tuples,
             **kwargs))
 
     def get_count(
@@ -154,6 +156,7 @@ def select(
             limit: int = None,
             page_size: int = 1000,
             page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> Generator[ManagedObject]:
         """ Query the database for managed objects and iterate over the
         results.
@@ -205,6 +208,10 @@ def select(
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (list[str] or dict[str, Any]):  Don't parse ManagedObjects,
+                but extract the values at certain JSON paths as tuples; If the path
+                is not defined in a result, None is used; Specify a dictionary to
+                define proper default values for each path.
 
         Returns:
             Generator for ManagedObject instances
@@ -234,6 +241,7 @@ def select(
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs)
 
     def _prepare_inventory_query(
@@ -336,10 +344,14 @@ def filter_none(**xs):
 
         return {query_key: query, **kwargs}
 
-    def _select(self, parse_fun, device_mode: bool, page_number, limit, **kwargs) -> Generator[Any]:
+    def _select(self, parse_fun, device_mode: bool, page_number, limit, as_tuples, **kwargs) -> Generator[Any]:
         """Generic select function to be used by derived classes as well."""
         base_query = self._prepare_inventory_query(device_mode, **kwargs)
-        return super()._iterate(base_query, page_number, limit, parse_fun)
+        return super()._iterate(
+            base_query,
+            page_number,
+            limit,
+            parse_fun if not as_tuples else (lambda x: get_all_by_path(x, as_tuples)))
 
     def create(self, *objects: ManagedObject):
         """Create managed objects within the database.
@@ -489,6 +501,7 @@ def select(  # noqa (order)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs,) -> Generator[Device]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Query the database for devices and iterate over the results.
@@ -543,6 +556,10 @@ def select(  # noqa (order)
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (list[str] or dict[str, Any]):  Don't parse Device objects,
+                but extract the values at certain JSON paths as tuples; If the path
+                is not defined in a result, None is used; Specify a dictionary to
+                define proper default values for each path.
 
         Returns:
             Generator for Device objects
@@ -571,6 +588,7 @@ def select(  # noqa (order)
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs)
 
     def get_all(  # noqa (changed signature)
@@ -596,6 +614,7 @@ def get_all(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> List[Device]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Query the database for devices and return the results as list.
@@ -628,6 +647,7 @@ def get_all(  # noqa (changed signature)
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs))
 
     def get_count(  # noqa (changed signature)
@@ -734,6 +754,7 @@ def select(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
+            as_tuples: list[str] | dict[str, Any] = None,
             **kwargs) -> Generator[DeviceGroup]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Select device groups by various parameters.
@@ -791,6 +812,11 @@ def select(  # noqa (changed signature)
                 parsed in one chunk). This is a performance related setting.
             page_number (int): Pull a specific page; this effectively disables
                 automatic follow-up page retrieval.
+            as_tuples: (list[str] or dict[str, Any]):  Don't parse DeviceGroup
+                objects, but extract the values at certain JSON paths as
+                tuples; If the path is not defined in a result, None is used;
+                Specify a dictionary to define proper default values for each
+                path.
 
         Returns:
             Generator of DeviceGroup instances
@@ -827,6 +853,7 @@ def select(  # noqa (changed signature)
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs)
 
     def get_count(  # noqa (changed signature)
@@ -896,7 +923,8 @@ def get_all(  # noqa (changed signature)
             limit: int = None,
             page_size: int = 100,
             page_number: int = None,
-            **kwargs) -> List[DeviceGroup]:
+            as_tuples: list[str] | dict[str, Any] = None,
+            **kwargs ) -> List[DeviceGroup]:
         # pylint: disable=arguments-differ, arguments-renamed
         """ Select managed objects by various parameters.
 
@@ -927,6 +955,7 @@ def get_all(  # noqa (changed signature)
             limit=limit,
             page_size=page_size,
             page_number=page_number,
+            as_tuples=as_tuples,
             **kwargs))
 
     def create(self, *groups):

tests/model/test_alarms.py

@@ -84,6 +84,29 @@ def test_select(fun, params, expected, not_expected):
         assert ne not in resource
 
 
+def test_select_as_tuples():
+    """Verify that select as tuples works as expected."""
+    jsons = [
+        {'type': 'type1', 'text': 'text1', 'source': 'source1', 'test_Fragment': {'key': 'value1', 'key2': 'value2'}},
+        {'type': 'type2', 'text': 'text2', 'source': 'source2', 'test_Fragment': {'key': 'value2'}},
+    ]
+
+    api = Alarms(c8y=Mock())
+    api.c8y.get = Mock(side_effect=[{'alarms': jsons}, {'alarms': []}])
+    result = api.get_all(as_tuples=['type', 'text', 'test_Fragment.key', 'test_Fragment.key2'])
+    assert result == [
+        ('type1', 'text1', 'value1', 'value2'),
+        ('type2', 'text2', 'value2', None),
+    ]
+
+    api.c8y.get = Mock(side_effect=[{'alarms': jsons}, {'alarms': []}])
+    result = api.get_all(as_tuples={'type': None, 'text': None, 'test_Fragment.key': None, 'test_Fragment.key2': '-'})
+    assert result == [
+        ('type1', 'text1', 'value1', 'value2'),
+        ('type2', 'text2', 'value2', '-'),
+    ]
+
+
 @pytest.mark.parametrize('fun', [
     Alarms.get_all,
     Alarms.count,