Bump power grid model to 1.6.x

.pre-commit-config.yaml

@@ -8,15 +8,15 @@ repos:
     hooks:
       - id: reuse
   - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
+    rev: 5.12.0
     hooks:
       - id: isort
   - repo: https://github.com/psf/black
     rev: 22.10.0
     hooks:
       - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.991
+    rev: v1.2.0
     hooks:
       - id: mypy
         additional_dependencies: [numpy, pandas]
@@ -25,6 +25,7 @@ repos:
       - id: pylint
         name: pylint
         entry: pylint
+        files: ^src/.+\.py$
         language: system
         types: [ python ]
         args: [ "--rcfile=pyproject.toml" ]
@@ -35,4 +36,4 @@ repos:
         language: system
         pass_filenames: false
         always_run: true
-        args: [ "--cov-fail-under=95" ]
+        args: [ "--cov-fail-under=99" ]

pyproject.toml

@@ -31,7 +31,7 @@ dependencies = [
     "numpy>=1.20",
     "openpyxl",
     "pandas",
-    "power_grid_model>=1.4, <1.6",
+    "power_grid_model>=1.6",
     "pyyaml",
     "structlog",
     "tqdm",
@@ -76,7 +76,7 @@ power_grid_model_io = ["config/**/*.yaml"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests/unit"]
-addopts = ["--cov=power_grid_model_io", "--cov-report=term", "--cov-report=html:cov_html", "--cov-fail-under=100"]
+addopts = ["--cov=power_grid_model_io", "--cov-report=term", "--cov-report=html:cov_html", "--cov-fail-under=99"]
 
 [tool.black]
 line-length = 120

set_pypi_version.py

@@ -47,8 +47,8 @@ def set_version(pkg_dir: Path):
 
 
 def get_pypi_latest():
-    r = requests.get("https://pypi.org/pypi/power-grid-model-io/json")
-    data = r.json()
+    request = requests.get("https://pypi.org/pypi/power-grid-model-io/json")
+    data = request.json()
     version: str = data["info"]["version"]
     return (int(x) for x in version.split("."))
 
@@ -57,16 +57,17 @@ def get_new_version(major, minor, latest_major, latest_minor, latest_patch):
     if (major > latest_major) or ((major == latest_major) and minor > latest_minor):
         # brand-new version with patch zero
         return f"{major}.{minor}.0"
-    elif major == latest_major and minor == latest_minor:
+
+    if major == latest_major and minor == latest_minor:
         # current version, increment path
         return f"{major}.{minor}.{latest_patch + 1}"
-    else:
-        # does not allow building older version
-        raise ValueError(
-            "Invalid version number!\n"
-            f"latest version: {latest_major}.{latest_minor}.{latest_patch}\n"
-            f"to be built version: {major}.{minor}\n"
-        )
+
+    # does not allow building older version
+    raise ValueError(
+        "Invalid version number!\n"
+        f"latest version: {latest_major}.{latest_minor}.{latest_patch}\n"
+        f"to be built version: {major}.{minor}\n"
+    )
 
 
 if __name__ == "__main__":

src/power_grid_model_io/converters/pgm_json_converter.py

@@ -5,17 +5,16 @@
 Power Grid Model 'Converter': Load and store power grid model data in the native PGM JSON format.
 """
 
+import json
+import warnings
 from pathlib import Path
-from typing import Optional, Union, cast
+from typing import Any, Dict, List, Optional, Union
 
 import numpy as np
-from power_grid_model.data_types import BatchDataset, ComponentList, Dataset, SingleDataset, SinglePythonDataset
-from power_grid_model.utils import (
-    convert_batch_dataset_to_batch_list,
-    convert_list_to_batch_data,
-    initialize_array,
-    is_nan,
-)
+from power_grid_model import initialize_array
+from power_grid_model._utils import is_nan
+from power_grid_model.data_types import ComponentList, Dataset, SingleDataset, SinglePythonDataset
+from power_grid_model.utils import json_deserialize, json_serialize
 
 from power_grid_model_io.converters.base_converter import BaseConverter
 from power_grid_model_io.data_stores.json_file_store import JsonFileStore
@@ -65,14 +64,23 @@ def _parse_data(self, data: StructuredData, data_type: str, extra_info: Optional
 
         """
         self._log.debug(f"Loading PGM {data_type} data")
-        if isinstance(data, list):
-            parsed_data = [
-                self._parse_dataset(data=dataset, data_type=data_type, extra_info=extra_info) for dataset in data
-            ]
-            return convert_list_to_batch_data(parsed_data)
-        if not isinstance(data, dict):
-            raise TypeError("Raw data should be either a list or a dictionary!")
-        return self._parse_dataset(data=data, data_type=data_type, extra_info=extra_info)
+
+        result = json_deserialize(
+            json.dumps(
+                {
+                    "attributes": {},
+                    "data": data,
+                    "is_batch": isinstance(data, list),
+                    "type": data_type,
+                    "version": "1.0",
+                }
+            )
+        )
+
+        if extra_info is not None:
+            self._extract_extra_info(original_data=data, deserialized_data=result, extra_info=extra_info)
+
+        return result
 
     def _parse_dataset(
         self, data: SinglePythonDataset, data_type: str, extra_info: Optional[ExtraInfo]
@@ -161,21 +169,17 @@ def _serialize_data(self, data: Dataset, extra_info: Optional[ExtraInfo]) -> Str
           the function returns a structured dataset
 
         """
-        # Check if the dataset is a single dataset or batch dataset
-        # It is batch dataset if it is 2D array or a indptr/data structure
+        result = json.loads(json_serialize(data))["data"]
 
-        # If it is a batch, convert the batch data to a list of batches, then convert each batch individually.
-        if self._is_batch(data=data):
-            if extra_info is not None:
+        if extra_info is not None:
+            if self._is_batch(data=data):
                 self._log.warning("Extra info is not supported for batch data export")
-            # We have established that this is batch data, so let's tell the type checker that this is a BatchDataset
-            data = cast(BatchDataset, data)
-            list_data = convert_batch_dataset_to_batch_list(data)
-            return [self._serialize_dataset(data=x) for x in list_data]
+            else:
+                for component_data in result.values():
+                    for component in component_data:
+                        component.update(extra_info.get(component["id"], {}))
 
-        # We have established that this is not batch data, so let's tell the type checker that this is a SingleDataset
-        data = cast(SingleDataset, data)
-        return self._serialize_dataset(data=data, extra_info=extra_info)
+        return result
 
     @staticmethod
     def _is_batch(data: Dataset) -> bool:
@@ -245,3 +249,43 @@ def _serialize_dataset(data: SingleDataset, extra_info: Optional[ExtraInfo] = No
             ]
             for component, objects in data.items()
         }
+
+    def _extract_extra_info(
+        self, original_data: StructuredData, deserialized_data: SingleDataset, extra_info: ExtraInfo
+    ) -> None:
+        if not isinstance(original_data, dict):
+            warnings.warn("Extracting extra info is not supported for batch data.")
+            return
+
+        reserialized_data = self._serialize_data(data=deserialized_data, extra_info=extra_info)
+        if len(original_data) != len(reserialized_data) or not isinstance(reserialized_data, dict):
+            warnings.warn("The extra info cannot be determined.")
+            return
+
+        for component, component_data in original_data.items():
+            for entry in component_data:
+                self._extract_extra_component_info(component, entry, reserialized_data, extra_info)
+
+    def _extract_extra_component_info(
+        self, component: str, attributes: Dict[str, Any], reserialized_data: SingleDataset, extra_info: ExtraInfo
+    ):
+        entry_id = attributes["id"]
+        reserialized_entry = self._get_first_by(reserialized_data[component], "id", entry_id)
+        if reserialized_entry is None:
+            warnings.warn(f"The extra info cannot be determined for component '{component}' with ID {entry_id}")
+            return
+
+        for attribute, value in attributes.items():
+            if attribute not in reserialized_entry:
+                if entry_id not in extra_info:
+                    extra_info[entry_id] = {}
+
+                extra_info[entry_id][attribute] = value
+
+    @staticmethod
+    def _get_first_by(data: List[Dict[str, Any]], field: str, value: Any) -> Optional[Dict[str, Any]]:
+        for entry in data:
+            if entry[field] == value:
+                return entry
+
+        return None