diff --git a/docs/conf.py b/docs/conf.py index 94ec06bf..1a3837ba 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,14 +24,9 @@ "sphinx.ext.intersphinx", "numpydoc", "hoverxref.extension", - "myst_parser", + "myst_nb", ] -source_suffix = { - ".rst": "restructuredtext", - ".md": "markdown", -} - templates_path = ["_templates"] exclude_patterns = ["_build", "Thumbs.db", ".DS_Store"] @@ -71,3 +66,8 @@ # -- sphinx.ext.intersphinx config ------------------------------------------- # For linking to power-grid-model's documentation. intersphinx_mapping = {"power-grid-model": ("https://power-grid-model.readthedocs.io/en/stable/", None)} + + +# -- Temporary config ----------- +# Disable notebook executions in examples directory +nb_execution_excludepatterns = ["examples/*.ipynb"] diff --git a/docs/converters/converter.md b/docs/converters/converter.md new file mode 100644 index 00000000..bb706640 --- /dev/null +++ b/docs/converters/converter.md @@ -0,0 +1,60 @@ + + +# Converters + +There are 4 types of converters present as of now. +Here, we shall discuss their basic structure and guidelines for building a custom converter. + +Use the examples notebooks to understand how to convert data from the respective formats. + +- **PGM JSON Converter:** Refer to the [PGM JSON Example](../examples/pgm_json_example.ipynb) +- **VisonExcelConverter** and **GaiaExcelConverter:** Refer to the [Vision and Gaia Example](../examples/vision_gaia_example.ipynb) +- **Pandapower Converter:** Converts [pandapower network](https://pandapower.readthedocs.io/en/stable/elements.html), which is a dictionary of dataframes, to power-grid-model data. + +Refer to [converters](../power_grid_model_io.md#converters) in API documentation for more details + +## Structure + +`VisonExcelConverter` and `GaiaExcelConverter` are inherited from [tabular converters](tabular_converter.md) for excel exports of vision and gaia respectively. +All 4 converters are derived from the base {py:class}`power_grid_model_io.converters.base_converter`. +The usable functions for loading, saving and converting the data are located in the base class. +The private functions (`_load_data`, `_parse_data` and `_serialize_data`) are overloaded based on the specific type of converter (ie. excel, json or pandapower). +It is recommended to create any custom converter in a similar way. + +## Instantiation + +A converter object can be instantiated in the following way. For eg, for a `PgmJsonConverter`, + +```python +from power_grid_model_io.converters.pgm_json_converter import PgmJsonConverter + +converter = PgmJsonConverter(source_file=source, destination_file=destination) +``` + +The usable methods of converters for loading and saving the data are described below. + +## Loading data + +Use the methods load_input_data(), load_update_data(), load_sym_output_data() or load_asym_output_data() to load the relevant data to the converter. +The Converter can be initialised with `source_file` containing path to source data. Or alternatively, the data can be provided as an argument to the load function. + +In addition to the power-grid-model input data, other miscellaneous information in the source file not used in calculations by power-grid-model gets stored under `extra_info` + +```python +input_data, extra_info = converter.load_input_data(data=example_data) +``` + +## Saving Data + +It is possible to save the data in the format of the converter. +The Converter can be instantiated with a path given to `destination_file`. +Alternatively, the destination path can be provided in the save function. +You can also add additional information about each component in the form of `extra_info` generated by [Load data](converter.md#load-data) to be saved along with it. + +```python +converter.save(example_data, extra_info=example_extra_info, destination=destination_path) +``` diff --git a/docs/converters/tabular_converter.md b/docs/converters/tabular_converter.md index 557b74f1..294c017b 100644 --- a/docs/converters/tabular_converter.md +++ b/docs/converters/tabular_converter.md @@ -10,7 +10,7 @@ each with multiple `columns`, possibly with a specific `unit`. Others may have a categorical value which needs to be mapped (i.e. open: 0, closed: 1); in general we'll call these `substitutions`. -# Mapping file +## Mapping file A mapping file is a yaml with three main sections `grid`, `units` and `substitutions`: ```yaml grid: @@ -146,7 +146,7 @@ The definitions above can be interpreted as: the word "none" should be replaced with the boolean `false` and the word "own" should be replaced with the value boolean `true`. -# AutoID +## AutoID The `id` field is special in the sense that each object should have a unique numerical id in power grid model. Therefore, each id definition is mapped to a numerical ID. Also each field name that ends with `node` is converted into the matching numerical ID. @@ -161,9 +161,9 @@ c = auto_id("Alpha") # c = 0 (because key "Alpha" already existed) item = auto_id[1] # item = "Bravo" ``` -See also [AutoID](/src/power_grid_model_io/utils/auto_id.py). +See also {py:class}`power_grid_model_io.utils.AutoID` -## Vision and Gaia +### Vision and Gaia For Vision and Gaia files, an extra trick is applied. Let's assume this mapping: ```yaml diff --git a/examples/data/tiny-net/input.json b/docs/examples/data/tiny-net/input.json similarity index 100% rename from examples/data/tiny-net/input.json rename to docs/examples/data/tiny-net/input.json diff --git a/examples/data/tiny-net/input.json.license b/docs/examples/data/tiny-net/input.json.license similarity index 100% rename from examples/data/tiny-net/input.json.license rename to docs/examples/data/tiny-net/input.json.license diff --git a/examples/data/tiny-net/params.json b/docs/examples/data/tiny-net/params.json similarity index 100% rename from examples/data/tiny-net/params.json rename to docs/examples/data/tiny-net/params.json diff --git a/examples/data/tiny-net/params.json.license b/docs/examples/data/tiny-net/params.json.license similarity index 100% rename from examples/data/tiny-net/params.json.license rename to docs/examples/data/tiny-net/params.json.license diff --git a/examples/data/tiny-net/sym_output.json b/docs/examples/data/tiny-net/sym_output.json similarity index 100% rename from examples/data/tiny-net/sym_output.json rename to docs/examples/data/tiny-net/sym_output.json diff --git a/examples/data/tiny-net/sym_output.json.license b/docs/examples/data/tiny-net/sym_output.json.license similarity index 100% rename from examples/data/tiny-net/sym_output.json.license rename to docs/examples/data/tiny-net/sym_output.json.license diff --git a/docs/examples/pgm_json_example.ipynb b/docs/examples/pgm_json_example.ipynb new file mode 100644 index 00000000..64fa51b1 --- /dev/null +++ b/docs/examples/pgm_json_example.ipynb @@ -0,0 +1,115 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# PGM JSON Converter\n", + "\n", + "Converts to and from the power-grid-model JSON data format. \n", + "This json format makes it convenient to evaluate the grid data in perspective of input to power-grid-model. \n", + "More details about the JSON format are mentioned in `power_grid_model.utils` and example of it is in [Make Test Dataset](power-grid-model:examples/Make%20Test%20Dataset.ipynb) in power-grid-model repository.\n", + "\n", + "Basic imports and configuration:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "from pathlib import Path\n", + "\n", + "import structlog\n", + "from power_grid_model import PowerGridModel\n", + "\n", + "from power_grid_model_io.converters.pgm_json_converter import PgmJsonConverter\n", + "\n", + "DATA_DIR = Path(\".\") / \"data\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define source and destination paths" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "input_file = DATA_DIR / \"tiny-net\" / \"input.json\"\n", + "output_file = DATA_DIR / \"tiny-net\" / \"sym_output.json\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Instantiate the converter, optionally with source and destination file paths. Use `load_input_data()` to convert to power-grid-model data. Additional information is stored in `extra_info`. Run powerflow calculation with the `input_data`." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2022-11-01 16:08.58 [debug ] Loading PGM input data\n" + ] + } + ], + "source": [ + "converter = PgmJsonConverter(source_file=input_file, destination_file=output_file)\n", + "input_data, extra_info = converter.load_input_data()\n", + "pgm = PowerGridModel(input_data=input_data)\n", + "output_data = pgm.calculate_state_estimation()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The converted data or loaded data can be saved. The file will be saved in `output_file` path or the destination path can be provided here optionally." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "converter.save(data=output_data, extra_info=extra_info)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/examples/pgm_json_example.ipynb.license b/docs/examples/pgm_json_example.ipynb.license new file mode 100644 index 00000000..f58b56c6 --- /dev/null +++ b/docs/examples/pgm_json_example.ipynb.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model project + +SPDX-License-Identifier: MPL-2.0 diff --git a/docs/examples/vision_gaia_example.ipynb b/docs/examples/vision_gaia_example.ipynb new file mode 100644 index 00000000..87991783 --- /dev/null +++ b/docs/examples/vision_gaia_example.ipynb @@ -0,0 +1,190 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Vision and Gaia Conversion\n", + "\n", + "This example illustrates conversion from Vision or Gaia excel export to power-grid-model input data. \n", + "They function in a similar way since both are [Tabular Converters](../converters/tabular_converter.md).\n", + "We can then calculate power-flow with it or convert to a different formats like PGM JSON.\n", + "\n", + "First we make some common imports and configuration for both converters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import logging\n", + "\n", + "import structlog\n", + "from power_grid_model import CalculationType, PowerGridModel\n", + "from power_grid_model.validation import assert_valid_input_data\n", + "\n", + "structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.INFO))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Define source and destination file paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "src = \"data/vision/example.xlsx\"\n", + "dst = \"data/vision/example_output.json\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Vision Conversion\n", + "\n", + "To save the vision file in excel format, please follow the instructions given in the [Vision Manual](https://phasetophase.nl/pdf/VisionEN.pdf). \n", + "\n", + "Import the vision converter" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "\n", + "from power_grid_model_io.converters.vision_excel_converter import VisionExcelConverter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can instantiate the Converter with the source data file path and destination file for saving the data in the PGM JSON format. Optionally they can be provided in the individually at their relevant methods too." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vision_converter = VisionExcelConverter(source_file=src)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Then convert the data by using `load_input_data()` method. The data which can be used by power-grid-model for powerflow calculation is in `input_data`. The additional information that is not used in the powerflow calculation but might be useful to have is in `extra_info`. We can also validate the converted data using the validation tool in power-grid-model: `assert_valid_input_data()`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "input_data, extra_info = vision_converter.load_input_data()\n", + "assert_valid_input_data(input_data, calculation_type=CalculationType.power_flow, symmetric=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Perform power flow calculation or state estimation in power-grid-model like you would normally." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "vision_grid = PowerGridModel(input_data=input_data)\n", + "output_data = vision_grid.calculate_power_flow()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Gaia Conversion\n", + "\n", + "```{note}\n", + "Gaia Conversion is under development currently and may not work as intended\n", + "```\n", + "\n", + "Please refer to Gaia manual from the software for instructions on excel export. Gaia Conversion is similar to vision conversion.\n", + "\n", + "Import Gaia Converter." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from power_grid_model_io.converters.vision_excel_converter import GaiaExcelConverter" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Convert and Run power flow in similar way" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "gaia_converter = GaiaExcelConverter(source_file=src)\n", + "input_data, extra_info = gaia_converter.load_input_data()\n", + "assert_valid_input_data(input_data, calculation_type=CalculationType.power_flow, symmetric=True)\n", + "gaia_grid = PowerGridModel(input_data=input_data)\n", + "output_data = gaia_grid.calculate_power_flow()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + }, + "vscode": { + "interpreter": { + "hash": "6191baaf994abc6c3e7d918f70d2cb3a487f327a3f3fca1519684915289f78b6" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/examples/vision_gaia_example.ipynb.license b/docs/examples/vision_gaia_example.ipynb.license new file mode 100644 index 00000000..f58b56c6 --- /dev/null +++ b/docs/examples/vision_gaia_example.ipynb.license @@ -0,0 +1,3 @@ +SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model project + +SPDX-License-Identifier: MPL-2.0 diff --git a/docs/index.md b/docs/index.md index ca39a5b2..30075d0d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -35,6 +35,15 @@ converters/converter.md converters/tabular_converter.md ``` +```{toctree} +:caption: "Examples" +:maxdepth: 2 +examples/pgm_json_example.ipynb +examples/vision_gaia_example.ipynb +``` + + + ```{toctree} :caption: "API Documentation" :maxdepth: 4 diff --git a/examples/pgm_json_conversion.py b/examples/pgm_json_conversion.py deleted file mode 100644 index 9444d240..00000000 --- a/examples/pgm_json_conversion.py +++ /dev/null @@ -1,29 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project -# -# SPDX-License-Identifier: MPL-2.0 - -import logging -from pathlib import Path - -import pandas as pd -import structlog -from power_grid_model import PowerGridModel - -from power_grid_model_io.converters.pgm_json_converter import PgmJsonConverter - -DATA_DIR = Path(__file__).parent / "data" - -if __name__ == "__main__": - structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.DEBUG)) - - input_file = DATA_DIR / "tiny-net" / "input.json" - output_file = DATA_DIR / "tiny-net" / "sym_output.json" - - converter = PgmJsonConverter(source_file=input_file, destination_file=output_file) - input_data, extra_info = converter.load_input_data() - pgm = PowerGridModel(input_data=input_data) - output_data = pgm.calculate_state_estimation() - converter.save(data=output_data, extra_info=extra_info) - - print("Nodes output: \n", pd.DataFrame(output_data["node"])) - print("Lines output: \n", pd.DataFrame(output_data["line"])) diff --git a/examples/vision_power_flow.py b/examples/vision_power_flow.py deleted file mode 100644 index 2d31b2e2..00000000 --- a/examples/vision_power_flow.py +++ /dev/null @@ -1,33 +0,0 @@ -# SPDX-FileCopyrightText: 2022 Contributors to the Power Grid Model IO project -# -# SPDX-License-Identifier: MPL-2.0 - -import logging - -import structlog -from power_grid_model import CalculationType, PowerGridModel -from power_grid_model.validation import assert_valid_input_data - -from power_grid_model_io.converters.pgm_json_converter import PgmJsonConverter -from power_grid_model_io.converters.vision_excel_converter import VisionExcelConverter - -structlog.configure(wrapper_class=structlog.make_filtering_bound_logger(logging.INFO)) - -# Source and destination file -src = "data/vision/example.xlsx" -dst = "data/vision/example_output.json" - -# Convert Vision file -vision_converter = VisionExcelConverter(source_file=src) -input_data, extra_info = vision_converter.load_input_data() - -# Validate data -assert_valid_input_data(input_data, calculation_type=CalculationType.power_flow, symmetric=True) - -# Perform power flow calculation -grid = PowerGridModel(input_data=input_data) -output_data = grid.calculate_power_flow() - -# Store the results in JSON format -converter = PgmJsonConverter(destination_file=dst) -converter.save(data=output_data, extra_info=extra_info) diff --git a/pyproject.toml b/pyproject.toml index 2fa46dcb..1e8ab1f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,7 +48,7 @@ dev = [ ] doc = [ "sphinx", - "myst-parser", + "myst-nb", "sphinx_rtd_theme", "readthedocs-sphinx-search", "sphinx-hoverxref",