optimizations to improve performance, add optional ormar-utils packag…

.github/workflows/benchmark.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1.4
         with:
-          version: 2.1.3
+          version: 2.3.2
           virtualenvs-create: false
 
       - name: Poetry details

.github/workflows/deploy-docs.yml

@@ -21,7 +21,7 @@ jobs:
       - name: Install Poetry
         uses: snok/install-poetry@v1.4
         with:
-          version: 2.1.3
+          version: 2.3.2
           virtualenvs-create: false
       - name: Install dependencies
         run: |

.github/workflows/lint.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

.github/workflows/python-publish.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: true
         virtualenvs-in-project: true
 

.github/workflows/test-package.yml

@@ -54,7 +54,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

.github/workflows/test_docs.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
     - name: Install dependencies
       run: |

.github/workflows/type-check.yml

@@ -26,7 +26,7 @@ jobs:
     - name: Install Poetry
       uses: snok/install-poetry@v1.4
       with:
-        version: 2.1.3
+        version: 2.3.2
         virtualenvs-create: false
 
     - name: Poetry details

benchmarks/conftest.py

@@ -1,7 +1,6 @@
 import asyncio
 import random
 import string
-import time
 
 import nest_asyncio
 import pytest
@@ -88,9 +87,6 @@ async def authors_in_db(num_models: int):
 
 
 @pytest_asyncio.fixture
-@pytest.mark.benchmark(
-    min_rounds=1, timer=time.process_time, disable_gc=True, warmup=False
-)
 async def aio_benchmark(benchmark):
     def _fixture_wrapper(func):
         def _func_wrapper(*args, **kwargs):

ormar/fields/parsers.py

@@ -12,6 +12,13 @@
 except ImportError:  # pragma: no cover
     import json  # type: ignore
 
+from ormar.utils.rust_utils import HAS_RUST, ormar_rust_utils
+
+if HAS_RUST:  # pragma: no cover
+    _rs_encode_bytes = ormar_rust_utils.encode_bytes
+    _rs_decode_bytes = ormar_rust_utils.decode_bytes
+    _rs_encode_json = ormar_rust_utils.encode_json
+
 
 def parse_bool(value: str) -> bool:
     return value == "true"
@@ -27,7 +34,9 @@ def encode_decimal(value: decimal.Decimal, precision: Optional[int] = None) -> f
     )
 
 
-def encode_bytes(value: Union[str, bytes], represent_as_string: bool = False) -> str:
+def _py_encode_bytes(  # pragma: no cover
+    value: Union[str, bytes], represent_as_string: bool = False
+) -> str:
     if represent_as_string:
         value = (
             value if isinstance(value, str) else base64.b64encode(value).decode("utf-8")
@@ -37,21 +46,23 @@ def encode_bytes(value: Union[str, bytes], represent_as_string: bool = False) ->
     return value
 
 
-def decode_bytes(value: str, represent_as_string: bool = False) -> bytes:
+def _py_decode_bytes(
+    value: str, represent_as_string: bool = False
+) -> bytes:  # pragma: no cover
     if represent_as_string:
         return value if isinstance(value, bytes) else base64.b64decode(value)
     return value if isinstance(value, bytes) else value.encode("utf-8")
 
 
-def encode_json(value: Any) -> Optional[str]:
+def _py_encode_json(value: Any) -> Optional[str]:  # pragma: no cover
     if isinstance(value, (datetime.date, datetime.datetime, datetime.time)):
         value = value.isoformat()
     value = json.dumps(value) if not isinstance(value, str) else re_dump_value(value)
     value = value.decode("utf-8") if isinstance(value, bytes) else value
     return value
 
 
-def re_dump_value(value: str) -> Union[str, bytes]:
+def re_dump_value(value: str) -> Union[str, bytes]:  # pragma: no cover
     """
     Re-dumps value due to different string representation in orjson and json
     :param value: string to re-dump
@@ -66,6 +77,11 @@ def re_dump_value(value: str) -> Union[str, bytes]:
     return result
 
 
+encode_bytes = _rs_encode_bytes if HAS_RUST else _py_encode_bytes
+decode_bytes = _rs_decode_bytes if HAS_RUST else _py_decode_bytes
+encode_json = _rs_encode_json if HAS_RUST else _py_encode_json
+
+
 ENCODERS_MAP: dict[type, Callable] = {
     datetime.datetime: lambda x: x.isoformat(),
     datetime.date: lambda x: x.isoformat(),

ormar/models/helpers/models.py

@@ -1,10 +1,13 @@
-import itertools
 from typing import TYPE_CHECKING, Any, ForwardRef
 
 import pydantic
 
 import ormar  # noqa: I100
 from ormar.models.helpers.pydantic import populate_pydantic_default_values
+from ormar.utils.rust_utils import HAS_RUST, ormar_rust_utils
+
+if HAS_RUST:  # pragma: no cover
+    _rs_group_related_list = ormar_rust_utils.group_related_list
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
@@ -112,19 +115,24 @@ def group_related_list(list_: list) -> dict:
     :return: list converted to dictionary to avoid repetition and group nested models
     :rtype: dict[str, list]
     """
-    result_dict: dict[str, Any] = dict()
-    list_.sort(key=lambda x: x.split("__")[0])
-    grouped = itertools.groupby(list_, key=lambda x: x.split("__")[0])
-    for key, group in grouped:
-        group_list = list(group)
-        new = sorted(
-            ["__".join(x.split("__")[1:]) for x in group_list if len(x.split("__")) > 1]
-        )
-        if any("__" in x for x in new):
-            result_dict[key] = group_related_list(new)
-        else:
-            result_dict.setdefault(key, []).extend(new)
-    return dict(sorted(result_dict.items(), key=lambda item: len(item[1])))
+    if HAS_RUST:  # pragma: no cover
+        return _rs_group_related_list(list_)
+    else:  # pragma: no cover
+        groups: dict[str, list[str]] = {}
+        for item in list_:
+            head, _, tail = item.partition("__")
+            groups.setdefault(head, [])
+            if tail:
+                groups[head].append(tail)
+
+        result_dict: dict[str, Any] = {}
+        for key in sorted(groups):
+            children = sorted(groups[key])
+            if any("__" in x for x in children):
+                result_dict[key] = group_related_list(children)
+            else:
+                result_dict.setdefault(key, []).extend(children)
+        return dict(sorted(result_dict.items(), key=lambda item: len(item[1])))
 
 
 def config_field_not_set(model: type["Model"], field_name: str) -> bool:

ormar/models/mixins/excludable_mixin.py

@@ -106,26 +106,17 @@ def own_table_columns(
         :rtype: list[str]
         """
         model_excludable = excludable.get(model_cls=model, alias=alias)  # type: ignore
-        columns = [
-            model.get_column_name_from_alias(col.name) if not use_alias else col.name
-            for col in model.ormar_config.table.columns
-        ]
-        field_names = [
-            model.get_column_name_from_alias(col.name)
-            for col in model.ormar_config.table.columns
-        ]
-        if model_excludable.include:
-            columns = [
-                col
-                for col, name in zip(columns, field_names)
-                if model_excludable.is_included(name)
-            ]
-        if model_excludable.exclude:
-            columns = [
-                col
-                for col, name in zip(columns, field_names)
-                if not model_excludable.is_excluded(name)
-            ]
+        has_include = bool(model_excludable.include)
+        has_exclude = bool(model_excludable.exclude)
+
+        columns = []
+        for col in model.ormar_config.table.columns:
+            field_name = model.get_column_name_from_alias(col.name)
+            if has_include and not model_excludable.is_included(field_name):
+                continue
+            if has_exclude and model_excludable.is_excluded(field_name):
+                continue
+            columns.append(col.name if use_alias else field_name)
 
         # always has to return pk column for ormar to work
         if add_pk_columns:

ormar/models/mixins/merge_mixin.py

@@ -2,6 +2,11 @@
 
 import ormar
 from ormar.queryset.utils import translate_list_to_dict
+from ormar.utils.rust_utils import HAS_RUST, ormar_rust_utils
+
+if HAS_RUST:  # pragma: no cover
+    _rs_group_by_pk = ormar_rust_utils.group_by_pk
+    _rs_plan_merge = ormar_rust_utils.plan_merge_items_lists
 
 if TYPE_CHECKING:  # pragma no cover
     from ormar import Model
@@ -56,14 +61,21 @@ def merge_instances_list(cls, result_rows: list["Model"]) -> list["Model"]:
         :rtype: list["Model"]
         """
         merged_rows: list["Model"] = []
-        grouped_instances: dict = {}
-
-        for model in result_rows:
-            grouped_instances.setdefault(model.pk, []).append(model)
 
-        for group in grouped_instances.values():
-            model = cls._recursive_add(group)[0]
-            merged_rows.append(model)
+        if HAS_RUST and result_rows:  # pragma: no cover
+            pks = [model.pk for model in result_rows]
+            index_groups = _rs_group_by_pk(pks)
+            for group_indices in index_groups:
+                group = [result_rows[i] for i in group_indices]
+                model = cls._recursive_add(group)[0]
+                merged_rows.append(model)
+        else:  # pragma: no cover
+            grouped_instances: dict = {}
+            for model in result_rows:
+                grouped_instances.setdefault(model.pk, []).append(model)
+            for group in grouped_instances.values():
+                model = cls._recursive_add(group)[0]
+                merged_rows.append(model)
 
         return merged_rows
 
@@ -149,18 +161,48 @@ def _merge_items_lists(
         :return: merged list of models
         :rtype: list[Model]
         """
-        value_to_set = [x for x in other_value]
-        for cur_field in current_field:
-            if cur_field in other_value:
-                old_value = next((x for x in other_value if x == cur_field), None)
-                new_val = cls.merge_two_instances(
-                    cur_field,
-                    cast("Model", old_value),
-                    relation_map=cur_field._skip_ellipsis(  # type: ignore
-                        relation_map, field_name, default_return=dict()
-                    ),
-                )
-                value_to_set = [x for x in value_to_set if x != cur_field] + [new_val]
-            else:
-                value_to_set.append(cur_field)
-        return value_to_set
+        if HAS_RUST:  # pragma: no cover
+            current_pks = [getattr(m, "pk", None) for m in current_field]
+            other_pks = [getattr(m, "pk", None) for m in other_value]
+            plan = _rs_plan_merge(current_pks, other_pks)
+            value_to_set = list(other_value)
+            for cur_idx, other_idx in plan:
+                cur_item = current_field[cur_idx]
+                if other_idx is not None:
+                    old_value = other_value[other_idx]
+                    new_val = cls.merge_two_instances(
+                        cur_item,
+                        cast("Model", old_value),
+                        relation_map=cur_item._skip_ellipsis(  # type: ignore
+                            relation_map, field_name, default_return=dict()
+                        ),
+                    )
+                    value_to_set = [
+                        x for x in value_to_set if getattr(x, "pk", None) != cur_item.pk
+                    ] + [new_val]
+                else:
+                    value_to_set.append(cur_item)
+            return value_to_set
+        else:  # pragma: no cover
+            other_by_pk: dict = {}
+            for idx, item in enumerate(other_value):
+                other_by_pk.setdefault(item.pk, idx)
+
+            value_to_set = list(other_value)
+            for cur_field in current_field:
+                other_idx = other_by_pk.get(cur_field.pk)
+                if other_idx is not None:
+                    old_value = other_value[other_idx]
+                    new_val = cls.merge_two_instances(
+                        cur_field,
+                        cast("Model", old_value),
+                        relation_map=cur_field._skip_ellipsis(  # type: ignore
+                            relation_map, field_name, default_return=dict()
+                        ),
+                    )
+                    value_to_set = [x for x in value_to_set if x.pk != cur_field.pk] + [
+                        new_val
+                    ]
+                else:
+                    value_to_set.append(cur_field)
+            return value_to_set