diff --git a/README.md b/README.md index 8947039..49796cf 100644 --- a/README.md +++ b/README.md @@ -81,6 +81,27 @@ class MyModel(models.Model): date_field = models.DateField(auto_now_add=True) ``` +#### DecimalField Support + +Django-mongo-backend supports `DecimalField` for storing decimal values as BSON Decimal128: + +- **Django 5.2+**: Use the built-in `django.db.models.DecimalField` directly +- **Django < 5.2**: Use `django_mongodb.models.DecimalField` for proper MongoDB compatibility + +```python +# For Django 5.2+ +from django.db import models + +class Product(models.Model): + price = models.DecimalField(max_digits=10, decimal_places=2) + +# For Django < 5.2 +from django_mongodb.models import DecimalField + +class Product(models.Model): + price = DecimalField(max_digits=10, decimal_places=2) +``` + Single table inheritance ```python diff --git a/django_mongodb/fields.py b/django_mongodb/fields.py new file mode 100644 index 0000000..14dfa98 --- /dev/null +++ b/django_mongodb/fields.py @@ -0,0 +1,23 @@ +from django.db import models + + +class DecimalField(models.DecimalField): + """ + Custom DecimalField that properly converts Python Decimal to BSON Decimal128 + for MongoDB operations. + + This custom field is required for Django < 5.2 to ensure proper conversion + of Decimal values during query operations. Starting from Django 5.2, the + built-in DecimalField should handle this conversion automatically. + + Note: When Django 5.2+ is the minimum supported version, this custom field + can be deprecated in favor of Django's built-in DecimalField. + """ + + def get_db_prep_value(self, value, connection, prepared=False): + value = super().get_db_prep_value(value, connection, prepared) + if hasattr(connection.ops, "adapt_decimalfield_value"): + return connection.ops.adapt_decimalfield_value( + value, max_digits=self.max_digits, decimal_places=self.decimal_places + ) + return value diff --git a/django_mongodb/models.py b/django_mongodb/models.py index 17f1601..560b7bc 100644 --- a/django_mongodb/models.py +++ b/django_mongodb/models.py @@ -4,6 +4,8 @@ from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ +from django_mongodb.fields import DecimalField + class ObjectIdFieldMixin: description = "MongoDB ObjectIdField" @@ -57,3 +59,6 @@ def get_internal_type(self): def rel_db_type(self, connection): return ObjectIdField().db_type(connection=connection) + + +__all__ = ["ObjectIdField", "ObjectIdAutoField", "DecimalField"] diff --git a/django_mongodb/operations.py b/django_mongodb/operations.py index 11a441e..8a93915 100644 --- a/django_mongodb/operations.py +++ b/django_mongodb/operations.py @@ -1,6 +1,8 @@ import datetime +from decimal import Decimal from bson import ObjectId +from bson.decimal128 import Decimal128 from django.conf import settings from django.db.backends.base.operations import BaseDatabaseOperations from django.utils.timezone import is_aware, make_aware @@ -59,6 +61,29 @@ def convert_datetime_value(self, value, expression, connection): return make_aware(value) return value + def convert_decimalfield_value(self, value, expression, connection): + """ + Convert BSON Decimal128 back to Python Decimal. + Also handles string values for backward compatibility. + """ + if value is None: + return None + + # If it's a Decimal128, convert to Python Decimal + if isinstance(value, Decimal128): + return value.to_decimal() + + # If it's a string, convert to Decimal + if isinstance(value, str): + return Decimal(value) + + # If it's already a Decimal, return as is + if isinstance(value, Decimal): + return value + + # Fallback to Decimal conversion + return Decimal(str(value)) + def get_db_converters(self, expression): converters = super().get_db_converters(expression) internal_type = expression.output_field.get_internal_type() @@ -69,4 +94,28 @@ def get_db_converters(self, expression): converters.append(self.convert_date_value) case "DateTimeField": converters.append(self.convert_datetime_value) + case "DecimalField": + converters.append(self.convert_decimalfield_value) return converters + + def adapt_decimalfield_value(self, value, max_digits=None, decimal_places=None): + """ + Transform a decimal.Decimal value to a BSON Decimal128 object. + """ + if value is None: + return None + + # If it's already a Decimal128, return as is + if isinstance(value, Decimal128): + return value + + # Convert to Decimal if it's a string + if isinstance(value, str): + value = Decimal(value) + + # Convert Python Decimal to BSON Decimal128 + if isinstance(value, Decimal): + return Decimal128(str(value)) + + # Fallback to string representation + return Decimal128(str(value)) diff --git a/django_mongodb/query.py b/django_mongodb/query.py index 68a8dc5..1b3b010 100644 --- a/django_mongodb/query.py +++ b/django_mongodb/query.py @@ -87,6 +87,13 @@ def _get_mongo_query(self, compiler, connection, is_search=False) -> dict: rhs = self.rhs if is_search and self.mongo_meta["search_fields"].get(lhs.attname): return {} + # Convert the rhs value using the field's database conversion method + if hasattr(lhs, "get_db_prep_value"): + # For In lookups, rhs is a list and we need to convert each item + if self.filter_operator == "$in" and isinstance(rhs, list | tuple): + rhs = [lhs.get_db_prep_value(item, connection) for item in rhs] + else: + rhs = lhs.get_db_prep_value(rhs, connection) return {lhs.column: {self.filter_operator: rhs}} def get_mongo_search(self, compiler, connection) -> dict: diff --git a/test/conftest.py b/test/conftest.py index 307bc79..eb662e6 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -14,11 +14,16 @@ def clean_db(): @pytest.fixture() def search_index(): + # Ensure the collection exists by creating a dummy document and then deleting it + collection = connections["mongodb"].cursor().connection["testapp_foomodel"] + collection.insert_one({"_id": "dummy"}) + collection.delete_one({"_id": "dummy"}) + try: - connections["mongodb"].cursor().connection["testapp_foomodel"].drop_search_index("default") + collection.drop_search_index("default") except Exception: pass - connections["mongodb"].cursor().connection["testapp_foomodel"].create_search_index( + collection.create_search_index( SearchIndexModel( { "analyzer": "lucene.standard", @@ -49,9 +54,7 @@ def search_index(): # we need to wait until the index is ready while True: time.sleep(1.0) - indexes = list( - connections["mongodb"].cursor().connection["testapp_foomodel"].list_search_indexes() - ) + indexes = list(collection.list_search_indexes()) if any(index["status"] == "READY" for index in indexes): break i += 1 diff --git a/test/test_decimal_field_compatibility.py b/test/test_decimal_field_compatibility.py new file mode 100644 index 0000000..341e56d --- /dev/null +++ b/test/test_decimal_field_compatibility.py @@ -0,0 +1,122 @@ +""" +Tests to ensure DecimalField compatibility across Django versions. +For Django < 5.2, we need to use the custom DecimalField from django_mongodb. +For Django >= 5.2, the built-in DecimalField should work correctly. +""" + +from decimal import Decimal + +import django +import pytest +from django.db import models + +from django_mongodb.models import DecimalField as MongoDecimalField + + +# Test model using built-in Django DecimalField +class BuiltinDecimalModel(models.Model): + value = models.DecimalField(max_digits=10, decimal_places=2) + + class Meta: + app_label = "testapp" + db_table = "test_builtin_decimal" + + +# Test model using custom MongoDB DecimalField +class MongoDecimalModel(models.Model): + value = MongoDecimalField(max_digits=10, decimal_places=2) + + class Meta: + app_label = "testapp" + db_table = "test_mongo_decimal" + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_custom_decimal_field_always_works(): + """Custom MongoDB DecimalField should work on all Django versions.""" + # Create and save + obj = MongoDecimalModel(value=Decimal("123.45")) + obj.save() + + # Retrieve + retrieved = MongoDecimalModel.objects.get(pk=obj.pk) + assert retrieved.value == Decimal("123.45") + + # Filter + filtered = MongoDecimalModel.objects.filter(value=Decimal("123.45")) + assert filtered.count() == 1 + + # Cleanup + MongoDecimalModel.objects.all().delete() + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_builtin_decimal_field_compatibility(): + """ + Built-in DecimalField should work on Django >= 5.2. + For Django < 5.2, this test demonstrates the issue that requires + using the custom DecimalField. + """ + django_version = tuple(map(int, django.__version__.split(".")[:2])) + + # Create and save (this should always work) + obj = BuiltinDecimalModel(value=Decimal("456.78")) + obj.save() + + # Retrieve (this should always work) + retrieved = BuiltinDecimalModel.objects.get(pk=obj.pk) + assert retrieved.value == Decimal("456.78") + + # Filter - this is where the issue occurs on Django < 5.2 + if django_version >= (5, 2): + # On Django 5.2+, built-in DecimalField should work + filtered = BuiltinDecimalModel.objects.filter(value=Decimal("456.78")) + assert filtered.count() == 1 + else: + # On Django < 5.2, filtering with built-in DecimalField fails + # with BSON encoding error + with pytest.raises(Exception) as exc_info: + list(BuiltinDecimalModel.objects.filter(value=Decimal("456.78"))) + assert "cannot encode object" in str(exc_info.value) or "Decimal" in str(exc_info.value) + + # Cleanup + BuiltinDecimalModel.objects.all().delete() + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_recommendation(): + """ + This test documents the recommended approach for different Django versions. + """ + django_version = tuple(map(int, django.__version__.split(".")[:2])) + + if django_version >= (5, 2): + # For Django 5.2+, both should work + recommendation = "Django >= 5.2: You can use either django.db.models.DecimalField or django_mongodb.models.DecimalField" + else: + # For Django < 5.2, custom field is required + recommendation = "Django < 5.2: You must use django_mongodb.models.DecimalField for proper MongoDB compatibility" + + # This assertion always passes - it's just to document the recommendation + assert recommendation is not None + + # Verify the recommendation by testing both fields + test_value = Decimal("999.99") + + # Custom field should always work + mongo_obj = MongoDecimalModel.objects.create(value=test_value) + assert MongoDecimalModel.objects.filter(value=test_value).count() == 1 + mongo_obj.delete() + + # Built-in field behavior depends on Django version + builtin_obj = BuiltinDecimalModel.objects.create(value=test_value) + if django_version >= (5, 2): + assert BuiltinDecimalModel.objects.filter(value=test_value).count() == 1 + else: + try: + # This will fail on Django < 5.2 + list(BuiltinDecimalModel.objects.filter(value=test_value)) + raise AssertionError("Expected BSON encoding error on Django < 5.2") + except Exception as e: + assert "cannot encode object" in str(e) or "Decimal" in str(e) + builtin_obj.delete() diff --git a/test/test_models.py b/test/test_models.py index 6ab0354..3fa0987 100644 --- a/test/test_models.py +++ b/test/test_models.py @@ -2,16 +2,22 @@ import os import time from datetime import timedelta +from decimal import Decimal import pytest from bson import ObjectId +from bson.decimal128 import Decimal128 +from django.conf import settings from django.contrib.postgres.search import SearchQuery, SearchVector +from django.db import models from django.utils.timezone import now +from pymongo import MongoClient from django_mongodb.expressions import RawMongoDBQuery from django_mongodb.query import RequiresSearchIndex from refapp.models import RefModel from testapp.models import ( + DecimalFieldModel, DifferentTableOneToOne, FooModel, RelatedModel, @@ -327,3 +333,244 @@ def test_reference_model(): assert len(RefModel.objects.all()) == 1 assert len(RefModel.objects.filter(name="foo").all()) == 1 RefModel.objects.filter(name="foo").delete() + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_basic(): + """Test basic decimal field CRUD operations with BSON Decimal128.""" + # Test create with default value + obj = DecimalFieldModel.objects.create() + assert obj.value == Decimal("0") + + # Test create with specific value + test_value = Decimal("123.45") + obj2 = DecimalFieldModel.objects.create(value=test_value) + obj2.refresh_from_db() + assert obj2.value == test_value + assert isinstance(obj2.value, Decimal) + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_precision(): + """Test decimal field precision preservation.""" + # Test various precision values + test_cases = [ + Decimal("0.01"), + Decimal("99999999.99"), # Max for 10 digits, 2 decimal places + Decimal("-123.45"), + Decimal("0.00"), + Decimal("1234567.89"), + ] + + for test_value in test_cases: + obj = DecimalFieldModel.objects.create(value=test_value) + obj.refresh_from_db() + assert obj.value == test_value + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_string_conversion(): + """Test storing and retrieving string decimals.""" + # Create object with string value + obj = DecimalFieldModel.objects.create(value="456.78") + obj.refresh_from_db() + assert obj.value == Decimal("456.78") + assert isinstance(obj.value, Decimal) + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_none_handling(): + """Test None/null handling in decimal fields.""" + + # Create a model with nullable decimal field for proper None testing + class NullableDecimalModel(models.Model): + value = models.DecimalField(null=True, blank=True, decimal_places=2, max_digits=10) + + class Meta: + app_label = "testapp" + db_table = "testapp_nullabledecimalmodel" + + # Create object with None value + obj = NullableDecimalModel.objects.create(value=None) + obj.refresh_from_db() + assert obj.value is None + + # Update None to a decimal value + obj.value = Decimal("123.45") + obj.save() + obj.refresh_from_db() + assert obj.value == Decimal("123.45") + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_filtering(): + """Test filtering operations on decimal fields.""" + DecimalFieldModel.objects.all().delete() + + # Create test data + values = [Decimal("10.50"), Decimal("20.75"), Decimal("30.00"), Decimal("5.25")] + for val in values: + DecimalFieldModel.objects.create(value=val) + + # Test exact match + assert DecimalFieldModel.objects.filter(value=Decimal("10.50")).count() == 1 + + # Test greater than + assert DecimalFieldModel.objects.filter(value__gt=Decimal("20")).count() == 2 + + # Test less than or equal + assert DecimalFieldModel.objects.filter(value__lte=Decimal("20.75")).count() == 3 + + # Test range + assert ( + DecimalFieldModel.objects.filter(value__gte=Decimal("10"), value__lte=Decimal("25")).count() + == 2 + ) + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_ordering(): + """Test ordering by decimal fields.""" + DecimalFieldModel.objects.all().delete() + + values = [Decimal("30.00"), Decimal("10.50"), Decimal("20.75"), Decimal("5.25")] + for val in values: + DecimalFieldModel.objects.create(value=val) + + # Test ascending order + ordered = list(DecimalFieldModel.objects.order_by("value").values_list("value", flat=True)) + assert ordered == [Decimal("5.25"), Decimal("10.50"), Decimal("20.75"), Decimal("30.00")] + + # Test descending order + ordered_desc = list( + DecimalFieldModel.objects.order_by("-value").values_list("value", flat=True) + ) + assert ordered_desc == [Decimal("30.00"), Decimal("20.75"), Decimal("10.50"), Decimal("5.25")] + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_update(): + """Test updating decimal fields.""" + obj = DecimalFieldModel.objects.create(value=Decimal("100.00")) + + # Update via save + obj.value = Decimal("200.50") + obj.save() + obj.refresh_from_db() + assert obj.value == Decimal("200.50") + + # Update via queryset + DecimalFieldModel.objects.filter(id=obj.id).update(value=Decimal("300.75")) + obj.refresh_from_db() + assert obj.value == Decimal("300.75") + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_edge_cases(): + """Test edge cases for decimal fields.""" + # Test zero + obj = DecimalFieldModel.objects.create(value=Decimal("0")) + obj.refresh_from_db() + assert obj.value == Decimal("0") + + # Test negative zero (should be normalized to zero) + obj2 = DecimalFieldModel.objects.create(value=Decimal("-0")) + obj2.refresh_from_db() + assert obj2.value == Decimal("0") + + # Test very small decimal + obj3 = DecimalFieldModel.objects.create(value=Decimal("0.01")) + obj3.refresh_from_db() + assert obj3.value == Decimal("0.01") + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_aggregation(): + """Test aggregation operations on decimal fields.""" + DecimalFieldModel.objects.all().delete() + + # Create test data + values = [Decimal("10.50"), Decimal("20.75"), Decimal("30.00")] + for val in values: + DecimalFieldModel.objects.create(value=val) + + # Test exists + assert DecimalFieldModel.objects.filter(value__gt=Decimal("0")).exists() + + # Test count + assert DecimalFieldModel.objects.filter(value__gte=Decimal("20")).count() == 2 + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_decimal_field_backward_compatibility(): + """Test backward compatibility with legacy string decimal storage.""" + + # Connect directly to MongoDB to insert string decimal values + db_settings = settings.DATABASES["mongodb"] + client = MongoClient(**db_settings["CLIENT"]) + db = client[db_settings["NAME"]] + collection = db["testapp_decimalfieldmodel"] + + # Clean up + collection.delete_many({}) + + # Insert legacy string decimal values directly + collection.insert_many( + [ + {"value": "123.45"}, # String decimal + {"value": Decimal128("678.90")}, # BSON Decimal128 + {"value": None}, # Null value + ] + ) + + # Test that Django can read all formats correctly + all_objects = list(DecimalFieldModel.objects.all()) + assert len(all_objects) == 3 + + # Check that we can read all values correctly + values = [obj.value for obj in all_objects] + assert None in values + assert Decimal("123.45") in values + assert Decimal("678.90") in values + + # Test filtering works with null values + assert DecimalFieldModel.objects.filter(value__isnull=True).count() == 1 + assert DecimalFieldModel.objects.filter(value__isnull=False).count() == 2 + + # Update all values to ensure they're stored as BSON Decimal128 + for obj in DecimalFieldModel.objects.all(): + if obj.value is not None: + obj.save() # This will convert string to Decimal128 + + # Now filtering should work correctly + assert DecimalFieldModel.objects.filter(value__gt=Decimal("200")).count() == 1 + assert DecimalFieldModel.objects.filter(value__lt=Decimal("200")).count() == 1 + + +@pytest.mark.django_db(databases=["mongodb"]) +def test_in_lookup_field_conversion(): + """Test that __in lookups properly convert field values.""" + # Test with ObjectId fields (FooModel) + item1 = FooModel.objects.create(name="test1", json_field={"foo": "bar"}) + item2 = FooModel.objects.create(name="test2", json_field={"foo": "baz"}) + + # Test ObjectId __in lookup + result = FooModel.objects.filter(id__in=[item1.id, item2.id]) + assert result.count() == 2 + + # Test with string IDs + result = FooModel.objects.filter(id__in=[str(item1.id), str(item2.id)]) + assert result.count() == 2 + + # Test with DecimalField + DecimalFieldModel.objects.all().delete() + DecimalFieldModel.objects.create(value=Decimal("10.50")) + DecimalFieldModel.objects.create(value=Decimal("20.75")) + + # Test Decimal __in lookup + result = DecimalFieldModel.objects.filter(value__in=[Decimal("10.50"), Decimal("20.75")]) + assert result.count() == 2 + + # Test with mixed types (should still work due to field conversion) + result = DecimalFieldModel.objects.filter(value__in=[Decimal("10.50"), "20.75"]) + assert result.count() == 2 diff --git a/testapp/models.py b/testapp/models.py index 3659090..c3c0474 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -1,7 +1,10 @@ +from decimal import Decimal + from django.db import models from django.db.models import JSONField from django_mongodb.managers import MongoManager +from django_mongodb.models import DecimalField class FooModel(models.Model): @@ -63,3 +66,11 @@ class DifferentTableOneToOne(models.Model): class RelatedModel(models.Model): name = models.CharField(max_length=100) foo = models.ForeignKey(FooModel, on_delete=models.CASCADE, related_name="related") + + +class DecimalFieldModel(models.Model): + value = DecimalField( + default=Decimal("0"), + decimal_places=2, + max_digits=10, + )