From edf401c5fa25e059d16d24692879999ae463d364 Mon Sep 17 00:00:00 2001
From: Lukasz Kawczynski <lukasz@kawczyn.ski>
Date: Tue, 3 Mar 2020 09:54:26 +0000
Subject: [PATCH] Preserve None values in DictFields.

Fixes #1378. Fixes #2051.
---
 mongoengine/base/document.py    | 25 +++++++++++++++++---
 tests/fields/test_dict_field.py | 41 +++++++++++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 3 deletions(-)

diff --git a/mongoengine/base/document.py b/mongoengine/base/document.py
index ad691362d..549910b23 100644
--- a/mongoengine/base/document.py
+++ b/mongoengine/base/document.py
@@ -644,6 +644,7 @@ def _delta(self):
 
         set_fields = self._get_changed_fields()
         unset_data = {}
+        deleted_sentinel = object()
         if hasattr(self, "_changed_fields"):
             set_data = {}
             # Fetch each set item from its path
@@ -660,7 +661,7 @@ def _delta(self):
                         d = d[int(p)]
                     elif hasattr(d, "get"):
                         # dict-like (dict, embedded document)
-                        d = d.get(p)
+                        d = d.get(p, deleted_sentinel)
                     new_path.append(p)
                 path = ".".join(new_path)
                 set_data[path] = d
@@ -669,10 +670,14 @@ def _delta(self):
             if "_id" in set_data:
                 del set_data["_id"]
 
+        DictField = _import_class("DictField")
+
         # Determine if any changed items were actually unset.
         for path, value in set_data.items():
-            if value or isinstance(
-                value, (numbers.Number, bool)
+            if (
+                value
+                and value != deleted_sentinel
+                or isinstance(value, (numbers.Number, bool))
             ):  # Account for 0 and True that are truthy
                 continue
 
@@ -690,7 +695,15 @@ def _delta(self):
             else:  # Perform a full lookup for lists / embedded lookups
                 d = self
                 db_field_name = parts.pop()
+                preserve = False
                 for p in parts:
+                    if value != deleted_sentinel and hasattr(d, "_fields"):
+                        # Preserve None values in DictFields
+                        field_name = d._reverse_db_field_map.get(p, p)
+                        if isinstance(d._fields.get(field_name), DictField):
+                            preserve = True
+                            break
+
                     if isinstance(d, list) and p.isdigit():
                         d = d[int(p)]
                     elif hasattr(d, "__getattribute__") and not isinstance(d, dict):
@@ -699,6 +712,9 @@ def _delta(self):
                     else:
                         d = d.get(p)
 
+                if preserve:
+                    continue
+
                 if hasattr(d, "_fields"):
                     field_name = d._reverse_db_field_map.get(
                         db_field_name, db_field_name
@@ -711,6 +727,9 @@ def _delta(self):
             if default is not None:
                 default = default() if callable(default) else default
 
+            if value == deleted_sentinel:
+                value = None
+
             if value != default:
                 continue
 
diff --git a/tests/fields/test_dict_field.py b/tests/fields/test_dict_field.py
index 44e628f6c..c5307aefc 100644
--- a/tests/fields/test_dict_field.py
+++ b/tests/fields/test_dict_field.py
@@ -354,3 +354,44 @@ class Simple(Document):
         assert isinstance(s.mapping7["someint"][0]["d"], Doc)
         assert isinstance(s.mapping8["someint"][0]["d"][0], Doc)
         assert isinstance(s.mapping9["someint"][0]["d"][0], Doc)
+
+    def test_dictfield_with_none_values(self):
+        class Doc(Document):
+            field = DictField()
+
+        Doc.drop_collection()
+        Doc(field={"key": "value", "key2": None}).save()
+        d = Doc.objects.first()
+        assert {"key": "value", "key2": None} == d.field
+
+        d.field["key"] = None
+        d.save()
+        d = Doc.objects.first()
+        assert {"key": None, "key2": None} == d.field
+
+        del d.field["key"]
+        d.save()
+        d = Doc.objects.first()
+        assert {"key2": None} == d.field
+
+    def test_embedded_dictfield_with_none_values(self):
+        class EmbeddedDoc(EmbeddedDocument):
+            field = DictField()
+
+        class Doc(Document):
+            field = ListField(EmbeddedDocumentField(EmbeddedDoc))
+
+        Doc.drop_collection()
+        Doc(field=[EmbeddedDoc(field={"key": "value", "key2": None})]).save()
+        d = Doc.objects.first()
+        assert {"key": "value", "key2": None} == d.field[0].field
+
+        d.field[0].field["key"] = None
+        d.save()
+        d = Doc.objects.first()
+        assert {"key": None, "key2": None} == d.field[0].field
+
+        del d.field[0].field["key"]
+        d.save()
+        d = Doc.objects.first()
+        assert {"key2": None} == d.field[0].field