Does not update document changes for mongdb 5.x

.gitignore

@@ -6,3 +6,4 @@
 /dist
 /build
 /.*
+/solr_doc_manager.egg-info

mongo_connector/doc_managers/solr_doc_manager.py

@@ -37,7 +37,7 @@
 from mongo_connector.doc_managers.doc_manager_base import DocManagerBase
 from mongo_connector.doc_managers.formatters import DocumentFlattener
 
-__version__ = '0.1.0'
+__version__ = '0.1.1'
 """Solr DocManager version."""
 
 
@@ -54,6 +54,80 @@
 decoder = json.JSONDecoder()
 
 
+
+class MongoUpdateSpecV1(object):
+    """
+     Examples of object received from mongo
+     For updating fields
+     {
+		"$v" : 1,
+		"$set" : {
+			"company_name" : "new comp name"
+		}
+	}
+
+    For deleting fields
+     {
+		"$v" : 1,
+		"$unset" : {
+			"company_name" : ""
+		}
+	}
+    """
+    def __init__(self, update_spec) -> None:
+        self.update_spec = update_spec
+
+    def is_a_update(self):
+        return '$set' in self.update_spec or '$unset' in self.update_spec
+
+    def set_fields(self):
+        return self.update_spec.get('$set',{}).items()
+
+    def unset_fields(self):
+        return self.update_spec.get('$unset',{}).keys()
+
+class MongoUpdateSpecV2(object):
+    """
+     Examples of object received from mongo
+     For updating fields
+    {
+		"$v" : 2,
+		"diff" : {
+			"u" : {
+				"company_name" : "new comp name",
+				"website_url" : "https://www.namename.com"
+			}
+		}
+	}
+    For deleting fields
+    {
+		"$v" : 2,
+		"diff" : {
+			"d" : {
+				"deleted_field1" : false,
+				"deleted_field2" : false
+			}
+		}
+	}
+    """
+    def __init__(self, update_spec) -> None:
+        self.update_spec = update_spec
+
+    def is_a_update(self):
+        diff = self.update_spec.get('diff',{})
+        return 'u' in diff or 'd' in diff
+
+    def set_fields(self):
+        return self.update_spec.get('diff',{}).get('u',{}).items()
+
+    def unset_fields(self):
+        return self.update_spec.get('diff',{}).get('d', {}).keys()
+
+
+def _parse_mongo_update_spec(update_spec):
+    version = update_spec.get('$v', -1)
+    return MongoUpdateSpecV2(update_spec) if version == 2 else MongoUpdateSpecV1(update_spec)
+
 class DocManager(DocManagerBase):
     """The DocManager class creates a connection to the backend engine and
     adds/removes documents, and in the case of rollback, searches for them.
@@ -194,36 +268,37 @@ def handle_command(self, doc, namespace, timestamp):
 
     def apply_update(self, doc, update_spec):
         """Override DocManagerBase.apply_update to have flat documents."""
+        mongo_spec = _parse_mongo_update_spec(update_spec)
         # Replace a whole document
-        if not '$set' in update_spec and not '$unset' in update_spec:
+        if not mongo_spec.is_a_update():
             # update_spec contains the new document.
             # Update the key in Solr based on the unique_key mentioned as
             # parameter.
             update_spec['_id'] = doc[self.unique_key]
             return update_spec
-        for to_set in update_spec.get("$set", []):
-            value = update_spec['$set'][to_set]
+        for set_field_name, field_value in mongo_spec.set_fields():
             # Find dotted-path to the value, remove that key from doc, then
             # put value at key:
             keys_to_pop = []
             for key in doc:
-                if key.startswith(to_set):
-                    if key == to_set or key[len(to_set)] == '.':
+                if key.startswith(set_field_name):
+                    if key == set_field_name or key[len(set_field_name)] == '.':
                         keys_to_pop.append(key)
             for key in keys_to_pop:
                 doc.pop(key)
-            doc[to_set] = value
-        for to_unset in update_spec.get("$unset", []):
+            doc[set_field_name] = field_value
+        for unset_field_name in mongo_spec.unset_fields():
             # MongoDB < 2.5.2 reports $unset for fields that don't exist within
             # the document being updated.
             keys_to_pop = []
             for key in doc:
-                if key.startswith(to_unset):
-                    if key == to_unset or key[len(to_unset)] == '.':
+                if key.startswith(unset_field_name):
+                    if key == unset_field_name or key[len(unset_field_name)] == '.':
                         keys_to_pop.append(key)
             for key in keys_to_pop:
                 doc.pop(key)
         return doc
+
 
     @wrap_exceptions
     def update(self, document_id, update_spec, namespace, timestamp):

setup.py

@@ -22,7 +22,7 @@
     long_description = None  # Install without README.rst
 
 setup(name='solr-doc-manager',
-      version='0.1.0',
+      version='0.1.1',
       description='Solr plugin for mongo-connector',
       long_description=long_description,
       platforms=['any'],

tests/test_solr_doc_manager.py

@@ -32,7 +32,7 @@ class TestSolrDocManager(SolrTestCase):
     def setUp(self):
         """Empty Solr at the start of every test
         """
-        self._remove()
+        #self._remove()
 
     def test_update(self):
         doc_id = '1'
@@ -73,6 +73,71 @@ def test_replacement_unique_key(self):
         replaced = docman.apply_update(from_solr, replacement)
         self.assertEqual('unique key replaced!', replaced['title'])
 
+    def test_setting_of_fields_for_v1_format(self):
+        docman = DocManager(solr_url, unique_key='id')
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        replacement = {'$v': 1, '$set':{ 'n1': 'changed name 1', 'n2':'changed name 2'}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertEqual('changed name 1', replaced['n1'])
+        self.assertEqual('changed name 2', replaced['n2'])
+        self.assertEqual('name 3', replaced['n3'])
+
+    def test_unsetting_of_fields_for_v1_format(self):
+        docman = DocManager(solr_url, unique_key='id')
+        # Document coming from Solr. 'id' is the unique key.
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        # Replacement coming from an oplog entry in MongoDB. 
+        replacement = {'$v': 1, '$unset':{ 'n1': '', 'n2':''}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertFalse('n1' in replaced)
+        self.assertFalse('n2' in replaced)
+
+        self.assertEqual('name 3', replaced['n3'])
+
+    def test_setting_of_fields_for_v2_format(self):
+        docman = DocManager(solr_url, unique_key='id')
+        # Document coming from Solr. 'id' is the unique key.
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        # Replacement coming from an oplog entry in MongoDB. 
+        replacement = {'$v': 2, 'diff': {'u':{ 'n1': 'changed name 1', 'n2':'changed name 2'}}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertEqual('changed name 1', replaced['n1'])
+        self.assertEqual('changed name 2', replaced['n2'])
+        self.assertEqual('name 3', replaced['n3'])
+
+    def test_unsetting_of_fields_for_v2_format(self):
+        docman = DocManager(solr_url, unique_key='id')
+        # Document coming from Solr. 'id' is the unique key.
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        # Replacement coming from an oplog entry in MongoDB. 
+        replacement = {'$v': 2, 'diff': {'d':{ 'n1': False, 'n2': False}}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertFalse('n1' in replaced)
+        self.assertFalse('n2' in replaced)
+
+        self.assertEqual('name 3', replaced['n3'])
+
+    def test_setting_to_v1_format_incase_of_missing_version_field(self):
+        docman = DocManager(solr_url, unique_key='id')
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        replacement = {'$set':{ 'n1': 'changed name 1', 'n2':'changed name 2'}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertEqual('changed name 1', replaced['n1'])
+        self.assertEqual('changed name 2', replaced['n2'])
+        self.assertEqual('name 3', replaced['n3'])
+
+    def test_unsetting_to_v1_format_incase_of_missing_version_field(self):
+        docman = DocManager(solr_url, unique_key='id')
+        # Document coming from Solr. 'id' is the unique key.
+        from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'}
+        # Replacement coming from an oplog entry in MongoDB. 
+        replacement = {'$unset':{ 'n1': '', 'n2':''}}
+        replaced = docman.apply_update(from_solr, replacement)
+        self.assertFalse('n1' in replaced)
+        self.assertFalse('n2' in replaced)
+
+        self.assertEqual('name 3', replaced['n3'])
+
     def test_upsert(self):
         """Ensure we can properly insert into Solr via DocManager.
         """