diff --git a/.gitignore b/.gitignore index bc1e1b4..3bd9c65 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ /dist /build /.* +/solr_doc_manager.egg-info \ No newline at end of file diff --git a/mongo_connector/doc_managers/solr_doc_manager.py b/mongo_connector/doc_managers/solr_doc_manager.py index c49dc35..d141345 100755 --- a/mongo_connector/doc_managers/solr_doc_manager.py +++ b/mongo_connector/doc_managers/solr_doc_manager.py @@ -37,7 +37,7 @@ from mongo_connector.doc_managers.doc_manager_base import DocManagerBase from mongo_connector.doc_managers.formatters import DocumentFlattener -__version__ = '0.1.0' +__version__ = '0.1.1' """Solr DocManager version.""" @@ -54,6 +54,80 @@ decoder = json.JSONDecoder() + +class MongoUpdateSpecV1(object): + """ + Examples of object received from mongo + For updating fields + { + "$v" : 1, + "$set" : { + "company_name" : "new comp name" + } + } + + For deleting fields + { + "$v" : 1, + "$unset" : { + "company_name" : "" + } + } + """ + def __init__(self, update_spec) -> None: + self.update_spec = update_spec + + def is_a_update(self): + return '$set' in self.update_spec or '$unset' in self.update_spec + + def set_fields(self): + return self.update_spec.get('$set',{}).items() + + def unset_fields(self): + return self.update_spec.get('$unset',{}).keys() + +class MongoUpdateSpecV2(object): + """ + Examples of object received from mongo + For updating fields + { + "$v" : 2, + "diff" : { + "u" : { + "company_name" : "new comp name", + "website_url" : "https://www.namename.com" + } + } + } + For deleting fields + { + "$v" : 2, + "diff" : { + "d" : { + "deleted_field1" : false, + "deleted_field2" : false + } + } + } + """ + def __init__(self, update_spec) -> None: + self.update_spec = update_spec + + def is_a_update(self): + diff = self.update_spec.get('diff',{}) + return 'u' in diff or 'd' in diff + + def set_fields(self): + return self.update_spec.get('diff',{}).get('u',{}).items() + + def unset_fields(self): + return self.update_spec.get('diff',{}).get('d', {}).keys() + + +def _parse_mongo_update_spec(update_spec): + version = update_spec.get('$v', -1) + return MongoUpdateSpecV2(update_spec) if version == 2 else MongoUpdateSpecV1(update_spec) + class DocManager(DocManagerBase): """The DocManager class creates a connection to the backend engine and adds/removes documents, and in the case of rollback, searches for them. @@ -194,36 +268,37 @@ def handle_command(self, doc, namespace, timestamp): def apply_update(self, doc, update_spec): """Override DocManagerBase.apply_update to have flat documents.""" + mongo_spec = _parse_mongo_update_spec(update_spec) # Replace a whole document - if not '$set' in update_spec and not '$unset' in update_spec: + if not mongo_spec.is_a_update(): # update_spec contains the new document. # Update the key in Solr based on the unique_key mentioned as # parameter. update_spec['_id'] = doc[self.unique_key] return update_spec - for to_set in update_spec.get("$set", []): - value = update_spec['$set'][to_set] + for set_field_name, field_value in mongo_spec.set_fields(): # Find dotted-path to the value, remove that key from doc, then # put value at key: keys_to_pop = [] for key in doc: - if key.startswith(to_set): - if key == to_set or key[len(to_set)] == '.': + if key.startswith(set_field_name): + if key == set_field_name or key[len(set_field_name)] == '.': keys_to_pop.append(key) for key in keys_to_pop: doc.pop(key) - doc[to_set] = value - for to_unset in update_spec.get("$unset", []): + doc[set_field_name] = field_value + for unset_field_name in mongo_spec.unset_fields(): # MongoDB < 2.5.2 reports $unset for fields that don't exist within # the document being updated. keys_to_pop = [] for key in doc: - if key.startswith(to_unset): - if key == to_unset or key[len(to_unset)] == '.': + if key.startswith(unset_field_name): + if key == unset_field_name or key[len(unset_field_name)] == '.': keys_to_pop.append(key) for key in keys_to_pop: doc.pop(key) return doc + @wrap_exceptions def update(self, document_id, update_spec, namespace, timestamp): diff --git a/setup.py b/setup.py index e62a61f..0f6b401 100644 --- a/setup.py +++ b/setup.py @@ -22,7 +22,7 @@ long_description = None # Install without README.rst setup(name='solr-doc-manager', - version='0.1.0', + version='0.1.1', description='Solr plugin for mongo-connector', long_description=long_description, platforms=['any'], diff --git a/tests/test_solr_doc_manager.py b/tests/test_solr_doc_manager.py index e7ba84a..1a8b956 100755 --- a/tests/test_solr_doc_manager.py +++ b/tests/test_solr_doc_manager.py @@ -32,7 +32,7 @@ class TestSolrDocManager(SolrTestCase): def setUp(self): """Empty Solr at the start of every test """ - self._remove() + #self._remove() def test_update(self): doc_id = '1' @@ -73,6 +73,71 @@ def test_replacement_unique_key(self): replaced = docman.apply_update(from_solr, replacement) self.assertEqual('unique key replaced!', replaced['title']) + def test_setting_of_fields_for_v1_format(self): + docman = DocManager(solr_url, unique_key='id') + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + replacement = {'$v': 1, '$set':{ 'n1': 'changed name 1', 'n2':'changed name 2'}} + replaced = docman.apply_update(from_solr, replacement) + self.assertEqual('changed name 1', replaced['n1']) + self.assertEqual('changed name 2', replaced['n2']) + self.assertEqual('name 3', replaced['n3']) + + def test_unsetting_of_fields_for_v1_format(self): + docman = DocManager(solr_url, unique_key='id') + # Document coming from Solr. 'id' is the unique key. + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + # Replacement coming from an oplog entry in MongoDB. + replacement = {'$v': 1, '$unset':{ 'n1': '', 'n2':''}} + replaced = docman.apply_update(from_solr, replacement) + self.assertFalse('n1' in replaced) + self.assertFalse('n2' in replaced) + + self.assertEqual('name 3', replaced['n3']) + + def test_setting_of_fields_for_v2_format(self): + docman = DocManager(solr_url, unique_key='id') + # Document coming from Solr. 'id' is the unique key. + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + # Replacement coming from an oplog entry in MongoDB. + replacement = {'$v': 2, 'diff': {'u':{ 'n1': 'changed name 1', 'n2':'changed name 2'}}} + replaced = docman.apply_update(from_solr, replacement) + self.assertEqual('changed name 1', replaced['n1']) + self.assertEqual('changed name 2', replaced['n2']) + self.assertEqual('name 3', replaced['n3']) + + def test_unsetting_of_fields_for_v2_format(self): + docman = DocManager(solr_url, unique_key='id') + # Document coming from Solr. 'id' is the unique key. + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + # Replacement coming from an oplog entry in MongoDB. + replacement = {'$v': 2, 'diff': {'d':{ 'n1': False, 'n2': False}}} + replaced = docman.apply_update(from_solr, replacement) + self.assertFalse('n1' in replaced) + self.assertFalse('n2' in replaced) + + self.assertEqual('name 3', replaced['n3']) + + def test_setting_to_v1_format_incase_of_missing_version_field(self): + docman = DocManager(solr_url, unique_key='id') + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + replacement = {'$set':{ 'n1': 'changed name 1', 'n2':'changed name 2'}} + replaced = docman.apply_update(from_solr, replacement) + self.assertEqual('changed name 1', replaced['n1']) + self.assertEqual('changed name 2', replaced['n2']) + self.assertEqual('name 3', replaced['n3']) + + def test_unsetting_to_v1_format_incase_of_missing_version_field(self): + docman = DocManager(solr_url, unique_key='id') + # Document coming from Solr. 'id' is the unique key. + from_solr = {'id': 1, 'n1': 'name 1', 'n2':'name 2', 'n3': 'name 3'} + # Replacement coming from an oplog entry in MongoDB. + replacement = {'$unset':{ 'n1': '', 'n2':''}} + replaced = docman.apply_update(from_solr, replacement) + self.assertFalse('n1' in replaced) + self.assertFalse('n2' in replaced) + + self.assertEqual('name 3', replaced['n3']) + def test_upsert(self): """Ensure we can properly insert into Solr via DocManager. """