From b5b75d3c334489db6330d6d28a8edf96a39fee0e Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Thu, 25 Apr 2013 11:51:02 -0700 Subject: [PATCH 01/14] Changed behavior of add, clear, and create to act more like Django --- mongom2m/fields.py | 59 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 18 deletions(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index e75d4e3..617460d 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -157,11 +157,21 @@ def __call__(self): def count(self): return len(self.objects) - def add(self, *objs): + def add(self, *objs, **kwargs): """ Add model instance(s) to the M2M field. The objects can be real Model instances or just ObjectIds (or strings representing ObjectIds). - """ + + Only supported kwarg is 'auto_save' + :param auto_save: Defaults to True. When a model is added to the M2M, + the behavior of Django is to create an entry in the + through-table, which essentially saves the list. In order to do + the equivalent, we need to save the model. However, that + behavior is not the same as Django either because Django doesn't + save the whole model object, so that's why this is optional. + Swings and Roundabouts. + """ + auto_save = kwargs.pop('auto_save', True) using = 'default' # should see if we can carry this over from somewhere add_objs = [] for obj in objs: @@ -180,32 +190,41 @@ def add(self, *objs): add_obj_ids = [str(obj['pk']) for obj in add_objs] # Send pre_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) + m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids) # Commit the add for obj in add_objs: self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) - + # Send post_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) - - return self - - def create(**kwargs): + m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids) + + if auto_save: + self.model_instance.save() + + def create(self, **kwargs): """ Create new model instance and add to the M2M field. """ + # See add() above for description of auto_save + auto_save = kwargs.pop('auto_save', True) + obj = self.rel.to(**kwargs) - self.add(obj) + self.add(obj, auto_save=auto_save) return obj - def remove(self, *objs): + def remove(self, *objs, **kwargs): """ Remove the specified object from the M2M field. The object can be a real model instance or an ObjectId or a string representing an ObjectId. The related object is not deleted, it's only removed from the list. + + Only supported kwarg is 'auto_save' + :param auto_save: See add() above for description """ + auto_save = kwargs.pop('auto_save', True) + obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) else ObjectId(obj.pk) for obj in objs]) # Calculate list of object ids that will be removed @@ -219,13 +238,16 @@ def remove(self, *objs): # Send the post_remove signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - - return self - - def clear(self): + + if auto_save: + self.model_instance.save() + + def clear(self, auto_save=True): """ - Clear all objecst in the list. The related objects are not + Clear all objects in the list. The related objects are not deleted from the database. + + :param auto_save: See add() above for description """ # Calculate list of object ids that will be removed removed_obj_ids = [str(obj['pk']) for obj in self.objects] @@ -238,9 +260,10 @@ def clear(self): # Send the post_clear signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) + + if auto_save: + self.model_instance.save() - return self - def __contains__(self, obj): """ Helper to enable 'object in container' by comparing IDs. From 95b64e9b4f31f1f7c02bdfd9b5c356de064121be Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Thu, 25 Apr 2013 15:13:52 -0700 Subject: [PATCH 02/14] Fixed some bugs with DB connection --- mongom2m/fields.py | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index 617460d..5e06dd3 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -1,11 +1,11 @@ -from djangotoolbox.fields import ListField, DictField, EmbeddedModelField, AbstractIterableField +# from djangotoolbox.fields import ListField, DictField, EmbeddedModelField, AbstractIterableField +from django.db import models, router from django.db.models.signals import m2m_changed -from django.db.models import get_model +# from django.db.models import get_model from django.db.models.fields.related import add_lazy_relation from django.utils.translation import ugettext_lazy as _ from django_mongodb_engine.contrib import MongoDBManager -from django.forms import ModelMultipleChoiceField -from django.db import models +# from django.forms import ModelMultipleChoiceField # How much to show when query set is viewed in the Python shell REPR_OUTPUT_SIZE = 20 @@ -24,7 +24,7 @@ class MongoDBM2MQuerySet(object): If embed=False, objects are always loaded from database. """ def __init__(self, rel, model, objects, use_cached, appear_as_relationship=(None, None, None, None, None)): - self.db = 'default' + self.db = router.db_for_read(rel.model if rel.model else rel.field.model) self.rel = rel self.objects = list(objects) # make a copy of the list to avoid problems self.model = model @@ -172,7 +172,8 @@ def add(self, *objs, **kwargs): Swings and Roundabouts. """ auto_save = kwargs.pop('auto_save', True) - using = 'default' # should see if we can carry this over from somewhere + using = router.db_for_write(self.model_instance if self.model_instance + else self.field.model) add_objs = [] for obj in objs: if isinstance(obj, (ObjectId, basestring)): @@ -190,14 +191,14 @@ def add(self, *objs, **kwargs): add_obj_ids = [str(obj['pk']) for obj in add_objs] # Send pre_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) # Commit the add for obj in add_objs: self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) # Send post_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) if auto_save: self.model_instance.save() @@ -359,7 +360,7 @@ def to_python(self, values): values = [values] self.objects = [self.to_python_embedded_instance(value) for value in values] - def get_db_prep_value_embedded_instance(self, obj): + def get_db_prep_value_embedded_instance(self, obj, connection, prepared=False): """ Convert an internal object value to database representation. """ @@ -375,18 +376,18 @@ def get_db_prep_value_embedded_instance(self, obj): values = {} for field in embedded_instance._meta.fields: value = field.pre_save(embedded_instance, add=True) - value = field.get_db_prep_value(value) + value = field.get_db_prep_value(value, connection=connection, prepared=prepared) values[field.column] = value # Convert primary key into an ObjectId so it's stored correctly values[self.rel.to._meta.pk.column] = ObjectId(values[self.rel.to._meta.pk.column]) return values - def get_db_prep_value(self): + def get_db_prep_value(self, connection, prepared=False): """ Convert the Django model instances managed by this manager into a special list that can be stored in MongoDB. """ - return [self.get_db_prep_value_embedded_instance(obj) for obj in self.objects] + return [self.get_db_prep_value_embedded_instance(obj, connection, prepared) for obj in self.objects] def create_through(field, model, to): """ @@ -622,7 +623,7 @@ def get_db_prep_value(self, value, connection, prepared=False): # Convert other values to manager objects first value = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed, value) # Let the manager to the conversion - return value.get_db_prep_value() + return value.get_db_prep_value(connection, prepared) def to_python(self, value): # The database value is a custom MongoDB list of ObjectIds and embedded models (if embed is enabled). From 77df23d399b5f55b013e83b367081eab89fa6758 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Fri, 26 Apr 2013 19:44:44 -0700 Subject: [PATCH 03/14] Fixed some bugs with A objects. Added a new "filter" option to search hosting models --- mongom2m/fields.py | 181 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 166 insertions(+), 15 deletions(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index 5e06dd3..94d9b84 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -1,21 +1,25 @@ -# from djangotoolbox.fields import ListField, DictField, EmbeddedModelField, AbstractIterableField from django.db import models, router +from django.db.models import Q from django.db.models.signals import m2m_changed -# from django.db.models import get_model from django.db.models.fields.related import add_lazy_relation from django.utils.translation import ugettext_lazy as _ -from django_mongodb_engine.contrib import MongoDBManager -# from django.forms import ModelMultipleChoiceField - -# How much to show when query set is viewed in the Python shell -REPR_OUTPUT_SIZE = 20 -# ObjectId has been moved to bson.objectid in newer versions of PyMongo +from django_mongodb_engine.contrib import MongoDBManager +from django_mongodb_engine.query import A +import django_mongodb_engine.query try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo from bson.objectid import ObjectId except ImportError: from pymongo.objectid import ObjectId +# How much to show when query set is viewed in the Python shell +REPR_OUTPUT_SIZE = 20 + + +class MongoDBM2MQueryError(Exception): pass + + class MongoDBM2MQuerySet(object): """ Helper for returning a set of objects from the managers. @@ -36,7 +40,7 @@ def __init__(self, rel, model, objects, use_cached, appear_as_relationship=(None self.objects = [{'pk':obj['pk'], 'obj':None} for obj in self.objects] def _get_obj(self, obj): - if not obj['obj']: + if not obj.get('obj'): # Load referred instance from db and keep in memory obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) if self.appear_as_relationship_model: @@ -532,6 +536,91 @@ def __set__(self, obj, value): """ obj.__dict__[self.field.name] = self.field.to_python(value) + def filter(self, *args, **kwargs): + """Enables queries on the host-model-level for contents of this field. + That means calling this filter will return instances of the + MongoDBManyToManyField host model, not instances of the related model. + + If embed=True, anything can be queried. If embed=False, then only + model objects (or ids) can be compared. In this case, the only accepted + argument is 'pk'. The reason for this is because related models are + stored by pk. + + Warning: Only the first call to this method will actually behave + correctly. If you string multiple calls to filter together, the + remaining filters after the first will all act on the fields + of the host model. + + Example: + >>>class M2MModel(models.Model): + >>> name = models.CharField() + >>> + >>>class Host(models.Model): + >>> m2m = MongoDBManyToManyField(M2MModel, embed=True) + >>> + >>> m = M2MModel.objects.get(name="foo") + >>> Host.m2m.filter(pk=m) + [] + >>> Host.m2m.filter(name="foo") + [] + + Important Distinction: + The above example is acting on the Host model *class*, not instance. + Calling filter on an *instance* of Host model would return instances of + the M2M related model. Example: + + >>> h = Host.objects.get(id=1) + >>> h.m2m.filter(name="foo") + [] + + Author's Note: + I very much dislike this solution, but a better solution eludes me + right now. In order to get the behavior Django has, django-nonrel or + djangotoolbox need to be changed to support manytomany fields. + """ + def raise_query_error(): + raise MongoDBM2MQueryError( + "Invalid query paramaters: '%s'. M2M Fields not using the " + "'embed=True' option can only filter on 'pk' because only " + "the related model's pk is stored for non-embedded M2Ms. " + "Note: M2M fields that are converted to 'embed=True' do " + "not convert the stored values automatically. Every " + "instance of the host-model must be re-saved after " + "converting the field." % args, kwargs) + + embedded = self.field._mm2m_embed + column = self.field.column + + updated_args = [] + # Iterate over the arguments and replace them with A objects + for field in args: + if isinstance(field, Q): + # Some args may be Qs. This function replaces the Q children + # with A() objects. + status = _replace_Q(field, column, + ["pk"] if not embedded else None) + if status: + updated_args.append(field) + else: + raise_query_error() + else: + # Anything else should be tuples of two items + updated_args.append( + (self.field.column, _combine_A(field[0], field[1]))) + + updated_kwargs = [] + # Iterate over the kwargs and combine them into A objects + for field, value in kwargs.iteritems(): + if not embedded and field != 'pk': + raise_query_error() + + # Have to build Q objects because all the arguments will have the + # same key in the kwargs otherwise + updated_kwargs.append(Q(**{column: _combine_A(field, value)})) + + return self.field.model.objects.filter(*(updated_args+updated_kwargs)) + + class MongoDBManyToManyRel(object): """ This object holds the information of the M2M relationship. @@ -588,7 +677,7 @@ def __init__(self, to, related_name=None, embed=False, default=None, *args, **kw self._mm2m_related_name = related_name self._mm2m_embed = embed models.Field.__init__(self, *args, **kwargs) - + def contribute_after_resolving(self, field, to, model): # Setup the main relation helper self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, self._mm2m_embed) @@ -607,15 +696,25 @@ def contribute_after_resolving(self, field, to, model): # Add the relationship descriptor to the model class for Django admin/forms to work setattr(model, self.name, MongoDBManyToManyRelationDescriptor(self, self.rel.through)) - def contribute_to_class(self, model, name, *args, **kwargs): + def contribute_to_class(self, model, name): self.__m2m_name = name # Call Field, not super, to skip Django's ManyToManyField extra stuff we don't need - models.Field.contribute_to_class(self, model, name, *args, **kwargs) + models.Field.contribute_to_class(self, model, name) # Do the rest after resolving the 'to' relation add_lazy_relation(model, self, self._mm2m_to_or_name, self.contribute_after_resolving) def db_type(self, *args, **kwargs): return 'list' + + def get_db_prep_lookup(self, lookup_type, value, connection, + prepared=False): + # This is necessary because the ManyToManyField.get_db_prep_lookup will + # convert 'A' objects into a unicode string. We don't want that. + if isinstance(value, A): + return value + else: + return super(MongoDBManyToManyField, self).get_db_prep_lookup( + lookup_type, value, connection, prepared) def get_db_prep_value(self, value, connection, prepared=False): # The Python value is a MongoDBM2MRelatedManager, and we'll store the models it contains as a special list. @@ -633,6 +732,58 @@ def to_python(self, value): manager.to_python(value) value = manager return value - -# def formfield(self, **kwargs): -# return super(MongoDBManyToManyField, self).formfield(**kwargs) + + +def _replace_Q(q, column, allowed_fields=None): + if not isinstance(q, Q): + raise ValueError("'q' must be of type Q, not: '%s'" % type(q)) + + # Iterate over the Q object's children. The children are either another Q, + # or a tuple of (,) + for child in q.children: + if isinstance(child, Q): + # If we have a Q in the children, let's recurse to fix it too + _replace_Q(child, column, allowed_fields) + elif isinstance(child, tuple): + # Otherwise we need to build an A(). Doing the index, remove, + # and insert to maintain the order of the children. I'm not sure + # changing the order matters, but I don't want to risk it. + index = q.children.index(child) + q.children.remove(child) + + # If allowed_fields is defined, this verifies them. E.g. ['pk'] + if allowed_fields and child[0] not in allowed_fields: + return False + # If all is well, build an A(), and insert back into the children + q.children.insert(index, (column, _combine_A(child[0], child[1]))) + else: + raise TypeError("Unknown type in Q.children") + return True + + +def _combine_A(field, value): + # The pk is actually stored as "id", so change it, we also need extract the + # pk from and models and wrap any IDs in an ObjectId, + if field in ('pk', 'id'): + field = "id" + if isinstance(value, models.Model): + # Specifically getattr field because we don't know if it's 'pk' + # or 'id' and they might not be the same thing. + value = getattr(value, field) + if not isinstance(value, ObjectId): + value = ObjectId(value) + + # If 'value' is already an A(), we need to extract the field part out + if isinstance(value, A): + field = "%s.%s" % (field, value.op) + value = value.val + return A(field, value) + + + +# Sort of hackish, but they left me no choice! Without this, 'A' objects are +# rejected for this field because it's not in "DJANGOTOOLBOX_FIELDS" +django_mongodb_engine.query.DJANGOTOOLBOX_FIELDS += \ + (MongoDBManyToManyField,) + + From 067030f470041dc8d2f5b448fb71c34fecf41f0f Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Mon, 29 Apr 2013 11:46:42 -0700 Subject: [PATCH 04/14] added exclude and get. also fixed a bug with pk=None --- mongom2m/fields.py | 42 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 5 deletions(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index 94d9b84..3afa2ee 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -536,7 +536,7 @@ def __set__(self, obj, value): """ obj.__dict__[self.field.name] = self.field.to_python(value) - def filter(self, *args, **kwargs): + def _filter_or_exclude(self, negate, *args, **kwargs): """Enables queries on the host-model-level for contents of this field. That means calling this filter will return instances of the MongoDBManyToManyField host model, not instances of the related model. @@ -580,13 +580,13 @@ def filter(self, *args, **kwargs): """ def raise_query_error(): raise MongoDBM2MQueryError( - "Invalid query paramaters: '%s'. M2M Fields not using the " + "Invalid query paramaters: '%s; %s'. M2M Fields not using the " "'embed=True' option can only filter on 'pk' because only " "the related model's pk is stored for non-embedded M2Ms. " "Note: M2M fields that are converted to 'embed=True' do " "not convert the stored values automatically. Every " "instance of the host-model must be re-saved after " - "converting the field." % args, kwargs) + "converting the field." % (args, kwargs)) embedded = self.field._mm2m_embed column = self.field.column @@ -618,7 +618,37 @@ def raise_query_error(): # same key in the kwargs otherwise updated_kwargs.append(Q(**{column: _combine_A(field, value)})) - return self.field.model.objects.filter(*(updated_args+updated_kwargs)) + query_args = updated_args + updated_kwargs + if negate: + return self.field.model.objects.exclude(*query_args) + else: + return self.field.model.objects.filter(*query_args) + + def filter(self, *args, **kwargs): + """See _filter_or_exclude() above for description""" + return self._filter_or_exclude(False, *args, **kwargs) + + def exclude(self, *args, **kwargs): + """See _filter_or_exclude() above for description""" + return self._filter_or_exclude(True, *args, **kwargs) + + def get(self, *args, **kwargs): + """Return a single object matching the query. + See _filter_or_exclude() above for more details. + """ + results = self.filter(*args, **kwargs) + num = len(results) + if num == 1: + return results[0] + elif num < 1: + raise self.field.model.DoesNotExist( + "%s matching query does not exist." + % self.field.model._meta.object_name) + else: + raise self.field.model.MultipleObjectsReturned( + "get() returned more than one %s -- it returned %s! " + "Lookup parameters were %s" + % (self.field.model._meta.object_name, num, kwargs)) class MongoDBManyToManyRel(object): @@ -770,7 +800,9 @@ def _combine_A(field, value): # Specifically getattr field because we don't know if it's 'pk' # or 'id' and they might not be the same thing. value = getattr(value, field) - if not isinstance(value, ObjectId): + + # If value is None, we want to leave it as None, otherwise wrap it + if value is not None and not isinstance(value, ObjectId): value = ObjectId(value) # If 'value' is already an A(), we need to extract the field part out From 58095a9baa04480eb9aa94cf2523de2647d2e5a5 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Mon, 29 Apr 2013 12:04:29 -0700 Subject: [PATCH 05/14] added docstring --- mongom2m/fields.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index 3afa2ee..77fb637 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -765,6 +765,22 @@ def to_python(self, value): def _replace_Q(q, column, allowed_fields=None): + """Replace the fields in the Q object with A() objects from 'column' + + :param q: The Q object to work on + :param column: The name of the column the A() objects should be attached to. + :param allowed_fields: If defined, only fields names listed in + 'allowed_fields' are allowed. + E.g. allowed_fields=["pk"]: Q(pk=1) is good, Q(name="tom") fails. + :returns: Boolean; False if 'allowed_fields' missed. True otherwise + + Example: + M2M field is called 'users' + _replace_Q(Q(name="Tom"), "users") would modify the given Q to be: + Q(users=A("name", "Tom")) + + That would generate the query: {"users.name":"Tom"} + """ if not isinstance(q, Q): raise ValueError("'q' must be of type Q, not: '%s'" % type(q)) @@ -781,7 +797,8 @@ def _replace_Q(q, column, allowed_fields=None): index = q.children.index(child) q.children.remove(child) - # If allowed_fields is defined, this verifies them. E.g. ['pk'] + # If allowed_fields is defined, this verifies that only those + # fields are present. E.g. ['pk'] if allowed_fields and child[0] not in allowed_fields: return False # If all is well, build an A(), and insert back into the children From 962e9d573b31e5171c5b41c5d8a42f11f4bf3dbc Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Tue, 30 Apr 2013 14:33:43 -0700 Subject: [PATCH 06/14] Updated the readme --- README.md | 94 +++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 81 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 69584bd..41077e1 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,34 @@ Django MongoDB ManyToManyField Implementation ============================================= -Created in 2012 by Kenneth Falck http://kfalck.net. +Created in 2012 by Kenneth Falck +Modified/Extended by Merchant Atlas Inc. 2013 Released under the standard BSD License (see below). Overview -------- -This is a simple implementation of ManyToManyFields for django-mongodb-engine. The MongoDBManyToManyField +This is a simple implementation of ManyToManyFields for django-mongodb-engine. The _MongoDBManyToManyField_ stores references to other Django model instances as ObjectIds in a MongoDB list field. -Optionally, MongoDBManyToManyField will also embed a "cached" copy of the instances inside the list. This +Optionally, _MongoDBManyToManyField_ will also embed a "cached" copy of the instances inside the list. This allows fast access to the data without having to query each related object from the database separately. -MongoDBManyToManyField attempts to work mostly in the same way as Django's built-in ManyToManyField. +_MongoDBManyToManyField_ attempts to work mostly in the same way as Django's built-in ManyToManyField. Related objects can be added and removed with the add(), remove(), clear() and create() methods. To enumerate the objects, the all() method returns a simulated QuerySet object which loads non-embedded objects automatically from the database when needed. -On the reverse side of the relation, an accessor property is added (usually called OtherModel.modelname\_set, +On the reverse side of the relation, an accessor property is added (usually called OtherModel.modelname\_set, can be overridden with the related\_name attribute) to return the related objects in the reverse direction. It uses MongoDB's raw\_query() to find all related model objects. Because of this, any data model that uses MongoDBManyToManyField() must have a default MongoDBManager() instead of Django's normal Manager(). -Django compability ------------------- +Django compatibility +-------------------- This implementation has been tweaked to be mostly compatible with Django admin, which means you can use TabularInlines or filter\_horizontal and filter\_vertical to administer the many-to-many fields. @@ -39,6 +40,7 @@ Don't be surprised, however, if some things don't work, because it's all emulate Usage ----- +### Define a field Example model using a many-to-many field: from django.db import models @@ -55,31 +57,96 @@ Example model using a many-to-many field: title = models.CharField(max_length=254) text = models.TextField() +### Add an instance To store categories in the field, you would first create the category and then add it: category = Category(title='foo') category.save() - + article = Article(title='bar') article.categories.add(category) - article.save() - + for cat in article.categories.all(): print cat.title for art in category.article_set.all(): print art.title +### Basic Querying +Querying with _MongoDBManyToManyField_ is similar to normal SQL Django, but with some caveats. +In order to have true Django behavior, we would have had to change some combination of: Django-nonrel, +Djangotoolbox, and mongodb-engine; so instead, we went a different route. + +How Django does it: + + Article.objects.filter(categories=category) + [, ] + + article.categories.all() + [, ] + +How _MongoDBManyToManyField_ does it: + + Article.categories.filter(pk=category) + [, ] + + article.categories.all() + [, ] + +### Embed models for performance and querying To enable embedding, just add the embed=True keyword argument to the field: class Article(models.Model): - categories = MongoDBManyToManyField(Category, embed=True) + categories = _MongoDBManyToManyField_(Category, embed=True) + +**Note about embedding**: If you change a _MongoDBManyToManyField_ from `embed=False` to +`embed=True` (or vice versa), the host model objects will not be automatically updated. +You will need to re-save every object of that model. +Example: + + for article in Article.objects.all(): + article.save() # Re-saving will now embed the categories automatically + +### Advanced Querying (Embedded models) +If you use `embed=True`, _MongoDBManyToManyField_ can do more than just query on 'pk'. +You can do any of: get, filter, and exclude; while using Q objects and A objects +(from mongodb-engine). _MongoDBManyToManyField_ will automatically convert them to +query correctly on the embedded document. +Note: The models have to be embedded because MongoDB doesn't support joins. + + Article.categories.filter(title="shirts") + Article.categories.filter(Q(title="shirts") | Q(title="hats")) + Article.categories.filter(Q(title="shirts") & ~Q(title="men's")) + + # If categories had an embedded model 'em', you could even query it with A() + Article.categories.filter(em=A("name", "em1")) + +### Limitations +There are some things that won't work with _MongoDBManyToManyField_: +#### Chaining multiple filters or excludes together + + # filter(title="hats").exclude(title="") will act on Article, not categories + Article.categories.filter(title="shirts").filter(title="hats").exclude(title="") + # However, the same can be accomplished with Q objects + Article.categories.filter(Q(title="shirts") & Q(title="hats") & ~Q(title="")) + +#### Double-underscore commands +This is actually an issue with djangotoolbox/mongobd-engine. Under the covers, +_MongoDBManyToManyField_ uses A() objects to generate the queries. Double-underscore +commands from A() objects don't work. + + # Will query the field 'title__contains', which doesn't exist + Article.categories.filter(title__contains="men") + + # Unfortunately, the only solution for this would be to do a raw_query + import re + Article.objects.raw_query({"categories.title": re.compile("men")}) Signals ------- -MongoDBManyToManyField supports Django's m2m\_changed signal, where the action can be: +_MongoDBManyToManyField_ supports Django's m2m\_changed signal, where the action can be: * pre\_add (triggered before adding object(s) to the field) * post\_add (triggered after adding object(s) to the field) @@ -136,8 +203,9 @@ Also make sure that the "id" field is properly indexed (see previous section). BSD License ----------- +Copyright (c) 2013 Merchant Atlas Inc. http://www.merchantatlas.com -Copyright (c) 2012 Kenneth Falck http://kfalck.net +[Original Work] Copyright (c) 2012 Kenneth Falck http://kfalck.net All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: From 98d143adcf58f17ccb40446b9c1dd094f2f10c0d Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Tue, 30 Apr 2013 16:40:47 -0700 Subject: [PATCH 07/14] Numerous changes including general code cleanup and fixing a bug with embedded model instances --- mongom2m/fields.py | 252 +++++++++++++++++++++++++++++++-------------- 1 file changed, 177 insertions(+), 75 deletions(-) diff --git a/mongom2m/fields.py b/mongom2m/fields.py index 77fb637..5d233b9 100644 --- a/mongom2m/fields.py +++ b/mongom2m/fields.py @@ -12,6 +12,7 @@ from bson.objectid import ObjectId except ImportError: from pymongo.objectid import ObjectId +from djangotoolbox.fields import ListField, EmbeddedModelField # How much to show when query set is viewed in the Python shell REPR_OUTPUT_SIZE = 20 @@ -27,17 +28,21 @@ class MongoDBM2MQuerySet(object): Lazily loads non-embedded objects when iterated. If embed=False, objects are always loaded from database. """ - def __init__(self, rel, model, objects, use_cached, appear_as_relationship=(None, None, None, None, None)): + def __init__(self, rel, model, objects, use_cached, + appear_as_relationship=(None, None, None, None, None)): self.db = router.db_for_read(rel.model if rel.model else rel.field.model) self.rel = rel self.objects = list(objects) # make a copy of the list to avoid problems self.model = model - self.appear_as_relationship_model, self.rel_model_instance, self.rel_to_instance, self.rel_model_name, self.rel_to_name = appear_as_relationship # appear as an intermediate m2m model + (self.appear_as_relationship_model, self.rel_model_instance, + self.rel_to_instance, self.rel_model_name, self.rel_to_name) = \ + appear_as_relationship # appear as an intermediate m2m model if self.appear_as_relationship_model: self.model = self.appear_as_relationship_model if not use_cached: # Reset any cached instances - self.objects = [{'pk':obj['pk'], 'obj':None} for obj in self.objects] + self.objects = [{'pk': obj['pk'], 'obj': None} + for obj in self.objects] def _get_obj(self, obj): if not obj.get('obj'): @@ -46,10 +51,15 @@ def _get_obj(self, obj): if self.appear_as_relationship_model: # Wrap us in a relationship class if self.rel_model_instance: - args = { 'pk':str(self.rel_model_instance.pk) + '$f$' + str(obj['pk']), self.rel_model_name:self.rel_model_instance, self.rel_to_name:obj['obj'] } + args = {'pk': "%s$f$%s" % + (self.rel_model_instance.pk, obj['pk']), + self.rel_model_name: self.rel_model_instance, + self.rel_to_name: obj['obj']} else: # Reverse - args = { 'pk':str(self.rel_to_instance.pk) + '$r$' + str(obj['pk']), self.rel_model_name:obj['obj'], self.rel_to_name:self.rel_to_instance } + args = {'pk': "%s$r$%s" % (self.rel_to_instance.pk, obj['pk']), + self.rel_model_name: obj['obj'], + self.rel_to_name: self.rel_to_instance } wrapper = self.appear_as_relationship_model(**args) return wrapper return obj['obj'] @@ -59,10 +69,11 @@ def __iter__(self): yield self._get_obj(obj) def __repr__(self): - data = list(self)[:REPR_OUTPUT_SIZE + 1] # limit list after conversion because mongodb doesn't use integer indices - if len(data) > REPR_OUTPUT_SIZE: + # limit list after conversion because mongodb doesn't use integer indices + data = list(self)[:REPR_OUTPUT_SIZE + 1] + if len(data) > REPR_OUTPUT_SIZE: data[-1] = "...(remaining elements truncated)..." - return repr(data) + return repr(data) def __getitem__(self, key): obj = self.objects[key] @@ -112,12 +123,16 @@ def all(self): pk = ObjectId(self.rel_field.pk) return self.model._default_manager.raw_query({name:pk}) - def _relationship_query_set(self, model, to_instance, model_module_name, to_module_name): + def _relationship_query_set(self, model, to_instance, model_module_name, + to_module_name): """ Emulate an intermediate 'through' relationship query set. """ objects = [{'pk':ObjectId(obj.pk), 'obj':obj} for obj in self.all()] - return MongoDBM2MQuerySet(self.rel, self.rel.to, objects, use_cached=True, appear_as_relationship=(model, None, to_instance, model_module_name, to_module_name)) + return MongoDBM2MQuerySet( + self.rel, self.rel.to, objects, use_cached=True, + appear_as_relationship=(model, None, to_instance, + model_module_name, to_module_name)) class MongoDBM2MReverseDescriptor(object): def __init__(self, model, field, rel, embed): @@ -129,7 +144,8 @@ def __init__(self, model, field, rel, embed): def __get__(self, instance, owner=None): if instance is None: return self - return MongoDBM2MReverseManager(instance, self.model, self.field, self.rel, self.embed) + return MongoDBM2MReverseManager(instance, self.model, self.field, + self.rel, self.embed) class MongoDBM2MRelatedManager(object): """ @@ -150,7 +166,9 @@ def _with_model_instance(self, model_instance): Create a new copy of this manager for a specific model instance. This is called when the field is being accessed through a model instance. """ - return MongoDBM2MRelatedManager(self.field, self.rel, self.embed, self.objects, model_instance=model_instance) + return MongoDBM2MRelatedManager( + self.field, self.rel, self.embed, + self.objects, model_instance=model_instance) def __call__(self): """ @@ -194,15 +212,21 @@ def add(self, *objs, **kwargs): # Calculate list of object ids that are being added add_obj_ids = [str(obj['pk']) for obj in add_objs] - # Send pre_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) + # Send pre_add signal (instance should be Through instance but it's the + # manager instance for now) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_add', reverse=False, model=self.rel.to, + pk_set=add_obj_ids, using=using) # Commit the add for obj in add_objs: self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) - # Send post_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) + # Send post_add signal (instance should be Through instance but it's + # the manager instance for now) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_add', reverse=False, model=self.rel.to, + pk_set=add_obj_ids, using=using) if auto_save: self.model_instance.save() @@ -230,19 +254,24 @@ def remove(self, *objs, **kwargs): """ auto_save = kwargs.pop('auto_save', True) - obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) else ObjectId(obj.pk) for obj in objs]) + obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) + else ObjectId(obj.pk) for obj in objs]) # Calculate list of object ids that will be removed removed_obj_ids = [str(obj['pk']) for obj in self.objects if obj['pk'] in obj_ids] # Send the pre_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) # Commit the remove self.objects = [obj for obj in self.objects if obj['pk'] not in obj_ids] # Send the post_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) if auto_save: self.model_instance.save() @@ -258,13 +287,17 @@ def clear(self, auto_save=True): removed_obj_ids = [str(obj['pk']) for obj in self.objects] # Send the pre_clear signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_clear', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) # Commit the clear self.objects = [] # Send the post_clear signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_clear', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) if auto_save: self.model_instance.save() @@ -293,7 +326,8 @@ def all(self, **kwargs): is enabled, returns embedded objects. Otherwise the query set will retrieve the objects from the database as needed. """ - return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=True, **kwargs) + return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, + use_cached=True, **kwargs) def ids(self): """ @@ -307,7 +341,8 @@ def objs(self): the database. This won't use embedded objects even if they exist. """ - return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=False) + return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, + use_cached=False) def to_python_embedded_instance(self, embedded_instance): """ @@ -316,10 +351,27 @@ def to_python_embedded_instance(self, embedded_instance): """ if isinstance(embedded_instance, ObjectId): # It's an object id, probably from a ListField(ForeignKey) migration - return {'pk':embedded_instance, 'obj':None} + return {'pk': embedded_instance, 'obj': None} elif isinstance(embedded_instance, basestring): - # Assume it's a string formatted object id, probably from a ListField(ForeignKey) migration - return {'pk':ObjectId(embedded_instance), 'obj':None} + # Assume it's a string formatted object id, probably from a + # ListField(ForeignKey) migration + return {'pk': ObjectId(embedded_instance), 'obj': None} + + elif isinstance(embedded_instance, tuple): + # This is the typical path for embedded instances (embed=True) + # The tuples is format: (, ) + cls, values = embedded_instance + if len(values) == 1 and 'id' in values and \ + len(cls._meta.fields) > 1: + # In this case, the user most likely just switched from + # embed=False to embed=True. We need to treat is as such + return self.to_python_embedded_instance( + {"id": ObjectId(values['id'])}) + else: + # Otherwise it's been embedded previously + instance = cls(**values) + return {'pk': ObjectId(instance.pk), 'obj': instance} + elif self.embed: # Try to load the embedded object contents if possible if isinstance(embedded_instance, dict): @@ -327,33 +379,42 @@ def to_python_embedded_instance(self, embedded_instance): data = {} for field in self.rel.to._meta.fields: try: - data[str(field.attname)] = embedded_instance[field.column] + data[str(field.attname)] = \ + embedded_instance[field.column] except KeyError: pass - # If we only got the id, give up to avoid creating an invalid/empty model instance + + # If we only got the id, give up to avoid creating an + # invalid/empty model instance if len(data) <= 1: - return {'pk':ObjectId(embedded_instance[self.rel.to._meta.pk.column]), 'obj':None} - # Otherwise create the model instance from the fields - obj = self.rel.to(**data) - # Make sure the pk in the model instance is a string (not ObjectId) to be compatible with django-mongodb-engine - if isinstance(obj.pk, ObjectId): - obj.pk = str(obj.pk) - return {'pk':ObjectId(obj.pk), 'obj':obj} + column = self.rel.to._meta.pk.column + return {'pk': ObjectId(embedded_instance[column]), + 'obj': None} + else: + # Otherwise create the model instance from the fields + obj = self.rel.to(**data) + # Make sure the pk in the model instance is a string + # (not ObjectId) to be compatible with django-mongodb-engine + if isinstance(obj.pk, ObjectId): + obj.pk = str(obj.pk) + return {'pk': ObjectId(obj.pk), 'obj': obj} else: # Assume it's already a model obj = embedded_instance - # Make sure the pk is a string (not ObjectId) to be compatible with django-mongodb-engine + # Make sure the pk is a string (not ObjectId) to be compatible + # with django-mongodb-engine if isinstance(obj.pk, ObjectId): obj.pk = str(obj.pk) - return {'pk':ObjectId(obj.pk), 'obj':obj} + return {'pk': ObjectId(obj.pk), 'obj': obj} else: # No embedded value, only ObjectId if isinstance(embedded_instance, dict): # Get the id value from the dict - return {'pk':ObjectId(embedded_instance[self.rel.to._meta.pk.column]), 'obj':None} + column = self.rel.to._meta.pk.column + return {'pk': ObjectId(embedded_instance[column]), 'obj': None} else: # Assume it's already a model - return {'pk':ObjectId(embedded_instance.pk), 'obj':None} + return {'pk': ObjectId(embedded_instance.pk), 'obj': None} def to_python(self, values): """ @@ -364,34 +425,37 @@ def to_python(self, values): values = [values] self.objects = [self.to_python_embedded_instance(value) for value in values] - def get_db_prep_value_embedded_instance(self, obj, connection, prepared=False): + def get_db_prep_value_embedded_instance(self, obj, connection): """ Convert an internal object value to database representation. """ if not obj: return None pk = obj['pk'] if not self.embed: - # Store only the ID - return { self.rel.to._meta.pk.column:pk } + # If we're not embedding, store only the ID + return {self.rel.to._meta.pk.column: pk} if not obj['obj']: # Retrieve the object from db for storing as embedded data obj['obj'] = self.rel.to.objects.get(pk=pk) embedded_instance = obj['obj'] values = {} + for field in embedded_instance._meta.fields: value = field.pre_save(embedded_instance, add=True) - value = field.get_db_prep_value(value, connection=connection, prepared=prepared) - values[field.column] = value + value = field.get_db_prep_save(value, connection=connection) + values[field] = value # Convert primary key into an ObjectId so it's stored correctly - values[self.rel.to._meta.pk.column] = ObjectId(values[self.rel.to._meta.pk.column]) + # values[self.rel.to._meta.pk] = ObjectId(values[self.rel.to._meta.pk]) return values def get_db_prep_value(self, connection, prepared=False): + """Convert the Django model instances managed by this manager into a + special list that can be stored in MongoDB. """ - Convert the Django model instances managed by this manager into a special list - that can be stored in MongoDB. - """ - return [self.get_db_prep_value_embedded_instance(obj, connection, prepared) for obj in self.objects] + values = [self.get_db_prep_value_embedded_instance(obj, connection) + for obj in self.objects] + return values + def create_through(field, model, to): """ @@ -531,8 +595,8 @@ def __get__(self, obj, type=None): def __set__(self, obj, value): """ - Attributes are being assigned to model instance. We redirect the assignments - to the model instance's fields instances. + Attributes are being assigned to model instance. We redirect the + assignments to the model instance's fields instances. """ obj.__dict__[self.field.name] = self.field.to_python(value) @@ -669,14 +733,13 @@ def __init__(self, field, to, related_name, embed): self.multiple = True self.limit_choices_to = {} - def is_hidden(self): return False def get_related_field(self, *args, **kwargs): return self.field -class MongoDBManyToManyField(models.ManyToManyField): +class MongoDBManyToManyField(models.ManyToManyField, ListField): """ A generic MongoDB many-to-many field that can store embedded copies of the referenced objects. Inherits from djangotoolbox.fields.ListField. @@ -701,41 +764,77 @@ class MongoDBManyToManyField(models.ManyToManyField): """ description = 'ManyToMany field with references and optional embedded objects' - def __init__(self, to, related_name=None, embed=False, default=None, *args, **kwargs): - # Call Field, not super, to skip Django's ManyToManyField extra stuff we don't need + def __init__(self, to, related_name=None, embed=False, *args, **kwargs): + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need self._mm2m_to_or_name = to self._mm2m_related_name = related_name self._mm2m_embed = embed - models.Field.__init__(self, *args, **kwargs) + if embed: + item_field = EmbeddedModelField(to) + else: + item_field = None + ListField.__init__(self, item_field, *args, **kwargs) def contribute_after_resolving(self, field, to, model): # Setup the main relation helper - self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, self._mm2m_embed) - # The field's default value will be an empty MongoDBM2MRelatedManager that's not connected to a model instance - self.default = MongoDBM2MRelatedManager(self, self.rel, self._mm2m_embed) + self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, + self._mm2m_embed) + # The field's default value will be an empty MongoDBM2MRelatedManager + # that's not connected to a model instance + self.default = MongoDBM2MRelatedManager(self, self.rel, + self._mm2m_embed) self.rel.model = model self.rel.through = create_through(self, self.rel.model, self.rel.to) # Determine related name automatically unless set if not self.rel.related_name: self.rel.related_name = model._meta.object_name.lower() + '_set' - #if hasattr(self.rel.to, self.rel.related_name): - # # Attribute name already taken, raise error - # raise Exception(u'Related name ' + unicode(self.rel.to._meta.object_name) + u'.' + unicode(self.rel.related_name) + u' is already used by another field, please choose another name with ' + unicode(name) + u' = ' + unicode(self.__class__.__name__) + u'(related_name=xxx)') + # Add the reverse relationship - setattr(self.rel.to, self.rel.related_name, MongoDBM2MReverseDescriptor(model, self, self.rel, self.rel.embed)) - # Add the relationship descriptor to the model class for Django admin/forms to work - setattr(model, self.name, MongoDBManyToManyRelationDescriptor(self, self.rel.through)) + setattr(self.rel.to, self.rel.related_name, + MongoDBM2MReverseDescriptor(model, self, self.rel, + self.rel.embed)) + # Add the relationship descriptor to the model class for Django + # admin/forms to work + setattr(model, self.name, + MongoDBManyToManyRelationDescriptor(self, self.rel.through)) def contribute_to_class(self, model, name): self.__m2m_name = name - # Call Field, not super, to skip Django's ManyToManyField extra stuff we don't need - models.Field.contribute_to_class(self, model, name) + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need + ListField.contribute_to_class(self, model, name) # Do the rest after resolving the 'to' relation - add_lazy_relation(model, self, self._mm2m_to_or_name, self.contribute_after_resolving) + add_lazy_relation(model, self, self._mm2m_to_or_name, + self.contribute_after_resolving) def db_type(self, *args, **kwargs): return 'list' + def get_internal_type(self): + return 'ListField' + + def formfield(self, **kwargs): + from django import forms + db = kwargs.pop('using', None) + defaults = { + 'form_class': forms.ModelMultipleChoiceField, + 'queryset': self.rel.to._default_manager.using(db).complex_filter( + self.rel.limit_choices_to) + } + defaults.update(kwargs) + # If initial is passed in, it's a list of related objects, but the + # MultipleChoiceField takes a list of IDs. + if defaults.get('initial') is not None: + initial = defaults['initial'] + if callable(initial): + initial = initial() + defaults['initial'] = [i._get_pk_val() for i in initial] + return models.Field.formfield(self, **defaults) + + def pre_save(self, model_instance, add): + return self.to_python(getattr(model_instance, self.attname)) + def get_db_prep_lookup(self, lookup_type, value, connection, prepared=False): # This is necessary because the ManyToManyField.get_db_prep_lookup will @@ -743,20 +842,23 @@ def get_db_prep_lookup(self, lookup_type, value, connection, if isinstance(value, A): return value else: - return super(MongoDBManyToManyField, self).get_db_prep_lookup( - lookup_type, value, connection, prepared) - + return models.ManyToManyField.get_db_prep_lookup( + self, lookup_type, value, connection, prepared) + def get_db_prep_value(self, value, connection, prepared=False): - # The Python value is a MongoDBM2MRelatedManager, and we'll store the models it contains as a special list. + # The Python value is a MongoDBM2MRelatedManager, and we'll store the + # models it contains as a special list. if not isinstance(value, MongoDBM2MRelatedManager): # Convert other values to manager objects first - value = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed, value) + value = MongoDBM2MRelatedManager(self, self.rel, + self.rel.embed, value) # Let the manager to the conversion return value.get_db_prep_value(connection, prepared) def to_python(self, value): - # The database value is a custom MongoDB list of ObjectIds and embedded models (if embed is enabled). - # We convert it into a MongoDBM2MRelatedManager object to hold the Django models. + # The database value is a custom MongoDB list of ObjectIds and embedded + # models (if embed is enabled). We convert it into a + # MongoDBM2MRelatedManager object to hold the Django models. if not isinstance(value, MongoDBM2MRelatedManager): manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) manager.to_python(value) From c44cb9fe5bcc24bc7bc283eb7a0dc332c4602c91 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Tue, 30 Apr 2013 16:49:48 -0700 Subject: [PATCH 08/14] Update README.md --- README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 41077e1..23332a4 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,7 @@ Django MongoDB ManyToManyField Implementation ============================================= -Created in 2012 by Kenneth Falck -Modified/Extended by Merchant Atlas Inc. 2013 +Created in 2012 by Kenneth Falck, Modified/Extended by Merchant Atlas Inc. 2013 Released under the standard BSD License (see below). @@ -93,7 +92,7 @@ How _MongoDBManyToManyField_ does it: article.categories.all() [, ] -### Embed models for performance and querying +### Embed Models for Performance and Querying To enable embedding, just add the embed=True keyword argument to the field: class Article(models.Model): @@ -124,6 +123,9 @@ Note: The models have to be embedded because MongoDB doesn't support joins. ### Limitations There are some things that won't work with _MongoDBManyToManyField_: #### Chaining multiple filters or excludes together +Under the covers, the initial filter is being called on the Article QuerySet, so what +gets returned is an Article QuerySet. That means calling filter() again will not get +any of the magic provided by _MongoDBManyToManyField_. # filter(title="hats").exclude(title="") will act on Article, not categories Article.categories.filter(title="shirts").filter(title="hats").exclude(title="") From 6392ccb2a9d4ca2ba3f1da51a8ed2e8ebfcb31ab Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Tue, 30 Apr 2013 17:56:22 -0700 Subject: [PATCH 09/14] Reorganized the directory structure --- LICENSE.txt | 4 +- README.md | 2 +- django_mongom2m/__init__.py | 16 + django_mongom2m/fields.py | 143 ++++++ .../fields.py => django_mongom2m/manager.py | 476 ++---------------- django_mongom2m/query.py | 95 ++++ django_mongom2m/utils.py | 182 +++++++ mongom2m/__init__.py | 0 setup.py | 10 +- 9 files changed, 485 insertions(+), 443 deletions(-) create mode 100644 django_mongom2m/__init__.py create mode 100644 django_mongom2m/fields.py rename mongom2m/fields.py => django_mongom2m/manager.py (53%) create mode 100644 django_mongom2m/query.py create mode 100644 django_mongom2m/utils.py delete mode 100644 mongom2m/__init__.py diff --git a/LICENSE.txt b/LICENSE.txt index 6d58702..dd33aaa 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,6 @@ -Copyright (c) 2012 Kenneth Falck http://kfalck.net +Copyright (c) 2013 Merchant Atlas Inc. http://www.merchantatlas.com + +[Original Work] Copyright (c) 2012 Kenneth Falck http://kfalck.net All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 41077e1..1304de0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ Usage Example model using a many-to-many field: from django.db import models - from mongom2m.fields import MongoDBManyToManyField + from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager class Category(models.Model): diff --git a/django_mongom2m/__init__.py b/django_mongom2m/__init__.py new file mode 100644 index 0000000..5b72140 --- /dev/null +++ b/django_mongom2m/__init__.py @@ -0,0 +1,16 @@ + +import django_mongodb_engine.query + +from . import fields +from . import manager +from . import query + + +# How much to show when query set is viewed in the Python shell +REPR_OUTPUT_SIZE = 20 + +# Sort of hackish, but they left me no choice! Without this, 'A' objects are +# rejected for this field because it's not in "DJANGOTOOLBOX_FIELDS" +django_mongodb_engine.query.DJANGOTOOLBOX_FIELDS += \ + (fields.MongoDBManyToManyField,) + diff --git a/django_mongom2m/fields.py b/django_mongom2m/fields.py new file mode 100644 index 0000000..ed8ae9e --- /dev/null +++ b/django_mongom2m/fields.py @@ -0,0 +1,143 @@ +from django.db import models +from django.db.models.fields.related import add_lazy_relation +from django_mongodb_engine.query import A +from djangotoolbox.fields import ListField, EmbeddedModelField + + +from .manager import (MongoDBManyToManyRel, MongoDBM2MRelatedManager, + MongoDBM2MReverseDescriptor, + MongoDBManyToManyRelationDescriptor) +from .utils import create_through + + +class MongoDBManyToManyField(models.ManyToManyField, ListField): + """ + A generic MongoDB many-to-many field that can store embedded copies of + the referenced objects. Inherits from djangotoolbox.fields.ListField. + + The field's value is a MongoDBM2MRelatedManager object that works similarly to Django's + RelatedManager objects, so you can add(), remove(), creaet() and clear() on it. + To access the related object instances, all() is supported. It will return + all the related instances, using the embedded copies if available. + + If you want the 'real' related (non-embedded) model instances, call all_objs() instead. + If you want the list of related ObjectIds, call all_refs() instead. + + The related model will also gain a new accessor method xxx_set() to make reverse queries. + That accessor is a MongoDBM2MReverseManager that provides an all() method to return + a QuerySet of related objects. + + For example, if you have an Article model with a MongoDBManyToManyField 'categories' + that refers to Category objects, you will have these methods: + + article.categories.all() - Returns all the categories that belong to the article + category.article_set.all() - Returns all the articles that belong to the category + """ + description = 'ManyToMany field with references and optional embedded objects' + + def __init__(self, to, related_name=None, embed=False, *args, **kwargs): + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need + self._mm2m_to_or_name = to + self._mm2m_related_name = related_name + self._mm2m_embed = embed + if embed: + item_field = EmbeddedModelField(to) + else: + item_field = None + ListField.__init__(self, item_field, *args, **kwargs) + + def contribute_after_resolving(self, field, to, model): + # Setup the main relation helper + self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, + self._mm2m_embed) + # The field's default value will be an empty MongoDBM2MRelatedManager + # that's not connected to a model instance + self.default = MongoDBM2MRelatedManager(self, self.rel, + self._mm2m_embed) + self.rel.model = model + self.rel.through = create_through(self, self.rel.model, self.rel.to) + # Determine related name automatically unless set + if not self.rel.related_name: + self.rel.related_name = model._meta.object_name.lower() + '_set' + + # Add the reverse relationship + setattr(self.rel.to, self.rel.related_name, + MongoDBM2MReverseDescriptor(model, self, self.rel, + self.rel.embed)) + # Add the relationship descriptor to the model class for Django + # admin/forms to work + setattr(model, self.name, + MongoDBManyToManyRelationDescriptor(self, self.rel.through)) + + def contribute_to_class(self, model, name): + self.__m2m_name = name + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need + ListField.contribute_to_class(self, model, name) + # Do the rest after resolving the 'to' relation + add_lazy_relation(model, self, self._mm2m_to_or_name, + self.contribute_after_resolving) + + def db_type(self, *args, **kwargs): + return 'list' + + def get_internal_type(self): + return 'ListField' + + def formfield(self, **kwargs): + from django import forms + db = kwargs.pop('using', None) + defaults = { + 'form_class': forms.ModelMultipleChoiceField, + 'queryset': self.rel.to._default_manager.using(db).complex_filter( + self.rel.limit_choices_to) + } + defaults.update(kwargs) + # If initial is passed in, it's a list of related objects, but the + # MultipleChoiceField takes a list of IDs. + if defaults.get('initial') is not None: + initial = defaults['initial'] + if callable(initial): + initial = initial() + defaults['initial'] = [i._get_pk_val() for i in initial] + return models.Field.formfield(self, **defaults) + + def pre_save(self, model_instance, add): + return self.to_python(getattr(model_instance, self.attname)) + + def get_db_prep_lookup(self, lookup_type, value, connection, + prepared=False): + # This is necessary because the ManyToManyField.get_db_prep_lookup will + # convert 'A' objects into a unicode string. We don't want that. + if isinstance(value, A): + return value + else: + return models.ManyToManyField.get_db_prep_lookup( + self, lookup_type, value, connection, prepared) + + def get_db_prep_value(self, value, connection, prepared=False): + # The Python value is a MongoDBM2MRelatedManager, and we'll store the + # models it contains as a special list. + if not isinstance(value, MongoDBM2MRelatedManager): + # Convert other values to manager objects first + value = MongoDBM2MRelatedManager(self, self.rel, + self.rel.embed, value) + # Let the manager to the conversion + return value.get_db_prep_value(connection, prepared) + + def to_python(self, value): + # The database value is a custom MongoDB list of ObjectIds and embedded + # models (if embed is enabled). We convert it into a + # MongoDBM2MRelatedManager object to hold the Django models. + if not isinstance(value, MongoDBM2MRelatedManager): + manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) + manager.to_python(value) + value = manager + return value + + + + + + diff --git a/mongom2m/fields.py b/django_mongom2m/manager.py similarity index 53% rename from mongom2m/fields.py rename to django_mongom2m/manager.py index 5d233b9..09e9a4c 100644 --- a/mongom2m/fields.py +++ b/django_mongom2m/manager.py @@ -1,108 +1,18 @@ + from django.db import models, router from django.db.models import Q from django.db.models.signals import m2m_changed -from django.db.models.fields.related import add_lazy_relation -from django.utils.translation import ugettext_lazy as _ - -from django_mongodb_engine.contrib import MongoDBManager -from django_mongodb_engine.query import A -import django_mongodb_engine.query try: # ObjectId has been moved to bson.objectid in newer versions of PyMongo from bson.objectid import ObjectId except ImportError: from pymongo.objectid import ObjectId -from djangotoolbox.fields import ListField, EmbeddedModelField - -# How much to show when query set is viewed in the Python shell -REPR_OUTPUT_SIZE = 20 +from .query import MongoDBM2MQuerySet, MongoDBM2MQueryError +from .utils import replace_Q, combine_A -class MongoDBM2MQueryError(Exception): pass -class MongoDBM2MQuerySet(object): - """ - Helper for returning a set of objects from the managers. - Works similarly to Django's own query set objects. - Lazily loads non-embedded objects when iterated. - If embed=False, objects are always loaded from database. - """ - def __init__(self, rel, model, objects, use_cached, - appear_as_relationship=(None, None, None, None, None)): - self.db = router.db_for_read(rel.model if rel.model else rel.field.model) - self.rel = rel - self.objects = list(objects) # make a copy of the list to avoid problems - self.model = model - (self.appear_as_relationship_model, self.rel_model_instance, - self.rel_to_instance, self.rel_model_name, self.rel_to_name) = \ - appear_as_relationship # appear as an intermediate m2m model - if self.appear_as_relationship_model: - self.model = self.appear_as_relationship_model - if not use_cached: - # Reset any cached instances - self.objects = [{'pk': obj['pk'], 'obj': None} - for obj in self.objects] - - def _get_obj(self, obj): - if not obj.get('obj'): - # Load referred instance from db and keep in memory - obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) - if self.appear_as_relationship_model: - # Wrap us in a relationship class - if self.rel_model_instance: - args = {'pk': "%s$f$%s" % - (self.rel_model_instance.pk, obj['pk']), - self.rel_model_name: self.rel_model_instance, - self.rel_to_name: obj['obj']} - else: - # Reverse - args = {'pk': "%s$r$%s" % (self.rel_to_instance.pk, obj['pk']), - self.rel_model_name: obj['obj'], - self.rel_to_name: self.rel_to_instance } - wrapper = self.appear_as_relationship_model(**args) - return wrapper - return obj['obj'] - - def __iter__(self): - for obj in self.objects: - yield self._get_obj(obj) - - def __repr__(self): - # limit list after conversion because mongodb doesn't use integer indices - data = list(self)[:REPR_OUTPUT_SIZE + 1] - if len(data) > REPR_OUTPUT_SIZE: - data[-1] = "...(remaining elements truncated)..." - return repr(data) - - def __getitem__(self, key): - obj = self.objects[key] - return self._get_obj(obj) - - def ordered(self, *args, **kwargs): - return self - - def __len__(self): - return len(self.objects) - - def using(self, db, *args, **kwargs): - self.db = db - return self - - def filter(self, *args, **kwargs): - return self - - def get(self, *args, **kwargs): - if 'pk' in kwargs: - pk = ObjectId(kwargs['pk']) - for obj in self.objects: - if pk == obj['pk']: - return self._get_obj(obj) - return None - - def count(self): - return len(self.objects) - class MongoDBM2MReverseManager(object): """ This manager is attached to the other side of M2M relationships @@ -114,7 +24,7 @@ def __init__(self, rel_field, model, field, rel, embed): self.field = field self.rel = rel self.embed = embed - + def all(self): """ Retrieve all related objects. @@ -122,7 +32,7 @@ def all(self): name = self.field.column + '.' + self.rel.model._meta.pk.column pk = ObjectId(self.rel_field.pk) return self.model._default_manager.raw_query({name:pk}) - + def _relationship_query_set(self, model, to_instance, model_module_name, to_module_name): """ @@ -140,7 +50,7 @@ def __init__(self, model, field, rel, embed): self.field = field self.rel = rel self.embed = embed - + def __get__(self, instance, owner=None): if instance is None: return self @@ -160,7 +70,7 @@ def __init__(self, field, rel, embed, objects=[], model_instance=None): self.rel = rel self.embed = embed self.objects = list(objects) # make copy of the list to avoid problems - + def _with_model_instance(self, model_instance): """ Create a new copy of this manager for a specific model instance. This @@ -169,16 +79,16 @@ def _with_model_instance(self, model_instance): return MongoDBM2MRelatedManager( self.field, self.rel, self.embed, self.objects, model_instance=model_instance) - + def __call__(self): """ This is used when creating a default value for the field """ return MongoDBM2MRelatedManager(self.field, self.rel, self.embed, self.objects) - + def count(self): return len(self.objects) - + def add(self, *objs, **kwargs): """ Add model instance(s) to the M2M field. The objects can be real @@ -208,16 +118,16 @@ def add(self, *objs, **kwargs): instance = obj if not pk in (obj['pk'] for obj in self.objects): add_objs.append({'pk':pk, 'obj':instance}) - + # Calculate list of object ids that are being added add_obj_ids = [str(obj['pk']) for obj in add_objs] - + # Send pre_add signal (instance should be Through instance but it's the # manager instance for now) m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) - + # Commit the add for obj in add_objs: self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) @@ -241,7 +151,7 @@ def create(self, **kwargs): obj = self.rel.to(**kwargs) self.add(obj, auto_save=auto_save) return obj - + def remove(self, *objs, **kwargs): """ Remove the specified object from the M2M field. @@ -256,18 +166,18 @@ def remove(self, *objs, **kwargs): obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) else ObjectId(obj.pk) for obj in objs]) - + # Calculate list of object ids that will be removed removed_obj_ids = [str(obj['pk']) for obj in self.objects if obj['pk'] in obj_ids] - + # Send the pre_remove signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - + # Commit the remove self.objects = [obj for obj in self.objects if obj['pk'] not in obj_ids] - + # Send the post_remove signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_remove', reverse=False, model=self.rel.to, @@ -285,15 +195,15 @@ def clear(self, auto_save=True): """ # Calculate list of object ids that will be removed removed_obj_ids = [str(obj['pk']) for obj in self.objects] - + # Send the pre_clear signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - + # Commit the clear self.objects = [] - + # Send the post_clear signal m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_clear', reverse=False, model=self.rel.to, @@ -301,7 +211,7 @@ def clear(self, auto_save=True): if auto_save: self.model_instance.save() - + def __contains__(self, obj): """ Helper to enable 'object in container' by comparing IDs. @@ -309,7 +219,7 @@ def __contains__(self, obj): if hasattr(obj, 'pk'): obj = obj.pk elif hasattr(obj, 'id'): obj = obj.id return ObjectId(obj) in [ObjectId(o['pk']) for o in self.objects] - + def __iter__(self): """ Iterator is used by Django admin's ModelMultipleChoiceField. @@ -319,7 +229,7 @@ def __iter__(self): # Load referred instance from db and keep in memory obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) yield obj['obj'] - + def all(self, **kwargs): """ Return all the related objects as a query set. If embedding @@ -328,13 +238,13 @@ def all(self, **kwargs): """ return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=True, **kwargs) - + def ids(self): """ Return a list of ObjectIds of all the related objects. """ return [obj['pk'] for obj in self.objects] - + def objs(self): """ Return the actual related model objects, loaded fresh from @@ -343,11 +253,11 @@ def objs(self): """ return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=False) - + def to_python_embedded_instance(self, embedded_instance): """ - Convert a single embedded instance value stored in the database to an object - we can store in the internal objects list. + Convert a single embedded instance value stored in the database to an + object we can store in the internal objects list. """ if isinstance(embedded_instance, ObjectId): # It's an object id, probably from a ListField(ForeignKey) migration @@ -415,7 +325,7 @@ def to_python_embedded_instance(self, embedded_instance): else: # Assume it's already a model return {'pk': ObjectId(embedded_instance.pk), 'obj': None} - + def to_python(self, values): """ Convert a database value to Django model instances managed by this manager. @@ -424,7 +334,7 @@ def to_python(self, values): # Single value given as parameter values = [values] self.objects = [self.to_python_embedded_instance(value) for value in values] - + def get_db_prep_value_embedded_instance(self, obj, connection): """ Convert an internal object value to database representation. @@ -447,7 +357,7 @@ def get_db_prep_value_embedded_instance(self, obj, connection): # Convert primary key into an ObjectId so it's stored correctly # values[self.rel.to._meta.pk] = ObjectId(values[self.rel.to._meta.pk]) return values - + def get_db_prep_value(self, connection, prepared=False): """Convert the Django model instances managed by this manager into a special list that can be stored in MongoDB. @@ -457,119 +367,13 @@ def get_db_prep_value(self, connection, prepared=False): return values -def create_through(field, model, to): - """ - Create a dummy 'through' model for MongoDBManyToMany relations. Django assumes there is a real - database model providing the relationship, so we simulate it. This model has to have - a ForeignKey relationship to both models. We will also override the save() and delete() - methods to pass the adding and removing of related objects to the relation manager. - """ - obj_name = model._meta.object_name + to._meta.object_name + 'Relationship' - to_module_name = to._meta.module_name - model_module_name = model._meta.module_name - class ThroughQuerySet(object): - def __init__(self, relationship_model, *args, **kwargs): - self.to = to - self.model = relationship_model - self.model_instance = None - self.related_manager = None - self.to_instance = None - self.db = 'default' - def filter(self, *args, **kwargs): - if model_module_name in kwargs: - # Relation, set up for querying by the model - self.model_instance = kwargs[model_module_name] - self.related_manager = getattr(self.model_instance, field.name) - # Now we know enough to retrieve the actual query set - queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) - return queryset - if to_module_name in kwargs: - # Reverse relation, set up for querying by the to model - self.to_instance = kwargs[to_module_name] - self.reverse_manager = getattr(self.to_instance, field.rel.related_name) - queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) - return queryset - return self - def exists(self, *args, **kwargs): - return False - def ordered(self, *args, **kwargs): - return self - def using(self, db, *args, **kwargs): - self.db = db - return self - def get(self, *args, **kwargs): - # Check if it's a magic key - if 'pk' in kwargs and isinstance(kwargs['pk'], basestring) and '$' in kwargs['pk']: - model_id, direction, to_id = kwargs['pk'].split('$', 2) - if direction == 'r': - # It's a reverse magic key - to_id, model_id = model_id, to_id - if direction == 'r': - # Query in reverse - self.to_instance = self.to.objects.get(pk=to_id) - self.reverse_manager = getattr(self.to_instance, field.rel.related_name) - queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) - obj = queryset.get(pk=model_id) - return obj - else: - self.model_instance = model.objects.get(pk=model_id) - self.related_manager = getattr(self.model_instance, field.name) - queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) - return queryset.get(pk=to_id) - # Normal key - return None - def __len__(self): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__len__)') - def __getitem__(self, key): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__getitem__)') - class ThroughManager(MongoDBManager): - def get_query_set(self): - return ThroughQuerySet(self.model) - class Through(models.Model): - class Meta: - auto_created = model - objects = ThroughManager() - locals()[to_module_name] = models.ForeignKey(to, null=True, blank=True) - locals()[model_module_name] = models.ForeignKey(model, null=True, blank=True) - def __unicode__(self): - return unicode(getattr(self, model_module_name)) + u' : ' + unicode(getattr(self, to_module_name)) - def save(self, *args, **kwargs): - # Don't actually save the model, convert to an add() call instead - obj = getattr(self, model_module_name) - manager = getattr(obj, field.name) - manager.add(getattr(self, to_module_name)) - obj.save() # must save parent model because Django admin won't - def delete(self, *args, **kwargs): - # Don't actually delete the model, convert to a delete() call instead - obj = getattr(self, model_module_name) - manager = getattr(obj, field.name) - manager.remove(getattr(self, to_module_name)) - obj.save() # must save parent model because Django admin won't - # Remove old model from Django's model registry, because it would be a duplicate - from django.db.models.loading import cache - model_dict = cache.app_models.get(Through._meta.app_label) - del model_dict[Through._meta.module_name] - # Rename the model - Through.__name__ = obj_name - Through._meta.app_label = model._meta.app_label - Through._meta.object_name = obj_name - Through._meta.module_name = obj_name.lower() - Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.module_name - Through._meta.verbose_name = _('%(model)s %(to)s relationship') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} - Through._meta.verbose_name_plural = _('%(model)s %(to)s relationships') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} - # Add new model to Django's model registry - cache.register_models(Through._meta.app_label, Through) - return Through - class MongoDBManyToManyRelationDescriptor(object): """ This descriptor returns the 'through' model used in Django admin to access the ManyToManyField objects for inlines. It's implemented by the MongoDBManyToManyThrough class, which simulates a data model. This class also handles the attribute assignment from the MongoDB raw fields, which must be properly converted to Python objects. - + In other words, when you have a many-to-many field called categories on model Article, this descriptor is the value of Article.categories. When you access the value Article.categories.through, you get the through attribute of this object. @@ -577,7 +381,7 @@ class MongoDBManyToManyRelationDescriptor(object): def __init__(self, field, through): self.field = field self.through = through - + def __get__(self, obj, type=None): """ A field is being accessed on a model instance. Add the model instance to the @@ -592,7 +396,7 @@ def __get__(self, obj, type=None): return manager else: return type.__dict__[self.field.name] - + def __set__(self, obj, value): """ Attributes are being assigned to model instance. We redirect the @@ -661,7 +465,7 @@ def raise_query_error(): if isinstance(field, Q): # Some args may be Qs. This function replaces the Q children # with A() objects. - status = _replace_Q(field, column, + status = replace_Q(field, column, ["pk"] if not embedded else None) if status: updated_args.append(field) @@ -670,7 +474,7 @@ def raise_query_error(): else: # Anything else should be tuples of two items updated_args.append( - (self.field.column, _combine_A(field[0], field[1]))) + (self.field.column, combine_A(field[0], field[1]))) updated_kwargs = [] # Iterate over the kwargs and combine them into A objects @@ -680,7 +484,7 @@ def raise_query_error(): # Have to build Q objects because all the arguments will have the # same key in the kwargs otherwise - updated_kwargs.append(Q(**{column: _combine_A(field, value)})) + updated_kwargs.append(Q(**{column: combine_A(field, value)})) query_args = updated_args + updated_kwargs if negate: @@ -732,209 +536,9 @@ def __init__(self, field, to, related_name, embed): # Required for Django admin/forms to work. self.multiple = True self.limit_choices_to = {} - + def is_hidden(self): return False - + def get_related_field(self, *args, **kwargs): return self.field - -class MongoDBManyToManyField(models.ManyToManyField, ListField): - """ - A generic MongoDB many-to-many field that can store embedded copies of - the referenced objects. Inherits from djangotoolbox.fields.ListField. - - The field's value is a MongoDBM2MRelatedManager object that works similarly to Django's - RelatedManager objects, so you can add(), remove(), creaet() and clear() on it. - To access the related object instances, all() is supported. It will return - all the related instances, using the embedded copies if available. - - If you want the 'real' related (non-embedded) model instances, call all_objs() instead. - If you want the list of related ObjectIds, call all_refs() instead. - - The related model will also gain a new accessor method xxx_set() to make reverse queries. - That accessor is a MongoDBM2MReverseManager that provides an all() method to return - a QuerySet of related objects. - - For example, if you have an Article model with a MongoDBManyToManyField 'categories' - that refers to Category objects, you will have these methods: - - article.categories.all() - Returns all the categories that belong to the article - category.article_set.all() - Returns all the articles that belong to the category - """ - description = 'ManyToMany field with references and optional embedded objects' - - def __init__(self, to, related_name=None, embed=False, *args, **kwargs): - # Call Field, not super, to skip Django's ManyToManyField extra stuff - # we don't need - self._mm2m_to_or_name = to - self._mm2m_related_name = related_name - self._mm2m_embed = embed - if embed: - item_field = EmbeddedModelField(to) - else: - item_field = None - ListField.__init__(self, item_field, *args, **kwargs) - - def contribute_after_resolving(self, field, to, model): - # Setup the main relation helper - self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, - self._mm2m_embed) - # The field's default value will be an empty MongoDBM2MRelatedManager - # that's not connected to a model instance - self.default = MongoDBM2MRelatedManager(self, self.rel, - self._mm2m_embed) - self.rel.model = model - self.rel.through = create_through(self, self.rel.model, self.rel.to) - # Determine related name automatically unless set - if not self.rel.related_name: - self.rel.related_name = model._meta.object_name.lower() + '_set' - - # Add the reverse relationship - setattr(self.rel.to, self.rel.related_name, - MongoDBM2MReverseDescriptor(model, self, self.rel, - self.rel.embed)) - # Add the relationship descriptor to the model class for Django - # admin/forms to work - setattr(model, self.name, - MongoDBManyToManyRelationDescriptor(self, self.rel.through)) - - def contribute_to_class(self, model, name): - self.__m2m_name = name - # Call Field, not super, to skip Django's ManyToManyField extra stuff - # we don't need - ListField.contribute_to_class(self, model, name) - # Do the rest after resolving the 'to' relation - add_lazy_relation(model, self, self._mm2m_to_or_name, - self.contribute_after_resolving) - - def db_type(self, *args, **kwargs): - return 'list' - - def get_internal_type(self): - return 'ListField' - - def formfield(self, **kwargs): - from django import forms - db = kwargs.pop('using', None) - defaults = { - 'form_class': forms.ModelMultipleChoiceField, - 'queryset': self.rel.to._default_manager.using(db).complex_filter( - self.rel.limit_choices_to) - } - defaults.update(kwargs) - # If initial is passed in, it's a list of related objects, but the - # MultipleChoiceField takes a list of IDs. - if defaults.get('initial') is not None: - initial = defaults['initial'] - if callable(initial): - initial = initial() - defaults['initial'] = [i._get_pk_val() for i in initial] - return models.Field.formfield(self, **defaults) - - def pre_save(self, model_instance, add): - return self.to_python(getattr(model_instance, self.attname)) - - def get_db_prep_lookup(self, lookup_type, value, connection, - prepared=False): - # This is necessary because the ManyToManyField.get_db_prep_lookup will - # convert 'A' objects into a unicode string. We don't want that. - if isinstance(value, A): - return value - else: - return models.ManyToManyField.get_db_prep_lookup( - self, lookup_type, value, connection, prepared) - - def get_db_prep_value(self, value, connection, prepared=False): - # The Python value is a MongoDBM2MRelatedManager, and we'll store the - # models it contains as a special list. - if not isinstance(value, MongoDBM2MRelatedManager): - # Convert other values to manager objects first - value = MongoDBM2MRelatedManager(self, self.rel, - self.rel.embed, value) - # Let the manager to the conversion - return value.get_db_prep_value(connection, prepared) - - def to_python(self, value): - # The database value is a custom MongoDB list of ObjectIds and embedded - # models (if embed is enabled). We convert it into a - # MongoDBM2MRelatedManager object to hold the Django models. - if not isinstance(value, MongoDBM2MRelatedManager): - manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) - manager.to_python(value) - value = manager - return value - - -def _replace_Q(q, column, allowed_fields=None): - """Replace the fields in the Q object with A() objects from 'column' - - :param q: The Q object to work on - :param column: The name of the column the A() objects should be attached to. - :param allowed_fields: If defined, only fields names listed in - 'allowed_fields' are allowed. - E.g. allowed_fields=["pk"]: Q(pk=1) is good, Q(name="tom") fails. - :returns: Boolean; False if 'allowed_fields' missed. True otherwise - - Example: - M2M field is called 'users' - _replace_Q(Q(name="Tom"), "users") would modify the given Q to be: - Q(users=A("name", "Tom")) - - That would generate the query: {"users.name":"Tom"} - """ - if not isinstance(q, Q): - raise ValueError("'q' must be of type Q, not: '%s'" % type(q)) - - # Iterate over the Q object's children. The children are either another Q, - # or a tuple of (,) - for child in q.children: - if isinstance(child, Q): - # If we have a Q in the children, let's recurse to fix it too - _replace_Q(child, column, allowed_fields) - elif isinstance(child, tuple): - # Otherwise we need to build an A(). Doing the index, remove, - # and insert to maintain the order of the children. I'm not sure - # changing the order matters, but I don't want to risk it. - index = q.children.index(child) - q.children.remove(child) - - # If allowed_fields is defined, this verifies that only those - # fields are present. E.g. ['pk'] - if allowed_fields and child[0] not in allowed_fields: - return False - # If all is well, build an A(), and insert back into the children - q.children.insert(index, (column, _combine_A(child[0], child[1]))) - else: - raise TypeError("Unknown type in Q.children") - return True - - -def _combine_A(field, value): - # The pk is actually stored as "id", so change it, we also need extract the - # pk from and models and wrap any IDs in an ObjectId, - if field in ('pk', 'id'): - field = "id" - if isinstance(value, models.Model): - # Specifically getattr field because we don't know if it's 'pk' - # or 'id' and they might not be the same thing. - value = getattr(value, field) - - # If value is None, we want to leave it as None, otherwise wrap it - if value is not None and not isinstance(value, ObjectId): - value = ObjectId(value) - - # If 'value' is already an A(), we need to extract the field part out - if isinstance(value, A): - field = "%s.%s" % (field, value.op) - value = value.val - return A(field, value) - - - -# Sort of hackish, but they left me no choice! Without this, 'A' objects are -# rejected for this field because it's not in "DJANGOTOOLBOX_FIELDS" -django_mongodb_engine.query.DJANGOTOOLBOX_FIELDS += \ - (MongoDBManyToManyField,) - - diff --git a/django_mongom2m/query.py b/django_mongom2m/query.py new file mode 100644 index 0000000..6759b9b --- /dev/null +++ b/django_mongom2m/query.py @@ -0,0 +1,95 @@ + +from django.db import router +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId + + + +class MongoDBM2MQueryError(Exception): pass + + +class MongoDBM2MQuerySet(object): + """ + Helper for returning a set of objects from the managers. + Works similarly to Django's own query set objects. + Lazily loads non-embedded objects when iterated. + If embed=False, objects are always loaded from database. + """ + def __init__(self, rel, model, objects, use_cached, + appear_as_relationship=(None, None, None, None, None)): + self.db = router.db_for_read(rel.model if rel.model else rel.field.model) + self.rel = rel + self.objects = list(objects) # make a copy of the list to avoid problems + self.model = model + (self.appear_as_relationship_model, self.rel_model_instance, + self.rel_to_instance, self.rel_model_name, self.rel_to_name) = \ + appear_as_relationship # appear as an intermediate m2m model + if self.appear_as_relationship_model: + self.model = self.appear_as_relationship_model + if not use_cached: + # Reset any cached instances + self.objects = [{'pk': obj['pk'], 'obj': None} + for obj in self.objects] + + def _get_obj(self, obj): + if not obj.get('obj'): + # Load referred instance from db and keep in memory + obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) + if self.appear_as_relationship_model: + # Wrap us in a relationship class + if self.rel_model_instance: + args = {'pk': "%s$f$%s" % + (self.rel_model_instance.pk, obj['pk']), + self.rel_model_name: self.rel_model_instance, + self.rel_to_name: obj['obj']} + else: + # Reverse + args = {'pk': "%s$r$%s" % (self.rel_to_instance.pk, obj['pk']), + self.rel_model_name: obj['obj'], + self.rel_to_name: self.rel_to_instance} + wrapper = self.appear_as_relationship_model(**args) + return wrapper + return obj['obj'] + + def __iter__(self): + for obj in self.objects: + yield self._get_obj(obj) + + def __repr__(self): + from . import REPR_OUTPUT_SIZE + # limit list after conversion because mongodb doesn't use integer indices + data = list(self)[:REPR_OUTPUT_SIZE + 1] + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + return repr(data) + + def __getitem__(self, key): + obj = self.objects[key] + return self._get_obj(obj) + + def ordered(self, *args, **kwargs): + return self + + def __len__(self): + return len(self.objects) + + def using(self, db, *args, **kwargs): + self.db = db + return self + + def filter(self, *args, **kwargs): + return self + + def get(self, *args, **kwargs): + if 'pk' in kwargs: + pk = ObjectId(kwargs['pk']) + for obj in self.objects: + if pk == obj['pk']: + return self._get_obj(obj) + return None + + def count(self): + return len(self.objects) diff --git a/django_mongom2m/utils.py b/django_mongom2m/utils.py new file mode 100644 index 0000000..a144b85 --- /dev/null +++ b/django_mongom2m/utils.py @@ -0,0 +1,182 @@ +from django.db import models +from django.db.models import Q +from django.utils.translation import ugettext_lazy as _ +from django_mongodb_engine.contrib import MongoDBManager +from django_mongodb_engine.query import A +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId + + +def create_through(field, model, to): + """ + Create a dummy 'through' model for MongoDBManyToMany relations. Django assumes there is a real + database model providing the relationship, so we simulate it. This model has to have + a ForeignKey relationship to both models. We will also override the save() and delete() + methods to pass the adding and removing of related objects to the relation manager. + """ + obj_name = model._meta.object_name + to._meta.object_name + 'Relationship' + to_module_name = to._meta.module_name + model_module_name = model._meta.module_name + class ThroughQuerySet(object): + def __init__(self, relationship_model, *args, **kwargs): + self.to = to + self.model = relationship_model + self.model_instance = None + self.related_manager = None + self.to_instance = None + self.db = 'default' + def filter(self, *args, **kwargs): + if model_module_name in kwargs: + # Relation, set up for querying by the model + self.model_instance = kwargs[model_module_name] + self.related_manager = getattr(self.model_instance, field.name) + # Now we know enough to retrieve the actual query set + queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) + return queryset + if to_module_name in kwargs: + # Reverse relation, set up for querying by the to model + self.to_instance = kwargs[to_module_name] + self.reverse_manager = getattr(self.to_instance, field.rel.related_name) + queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) + return queryset + return self + def exists(self, *args, **kwargs): + return False + def ordered(self, *args, **kwargs): + return self + def using(self, db, *args, **kwargs): + self.db = db + return self + def get(self, *args, **kwargs): + # Check if it's a magic key + if 'pk' in kwargs and isinstance(kwargs['pk'], basestring) and '$' in kwargs['pk']: + model_id, direction, to_id = kwargs['pk'].split('$', 2) + if direction == 'r': + # It's a reverse magic key + to_id, model_id = model_id, to_id + if direction == 'r': + # Query in reverse + self.to_instance = self.to.objects.get(pk=to_id) + self.reverse_manager = getattr(self.to_instance, field.rel.related_name) + queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) + obj = queryset.get(pk=model_id) + return obj + else: + self.model_instance = model.objects.get(pk=model_id) + self.related_manager = getattr(self.model_instance, field.name) + queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) + return queryset.get(pk=to_id) + # Normal key + return None + def __len__(self): + # Won't work, must be accessed through filter() + raise Exception('ThroughQuerySet relation unknown (__len__)') + def __getitem__(self, key): + # Won't work, must be accessed through filter() + raise Exception('ThroughQuerySet relation unknown (__getitem__)') + class ThroughManager(MongoDBManager): + def get_query_set(self): + return ThroughQuerySet(self.model) + class Through(models.Model): + class Meta: + auto_created = model + objects = ThroughManager() + locals()[to_module_name] = models.ForeignKey(to, null=True, blank=True) + locals()[model_module_name] = models.ForeignKey(model, null=True, blank=True) + def __unicode__(self): + return unicode(getattr(self, model_module_name)) + u' : ' + unicode(getattr(self, to_module_name)) + def save(self, *args, **kwargs): + # Don't actually save the model, convert to an add() call instead + obj = getattr(self, model_module_name) + manager = getattr(obj, field.name) + manager.add(getattr(self, to_module_name)) + obj.save() # must save parent model because Django admin won't + def delete(self, *args, **kwargs): + # Don't actually delete the model, convert to a delete() call instead + obj = getattr(self, model_module_name) + manager = getattr(obj, field.name) + manager.remove(getattr(self, to_module_name)) + obj.save() # must save parent model because Django admin won't + # Remove old model from Django's model registry, because it would be a duplicate + from django.db.models.loading import cache + model_dict = cache.app_models.get(Through._meta.app_label) + del model_dict[Through._meta.module_name] + # Rename the model + Through.__name__ = obj_name + Through._meta.app_label = model._meta.app_label + Through._meta.object_name = obj_name + Through._meta.module_name = obj_name.lower() + Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.module_name + Through._meta.verbose_name = _('%(model)s %(to)s relationship') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} + Through._meta.verbose_name_plural = _('%(model)s %(to)s relationships') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} + # Add new model to Django's model registry + cache.register_models(Through._meta.app_label, Through) + return Through + + +def replace_Q(q, column, allowed_fields=None): + """Replace the fields in the Q object with A() objects from 'column' + + :param q: The Q object to work on + :param column: The name of the column the A() objects should be attached to. + :param allowed_fields: If defined, only fields names listed in + 'allowed_fields' are allowed. + E.g. allowed_fields=["pk"]: Q(pk=1) is good, Q(name="tom") fails. + :returns: Boolean; False if 'allowed_fields' missed. True otherwise + + Example: + M2M field is called 'users' + _replace_Q(Q(name="Tom"), "users") would modify the given Q to be: + Q(users=A("name", "Tom")) + + That would generate the query: {"users.name":"Tom"} + """ + if not isinstance(q, Q): + raise ValueError("'q' must be of type Q, not: '%s'" % type(q)) + + # Iterate over the Q object's children. The children are either another Q, + # or a tuple of (,) + for child in q.children: + if isinstance(child, Q): + # If we have a Q in the children, let's recurse to fix it too + replace_Q(child, column, allowed_fields) + elif isinstance(child, tuple): + # Otherwise we need to build an A(). Doing the index, remove, + # and insert to maintain the order of the children. I'm not sure + # changing the order matters, but I don't want to risk it. + index = q.children.index(child) + q.children.remove(child) + + # If allowed_fields is defined, this verifies that only those + # fields are present. E.g. ['pk'] + if allowed_fields and child[0] not in allowed_fields: + return False + # If all is well, build an A(), and insert back into the children + q.children.insert(index, (column, combine_A(child[0], child[1]))) + else: + raise TypeError("Unknown type in Q.children") + return True + + +def combine_A(field, value): + # The pk is actually stored as "id", so change it, we also need extract the + # pk from and models and wrap any IDs in an ObjectId, + if field in ('pk', 'id'): + field = "id" + if isinstance(value, models.Model): + # Specifically getattr field because we don't know if it's 'pk' + # or 'id' and they might not be the same thing. + value = getattr(value, field) + + # If value is None, we want to leave it as None, otherwise wrap it + if value is not None and not isinstance(value, ObjectId): + value = ObjectId(value) + + # If 'value' is already an A(), we need to extract the field part out + if isinstance(value, A): + field = "%s.%s" % (field, value.op) + value = value.val + return A(field, value) diff --git a/mongom2m/__init__.py b/mongom2m/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/setup.py b/setup.py index 7da669b..c86d423 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,11 @@ setup( name='django-mongom2m', - version='0.1.0', - author=u'Kenneth Falck', - author_email='kennu@iki.fi', - packages=['mongom2m', 'mongom2m_testapp'], - url='https://github.com/kennu/django-mongom2m', + version='0.2.0', + author=u'Merchant Atlas Inc.', + author_email='support@merchantatlas.com', + packages=['django_mongom2m'], + url='https://github.com/mobilespinach/django-mongom2m', license='BSD licence, see LICENCE.txt', description='A ManyToManyField for django-mongodb-engine', long_description='A ManyToManyField for django-mongodb-engine', From d2517e5eec8fc89f5f42e3314efc722311542539 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Thu, 14 Nov 2013 21:00:35 -0800 Subject: [PATCH 10/14] fixed a bug that would prevent this from working with Deferred fields --- django_mongom2m/fields.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/django_mongom2m/fields.py b/django_mongom2m/fields.py index ed8ae9e..378ccc7 100644 --- a/django_mongom2m/fields.py +++ b/django_mongom2m/fields.py @@ -1,9 +1,9 @@ from django.db import models from django.db.models.fields.related import add_lazy_relation +from django.db.models.query_utils import DeferredAttribute from django_mongodb_engine.query import A from djangotoolbox.fields import ListField, EmbeddedModelField - from .manager import (MongoDBManyToManyRel, MongoDBM2MRelatedManager, MongoDBM2MReverseDescriptor, MongoDBManyToManyRelationDescriptor) @@ -130,7 +130,9 @@ def to_python(self, value): # The database value is a custom MongoDB list of ObjectIds and embedded # models (if embed is enabled). We convert it into a # MongoDBM2MRelatedManager object to hold the Django models. - if not isinstance(value, MongoDBM2MRelatedManager): + + if not isinstance(value, MongoDBM2MRelatedManager) and \ + not isinstance(value, DeferredAttribute): manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) manager.to_python(value) value = manager From 17d2d1b6916bf888092e77c8833602f93ddb56f1 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Thu, 14 Nov 2013 21:01:19 -0800 Subject: [PATCH 11/14] upped the version for bug fix --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c86d423..9e79fe2 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='django-mongom2m', - version='0.2.0', + version='0.2.1', author=u'Merchant Atlas Inc.', author_email='support@merchantatlas.com', packages=['django_mongom2m'], From bb5b4e70b66c5e8c8712de6463f665a17f4f253b Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Sun, 12 Jan 2014 15:27:31 -0800 Subject: [PATCH 12/14] Fixed a bug created by changes to django-toolbox --- django_mongom2m/manager.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/django_mongom2m/manager.py b/django_mongom2m/manager.py index 09e9a4c..2b11266 100644 --- a/django_mongom2m/manager.py +++ b/django_mongom2m/manager.py @@ -278,6 +278,9 @@ def to_python_embedded_instance(self, embedded_instance): return self.to_python_embedded_instance( {"id": ObjectId(values['id'])}) else: + if isinstance(values, tuple): + # In some versions of django-toolbox, 'values' is a tuple. + values = dict(values) # Otherwise it's been embedded previously instance = cls(**values) return {'pk': ObjectId(instance.pk), 'obj': instance} @@ -328,12 +331,14 @@ def to_python_embedded_instance(self, embedded_instance): def to_python(self, values): """ - Convert a database value to Django model instances managed by this manager. + Convert a database value to Django model instances managed by this + manager. """ if isinstance(values, models.Model): # Single value given as parameter values = [values] - self.objects = [self.to_python_embedded_instance(value) for value in values] + self.objects = [self.to_python_embedded_instance(value) + for value in values] def get_db_prep_value_embedded_instance(self, obj, connection): """ From 298a512f0dc4b606594518119d9b6970b16f2819 Mon Sep 17 00:00:00 2001 From: Michael Steffeck Date: Sun, 12 Jan 2014 15:28:30 -0800 Subject: [PATCH 13/14] Fixed a bug created by changes to django-toolbox --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9e79fe2..c0cd7ad 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup( name='django-mongom2m', - version='0.2.1', + version='0.2.2', author=u'Merchant Atlas Inc.', author_email='support@merchantatlas.com', packages=['django_mongom2m'], From 0c450753c1a3b759c745cab1f1d08d7eaf5c18da Mon Sep 17 00:00:00 2001 From: huand Date: Mon, 9 Feb 2015 11:15:15 +0800 Subject: [PATCH 14/14] Fix support for nonrel-1.6 --- README.md | 32 ++++++- django_mongom2m/fields.py | 22 ++++- django_mongom2m/manager.py | 70 +++++++++++--- django_mongom2m/query.py | 117 +++++++++++++++++++++-- django_mongom2m/utils.py | 52 +++++++++-- mongom2m_testapp/models.py | 2 +- mongom2m_testapp/tests.py | 184 +++++++++++++++++++++++++++++++------ 7 files changed, 414 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 5086113..61eabb6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,8 @@ Django MongoDB ManyToManyField Implementation ============================================= -Created in 2012 by Kenneth Falck, Modified/Extended by Merchant Atlas Inc. 2013 +Created in 2012 by Kenneth Falck, Modified/Extended by: +* Merchant Atlas Inc. 2013 Released under the standard BSD License (see below). @@ -35,6 +36,7 @@ TabularInlines or filter\_horizontal and filter\_vertical to administer the many Don't be surprised, however, if some things don't work, because it's all emulated. There is no real "through" table in the database to provide the many-to-many association. +Supported version: [django-nonrel-1.6](https://github.com/django-nonrel/django/tree/nonrel-1.6) Usage ----- @@ -45,11 +47,11 @@ Example model using a many-to-many field: from django.db import models from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager - + class Category(models.Model): objects = MongoDBManager() title = models.CharField(max_length=254) - + class Article(models.Model): objects = MongoDBManager() categories = MongoDBManyToManyField(Category) @@ -67,7 +69,7 @@ To store categories in the field, you would first create the category and then a for cat in article.categories.all(): print cat.title - + for art in category.article_set.all(): print art.title @@ -106,6 +108,26 @@ Example: for article in Article.objects.all(): article.save() # Re-saving will now embed the categories automatically +### Query with or without cache +To query with or without cache, just passing `use_cached=` argument to supported query. + +Examples: + + # return all instances in cache + article.categories.all(use_cached=False) + # return a list of ids + # to be compatible with admin site, values_list use `use_cached=False` by default + article.categories.values_list('pk', flat=True) + +### Refresh cache +To remove instances already deleted by other other actions: + + article.categories.remove_nonexists() + +To refresh cache and remove instances already deleted: + + article.categories.reload_from_db() + ### Advanced Querying (Embedded models) If you use `embed=True`, _MongoDBManyToManyField_ can do more than just query on 'pk'. You can do any of: get, filter, and exclude; while using Q objects and A objects @@ -124,7 +146,7 @@ Note: The models have to be embedded because MongoDB doesn't support joins. There are some things that won't work with _MongoDBManyToManyField_: #### Chaining multiple filters or excludes together Under the covers, the initial filter is being called on the Article QuerySet, so what -gets returned is an Article QuerySet. That means calling filter() again will not get +gets returned is an Article QuerySet. That means calling filter() again will not get any of the magic provided by _MongoDBManyToManyField_. # filter(title="hats").exclude(title="") will act on Article, not categories diff --git a/django_mongom2m/fields.py b/django_mongom2m/fields.py index 378ccc7..58c3cd6 100644 --- a/django_mongom2m/fields.py +++ b/django_mongom2m/fields.py @@ -1,5 +1,5 @@ from django.db import models -from django.db.models.fields.related import add_lazy_relation +from django.db.models.fields.related import add_lazy_relation, RelatedObject from django.db.models.query_utils import DeferredAttribute from django_mongodb_engine.query import A from djangotoolbox.fields import ListField, EmbeddedModelField @@ -34,6 +34,8 @@ class MongoDBManyToManyField(models.ManyToManyField, ListField): category.article_set.all() - Returns all the articles that belong to the category """ description = 'ManyToMany field with references and optional embedded objects' + generate_reverse_relation = False + requires_unique_target = False def __init__(self, to, related_name=None, embed=False, *args, **kwargs): # Call Field, not super, to skip Django's ManyToManyField extra stuff @@ -69,7 +71,14 @@ def contribute_after_resolving(self, field, to, model): # admin/forms to work setattr(model, self.name, MongoDBManyToManyRelationDescriptor(self, self.rel.through)) - + #TODO: deprecated self.related used in django nonrel-1.6, remove later + other = self.rel.to + self.do_related_class(other, model) + + + def do_related_class(self, other, cls): + self.related = RelatedObject(other, cls, self) + def contribute_to_class(self, model, name): self.__m2m_name = name # Call Field, not super, to skip Django's ManyToManyField extra stuff @@ -106,6 +115,15 @@ def formfield(self, **kwargs): def pre_save(self, model_instance, add): return self.to_python(getattr(model_instance, self.attname)) + #get_db_prep_save changed for ListField from djangotoolbox 1.6.2 + #using Field.get_db_prep_save which is compitable + def get_db_prep_save(self, value, connection): + """ + Returns field's value prepared for saving into a database. + """ + return self.get_db_prep_value(value, connection=connection, + prepared=False) + def get_db_prep_lookup(self, lookup_type, value, connection, prepared=False): # This is necessary because the ManyToManyField.get_db_prep_lookup will diff --git a/django_mongom2m/manager.py b/django_mongom2m/manager.py index 2b11266..1800867 100644 --- a/django_mongom2m/manager.py +++ b/django_mongom2m/manager.py @@ -2,6 +2,8 @@ from django.db import models, router from django.db.models import Q from django.db.models.signals import m2m_changed +from .utils import get_exists_ids + try: # ObjectId has been moved to bson.objectid in newer versions of PyMongo from bson.objectid import ObjectId @@ -10,7 +12,7 @@ from .query import MongoDBM2MQuerySet, MongoDBM2MQueryError from .utils import replace_Q, combine_A - +import warnings class MongoDBM2MReverseManager(object): @@ -152,6 +154,24 @@ def create(self, **kwargs): self.add(obj, auto_save=auto_save) return obj + def _remove_by_id_strings(self, removed_obj_ids): + ''' + Remove specified objects by list of id strings + ''' + # Send the pre_remove signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + # Commit the remove + self.objects = [obj for obj in self.objects if str(obj['pk']) not in removed_obj_ids] + + # Send the post_remove signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + def remove(self, *objs, **kwargs): """ Remove the specified object from the M2M field. @@ -169,19 +189,39 @@ def remove(self, *objs, **kwargs): # Calculate list of object ids that will be removed removed_obj_ids = [str(obj['pk']) for obj in self.objects if obj['pk'] in obj_ids] + self._remove_by_id_strings(removed_obj_ids) - # Send the pre_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, - action='pre_remove', reverse=False, model=self.rel.to, - pk_set=removed_obj_ids) + if auto_save: + self.model_instance.save() - # Commit the remove - self.objects = [obj for obj in self.objects if obj['pk'] not in obj_ids] + def remove_nonexists(self, **kwargs): + """ + remove objects not exist in db - # Send the post_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, - action='post_remove', reverse=False, model=self.rel.to, - pk_set=removed_obj_ids) + :param auto_save: See add() above for description + """ + auto_save = kwargs.pop('auto_save', True) + + exists_ids = [obj['_id'] for obj in get_exists_ids(self.model_instance, self.rel, self.objects)] + removed_obj_ids = [str(obj['pk']) for obj in self.objects if (not obj['pk'] in exists_ids)] + self._remove_by_id_strings(removed_obj_ids) + + if auto_save: + self.model_instance.save() + + + def reload_from_db(self, **kwargs): + """ + Reload all objs from db, and remove objs not exists + + A short cut using all(use_cached=False),. + TODO: dev a more effiecient method + """ + auto_save = kwargs.pop('auto_save', True) + + all_objects_from_db = [obj for obj in self.all(use_cached=False)] + self.clear(auto_save=False) + self.add(*all_objects_from_db, auto_save=False) if auto_save: self.model_instance.save() @@ -237,7 +277,7 @@ def all(self, **kwargs): will retrieve the objects from the database as needed. """ return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, - use_cached=True, **kwargs) + **kwargs) def ids(self): """ @@ -250,7 +290,11 @@ def objs(self): Return the actual related model objects, loaded fresh from the database. This won't use embedded objects even if they exist. + + Depreciated by all(use_cached=False) """ + warnings.warn('MongoDBM2MRelatedManager.objs depreciated by all(use_cached=False)', + DeprecationWarning) return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=False) @@ -533,6 +577,8 @@ class MongoDBManyToManyRel(object): def __init__(self, field, to, related_name, embed): self.model = None # added later from contribute_to_class self.through = None # added later from contribute_to_class + #for django.core.management.validation + self.related_query_name = None self.field = field self.to = to self.related_name = related_name diff --git a/django_mongom2m/query.py b/django_mongom2m/query.py index 6759b9b..08ed428 100644 --- a/django_mongom2m/query.py +++ b/django_mongom2m/query.py @@ -1,5 +1,6 @@ from django.db import router +from .utils import get_exists_ids try: # ObjectId has been moved to bson.objectid in newer versions of PyMongo from bson.objectid import ObjectId @@ -18,26 +19,41 @@ class MongoDBM2MQuerySet(object): Lazily loads non-embedded objects when iterated. If embed=False, objects are always loaded from database. """ - def __init__(self, rel, model, objects, use_cached, - appear_as_relationship=(None, None, None, None, None)): + def __init__(self, rel, model, objects, + use_cached=True, + appear_as_relationship=(None, None, None, None, None), + **kwargs): self.db = router.db_for_read(rel.model if rel.model else rel.field.model) self.rel = rel self.objects = list(objects) # make a copy of the list to avoid problems + self.model = model (self.appear_as_relationship_model, self.rel_model_instance, self.rel_to_instance, self.rel_model_name, self.rel_to_name) = \ appear_as_relationship # appear as an intermediate m2m model if self.appear_as_relationship_model: self.model = self.appear_as_relationship_model - if not use_cached: + self.use_cached = use_cached + if not self.use_cached: # Reset any cached instances self.objects = [{'pk': obj['pk'], 'obj': None} for obj in self.objects] + #whether clear none exists objs for potential trouble + self.exists_in_db_only = kwargs.get('exists_in_db_only', False) + if self.exists_in_db_only: + #using only objects stored in db + exists_ids = [obj['_id'] for obj in get_exists_ids(self.model, self.rel, self.objects)] + self.objects = [obj for obj in self.objects if obj['pk'] in exists_ids] + def _get_obj(self, obj): if not obj.get('obj'): - # Load referred instance from db and keep in memory - obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) + try: + # Load referred instance from db and keep in memory + obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) + except self.rel.to.DoesNotExist: + pass + # obj['obj'] will be None if self.appear_as_relationship_model: # Wrap us in a relationship class if self.rel_model_instance: @@ -55,8 +71,11 @@ def _get_obj(self, obj): return obj['obj'] def __iter__(self): - for obj in self.objects: - yield self._get_obj(obj) + for obj in list(self.objects): + #ignore obj of nowhere + obj_cached_or_loaded = self._get_obj(obj) + if not obj_cached_or_loaded is None: + yield obj_cached_or_loaded def __repr__(self): from . import REPR_OUTPUT_SIZE @@ -93,3 +112,87 @@ def get(self, *args, **kwargs): def count(self): return len(self.objects) + + def _clone(self, klass=None, setup=False, **kwargs): + ''' + return a clone of self queryset + works similar to django.db.models.query.QuerySet.clone + ''' + if klass is None: + klass = self.__class__ + #copy self.objects + objects = list(self.objects) + c = klass(rel=self.rel, model=self.model, objects=objects, + use_cached=self.use_cached, + appear_as_relationship=( + self.appear_as_relationship_model, + self.rel_model_instance, + self.rel_to_instance, + self.rel_model_name, self.rel_to_name), + ) + c.__dict__.update(kwargs) + #no use for now + if setup and hasattr(c, '_setup_query'): + c._setup_query() + return c + + def values_list(self, *fields, **kwargs): + ''' + Emulate QuerySet.values_list required by django.contrib.admin + ''' + flat = kwargs.pop('flat', False) + #required True for django.contrib.admin + exists_in_db_only = kwargs.pop('exists_in_db_only', self.exists_in_db_only) + if kwargs: + raise TypeError('Unexpected keyword arguments to values_list: %s' + % (list(kwargs),)) + if flat and len(fields) > 1: + raise TypeError("'flat' is not valid when values_list is called with more than one field.") + return self._clone(klass=MongoDBM2MValuesListQuerySet, setup=True, + flat=flat, + exists_in_db_only=exists_in_db_only, + _fields=fields) + +class MongoDBM2MValuesListQuerySet(MongoDBM2MQuerySet): + ''' + simulate ValuesListQuerySet, using objects instead of query + ''' + def iterator(self): + ''' + iterator yield only fields requested + ''' + for obj in list(self.objects): + obj_cached_or_loaded = self._get_obj(obj) + # skip when obj not in cached and db + if obj_cached_or_loaded is None: + pass + else: + #behavior same as ValuesListQuerySet.iterator + if self.flat and len(self._fields) == 1: + field = self._fields[0] + if hasattr(obj['obj'], field): + yield obj['obj'].__getattribute__(field) + else: + yield None + else: + row = list() + for field in self._fields: + if hasattr(obj['obj'], field): + row.append(obj['obj'].__getattribute__(field)) + else: + row.append(None) + yield tuple(row) + + def __iter__(self): + for item in self.iterator(): + yield item + + def _clone(self, *args, **kwargs): + ''' + override MongoDBM2MQuerySet._clone, clone this query set + ''' + clone = super(MongoDBM2MValuesListQuerySet, self)._clone(*args, **kwargs) + if not hasattr(clone, "flat"): + # Only assign flat if the clone didn't already get it from kwargs + clone.flat = self.flat + return clone diff --git a/django_mongom2m/utils.py b/django_mongom2m/utils.py index a144b85..5f0742d 100644 --- a/django_mongom2m/utils.py +++ b/django_mongom2m/utils.py @@ -1,4 +1,4 @@ -from django.db import models +from django.db import models, router, connections from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django_mongodb_engine.contrib import MongoDBManager @@ -27,7 +27,8 @@ def __init__(self, relationship_model, *args, **kwargs): self.model_instance = None self.related_manager = None self.to_instance = None - self.db = 'default' + #avoiding using backends other than django-mongodb-engine + self.db = router.db_for_write(self.model) def filter(self, *args, **kwargs): if model_module_name in kwargs: # Relation, set up for querying by the model @@ -45,6 +46,11 @@ def filter(self, *args, **kwargs): return self def exists(self, *args, **kwargs): return False + def none(self, *args, **kwargs): + if self.filter() != self: + return self.filter().none() + else: + return self def ordered(self, *args, **kwargs): return self def using(self, db, *args, **kwargs): @@ -72,11 +78,27 @@ def get(self, *args, **kwargs): # Normal key return None def __len__(self): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__len__)') + ''' + hack: return 0 if filter is not ready, else using filter().__len__ + ''' + if self.filter() != self: + return self.filter().__len__() + else: + return 0 def __getitem__(self, key): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__getitem__)') + ''' + hack: admin site will try __getitem___ but only catch IndexError + raise IndexError if filter is not ready, + else using filter().__getitem___ + ''' + if self.filter() != self: + import ipdb;ipdb.set_trace() + return self.filter().__getitem__(key) + else: + # Won't work, must be accessed through filter() + # hack: raise as IndexError for admin + raise IndexError('ThroughQuerySet relation unknown (__getitem__)') + class ThroughManager(MongoDBManager): def get_query_set(self): return ThroughQuerySet(self.model) @@ -108,8 +130,9 @@ def delete(self, *args, **kwargs): Through.__name__ = obj_name Through._meta.app_label = model._meta.app_label Through._meta.object_name = obj_name - Through._meta.module_name = obj_name.lower() - Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.module_name + #.model_name is used instead of .module_name since 1.6 + Through._meta.model_name = obj_name.lower() + Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.model_name Through._meta.verbose_name = _('%(model)s %(to)s relationship') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} Through._meta.verbose_name_plural = _('%(model)s %(to)s relationships') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} # Add new model to Django's model registry @@ -180,3 +203,16 @@ def combine_A(field, value): field = "%s.%s" % (field, value.op) value = value.val return A(field, value) + +def get_exists_ids(model, rel, objects): + ''' + return a cursor return exists ids cached in m2mfield + + :param model: to determine db + :param rel: to determine table used for m2mfield + :param objects: objects for checking existence + ''' + db = connections[router.db_for_write(model)] + conn = db.get_collection(rel.to._meta.db_table) + ids = [obj['pk'] for obj in objects] + return conn.find({"_id":{"$in":ids}},{"_id":1}).limit(len(objects)) diff --git a/mongom2m_testapp/models.py b/mongom2m_testapp/models.py index 6fe73b1..6cb0710 100644 --- a/mongom2m_testapp/models.py +++ b/mongom2m_testapp/models.py @@ -1,6 +1,6 @@ from django.db import models from djangotoolbox.fields import ListField, EmbeddedModelField -from mongom2m.fields import MongoDBManyToManyField +from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager class TestCategory(models.Model): diff --git a/mongom2m_testapp/tests.py b/mongom2m_testapp/tests.py index 167cec7..db29d24 100644 --- a/mongom2m_testapp/tests.py +++ b/mongom2m_testapp/tests.py @@ -1,11 +1,15 @@ from django.test import TestCase from django.db import models from django.db.models.signals import m2m_changed -from mongom2m.fields import MongoDBManyToManyField +from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager from djangotoolbox.fields import ListField, EmbeddedModelField from models import TestArticle, TestCategory, TestTag, TestAuthor, TestBook#, TestOldArticle, TestOldEmbeddedArticle -from pymongo.objectid import ObjectId +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId import sys class MongoDBManyToManyFieldTest(TestCase): @@ -67,10 +71,95 @@ def test_m2m(self): self.assertEqual(category4.testarticle_set.all().count(), 2) self.assertEqual(category4.testarticle_set.all()[0].title, 'test article 2') self.assertEqual(category4.testarticle_set.all()[1].title, 'test article 3') - + #tests on delete + #del tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_tag2.delete() + + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag still in all cached + self.assertIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + new_article2.save() + #Verify that all() and objs() dont change model in db + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag still in all cached + self.assertIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.reload_from_db() + new_article2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag not in all cached + self.assertNotIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + + #test change + #add back tag2 + tag2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.add(tag2) + new_article2.save() + #change tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_tag2.name = 'new tag 2' + new_tag2.save() + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that old name still in all cached + self.assertEqual(tag2.name, new_article2.tags.all()[0].name) + # Verify that old name not in all without cached + self.assertNotEqual(tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new name in all without cached + self.assertEqual(new_tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new_name in .objs + self.assertEqual(new_tag2.name, new_article2.tags.objs()[0].name) + new_article2.save() + + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.reload_from_db() + new_article2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that old name not in all cached + self.assertNotEqual(tag2.name, new_article2.tags.all()[0].name) + # Verify that new name in all cached + self.assertEqual(new_tag2.name, new_article2.tags.all()[0].name) + # Verify that new name in all without cached + self.assertEqual(new_tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new_name in .objs + self.assertEqual(new_tag2.name, new_article2.tags.objs()[0].name) + + #delete tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + tag2 = TestTag.objects.get(pk=tag2.pk) + tag2.delete() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify tag2 still in cache + self.assertIn(new_tag2, new_article2.tags.all()) + #remove nonexists + new_article2.tags.remove_nonexists() + new_article2.save() + # Verify tag2 removed from cache + self.assertNotIn(new_tag2, new_article2.tags.all()) + + + def test_migrations(self): """ Test migrating from an existing ListField(ForeignKey) field + Note: migrating is not directly supported with option embed=True + Workaround: ListField(ForeignKey) -> MongoDBManyToManyField(model) + -> MongoDBManyToManyField(model, embed=True) """ # Create test categories category1 = TestCategory(title='test cat 1') @@ -82,45 +171,81 @@ def test_migrations(self): tag1.save() tag2 = TestTag(name='test tag 2') tag2.save() - + # Create the old data - this model uses ListField(ForeignKey) fields class TestOldArticle(models.Model): class Meta: # Used for testing migrations using same MongoDB collection - db_table = 'mongom2m_testapp_testarticle' - + db_table = TestArticle._meta.db_table + objects = MongoDBManager() main_category = models.ForeignKey(TestCategory, related_name='main_oldarticles') categories = ListField(models.ForeignKey(TestCategory)) tags = ListField(models.ForeignKey(TestTag)) title = models.CharField(max_length=254) text = models.TextField() - + def __unicode__(self): return self.title - + old_article = TestOldArticle(title='old article 1', text='old article text 1', main_category=category1, categories=[category1.id, category2.id], tags=[tag1.id, tag2.id]) old_article.save() - - # Now use the new model to access the old data. - new_article = TestArticle.objects.get(title='old article 1') - + + class TestTransferArticle(models.Model): + ''' + with tags not embedded + ''' + class Meta: + db_table = TestArticle._meta.db_table + objects = MongoDBManager() + main_category = models.ForeignKey(TestCategory, related_name='main_articles') + categories = MongoDBManyToManyField(TestCategory) + #without embed option + tags = MongoDBManyToManyField(TestTag, related_name='articles') + title = models.CharField(max_length=254) + text = models.TextField() + + def __unicode__(self): + return self.title + + # Now use the transfer model to access the old data. + new_article = TestTransferArticle.objects.get(title='old article 1') + # Make sure the fields were loaded correctly self.assertEqual(set(cat.title for cat in new_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in new_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in new_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in new_article.tags.all()), set((tag1.id, tag2.id))) - + # Re-save and reload the data to migrate it in MongoDB new_article.save() - migrated_article = TestArticle.objects.get(title='old article 1') - + migrated_article = TestTransferArticle.objects.get(title='old article 1') + # Make sure the fields are still loaded correctly self.assertEqual(set(cat.title for cat in migrated_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in migrated_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in migrated_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in migrated_article.tags.all()), set((tag1.id, tag2.id))) - + + # Now use the new model to access the old data. + new_article_final = TestArticle.objects.get(title='old article 1') + + # Make sure the fields were loaded correctly + self.assertEqual(set(cat.title for cat in new_article_final.categories.all()), set(('test cat 1', 'test cat 2'))) + self.assertEqual(set(cat.id for cat in new_article_final.categories.all()), set((category1.id, category2.id))) + self.assertEqual(set(tag.name for tag in new_article_final.tags.all()), set(('test tag 1', 'test tag 2'))) + self.assertEqual(set(tag.id for tag in new_article_final.tags.all()), set((tag1.id, tag2.id))) + + # Re-save and reload the data to migrate it in MongoDB + new_article_final.save() + migrated_article_final= TestArticle.objects.get(title='old article 1') + + # Make sure the fields are still loaded correctly + self.assertEqual(set(cat.title for cat in migrated_article_final.categories.all()), set(('test cat 1', 'test cat 2'))) + self.assertEqual(set(cat.id for cat in migrated_article_final.categories.all()), set((category1.id, category2.id))) + self.assertEqual(set(tag.name for tag in migrated_article_final.tags.all()), set(('test tag 1', 'test tag 2'))) + self.assertEqual(set(tag.id for tag in migrated_article_final.tags.all()), set((tag1.id, tag2.id))) + def test_embedded_migrations(self): """ Test migrating from an existing ListField(EmbeddedModelField) @@ -135,45 +260,45 @@ def test_embedded_migrations(self): tag1.save() tag2 = TestTag(name='test tag 2') tag2.save() - + # Create the old data - this model uses ListField(EmbeddedModelField) fields class TestOldEmbeddedArticle(models.Model): class Meta: # Used for testing migrations using same MongoDB collection - db_table = 'mongom2m_testapp_testarticle' - + db_table = TestArticle._meta.db_table + objects = MongoDBManager() main_category = models.ForeignKey(TestCategory, related_name='main_oldembeddedarticles') categories = ListField(EmbeddedModelField(TestCategory)) tags = ListField(EmbeddedModelField(TestTag)) title = models.CharField(max_length=254) text = models.TextField() - + def __unicode__(self): return self.title - + old_article = TestOldEmbeddedArticle(title='old embedded article 1', text='old embedded article text 1', main_category=category1, categories=[category1, category2], tags=[tag1, tag2]) old_article.save() - + # Now use the new model to access the old data. new_article = TestArticle.objects.get(title='old embedded article 1') - + # Make sure the fields were loaded correctly self.assertEqual(set(cat.title for cat in new_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in new_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in new_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in new_article.tags.all()), set((tag1.id, tag2.id))) - + # Re-save and reload the data to migrate it in MongoDB new_article.save() migrated_article = TestArticle.objects.get(title='old embedded article 1') - + # Make sure the fields are still loaded correctly self.assertEqual(set(cat.title for cat in migrated_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in migrated_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in migrated_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in migrated_article.tags.all()), set((tag1.id, tag2.id))) - + def test_signals(self): """ Test signals emitted by various M2M operations. @@ -188,7 +313,7 @@ def test_signals(self): tag2 = TestTag(name='test tag 2') tag2.save() article = TestArticle(title='test article 1', text='test article 1 text', main_category=category1) - + # Test pre_add / post_add self.on_add_called = 0 def on_add(sender, instance, action, reverse, model, pk_set, *args, **kwargs): @@ -209,7 +334,7 @@ def on_add(sender, instance, action, reverse, model, pk_set, *args, **kwargs): article.categories.add(category1) self.assertEqual(self.on_add_called, 2) m2m_changed.disconnect(on_add) - + # Test pre_remove / post_remove self.on_remove_called = 0 def on_remove(sender, instance, action, reverse, model, pk_set, *args, **kwargs): @@ -230,7 +355,7 @@ def on_remove(sender, instance, action, reverse, model, pk_set, *args, **kwargs) article.categories.remove(category1) self.assertEqual(self.on_remove_called, 2) m2m_changed.disconnect(on_remove) - + # Test pre_clear / post_clear article.categories.add(category1) self.on_clear_called = 0 @@ -252,4 +377,3 @@ def on_clear(sender, instance, action, reverse, model, pk_set, *args, **kwargs): article.categories.clear() self.assertEqual(self.on_clear_called, 2) m2m_changed.disconnect(on_clear) -