diff --git a/LICENSE.txt b/LICENSE.txt index 6d58702..dd33aaa 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,6 @@ -Copyright (c) 2012 Kenneth Falck http://kfalck.net +Copyright (c) 2013 Merchant Atlas Inc. http://www.merchantatlas.com + +[Original Work] Copyright (c) 2012 Kenneth Falck http://kfalck.net All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/README.md b/README.md index 69584bd..61eabb6 100644 --- a/README.md +++ b/README.md @@ -1,33 +1,34 @@ Django MongoDB ManyToManyField Implementation ============================================= -Created in 2012 by Kenneth Falck http://kfalck.net. +Created in 2012 by Kenneth Falck, Modified/Extended by: +* Merchant Atlas Inc. 2013 Released under the standard BSD License (see below). Overview -------- -This is a simple implementation of ManyToManyFields for django-mongodb-engine. The MongoDBManyToManyField +This is a simple implementation of ManyToManyFields for django-mongodb-engine. The _MongoDBManyToManyField_ stores references to other Django model instances as ObjectIds in a MongoDB list field. -Optionally, MongoDBManyToManyField will also embed a "cached" copy of the instances inside the list. This +Optionally, _MongoDBManyToManyField_ will also embed a "cached" copy of the instances inside the list. This allows fast access to the data without having to query each related object from the database separately. -MongoDBManyToManyField attempts to work mostly in the same way as Django's built-in ManyToManyField. +_MongoDBManyToManyField_ attempts to work mostly in the same way as Django's built-in ManyToManyField. Related objects can be added and removed with the add(), remove(), clear() and create() methods. To enumerate the objects, the all() method returns a simulated QuerySet object which loads non-embedded objects automatically from the database when needed. -On the reverse side of the relation, an accessor property is added (usually called OtherModel.modelname\_set, +On the reverse side of the relation, an accessor property is added (usually called OtherModel.modelname\_set, can be overridden with the related\_name attribute) to return the related objects in the reverse direction. It uses MongoDB's raw\_query() to find all related model objects. Because of this, any data model that uses MongoDBManyToManyField() must have a default MongoDBManager() instead of Django's normal Manager(). -Django compability ------------------- +Django compatibility +-------------------- This implementation has been tweaked to be mostly compatible with Django admin, which means you can use TabularInlines or filter\_horizontal and filter\_vertical to administer the many-to-many fields. @@ -35,51 +36,141 @@ TabularInlines or filter\_horizontal and filter\_vertical to administer the many Don't be surprised, however, if some things don't work, because it's all emulated. There is no real "through" table in the database to provide the many-to-many association. +Supported version: [django-nonrel-1.6](https://github.com/django-nonrel/django/tree/nonrel-1.6) Usage ----- +### Define a field Example model using a many-to-many field: from django.db import models - from mongom2m.fields import MongoDBManyToManyField + from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager - + class Category(models.Model): objects = MongoDBManager() title = models.CharField(max_length=254) - + class Article(models.Model): objects = MongoDBManager() categories = MongoDBManyToManyField(Category) title = models.CharField(max_length=254) text = models.TextField() +### Add an instance To store categories in the field, you would first create the category and then add it: category = Category(title='foo') category.save() - + article = Article(title='bar') article.categories.add(category) - article.save() - + for cat in article.categories.all(): print cat.title - + for art in category.article_set.all(): print art.title +### Basic Querying +Querying with _MongoDBManyToManyField_ is similar to normal SQL Django, but with some caveats. +In order to have true Django behavior, we would have had to change some combination of: Django-nonrel, +Djangotoolbox, and mongodb-engine; so instead, we went a different route. + +How Django does it: + + Article.objects.filter(categories=category) + [, ] + + article.categories.all() + [, ] + +How _MongoDBManyToManyField_ does it: + + Article.categories.filter(pk=category) + [, ] + + article.categories.all() + [, ] + +### Embed Models for Performance and Querying To enable embedding, just add the embed=True keyword argument to the field: class Article(models.Model): - categories = MongoDBManyToManyField(Category, embed=True) + categories = _MongoDBManyToManyField_(Category, embed=True) + +**Note about embedding**: If you change a _MongoDBManyToManyField_ from `embed=False` to +`embed=True` (or vice versa), the host model objects will not be automatically updated. +You will need to re-save every object of that model. +Example: + + for article in Article.objects.all(): + article.save() # Re-saving will now embed the categories automatically + +### Query with or without cache +To query with or without cache, just passing `use_cached=` argument to supported query. + +Examples: + + # return all instances in cache + article.categories.all(use_cached=False) + # return a list of ids + # to be compatible with admin site, values_list use `use_cached=False` by default + article.categories.values_list('pk', flat=True) + +### Refresh cache +To remove instances already deleted by other other actions: + + article.categories.remove_nonexists() + +To refresh cache and remove instances already deleted: + + article.categories.reload_from_db() + +### Advanced Querying (Embedded models) +If you use `embed=True`, _MongoDBManyToManyField_ can do more than just query on 'pk'. +You can do any of: get, filter, and exclude; while using Q objects and A objects +(from mongodb-engine). _MongoDBManyToManyField_ will automatically convert them to +query correctly on the embedded document. +Note: The models have to be embedded because MongoDB doesn't support joins. + + Article.categories.filter(title="shirts") + Article.categories.filter(Q(title="shirts") | Q(title="hats")) + Article.categories.filter(Q(title="shirts") & ~Q(title="men's")) + + # If categories had an embedded model 'em', you could even query it with A() + Article.categories.filter(em=A("name", "em1")) + +### Limitations +There are some things that won't work with _MongoDBManyToManyField_: +#### Chaining multiple filters or excludes together +Under the covers, the initial filter is being called on the Article QuerySet, so what +gets returned is an Article QuerySet. That means calling filter() again will not get +any of the magic provided by _MongoDBManyToManyField_. + + # filter(title="hats").exclude(title="") will act on Article, not categories + Article.categories.filter(title="shirts").filter(title="hats").exclude(title="") + # However, the same can be accomplished with Q objects + Article.categories.filter(Q(title="shirts") & Q(title="hats") & ~Q(title="")) + +#### Double-underscore commands +This is actually an issue with djangotoolbox/mongobd-engine. Under the covers, +_MongoDBManyToManyField_ uses A() objects to generate the queries. Double-underscore +commands from A() objects don't work. + + # Will query the field 'title__contains', which doesn't exist + Article.categories.filter(title__contains="men") + + # Unfortunately, the only solution for this would be to do a raw_query + import re + Article.objects.raw_query({"categories.title": re.compile("men")}) Signals ------- -MongoDBManyToManyField supports Django's m2m\_changed signal, where the action can be: +_MongoDBManyToManyField_ supports Django's m2m\_changed signal, where the action can be: * pre\_add (triggered before adding object(s) to the field) * post\_add (triggered after adding object(s) to the field) @@ -136,8 +227,9 @@ Also make sure that the "id" field is properly indexed (see previous section). BSD License ----------- +Copyright (c) 2013 Merchant Atlas Inc. http://www.merchantatlas.com -Copyright (c) 2012 Kenneth Falck http://kfalck.net +[Original Work] Copyright (c) 2012 Kenneth Falck http://kfalck.net All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/django_mongom2m/__init__.py b/django_mongom2m/__init__.py new file mode 100644 index 0000000..5b72140 --- /dev/null +++ b/django_mongom2m/__init__.py @@ -0,0 +1,16 @@ + +import django_mongodb_engine.query + +from . import fields +from . import manager +from . import query + + +# How much to show when query set is viewed in the Python shell +REPR_OUTPUT_SIZE = 20 + +# Sort of hackish, but they left me no choice! Without this, 'A' objects are +# rejected for this field because it's not in "DJANGOTOOLBOX_FIELDS" +django_mongodb_engine.query.DJANGOTOOLBOX_FIELDS += \ + (fields.MongoDBManyToManyField,) + diff --git a/django_mongom2m/fields.py b/django_mongom2m/fields.py new file mode 100644 index 0000000..58c3cd6 --- /dev/null +++ b/django_mongom2m/fields.py @@ -0,0 +1,163 @@ +from django.db import models +from django.db.models.fields.related import add_lazy_relation, RelatedObject +from django.db.models.query_utils import DeferredAttribute +from django_mongodb_engine.query import A +from djangotoolbox.fields import ListField, EmbeddedModelField + +from .manager import (MongoDBManyToManyRel, MongoDBM2MRelatedManager, + MongoDBM2MReverseDescriptor, + MongoDBManyToManyRelationDescriptor) +from .utils import create_through + + +class MongoDBManyToManyField(models.ManyToManyField, ListField): + """ + A generic MongoDB many-to-many field that can store embedded copies of + the referenced objects. Inherits from djangotoolbox.fields.ListField. + + The field's value is a MongoDBM2MRelatedManager object that works similarly to Django's + RelatedManager objects, so you can add(), remove(), creaet() and clear() on it. + To access the related object instances, all() is supported. It will return + all the related instances, using the embedded copies if available. + + If you want the 'real' related (non-embedded) model instances, call all_objs() instead. + If you want the list of related ObjectIds, call all_refs() instead. + + The related model will also gain a new accessor method xxx_set() to make reverse queries. + That accessor is a MongoDBM2MReverseManager that provides an all() method to return + a QuerySet of related objects. + + For example, if you have an Article model with a MongoDBManyToManyField 'categories' + that refers to Category objects, you will have these methods: + + article.categories.all() - Returns all the categories that belong to the article + category.article_set.all() - Returns all the articles that belong to the category + """ + description = 'ManyToMany field with references and optional embedded objects' + generate_reverse_relation = False + requires_unique_target = False + + def __init__(self, to, related_name=None, embed=False, *args, **kwargs): + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need + self._mm2m_to_or_name = to + self._mm2m_related_name = related_name + self._mm2m_embed = embed + if embed: + item_field = EmbeddedModelField(to) + else: + item_field = None + ListField.__init__(self, item_field, *args, **kwargs) + + def contribute_after_resolving(self, field, to, model): + # Setup the main relation helper + self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, + self._mm2m_embed) + # The field's default value will be an empty MongoDBM2MRelatedManager + # that's not connected to a model instance + self.default = MongoDBM2MRelatedManager(self, self.rel, + self._mm2m_embed) + self.rel.model = model + self.rel.through = create_through(self, self.rel.model, self.rel.to) + # Determine related name automatically unless set + if not self.rel.related_name: + self.rel.related_name = model._meta.object_name.lower() + '_set' + + # Add the reverse relationship + setattr(self.rel.to, self.rel.related_name, + MongoDBM2MReverseDescriptor(model, self, self.rel, + self.rel.embed)) + # Add the relationship descriptor to the model class for Django + # admin/forms to work + setattr(model, self.name, + MongoDBManyToManyRelationDescriptor(self, self.rel.through)) + #TODO: deprecated self.related used in django nonrel-1.6, remove later + other = self.rel.to + self.do_related_class(other, model) + + + def do_related_class(self, other, cls): + self.related = RelatedObject(other, cls, self) + + def contribute_to_class(self, model, name): + self.__m2m_name = name + # Call Field, not super, to skip Django's ManyToManyField extra stuff + # we don't need + ListField.contribute_to_class(self, model, name) + # Do the rest after resolving the 'to' relation + add_lazy_relation(model, self, self._mm2m_to_or_name, + self.contribute_after_resolving) + + def db_type(self, *args, **kwargs): + return 'list' + + def get_internal_type(self): + return 'ListField' + + def formfield(self, **kwargs): + from django import forms + db = kwargs.pop('using', None) + defaults = { + 'form_class': forms.ModelMultipleChoiceField, + 'queryset': self.rel.to._default_manager.using(db).complex_filter( + self.rel.limit_choices_to) + } + defaults.update(kwargs) + # If initial is passed in, it's a list of related objects, but the + # MultipleChoiceField takes a list of IDs. + if defaults.get('initial') is not None: + initial = defaults['initial'] + if callable(initial): + initial = initial() + defaults['initial'] = [i._get_pk_val() for i in initial] + return models.Field.formfield(self, **defaults) + + def pre_save(self, model_instance, add): + return self.to_python(getattr(model_instance, self.attname)) + + #get_db_prep_save changed for ListField from djangotoolbox 1.6.2 + #using Field.get_db_prep_save which is compitable + def get_db_prep_save(self, value, connection): + """ + Returns field's value prepared for saving into a database. + """ + return self.get_db_prep_value(value, connection=connection, + prepared=False) + + def get_db_prep_lookup(self, lookup_type, value, connection, + prepared=False): + # This is necessary because the ManyToManyField.get_db_prep_lookup will + # convert 'A' objects into a unicode string. We don't want that. + if isinstance(value, A): + return value + else: + return models.ManyToManyField.get_db_prep_lookup( + self, lookup_type, value, connection, prepared) + + def get_db_prep_value(self, value, connection, prepared=False): + # The Python value is a MongoDBM2MRelatedManager, and we'll store the + # models it contains as a special list. + if not isinstance(value, MongoDBM2MRelatedManager): + # Convert other values to manager objects first + value = MongoDBM2MRelatedManager(self, self.rel, + self.rel.embed, value) + # Let the manager to the conversion + return value.get_db_prep_value(connection, prepared) + + def to_python(self, value): + # The database value is a custom MongoDB list of ObjectIds and embedded + # models (if embed is enabled). We convert it into a + # MongoDBM2MRelatedManager object to hold the Django models. + + if not isinstance(value, MongoDBM2MRelatedManager) and \ + not isinstance(value, DeferredAttribute): + manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) + manager.to_python(value) + value = manager + return value + + + + + + diff --git a/django_mongom2m/manager.py b/django_mongom2m/manager.py new file mode 100644 index 0000000..1800867 --- /dev/null +++ b/django_mongom2m/manager.py @@ -0,0 +1,595 @@ + +from django.db import models, router +from django.db.models import Q +from django.db.models.signals import m2m_changed +from .utils import get_exists_ids + +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId + +from .query import MongoDBM2MQuerySet, MongoDBM2MQueryError +from .utils import replace_Q, combine_A +import warnings + + +class MongoDBM2MReverseManager(object): + """ + This manager is attached to the other side of M2M relationships + and will return query sets that fetch related objects. + """ + def __init__(self, rel_field, model, field, rel, embed): + self.rel_field = rel_field + self.model = model + self.field = field + self.rel = rel + self.embed = embed + + def all(self): + """ + Retrieve all related objects. + """ + name = self.field.column + '.' + self.rel.model._meta.pk.column + pk = ObjectId(self.rel_field.pk) + return self.model._default_manager.raw_query({name:pk}) + + def _relationship_query_set(self, model, to_instance, model_module_name, + to_module_name): + """ + Emulate an intermediate 'through' relationship query set. + """ + objects = [{'pk':ObjectId(obj.pk), 'obj':obj} for obj in self.all()] + return MongoDBM2MQuerySet( + self.rel, self.rel.to, objects, use_cached=True, + appear_as_relationship=(model, None, to_instance, + model_module_name, to_module_name)) + +class MongoDBM2MReverseDescriptor(object): + def __init__(self, model, field, rel, embed): + self.model = model + self.field = field + self.rel = rel + self.embed = embed + + def __get__(self, instance, owner=None): + if instance is None: + return self + return MongoDBM2MReverseManager(instance, self.model, self.field, + self.rel, self.embed) + +class MongoDBM2MRelatedManager(object): + """ + This manager manages the related objects stored in a MongoDBManyToManyField. + They can be embedded or stored as relations (ObjectIds) only. + Internally, we store the objects as dicts that contain keys pk and obj. + The obj key is None when the object has not yet been loaded from the db. + """ + def __init__(self, field, rel, embed, objects=[], model_instance=None): + self.model_instance = model_instance + self.field = field + self.rel = rel + self.embed = embed + self.objects = list(objects) # make copy of the list to avoid problems + + def _with_model_instance(self, model_instance): + """ + Create a new copy of this manager for a specific model instance. This + is called when the field is being accessed through a model instance. + """ + return MongoDBM2MRelatedManager( + self.field, self.rel, self.embed, + self.objects, model_instance=model_instance) + + def __call__(self): + """ + This is used when creating a default value for the field + """ + return MongoDBM2MRelatedManager(self.field, self.rel, self.embed, self.objects) + + def count(self): + return len(self.objects) + + def add(self, *objs, **kwargs): + """ + Add model instance(s) to the M2M field. The objects can be real + Model instances or just ObjectIds (or strings representing ObjectIds). + + Only supported kwarg is 'auto_save' + :param auto_save: Defaults to True. When a model is added to the M2M, + the behavior of Django is to create an entry in the + through-table, which essentially saves the list. In order to do + the equivalent, we need to save the model. However, that + behavior is not the same as Django either because Django doesn't + save the whole model object, so that's why this is optional. + Swings and Roundabouts. + """ + auto_save = kwargs.pop('auto_save', True) + using = router.db_for_write(self.model_instance if self.model_instance + else self.field.model) + add_objs = [] + for obj in objs: + if isinstance(obj, (ObjectId, basestring)): + # It's an ObjectId + pk = ObjectId(obj) + instance = None + else: + # It's a model object + pk = ObjectId(obj.pk) + instance = obj + if not pk in (obj['pk'] for obj in self.objects): + add_objs.append({'pk':pk, 'obj':instance}) + + # Calculate list of object ids that are being added + add_obj_ids = [str(obj['pk']) for obj in add_objs] + + # Send pre_add signal (instance should be Through instance but it's the + # manager instance for now) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_add', reverse=False, model=self.rel.to, + pk_set=add_obj_ids, using=using) + + # Commit the add + for obj in add_objs: + self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) + + # Send post_add signal (instance should be Through instance but it's + # the manager instance for now) + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_add', reverse=False, model=self.rel.to, + pk_set=add_obj_ids, using=using) + + if auto_save: + self.model_instance.save() + + def create(self, **kwargs): + """ + Create new model instance and add to the M2M field. + """ + # See add() above for description of auto_save + auto_save = kwargs.pop('auto_save', True) + + obj = self.rel.to(**kwargs) + self.add(obj, auto_save=auto_save) + return obj + + def _remove_by_id_strings(self, removed_obj_ids): + ''' + Remove specified objects by list of id strings + ''' + # Send the pre_remove signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + # Commit the remove + self.objects = [obj for obj in self.objects if str(obj['pk']) not in removed_obj_ids] + + # Send the post_remove signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_remove', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + + def remove(self, *objs, **kwargs): + """ + Remove the specified object from the M2M field. + The object can be a real model instance or an ObjectId or + a string representing an ObjectId. The related object is + not deleted, it's only removed from the list. + + Only supported kwarg is 'auto_save' + :param auto_save: See add() above for description + """ + auto_save = kwargs.pop('auto_save', True) + + obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) + else ObjectId(obj.pk) for obj in objs]) + + # Calculate list of object ids that will be removed + removed_obj_ids = [str(obj['pk']) for obj in self.objects if obj['pk'] in obj_ids] + self._remove_by_id_strings(removed_obj_ids) + + if auto_save: + self.model_instance.save() + + def remove_nonexists(self, **kwargs): + """ + remove objects not exist in db + + :param auto_save: See add() above for description + """ + auto_save = kwargs.pop('auto_save', True) + + exists_ids = [obj['_id'] for obj in get_exists_ids(self.model_instance, self.rel, self.objects)] + removed_obj_ids = [str(obj['pk']) for obj in self.objects if (not obj['pk'] in exists_ids)] + self._remove_by_id_strings(removed_obj_ids) + + if auto_save: + self.model_instance.save() + + + def reload_from_db(self, **kwargs): + """ + Reload all objs from db, and remove objs not exists + + A short cut using all(use_cached=False),. + TODO: dev a more effiecient method + """ + auto_save = kwargs.pop('auto_save', True) + + all_objects_from_db = [obj for obj in self.all(use_cached=False)] + self.clear(auto_save=False) + self.add(*all_objects_from_db, auto_save=False) + + if auto_save: + self.model_instance.save() + + def clear(self, auto_save=True): + """ + Clear all objects in the list. The related objects are not + deleted from the database. + + :param auto_save: See add() above for description + """ + # Calculate list of object ids that will be removed + removed_obj_ids = [str(obj['pk']) for obj in self.objects] + + # Send the pre_clear signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='pre_clear', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + # Commit the clear + self.objects = [] + + # Send the post_clear signal + m2m_changed.send(self.rel.through, instance=self.model_instance, + action='post_clear', reverse=False, model=self.rel.to, + pk_set=removed_obj_ids) + + if auto_save: + self.model_instance.save() + + def __contains__(self, obj): + """ + Helper to enable 'object in container' by comparing IDs. + """ + if hasattr(obj, 'pk'): obj = obj.pk + elif hasattr(obj, 'id'): obj = obj.id + return ObjectId(obj) in [ObjectId(o['pk']) for o in self.objects] + + def __iter__(self): + """ + Iterator is used by Django admin's ModelMultipleChoiceField. + """ + for obj in self.objects: + if not obj['obj']: + # Load referred instance from db and keep in memory + obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) + yield obj['obj'] + + def all(self, **kwargs): + """ + Return all the related objects as a query set. If embedding + is enabled, returns embedded objects. Otherwise the query set + will retrieve the objects from the database as needed. + """ + return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, + **kwargs) + + def ids(self): + """ + Return a list of ObjectIds of all the related objects. + """ + return [obj['pk'] for obj in self.objects] + + def objs(self): + """ + Return the actual related model objects, loaded fresh from + the database. This won't use embedded objects even if they + exist. + + Depreciated by all(use_cached=False) + """ + warnings.warn('MongoDBM2MRelatedManager.objs depreciated by all(use_cached=False)', + DeprecationWarning) + return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, + use_cached=False) + + def to_python_embedded_instance(self, embedded_instance): + """ + Convert a single embedded instance value stored in the database to an + object we can store in the internal objects list. + """ + if isinstance(embedded_instance, ObjectId): + # It's an object id, probably from a ListField(ForeignKey) migration + return {'pk': embedded_instance, 'obj': None} + elif isinstance(embedded_instance, basestring): + # Assume it's a string formatted object id, probably from a + # ListField(ForeignKey) migration + return {'pk': ObjectId(embedded_instance), 'obj': None} + + elif isinstance(embedded_instance, tuple): + # This is the typical path for embedded instances (embed=True) + # The tuples is format: (, ) + cls, values = embedded_instance + if len(values) == 1 and 'id' in values and \ + len(cls._meta.fields) > 1: + # In this case, the user most likely just switched from + # embed=False to embed=True. We need to treat is as such + return self.to_python_embedded_instance( + {"id": ObjectId(values['id'])}) + else: + if isinstance(values, tuple): + # In some versions of django-toolbox, 'values' is a tuple. + values = dict(values) + # Otherwise it's been embedded previously + instance = cls(**values) + return {'pk': ObjectId(instance.pk), 'obj': instance} + + elif self.embed: + # Try to load the embedded object contents if possible + if isinstance(embedded_instance, dict): + # Convert the embedded value from dict to model + data = {} + for field in self.rel.to._meta.fields: + try: + data[str(field.attname)] = \ + embedded_instance[field.column] + except KeyError: + pass + + # If we only got the id, give up to avoid creating an + # invalid/empty model instance + if len(data) <= 1: + column = self.rel.to._meta.pk.column + return {'pk': ObjectId(embedded_instance[column]), + 'obj': None} + else: + # Otherwise create the model instance from the fields + obj = self.rel.to(**data) + # Make sure the pk in the model instance is a string + # (not ObjectId) to be compatible with django-mongodb-engine + if isinstance(obj.pk, ObjectId): + obj.pk = str(obj.pk) + return {'pk': ObjectId(obj.pk), 'obj': obj} + else: + # Assume it's already a model + obj = embedded_instance + # Make sure the pk is a string (not ObjectId) to be compatible + # with django-mongodb-engine + if isinstance(obj.pk, ObjectId): + obj.pk = str(obj.pk) + return {'pk': ObjectId(obj.pk), 'obj': obj} + else: + # No embedded value, only ObjectId + if isinstance(embedded_instance, dict): + # Get the id value from the dict + column = self.rel.to._meta.pk.column + return {'pk': ObjectId(embedded_instance[column]), 'obj': None} + else: + # Assume it's already a model + return {'pk': ObjectId(embedded_instance.pk), 'obj': None} + + def to_python(self, values): + """ + Convert a database value to Django model instances managed by this + manager. + """ + if isinstance(values, models.Model): + # Single value given as parameter + values = [values] + self.objects = [self.to_python_embedded_instance(value) + for value in values] + + def get_db_prep_value_embedded_instance(self, obj, connection): + """ + Convert an internal object value to database representation. + """ + if not obj: return None + pk = obj['pk'] + if not self.embed: + # If we're not embedding, store only the ID + return {self.rel.to._meta.pk.column: pk} + if not obj['obj']: + # Retrieve the object from db for storing as embedded data + obj['obj'] = self.rel.to.objects.get(pk=pk) + embedded_instance = obj['obj'] + values = {} + + for field in embedded_instance._meta.fields: + value = field.pre_save(embedded_instance, add=True) + value = field.get_db_prep_save(value, connection=connection) + values[field] = value + # Convert primary key into an ObjectId so it's stored correctly + # values[self.rel.to._meta.pk] = ObjectId(values[self.rel.to._meta.pk]) + return values + + def get_db_prep_value(self, connection, prepared=False): + """Convert the Django model instances managed by this manager into a + special list that can be stored in MongoDB. + """ + values = [self.get_db_prep_value_embedded_instance(obj, connection) + for obj in self.objects] + return values + + +class MongoDBManyToManyRelationDescriptor(object): + """ + This descriptor returns the 'through' model used in Django admin to access the + ManyToManyField objects for inlines. It's implemented by the MongoDBManyToManyThrough + class, which simulates a data model. This class also handles the attribute assignment + from the MongoDB raw fields, which must be properly converted to Python objects. + + In other words, when you have a many-to-many field called categories on model Article, + this descriptor is the value of Article.categories. When you access the value + Article.categories.through, you get the through attribute of this object. + """ + def __init__(self, field, through): + self.field = field + self.through = through + + def __get__(self, obj, type=None): + """ + A field is being accessed on a model instance. Add the model instance to the + related manager so we can use it for signals etc. + """ + if obj: + manager = obj.__dict__[self.field.name] + if not manager.model_instance: + manager = manager._with_model_instance(obj) + # Store it in the model for future reference + obj.__dict__[self.field.name] = manager + return manager + else: + return type.__dict__[self.field.name] + + def __set__(self, obj, value): + """ + Attributes are being assigned to model instance. We redirect the + assignments to the model instance's fields instances. + """ + obj.__dict__[self.field.name] = self.field.to_python(value) + + def _filter_or_exclude(self, negate, *args, **kwargs): + """Enables queries on the host-model-level for contents of this field. + That means calling this filter will return instances of the + MongoDBManyToManyField host model, not instances of the related model. + + If embed=True, anything can be queried. If embed=False, then only + model objects (or ids) can be compared. In this case, the only accepted + argument is 'pk'. The reason for this is because related models are + stored by pk. + + Warning: Only the first call to this method will actually behave + correctly. If you string multiple calls to filter together, the + remaining filters after the first will all act on the fields + of the host model. + + Example: + >>>class M2MModel(models.Model): + >>> name = models.CharField() + >>> + >>>class Host(models.Model): + >>> m2m = MongoDBManyToManyField(M2MModel, embed=True) + >>> + >>> m = M2MModel.objects.get(name="foo") + >>> Host.m2m.filter(pk=m) + [] + >>> Host.m2m.filter(name="foo") + [] + + Important Distinction: + The above example is acting on the Host model *class*, not instance. + Calling filter on an *instance* of Host model would return instances of + the M2M related model. Example: + + >>> h = Host.objects.get(id=1) + >>> h.m2m.filter(name="foo") + [] + + Author's Note: + I very much dislike this solution, but a better solution eludes me + right now. In order to get the behavior Django has, django-nonrel or + djangotoolbox need to be changed to support manytomany fields. + """ + def raise_query_error(): + raise MongoDBM2MQueryError( + "Invalid query paramaters: '%s; %s'. M2M Fields not using the " + "'embed=True' option can only filter on 'pk' because only " + "the related model's pk is stored for non-embedded M2Ms. " + "Note: M2M fields that are converted to 'embed=True' do " + "not convert the stored values automatically. Every " + "instance of the host-model must be re-saved after " + "converting the field." % (args, kwargs)) + + embedded = self.field._mm2m_embed + column = self.field.column + + updated_args = [] + # Iterate over the arguments and replace them with A objects + for field in args: + if isinstance(field, Q): + # Some args may be Qs. This function replaces the Q children + # with A() objects. + status = replace_Q(field, column, + ["pk"] if not embedded else None) + if status: + updated_args.append(field) + else: + raise_query_error() + else: + # Anything else should be tuples of two items + updated_args.append( + (self.field.column, combine_A(field[0], field[1]))) + + updated_kwargs = [] + # Iterate over the kwargs and combine them into A objects + for field, value in kwargs.iteritems(): + if not embedded and field != 'pk': + raise_query_error() + + # Have to build Q objects because all the arguments will have the + # same key in the kwargs otherwise + updated_kwargs.append(Q(**{column: combine_A(field, value)})) + + query_args = updated_args + updated_kwargs + if negate: + return self.field.model.objects.exclude(*query_args) + else: + return self.field.model.objects.filter(*query_args) + + def filter(self, *args, **kwargs): + """See _filter_or_exclude() above for description""" + return self._filter_or_exclude(False, *args, **kwargs) + + def exclude(self, *args, **kwargs): + """See _filter_or_exclude() above for description""" + return self._filter_or_exclude(True, *args, **kwargs) + + def get(self, *args, **kwargs): + """Return a single object matching the query. + See _filter_or_exclude() above for more details. + """ + results = self.filter(*args, **kwargs) + num = len(results) + if num == 1: + return results[0] + elif num < 1: + raise self.field.model.DoesNotExist( + "%s matching query does not exist." + % self.field.model._meta.object_name) + else: + raise self.field.model.MultipleObjectsReturned( + "get() returned more than one %s -- it returned %s! " + "Lookup parameters were %s" + % (self.field.model._meta.object_name, num, kwargs)) + + +class MongoDBManyToManyRel(object): + """ + This object holds the information of the M2M relationship. + It's accessed by Django admin/forms in various contexts, and we also + use it internally. We try to simulate what's needed by Django. + """ + def __init__(self, field, to, related_name, embed): + self.model = None # added later from contribute_to_class + self.through = None # added later from contribute_to_class + #for django.core.management.validation + self.related_query_name = None + self.field = field + self.to = to + self.related_name = related_name + self.embed = embed + self.field_name = self.to._meta.pk.name + # Required for Django admin/forms to work. + self.multiple = True + self.limit_choices_to = {} + + def is_hidden(self): + return False + + def get_related_field(self, *args, **kwargs): + return self.field diff --git a/django_mongom2m/query.py b/django_mongom2m/query.py new file mode 100644 index 0000000..08ed428 --- /dev/null +++ b/django_mongom2m/query.py @@ -0,0 +1,198 @@ + +from django.db import router +from .utils import get_exists_ids +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId + + + +class MongoDBM2MQueryError(Exception): pass + + +class MongoDBM2MQuerySet(object): + """ + Helper for returning a set of objects from the managers. + Works similarly to Django's own query set objects. + Lazily loads non-embedded objects when iterated. + If embed=False, objects are always loaded from database. + """ + def __init__(self, rel, model, objects, + use_cached=True, + appear_as_relationship=(None, None, None, None, None), + **kwargs): + self.db = router.db_for_read(rel.model if rel.model else rel.field.model) + self.rel = rel + self.objects = list(objects) # make a copy of the list to avoid problems + + self.model = model + (self.appear_as_relationship_model, self.rel_model_instance, + self.rel_to_instance, self.rel_model_name, self.rel_to_name) = \ + appear_as_relationship # appear as an intermediate m2m model + if self.appear_as_relationship_model: + self.model = self.appear_as_relationship_model + self.use_cached = use_cached + if not self.use_cached: + # Reset any cached instances + self.objects = [{'pk': obj['pk'], 'obj': None} + for obj in self.objects] + #whether clear none exists objs for potential trouble + self.exists_in_db_only = kwargs.get('exists_in_db_only', False) + if self.exists_in_db_only: + #using only objects stored in db + exists_ids = [obj['_id'] for obj in get_exists_ids(self.model, self.rel, self.objects)] + self.objects = [obj for obj in self.objects if obj['pk'] in exists_ids] + + + def _get_obj(self, obj): + if not obj.get('obj'): + try: + # Load referred instance from db and keep in memory + obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) + except self.rel.to.DoesNotExist: + pass + # obj['obj'] will be None + if self.appear_as_relationship_model: + # Wrap us in a relationship class + if self.rel_model_instance: + args = {'pk': "%s$f$%s" % + (self.rel_model_instance.pk, obj['pk']), + self.rel_model_name: self.rel_model_instance, + self.rel_to_name: obj['obj']} + else: + # Reverse + args = {'pk': "%s$r$%s" % (self.rel_to_instance.pk, obj['pk']), + self.rel_model_name: obj['obj'], + self.rel_to_name: self.rel_to_instance} + wrapper = self.appear_as_relationship_model(**args) + return wrapper + return obj['obj'] + + def __iter__(self): + for obj in list(self.objects): + #ignore obj of nowhere + obj_cached_or_loaded = self._get_obj(obj) + if not obj_cached_or_loaded is None: + yield obj_cached_or_loaded + + def __repr__(self): + from . import REPR_OUTPUT_SIZE + # limit list after conversion because mongodb doesn't use integer indices + data = list(self)[:REPR_OUTPUT_SIZE + 1] + if len(data) > REPR_OUTPUT_SIZE: + data[-1] = "...(remaining elements truncated)..." + return repr(data) + + def __getitem__(self, key): + obj = self.objects[key] + return self._get_obj(obj) + + def ordered(self, *args, **kwargs): + return self + + def __len__(self): + return len(self.objects) + + def using(self, db, *args, **kwargs): + self.db = db + return self + + def filter(self, *args, **kwargs): + return self + + def get(self, *args, **kwargs): + if 'pk' in kwargs: + pk = ObjectId(kwargs['pk']) + for obj in self.objects: + if pk == obj['pk']: + return self._get_obj(obj) + return None + + def count(self): + return len(self.objects) + + def _clone(self, klass=None, setup=False, **kwargs): + ''' + return a clone of self queryset + works similar to django.db.models.query.QuerySet.clone + ''' + if klass is None: + klass = self.__class__ + #copy self.objects + objects = list(self.objects) + c = klass(rel=self.rel, model=self.model, objects=objects, + use_cached=self.use_cached, + appear_as_relationship=( + self.appear_as_relationship_model, + self.rel_model_instance, + self.rel_to_instance, + self.rel_model_name, self.rel_to_name), + ) + c.__dict__.update(kwargs) + #no use for now + if setup and hasattr(c, '_setup_query'): + c._setup_query() + return c + + def values_list(self, *fields, **kwargs): + ''' + Emulate QuerySet.values_list required by django.contrib.admin + ''' + flat = kwargs.pop('flat', False) + #required True for django.contrib.admin + exists_in_db_only = kwargs.pop('exists_in_db_only', self.exists_in_db_only) + if kwargs: + raise TypeError('Unexpected keyword arguments to values_list: %s' + % (list(kwargs),)) + if flat and len(fields) > 1: + raise TypeError("'flat' is not valid when values_list is called with more than one field.") + return self._clone(klass=MongoDBM2MValuesListQuerySet, setup=True, + flat=flat, + exists_in_db_only=exists_in_db_only, + _fields=fields) + +class MongoDBM2MValuesListQuerySet(MongoDBM2MQuerySet): + ''' + simulate ValuesListQuerySet, using objects instead of query + ''' + def iterator(self): + ''' + iterator yield only fields requested + ''' + for obj in list(self.objects): + obj_cached_or_loaded = self._get_obj(obj) + # skip when obj not in cached and db + if obj_cached_or_loaded is None: + pass + else: + #behavior same as ValuesListQuerySet.iterator + if self.flat and len(self._fields) == 1: + field = self._fields[0] + if hasattr(obj['obj'], field): + yield obj['obj'].__getattribute__(field) + else: + yield None + else: + row = list() + for field in self._fields: + if hasattr(obj['obj'], field): + row.append(obj['obj'].__getattribute__(field)) + else: + row.append(None) + yield tuple(row) + + def __iter__(self): + for item in self.iterator(): + yield item + + def _clone(self, *args, **kwargs): + ''' + override MongoDBM2MQuerySet._clone, clone this query set + ''' + clone = super(MongoDBM2MValuesListQuerySet, self)._clone(*args, **kwargs) + if not hasattr(clone, "flat"): + # Only assign flat if the clone didn't already get it from kwargs + clone.flat = self.flat + return clone diff --git a/django_mongom2m/utils.py b/django_mongom2m/utils.py new file mode 100644 index 0000000..5f0742d --- /dev/null +++ b/django_mongom2m/utils.py @@ -0,0 +1,218 @@ +from django.db import models, router, connections +from django.db.models import Q +from django.utils.translation import ugettext_lazy as _ +from django_mongodb_engine.contrib import MongoDBManager +from django_mongodb_engine.query import A +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId + + +def create_through(field, model, to): + """ + Create a dummy 'through' model for MongoDBManyToMany relations. Django assumes there is a real + database model providing the relationship, so we simulate it. This model has to have + a ForeignKey relationship to both models. We will also override the save() and delete() + methods to pass the adding and removing of related objects to the relation manager. + """ + obj_name = model._meta.object_name + to._meta.object_name + 'Relationship' + to_module_name = to._meta.module_name + model_module_name = model._meta.module_name + class ThroughQuerySet(object): + def __init__(self, relationship_model, *args, **kwargs): + self.to = to + self.model = relationship_model + self.model_instance = None + self.related_manager = None + self.to_instance = None + #avoiding using backends other than django-mongodb-engine + self.db = router.db_for_write(self.model) + def filter(self, *args, **kwargs): + if model_module_name in kwargs: + # Relation, set up for querying by the model + self.model_instance = kwargs[model_module_name] + self.related_manager = getattr(self.model_instance, field.name) + # Now we know enough to retrieve the actual query set + queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) + return queryset + if to_module_name in kwargs: + # Reverse relation, set up for querying by the to model + self.to_instance = kwargs[to_module_name] + self.reverse_manager = getattr(self.to_instance, field.rel.related_name) + queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) + return queryset + return self + def exists(self, *args, **kwargs): + return False + def none(self, *args, **kwargs): + if self.filter() != self: + return self.filter().none() + else: + return self + def ordered(self, *args, **kwargs): + return self + def using(self, db, *args, **kwargs): + self.db = db + return self + def get(self, *args, **kwargs): + # Check if it's a magic key + if 'pk' in kwargs and isinstance(kwargs['pk'], basestring) and '$' in kwargs['pk']: + model_id, direction, to_id = kwargs['pk'].split('$', 2) + if direction == 'r': + # It's a reverse magic key + to_id, model_id = model_id, to_id + if direction == 'r': + # Query in reverse + self.to_instance = self.to.objects.get(pk=to_id) + self.reverse_manager = getattr(self.to_instance, field.rel.related_name) + queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) + obj = queryset.get(pk=model_id) + return obj + else: + self.model_instance = model.objects.get(pk=model_id) + self.related_manager = getattr(self.model_instance, field.name) + queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) + return queryset.get(pk=to_id) + # Normal key + return None + def __len__(self): + ''' + hack: return 0 if filter is not ready, else using filter().__len__ + ''' + if self.filter() != self: + return self.filter().__len__() + else: + return 0 + def __getitem__(self, key): + ''' + hack: admin site will try __getitem___ but only catch IndexError + raise IndexError if filter is not ready, + else using filter().__getitem___ + ''' + if self.filter() != self: + import ipdb;ipdb.set_trace() + return self.filter().__getitem__(key) + else: + # Won't work, must be accessed through filter() + # hack: raise as IndexError for admin + raise IndexError('ThroughQuerySet relation unknown (__getitem__)') + + class ThroughManager(MongoDBManager): + def get_query_set(self): + return ThroughQuerySet(self.model) + class Through(models.Model): + class Meta: + auto_created = model + objects = ThroughManager() + locals()[to_module_name] = models.ForeignKey(to, null=True, blank=True) + locals()[model_module_name] = models.ForeignKey(model, null=True, blank=True) + def __unicode__(self): + return unicode(getattr(self, model_module_name)) + u' : ' + unicode(getattr(self, to_module_name)) + def save(self, *args, **kwargs): + # Don't actually save the model, convert to an add() call instead + obj = getattr(self, model_module_name) + manager = getattr(obj, field.name) + manager.add(getattr(self, to_module_name)) + obj.save() # must save parent model because Django admin won't + def delete(self, *args, **kwargs): + # Don't actually delete the model, convert to a delete() call instead + obj = getattr(self, model_module_name) + manager = getattr(obj, field.name) + manager.remove(getattr(self, to_module_name)) + obj.save() # must save parent model because Django admin won't + # Remove old model from Django's model registry, because it would be a duplicate + from django.db.models.loading import cache + model_dict = cache.app_models.get(Through._meta.app_label) + del model_dict[Through._meta.module_name] + # Rename the model + Through.__name__ = obj_name + Through._meta.app_label = model._meta.app_label + Through._meta.object_name = obj_name + #.model_name is used instead of .module_name since 1.6 + Through._meta.model_name = obj_name.lower() + Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.model_name + Through._meta.verbose_name = _('%(model)s %(to)s relationship') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} + Through._meta.verbose_name_plural = _('%(model)s %(to)s relationships') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} + # Add new model to Django's model registry + cache.register_models(Through._meta.app_label, Through) + return Through + + +def replace_Q(q, column, allowed_fields=None): + """Replace the fields in the Q object with A() objects from 'column' + + :param q: The Q object to work on + :param column: The name of the column the A() objects should be attached to. + :param allowed_fields: If defined, only fields names listed in + 'allowed_fields' are allowed. + E.g. allowed_fields=["pk"]: Q(pk=1) is good, Q(name="tom") fails. + :returns: Boolean; False if 'allowed_fields' missed. True otherwise + + Example: + M2M field is called 'users' + _replace_Q(Q(name="Tom"), "users") would modify the given Q to be: + Q(users=A("name", "Tom")) + + That would generate the query: {"users.name":"Tom"} + """ + if not isinstance(q, Q): + raise ValueError("'q' must be of type Q, not: '%s'" % type(q)) + + # Iterate over the Q object's children. The children are either another Q, + # or a tuple of (,) + for child in q.children: + if isinstance(child, Q): + # If we have a Q in the children, let's recurse to fix it too + replace_Q(child, column, allowed_fields) + elif isinstance(child, tuple): + # Otherwise we need to build an A(). Doing the index, remove, + # and insert to maintain the order of the children. I'm not sure + # changing the order matters, but I don't want to risk it. + index = q.children.index(child) + q.children.remove(child) + + # If allowed_fields is defined, this verifies that only those + # fields are present. E.g. ['pk'] + if allowed_fields and child[0] not in allowed_fields: + return False + # If all is well, build an A(), and insert back into the children + q.children.insert(index, (column, combine_A(child[0], child[1]))) + else: + raise TypeError("Unknown type in Q.children") + return True + + +def combine_A(field, value): + # The pk is actually stored as "id", so change it, we also need extract the + # pk from and models and wrap any IDs in an ObjectId, + if field in ('pk', 'id'): + field = "id" + if isinstance(value, models.Model): + # Specifically getattr field because we don't know if it's 'pk' + # or 'id' and they might not be the same thing. + value = getattr(value, field) + + # If value is None, we want to leave it as None, otherwise wrap it + if value is not None and not isinstance(value, ObjectId): + value = ObjectId(value) + + # If 'value' is already an A(), we need to extract the field part out + if isinstance(value, A): + field = "%s.%s" % (field, value.op) + value = value.val + return A(field, value) + +def get_exists_ids(model, rel, objects): + ''' + return a cursor return exists ids cached in m2mfield + + :param model: to determine db + :param rel: to determine table used for m2mfield + :param objects: objects for checking existence + ''' + db = connections[router.db_for_write(model)] + conn = db.get_collection(rel.to._meta.db_table) + ids = [obj['pk'] for obj in objects] + return conn.find({"_id":{"$in":ids}},{"_id":1}).limit(len(objects)) diff --git a/mongom2m/__init__.py b/mongom2m/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/mongom2m/fields.py b/mongom2m/fields.py deleted file mode 100644 index e75d4e3..0000000 --- a/mongom2m/fields.py +++ /dev/null @@ -1,614 +0,0 @@ -from djangotoolbox.fields import ListField, DictField, EmbeddedModelField, AbstractIterableField -from django.db.models.signals import m2m_changed -from django.db.models import get_model -from django.db.models.fields.related import add_lazy_relation -from django.utils.translation import ugettext_lazy as _ -from django_mongodb_engine.contrib import MongoDBManager -from django.forms import ModelMultipleChoiceField -from django.db import models - -# How much to show when query set is viewed in the Python shell -REPR_OUTPUT_SIZE = 20 - -# ObjectId has been moved to bson.objectid in newer versions of PyMongo -try: - from bson.objectid import ObjectId -except ImportError: - from pymongo.objectid import ObjectId - -class MongoDBM2MQuerySet(object): - """ - Helper for returning a set of objects from the managers. - Works similarly to Django's own query set objects. - Lazily loads non-embedded objects when iterated. - If embed=False, objects are always loaded from database. - """ - def __init__(self, rel, model, objects, use_cached, appear_as_relationship=(None, None, None, None, None)): - self.db = 'default' - self.rel = rel - self.objects = list(objects) # make a copy of the list to avoid problems - self.model = model - self.appear_as_relationship_model, self.rel_model_instance, self.rel_to_instance, self.rel_model_name, self.rel_to_name = appear_as_relationship # appear as an intermediate m2m model - if self.appear_as_relationship_model: - self.model = self.appear_as_relationship_model - if not use_cached: - # Reset any cached instances - self.objects = [{'pk':obj['pk'], 'obj':None} for obj in self.objects] - - def _get_obj(self, obj): - if not obj['obj']: - # Load referred instance from db and keep in memory - obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) - if self.appear_as_relationship_model: - # Wrap us in a relationship class - if self.rel_model_instance: - args = { 'pk':str(self.rel_model_instance.pk) + '$f$' + str(obj['pk']), self.rel_model_name:self.rel_model_instance, self.rel_to_name:obj['obj'] } - else: - # Reverse - args = { 'pk':str(self.rel_to_instance.pk) + '$r$' + str(obj['pk']), self.rel_model_name:obj['obj'], self.rel_to_name:self.rel_to_instance } - wrapper = self.appear_as_relationship_model(**args) - return wrapper - return obj['obj'] - - def __iter__(self): - for obj in self.objects: - yield self._get_obj(obj) - - def __repr__(self): - data = list(self)[:REPR_OUTPUT_SIZE + 1] # limit list after conversion because mongodb doesn't use integer indices - if len(data) > REPR_OUTPUT_SIZE: - data[-1] = "...(remaining elements truncated)..." - return repr(data) - - def __getitem__(self, key): - obj = self.objects[key] - return self._get_obj(obj) - - def ordered(self, *args, **kwargs): - return self - - def __len__(self): - return len(self.objects) - - def using(self, db, *args, **kwargs): - self.db = db - return self - - def filter(self, *args, **kwargs): - return self - - def get(self, *args, **kwargs): - if 'pk' in kwargs: - pk = ObjectId(kwargs['pk']) - for obj in self.objects: - if pk == obj['pk']: - return self._get_obj(obj) - return None - - def count(self): - return len(self.objects) - -class MongoDBM2MReverseManager(object): - """ - This manager is attached to the other side of M2M relationships - and will return query sets that fetch related objects. - """ - def __init__(self, rel_field, model, field, rel, embed): - self.rel_field = rel_field - self.model = model - self.field = field - self.rel = rel - self.embed = embed - - def all(self): - """ - Retrieve all related objects. - """ - name = self.field.column + '.' + self.rel.model._meta.pk.column - pk = ObjectId(self.rel_field.pk) - return self.model._default_manager.raw_query({name:pk}) - - def _relationship_query_set(self, model, to_instance, model_module_name, to_module_name): - """ - Emulate an intermediate 'through' relationship query set. - """ - objects = [{'pk':ObjectId(obj.pk), 'obj':obj} for obj in self.all()] - return MongoDBM2MQuerySet(self.rel, self.rel.to, objects, use_cached=True, appear_as_relationship=(model, None, to_instance, model_module_name, to_module_name)) - -class MongoDBM2MReverseDescriptor(object): - def __init__(self, model, field, rel, embed): - self.model = model - self.field = field - self.rel = rel - self.embed = embed - - def __get__(self, instance, owner=None): - if instance is None: - return self - return MongoDBM2MReverseManager(instance, self.model, self.field, self.rel, self.embed) - -class MongoDBM2MRelatedManager(object): - """ - This manager manages the related objects stored in a MongoDBManyToManyField. - They can be embedded or stored as relations (ObjectIds) only. - Internally, we store the objects as dicts that contain keys pk and obj. - The obj key is None when the object has not yet been loaded from the db. - """ - def __init__(self, field, rel, embed, objects=[], model_instance=None): - self.model_instance = model_instance - self.field = field - self.rel = rel - self.embed = embed - self.objects = list(objects) # make copy of the list to avoid problems - - def _with_model_instance(self, model_instance): - """ - Create a new copy of this manager for a specific model instance. This - is called when the field is being accessed through a model instance. - """ - return MongoDBM2MRelatedManager(self.field, self.rel, self.embed, self.objects, model_instance=model_instance) - - def __call__(self): - """ - This is used when creating a default value for the field - """ - return MongoDBM2MRelatedManager(self.field, self.rel, self.embed, self.objects) - - def count(self): - return len(self.objects) - - def add(self, *objs): - """ - Add model instance(s) to the M2M field. The objects can be real - Model instances or just ObjectIds (or strings representing ObjectIds). - """ - using = 'default' # should see if we can carry this over from somewhere - add_objs = [] - for obj in objs: - if isinstance(obj, (ObjectId, basestring)): - # It's an ObjectId - pk = ObjectId(obj) - instance = None - else: - # It's a model object - pk = ObjectId(obj.pk) - instance = obj - if not pk in (obj['pk'] for obj in self.objects): - add_objs.append({'pk':pk, 'obj':instance}) - - # Calculate list of object ids that are being added - add_obj_ids = [str(obj['pk']) for obj in add_objs] - - # Send pre_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) - - # Commit the add - for obj in add_objs: - self.objects.append({'pk':obj['pk'], 'obj':obj['obj']}) - - # Send post_add signal (instance should be Through instance but it's the manager instance for now) - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_add', reverse=False, model=self.rel.to, pk_set=add_obj_ids, using=using) - - return self - - def create(**kwargs): - """ - Create new model instance and add to the M2M field. - """ - obj = self.rel.to(**kwargs) - self.add(obj) - return obj - - def remove(self, *objs): - """ - Remove the specified object from the M2M field. - The object can be a real model instance or an ObjectId or - a string representing an ObjectId. The related object is - not deleted, it's only removed from the list. - """ - obj_ids = set([ObjectId(obj) if isinstance(obj, (ObjectId, basestring)) else ObjectId(obj.pk) for obj in objs]) - - # Calculate list of object ids that will be removed - removed_obj_ids = [str(obj['pk']) for obj in self.objects if obj['pk'] in obj_ids] - - # Send the pre_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - - # Commit the remove - self.objects = [obj for obj in self.objects if obj['pk'] not in obj_ids] - - # Send the post_remove signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_remove', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - - return self - - def clear(self): - """ - Clear all objecst in the list. The related objects are not - deleted from the database. - """ - # Calculate list of object ids that will be removed - removed_obj_ids = [str(obj['pk']) for obj in self.objects] - - # Send the pre_clear signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='pre_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - - # Commit the clear - self.objects = [] - - # Send the post_clear signal - m2m_changed.send(self.rel.through, instance=self.model_instance, action='post_clear', reverse=False, model=self.rel.to, pk_set=removed_obj_ids) - - return self - - def __contains__(self, obj): - """ - Helper to enable 'object in container' by comparing IDs. - """ - if hasattr(obj, 'pk'): obj = obj.pk - elif hasattr(obj, 'id'): obj = obj.id - return ObjectId(obj) in [ObjectId(o['pk']) for o in self.objects] - - def __iter__(self): - """ - Iterator is used by Django admin's ModelMultipleChoiceField. - """ - for obj in self.objects: - if not obj['obj']: - # Load referred instance from db and keep in memory - obj['obj'] = self.rel.to.objects.get(pk=obj['pk']) - yield obj['obj'] - - def all(self, **kwargs): - """ - Return all the related objects as a query set. If embedding - is enabled, returns embedded objects. Otherwise the query set - will retrieve the objects from the database as needed. - """ - return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=True, **kwargs) - - def ids(self): - """ - Return a list of ObjectIds of all the related objects. - """ - return [obj['pk'] for obj in self.objects] - - def objs(self): - """ - Return the actual related model objects, loaded fresh from - the database. This won't use embedded objects even if they - exist. - """ - return MongoDBM2MQuerySet(self.rel, self.rel.to, self.objects, use_cached=False) - - def to_python_embedded_instance(self, embedded_instance): - """ - Convert a single embedded instance value stored in the database to an object - we can store in the internal objects list. - """ - if isinstance(embedded_instance, ObjectId): - # It's an object id, probably from a ListField(ForeignKey) migration - return {'pk':embedded_instance, 'obj':None} - elif isinstance(embedded_instance, basestring): - # Assume it's a string formatted object id, probably from a ListField(ForeignKey) migration - return {'pk':ObjectId(embedded_instance), 'obj':None} - elif self.embed: - # Try to load the embedded object contents if possible - if isinstance(embedded_instance, dict): - # Convert the embedded value from dict to model - data = {} - for field in self.rel.to._meta.fields: - try: - data[str(field.attname)] = embedded_instance[field.column] - except KeyError: - pass - # If we only got the id, give up to avoid creating an invalid/empty model instance - if len(data) <= 1: - return {'pk':ObjectId(embedded_instance[self.rel.to._meta.pk.column]), 'obj':None} - # Otherwise create the model instance from the fields - obj = self.rel.to(**data) - # Make sure the pk in the model instance is a string (not ObjectId) to be compatible with django-mongodb-engine - if isinstance(obj.pk, ObjectId): - obj.pk = str(obj.pk) - return {'pk':ObjectId(obj.pk), 'obj':obj} - else: - # Assume it's already a model - obj = embedded_instance - # Make sure the pk is a string (not ObjectId) to be compatible with django-mongodb-engine - if isinstance(obj.pk, ObjectId): - obj.pk = str(obj.pk) - return {'pk':ObjectId(obj.pk), 'obj':obj} - else: - # No embedded value, only ObjectId - if isinstance(embedded_instance, dict): - # Get the id value from the dict - return {'pk':ObjectId(embedded_instance[self.rel.to._meta.pk.column]), 'obj':None} - else: - # Assume it's already a model - return {'pk':ObjectId(embedded_instance.pk), 'obj':None} - - def to_python(self, values): - """ - Convert a database value to Django model instances managed by this manager. - """ - if isinstance(values, models.Model): - # Single value given as parameter - values = [values] - self.objects = [self.to_python_embedded_instance(value) for value in values] - - def get_db_prep_value_embedded_instance(self, obj): - """ - Convert an internal object value to database representation. - """ - if not obj: return None - pk = obj['pk'] - if not self.embed: - # Store only the ID - return { self.rel.to._meta.pk.column:pk } - if not obj['obj']: - # Retrieve the object from db for storing as embedded data - obj['obj'] = self.rel.to.objects.get(pk=pk) - embedded_instance = obj['obj'] - values = {} - for field in embedded_instance._meta.fields: - value = field.pre_save(embedded_instance, add=True) - value = field.get_db_prep_value(value) - values[field.column] = value - # Convert primary key into an ObjectId so it's stored correctly - values[self.rel.to._meta.pk.column] = ObjectId(values[self.rel.to._meta.pk.column]) - return values - - def get_db_prep_value(self): - """ - Convert the Django model instances managed by this manager into a special list - that can be stored in MongoDB. - """ - return [self.get_db_prep_value_embedded_instance(obj) for obj in self.objects] - -def create_through(field, model, to): - """ - Create a dummy 'through' model for MongoDBManyToMany relations. Django assumes there is a real - database model providing the relationship, so we simulate it. This model has to have - a ForeignKey relationship to both models. We will also override the save() and delete() - methods to pass the adding and removing of related objects to the relation manager. - """ - obj_name = model._meta.object_name + to._meta.object_name + 'Relationship' - to_module_name = to._meta.module_name - model_module_name = model._meta.module_name - class ThroughQuerySet(object): - def __init__(self, relationship_model, *args, **kwargs): - self.to = to - self.model = relationship_model - self.model_instance = None - self.related_manager = None - self.to_instance = None - self.db = 'default' - def filter(self, *args, **kwargs): - if model_module_name in kwargs: - # Relation, set up for querying by the model - self.model_instance = kwargs[model_module_name] - self.related_manager = getattr(self.model_instance, field.name) - # Now we know enough to retrieve the actual query set - queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) - return queryset - if to_module_name in kwargs: - # Reverse relation, set up for querying by the to model - self.to_instance = kwargs[to_module_name] - self.reverse_manager = getattr(self.to_instance, field.rel.related_name) - queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) - return queryset - return self - def exists(self, *args, **kwargs): - return False - def ordered(self, *args, **kwargs): - return self - def using(self, db, *args, **kwargs): - self.db = db - return self - def get(self, *args, **kwargs): - # Check if it's a magic key - if 'pk' in kwargs and isinstance(kwargs['pk'], basestring) and '$' in kwargs['pk']: - model_id, direction, to_id = kwargs['pk'].split('$', 2) - if direction == 'r': - # It's a reverse magic key - to_id, model_id = model_id, to_id - if direction == 'r': - # Query in reverse - self.to_instance = self.to.objects.get(pk=to_id) - self.reverse_manager = getattr(self.to_instance, field.rel.related_name) - queryset = self.reverse_manager._relationship_query_set(self.model, self.to_instance, model_module_name, to_module_name).using(self.db) - obj = queryset.get(pk=model_id) - return obj - else: - self.model_instance = model.objects.get(pk=model_id) - self.related_manager = getattr(self.model_instance, field.name) - queryset = self.related_manager.all(appear_as_relationship=(self.model, self.model_instance, None, model_module_name, to_module_name)).using(self.db) - return queryset.get(pk=to_id) - # Normal key - return None - def __len__(self): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__len__)') - def __getitem__(self, key): - # Won't work, must be accessed through filter() - raise Exception('ThroughQuerySet relation unknown (__getitem__)') - class ThroughManager(MongoDBManager): - def get_query_set(self): - return ThroughQuerySet(self.model) - class Through(models.Model): - class Meta: - auto_created = model - objects = ThroughManager() - locals()[to_module_name] = models.ForeignKey(to, null=True, blank=True) - locals()[model_module_name] = models.ForeignKey(model, null=True, blank=True) - def __unicode__(self): - return unicode(getattr(self, model_module_name)) + u' : ' + unicode(getattr(self, to_module_name)) - def save(self, *args, **kwargs): - # Don't actually save the model, convert to an add() call instead - obj = getattr(self, model_module_name) - manager = getattr(obj, field.name) - manager.add(getattr(self, to_module_name)) - obj.save() # must save parent model because Django admin won't - def delete(self, *args, **kwargs): - # Don't actually delete the model, convert to a delete() call instead - obj = getattr(self, model_module_name) - manager = getattr(obj, field.name) - manager.remove(getattr(self, to_module_name)) - obj.save() # must save parent model because Django admin won't - # Remove old model from Django's model registry, because it would be a duplicate - from django.db.models.loading import cache - model_dict = cache.app_models.get(Through._meta.app_label) - del model_dict[Through._meta.module_name] - # Rename the model - Through.__name__ = obj_name - Through._meta.app_label = model._meta.app_label - Through._meta.object_name = obj_name - Through._meta.module_name = obj_name.lower() - Through._meta.db_table = Through._meta.app_label + '_' + Through._meta.module_name - Through._meta.verbose_name = _('%(model)s %(to)s relationship') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} - Through._meta.verbose_name_plural = _('%(model)s %(to)s relationships') % {'model':model._meta.verbose_name, 'to':to._meta.verbose_name} - # Add new model to Django's model registry - cache.register_models(Through._meta.app_label, Through) - return Through - -class MongoDBManyToManyRelationDescriptor(object): - """ - This descriptor returns the 'through' model used in Django admin to access the - ManyToManyField objects for inlines. It's implemented by the MongoDBManyToManyThrough - class, which simulates a data model. This class also handles the attribute assignment - from the MongoDB raw fields, which must be properly converted to Python objects. - - In other words, when you have a many-to-many field called categories on model Article, - this descriptor is the value of Article.categories. When you access the value - Article.categories.through, you get the through attribute of this object. - """ - def __init__(self, field, through): - self.field = field - self.through = through - - def __get__(self, obj, type=None): - """ - A field is being accessed on a model instance. Add the model instance to the - related manager so we can use it for signals etc. - """ - if obj: - manager = obj.__dict__[self.field.name] - if not manager.model_instance: - manager = manager._with_model_instance(obj) - # Store it in the model for future reference - obj.__dict__[self.field.name] = manager - return manager - else: - return type.__dict__[self.field.name] - - def __set__(self, obj, value): - """ - Attributes are being assigned to model instance. We redirect the assignments - to the model instance's fields instances. - """ - obj.__dict__[self.field.name] = self.field.to_python(value) - -class MongoDBManyToManyRel(object): - """ - This object holds the information of the M2M relationship. - It's accessed by Django admin/forms in various contexts, and we also - use it internally. We try to simulate what's needed by Django. - """ - def __init__(self, field, to, related_name, embed): - self.model = None # added later from contribute_to_class - self.through = None # added later from contribute_to_class - self.field = field - self.to = to - self.related_name = related_name - self.embed = embed - self.field_name = self.to._meta.pk.name - # Required for Django admin/forms to work. - self.multiple = True - self.limit_choices_to = {} - - - def is_hidden(self): - return False - - def get_related_field(self, *args, **kwargs): - return self.field - -class MongoDBManyToManyField(models.ManyToManyField): - """ - A generic MongoDB many-to-many field that can store embedded copies of - the referenced objects. Inherits from djangotoolbox.fields.ListField. - - The field's value is a MongoDBM2MRelatedManager object that works similarly to Django's - RelatedManager objects, so you can add(), remove(), creaet() and clear() on it. - To access the related object instances, all() is supported. It will return - all the related instances, using the embedded copies if available. - - If you want the 'real' related (non-embedded) model instances, call all_objs() instead. - If you want the list of related ObjectIds, call all_refs() instead. - - The related model will also gain a new accessor method xxx_set() to make reverse queries. - That accessor is a MongoDBM2MReverseManager that provides an all() method to return - a QuerySet of related objects. - - For example, if you have an Article model with a MongoDBManyToManyField 'categories' - that refers to Category objects, you will have these methods: - - article.categories.all() - Returns all the categories that belong to the article - category.article_set.all() - Returns all the articles that belong to the category - """ - description = 'ManyToMany field with references and optional embedded objects' - - def __init__(self, to, related_name=None, embed=False, default=None, *args, **kwargs): - # Call Field, not super, to skip Django's ManyToManyField extra stuff we don't need - self._mm2m_to_or_name = to - self._mm2m_related_name = related_name - self._mm2m_embed = embed - models.Field.__init__(self, *args, **kwargs) - - def contribute_after_resolving(self, field, to, model): - # Setup the main relation helper - self.rel = MongoDBManyToManyRel(self, to, self._mm2m_related_name, self._mm2m_embed) - # The field's default value will be an empty MongoDBM2MRelatedManager that's not connected to a model instance - self.default = MongoDBM2MRelatedManager(self, self.rel, self._mm2m_embed) - self.rel.model = model - self.rel.through = create_through(self, self.rel.model, self.rel.to) - # Determine related name automatically unless set - if not self.rel.related_name: - self.rel.related_name = model._meta.object_name.lower() + '_set' - #if hasattr(self.rel.to, self.rel.related_name): - # # Attribute name already taken, raise error - # raise Exception(u'Related name ' + unicode(self.rel.to._meta.object_name) + u'.' + unicode(self.rel.related_name) + u' is already used by another field, please choose another name with ' + unicode(name) + u' = ' + unicode(self.__class__.__name__) + u'(related_name=xxx)') - # Add the reverse relationship - setattr(self.rel.to, self.rel.related_name, MongoDBM2MReverseDescriptor(model, self, self.rel, self.rel.embed)) - # Add the relationship descriptor to the model class for Django admin/forms to work - setattr(model, self.name, MongoDBManyToManyRelationDescriptor(self, self.rel.through)) - - def contribute_to_class(self, model, name, *args, **kwargs): - self.__m2m_name = name - # Call Field, not super, to skip Django's ManyToManyField extra stuff we don't need - models.Field.contribute_to_class(self, model, name, *args, **kwargs) - # Do the rest after resolving the 'to' relation - add_lazy_relation(model, self, self._mm2m_to_or_name, self.contribute_after_resolving) - - def db_type(self, *args, **kwargs): - return 'list' - - def get_db_prep_value(self, value, connection, prepared=False): - # The Python value is a MongoDBM2MRelatedManager, and we'll store the models it contains as a special list. - if not isinstance(value, MongoDBM2MRelatedManager): - # Convert other values to manager objects first - value = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed, value) - # Let the manager to the conversion - return value.get_db_prep_value() - - def to_python(self, value): - # The database value is a custom MongoDB list of ObjectIds and embedded models (if embed is enabled). - # We convert it into a MongoDBM2MRelatedManager object to hold the Django models. - if not isinstance(value, MongoDBM2MRelatedManager): - manager = MongoDBM2MRelatedManager(self, self.rel, self.rel.embed) - manager.to_python(value) - value = manager - return value - -# def formfield(self, **kwargs): -# return super(MongoDBManyToManyField, self).formfield(**kwargs) diff --git a/mongom2m_testapp/models.py b/mongom2m_testapp/models.py index 6fe73b1..6cb0710 100644 --- a/mongom2m_testapp/models.py +++ b/mongom2m_testapp/models.py @@ -1,6 +1,6 @@ from django.db import models from djangotoolbox.fields import ListField, EmbeddedModelField -from mongom2m.fields import MongoDBManyToManyField +from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager class TestCategory(models.Model): diff --git a/mongom2m_testapp/tests.py b/mongom2m_testapp/tests.py index 167cec7..db29d24 100644 --- a/mongom2m_testapp/tests.py +++ b/mongom2m_testapp/tests.py @@ -1,11 +1,15 @@ from django.test import TestCase from django.db import models from django.db.models.signals import m2m_changed -from mongom2m.fields import MongoDBManyToManyField +from django_mongom2m.fields import MongoDBManyToManyField from django_mongodb_engine.contrib import MongoDBManager from djangotoolbox.fields import ListField, EmbeddedModelField from models import TestArticle, TestCategory, TestTag, TestAuthor, TestBook#, TestOldArticle, TestOldEmbeddedArticle -from pymongo.objectid import ObjectId +try: + # ObjectId has been moved to bson.objectid in newer versions of PyMongo + from bson.objectid import ObjectId +except ImportError: + from pymongo.objectid import ObjectId import sys class MongoDBManyToManyFieldTest(TestCase): @@ -67,10 +71,95 @@ def test_m2m(self): self.assertEqual(category4.testarticle_set.all().count(), 2) self.assertEqual(category4.testarticle_set.all()[0].title, 'test article 2') self.assertEqual(category4.testarticle_set.all()[1].title, 'test article 3') - + #tests on delete + #del tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_tag2.delete() + + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag still in all cached + self.assertIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + new_article2.save() + #Verify that all() and objs() dont change model in db + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag still in all cached + self.assertIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.reload_from_db() + new_article2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that deleted tag not in all cached + self.assertNotIn(tag2, new_article2.tags.all()) + # Verify that deleted tag not in all without cached + self.assertNotIn(tag2, new_article2.tags.all(use_cached=False)) + # Verify that deleted tag not in .objs + self.assertNotIn(tag2, new_article2.tags.objs()) + + #test change + #add back tag2 + tag2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.add(tag2) + new_article2.save() + #change tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_tag2.name = 'new tag 2' + new_tag2.save() + new_tag2 = TestTag.objects.get(pk=tag2.pk) + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that old name still in all cached + self.assertEqual(tag2.name, new_article2.tags.all()[0].name) + # Verify that old name not in all without cached + self.assertNotEqual(tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new name in all without cached + self.assertEqual(new_tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new_name in .objs + self.assertEqual(new_tag2.name, new_article2.tags.objs()[0].name) + new_article2.save() + + new_article2 = TestArticle.objects.get(pk=article2.pk) + new_article2.tags.reload_from_db() + new_article2.save() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify that old name not in all cached + self.assertNotEqual(tag2.name, new_article2.tags.all()[0].name) + # Verify that new name in all cached + self.assertEqual(new_tag2.name, new_article2.tags.all()[0].name) + # Verify that new name in all without cached + self.assertEqual(new_tag2.name, new_article2.tags.all(use_cached=False)[0].name) + # Verify that new_name in .objs + self.assertEqual(new_tag2.name, new_article2.tags.objs()[0].name) + + #delete tag2 + new_tag2 = TestTag.objects.get(pk=tag2.pk) + tag2 = TestTag.objects.get(pk=tag2.pk) + tag2.delete() + new_article2 = TestArticle.objects.get(pk=article2.pk) + # Verify tag2 still in cache + self.assertIn(new_tag2, new_article2.tags.all()) + #remove nonexists + new_article2.tags.remove_nonexists() + new_article2.save() + # Verify tag2 removed from cache + self.assertNotIn(new_tag2, new_article2.tags.all()) + + + def test_migrations(self): """ Test migrating from an existing ListField(ForeignKey) field + Note: migrating is not directly supported with option embed=True + Workaround: ListField(ForeignKey) -> MongoDBManyToManyField(model) + -> MongoDBManyToManyField(model, embed=True) """ # Create test categories category1 = TestCategory(title='test cat 1') @@ -82,45 +171,81 @@ def test_migrations(self): tag1.save() tag2 = TestTag(name='test tag 2') tag2.save() - + # Create the old data - this model uses ListField(ForeignKey) fields class TestOldArticle(models.Model): class Meta: # Used for testing migrations using same MongoDB collection - db_table = 'mongom2m_testapp_testarticle' - + db_table = TestArticle._meta.db_table + objects = MongoDBManager() main_category = models.ForeignKey(TestCategory, related_name='main_oldarticles') categories = ListField(models.ForeignKey(TestCategory)) tags = ListField(models.ForeignKey(TestTag)) title = models.CharField(max_length=254) text = models.TextField() - + def __unicode__(self): return self.title - + old_article = TestOldArticle(title='old article 1', text='old article text 1', main_category=category1, categories=[category1.id, category2.id], tags=[tag1.id, tag2.id]) old_article.save() - - # Now use the new model to access the old data. - new_article = TestArticle.objects.get(title='old article 1') - + + class TestTransferArticle(models.Model): + ''' + with tags not embedded + ''' + class Meta: + db_table = TestArticle._meta.db_table + objects = MongoDBManager() + main_category = models.ForeignKey(TestCategory, related_name='main_articles') + categories = MongoDBManyToManyField(TestCategory) + #without embed option + tags = MongoDBManyToManyField(TestTag, related_name='articles') + title = models.CharField(max_length=254) + text = models.TextField() + + def __unicode__(self): + return self.title + + # Now use the transfer model to access the old data. + new_article = TestTransferArticle.objects.get(title='old article 1') + # Make sure the fields were loaded correctly self.assertEqual(set(cat.title for cat in new_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in new_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in new_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in new_article.tags.all()), set((tag1.id, tag2.id))) - + # Re-save and reload the data to migrate it in MongoDB new_article.save() - migrated_article = TestArticle.objects.get(title='old article 1') - + migrated_article = TestTransferArticle.objects.get(title='old article 1') + # Make sure the fields are still loaded correctly self.assertEqual(set(cat.title for cat in migrated_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in migrated_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in migrated_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in migrated_article.tags.all()), set((tag1.id, tag2.id))) - + + # Now use the new model to access the old data. + new_article_final = TestArticle.objects.get(title='old article 1') + + # Make sure the fields were loaded correctly + self.assertEqual(set(cat.title for cat in new_article_final.categories.all()), set(('test cat 1', 'test cat 2'))) + self.assertEqual(set(cat.id for cat in new_article_final.categories.all()), set((category1.id, category2.id))) + self.assertEqual(set(tag.name for tag in new_article_final.tags.all()), set(('test tag 1', 'test tag 2'))) + self.assertEqual(set(tag.id for tag in new_article_final.tags.all()), set((tag1.id, tag2.id))) + + # Re-save and reload the data to migrate it in MongoDB + new_article_final.save() + migrated_article_final= TestArticle.objects.get(title='old article 1') + + # Make sure the fields are still loaded correctly + self.assertEqual(set(cat.title for cat in migrated_article_final.categories.all()), set(('test cat 1', 'test cat 2'))) + self.assertEqual(set(cat.id for cat in migrated_article_final.categories.all()), set((category1.id, category2.id))) + self.assertEqual(set(tag.name for tag in migrated_article_final.tags.all()), set(('test tag 1', 'test tag 2'))) + self.assertEqual(set(tag.id for tag in migrated_article_final.tags.all()), set((tag1.id, tag2.id))) + def test_embedded_migrations(self): """ Test migrating from an existing ListField(EmbeddedModelField) @@ -135,45 +260,45 @@ def test_embedded_migrations(self): tag1.save() tag2 = TestTag(name='test tag 2') tag2.save() - + # Create the old data - this model uses ListField(EmbeddedModelField) fields class TestOldEmbeddedArticle(models.Model): class Meta: # Used for testing migrations using same MongoDB collection - db_table = 'mongom2m_testapp_testarticle' - + db_table = TestArticle._meta.db_table + objects = MongoDBManager() main_category = models.ForeignKey(TestCategory, related_name='main_oldembeddedarticles') categories = ListField(EmbeddedModelField(TestCategory)) tags = ListField(EmbeddedModelField(TestTag)) title = models.CharField(max_length=254) text = models.TextField() - + def __unicode__(self): return self.title - + old_article = TestOldEmbeddedArticle(title='old embedded article 1', text='old embedded article text 1', main_category=category1, categories=[category1, category2], tags=[tag1, tag2]) old_article.save() - + # Now use the new model to access the old data. new_article = TestArticle.objects.get(title='old embedded article 1') - + # Make sure the fields were loaded correctly self.assertEqual(set(cat.title for cat in new_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in new_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in new_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in new_article.tags.all()), set((tag1.id, tag2.id))) - + # Re-save and reload the data to migrate it in MongoDB new_article.save() migrated_article = TestArticle.objects.get(title='old embedded article 1') - + # Make sure the fields are still loaded correctly self.assertEqual(set(cat.title for cat in migrated_article.categories.all()), set(('test cat 1', 'test cat 2'))) self.assertEqual(set(cat.id for cat in migrated_article.categories.all()), set((category1.id, category2.id))) self.assertEqual(set(tag.name for tag in migrated_article.tags.all()), set(('test tag 1', 'test tag 2'))) self.assertEqual(set(tag.id for tag in migrated_article.tags.all()), set((tag1.id, tag2.id))) - + def test_signals(self): """ Test signals emitted by various M2M operations. @@ -188,7 +313,7 @@ def test_signals(self): tag2 = TestTag(name='test tag 2') tag2.save() article = TestArticle(title='test article 1', text='test article 1 text', main_category=category1) - + # Test pre_add / post_add self.on_add_called = 0 def on_add(sender, instance, action, reverse, model, pk_set, *args, **kwargs): @@ -209,7 +334,7 @@ def on_add(sender, instance, action, reverse, model, pk_set, *args, **kwargs): article.categories.add(category1) self.assertEqual(self.on_add_called, 2) m2m_changed.disconnect(on_add) - + # Test pre_remove / post_remove self.on_remove_called = 0 def on_remove(sender, instance, action, reverse, model, pk_set, *args, **kwargs): @@ -230,7 +355,7 @@ def on_remove(sender, instance, action, reverse, model, pk_set, *args, **kwargs) article.categories.remove(category1) self.assertEqual(self.on_remove_called, 2) m2m_changed.disconnect(on_remove) - + # Test pre_clear / post_clear article.categories.add(category1) self.on_clear_called = 0 @@ -252,4 +377,3 @@ def on_clear(sender, instance, action, reverse, model, pk_set, *args, **kwargs): article.categories.clear() self.assertEqual(self.on_clear_called, 2) m2m_changed.disconnect(on_clear) - diff --git a/setup.py b/setup.py index 7da669b..c0cd7ad 100644 --- a/setup.py +++ b/setup.py @@ -2,11 +2,11 @@ setup( name='django-mongom2m', - version='0.1.0', - author=u'Kenneth Falck', - author_email='kennu@iki.fi', - packages=['mongom2m', 'mongom2m_testapp'], - url='https://github.com/kennu/django-mongom2m', + version='0.2.2', + author=u'Merchant Atlas Inc.', + author_email='support@merchantatlas.com', + packages=['django_mongom2m'], + url='https://github.com/mobilespinach/django-mongom2m', license='BSD licence, see LICENCE.txt', description='A ManyToManyField for django-mongodb-engine', long_description='A ManyToManyField for django-mongodb-engine',