v0.1.4

Author: maxtepkeev

HISTORY.rst

@@ -0,0 +1,29 @@
+.. :changelog:
+
+History
+-------
+
+0.1.4 (2013-06-01)
+~~~~~~~~~~~~~~~~~~
+
+- Packaging fix
+
+0.1.3 (2013-06-01)
+~~~~~~~~~~~~~~~~~~
+
+- Packaging fix
+
+0.1.2 (2013-06-01)
+~~~~~~~~~~~~~~~~~~
+
+- Packaging fix
+
+0.1.1 (2013-06-01)
+~~~~~~~~~~~~~~~~~~
+
+- Packaging fix
+
+0.1.0 (2013-06-01)
+~~~~~~~~~~~~~~~~~~
+
+- Initial release

LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2013 Max Tepkeev
+All rights reserved.
+ 
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+ 
+    1. Redistributions of source code must retain the above copyright notice,
+       this list of conditions and the following disclaimer.
+    
+    2. Redistributions in binary form must reproduce the above copyright
+       notice, this list of conditions and the following disclaimer in the
+       documentation and/or other materials provided with the distribution.
+ 
+    3. Neither the name of this project nor the names of its contributors may be
+       used to endorse or promote products derived from this software without
+       specific prior written permission.
+ 
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

MANIFEST.in

@@ -0,0 +1,3 @@
+include README.rst
+include HISTORY.rst
+include LICENSE

README.md

@@ -0,0 +1,144 @@
+Django DB Parti
+===============
+
+Django DB Parti is a package for Django which aim is to make table partitioning on the fly.
+Partitioning is a division of one large table into smaller tables which represent that table.
+Partitioning is usually done for manageability, performance or availability reasons. If you
+are unsure whether you need partitioning or not, then you almost certainly don't need it.
+
+Requirements
+------------
+
+* Django 1.5 (http://www.djangoproject.com) (may work with older versions, but untested)
+
+Installation
+------------
+
+From pypi_::
+
+    $ pip install django-db-parti
+
+or clone from github_::
+
+    $ git clone git://github.com/maxtepkeev/django-db-parti.git
+
+Configuration
+-------------
+
+Add dbparti to PYTHONPATH and installed applications::
+
+    INSTALLED_APPS = (
+        ...
+        'dbparti'
+    )
+
+Create the model as usual which will represent the partitioned table, if you are using South
+for migrations, you can also create the model as usual. No additional steps required. After that
+we need to make a few changes to the model:
+
+\1) In models.py add the following import statement at the top of the file::
+
+    from dbparti.models import Partitionable
+
+\2) Make your model to inherit from Partitionable, to do that change::
+
+    class YourModelName(models.Model):
+
+to::
+
+    class YourModelName(Partitionable):
+
+\3) Optionally add a Meta class to your model with a few settings (or if you already have a Meta class change it as the following)::
+
+    class Meta(Partitionable.Meta):
+        partition_range = 'month'
+        partition_column = 'partdate'
+
+That's it! Easy right?! Now a few words about what we just did. We made our model to inherit from Partitionable, also we
+used "month" as partition range and "partdate" as partition column, that means that from now on, a new partition will be
+created every month and a value from partdate column will be used for that. You can also customize how data from that model
+will be displayed in the Django admin interface, for that you need to do the following:
+
+\1) In admin.py add the following import statement at the top of the file::
+
+    from dbparti.admin import PartitionableAdmin
+
+\2) Create admin model as usual and then change::
+
+    class YourAdminModelName(admin.ModelAdmin):
+
+to::
+
+    class YourAdminModelName(PartitionableAdmin):
+
+\3) Optionally add a setting which tells how records are displayed in Django admin interface (more on that below)::
+
+    partition_show = 'all'
+
+Available settings
+------------------
+
+Model settings:
+~~~~~~~~~~~~~~~
+
+All model settings are done inside model's Meta class which should inherit from Partitionable.Meta
+
+``partition_range`` - how often a new partition will be created, currently accepts the following:
+
+* week
+* month (default)
+
+``partition_column`` - column name, which value will be used to determine which partition record belongs to:
+
+* partdate (default)
+
+ModelAdmin settings:
+~~~~~~~~~~~~~~~~~~~~
+
+All model admin settings are done inside model admin class itself
+
+``partition_show`` - data from which partition will be shown in Django admin, the following values are possible:
+
+* all (default)
+* current
+* previous
+
+Example
+-------
+
+Let's imagine that we would like to create a table for storing log files. Without partitioning our table would have
+millions of rows very soon and as the table grows performance will become slower. With partitioning we can tell database
+that we want a new table to be created every month and that we will use a value from partdate to determine to which partition
+every new record belongs to. To be more specific let's call our table "logdata", it will have only 3 columns: id, content and
+logdate. Now when we insert the following record: id='1', content='blablabla', logdate='2013-05-20', this record will be
+inserted not to our "logdata" table but to the "logdata_y2013m05", then if we insert another record like that: id='2',
+content='yadayadayada', logdate='2013-07-16' it will be inserted to the table "logdata_y2013m07" BUT the great thing about
+all of that is that you are doing your inserts/updates/selects to the table "logdata"! Again, your are working with the table
+"logdata" as usual and you don't may even know that actually your data is stored in a lot of different tables, everything is
+done for you automatically at the database level, isn't that cool ?!
+
+Backends
+--------
+
+Django DB Parti is designed in a modular way, so new db backends can be added easily, currently the following backends are available:
+
+* postgresql
+
+Limitations
+-----------
+
+Currently partitioning is only possible on a date basis, so you can't partition for example by ZIP code or something else. Other
+partitioning options will be added in next releases.
+
+Contact & Support
+-----------------
+
+I will be glad to get your feedback, pull requests, issues, whatever. Feel free to contact me for any questions.
+
+Copyright & License
+-------------------
+
+``django-db-parti`` is protected by BSD licence.
+
+.. _pypi: https://pypi.python.org/pypi/django-db-parti
+.. _github: https://github.com/maxtepkeev/django-db-parti

README.rst

@@ -0,0 +1,29 @@
+from django.contrib import admin
+from dbparti.utilities import DateTimeMixin
+
+
+class PartitionableAdmin(DateTimeMixin, admin.ModelAdmin):
+    partition_show = 'all'
+
+    def __init__(self, *args, **kwargs):
+        admin.ModelAdmin.__init__(self, *args, **kwargs)
+        DateTimeMixin.__init__(
+            self,
+            self.partition_show,
+            self.opts.partition_range,
+            None,
+            self.opts.get_field(self.opts.partition_column).get_internal_type(),
+        )
+
+    def queryset(self, request):
+        fday, lday = self.get_partition_show_period(self.get_datetime_string('date'))
+        qs = super(PartitionableAdmin, self).queryset(request)
+
+        if fday is None and lday is None:
+            qs = qs.all()
+        else:
+            qs = qs.filter(
+                **{self.opts.partition_column + '__gte': fday, self.opts.partition_column + '__lte': lday}
+            )
+
+        return qs

dbparti/__init__.py

@@ -0,0 +1,81 @@
+from django.db import connection, transaction
+
+
+class Postgresql(object):
+    def __init__(self, table, partition_column):
+        self.table = table
+        self.cursor = connection.cursor()
+        self.partition_column = partition_column
+
+    def partition_exists(self, partition_name):
+        self.cursor.execute('SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_name=%s)',
+            (self.table + partition_name,))
+        return self.cursor.fetchone()[0]
+
+    def create_partition(self, partition_name, datetype, fday, lday):
+        self.cursor.execute('''
+            -- We need to create a table first
+            CREATE TABLE {child_table} (
+                CHECK ( {partition_column} >= {datetype} '{fday}' AND {partition_column} <= {datetype} '{lday}' )
+            ) INHERITS ({parent_table});
+
+            -- Then we create an index to speed up things a little
+            CREATE INDEX {child_table}_{partition_column} ON {child_table} ({partition_column});
+
+            -- Now we need to create a before insert function
+            CREATE OR REPLACE FUNCTION {parent_table}_insert_child()
+            RETURNS TRIGGER AS $$
+                BEGIN
+                    INSERT INTO {child_table} VALUES (NEW.*);
+                    RETURN NEW;
+                END;
+            $$ LANGUAGE plpgsql;
+
+            -- Then we create a trigger which calls the before insert function
+            DO $$
+            BEGIN
+            IF NOT EXISTS(
+                SELECT 1 
+                FROM information_schema.triggers
+                WHERE event_object_table = '{parent_table}'
+                AND trigger_name = 'before_insert_{parent_table}_trigger'
+            ) THEN
+                CREATE TRIGGER before_insert_{parent_table}_trigger
+                    BEFORE INSERT ON {parent_table}
+                    FOR EACH ROW EXECUTE PROCEDURE {parent_table}_insert_child();
+            END IF;
+            END $$;
+
+            -- Then we create a function to delete duplicate row from the master table after insert
+            CREATE OR REPLACE FUNCTION {parent_table}_delete_master()
+            RETURNS TRIGGER AS $$
+                BEGIN
+                    DELETE FROM ONLY {parent_table} WHERE id = NEW.id;
+                    RETURN NEW;
+                END;
+            $$ LANGUAGE plpgsql;
+
+            -- Lastly we create the after insert trigger that calls the after insert function
+            DO $$
+            BEGIN
+            IF NOT EXISTS(
+                SELECT 1 
+                FROM information_schema.triggers
+                WHERE event_object_table = '{parent_table}'
+                AND trigger_name = 'after_insert_{parent_table}_trigger'
+            ) THEN
+                CREATE TRIGGER after_insert_{parent_table}_trigger
+                    AFTER INSERT ON {parent_table}
+                    FOR EACH ROW EXECUTE PROCEDURE {parent_table}_delete_master();
+            END IF;
+            END $$;
+        '''.format(
+            child_table=self.table + partition_name,
+            parent_table=self.table,
+            partition_column=self.partition_column,
+            fday=fday,
+            lday=lday,
+            datetype='DATE' if datetype == 'date' else 'TIMESTAMP',
+        ))
+
+        transaction.commit_unless_managed()

dbparti/admin.py

@@ -0,0 +1,33 @@
+from django.db import models, connection
+from dbparti.utilities import DateTimeMixin
+
+
+models.options.DEFAULT_NAMES += ('partition_range', 'partition_column')
+
+class Partitionable(DateTimeMixin, models.Model):
+    def __init__(self, *args, **kwargs):
+        models.Model.__init__(self, *args, **kwargs)
+        DateTimeMixin.__init__(
+            self,
+            False,
+            self._meta.partition_range,
+            getattr(self, self._meta.partition_column),
+            self._meta.get_field(self._meta.partition_column).get_internal_type(),
+        )
+
+        self.db = getattr(__import__('dbparti.backends.' + connection.vendor,
+            fromlist=[connection.vendor.capitalize()]), connection.vendor.capitalize())(
+                self._meta.db_table, self._meta.partition_column)
+
+    def save(self, *args, **kwargs):
+        fday, lday = self.get_partition_range_period(self.get_datetime_string('date'))
+
+        if not self.db.partition_exists(self.get_partition_name()):
+            self.db.create_partition(self.get_partition_name(), self.get_datetype(), fday, lday)
+
+        super(Partitionable, self).save(*args, **kwargs)
+
+    class Meta:
+        abstract = True
+        partition_range = 'month'
+        partition_column = 'partdate'