django-kvstore-1.0/000755 000765 000024 00000000000 11327645203 014365 5ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore/000755 000765 000024 00000000000 11327645203 017404 5ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore.egg-info/000755 000765 000024 00000000000 11327645203 021076 5ustar00bradstaff000000 000000 django-kvstore-1.0/PKG-INFO000644 000765 000024 00000005066 11327645203 015471 0ustar00bradstaff000000 000000 Metadata-Version: 1.1 Name: django-kvstore Version: 1.0 Summary: An extensible key-value store backend for Django applications. Home-page: http://github.com/sixapart/django-kvstore Author: Six Apart Ltd. Author-email: python@sixapart.com License: UNKNOWN Description: An extensible key-value store backend for Django applications. This module provides an abstraction layer for accessing a key-value storage. Configuring your key-value store is a matter of adding a statement in this form to your Django settings module:: KEY_VALUE_STORE_BACKEND = 'scheme://store?parameters' Where ``scheme`` is one of the following, persistent stores: * db (local table accessed through Django's database connection) * googleappengine (Google AppEngine data store) * sdb (Amazon SimpleDB) * tokyotyrant (Tokyo Tyrant) And some non-persistent stores, provided mainly for testing purposes: * locmem * memcached ``store`` and ``parameters`` varies from one backend to another. Refer to the documentation included in each backend implementation for further details. You can define a django_kvstore-backed custom model, in a fashion similar to Django models (although it does not support querying, except by primary key lookup). Here's an example of a custom model class using django_kvstore:: from django_kvstore import models class MyData(models.Model): my_key = models.Field(pk=True) foo = models.Field() bar = models.Field() Typical usage for such a model:: key = "something_unique" data = MyData.get(key) if data is None: data = MyData(my_key=key) data.foo = "foo" data.bar = "bar" data.save() and deletion:: key = "something_unique" data = MyData.get(key) if data is not None: data.delete() Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Console Classifier: Environment :: Web Environment Classifier: Framework :: Django Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Topic :: Internet :: WWW/HTTP Requires: Django(>=1.1.1) Provides: django_kvstore django-kvstore-1.0/setup.cfg000644 000765 000024 00000000073 11327645203 016206 0ustar00bradstaff000000 000000 [egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 django-kvstore-1.0/setup.py000644 000765 000024 00000005177 11327621224 016106 0ustar00bradstaff000000 000000 #!/usr/bin/env python # Copyright (c) 2009 Six Apart Ltd. # All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # # * Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. # # * Redistributions in binary form must reproduce the above copyright notice, # this list of conditions and the following disclaimer in the documentation # and/or other materials provided with the distribution. # # * Neither the name of Six Apart Ltd. nor the names of its contributors may # be used to endorse or promote products derived from this software without # specific prior written permission. # # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE # ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS # INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN # CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE # POSSIBILITY OF SUCH DAMAGE. from setuptools import setup, find_packages from os.path import join, dirname try: long_description = open(join(dirname(__file__), 'README.rst')).read() except Exception: long_description = None setup( name='django-kvstore', version='1.0', description='An extensible key-value store backend for Django applications.', author='Six Apart Ltd.', author_email='python@sixapart.com', url='http://github.com/sixapart/django-kvstore', long_description=long_description, classifiers=[ 'Development Status :: 4 - Beta', 'Environment :: Console', 'Environment :: Web Environment', 'Framework :: Django', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Programming Language :: Python', 'Topic :: Internet :: WWW/HTTP', ], packages=find_packages(), provides=['django_kvstore'], include_package_data=True, zip_safe=True, requires=['Django(>=1.1.1)'], install_requires=['Django>=1.1.1'], ) django-kvstore-1.0/django_kvstore.egg-info/dependency_links.txt000644 000765 000024 00000000001 11327645203 025144 0ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore.egg-info/PKG-INFO000644 000765 000024 00000005066 11327645203 022202 0ustar00bradstaff000000 000000 Metadata-Version: 1.1 Name: django-kvstore Version: 1.0 Summary: An extensible key-value store backend for Django applications. Home-page: http://github.com/sixapart/django-kvstore Author: Six Apart Ltd. Author-email: python@sixapart.com License: UNKNOWN Description: An extensible key-value store backend for Django applications. This module provides an abstraction layer for accessing a key-value storage. Configuring your key-value store is a matter of adding a statement in this form to your Django settings module:: KEY_VALUE_STORE_BACKEND = 'scheme://store?parameters' Where ``scheme`` is one of the following, persistent stores: * db (local table accessed through Django's database connection) * googleappengine (Google AppEngine data store) * sdb (Amazon SimpleDB) * tokyotyrant (Tokyo Tyrant) And some non-persistent stores, provided mainly for testing purposes: * locmem * memcached ``store`` and ``parameters`` varies from one backend to another. Refer to the documentation included in each backend implementation for further details. You can define a django_kvstore-backed custom model, in a fashion similar to Django models (although it does not support querying, except by primary key lookup). Here's an example of a custom model class using django_kvstore:: from django_kvstore import models class MyData(models.Model): my_key = models.Field(pk=True) foo = models.Field() bar = models.Field() Typical usage for such a model:: key = "something_unique" data = MyData.get(key) if data is None: data = MyData(my_key=key) data.foo = "foo" data.bar = "bar" data.save() and deletion:: key = "something_unique" data = MyData.get(key) if data is not None: data.delete() Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Console Classifier: Environment :: Web Environment Classifier: Framework :: Django Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Operating System :: MacOS :: MacOS X Classifier: Operating System :: POSIX Classifier: Programming Language :: Python Classifier: Topic :: Internet :: WWW/HTTP Requires: Django(>=1.1.1) Provides: django_kvstore django-kvstore-1.0/django_kvstore.egg-info/requires.txt000644 000765 000024 00000000015 11327645203 023472 0ustar00bradstaff000000 000000 Django>=1.1.1django-kvstore-1.0/django_kvstore.egg-info/SOURCES.txt000644 000765 000024 00000001064 11327645203 022763 0ustar00bradstaff000000 000000 setup.py django_kvstore/__init__.py django_kvstore/models.py django_kvstore.egg-info/PKG-INFO django_kvstore.egg-info/SOURCES.txt django_kvstore.egg-info/dependency_links.txt django_kvstore.egg-info/requires.txt django_kvstore.egg-info/top_level.txt django_kvstore.egg-info/zip-safe django_kvstore/backends/__init__.py django_kvstore/backends/base.py django_kvstore/backends/db.py django_kvstore/backends/googleappengine.py django_kvstore/backends/locmem.py django_kvstore/backends/memcached.py django_kvstore/backends/sdb.py django_kvstore/backends/tokyotyrant.pydjango-kvstore-1.0/django_kvstore.egg-info/top_level.txt000644 000765 000024 00000000017 11327645203 023626 0ustar00bradstaff000000 000000 django_kvstore django-kvstore-1.0/django_kvstore.egg-info/zip-safe000644 000765 000024 00000000001 11327642511 022525 0ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore/__init__.py000644 000765 000024 00000004026 11327642305 021517 0ustar00bradstaff000000 000000 """ An extensible key-value store backend for Django applications. This package defines a set of key-value store backends that all conform to a simple API. A key-value store is a simple data storage backend that is similar to a cache. Unlike a cache, items are persisted to disk, and are not lost when the backend restarts.""" __version__ = '1.0' __date__ = '26 December 2010' __author__ = 'Six Apart Ltd.' __credits__ = """Mike Malone Brad Choate""" from cgi import parse_qsl from django.conf import settings from django.core import signals # Names for use in settings file --> name of module in "backends" directory. # Any backend scheeme that is not in this dictionary is treated as a Python # import path to a custom backend. BACKENDS = { 'memcached': 'memcached', 'tokyotyrant': 'tokyotyrant', 'locmem': 'locmem', 'db': 'db', 'simpledb': 'sdb', 'googleappengine': 'googleappengine', } class InvalidKeyValueStoreBackend(Exception): pass def get_kvstore(backend_uri): if backend_uri.find(':') == -1: raise InvalidKeyValueStoreBackend("Backend URI must start with scheme://") scheme, rest = backend_uri.split(':', 1) if not rest.startswith('//'): raise InvalidKeyValueStoreBackend("Backend URI must start with scheme://") host = rest[2:] qpos = rest.find('?') if qpos != -1: params = dict(parse_qsl(rest[qpos+1:])) host = rest[2:qpos] else: params = {} if host.endswith('/'): host = host[:-1] if scheme in BACKENDS: module = __import__('django_kvstore.backends.%s' % BACKENDS[scheme], {}, {}, ['']) else: module = __import__(scheme, {}, {}, ['']) return getattr(module, 'StorageClass')(host, params) kvstore = get_kvstore(settings.KEY_VALUE_STORE_BACKEND) """A handle to the configured key-value store.""" # Some kv store backends need to do a cleanup at the end of # a request cycle. If the cache provides a close() method, wire # it up here. if hasattr(kvstore, 'close'): signals.request_finished.connect(kvstore.close) django-kvstore-1.0/django_kvstore/backends/000755 000765 000024 00000000000 11327645203 021156 5ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore/models.py000644 000765 000024 00000006623 11327413671 021252 0ustar00bradstaff000000 000000 from django_kvstore import kvstore class FieldError(Exception): pass KV_PREFIX = '__KV_STORE_::' def generate_key(cls, pk): return str('%s%s.%s:%s' % (KV_PREFIX, cls.__module__, cls.__name__, pk)) class Field(object): def __init__(self, default=None, pk=False): self.default = default self.pk = pk def install(self, name, cls): setattr(cls, name, self.default) def decode(self, value): """Decodes an object from the datastore into a python object.""" return value def encode(self, value): """Encodes an object into a value suitable for the backend datastore.""" return value class ModelMetaclass(type): """ Metaclass for `kvstore.models.Model` instances. Installs `kvstore.models.Field` and `kvstore.models.Key` instances declared as attributes of the new class. """ def __new__(cls, name, bases, attrs): fields = {} for base in bases: if isinstance(base, ModelMetaclass): fields.update(base.fields) new_fields = {} # Move all the class's attributes that are Fields to the fields set. for attrname, field in attrs.items(): if isinstance(field, Field): new_fields[attrname] = field if field.pk: # Add key_field attr so we know what the key is if 'key_field' in attrs: raise FieldError("Multiple key fields defined for model '%s'" % name) attrs['key_field'] = attrname elif attrname in fields: # Throw out any parent fields that the subclass defined as # something other than a field del fields[attrname] fields.update(new_fields) attrs['fields'] = fields new_cls = super(ModelMetaclass, cls).__new__(cls, name, bases, attrs) for field, value in new_fields.items(): new_cls.add_to_class(field, value) return new_cls def add_to_class(cls, name, value): if hasattr(value, 'install'): value.install(name, cls) else: setattr(cls, name, value) class Model(object): __metaclass__ = ModelMetaclass def __init__(self, **kwargs): for name, value in kwargs.items(): setattr(self, name, value) def to_dict(self): d = {} for name, field in self.fields.items(): d[name] = field.encode(getattr(self, name)) return d def save(self): d = self.to_dict() kvstore.set(generate_key(self.__class__, self._get_pk_value()), d) def _get_pk_value(self): return getattr(self, self.key_field) @classmethod def from_dict(cls, fields): for name, value in fields.items(): # Keys can't be unicode to work as **kwargs. Must delete and re-add # otherwise the dict won't change the type of the key. if name in cls.fields: if isinstance(name, unicode): del fields[name] name = name.encode('utf-8') fields[name] = cls.fields[name].decode(value) else: del fields[name] return cls(**fields) @classmethod def get(cls, id): fields = kvstore.get(generate_key(cls, id)) if fields is None: return None return cls.from_dict(fields) django-kvstore-1.0/django_kvstore/backends/__init__.py000644 000765 000024 00000000000 11327413671 023257 0ustar00bradstaff000000 000000 django-kvstore-1.0/django_kvstore/backends/base.py000644 000765 000024 00000001347 11327413671 022451 0ustar00bradstaff000000 000000 "Base key-value store abstract class." from django.core.exceptions import ImproperlyConfigured class InvalidKeyValueStoreBackendError(ImproperlyConfigured): pass class BaseStorage(object): def __init__(self, *args, **kwargs): pass def set(self, key, value): """Set a value in the key-value store.""" raise NotImplementedError def delete(self, key): """Delete a key from the key-value store. Fail silently.""" raise NotImplementedError def has_key(self, key): """Returns True if the key is in the store.""" return self.get(key) is not None def __contains__(self, key): """Returns true if the key is in the store.""" return self.has_key(key) django-kvstore-1.0/django_kvstore/backends/db.py000644 000765 000024 00000004454 11327413671 022126 0ustar00bradstaff000000 000000 """ Database key-value store backend. Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'db://table_name' You will need to create a database table for storing the key-value pairs when using this backend. The table should have two columns, 'kee' - capable of storing up to 255 characters, and 'value', which should be a text type (character blob). You can declare a regular Django model for this table, if you want Django's ``syncdb`` command to create it for you. Just make sure the table name Django uses is the same table name provided in the ``KEY_VALUE_STORE_BACKEND`` setting. """ import base64 from django_kvstore.backends.base import BaseStorage from django.db import connection, transaction, DatabaseError try: import cPickle as pickle except ImportError: import pickle class StorageClass(BaseStorage): def __init__(self, table, params): BaseStorage.__init__(self, params) self._table = table def get(self, key): cursor = connection.cursor() cursor.execute("SELECT kee, value FROM %s WHERE kee = %%s" % self._table, [key]) row = cursor.fetchone() if row is None: return None return pickle.loads(base64.decodestring(row[1])) def set(self, key, value): encoded = base64.encodestring(pickle.dumps(value, 2)).strip() cursor = connection.cursor() cursor.execute("SELECT kee FROM %s WHERE kee = %%s" % self._table, [key]) try: if cursor.fetchone(): cursor.execute("UPDATE %s SET value = %%s WHERE kee = %%s" % self._table, [encoded, key]) else: cursor.execute("INSERT INTO %s (kee, value) VALUES (%%s, %%s)" % self._table, [key, encoded]) except DatabaseError, e: # To be threadsafe, updates/inserts are allowed to fail silently return False else: transaction.commit_unless_managed() return True def delete(self, key): cursor = connection.cursor() cursor.execute("DELETE FROM %s WHERE kee = %%s" % self._table, [key]) transaction.commit_unless_managed() def has_key(self, key): cursor = connection.cursor() cursor.execute("SELECT kee FROM %s WHERE kee = %%s" % self._table, [key]) return cursor.fetchone() is not None django-kvstore-1.0/django_kvstore/backends/googleappengine.py000644 000765 000024 00000002576 11327642140 024702 0ustar00bradstaff000000 000000 """ A Google AppEngine key-value store backend. Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'appengine://' """ import base64 from base import BaseStorage, InvalidKeyValueStoreBackendError try: import cPickle as pickle except ImportError: import pickle try: from google.appengine.ext import db except ImportError: raise InvalidKeyValueStoreBackendError("googleappengine key-value store backend requires google.appengine.ext.db import") class DjangoKVStore(db.Model): value = db.BlobProperty() class StorageClass(BaseStorage): def __init__(self, table=None, params=None): BaseStorage.__init__(self, params) self._model = DjangoKVStore def _get(self, key): return self._model.get_by_key_name('k:' + key) def get(self, key): row = self._get(key) if row is None: return None return pickle.loads(base64.decodestring(row.value)) def set(self, key, value): encoded = base64.encodestring(pickle.dumps(value, 2)).strip() row = self._get(key) if row is None: row = self._model(key_name='k:'+key) row.value = encoded row.save() return True def delete(self, key): row = self._get(key) if row is not None: row.delete() def has_key(self, key): return self._get(key) is not None django-kvstore-1.0/django_kvstore/backends/locmem.py000644 000765 000024 00000002741 11327413671 023012 0ustar00bradstaff000000 000000 """ Thread-safe in-memory key-value store backend. Just for testing. This isn't persistent. Don't actually use it. Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'locmem://' """ try: import cPickle as pickle except ImportError: import pickle from base import BaseStorage from django.utils.synch import RWLock class StorageClass(BaseStorage): def __init__(self, _, params): BaseStorage.__init__(self, params) self._db = {} self._lock = RWLock() def set(self, key, value): self._lock.writer_enters() try: self._db[key] = pickle.dumps(value) finally: self._lock.writer_leaves() def get(self, key): self._lock.reader_enters() # Python 2.3 and 2.4 don't allow combined try-except-finally blocks. try: try: return pickle.loads(self._db[key]) except KeyError: return None finally: self._lock.reader_leaves() def delete(self, key): self._lock.write_enters() # Python 2.3 and 2.4 don't allow combined try-except-finally blocks. try: try: del self._db[key] except KeyError: pass finally: self._lock.writer_leaves() def has_key(self, key): self._lock.reader_enters() try: return key in self._db finally: self._lcok.reader_leaves() django-kvstore-1.0/django_kvstore/backends/memcached.py000644 000765 000024 00000002307 11327413671 023442 0ustar00bradstaff000000 000000 """ Memcache key-value store backend Just for testing. This isn't persistent. Don't actually use it. Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'memcached://hostname:port' """ from base import BaseStorage, InvalidKeyValueStoreBackendError from django.utils.encoding import smart_unicode, smart_str try: import cmemcache as memcache except ImportError: try: import memcache except: raise InvalidKeyValueStoreBackendError("Memcached key-value store backend requires either the 'memcache' or 'cmemcache' library") class StorageClass(BaseStorage): def __init__(self, server, params): BaseStorage.__init__(self, params) self._db = memcache.Client(server.split(';')) def set(self, key, value): if isinstance(value, unicode): value = value.encode('utf-8') self._db.set(smart_str(key), value, 0) def get(self, key): val = self._db.get(smart_str(key)) if isinstance(val, basestring): return smart_unicode(val) else: return val def delete(self, key): self._db.delete(smart_str(key)) def close(self, **kwargs): self._db.disconnect_all() django-kvstore-1.0/django_kvstore/backends/sdb.py000644 000765 000024 00000003226 11327413671 022305 0ustar00bradstaff000000 000000 """ Amazon SimpleDB key-value store backend Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'sdb://?aws_access_key=&aws_secret_access_key=' """ from base import BaseStorage, InvalidKeyValueStoreBackendError from django.core.exceptions import ImproperlyConfigured from django.utils.encoding import smart_unicode, smart_str from django.utils import simplejson try: import simpledb except ImportError: raise InvalidKeyValueStoreBackendError("SipmleDB key-value store backend requires the 'python-simpledb' library") class StorageClass(BaseStorage): def __init__(self, domain, params): BaseStorage.__init__(self, params) params = dict(params) try: aws_access_key = params['aws_access_key'] aws_secret_access_key = params['aws_secret_access_key'] except KeyError: raise ImproperlyConfigured("Incomplete configuration of SimpleDB key-value store. Required parameters: 'aws_access_key', and 'aws_secret_access_key'.") self._db = simpledb.SimpleDB(aws_access_key, aws_secret_access_key) self._domain = self._db[domain] def set(self, key, value): if isinstance(value, unicode): value = value.encode('utf-8') self._domain[smart_str(key)] = {'value': simplejson.dumps(value)} def get(self, key): val = self._domain[smart_str(key)].get('value', None) if isinstance(val, basestring): return simplejson.loads(val) else: return val def delete(self, key): del self._domain[smart_str(key)] def close(self, **kwargs): pass django-kvstore-1.0/django_kvstore/backends/tokyotyrant.py000644 000765 000024 00000003233 11327413671 024142 0ustar00bradstaff000000 000000 """ Memcache key-value store backend Just for testing. This isn't persistent. Don't actually use it. Example configuration for Django settings: KEY_VALUE_STORE_BACKEND = 'tokyotyrant://hostname:port """ from base import BaseStorage, InvalidKeyValueStoreBackendError from django.core.exceptions import ImproperlyConfigured from django.utils.encoding import smart_unicode, smart_str from django.utils import simplejson try: import pytyrant except ImportError: raise InvalidKeyValueStoreBackendError("Tokyotyrant key-value store backend requires the 'pytyrant' library") class StorageClass(BaseStorage): def __init__(self, server, params): BaseStorage.__init__(self, params) host, port = server.split(':') try: port = int(port) except ValueError: raise ImproperlyConfigured("Invalid port provided for tokyo-tyrant key-value store backend") self._db = pytyrant.PyTyrant.open(host, port) def set(self, key, value): if isinstance(value, unicode): value = value.encode('utf-8') self._db[smart_str(key)] = simplejson.dumps(value) def get(self, key): val = self._db.get(smart_str(key)) if isinstance(val, basestring): return simplejson.loads(val) else: return val def delete(self, key): del self._db[smart_str(key)] def close(self, **kwargs): pass # Er, should be closing after each request..? But throws # a 'Bad File Descriptor' exception if we do (presumably because # something's trying to use a connection that's already been # closed... #self._db.close()