python-dogpile.core-0.4.1+dfsg1/0000755000175000017500000000000012213266700015106 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/dogpile/0000755000175000017500000000000012076552100016530 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/dogpile/__init__.py0000644000175000017500000000036412076551651020656 0ustar zigozigo# See http://peak.telecommunity.com/DevCenter/setuptools#namespace-packages try: __import__('pkg_resources').declare_namespace(__name__) except ImportError: from pkgutil import extend_path __path__ = extend_path(__path__, __name__) python-dogpile.core-0.4.1+dfsg1/dogpile/core/0000755000175000017500000000000012076552100017460 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/dogpile/core/dogpile.py0000644000175000017500000001301012076551651021462 0ustar zigozigoimport time import logging log = logging.getLogger(__name__) class NeedRegenerationException(Exception): """An exception that when raised in the 'with' block, forces the 'has_value' flag to False and incurs a regeneration of the value. """ NOT_REGENERATED = object() class Lock(object): """Dogpile lock class. Provides an interface around an arbitrary mutex that allows one thread/process to be elected as the creator of a new value, while other threads/processes continue to return the previous version of that value. .. versionadded:: 0.4.0 The :class:`.Lock` class was added as a single-use object representing the dogpile API without dependence on any shared state between multiple instances. :param mutex: A mutex object that provides ``acquire()`` and ``release()`` methods. :param creator: Callable which returns a tuple of the form (new_value, creation_time). "new_value" should be a newly generated value representing completed state. "creation_time" should be a floating point time value which is relative to Python's ``time.time()`` call, representing the time at which the value was created. This time value should be associated with the created value. :param value_and_created_fn: Callable which returns a tuple of the form (existing_value, creation_time). This basically should return what the last local call to the ``creator()`` callable has returned, i.e. the value and the creation time, which would be assumed here to be from a cache. If the value is not available, the :class:`.NeedRegenerationException` exception should be thrown. :param expiretime: Expiration time in seconds. Set to ``None`` for never expires. This timestamp is compared to the creation_time result and ``time.time()`` to determine if the value returned by value_and_created_fn is "expired". :param async_creator: A callable. If specified, this callable will be passed the mutex as an argument and is responsible for releasing the mutex after it finishes some asynchronous value creation. The intent is for this to be used to defer invocation of the creator callable until some later time. .. versionadded:: 0.4.1 added the async_creator argument. """ def __init__(self, mutex, creator, value_and_created_fn, expiretime, async_creator=None, ): self.mutex = mutex self.creator = creator self.value_and_created_fn = value_and_created_fn self.expiretime = expiretime self.async_creator = async_creator def _is_expired(self, createdtime): """Return true if the expiration time is reached, or no value is available.""" return not self._has_value(createdtime) or \ ( self.expiretime is not None and time.time() - createdtime > self.expiretime ) def _has_value(self, createdtime): """Return true if the creation function has proceeded at least once.""" return createdtime > 0 def _enter(self): value_fn = self.value_and_created_fn try: value = value_fn() value, createdtime = value except NeedRegenerationException: log.debug("NeedRegenerationException") value = NOT_REGENERATED createdtime = -1 generated = self._enter_create(createdtime) if generated is not NOT_REGENERATED: generated, createdtime = generated return generated elif value is NOT_REGENERATED: try: value, createdtime = value_fn() return value except NeedRegenerationException: raise Exception("Generation function should " "have just been called by a concurrent " "thread.") else: return value def _enter_create(self, createdtime): if not self._is_expired(createdtime): return NOT_REGENERATED async = False if self._has_value(createdtime): if not self.mutex.acquire(False): log.debug("creation function in progress " "elsewhere, returning") return NOT_REGENERATED else: log.debug("no value, waiting for create lock") self.mutex.acquire() try: log.debug("value creation lock %r acquired" % self.mutex) # see if someone created the value already try: value, createdtime = self.value_and_created_fn() except NeedRegenerationException: pass else: if not self._is_expired(createdtime): log.debug("value already present") return value, createdtime elif self.async_creator: log.debug("Passing creation lock to async runner") self.async_creator(self.mutex) async = True return value, createdtime log.debug("Calling creation function") created = self.creator() return created finally: if not async: self.mutex.release() log.debug("Released creation lock") def __enter__(self): return self._enter() def __exit__(self, type, value, traceback): pass python-dogpile.core-0.4.1+dfsg1/dogpile/core/legacy.py0000644000175000017500000001152212076551651021311 0ustar zigozigofrom __future__ import with_statement from .util import threading from .readwrite_lock import ReadWriteMutex from .dogpile import Lock import time import contextlib class Dogpile(object): """Dogpile lock class. .. deprecated:: 0.4.0 The :class:`.Lock` object specifies the full API of the :class:`.Dogpile` object in a single way, rather than providing multiple modes of usage which don't necessarily work in the majority of cases. :class:`.Dogpile` is now a wrapper around the :class:`.Lock` object which provides dogpile.core's original usage pattern. This usage pattern began as something simple, but was not of general use in real-world caching environments without several extra complicating factors; the :class:`.Lock` object presents the "real-world" API more succinctly, and also fixes a cross-process concurrency issue. :param expiretime: Expiration time in seconds. Set to ``None`` for never expires. :param init: if True, set the 'createdtime' to the current time. :param lock: a mutex object that provides ``acquire()`` and ``release()`` methods. """ def __init__(self, expiretime, init=False, lock=None): """Construct a new :class:`.Dogpile`. """ if lock: self.dogpilelock = lock else: self.dogpilelock = threading.Lock() self.expiretime = expiretime if init: self.createdtime = time.time() createdtime = -1 """The last known 'creation time' of the value, stored as an epoch (i.e. from ``time.time()``). If the value here is -1, it is assumed the value should recreate immediately. """ def acquire(self, creator, value_fn=None, value_and_created_fn=None): """Acquire the lock, returning a context manager. :param creator: Creation function, used if this thread is chosen to create a new value. :param value_fn: Optional function that returns the value from some datasource. Will be returned if regeneration is not needed. :param value_and_created_fn: Like value_fn, but returns a tuple of (value, createdtime). The returned createdtime will replace the "createdtime" value on this dogpile lock. This option removes the need for the dogpile lock itself to remain persistent across usages; another dogpile can come along later and pick up where the previous one left off. """ if value_and_created_fn is None: if value_fn is None: def value_and_created_fn(): return None, self.createdtime else: def value_and_created_fn(): return value_fn(), self.createdtime def creator_wrapper(): value = creator() self.createdtime = time.time() return value, self.createdtime else: def creator_wrapper(): value = creator() self.createdtime = time.time() return value return Lock( self.dogpilelock, creator_wrapper, value_and_created_fn, self.expiretime ) @property def is_expired(self): """Return true if the expiration time is reached, or no value is available.""" return not self.has_value or \ ( self.expiretime is not None and time.time() - self.createdtime > self.expiretime ) @property def has_value(self): """Return true if the creation function has proceeded at least once.""" return self.createdtime > 0 class SyncReaderDogpile(Dogpile): """Provide a read-write lock function on top of the :class:`.Dogpile` class. .. deprecated:: 0.4.0 The :class:`.ReadWriteMutex` object can be used directly. """ def __init__(self, *args, **kw): super(SyncReaderDogpile, self).__init__(*args, **kw) self.readwritelock = ReadWriteMutex() @contextlib.contextmanager def acquire_write_lock(self): """Return the "write" lock context manager. This will provide a section that is mutexed against all readers/writers for the dogpile-maintained value. """ self.readwritelock.acquire_write_lock() try: yield finally: self.readwritelock.release_write_lock() @contextlib.contextmanager def acquire(self, *arg, **kw): with super(SyncReaderDogpile, self).acquire(*arg, **kw) as value: self.readwritelock.acquire_read_lock() try: yield value finally: self.readwritelock.release_read_lock() python-dogpile.core-0.4.1+dfsg1/dogpile/core/__init__.py0000644000175000017500000000052212076551651021602 0ustar zigozigofrom .dogpile import NeedRegenerationException, Lock from .nameregistry import NameRegistry from .readwrite_lock import ReadWriteMutex from .legacy import Dogpile, SyncReaderDogpile __all__ = [ 'Dogpile', 'SyncReaderDogpile', 'NeedRegenerationException', 'NameRegistry', 'ReadWriteMutex', 'Lock'] __version__ = '0.4.1' python-dogpile.core-0.4.1+dfsg1/dogpile/core/nameregistry.py0000644000175000017500000000505412076551651022561 0ustar zigozigofrom .util import threading import weakref class NameRegistry(object): """Generates and return an object, keeping it as a singleton for a certain identifier for as long as its strongly referenced. e.g.:: class MyFoo(object): "some important object." def __init__(self, identifier): self.identifier = identifier registry = NameRegistry(MyFoo) # thread 1: my_foo = registry.get("foo1") # thread 2 my_foo = registry.get("foo1") Above, ``my_foo`` in both thread #1 and #2 will be *the same object*. The constructor for ``MyFoo`` will be called once, passing the identifier ``foo1`` as the argument. When thread 1 and thread 2 both complete or otherwise delete references to ``my_foo``, the object is *removed* from the :class:`.NameRegistry` as a result of Python garbage collection. :param creator: A function that will create a new value, given the identifier passed to the :meth:`.NameRegistry.get` method. """ _locks = weakref.WeakValueDictionary() _mutex = threading.RLock() def __init__(self, creator): """Create a new :class:`.NameRegistry`. """ self._values = weakref.WeakValueDictionary() self._mutex = threading.RLock() self.creator = creator def get(self, identifier, *args, **kw): """Get and possibly create the value. :param identifier: Hash key for the value. If the creation function is called, this identifier will also be passed to the creation function. :param \*args, \**kw: Additional arguments which will also be passed to the creation function if it is called. """ try: if identifier in self._values: return self._values[identifier] else: return self._sync_get(identifier, *args, **kw) except KeyError: return self._sync_get(identifier, *args, **kw) def _sync_get(self, identifier, *args, **kw): self._mutex.acquire() try: try: if identifier in self._values: return self._values[identifier] else: self._values[identifier] = value = self.creator(identifier, *args, **kw) return value except KeyError: self._values[identifier] = value = self.creator(identifier, *args, **kw) return value finally: self._mutex.release() python-dogpile.core-0.4.1+dfsg1/dogpile/core/readwrite_lock.py0000644000175000017500000001062412076551651023045 0ustar zigozigofrom .util import threading import logging log = logging.getLogger(__name__) class LockError(Exception): pass class ReadWriteMutex(object): """A mutex which allows multiple readers, single writer. :class:`.ReadWriteMutex` uses a Python ``threading.Condition`` to provide this functionality across threads within a process. The Beaker package also contained a file-lock based version of this concept, so that readers/writers could be synchronized across processes with a common filesystem. A future Dogpile release may include this additional class at some point. """ def __init__(self): # counts how many asynchronous methods are executing self.async = 0 # pointer to thread that is the current sync operation self.current_sync_operation = None # condition object to lock on self.condition = threading.Condition(threading.Lock()) def acquire_read_lock(self, wait = True): """Acquire the 'read' lock.""" self.condition.acquire() try: # see if a synchronous operation is waiting to start # or is already running, in which case we wait (or just # give up and return) if wait: while self.current_sync_operation is not None: self.condition.wait() else: if self.current_sync_operation is not None: return False self.async += 1 log.debug("%s acquired read lock", self) finally: self.condition.release() if not wait: return True def release_read_lock(self): """Release the 'read' lock.""" self.condition.acquire() try: self.async -= 1 # check if we are the last asynchronous reader thread # out the door. if self.async == 0: # yes. so if a sync operation is waiting, notifyAll to wake # it up if self.current_sync_operation is not None: self.condition.notifyAll() elif self.async < 0: raise LockError("Synchronizer error - too many " "release_read_locks called") log.debug("%s released read lock", self) finally: self.condition.release() def acquire_write_lock(self, wait = True): """Acquire the 'write' lock.""" self.condition.acquire() try: # here, we are not a synchronous reader, and after returning, # assuming waiting or immediate availability, we will be. if wait: # if another sync is working, wait while self.current_sync_operation is not None: self.condition.wait() else: # if another sync is working, # we dont want to wait, so forget it if self.current_sync_operation is not None: return False # establish ourselves as the current sync # this indicates to other read/write operations # that they should wait until this is None again self.current_sync_operation = threading.currentThread() # now wait again for asyncs to finish if self.async > 0: if wait: # wait self.condition.wait() else: # we dont want to wait, so forget it self.current_sync_operation = None return False log.debug("%s acquired write lock", self) finally: self.condition.release() if not wait: return True def release_write_lock(self): """Release the 'write' lock.""" self.condition.acquire() try: if self.current_sync_operation is not threading.currentThread(): raise LockError("Synchronizer error - current thread doesn't " "have the write lock") # reset the current sync operation so # another can get it self.current_sync_operation = None # tell everyone to get ready self.condition.notifyAll() log.debug("%s released write lock", self) finally: # everyone go !! self.condition.release() python-dogpile.core-0.4.1+dfsg1/dogpile/core/util.py0000644000175000017500000000020512076551651021016 0ustar zigozigoimport sys py3k = sys.version_info >= (3, 0) try: import threading except ImportError: import dummy_threading as threading python-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/0000755000175000017500000000000012076552100021151 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/SOURCES.txt0000644000175000017500000000300512076552100023033 0ustar zigozigoLICENSE MANIFEST.in README.rst nose_logging_config.ini setup.cfg setup.py docs/api.html docs/changelog.html docs/front.html docs/genindex.html docs/index.html docs/search.html docs/searchindex.js docs/usage.html docs/_sources/api.txt docs/_sources/changelog.txt docs/_sources/front.txt docs/_sources/index.txt docs/_sources/usage.txt docs/_static/basic.css docs/_static/comment-bright.png docs/_static/comment-close.png docs/_static/comment.png docs/_static/doctools.js docs/_static/down-pressed.png docs/_static/down.png docs/_static/file.png docs/_static/jquery.js docs/_static/minus.png docs/_static/nature.css docs/_static/plus.png docs/_static/pygments.css docs/_static/searchtools.js docs/_static/underscore.js docs/_static/up-pressed.png docs/_static/up.png docs/_static/websupport.js docs/build/Makefile docs/build/api.rst docs/build/builder.py docs/build/changelog.rst docs/build/conf.py docs/build/front.rst docs/build/index.rst docs/build/requirements.txt docs/build/usage.rst dogpile/__init__.py dogpile.core.egg-info/PKG-INFO dogpile.core.egg-info/SOURCES.txt dogpile.core.egg-info/dependency_links.txt dogpile.core.egg-info/namespace_packages.txt dogpile.core.egg-info/not-zip-safe dogpile.core.egg-info/top_level.txt dogpile/core/__init__.py dogpile/core/dogpile.py dogpile/core/legacy.py dogpile/core/nameregistry.py dogpile/core/readwrite_lock.py dogpile/core/util.py tests/__init__.py tests/core/__init__.py tests/core/test_backgrounding.py tests/core/test_dogpile.py tests/core/test_lock.py tests/core/test_nameregistry.pypython-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/PKG-INFO0000644000175000017500000000646112076552100022255 0ustar zigozigoMetadata-Version: 1.0 Name: dogpile.core Version: 0.4.1 Summary: A 'dogpile' lock, typically used as a component of a larger caching solution Home-page: http://bitbucket.org/zzzeek/dogpile.core Author: Mike Bayer Author-email: mike_mp@zzzcomputing.com License: BSD Description: dogpile.core ============ A "dogpile" lock, one which allows a single thread to generate an expensive resource while other threads use the "old" value, until the "new" value is ready. Dogpile is basically the locking code extracted from the Beaker package, for simple and generic usage. Usage ----- A simple example:: from dogpile.core import Dogpile # store a reference to a "resource", some # object that is expensive to create. the_resource = [None] def some_creation_function(): # create the resource here the_resource[0] = create_some_resource() def use_the_resource(): # some function that uses # the resource. Won't reach # here until some_creation_function() # has completed at least once. the_resource[0].do_something() # create Dogpile with 3600 second # expiry time dogpile = Dogpile(3600) with dogpile.acquire(some_creation_function): use_the_resource() Above, ``some_creation_function()`` will be called when ``Dogpile.acquire()`` is first called. The remainder of the ``with`` block then proceeds. Concurrent threads which call ``Dogpile.acquire()`` during this initial period will be blocked until ``some_creation_function()`` completes. Once the creation function has completed successfully the first time, new calls to ``Dogpile.acquire()`` will call ``some_creation_function()`` each time the "expiretime" has been reached, allowing only a single thread to call the function. Concurrent threads which call ``Dogpile.acquire()`` during this period will fall through, and not be blocked. It is expected that the "stale" version of the resource remain available at this time while the new one is generated. dogpile.core is at the core of the `dogpile.cache `_ package which provides for a basic cache API and sample backends based on the dogpile concept. Development Status ------------------- dogpile.core has been in use in a small number of production environments for a period of months, and as of 0.3.2 has entered beta status. No issues have been reported yet with its core synchronization model, and overall the project hasn't seen many changes. Most development continues within dogpile.cache. Keywords: caching Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 python-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/namespace_packages.txt0000644000175000017500000000001012076552100025473 0ustar zigozigodogpile python-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/not-zip-safe0000644000175000017500000000000112076551774023417 0ustar zigozigo python-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/top_level.txt0000644000175000017500000000001012076552100023672 0ustar zigozigodogpile python-dogpile.core-0.4.1+dfsg1/dogpile.core.egg-info/dependency_links.txt0000644000175000017500000000000112076552100025217 0ustar zigozigo python-dogpile.core-0.4.1+dfsg1/setup.py0000644000175000017500000000217312076551651016634 0ustar zigozigoimport os import sys import re from setuptools import setup, find_packages v = open(os.path.join(os.path.dirname(__file__), 'dogpile', 'core', '__init__.py')) VERSION = re.compile(r".*__version__ = '(.*?)'", re.S).match(v.read()).group(1) v.close() readme = os.path.join(os.path.dirname(__file__), 'README.rst') setup(name='dogpile.core', version=VERSION, description="A 'dogpile' lock, typically used as a component of a larger caching solution", long_description=open(readme).read(), classifiers=[ 'Development Status :: 4 - Beta', 'Intended Audience :: Developers', 'License :: OSI Approved :: BSD License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', ], keywords='caching', author='Mike Bayer', author_email='mike_mp@zzzcomputing.com', url='http://bitbucket.org/zzzeek/dogpile.core', license='BSD', packages=find_packages('.', exclude=['ez_setup', 'tests*']), namespace_packages=['dogpile'], zip_safe=False, install_requires=[], test_suite='nose.collector', tests_require=['nose'], ) python-dogpile.core-0.4.1+dfsg1/tests/0000755000175000017500000000000012076552100016247 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/tests/__init__.py0000644000175000017500000000000012076551651020360 0ustar zigozigopython-dogpile.core-0.4.1+dfsg1/tests/core/0000755000175000017500000000000012076552100017177 5ustar zigozigopython-dogpile.core-0.4.1+dfsg1/tests/core/__init__.py0000644000175000017500000000000012076551651021310 0ustar zigozigopython-dogpile.core-0.4.1+dfsg1/tests/core/test_lock.py0000644000175000017500000001736712076551651021570 0ustar zigozigofrom unittest import TestCase import time import threading from dogpile.core import Lock, NeedRegenerationException from dogpile.core.nameregistry import NameRegistry import contextlib import math import logging log = logging.getLogger(__name__) class ConcurrencyTest(TestCase): # expiretime, time to create, num usages, time spend using, delay btw usage _assertion_lock = threading.Lock() def test_quick(self): self._test_multi( 10, 2, .5, 50, .05, .1, ) def test_slow(self): self._test_multi( 10, 5, 2, 50, .1, .1, ) def test_return_while_in_progress(self): self._test_multi( 10, 5, 2, 50, 1, .1 ) def test_get_value_plus_created_long_create(self): self._test_multi( 10, 2, 2.5, 50, .05, .1, ) def test_get_value_plus_created_registry_unsafe_cache(self): self._test_multi( 10, 1, .6, 100, .05, .1, cache_expire_time='unsafe' ) def test_get_value_plus_created_registry_safe_cache_quick(self): self._test_multi( 10, 2, .5, 50, .05, .1, cache_expire_time='safe' ) def test_get_value_plus_created_registry_safe_cache_slow(self): self._test_multi( 10, 5, 2, 50, .1, .1, cache_expire_time='safe' ) def _assert_synchronized(self): acq = self._assertion_lock.acquire(False) assert acq, "Could not acquire" @contextlib.contextmanager def go(): try: yield {} except: raise finally: self._assertion_lock.release() return go() def _assert_log(self, cond, msg, *args): if cond: log.debug(msg, *args) else: log.error("Assertion failed: " + msg, *args) assert False, msg % args def _test_multi(self, num_threads, expiretime, creation_time, num_usages, usage_time, delay_time, cache_expire_time=None): mutex = threading.Lock() unsafe_cache = False if cache_expire_time: if cache_expire_time == 'unsafe': unsafe_cache = True cache_expire_time = expiretime * .8 elif cache_expire_time == 'safe': cache_expire_time = (expiretime + creation_time) * 1.1 else: assert False, cache_expire_time log.info("Cache expire time: %s", cache_expire_time) effective_expiretime = min(cache_expire_time, expiretime) else: effective_expiretime = expiretime effective_creation_time = creation_time max_stale = (effective_expiretime + effective_creation_time + usage_time + delay_time) * 1.1 the_resource = [] slow_waiters = [0] failures = [0] def create_resource(): with self._assert_synchronized(): log.debug("creating resource, will take %f sec" % creation_time) time.sleep(creation_time) the_resource.append(time.time()) value = the_resource[-1] log.debug("finished creating resource") return value, time.time() def get_value(): if not the_resource: raise NeedRegenerationException() if cache_expire_time: if time.time() - the_resource[-1] > cache_expire_time: # should never hit a cache invalidation # if we've set expiretime below the cache # expire time (assuming a cache which # honors this). self._assert_log( cache_expire_time < expiretime, "Cache expiration hit, cache " "expire time %s, expiretime %s", cache_expire_time, expiretime, ) raise NeedRegenerationException() return the_resource[-1], the_resource[-1] def use_dogpile(): try: for i in range(num_usages): now = time.time() with Lock(mutex, create_resource, get_value, expiretime) as value: waited = time.time() - now if waited > .01: slow_waiters[0] += 1 check_value(value, waited) time.sleep(usage_time) time.sleep(delay_time) except: log.error("thread failed", exc_info=True) failures[0] += 1 def check_value(value, waited): assert value # time since the current resource was # created time_since_create = time.time() - value self._assert_log( time_since_create < max_stale, "Time since create %.4f max stale time %s, " "total waited %s", time_since_create, max_stale, slow_waiters[0] ) started_at = time.time() threads = [] for i in range(num_threads): t = threading.Thread(target=use_dogpile) t.start() threads.append(t) for t in threads: t.join() actual_run_time = time.time() - started_at # time spent starts with num usages * time per usage, with a 10% fudge expected_run_time = (num_usages * (usage_time + delay_time)) * 1.1 expected_generations = math.ceil(expected_run_time / effective_expiretime) if unsafe_cache: expected_slow_waiters = expected_generations * num_threads else: expected_slow_waiters = expected_generations + num_threads - 1 # time spent also increments by one wait period in the beginning... expected_run_time += effective_creation_time # and a fudged version of the periodic waiting time anticipated # for a single thread... expected_run_time += (expected_slow_waiters * effective_creation_time) / num_threads expected_run_time *= 1.1 log.info("Test Summary") log.info("num threads: %s; expiretime: %s; creation_time: %s; " "num_usages: %s; " "usage_time: %s; delay_time: %s", num_threads, expiretime, creation_time, num_usages, usage_time, delay_time ) log.info("cache expire time: %s; unsafe cache: %s", cache_expire_time, unsafe_cache) log.info("Estimated run time %.2f actual run time %.2f", expected_run_time, actual_run_time) log.info("Effective expiretime (min(cache_exp_time, exptime)) %s", effective_expiretime) log.info("Expected slow waits %s, Total slow waits %s", expected_slow_waiters, slow_waiters[0]) log.info("Total generations %s Max generations expected %s" % ( len(the_resource), expected_generations )) assert not failures[0], "%s failures occurred" % failures[0] assert actual_run_time <= expected_run_time assert slow_waiters[0] <= expected_slow_waiters, \ "Number of slow waiters %s exceeds expected slow waiters %s" % ( slow_waiters[0], expected_slow_waiters ) assert len(the_resource) <= expected_generations,\ "Number of resource generations %d exceeded "\ "expected %d" % (len(the_resource), expected_generations) python-dogpile.core-0.4.1+dfsg1/tests/core/test_nameregistry.py0000644000175000017500000000306612076551651023340 0ustar zigozigofrom unittest import TestCase import time import threading from dogpile.core.nameregistry import NameRegistry import random import logging log = logging.getLogger(__name__) class NameRegistryTest(TestCase): def test_name_registry(self): success = [True] num_operations = [0] def create(identifier): log.debug("Creator running for id: " + identifier) return threading.Lock() registry = NameRegistry(create) baton = { "beans":False, "means":False, "please":False } def do_something(name): for iteration in range(20): name = list(baton)[random.randint(0, 2)] lock = registry.get(name) lock.acquire() try: if baton[name]: success[0] = False log.debug("Baton is already populated") break baton[name] = True try: time.sleep(random.random() * .01) finally: num_operations[0] += 1 baton[name] = False finally: lock.release() log.debug("thread completed operations") threads = [] for id_ in range(1, 20): t = threading.Thread(target=do_something, args=("somename",)) t.start() threads.append(t) for t in threads: t.join() assert success[0] python-dogpile.core-0.4.1+dfsg1/tests/core/test_backgrounding.py0000644000175000017500000000112112076551651023432 0ustar zigozigoimport unittest import threading import dogpile.core class TestAsyncRunner(unittest.TestCase): def test_async_release(self): self.called = False def runner(mutex): self.called = True mutex.release() mutex = threading.Lock() create = lambda: ("value", 1) get = lambda: ("value", 1) expiretime = 1 assert not self.called with dogpile.core.Lock(mutex, create, get, expiretime, runner) as l: assert self.called assert self.called if __name__ == '__main__': unittest.main() python-dogpile.core-0.4.1+dfsg1/tests/core/test_dogpile.py0000644000175000017500000003042012076551651022244 0ustar zigozigofrom unittest import TestCase import time import threading from dogpile.core import Dogpile, SyncReaderDogpile, NeedRegenerationException from dogpile.core.nameregistry import NameRegistry import contextlib import math import logging log = logging.getLogger(__name__) class ConcurrencyTest(TestCase): # expiretime, time to create, num usages, time spend using, delay btw usage timings = [ # quick one (2, .5, 50, .05, .1), # slow creation time (5, 2, 50, .1, .1), ] _assertion_lock = threading.Lock() def test_rudimental(self): for exp, crt, nu, ut, dt in self.timings: self._test_multi( 10, exp, crt, nu, ut, dt, ) def test_rudimental_slow_write(self): self._test_multi( 10, 2, .5, 50, .05, .1, slow_write_time=2 ) def test_return_while_in_progress(self): self._test_multi( 10, 5, 2, 50, 1, .1, inline_create='get_value' ) def test_rudimental_long_create(self): self._test_multi( 10, 2, 2.5, 50, .05, .1, ) def test_get_value_plus_created_slow_write(self): self._test_multi( 10, 2, .5, 50, .05, .1, inline_create='get_value_plus_created', slow_write_time=2 ) def test_get_value_plus_created_long_create(self): self._test_multi( 10, 2, 2.5, 50, .05, .1, inline_create='get_value_plus_created', ) def test_get_value_plus_created_registry_unsafe_cache(self): self._test_multi( 10, 1, .6, 100, .05, .1, inline_create='get_value_plus_created', cache_expire_time='unsafe' ) def test_get_value_plus_created_registry_safe_cache(self): for exp, crt, nu, ut, dt in self.timings: self._test_multi( 10, exp, crt, nu, ut, dt, inline_create='get_value_plus_created', cache_expire_time='safe' ) def _assert_synchronized(self): acq = self._assertion_lock.acquire(False) assert acq, "Could not acquire" @contextlib.contextmanager def go(): try: yield {} except: raise finally: self._assertion_lock.release() return go() def _assert_log(self, cond, msg, *args): if cond: log.debug(msg, *args) else: log.error("Assertion failed: " + msg, *args) assert False, msg % args def _test_multi(self, num_threads, expiretime, creation_time, num_usages, usage_time, delay_time, cache_expire_time=None, slow_write_time=None, inline_create='rudimental'): if slow_write_time: dogpile_cls = SyncReaderDogpile else: dogpile_cls = Dogpile # the registry feature should not be used # unless the value + created time func is used. use_registry = inline_create == 'get_value_plus_created' if use_registry: reg = NameRegistry(lambda key, exptime: dogpile_cls(exptime)) get_dogpile = lambda: reg.get("somekey", expiretime) else: dogpile = dogpile_cls(expiretime) get_dogpile = lambda: dogpile unsafe_cache = False if cache_expire_time: if cache_expire_time == 'unsafe': unsafe_cache = True cache_expire_time = expiretime *.8 elif cache_expire_time == 'safe': cache_expire_time = (expiretime + creation_time) * 1.1 else: assert False, cache_expire_time log.info("Cache expire time: %s", cache_expire_time) effective_expiretime = min(cache_expire_time, expiretime) else: effective_expiretime = expiretime effective_creation_time= creation_time if slow_write_time: effective_creation_time += slow_write_time max_stale = (effective_expiretime + effective_creation_time + usage_time + delay_time) * 1.1 the_resource = [] slow_waiters = [0] failures = [0] def create_impl(dogpile): log.debug("creating resource...") time.sleep(creation_time) if slow_write_time: with dogpile.acquire_write_lock(): saved = list(the_resource) # clear out the resource dict so that # usage threads hitting it will # raise the_resource[:] = [] time.sleep(slow_write_time) the_resource[:] = saved the_resource.append(time.time()) return the_resource[-1] if inline_create == 'get_value_plus_created': def create_resource(dogpile): with self._assert_synchronized(): value = create_impl(dogpile) return value, time.time() else: def create_resource(dogpile): with self._assert_synchronized(): return create_impl(dogpile) if cache_expire_time: def get_value(): if not the_resource: raise NeedRegenerationException() if time.time() - the_resource[-1] > cache_expire_time: # should never hit a cache invalidation # if we've set expiretime below the cache # expire time (assuming a cache which # honors this). self._assert_log( cache_expire_time < expiretime, "Cache expiration hit, cache " "expire time %s, expiretime %s", cache_expire_time, expiretime, ) raise NeedRegenerationException() if inline_create == 'get_value_plus_created': return the_resource[-1], the_resource[-1] else: return the_resource[-1] else: def get_value(): if not the_resource: raise NeedRegenerationException() if inline_create == 'get_value_plus_created': return the_resource[-1], the_resource[-1] else: return the_resource[-1] if inline_create == 'rudimental': assert not cache_expire_time @contextlib.contextmanager def enter_dogpile_block(dogpile): with dogpile.acquire(lambda: create_resource(dogpile)) as x: yield the_resource[-1] elif inline_create == 'get_value': @contextlib.contextmanager def enter_dogpile_block(dogpile): with dogpile.acquire( lambda: create_resource(dogpile), get_value ) as rec: yield rec elif inline_create == 'get_value_plus_created': @contextlib.contextmanager def enter_dogpile_block(dogpile): with dogpile.acquire( lambda: create_resource(dogpile), value_and_created_fn=get_value ) as rec: yield rec else: assert False, inline_create def use_dogpile(): try: for i in range(num_usages): dogpile = get_dogpile() now = time.time() with enter_dogpile_block(dogpile) as value: waited = time.time() - now if waited > .01: slow_waiters[0] += 1 check_value(value, waited) time.sleep(usage_time) time.sleep(delay_time) except: log.error("thread failed", exc_info=True) failures[0] += 1 def check_value(value, waited): assert value, repr(value) # time since the current resource was # created time_since_create = time.time() - value self._assert_log( time_since_create < max_stale, "Time since create %.4f max stale time %s, " "total waited %s", time_since_create, max_stale, slow_waiters[0] ) started_at = time.time() threads = [] for i in range(num_threads): t = threading.Thread(target=use_dogpile) t.start() threads.append(t) for t in threads: t.join() actual_run_time = time.time() - started_at # time spent starts with num usages * time per usage, with a 10% fudge expected_run_time = (num_usages * (usage_time + delay_time)) * 1.1 expected_generations = math.ceil(expected_run_time / effective_expiretime) if unsafe_cache: expected_slow_waiters = expected_generations * num_threads else: expected_slow_waiters = expected_generations + num_threads - 1 if slow_write_time: expected_slow_waiters = num_threads * expected_generations # time spent also increments by one wait period in the beginning... expected_run_time += effective_creation_time # and a fudged version of the periodic waiting time anticipated # for a single thread... expected_run_time += (expected_slow_waiters * effective_creation_time) / num_threads expected_run_time *= 1.1 log.info("Test Summary") log.info("num threads: %s; expiretime: %s; creation_time: %s; " "num_usages: %s; " "usage_time: %s; delay_time: %s", num_threads, expiretime, creation_time, num_usages, usage_time, delay_time ) log.info("cache expire time: %s; unsafe cache: %s slow " "write time: %s; inline: %s; registry: %s", cache_expire_time, unsafe_cache, slow_write_time, inline_create, use_registry) log.info("Estimated run time %.2f actual run time %.2f", expected_run_time, actual_run_time) log.info("Effective expiretime (min(cache_exp_time, exptime)) %s", effective_expiretime) log.info("Expected slow waits %s, Total slow waits %s", expected_slow_waiters, slow_waiters[0]) log.info("Total generations %s Max generations expected %s" % ( len(the_resource), expected_generations )) assert not failures[0], "%s failures occurred" % failures[0] assert actual_run_time <= expected_run_time assert slow_waiters[0] <= expected_slow_waiters, \ "Number of slow waiters %s exceeds expected slow waiters %s" % ( slow_waiters[0], expected_slow_waiters ) assert len(the_resource) <= expected_generations,\ "Number of resource generations %d exceeded "\ "expected %d" % (len(the_resource), expected_generations) class DogpileTest(TestCase): def test_single_create(self): dogpile = Dogpile(2) the_resource = [0] def create_resource(): the_resource[0] += 1 with dogpile.acquire(create_resource): assert the_resource[0] == 1 with dogpile.acquire(create_resource): assert the_resource[0] == 1 time.sleep(2) with dogpile.acquire(create_resource): assert the_resource[0] == 2 with dogpile.acquire(create_resource): assert the_resource[0] == 2 def test_no_expiration(self): dogpile = Dogpile(None) the_resource = [0] def create_resource(): the_resource[0] += 1 with dogpile.acquire(create_resource): assert the_resource[0] == 1 with dogpile.acquire(create_resource): assert the_resource[0] == 1 python-dogpile.core-0.4.1+dfsg1/MANIFEST.in0000644000175000017500000000035212076551651016655 0ustar zigozigorecursive-include docs *.html *.css *.txt *.js *.jpg *.png *.py Makefile *.rst *.sty recursive-include tests *.py *.dat include README* LICENSE distribute_setup.py CHANGES* test.cfg nose_logging_config.ini prune docs/build/output python-dogpile.core-0.4.1+dfsg1/LICENSE0000644000175000017500000000265012076551651016127 0ustar zigozigoCopyright (c) 2011-2013 Mike Bayer All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The name of the author or contributors may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. python-dogpile.core-0.4.1+dfsg1/PKG-INFO0000644000175000017500000000646112076552100016211 0ustar zigozigoMetadata-Version: 1.0 Name: dogpile.core Version: 0.4.1 Summary: A 'dogpile' lock, typically used as a component of a larger caching solution Home-page: http://bitbucket.org/zzzeek/dogpile.core Author: Mike Bayer Author-email: mike_mp@zzzcomputing.com License: BSD Description: dogpile.core ============ A "dogpile" lock, one which allows a single thread to generate an expensive resource while other threads use the "old" value, until the "new" value is ready. Dogpile is basically the locking code extracted from the Beaker package, for simple and generic usage. Usage ----- A simple example:: from dogpile.core import Dogpile # store a reference to a "resource", some # object that is expensive to create. the_resource = [None] def some_creation_function(): # create the resource here the_resource[0] = create_some_resource() def use_the_resource(): # some function that uses # the resource. Won't reach # here until some_creation_function() # has completed at least once. the_resource[0].do_something() # create Dogpile with 3600 second # expiry time dogpile = Dogpile(3600) with dogpile.acquire(some_creation_function): use_the_resource() Above, ``some_creation_function()`` will be called when ``Dogpile.acquire()`` is first called. The remainder of the ``with`` block then proceeds. Concurrent threads which call ``Dogpile.acquire()`` during this initial period will be blocked until ``some_creation_function()`` completes. Once the creation function has completed successfully the first time, new calls to ``Dogpile.acquire()`` will call ``some_creation_function()`` each time the "expiretime" has been reached, allowing only a single thread to call the function. Concurrent threads which call ``Dogpile.acquire()`` during this period will fall through, and not be blocked. It is expected that the "stale" version of the resource remain available at this time while the new one is generated. dogpile.core is at the core of the `dogpile.cache `_ package which provides for a basic cache API and sample backends based on the dogpile concept. Development Status ------------------- dogpile.core has been in use in a small number of production environments for a period of months, and as of 0.3.2 has entered beta status. No issues have been reported yet with its core synchronization model, and overall the project hasn't seen many changes. Most development continues within dogpile.cache. Keywords: caching Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: BSD License Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 3 python-dogpile.core-0.4.1+dfsg1/README.rst0000644000175000017500000000437112076551651016613 0ustar zigozigodogpile.core ============ A "dogpile" lock, one which allows a single thread to generate an expensive resource while other threads use the "old" value, until the "new" value is ready. Dogpile is basically the locking code extracted from the Beaker package, for simple and generic usage. Usage ----- A simple example:: from dogpile.core import Dogpile # store a reference to a "resource", some # object that is expensive to create. the_resource = [None] def some_creation_function(): # create the resource here the_resource[0] = create_some_resource() def use_the_resource(): # some function that uses # the resource. Won't reach # here until some_creation_function() # has completed at least once. the_resource[0].do_something() # create Dogpile with 3600 second # expiry time dogpile = Dogpile(3600) with dogpile.acquire(some_creation_function): use_the_resource() Above, ``some_creation_function()`` will be called when ``Dogpile.acquire()`` is first called. The remainder of the ``with`` block then proceeds. Concurrent threads which call ``Dogpile.acquire()`` during this initial period will be blocked until ``some_creation_function()`` completes. Once the creation function has completed successfully the first time, new calls to ``Dogpile.acquire()`` will call ``some_creation_function()`` each time the "expiretime" has been reached, allowing only a single thread to call the function. Concurrent threads which call ``Dogpile.acquire()`` during this period will fall through, and not be blocked. It is expected that the "stale" version of the resource remain available at this time while the new one is generated. dogpile.core is at the core of the `dogpile.cache `_ package which provides for a basic cache API and sample backends based on the dogpile concept. Development Status ------------------- dogpile.core has been in use in a small number of production environments for a period of months, and as of 0.3.2 has entered beta status. No issues have been reported yet with its core synchronization model, and overall the project hasn't seen many changes. Most development continues within dogpile.cache. python-dogpile.core-0.4.1+dfsg1/setup.cfg0000644000175000017500000000024412076552100016726 0ustar zigozigo[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 [upload_docs] upload-dir = docs/build/output/html [nosetests] logging-config = nose_logging_config.ini python-dogpile.core-0.4.1+dfsg1/nose_logging_config.ini0000644000175000017500000000102712076551651021617 0ustar zigozigo# nose specific logging [loggers] keys = root, dogpilecore, tests [handlers] keys = console [formatters] keys = generic [logger_root] level = CRITICAL handlers = console [logger_dogpilecore] level = DEBUG qualname = dogpile.core handlers = [logger_tests] level = DEBUG qualname = tests handlers = [handler_console] class = StreamHandler args = (sys.stderr,) level = NOTSET formatter = generic [formatter_generic] format = %(asctime)s,%(msecs)03d %(levelname)-5.5s [%(name)s] [%(thread)s] %(message)s datefmt = %Y-%m-%d %H:%M:%S