pax_global_header00006660000000000000000000000064140520403570014512gustar00rootroot0000000000000052 comment=4a8211158f8424ba621d70b70beba7f908e30671 aioprocessing-2.0.0/000077500000000000000000000000001405204035700143565ustar00rootroot00000000000000aioprocessing-2.0.0/.github/000077500000000000000000000000001405204035700157165ustar00rootroot00000000000000aioprocessing-2.0.0/.github/workflows/000077500000000000000000000000001405204035700177535ustar00rootroot00000000000000aioprocessing-2.0.0/.github/workflows/run_tests.yml000066400000000000000000000015341405204035700225270ustar00rootroot00000000000000name: aioprocessing tests on: pull_request: null push: branches: - master jobs: tests: runs-on: ubuntu-latest strategy: matrix: python: ['3.5', '3.6', '3.7', '3.8', '3.9'] name: aioprocessing ${{ matrix.python }} tests steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: ${{ matrix.python }} - name: Install Flake8 run: pip install flake8 - run: flake8 . - run: python runtests.py -v --failfast timeout-minutes: 1 # tests should also pass when using multiprocess (dill) - run: pip install multiprocess - run: python runtests.py -v --failfast timeout-minutes: 1 aioprocessing-2.0.0/.gitignore000066400000000000000000000000271405204035700163450ustar00rootroot00000000000000.idea *.pyc *.egg-infoaioprocessing-2.0.0/AUTHORS000066400000000000000000000000421405204035700154220ustar00rootroot00000000000000Dan O'Reilly (oreilldf@gmail.com) aioprocessing-2.0.0/LICENSE.txt000066400000000000000000000027571405204035700162140ustar00rootroot00000000000000Copyright (c) 2014-2020, Dan O'Reilly All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. The views and conclusions contained in the software and documentation are those of the authors and should not be interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. aioprocessing-2.0.0/MANIFEST.in000066400000000000000000000001261405204035700161130ustar00rootroot00000000000000include AUTHORS include LICENSE.txt include README.md include tests/*.py include *.py aioprocessing-2.0.0/README.md000066400000000000000000000140301405204035700156330ustar00rootroot00000000000000aioprocessing ============= [![Build Status](https://github.com/dano/aioprocessing/workflows/aioprocessing%20tests/badge.svg?branch=master)](https://github.com/dano/aioprocessing/actions) `aioprocessing` provides asynchronous, [`asyncio`](https://docs.python.org/3/library/asyncio.html) compatible, coroutine versions of many blocking instance methods on objects in the [`multiprocessing`](https://docs.python.org/3/library/multiprocessing.html) library. To use [`dill`](https://pypi.org/project/dill) for universal pickling, install using `pip install aioprocessing[dill]`. Here's an example demonstrating the `aioprocessing` versions of `Event`, `Queue`, and `Lock`: ```python import time import asyncio import aioprocessing def func(queue, event, lock, items): """ Demo worker function. This worker function runs in its own process, and uses normal blocking calls to aioprocessing objects, exactly the way you would use oridinary multiprocessing objects. """ with lock: event.set() for item in items: time.sleep(3) queue.put(item+5) queue.close() async def example(queue, event, lock): l = [1,2,3,4,5] p = aioprocessing.AioProcess(target=func, args=(queue, event, lock, l)) p.start() while True: result = await queue.coro_get() if result is None: break print("Got result {}".format(result)) await p.coro_join() async def example2(queue, event, lock): await event.coro_wait() async with lock: await queue.coro_put(78) await queue.coro_put(None) # Shut down the worker if __name__ == "__main__": loop = asyncio.get_event_loop() queue = aioprocessing.AioQueue() lock = aioprocessing.AioLock() event = aioprocessing.AioEvent() tasks = [ asyncio.ensure_future(example(queue, event, lock)), asyncio.ensure_future(example2(queue, event, lock)), ] loop.run_until_complete(asyncio.wait(tasks)) loop.close() ``` The aioprocessing objects can be used just like their multiprocessing equivalents - as they are in `func` above - but they can also be seamlessly used inside of `asyncio` coroutines, without ever blocking the event loop. What's new ---------- `v2.0.0` - Add support for universal pickling using [`dill`](https://github.com/uqfoundation/dill), installable with `pip install aioprocessing[dill]`. The library will now attempt to import [`multiprocess`](https://github.com/uqfoundation/multiprocess), falling back to stdlib `multiprocessing`. Force stdlib behaviour by setting a non-empty environment variable `AIOPROCESSING_DILL_DISABLED=1`. This can be used to avoid [errors](https://github.com/dano/aioprocessing/pull/36#discussion_r631178933) when attempting to combine `aioprocessing[dill]` with stdlib `multiprocessing` based objects like `concurrent.futures.ProcessPoolExecutor`. How does it work? ----------------- In most cases, this library makes blocking calls to `multiprocessing` methods asynchronous by executing the call in a [`ThreadPoolExecutor`](https://docs.python.org/3/library/concurrent.futures.html#threadpoolexecutor), using [`asyncio.run_in_executor()`](https://docs.python.org/3/library/asyncio-eventloop.html#asyncio.BaseEventLoop.run_in_executor). It does *not* re-implement multiprocessing using asynchronous I/O. This means there is extra overhead added when you use `aioprocessing` objects instead of `multiprocessing` objects, because each one is generally introducing a `ThreadPoolExecutor` containing at least one [`threading.Thread`](https://docs.python.org/2/library/threading.html#thread-objects). It also means that all the normal risks you get when you mix threads with fork apply here, too (See http://bugs.python.org/issue6721 for more info). The one exception to this is `aioprocessing.AioPool`, which makes use of the existing `callback` and `error_callback` keyword arguments in the various [`Pool.*_async`](https://docs.python.org/3/library/multiprocessing.html#multiprocessing.pool.Pool.apply_async) methods to run them as `asyncio` coroutines. Note that `multiprocessing.Pool` is actually using threads internally, so the thread/fork mixing caveat still applies. Each `multiprocessing` class is replaced by an equivalent `aioprocessing` class, distinguished by the `Aio` prefix. So, `Pool` becomes `AioPool`, etc. All methods that could block on I/O also have a coroutine version that can be used with `asyncio`. For example, `multiprocessing.Lock.acquire()` can be replaced with `aioprocessing.AioLock.coro_acquire()`. You can pass an `asyncio` EventLoop object to any `coro_*` method using the `loop` keyword argument. For example, `lock.coro_acquire(loop=my_loop)`. Note that you can also use the `aioprocessing` synchronization primitives as replacements for their equivalent `threading` primitives, in single-process, multi-threaded programs that use `asyncio`. What parts of multiprocessing are supported? -------------------------------------------- Most of them! All methods that could do blocking I/O in the following objects have equivalent versions in `aioprocessing` that extend the `multiprocessing` versions by adding coroutine versions of all the blocking methods. - `Pool` - `Process` - `Pipe` - `Lock` - `RLock` - `Semaphore` - `BoundedSemaphore` - `Event` - `Condition` - `Barrier` - `connection.Connection` - `connection.Listener` - `connection.Client` - `Queue` - `JoinableQueue` - `SimpleQueue` - All `managers.SyncManager` `Proxy` versions of the items above (`SyncManager.Queue`, `SyncManager.Lock()`, etc.). What versions of Python are compatible? --------------------------------------- `aioprocessing` will work out of the box on Python 3.5+. Gotchas ------- Keep in mind that, while the API exposes coroutines for interacting with `multiprocessing` APIs, internally they are almost always being delegated to a `ThreadPoolExecutor`, this means the caveats that apply with using `ThreadPoolExecutor` with `asyncio` apply: namely, you won't be able to cancel any of the coroutines, because the work being done in the worker thread can't be interrupted. aioprocessing-2.0.0/aioprocessing/000077500000000000000000000000001405204035700172235ustar00rootroot00000000000000aioprocessing-2.0.0/aioprocessing/.gitignore000066400000000000000000000000141405204035700212060ustar00rootroot00000000000000*.swp *.pyc aioprocessing-2.0.0/aioprocessing/__init__.py000066400000000000000000000134541405204035700213430ustar00rootroot00000000000000from . import mp as multiprocessing # noqa from .connection import * # noqa from .managers import * # noqa __all__ = [ "AioProcess", "AioManager", "AioPipe", "AioQueue", "AioSimpleQueue", "AioJoinableQueue", "AioLock", "AioRLock", "AioCondition", "AioPool", "AioSemaphore", "AioBoundedSemaphore", "AioEvent", "AioBarrier", ] # version is a human-readable version number. # version_info is a four-tuple for programmatic comparison. The first # three numbers are the components of the version number. The fourth # is zero for an official release, positive for a development branch, # or negative for a release candidate or beta (after the base version # number has been incremented) version = "2.0.0" version_info = (2, 0, 0, 0) if hasattr(multiprocessing, "get_context"): def _get_context(): return multiprocessing.get_context() has_context = True else: def _get_context(): return None has_context = False def AioProcess( group=None, target=None, name=None, args=(), kwargs=None, *, daemon=None, context=None ): """ Returns an asyncio-friendly version of multiprocessing.Process. Provides the following coroutines: coro_join() """ if kwargs is None: kwargs = {} context = context if context else _get_context() from .process import AioProcess return AioProcess( group=group, target=target, name=name, args=args, kwargs=kwargs, daemon=daemon, ctx=context, ) def AioPool( processes=None, initializer=None, initargs=(), maxtasksperchild=None, *, context=None ): """ Returns an asyncio-friendly version of multiprocessing.Pool. Provides the following coroutines: coro_join() coro_apply() coro_map() coro_starmap() """ context = context if context else _get_context() from .pool import AioPool return AioPool( processes=processes, initializer=initializer, initargs=initargs, maxtasksperchild=maxtasksperchild, ctx=context, ) def AioManager(*, context=None): """ Starts and returns an asyncio-friendly mp.SyncManager. Provides the follow asyncio-friendly objects: AioQueue AioBarrier AioBoundedSemaphore AioCondition AioEvent AioLock AioRLock AioSemaphore """ context = context if context else _get_context() from .managers import AioSyncManager # For Python 3.3 support, don't always pass ctx. kwargs = {"ctx": context} if has_context else {} m = AioSyncManager(**kwargs) m.start() return m def AioPipe(duplex=True): """ Returns a pair of AioConnection objects. """ from .connection import AioConnection conn1, conn2 = multiprocessing.Pipe(duplex=duplex) # Transform the returned connection instances into # instance of AioConnection. conn1 = AioConnection(conn1) conn2 = AioConnection(conn2) return conn1, conn2 # queues def AioQueue(maxsize=0, *, context=None): """ Returns an asyncio-friendly version of a multiprocessing.Queue Returns an AioQueue objects with the given context. If a context is not provided, the default for the platform will be used. """ context = context = context if context else _get_context() from .queues import AioQueue return AioQueue(maxsize, ctx=context) def AioJoinableQueue(maxsize=0, *, context=None): """ Returns an asyncio-friendly version of a multiprocessing.JoinableQueue Returns an AioJoinableQueue object with the given context. If a context is not provided, the default for the platform will be used. """ context = context = context if context else _get_context() from .queues import AioJoinableQueue return AioJoinableQueue(maxsize, ctx=context) def AioSimpleQueue(*, context=None): """ Returns an asyncio-friendly version of a multiprocessing.SimpleQueue Returns an AioSimpleQueue object with the given context. If a context is not provided, the default for the platform will be used. """ context = context = context if context else _get_context() from .queues import AioSimpleQueue return AioSimpleQueue(ctx=context) # locks def AioLock(*, context=None): """ Returns a non-recursive lock object. """ context = context = context if context else _get_context() from .locks import AioLock return AioLock(ctx=context) def AioRLock(*, context=None): """ Returns a recursive lock object. """ context = context = context if context else _get_context() from .locks import AioRLock return AioRLock(ctx=context) def AioCondition(lock=None, *, context=None): """ Returns a condition object. """ context = context = context if context else _get_context() from .locks import AioCondition return AioCondition(lock, ctx=context) def AioSemaphore(value=1, *, context=None): """ Returns a semaphore object. """ context = context = context if context else _get_context() from .locks import AioSemaphore return AioSemaphore(value, ctx=context) def AioBoundedSemaphore(value=1, *, context=None): """ Returns a bounded semaphore object. """ context = context = context if context else _get_context() from .locks import AioBoundedSemaphore return AioBoundedSemaphore(value, ctx=context) def AioEvent(*, context=None): """ Returns an event object. """ context = context = context if context else _get_context() from .locks import AioEvent return AioEvent(ctx=context) def AioBarrier(parties, action=None, timeout=None, *, context=None): """ Returns a barrier object. """ context = context = context if context else _get_context() from .locks import AioBarrier return AioBarrier(parties, action, timeout, ctx=context) aioprocessing-2.0.0/aioprocessing/connection.py000066400000000000000000000034201405204035700217330ustar00rootroot00000000000000from concurrent.futures import ThreadPoolExecutor from .mp import connection as _connection from .executor import CoroBuilder from .util import run_in_executor __all__ = ["AioConnection"] class AioConnection(metaclass=CoroBuilder): coroutines = [ "recv", "poll", "send_bytes", "recv_bytes", "recv_bytes_into", "send", ] def __init__(self, obj): """ Initialize the AioConnection. obj - a multiprocessing.Connection object. """ super().__init__() self._obj = obj def __enter__(self): self._obj.__enter__() return self def __exit__(self, *args, **kwargs): self._obj.__exit__(*args, **kwargs) def AioClient(*args, **kwargs): """ Returns an AioConnection instance. """ conn = _connection.Client(*args, **kwargs) return AioConnection(conn) class AioListener(metaclass=CoroBuilder): delegate = _connection.Listener coroutines = ["accept"] def accept(self): conn = self._obj.accept() return AioConnection(conn) def __enter__(self): self._obj.__enter__() return self def __exit__(self, *args, **kwargs): self._obj.__exit__(*args, **kwargs) def coro_deliver_challenge(*args, **kwargs): executor = ThreadPoolExecutor(max_workers=1) return run_in_executor( executor, _connection.deliver_challenge, *args, **kwargs ) def coro_answer_challenge(*args, **kwargs): executor = ThreadPoolExecutor(max_workers=1) return run_in_executor( executor, _connection.answer_challenge, *args, **kwargs ) def coro_wait(*args, **kwargs): executor = ThreadPoolExecutor(max_workers=1) return run_in_executor( executor, _connection.wait, *args, **kwargs ) aioprocessing-2.0.0/aioprocessing/executor.py000066400000000000000000000154471405204035700214460ustar00rootroot00000000000000from functools import wraps from concurrent.futures import ThreadPoolExecutor from . import util from .mp import cpu_count def init_executor(func): @wraps(func) def wrapper(self, *args, **kwargs): if not hasattr(self, "_executor"): self._executor = self._get_executor() return func(self, *args, **kwargs) return wrapper class _ExecutorMixin: """ A Mixin that provides asynchronous functionality. This mixin provides methods that allow a class to run blocking methods via asyncio in a ThreadPoolExecutor. It also provides methods that attempt to keep the object picklable despite having a non-picklable ThreadPoolExecutor as part of its state. """ pool_workers = cpu_count() @init_executor def run_in_executor(self, callback, *args, loop=None, **kwargs): """ Wraps run_in_executor so we can support kwargs. BaseEventLoop.run_in_executor does not support kwargs, so we wrap our callback in a lambda if kwargs are provided. """ return util.run_in_executor( self._executor, callback, *args, loop=loop, **kwargs ) @init_executor def run_in_thread(self, callback, *args, **kwargs): """ Runs a method in an executor thread. This is used when a method must be run in a thread (e.g. to that a lock is released in the same thread it was acquired), but should be run in a blocking way. """ fut = self._executor.submit(callback, *args, **kwargs) return fut.result() def _get_executor(self): return ThreadPoolExecutor(max_workers=self.pool_workers) def __getattr__(self, attr): assert attr != "_obj", ( "Make sure that your Class has a " '"delegate" assigned' ) if ( self._obj and hasattr(self._obj, attr) and not attr.startswith("__") ): return getattr(self._obj, attr) raise AttributeError(attr) def __getstate__(self): self_dict = self.__dict__.copy() self_dict["_executor"] = None return self_dict def __setstate__(self, state): self.__dict__.update(state) self._executor = self._get_executor() class CoroBuilder(type): """ Metaclass for adding coroutines to a class. This metaclass has two main roles: 1) Make _ExecutorMixin a parent of the given class 2) For every function name listed in the class attribute "coroutines", add a new instance method to the class called "coro_", which is a coroutine that calls func_name in a ThreadPoolExecutor. Each wrapper class that uses this metaclass can define three class attributes that will influence the behavior of the metaclass: coroutines - A list of methods that should get coroutine versions in the wrapper. For example: coroutines = ['acquire', 'wait'] Will mean the class gets coro_acquire and coro_wait methods. delegate - The class object that is being wrapped. This object will be instantiated when the wrapper class is instantiated, and will be set to the `_obj` attribute of the instance. pool_workers - The number of workers in the ThreadPoolExecutor internally used by the wrapper class. This defaults to cpu_count(), but for classes that need to acquire locks, it should always be set to 1. """ def __new__(cls, clsname, bases, dct, **kwargs): coro_list = dct.get("coroutines", []) existing_coros = set() def find_existing_coros(d): for attr in d: if attr.startswith("coro_") or attr.startswith("thread_"): existing_coros.add(attr) # Determine if any bases include the coroutines attribute, or # if either this class or a base class provides an actual # implementation for a coroutine method. find_existing_coros(dct) for b in bases: b_dct = b.__dict__ coro_list.extend(b_dct.get("coroutines", [])) find_existing_coros(b_dct) # Add _ExecutorMixin to bases. if _ExecutorMixin not in bases: bases += (_ExecutorMixin,) # Add coro funcs to dct, but only if a definition # is not already provided by dct or one of our bases. for func in coro_list: coro_name = "coro_{}".format(func) if coro_name not in existing_coros: dct[coro_name] = cls.coro_maker(func) return super().__new__(cls, clsname, bases, dct) def __init__(cls, name, bases, dct): """ Properly initialize a coroutine wrapper class. Sets pool_workers and delegate on the class, and also adds an __init__ method to it that instantiates the delegate with the proper context. """ super().__init__(name, bases, dct) pool_workers = dct.get("pool_workers") delegate = dct.get("delegate") old_init = dct.get("__init__") # Search bases for values we care about, if we didn't # find them on the current class. for b in bases: b_dct = b.__dict__ if not pool_workers: pool_workers = b_dct.get("pool_workers") if not delegate: delegate = b_dct.get("delegate") if not old_init: old_init = b_dct.get("__init__") cls.delegate = delegate # If we found a value for pool_workers, set it. If not, # ExecutorMixin sets a default that will be used. if pool_workers: cls.pool_workers = pool_workers # Here's the __init__ we want every wrapper class to use. # It just instantiates the delegate mp object using the # correct context. @wraps(old_init) def init_func(self, *args, **kwargs): # Be sure to call the original __init__, if there # was one. if old_init: old_init(self, *args, **kwargs) # If we're wrapping a mp object, instantiate it here. # If a context was specified, we instaniate the mp class # using that context. Otherwise, we'll just use the default # context. if cls.delegate: ctx = kwargs.pop("ctx", None) if ctx: clz = getattr(ctx, cls.delegate.__name__) else: clz = cls.delegate self._obj = clz(*args, **kwargs) cls.__init__ = init_func @staticmethod def coro_maker(func): def coro_func(self, *args, loop=None, **kwargs): return self.run_in_executor( getattr(self, func), *args, loop=loop, **kwargs ) return coro_func aioprocessing-2.0.0/aioprocessing/locks.py000066400000000000000000000065731405204035700207230ustar00rootroot00000000000000from .executor import CoroBuilder from .mp import ( Event, Lock, RLock, BoundedSemaphore, Condition, Semaphore, Barrier, util as _util, ) __all__ = [ "AioLock", "AioRLock", "AioBarrier", "AioCondition", "AioEvent", "AioSemaphore", "AioBoundedSemaphore", ] class _ContextManager: """ Context manager. This enables the following idiom for acquiring and releasing a lock around a block: async with lock: """ def __init__(self, lock): self._lock = lock def __enter__(self): # We have no use for the "as ..." clause in the with # statement for locks. return None def __exit__(self, *args): try: self._lock.release() finally: self._lock = None # Crudely prevent reuse. class AioBaseLock(metaclass=CoroBuilder): pool_workers = 1 coroutines = ["acquire", "release"] def __init__(self, *args, **kwargs): self._threaded_acquire = False def _after_fork(obj): obj._threaded_acquire = False _util.register_after_fork(self, _after_fork) def coro_acquire(self, *args, **kwargs): """ Non-blocking acquire. We need a custom implementation here, because we need to set the _threaded_acquire attribute to True once we have the lock. This attribute is used by release() to determine whether the lock should be released in the main thread, or in the Executor thread. """ def lock_acquired(fut): if fut.result(): self._threaded_acquire = True out = self.run_in_executor(self._obj.acquire, *args, **kwargs) out.add_done_callback(lock_acquired) return out def __getstate__(self): state = super().__getstate__() state["_threaded_acquire"] = False return state def __setstate__(self, state): super().__setstate__(state) def release(self): """ Release the lock. If the lock was acquired in the same process via coro_acquire, we need to release the lock in the ThreadPoolExecutor's thread. """ if self._threaded_acquire: out = self.run_in_thread(self._obj.release) else: out = self._obj.release() self._threaded_acquire = False return out async def __aenter__(self): await self.coro_acquire() return None async def __aexit__(self, *args, **kwargs): self.release() def __enter__(self): return self._obj.__enter__() def __exit__(self, *args, **kwargs): return self._obj.__exit__(*args, **kwargs) async def __aiter__(self): await self.coro_acquire() return _ContextManager(self) class AioBaseWaiter(metaclass=CoroBuilder): pool_workers = 1 coroutines = ["wait"] class AioBarrier(AioBaseWaiter): delegate = Barrier pass class AioCondition(AioBaseLock, AioBaseWaiter): delegate = Condition pool_workers = 1 coroutines = ["wait_for", "notify", "notify_all"] class AioEvent(AioBaseWaiter): delegate = Event class AioLock(AioBaseLock): delegate = Lock class AioRLock(AioBaseLock): delegate = RLock class AioSemaphore(AioBaseLock): delegate = Semaphore class AioBoundedSemaphore(AioBaseLock): delegate = BoundedSemaphore aioprocessing-2.0.0/aioprocessing/managers.py000077500000000000000000000121141405204035700213740ustar00rootroot00000000000000import asyncio from multiprocessing.util import register_after_fork from queue import Queue from threading import ( Barrier, BoundedSemaphore, Condition, Event, Lock, RLock, Semaphore, ) from aioprocessing.locks import _ContextManager from .executor import _ExecutorMixin from .mp import managers as _managers AioBaseQueueProxy = _managers.MakeProxyType( "AioQueueProxy", ( "task_done", "get", "qsize", "put", "put_nowait", "get_nowait", "empty", "join", "_qsize", "full", ), ) class _AioProxyMixin(_ExecutorMixin): _obj = None def _async_call(self, method, *args, loop=None, **kwargs): return asyncio.ensure_future( self.run_in_executor( self._callmethod, method, args, kwargs, loop=loop ) ) class ProxyCoroBuilder(type): """ Build coroutines to proxy functions. """ def __new__(cls, clsname, bases, dct): coro_list = dct.get("coroutines", []) existing_coros = set() def find_existing_coros(d): for attr in d: if attr.startswith("coro_") or attr.startswith("thread_"): existing_coros.add(attr) # Determine if any bases include the coroutines attribute, or # if either this class or a base class provides an actual # implementation for a coroutine method. find_existing_coros(dct) for b in bases: b_dct = b.__dict__ coro_list.extend(b_dct.get("coroutines", [])) find_existing_coros(b_dct) bases += (_AioProxyMixin,) for func in coro_list: coro_name = "coro_{}".format(func) if coro_name not in existing_coros: dct[coro_name] = cls.coro_maker(func) return super().__new__(cls, clsname, bases, dct) @staticmethod def coro_maker(func): def coro_func(self, *args, loop=None, **kwargs): return self._async_call(func, *args, loop=loop, **kwargs) return coro_func class AioQueueProxy(AioBaseQueueProxy, metaclass=ProxyCoroBuilder): """ A Proxy object for AioQueue. Provides coroutines for calling 'get' and 'put' on the proxy. """ coroutines = ["get", "put"] class AioAcquirerProxy(_managers.AcquirerProxy, metaclass=ProxyCoroBuilder): pool_workers = 1 coroutines = ["acquire", "release"] def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._threaded_acquire = False def _after_fork(obj): obj._threaded_acquire = False register_after_fork(self, _after_fork) def coro_acquire(self, *args, **kwargs): """ Non-blocking acquire. We need a custom implementation here, because we need to set the _threaded_acquire attribute to True once we have the lock. This attribute is used by release() to determine whether the lock should be released in the main thread, or in the Executor thread. """ def lock_acquired(fut): if fut.result(): self._threaded_acquire = True out = self.run_in_executor(self.acquire, *args, **kwargs) out.add_done_callback(lock_acquired) return out def __getstate__(self): state = super().__getstate__() state["_threaded_acquire"] = False return state def __setstate__(self, state): super().__setstate__(state) def release(self): """ Release the lock. If the lock was acquired in the same process via coro_acquire, we need to release the lock in the ThreadPoolExecutor's thread. """ if self._threaded_acquire: out = self.run_in_thread(super().release) else: out = super().release() self._threaded_acquire = False return out async def __aenter__(self): await self.coro_acquire() return None async def __aexit__(self, *args, **kwargs): self.release() def __iter__(self): yield from self.coro_acquire() return _ContextManager(self) class AioBarrierProxy(_managers.BarrierProxy, metaclass=ProxyCoroBuilder): coroutines = ["wait"] class AioEventProxy(_managers.EventProxy, metaclass=ProxyCoroBuilder): coroutines = ["wait"] class AioConditionProxy(_managers.ConditionProxy, metaclass=ProxyCoroBuilder): coroutines = ["wait", "wait_for"] class AioSyncManager(_managers.SyncManager): """ A mp.Manager that provides asyncio-friendly objects. """ pass AioSyncManager.register("AioQueue", Queue, AioQueueProxy) AioSyncManager.register("AioBarrier", Barrier, AioQueueProxy) AioSyncManager.register( "AioBoundedSemaphore", BoundedSemaphore, AioAcquirerProxy ) AioSyncManager.register("AioCondition", Condition, AioConditionProxy) AioSyncManager.register("AioEvent", Event, AioQueueProxy) AioSyncManager.register("AioLock", Lock, AioAcquirerProxy) AioSyncManager.register("AioRLock", RLock, AioAcquirerProxy) AioSyncManager.register("AioSemaphore", Semaphore, AioAcquirerProxy) aioprocessing-2.0.0/aioprocessing/mp.py000066400000000000000000000004661405204035700202170ustar00rootroot00000000000000# flake8: noqa import os try: if os.environ.get("AIOPROCESSING_DILL_DISABLED"): raise ImportError from multiprocess import * from multiprocess import connection, managers, util except ImportError: from multiprocessing import * from multiprocessing import connection, managers, util aioprocessing-2.0.0/aioprocessing/pool.py000066400000000000000000000032041405204035700205450ustar00rootroot00000000000000from asyncio import Future import asyncio from .executor import CoroBuilder from .mp import Pool __all__ = ["AioPool"] class AioPool(metaclass=CoroBuilder): delegate = Pool coroutines = ["join"] def _coro_func(self, funcname, *args, loop=None, **kwargs): """ Call the given function, and wrap the reuslt in a Future. funcname should be the name of a function which takes `callback` and `error_callback` keyword arguments (e.g. apply_async). """ if not loop: loop = asyncio.get_event_loop() fut = Future() def set_result(result): loop.call_soon_threadsafe(fut.set_result, result) def set_exc(exc): loop.call_soon_threadsafe(fut.set_exception, exc) func = getattr(self._obj, funcname) func(*args, callback=set_result, error_callback=set_exc, **kwargs) return fut def coro_apply(self, func, args=(), kwds=None, *, loop=None): if kwds is None: kwds = {} return self._coro_func( "apply_async", func, args=args, kwds=kwds, loop=loop ) def coro_map(self, func, iterable, chunksize=None, *, loop=None): return self._coro_func( "map_async", func, iterable, chunksize=chunksize, loop=loop ) def coro_starmap(self, func, iterable, chunksize=None, *, loop=None): return self._coro_func( "starmap_async", func, iterable, chunksize=chunksize, loop=loop ) def __enter__(self): self._obj.__enter__() return self def __exit__(self, *args, **kwargs): self._obj.__exit__(*args, **kwargs) aioprocessing-2.0.0/aioprocessing/process.py000066400000000000000000000002601405204035700212510ustar00rootroot00000000000000from .executor import CoroBuilder from .mp import Process __all__ = ["AioProcess"] class AioProcess(metaclass=CoroBuilder): delegate = Process coroutines = ["join"] aioprocessing-2.0.0/aioprocessing/queues.py000066400000000000000000000016651405204035700211140ustar00rootroot00000000000000from .executor import CoroBuilder from .mp import Queue, SimpleQueue, JoinableQueue class AioBaseQueue(metaclass=CoroBuilder): coroutines = ["get", "put"] class AioSimpleQueue(AioBaseQueue): """ An asyncio-friendly version of mp.SimpleQueue. Provides two coroutines: coro_get and coro_put, which are asynchronous version of get and put, respectively. """ delegate = SimpleQueue class AioQueue(AioBaseQueue): """ An asyncio-friendly version of mp.SimpleQueue. Provides two coroutines: coro_get and coro_put, which are asynchronous version of get and put, respectively. """ delegate = Queue class AioJoinableQueue(AioBaseQueue): """ An asyncio-friendly version of mp.JoinableQueue. Provides three coroutines: coro_get, coro_put, and coro_join, which are asynchronous version of get put, and join, respectively. """ coroutines = ["join"] delegate = JoinableQueue aioprocessing-2.0.0/aioprocessing/util.py000066400000000000000000000005161405204035700205540ustar00rootroot00000000000000import asyncio def run_in_executor(executor, callback, *args, loop=None, **kwargs): if not loop: loop = asyncio.get_event_loop() if kwargs: return loop.run_in_executor( executor, lambda: callback(*args, **kwargs) ) else: return loop.run_in_executor(executor, callback, *args) aioprocessing-2.0.0/ez_setup.py000066400000000000000000000243541405204035700165760ustar00rootroot00000000000000#!/usr/bin/env python """Bootstrap setuptools installation To use setuptools in your package's setup.py, include this file in the same directory and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() To require a specific version of setuptools, set a download mirror, or use an alternate download directory, simply supply the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import zipfile import optparse import subprocess import platform import textwrap import contextlib from distutils import log try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen try: from site import USER_SITE except ImportError: USER_SITE = None DEFAULT_VERSION = "5.7" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" def _python_cmd(*args): """ Return True if the command succeeded. """ args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(archive_filename, install_args=()): with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 def _build_egg(egg, archive_filename, to_dir): with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') class ContextualZipFile(zipfile.ZipFile): """ Supplement ZipFile class to support context manager for Python 2.6 """ def __enter__(self): return self def __exit__(self, type, value, traceback): self.close() def __new__(cls, *args, **kwargs): """ Construct a ZipFile or ContextualZipFile as appropriate """ if hasattr(zipfile.ZipFile, '__exit__'): return zipfile.ZipFile(*args, **kwargs) return super(ContextualZipFile, cls).__new__(cls) @contextlib.contextmanager def archive_context(filename): # extracting the archive tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) with ContextualZipFile(filename) as archive: archive.extractall() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): archive = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: del sys.modules['pkg_resources'] import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15): to_dir = os.path.abspath(to_dir) rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) try: import pkg_resources except ImportError: return _do_download(version, download_base, to_dir, download_delay) try: pkg_resources.require("setuptools>=" + version) return except pkg_resources.DistributionNotFound: return _do_download(version, download_base, to_dir, download_delay) except pkg_resources.VersionConflict as VC_err: if imported: msg = textwrap.dedent(""" The required version of setuptools (>={version}) is not available, and can't be installed while this script is running. Please install a more recent version first, using 'easy_install -U setuptools'. (Currently using {VC_err.args[0]!r}) """).format(VC_err=VC_err, version=version) sys.stderr.write(msg) sys.exit(2) # otherwise, reload ok del pkg_resources, sys.modules['pkg_resources'] return _do_download(version, download_base, to_dir, download_delay) def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell (which will validate trust). Raise an exception if the command cannot complete. """ target = os.path.abspath(target) ps_cmd = ( "[System.Net.WebRequest]::DefaultWebProxy.Credentials = " "[System.Net.CredentialCache]::DefaultCredentials; " "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars() ) cmd = [ 'powershell', '-Command', ps_cmd, ] _clean_check(cmd, target) def has_powershell(): if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] with open(os.path.devnull, 'wb') as devnull: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except Exception: return False return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """ Use Python to download the file, even though it cannot authenticate the connection. """ src = urlopen(url) try: # Read all the data in one block. data = src.read() finally: src.close() # Write all the data in one block to avoid creating a partial file. with open(target, "wb") as dst: dst.write(data) download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = ( download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ) viable_downloaders = (dl for dl in downloaders if dl.viable()) return next(viable_downloaders, None) def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): """ Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an sdist for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version url = download_base + zip_name saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package """ return ['--user'] if options.user_install else [] def _parse_args(): """ Parse the command line for options """ parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) parser.add_option( '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) options, args = parser.parse_args() # positional arguments are ignored return options def main(): """Install or upgrade setuptools and EasyInstall""" options = _parse_args() archive = download_setuptools( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, ) return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) aioprocessing-2.0.0/requirements.txt000066400000000000000000000000071405204035700176370ustar00rootroot00000000000000flake8 aioprocessing-2.0.0/runtests.py000066400000000000000000000221551405204035700166240ustar00rootroot00000000000000#!/usr/bin/python3 """ Run aioprocessing unit tests. Usage: python3 runtests.py [flags] [pattern] ... Patterns are matched against the fully qualified name of the test, including package, module, class and method, e.g. 'tests.test_events.PolicyTests.testPolicy'. For full help, try --help. runtests.py --coverage is equivalent of: $(COVERAGE) run --branch runtests.py -v $(COVERAGE) html $(list of files) $(COVERAGE) report -m $(list of files) """ # Originally written by Beech Horn (for NDB). import argparse import gc import logging import os import re import shutil import sys import unittest import traceback import textwrap import importlib.machinery try: import coverage except ImportError: coverage = None from unittest.signals import installHandler assert sys.version >= '3.5', 'Please use Python 3.5 or higher.' ARGS = argparse.ArgumentParser(description="Run all unittests.") ARGS.add_argument( '-v', action="store", dest='verbose', nargs='?', const=1, type=int, default=0, help='verbose') ARGS.add_argument( '-x', action="store_true", dest='exclude', help='exclude tests') ARGS.add_argument( '-f', '--failfast', action="store_true", default=False, dest='failfast', help='Stop on first fail or error') ARGS.add_argument( '-c', '--catch', action="store_true", default=False, dest='catchbreak', help='Catch control-C and display results') ARGS.add_argument( '--forever', action="store_true", dest='forever', default=False, help='run tests forever to catch sporadic errors') ARGS.add_argument( '--findleaks', action='store_true', dest='findleaks', help='detect tests that leak memory') ARGS.add_argument( '-q', action="store_true", dest='quiet', help='quiet') ARGS.add_argument( '--tests', action="store", dest='testsdir', default='tests', help='tests directory') ARGS.add_argument( '--coverage', action="store_true", dest='coverage', help='enable html coverage report') ARGS.add_argument( 'pattern', action="store", nargs="*", help='optional regex patterns to match test ids (default all tests)') COV_ARGS = argparse.ArgumentParser(description="Run all unittests.") COV_ARGS.add_argument( '--coverage', action="store", dest='coverage', nargs='?', const='', help='enable coverage report and provide python files directory') def load_modules(basedir, suffix='.py', *, verbose=False): def list_dir(prefix, dir): files = [] modpath = os.path.join(dir, '__init__.py') if os.path.isfile(modpath): mod = os.path.split(dir)[-1] files.append(('{}{}'.format(prefix, mod), modpath)) prefix = '{}{}.'.format(prefix, mod) for name in os.listdir(dir): path = os.path.join(dir, name) if os.path.isdir(path): files.extend(list_dir('{}{}.'.format(prefix, name), path)) else: if (name != '__init__.py' and name.endswith(suffix) and not name.startswith(('.', '_'))): files.append(('{}{}'.format(prefix, name[:-3]), path)) return files mods = [] for modname, sourcefile in list_dir('', basedir): if modname == 'runtests': continue try: loader = importlib.machinery.SourceFileLoader(modname, sourcefile) mods.append((loader.load_module(), sourcefile)) except SyntaxError: raise except Exception as err: print("Skipping '{}': {}".format(modname, err), file=sys.stderr) if verbose: try: traceback.print_exc() except Exception: pass return mods class TestsFinder: def __init__(self, testsdir, includes=(), excludes=(), *, verbose=False): self._testsdir = testsdir self._includes = includes self._excludes = excludes self._verbose = verbose self.find_available_tests() def find_available_tests(self): """ Find available test classes without instantiating them. """ self._test_factories = [] mods = [mod for mod, _ in load_modules(self._testsdir, verbose=self._verbose)] for mod in mods: for name in set(dir(mod)): if name.endswith('Test'): self._test_factories.append(getattr(mod, name)) def load_tests(self): """ Load test cases from the available test classes and apply optional include / exclude filters. """ loader = unittest.TestLoader() suite = unittest.TestSuite() for test_factory in self._test_factories: tests = loader.loadTestsFromTestCase(test_factory) if self._includes: tests = [test for test in tests if any(re.search(pat, test.id()) for pat in self._includes)] if self._excludes: tests = [test for test in tests if not any(re.search(pat, test.id()) for pat in self._excludes)] suite.addTests(tests) return suite class TestResult(unittest.TextTestResult): def __init__(self, stream, descriptions, verbosity): super().__init__(stream, descriptions, verbosity) self.leaks = [] def startTest(self, test): super().startTest(test) gc.collect() def addSuccess(self, test): super().addSuccess(test) gc.collect() if gc.garbage: if self.showAll: self.stream.writeln( " Warning: test created {} uncollectable " "object(s).".format(len(gc.garbage))) # move the uncollectable objects somewhere so we don't see # them again self.leaks.append((self.getDescription(test), gc.garbage[:])) del gc.garbage[:] class TestRunner(unittest.TextTestRunner): resultclass = TestResult def run(self, test): result = super().run(test) if result.leaks: self.stream.writeln("{} tests leaks:".format(len(result.leaks))) for name, leaks in result.leaks: self.stream.writeln(' '*4 + name + ':') for leak in leaks: self.stream.writeln(' '*8 + repr(leak)) return result def runtests(): args = ARGS.parse_args() if args.coverage and coverage is None: URL = "bitbucket.org/pypa/setuptools/raw/bootstrap/ez_setup.py" print(textwrap.dedent(""" coverage package is not installed. To install coverage3 for Python 3, you need: - Setuptools (https://pypi.python.org/pypi/setuptools) What worked for me: - download {0} * curl -O https://{0} - python3 ez_setup.py - python3 -m easy_install coverage """.format(URL)).strip()) sys.exit(1) testsdir = os.path.abspath(args.testsdir) if not os.path.isdir(testsdir): print("Tests directory is not found: {}\n".format(testsdir)) ARGS.print_help() return excludes = includes = [] if args.exclude: excludes = args.pattern else: includes = args.pattern v = 0 if args.quiet else args.verbose + 1 failfast = args.failfast catchbreak = args.catchbreak findleaks = args.findleaks runner_factory = TestRunner if findleaks else unittest.TextTestRunner if args.coverage: cov = coverage.coverage(branch=True, source=['aioprocessing'],) cov.start() finder = TestsFinder(args.testsdir, includes, excludes, verbose=args.verbose) logger = logging.getLogger() if v == 0: logger.setLevel(logging.CRITICAL) elif v == 1: logger.setLevel(logging.ERROR) elif v == 2: logger.setLevel(logging.WARNING) elif v == 3: logger.setLevel(logging.INFO) elif v >= 4: logger.setLevel(logging.DEBUG) if catchbreak: installHandler() try: if args.forever: while True: tests = finder.load_tests() result = runner_factory(verbosity=v, failfast=failfast).run(tests) if not result.wasSuccessful(): sys.exit(1) else: tests = finder.load_tests() result = runner_factory(verbosity=v, failfast=failfast).run(tests) sys.exit(not result.wasSuccessful()) finally: if args.coverage: cov.stop() cov.save() if os.path.exists('htmlcov'): shutil.rmtree('htmlcov') cov.html_report(directory='htmlcov') print("\nCoverage report:") cov.report(show_missing=False) here = os.path.dirname(os.path.abspath(__file__)) print("\nFor html report:") print("open file://{}/htmlcov/index.html".format(here)) if __name__ == '__main__': runtests() aioprocessing-2.0.0/setup.cfg000066400000000000000000000000361405204035700161760ustar00rootroot00000000000000[flake8] exclude = ez_setup.pyaioprocessing-2.0.0/setup.py000066400000000000000000000027501405204035700160740ustar00rootroot00000000000000from setuptools import setup, find_packages import aioprocessing with open("README.md", "r") as f: readme = f.read() setup( name="aioprocessing", version=aioprocessing.version, packages=find_packages(exclude=["tests"]), author="Dan O'Reilly", author_email="oreilldf@gmail.com", description=( "A Python 3.5+ library that integrates " "the multiprocessing module with asyncio." ), zip_safe=False, license="BSD", extras_require={"dill": ["multiprocess"]}, keywords="asyncio multiprocessing coroutine", url="https://github.com/dano/aioprocessing", long_description=readme, long_description_content_type="text/markdown", classifiers=[ "Development Status :: 3 - Alpha", "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: BSD License", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Topic :: Software Development :: Libraries :: Python Modules", ], ) aioprocessing-2.0.0/tester.py000077500000000000000000000016371405204035700162500ustar00rootroot00000000000000#!/usr/bin/python3 import asyncio from aioprocessing import AioManager from concurrent.futures import ProcessPoolExecutor async def _do_coro_proc_work(q, val, val2): ok = val + val2 # await asyncio.sleep(4) print("Passing {} to parent".format(ok)) await q.coro_put(ok) item = q.get() print("got {} back from parent".format(item)) def do_coro_proc_work(q, val, val2): loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) loop.run_until_complete(_do_coro_proc_work(q, val, val2)) async def do_work(q): print("hi") loop.run_in_executor(ProcessPoolExecutor(), do_coro_proc_work, q, 1, 2) item = await q.coro_get() print("Got {} from worker".format(item)) item = item + 25 q.put(item) if __name__ == "__main__": m = AioManager() q = m.AioQueue() loop = asyncio.get_event_loop() loop.run_until_complete(do_work(q)) aioprocessing-2.0.0/tests/000077500000000000000000000000001405204035700155205ustar00rootroot00000000000000aioprocessing-2.0.0/tests/.gitignore000066400000000000000000000000141405204035700175030ustar00rootroot00000000000000*.swp *.pyc aioprocessing-2.0.0/tests/__init__.py000066400000000000000000000000001405204035700176170ustar00rootroot00000000000000aioprocessing-2.0.0/tests/_base_test.py000066400000000000000000000015711405204035700202060ustar00rootroot00000000000000import asyncio import unittest import aioprocessing.mp as multiprocessing class BaseTest(unittest.TestCase): def setUp(self): self.loop = asyncio.get_event_loop() def assertReturnsIfImplemented(self, value, func, *args): try: res = func(*args) except NotImplementedError: pass else: return self.assertEqual(value, res) class _GenMixin: initargs = () args = () def test_loop(self): loop = asyncio.new_event_loop() getattr(self.inst, self.meth)(*self.args, loop=loop) self._after() @unittest.skipIf( not hasattr(multiprocessing, "get_context"), "Not supported in this version of Python", ) def test_ctx(self): ctx = multiprocessing.get_context("spawn") self.Obj(*self.initargs, context=ctx) def _after(self): pass aioprocessing-2.0.0/tests/connection_tests.py000066400000000000000000000073211405204035700214560ustar00rootroot00000000000000import unittest from array import array import aioprocessing import aioprocessing.mp as multiprocessing from aioprocessing.connection import AioConnection, AioListener, AioClient from aioprocessing.mp import Process from ._base_test import BaseTest def conn_send(conn, val): conn.send(val) def client_sendback(event, address, authkey): event.wait() conn = multiprocessing.connection.Client(address, authkey=authkey) got = conn.recv() conn.send(got + got) conn.close() def listener_sendback(event, address, authkey): listener = multiprocessing.connection.Listener(address, authkey=authkey) event.set() conn = listener.accept() inval = conn.recv() conn.send_bytes(array("i", [inval, inval + 1, inval + 2, inval + 3])) conn.close() class PipeTest(BaseTest): def test_pipe(self): conn1, conn2 = aioprocessing.AioPipe() val = 25 p = Process(target=conn_send, args=(conn1, val)) p.start() async def conn_recv(): out = await conn2.coro_recv() self.assertEqual(out, val) self.loop.run_until_complete(conn_recv()) class ListenerTest(BaseTest): def test_listener(self): address = ("localhost", 8999) authkey = b"abcdefg" event = multiprocessing.Event() p = Process(target=client_sendback, args=(event, address, authkey)) p.start() listener = AioListener(address, authkey=authkey) try: event.set() conn = listener.accept() self.assertIsInstance(conn, AioConnection) conn.send("") conn.close() event.clear() p.join() p = Process(target=client_sendback, args=(event, address, authkey)) p.start() def conn_accept(): fut = listener.coro_accept() event.set() conn = yield from fut self.assertIsInstance(conn, AioConnection) yield from conn.coro_send("hi there") back = yield from conn.coro_recv() self.assertEqual(back, "hi therehi there") conn.close() self.loop.run_until_complete(conn_accept()) p.join() finally: listener.close() def test_client(self): address = ("localhost", 8999) authkey = b"abcdefg" event = multiprocessing.Event() p = Process(target=listener_sendback, args=(event, address, authkey)) p.start() event.wait() conn = AioClient(address, authkey=authkey) self.assertIsInstance(conn, AioConnection) def do_work(): yield from conn.coro_send(25) arr = array("i", [0, 0, 0, 0]) yield from conn.coro_recv_bytes_into(arr) self.assertEqual(arr, array("i", [25, 26, 27, 28])) conn.close() self.loop.run_until_complete(do_work()) p.join() def test_listener_ctxmgr(self): address = ("localhost", 8999) authkey = b"abcdefg" with AioListener(address, authkey=authkey) as listener: self.assertIsInstance(listener, AioListener) self.assertRaises(OSError, listener.accept) def test_client_ctxmgr(self): address = ("localhost", 8999) authkey = b"abcdefg" event = multiprocessing.Event() p = Process(target=listener_sendback, args=(event, address, authkey)) p.daemon = True p.start() event.wait() with AioClient(address, authkey=authkey) as conn: self.assertIsInstance(conn, AioConnection) self.assertRaises(OSError, conn.send, "hi") p.terminate() p.join() if __name__ == "__main__": unittest.main() aioprocessing-2.0.0/tests/lock_tests.py000077500000000000000000000246641405204035700202630ustar00rootroot00000000000000import sys import time import asyncio import unittest import traceback import aioprocessing import aioprocessing.mp as multiprocessing from aioprocessing.mp import Process, Event, Queue, get_all_start_methods try: from aioprocessing.mp import get_context except ImportError: def get_context(param): pass from ._base_test import BaseTest, _GenMixin MANAGER_TYPE = 1 STANDARD_TYPE = 2 def get_value(self): try: return self.get_value() except AttributeError: try: return self._Semaphore__value except AttributeError: try: return self._value except AttributeError: raise NotImplementedError def do_lock_acquire(lock, e): lock.acquire() e.set() time.sleep(2) lock.release() def sync_lock(lock, event, event2, queue): event2.wait() queue.put(lock.acquire(False)) event.set() lock.acquire() lock.release() class GenAioLockTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioLock self.inst = self.Obj() self.meth = "coro_acquire" class GenAioManagerLockTest(GenAioLockTest): def setUp(self): super().setUp() self.manager = aioprocessing.AioManager() self.Obj = self.manager.AioLock self.inst = self.Obj() @unittest.skipIf( not hasattr(multiprocessing, "get_context"), "No get_context method" ) def test_ctx(self): pass class GenAioRLockTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioRLock self.inst = self.Obj() self.meth = "coro_acquire" class GenAioConditionTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioCondition self.inst = self.Obj() self.meth = "coro_acquire" class GenAioSemaphoreTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioSemaphore self.inst = self.Obj() self.meth = "coro_acquire" class GenAioEventTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioEvent self.inst = self.Obj() self.meth = "coro_wait" def _after(self): self.inst.set() class GenAioBarrierTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioBarrier self.inst = self.Obj(1) self.initargs = (1,) self.meth = "coro_wait" class LoopLockTest(BaseTest): def setUp(self): pass def test_lock_with_loop(self): loop = asyncio.new_event_loop() lock = aioprocessing.AioLock() async def do_async_lock(): await lock.coro_acquire(loop=loop) loop.run_until_complete(do_async_lock()) class LockTest(BaseTest): def setUp(self): super().setUp() self.type_ = STANDARD_TYPE self.lock = aioprocessing.AioLock() def test_lock(self): self.assertEqual(True, self.lock.acquire()) self.assertEqual(False, self.lock.acquire(False)) self.assertEqual(None, self.lock.release()) def test_lock_async(self): async def do_async_lock(): self.assertEqual(True, (await self.lock.coro_acquire())) self.assertEqual(None, self.lock.release()) self.loop.run_until_complete(do_async_lock()) def test_lock_cm(self): event = Event() event2 = Event() q = Queue() async def with_lock(): async with self.lock: event2.set() await asyncio.sleep(1) event.wait() p = Process(target=sync_lock, args=(self.lock, event, event2, q)) p.start() self.loop.run_until_complete(with_lock()) p.join() self.assertFalse(q.get()) def test_lock_multiproc(self): e = Event() async def do_async_lock(): self.assertEqual(False, (await self.lock.coro_acquire(False))) self.assertEqual( True, (await self.lock.coro_acquire(timeout=4)) ) p = Process(target=do_lock_acquire, args=(self.lock, e)) p.start() e.wait() self.loop.run_until_complete(do_async_lock()) class LockManagerTest(LockTest): def setUp(self): super().setUp() self.type_ = MANAGER_TYPE self.manager = aioprocessing.AioManager() self.lock = self.manager.AioLock() def tearDown(self): super().tearDown() self.manager.shutdown() self.manager.join() class RLockTest(LockTest): def setUp(self): super().setUp() self.lock = aioprocessing.AioRLock() def test_lock(self): self.assertEqual(True, self.lock.acquire()) self.assertEqual(True, self.lock.acquire(False)) self.assertEqual(None, self.lock.release()) class RLockManagerTest(RLockTest): def setUp(self): super().setUp() self.type_ = MANAGER_TYPE self.manager = aioprocessing.AioManager() self.lock = self.manager.AioRLock() def tearDown(self): super().tearDown() self.manager.shutdown() self.manager.join() def mix_release(lock, q): try: try: lock.release() except (ValueError, AssertionError): pass else: q.put("Didn't get excepted AssertionError") lock.acquire() lock.release() q.put(True) except Exception: exc = traceback.format_exception(*sys.exc_info()) q.put(exc) class LockMixingTest(BaseTest): def setUp(self): super().setUp() self.lock = aioprocessing.AioRLock() def test_sync_lock(self): self.lock.acquire() self.lock.release() def test_mix_async_to_sync(self): async def do_acquire(): await self.lock.coro_acquire() self.loop.run_until_complete(do_acquire()) self.lock.release() def test_mix_with_procs(self): async def do_acquire(): await self.lock.coro_acquire() q = Queue() p = Process(target=mix_release, args=(self.lock, q)) self.loop.run_until_complete(do_acquire()) p.start() self.lock.release() out = q.get(timeout=5) p.join() self.assertTrue(isinstance(out, bool)) class SpawnLockMixingTest(LockMixingTest): def setUp(self): super().setUp() context = get_context("spawn") self.lock = aioprocessing.AioLock(context=context) if "forkserver" in get_all_start_methods(): class ForkServerLockMixingTest(LockMixingTest): def setUp(self): super().setUp() context = get_context("forkserver") self.lock = aioprocessing.AioLock(context=context) class SemaphoreTest(BaseTest): def setUp(self): super().setUp() self.sem = aioprocessing.AioSemaphore(2) def _test_semaphore(self, sem): self.assertReturnsIfImplemented(2, get_value, sem) self.assertEqual(True, sem.acquire()) self.assertReturnsIfImplemented(1, get_value, sem) async def sem_acquire(): self.assertEqual(True, (await sem.coro_acquire())) self.loop.run_until_complete(sem_acquire()) self.assertReturnsIfImplemented(0, get_value, sem) self.assertEqual(False, sem.acquire(False)) self.assertReturnsIfImplemented(0, get_value, sem) self.assertEqual(None, sem.release()) self.assertReturnsIfImplemented(1, get_value, sem) self.assertEqual(None, sem.release()) self.assertReturnsIfImplemented(2, get_value, sem) def test_semaphore(self): sem = self.sem self._test_semaphore(sem) self.assertEqual(None, sem.release()) self.assertReturnsIfImplemented(3, get_value, sem) self.assertEqual(None, sem.release()) self.assertReturnsIfImplemented(4, get_value, sem) class BoundedSemaphoreTest(SemaphoreTest): def setUp(self): super().setUp() self.sem = aioprocessing.AioBoundedSemaphore(2) def test_semaphore(self): self._test_semaphore(self.sem) def barrier_wait(barrier, event): event.set() barrier.wait() class BarrierTest(BaseTest): def setUp(self): super().setUp() self.barrier = aioprocessing.AioBarrier(2) def _wait_barrier(self): self.barrier.wait() def test_barrier(self): fut = None async def wait_barrier_async(): await self.barrier.coro_wait() async def wait_barrier(): nonlocal fut fut = asyncio.ensure_future(wait_barrier_async()) await asyncio.sleep(0.5) self.assertEqual(1, self.barrier.n_waiting) self.barrier.wait() # t = threading.Thread(target=self._wait_barrier) # t.start() self.loop.run_until_complete(wait_barrier()) self.loop.run_until_complete(fut) def test_barrier_multiproc(self): event = Event() p = Process(target=barrier_wait, args=(self.barrier, event)) p.start() async def wait_barrier(): event.wait() await asyncio.sleep(0.2) self.assertEqual(1, self.barrier.n_waiting) await self.barrier.coro_wait() self.loop.run_until_complete(wait_barrier()) p.join() def set_event(event): event.set() class EventTest(BaseTest): def setUp(self): super().setUp() self.event = aioprocessing.AioEvent() def test_event(self): p = Process(target=set_event, args=(self.event,)) async def wait_event(): await self.event.coro_wait() p.start() self.loop.run_until_complete(wait_event()) p.join() def cond_notify(cond, event): time.sleep(2) event.set() cond.acquire() cond.notify_all() cond.release() class ConditionTest(BaseTest): def setUp(self): super().setUp() self.cond = aioprocessing.AioCondition() def test_cond(self): event = Event() def pred(): return event.is_set() async def wait_for_pred(): await self.cond.coro_acquire() await self.cond.coro_wait_for(pred) self.cond.release() p = Process(target=cond_notify, args=(self.cond, event)) p.start() self.loop.run_until_complete(wait_for_pred()) p.join() if __name__ == "__main__": unittest.main() aioprocessing-2.0.0/tests/pickle_test.py000066400000000000000000000005731405204035700204050ustar00rootroot00000000000000import pickle import unittest from aioprocessing.executor import _ExecutorMixin class PickleTest(unittest.TestCase): def test_pickle_queue(self): q = _ExecutorMixin() q.test = "abc" pickled = pickle.dumps(q) unpickled = pickle.loads(pickled) self.assertEqual(q.test, unpickled.test) if __name__ == "__main__": unittest.main() aioprocessing-2.0.0/tests/pool_test.py000066400000000000000000000034541405204035700201100ustar00rootroot00000000000000import aioprocessing from ._base_test import BaseTest, _GenMixin def work_func(a, b): c = a * b return c def map_func(z): return z * 3 def starmap(func, it): return map(func, *zip(*it)) class GenAioPoolTest(BaseTest, _GenMixin): def setUp(self): super().setUp() self.Obj = aioprocessing.AioPool self.inst = self.Obj(1) self.initargs = (1,) self.meth = "coro_map" self.args = (map_func, [1, 2, 3]) class PoolTest(BaseTest): def setUp(self): super().setUp() self.pool = aioprocessing.AioPool() def tearDown(self): super().tearDown() self.pool.close() self.pool.join() def test_ctx_mgr(self): with aioprocessing.AioPool() as pool: self.assertIsInstance(pool, aioprocessing.pool.AioPool) self.assertRaises(ValueError, pool.map, map_func, [1]) def test_coro_apply(self): async def do_apply(): out = await self.pool.coro_apply(work_func, args=(2, 3)) self.assertEqual(out, 6) self.loop.run_until_complete(do_apply()) def test_coro_map(self): it = list(range(5)) async def do_map(): out = await self.pool.coro_map(map_func, it) self.assertEqual(out, list(map(map_func, it))) self.loop.run_until_complete(do_map()) def test_coro_starmap(self): it = list(zip(range(5), range(5, 10))) async def do_starmap(): out = await self.pool.coro_starmap(work_func, it) self.assertEqual(out, list(starmap(work_func, it))) self.loop.run_until_complete(do_starmap()) def test_coro_join(self): async def do_join(): await self.pool.coro_join() self.pool.close() self.loop.run_until_complete(do_join()) aioprocessing-2.0.0/tests/process_test.py000066400000000000000000000013771405204035700206170ustar00rootroot00000000000000import unittest import aioprocessing.mp as multiprocessing import aioprocessing from ._base_test import BaseTest, _GenMixin def f(q, a, b): q.put((a, b)) def dummy(): pass class GenAioProcessTest(BaseTest, _GenMixin): def setUp(self): self.Obj = aioprocessing.AioProcess self.inst = self.Obj(target=dummy) self.meth = "coro_join" class ProcessTest(BaseTest): def test_pickle_queue(self): t = ("a", "b") q = multiprocessing.Queue() p = aioprocessing.AioProcess(target=f, args=(q,) + t) p.start() async def join(): await p.coro_join() self.loop.run_until_complete(join()) self.assertEqual(q.get(), t) if __name__ == "__main__": unittest.main() aioprocessing-2.0.0/tests/queue_test.py000066400000000000000000000061771405204035700202700ustar00rootroot00000000000000import unittest from concurrent.futures import ProcessPoolExecutor import aioprocessing from aioprocessing.mp import Process, Event, util from ._base_test import BaseTest, _GenMixin def queue_put(q, val): val = q.put(val) return val def queue_get(q, e): val = q.get() e.set() q.put(val) class GenQueueMixin(_GenMixin): def setUp(self): super().setUp() self.inst = self.Obj() self.meth = "coro_get" def _after(self): self.inst.put(1) class GenAioQueueTest(GenQueueMixin, BaseTest): def setUp(self): self.Obj = aioprocessing.AioQueue super().setUp() class GenAioSimpleQueueTest(GenQueueMixin, BaseTest): def setUp(self): self.Obj = aioprocessing.AioSimpleQueue super().setUp() class GenAioJoinableQueueTest(GenQueueMixin, BaseTest): def setUp(self): self.Obj = aioprocessing.AioJoinableQueue super().setUp() class QueueTest(BaseTest): def test_blocking_put(self): q = aioprocessing.AioQueue() async def queue_put(): await q.coro_put(1) self.loop.run_until_complete(queue_put()) self.assertEqual(q.get(), 1) def test_put_get(self): q = aioprocessing.AioQueue() val = 1 p = Process(target=queue_put, args=(q, val)) async def queue_get(): ret = await q.coro_get() self.assertEqual(ret, val) p.start() self.loop.run_until_complete(queue_get()) p.join() def test_get_put(self): q = aioprocessing.AioQueue() e = Event() val = 2 async def queue_put(): await q.coro_put(val) p = Process(target=queue_get, args=(q, e)) p.start() self.loop.run_until_complete(queue_put()) e.wait() out = q.get() p.join() self.assertEqual(out, val) def test_simple_queue(self): q = aioprocessing.AioSimpleQueue() val = 8 async def queue_put(): await q.coro_put(val) self.loop.run_until_complete(queue_put()) out = q.get() self.assertEqual(val, out) class ManagerQueueTest(BaseTest): @unittest.skipIf( "multiprocess.util" in str(util), "concurrent.futures is not yet supported by uqfoundation " "(https://github.com/uqfoundation/pathos/issues/90)" ) def test_executor(self): m = aioprocessing.AioManager() q = m.AioQueue() p = ProcessPoolExecutor(max_workers=1) val = 4 def submit(): yield p.submit(queue_put, q, val) next(submit()) async def queue_get(): out = await q.coro_get() self.assertEqual(out, val) await q.coro_put(5) self.loop.run_until_complete(queue_get()) returned = q.get() self.assertEqual(returned, 5) p.shutdown() class JoinableQueueTest(BaseTest): def test_join_empty_queue(self): q = aioprocessing.AioJoinableQueue() async def join(): await q.coro_join() self.loop.run_until_complete(join()) if __name__ == "__main__": unittest.main()