socketpool-0.5.3/0000775000175000017500000000000012210134336014766 5ustar benoitcbenoitc00000000000000socketpool-0.5.3/MANIFEST.in0000664000175000017500000000021212210132231016507 0ustar benoitcbenoitc00000000000000include NOTICE include LICENSE include README.rst include THANKS include UNLICENSE recursive-include examples * recursive-include tests * socketpool-0.5.3/PKG-INFO0000664000175000017500000001227112210134336016066 0ustar benoitcbenoitc00000000000000Metadata-Version: 1.1 Name: socketpool Version: 0.5.3 Summary: Python socket pool Home-page: http://github.com/benoitc/socketpool Author: UNKNOWN Author-email: UNKNOWN License: BSD Description: socketpool ---------- Socketpool - a simple Python socket pool. Socket pool is a simple socket pool that suports multiple factories and backends. It can easily be used by gevent, eventlet or any other library. Usage ----- socketpool offers 3 main classes, a `ConnectionPool` class able to accept a factory and a backend, `Connector` an interface class inherited by all connectors and a default TCP connector `TcpConnector` . Example of a simple echo client using Gevent -------------------------------------------- :: import gevent from gevent.server import StreamServer from socketpool import ConnectionPool, TcpConnector # this handler will be run for each incoming connection # in a dedicated greenlet def echo(sock, address): print ('New connection from %s:%s' % address) while True: data = sock.recv(1024) if not data: break sock.send(data) print ("echoed %r" % data) if __name__ == '__main__': import time options = {'host': 'localhost', 'port': 6000} pool = ConnectionPool(factory=TcpConnector, backend="gevent") server = StreamServer(('localhost', 6000), echo) gevent.spawn(server.serve_forever) def runpool(data): print 'ok' with pool.connection(**options) as conn: print 'sending' sent = conn.send(data) print 'send %d bytes' % sent echo_data = conn.recv(1024) print "got %s" % data assert data == echo_data start = time.time() jobs = [gevent.spawn(runpool, "blahblah") for _ in xrange(20)] gevent.joinall(jobs) delay = time.time() - start Example of a connector ---------------------- :: class TcpConnector(Connector): def __init__(self, host, port, backend_mod, pool=None): self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM) self._s.connect((host, port)) self.host = host self.port = port self._connected = True self._life = time.time() self._pool = pool def __del__(self): self.release() def matches(self, **match_options): target_host = match_options.get('host') target_port = match_options.get('port') return target_host == self.host and target_port == self.port def is_connected(self): return self._connected def handle_exception(self, exception): print 'got an exception' print str(exception) def get_lifetime(self): return self._life def invalidate(self): self._s.close() self._connected = False self._life = -1 def release(self): if self._pool is not None: if self._connected: self._pool.release_connection(self) else: self._pool = None def send(self, data): return self._s.send(data) def recv(self, size=1024): return self._s.recv(size) Authors ------- - Benoît Chesneau (benoitc) - Tarek Ziade (tarek) License ------- socketpool is available in the public domain (see UNLICENSE). socketpool is also optionally available under the MIT License (see LICENSE), meant especially for jurisdictions that do not recognize public domain works. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Web Environment Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Software Development :: Libraries socketpool-0.5.3/tests/0000775000175000017500000000000012210134336016130 5ustar benoitcbenoitc00000000000000socketpool-0.5.3/tests/test_backend_finding.py0000664000175000017500000000144712210132231022624 0ustar benoitcbenoitc00000000000000import pytest from socketpool.pool import ConnectionPool from socketpool.conn import TcpConnector def make_and_check_backend(expected_name, **kw): pool = ConnectionPool(TcpConnector, **kw) assert pool.backend == expected_name return pool def test_default_backend(): make_and_check_backend('thread') def test_thread_backend(): make_and_check_backend('thread', backend='thread') def test_gevent_backend(): pytest.importorskip('gevent') make_and_check_backend('gevent', backend='gevent') def test_eventlet_backend(): pytest.importorskip('eventlet') make_and_check_backend('eventlet', backend='eventlet') def test_thread_backend_as_module(): from socketpool import backend_thread make_and_check_backend('socketpool.backend_thread', backend=backend_thread) socketpool-0.5.3/tests/test_pool_01.py0000664000175000017500000000113412210132231021001 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import unittest from socketpool import ConnectionPool, Connector from socketpool.pool import MaxTriesError class MessyConnector(Connector): def __init__(self, **options): pass def is_connected(self): return False def invalidate(self): pass class PoolTestCase(unittest.TestCase): def test_size_on_isconnected_failure(self): pool = ConnectionPool(MessyConnector) self.assert_(pool.size == 0) self.assertRaises(MaxTriesError, pool.get) socketpool-0.5.3/setup.cfg0000664000175000017500000000007312210134336016607 0ustar benoitcbenoitc00000000000000[egg_info] tag_build = tag_date = 0 tag_svn_revision = 0 socketpool-0.5.3/setup.py0000775000175000017500000000226012210132231016473 0ustar benoitcbenoitc00000000000000#!/usr/bin/env python # -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import os from setuptools import setup, find_packages CLASSIFIERS = [ 'Development Status :: 4 - Beta', 'Environment :: Web Environment', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Topic :: Software Development :: Libraries'] # read long description with open(os.path.join(os.path.dirname(__file__), 'README.rst')) as f: long_description = f.read() DATA_FILES = [ ('socketpool', ["LICENSE", "MANIFEST.in", "NOTICE", "README.rst", "THANKS", "UNLICENSE"]) ] setup(name='socketpool', version='0.5.3', description = 'Python socket pool', long_description = long_description, classifiers = CLASSIFIERS, license = 'BSD', url = 'http://github.com/benoitc/socketpool', packages=find_packages(), data_files = DATA_FILES) socketpool-0.5.3/THANKS0000664000175000017500000000011012210132231015661 0ustar benoitcbenoitc00000000000000Ronny Pfannschmidt elij socketpool-0.5.3/README.rst0000664000175000017500000000676212210132231016460 0ustar benoitcbenoitc00000000000000socketpool ---------- Socketpool - a simple Python socket pool. Socket pool is a simple socket pool that suports multiple factories and backends. It can easily be used by gevent, eventlet or any other library. Usage ----- socketpool offers 3 main classes, a `ConnectionPool` class able to accept a factory and a backend, `Connector` an interface class inherited by all connectors and a default TCP connector `TcpConnector` . Example of a simple echo client using Gevent -------------------------------------------- :: import gevent from gevent.server import StreamServer from socketpool import ConnectionPool, TcpConnector # this handler will be run for each incoming connection # in a dedicated greenlet def echo(sock, address): print ('New connection from %s:%s' % address) while True: data = sock.recv(1024) if not data: break sock.send(data) print ("echoed %r" % data) if __name__ == '__main__': import time options = {'host': 'localhost', 'port': 6000} pool = ConnectionPool(factory=TcpConnector, backend="gevent") server = StreamServer(('localhost', 6000), echo) gevent.spawn(server.serve_forever) def runpool(data): print 'ok' with pool.connection(**options) as conn: print 'sending' sent = conn.send(data) print 'send %d bytes' % sent echo_data = conn.recv(1024) print "got %s" % data assert data == echo_data start = time.time() jobs = [gevent.spawn(runpool, "blahblah") for _ in xrange(20)] gevent.joinall(jobs) delay = time.time() - start Example of a connector ---------------------- :: class TcpConnector(Connector): def __init__(self, host, port, backend_mod, pool=None): self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM) self._s.connect((host, port)) self.host = host self.port = port self._connected = True self._life = time.time() self._pool = pool def __del__(self): self.release() def matches(self, **match_options): target_host = match_options.get('host') target_port = match_options.get('port') return target_host == self.host and target_port == self.port def is_connected(self): return self._connected def handle_exception(self, exception): print 'got an exception' print str(exception) def get_lifetime(self): return self._life def invalidate(self): self._s.close() self._connected = False self._life = -1 def release(self): if self._pool is not None: if self._connected: self._pool.release_connection(self) else: self._pool = None def send(self, data): return self._s.send(data) def recv(self, size=1024): return self._s.recv(size) Authors ------- - Benoît Chesneau (benoitc) - Tarek Ziade (tarek) License ------- socketpool is available in the public domain (see UNLICENSE). socketpool is also optionally available under the MIT License (see LICENSE), meant especially for jurisdictions that do not recognize public domain works. socketpool-0.5.3/socketpool.egg-info/0000775000175000017500000000000012210134336020642 5ustar benoitcbenoitc00000000000000socketpool-0.5.3/socketpool.egg-info/PKG-INFO0000664000175000017500000001227112210134336021742 0ustar benoitcbenoitc00000000000000Metadata-Version: 1.1 Name: socketpool Version: 0.5.3 Summary: Python socket pool Home-page: http://github.com/benoitc/socketpool Author: UNKNOWN Author-email: UNKNOWN License: BSD Description: socketpool ---------- Socketpool - a simple Python socket pool. Socket pool is a simple socket pool that suports multiple factories and backends. It can easily be used by gevent, eventlet or any other library. Usage ----- socketpool offers 3 main classes, a `ConnectionPool` class able to accept a factory and a backend, `Connector` an interface class inherited by all connectors and a default TCP connector `TcpConnector` . Example of a simple echo client using Gevent -------------------------------------------- :: import gevent from gevent.server import StreamServer from socketpool import ConnectionPool, TcpConnector # this handler will be run for each incoming connection # in a dedicated greenlet def echo(sock, address): print ('New connection from %s:%s' % address) while True: data = sock.recv(1024) if not data: break sock.send(data) print ("echoed %r" % data) if __name__ == '__main__': import time options = {'host': 'localhost', 'port': 6000} pool = ConnectionPool(factory=TcpConnector, backend="gevent") server = StreamServer(('localhost', 6000), echo) gevent.spawn(server.serve_forever) def runpool(data): print 'ok' with pool.connection(**options) as conn: print 'sending' sent = conn.send(data) print 'send %d bytes' % sent echo_data = conn.recv(1024) print "got %s" % data assert data == echo_data start = time.time() jobs = [gevent.spawn(runpool, "blahblah") for _ in xrange(20)] gevent.joinall(jobs) delay = time.time() - start Example of a connector ---------------------- :: class TcpConnector(Connector): def __init__(self, host, port, backend_mod, pool=None): self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM) self._s.connect((host, port)) self.host = host self.port = port self._connected = True self._life = time.time() self._pool = pool def __del__(self): self.release() def matches(self, **match_options): target_host = match_options.get('host') target_port = match_options.get('port') return target_host == self.host and target_port == self.port def is_connected(self): return self._connected def handle_exception(self, exception): print 'got an exception' print str(exception) def get_lifetime(self): return self._life def invalidate(self): self._s.close() self._connected = False self._life = -1 def release(self): if self._pool is not None: if self._connected: self._pool.release_connection(self) else: self._pool = None def send(self, data): return self._s.send(data) def recv(self, size=1024): return self._s.recv(size) Authors ------- - Benoît Chesneau (benoitc) - Tarek Ziade (tarek) License ------- socketpool is available in the public domain (see UNLICENSE). socketpool is also optionally available under the MIT License (see LICENSE), meant especially for jurisdictions that do not recognize public domain works. Platform: UNKNOWN Classifier: Development Status :: 4 - Beta Classifier: Environment :: Web Environment Classifier: Intended Audience :: Developers Classifier: License :: OSI Approved :: MIT License Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Programming Language :: Python :: 2.6 Classifier: Programming Language :: Python :: 2.7 Classifier: Programming Language :: Python :: 3 Classifier: Topic :: Software Development :: Libraries socketpool-0.5.3/socketpool.egg-info/top_level.txt0000664000175000017500000000001312210134336023366 0ustar benoitcbenoitc00000000000000socketpool socketpool-0.5.3/socketpool.egg-info/SOURCES.txt0000664000175000017500000000076012210134336022531 0ustar benoitcbenoitc00000000000000LICENSE MANIFEST.in NOTICE README.rst THANKS UNLICENSE setup.py examples/test_eventlet.py examples/test_gevent.py examples/test_threaded.py socketpool/__init__.py socketpool/backend_eventlet.py socketpool/backend_gevent.py socketpool/backend_thread.py socketpool/conn.py socketpool/pool.py socketpool/util.py socketpool.egg-info/PKG-INFO socketpool.egg-info/SOURCES.txt socketpool.egg-info/dependency_links.txt socketpool.egg-info/top_level.txt tests/test_backend_finding.py tests/test_pool_01.pysocketpool-0.5.3/socketpool.egg-info/dependency_links.txt0000664000175000017500000000000112210134336024710 0ustar benoitcbenoitc00000000000000 socketpool-0.5.3/examples/0000775000175000017500000000000012210134336016604 5ustar benoitcbenoitc00000000000000socketpool-0.5.3/examples/test_gevent.py0000664000175000017500000000271212210132231021477 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import gevent from gevent.server import StreamServer from socketpool.pool import ConnectionPool from socketpool.conn import TcpConnector # this handler will be run for each incoming connection in a dedicated greenlet def echo(sock, address): print ('New connection from %s:%s' % address) while True: data = sock.recv(1024) if not data: break sock.send(data) print ("echoed %r" % data) if __name__ == '__main__': import time options = {'host': 'localhost', 'port': 6000} pool = ConnectionPool(factory=TcpConnector, backend="gevent") server = StreamServer(('localhost', 6000), echo) gevent.spawn(server.serve_forever) def runpool(data): with pool.connection(**options) as conn: print ("conn: pool size: %s" % pool.size) sent = conn.send(data) echo_data = conn.recv(1024) assert data == echo_data start = time.time() jobs = [gevent.spawn(runpool, "blahblah") for _ in xrange(50)] gevent.joinall(jobs) delay = time.time() - start print ("final pool size: %s" % pool.size) with pool.connection(**options) as conn: print ("conn: pool size: %s" % pool.size) sent = conn.send("hello") echo_data = conn.recv(1024) assert "hello" == echo_data print ("final pool size: %s" % pool.size) socketpool-0.5.3/examples/test_threaded.py0000664000175000017500000000444412210132231021773 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import socket import sys import threading try: from queue import * except ImportError: from Queue import * try: import SocketServer as socketserver except ImportError: import socketserver import time from socketpool.pool import ConnectionPool from socketpool.conn import TcpConnector PY3 = sys.version_info[0] == 3 if sys.version_info[0] == 3: def s2b(s): return s.encode('latin1') else: def s2b(s): return s class EchoHandler(socketserver.BaseRequestHandler): def handle(self): while True: data = self.request.recv(1024) if not data: break self.request.send(data) print("echoed %r" % data) class EchoServer(socketserver.ThreadingMixIn, socketserver.TCPServer): pass if __name__ == "__main__": # Port 0 means to select an arbitrary unused port HOST, PORT = "localhost", 0 server = EchoServer((HOST, PORT), EchoHandler) ip, port = server.server_address # Start a thread with the server -- that thread will then start one # more thread for each request server_thread = threading.Thread(target=server.serve_forever, kwargs={"poll_interval":0.5}) # Exit the server thread when the main thread terminates server_thread.daemon = True server_thread.start() options = {'host': ip, 'port': port} pool = ConnectionPool(factory=TcpConnector, options=options) q = Queue() def runpool(): while True: try: data = q.get(False) except Empty: break try: with pool.connection() as conn: print("conn: pool size: %s" % pool.size) sent = conn.send(data) echo = conn.recv(1024) print("got %s" % data) assert data == echo finally: q.task_done() for i in range(20): q.put(s2b("Hello World %s" % i), False) for i in range(4): th = threading.Thread(target=runpool) th.daemnon = True th.start() q.join() print ("final pool size: %s" % pool.size) pool.release_all() server.shutdown() socketpool-0.5.3/examples/test_eventlet.py0000664000175000017500000000371412210132231022040 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import eventlet from socketpool.pool import ConnectionPool from socketpool.conn import TcpConnector # this handler will be run for each incoming connection in a dedicated greenlet class EchoServer(object): def __init__(self, host, port): self.host = host self.port = port self.spool = eventlet.GreenPool() self.running = False self.server = None def start(self): eventlet.spawn(self.run) def run(self): self.server = eventlet.listen((self.host, self.port)) self.running = True while self.running: try: sock, address = self.server.accept() print "accepted", address self.spool.spawn_n(self.handle, sock, address) except (SystemExit, KeyboardInterrupt): break def handle(self, sock, address): print ('New connection from %s:%s' % address) while True: data = sock.recv(1024) if not data: break sock.send(data) print ("echoed %r" % data) def stop(self): self.running = False if __name__ == '__main__': import time options = {'host': 'localhost', 'port': 6000} pool = ConnectionPool(factory=TcpConnector, options=options, backend="eventlet") server = EchoServer('localhost', 6000) server.start() epool = eventlet.GreenPool() def runpool(data): print 'ok' with pool.connection() as conn: print 'sending' sent = conn.send(data) print 'send %d bytes' % sent echo_data = conn.recv(1024) print "got %s" % data assert data == echo_data start = time.time() _ = [epool.spawn(runpool, "blahblah") for _ in xrange(20)] epool.waitall() server.stop() delay = time.time() - start socketpool-0.5.3/NOTICE0000664000175000017500000000051112210132231015657 0ustar benoitcbenoitc00000000000000socketpool ---------- 2012 (c) Benoît Chesneau 2012 (c) Tarek Ziade socketpool is available in the public domain (see UNLICENSE). socketpool is also optionally available under the MIT License (see LICENSE), meant especially for jurisdictions that do not recognize public domain works. socketpool-0.5.3/LICENSE0000664000175000017500000000213012210132231015757 0ustar benoitcbenoitc000000000000002012 (c) Benoît Chesneau 2012 (c) Tarek Ziade Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. socketpool-0.5.3/socketpool/0000775000175000017500000000000012210134336017150 5ustar benoitcbenoitc00000000000000socketpool-0.5.3/socketpool/backend_gevent.py0000664000175000017500000000214012210132231022446 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import gevent from gevent import select from gevent import socket from gevent import queue from socketpool.pool import ConnectionPool try: from gevent import lock except ImportError: #gevent < 1.0b2 from gevent import coros as lock sleep = gevent.sleep Semaphore = lock.BoundedSemaphore Socket = socket.socket Select = select.select class PriorityQueue(queue.PriorityQueue): def __next__(self): try: result = self.get(block=False) except queue.Empty: raise StopIteration return result next = __next__ class ConnectionReaper(gevent.Greenlet): running = False def __init__(self, pool, delay=150): self.pool = pool self.delay = delay gevent.Greenlet.__init__(self) def _run(self): self.running = True while True: gevent.sleep(self.delay) self.pool.murder_connections() def ensure_started(self): if not self.running or self.ready(): self.start() socketpool-0.5.3/socketpool/__init__.py0000664000175000017500000000030212210132231021244 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. from socketpool.pool import ConnectionPool from socketpool.conn import Connector, TcpConnector socketpool-0.5.3/socketpool/backend_thread.py0000664000175000017500000000225412210132231022433 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import select import socket import threading import time try: import Queue as queue except ImportError: # py3 import queue Select = select.select Socket = socket.socket sleep = time.sleep Semaphore = threading.BoundedSemaphore class PriorityQueue(queue.PriorityQueue): def __iter__(self): return self def __next__(self): try: result = self.get(block=False) except queue.Empty: raise StopIteration return result next = __next__ class ConnectionReaper(threading.Thread): """ connection reaper thread. Open a thread that will murder iddle connections after a delay """ running = False def __init__(self, pool, delay=600): self.pool = pool self.delay = delay threading.Thread.__init__(self) self.setDaemon(True) def run(self): self.running = True while True: time.sleep(self.delay) self.pool.murder_connections() def ensure_started(self): if not self.running and not self.isAlive(): self.start() socketpool-0.5.3/socketpool/pool.py0000664000175000017500000001461712210132231020474 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import contextlib import sys import time from socketpool.util import load_backend class MaxTriesError(Exception): pass class MaxConnectionsError(Exception): pass class ConnectionPool(object): """Pool of connections This is the main object to maintain connection. Connections are created using the factory instance passed as an option. Options: -------- :attr factory: Instance of socketpool.Connector. See socketpool.conn.TcpConnector for an example :attr retry_max: int, default 3. Numbr of times to retry a connection before raising the MaxTriesError exception. :attr max_lifetime: int, default 600. time in ms we keep a connection in the pool :attr max_size: int, default 10. Maximum number of connections we keep in the pool. :attr options: Options to pass to the factory :attr reap_connection: boolean, default is true. If true a process will be launched in background to kill idle connections. :attr backend: string, default is thread. The socket pool can use different backend to handle process and connections. For now the backends "thread", "gevent" and "eventlet" are supported. But you can add your own backend if you want. For an example of backend, look at the module socketpool.gevent_backend. """ def __init__(self, factory, retry_max=3, retry_delay=.1, timeout=-1, max_lifetime=600., max_size=10, options=None, reap_connections=True, reap_delay=1, backend="thread"): if isinstance(backend, str): self.backend_mod = load_backend(backend) self.backend = backend else: self.backend_mod = backend self.backend = str(getattr(backend, '__name__', backend)) self.max_size = max_size self.pool = getattr(self.backend_mod, 'PriorityQueue')() self._free_conns = 0 self.factory = factory self.retry_max = retry_max self.retry_delay = retry_delay self.timeout = timeout self.max_lifetime = max_lifetime if options is None: self.options = {"backend_mod": self.backend_mod, "pool": self} else: self.options = options self.options["backend_mod"] = self.backend_mod self.options["pool"] = self # bounded semaphore to make self._alive 'safe' self._sem = self.backend_mod.Semaphore(1) self._reaper = None if reap_connections: self.reap_delay = reap_delay self.start_reaper() def too_old(self, conn): return time.time() - conn.get_lifetime() > self.max_lifetime def murder_connections(self): current_pool_size = self.pool.qsize() if current_pool_size > 0: for priority, candidate in self.pool: current_pool_size -= 1 if not self.too_old(candidate): self.pool.put((priority, candidate)) else: self._reap_connection(candidate) if current_pool_size <= 0: break def start_reaper(self): self._reaper = self.backend_mod.ConnectionReaper(self, delay=self.reap_delay) self._reaper.ensure_started() def _reap_connection(self, conn): if conn.is_connected(): conn.invalidate() @property def size(self): return self.pool.qsize() def release_all(self): if self.pool.qsize(): for priority, conn in self.pool: self._reap_connection(conn) def release_connection(self, conn): if self._reaper is not None: self._reaper.ensure_started() with self._sem: if self.pool.qsize() < self.max_size: connected = conn.is_connected() if connected and not self.too_old(conn): self.pool.put((conn.get_lifetime(), conn)) else: self._reap_connection(conn) else: self._reap_connection(conn) def get(self, **options): options.update(self.options) found = None i = self.pool.qsize() tries = 0 last_error = None unmatched = [] while tries < self.retry_max: # first let's try to find a matching one from pool if self.pool.qsize(): for priority, candidate in self.pool: i -= 1 if self.too_old(candidate): # let's drop it self._reap_connection(candidate) continue matches = candidate.matches(**options) if not matches: # let's put it back unmatched.append((priority, candidate)) else: if candidate.is_connected(): found = candidate break else: # conn is dead for some reason. # reap it. self._reap_connection(candidate) if i <= 0: break if unmatched: for candidate in unmatched: self.pool.put(candidate) # we got one.. we use it if found is not None: return found try: new_item = self.factory(**options) except Exception as e: last_error = e else: # we should be connected now if new_item.is_connected(): with self._sem: return new_item tries += 1 self.backend_mod.sleep(self.retry_delay) if last_error is None: raise MaxTriesError() else: raise last_error @contextlib.contextmanager def connection(self, **options): conn = self.get(**options) try: yield conn # what to do in case of success except Exception as e: conn.handle_exception(e) finally: self.release_connection(conn) socketpool-0.5.3/socketpool/backend_eventlet.py0000664000175000017500000000232512210132231023011 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import eventlet from eventlet.green import select from eventlet.green import socket from eventlet import queue from socketpool.pool import ConnectionPool sleep = eventlet.sleep Socket = socket.socket Select = select.select Semaphore = eventlet.semaphore.BoundedSemaphore class PriorityQueue(queue.PriorityQueue): def __iter__(self): return self def __next__(self): try: result = self.get(block=False) except queue.Empty: raise StopIteration return result next = __next__ class ConnectionReaper(object): running = False def __init__(self, pool, delay=150): self.pool = pool self.delay = delay def start(self): self.running = True g = eventlet.spawn(self._exec) g.link(self._exit) def _exit(self, g): try: g.wait() except: pass self.running = False def _exec(self): while True: eventlet.sleep(self.delay) self.pool.murder_connections() def ensure_started(self): if not self.running: self.start() socketpool-0.5.3/socketpool/conn.py0000664000175000017500000000405112210132231020447 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import select import socket import time import random from socketpool import util class Connector(object): def matches(self, **match_options): raise NotImplementedError() def is_connected(self): raise NotImplementedError() def handle_exception(self, exception): raise NotImplementedError() def get_lifetime(self): raise NotImplementedError() def invalidate(self): raise NotImplementedError() class TcpConnector(Connector): def __init__(self, host, port, backend_mod, pool=None): self._s = backend_mod.Socket(socket.AF_INET, socket.SOCK_STREAM) self._s.connect((host, port)) self.host = host self.port = port self.backend_mod = backend_mod self._connected = True # use a 'jiggle' value to make sure there is some # randomization to expiry, to avoid many conns expiring very # closely together. self._life = time.time() - random.randint(0, 10) self._pool = pool def __del__(self): self.release() def matches(self, **match_options): target_host = match_options.get('host') target_port = match_options.get('port') return target_host == self.host and target_port == self.port def is_connected(self): if self._connected: return util.is_connected(self._s) return False def handle_exception(self, exception): print('got an exception') print(str(exception)) def get_lifetime(self): return self._life def invalidate(self): self._s.close() self._connected = False self._life = -1 def release(self): if self._pool is not None: if self._connected: self._pool.release_connection(self) else: self._pool = None def send(self, data): return self._s.send(data) def recv(self, size=1024): return self._s.recv(size) socketpool-0.5.3/socketpool/util.py0000664000175000017500000001067512210132231020500 0ustar benoitcbenoitc00000000000000# -*- coding: utf-8 - # # This file is part of socketpool. # See the NOTICE for more information. import errno import os import platform import select import socket import sys try: from importlib import import_module except ImportError: import sys def _resolve_name(name, package, level): """Return the absolute name of the module to be imported.""" if not hasattr(package, 'rindex'): raise ValueError("'package' not set to a string") dot = len(package) for x in range(level, 1, -1): try: dot = package.rindex('.', 0, dot) except ValueError: raise ValueError("attempted relative import beyond top-level " "package") return "%s.%s" % (package[:dot], name) def import_module(name, package=None): """Import a module. The 'package' argument is required when performing a relative import. It specifies the package to use as the anchor point from which to resolve the relative import to an absolute import. """ if name.startswith('.'): if not package: raise TypeError("relative imports require the 'package' argument") level = 0 for character in name: if character != '.': break level += 1 name = _resolve_name(name[level:], package, level) __import__(name) return sys.modules[name] def load_backend(backend_name): """ load pool backend. If this is an external module it should be passed as "somelib.backend_mod", for socketpool backend you can just pass the name. Supported backend are : - thread: connection are maintained in a threadsafe queue. - gevent: support gevent - eventlet: support eventlet """ try: if len(backend_name.split(".")) > 1: mod = import_module(backend_name) else: mod = import_module("socketpool.backend_%s" % backend_name) return mod except ImportError: error_msg = "%s isn't a socketpool backend" % backend_name raise ImportError(error_msg) def can_use_kqueue(): # See Issue #15. kqueue doesn't work on OS X 10.6 and below. if not hasattr(select, "kqueue"): return False if platform.system() == 'Darwin' and platform.mac_ver()[0] < '10.7': return False return True def is_connected(skt): try: fno = skt.fileno() except socket.error as e: if e[0] == errno.EBADF: return False raise try: if hasattr(select, "epoll"): ep = select.epoll() ep.register(fno, select.EPOLLOUT | select.EPOLLIN) events = ep.poll(0) for fd, ev in events: if fno == fd and \ (ev & select.EPOLLOUT or ev & select.EPOLLIN): ep.unregister(fno) return True ep.unregister(fno) elif hasattr(select, "poll"): p = select.poll() p.register(fno, select.POLLOUT | select.POLLIN) events = p.poll(0) for fd, ev in events: if fno == fd and \ (ev & select.POLLOUT or ev & select.POLLIN): p.unregister(fno) return True p.unregister(fno) elif can_use_kqueue(): kq = select.kqueue() events = [ select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_ADD), select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_ADD) ] kq.control(events, 0) kevents = kq.control(None, 4, 0) for ev in kevents: if ev.ident == fno: if ev.flags & select.KQ_EV_ERROR: return False else: return True # delete events = [ select.kevent(fno, select.KQ_FILTER_READ, select.KQ_EV_DELETE), select.kevent(fno, select.KQ_FILTER_WRITE, select.KQ_EV_DELETE) ] kq.control(events, 0) kq.close() return True else: r, _, _ = select.select([fno], [], [], 0) if not r: return True except IOError: pass except (ValueError, select.error,) as e: pass return False socketpool-0.5.3/UNLICENSE0000664000175000017500000000227312210132231016232 0ustar benoitcbenoitc00000000000000This is free and unencumbered software released into the public domain. Anyone is free to copy, modify, publish, use, compile, sell, or distribute this software, either in source code form or as a compiled binary, for any purpose, commercial or non-commercial, and by any means. In jurisdictions that recognize copyright laws, the author or authors of this software dedicate any and all copyright interest in the software to the public domain. We make this dedication for the benefit of the public at large and to the detriment of our heirs and successors. We intend this dedication to be an overt act of relinquishment in perpetuity of all present and future rights to this software under copyright law. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. For more information, please refer to