pysendfile-2.0.0/0000775000175000017500000000000011703650243015620 5ustar giampaologiampaolo00000000000000pysendfile-2.0.0/setup.py0000664000175000017500000000714011703650167017341 0ustar giampaologiampaolo00000000000000#!/usr/bin/env python # $Id: setup.py 113 2012-01-12 21:28:21Z g.rodola@gmail.com $ # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2012 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== import sys try: from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension if sys.version_info < (2, 5): sys.exit('python version not supported (< 2.5)') if 'sunos' in sys.platform: libraries = ["sendfile"] else: libraries = [] name = 'pysendfile' version = '2.0.0' download_url = "http://pysendfile.googlecode.com/files/" + name + "-" + \ version + ".tar.gz" def main(): setup(name=name, url='http://code.google.com/p/pysendfile/', version=version, description='A Python interface to sendfile(2)', long_description=open('README', 'r').read(), author='Giampaolo Rodola', author_email='g.rodola@gmail.com', download_url=download_url, platforms='UNIX', license='MIT', keywords=['sendfile', 'python', 'performance', 'ftp'], classifiers = [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Operating System :: POSIX :: SunOS/Solaris', 'Operating System :: POSIX :: AIX', 'Programming Language :: C', 'Programming Language :: Python :: 2.4', 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.0', 'Programming Language :: Python :: 3.1', 'Programming Language :: Python :: 3.2', 'Topic :: System :: Networking', 'Topic :: System :: Operating System', 'Topic :: Internet :: File Transfer Protocol (FTP)', 'Topic :: Internet :: WWW/HTTP', 'License :: OSI Approved :: MIT License', ], ext_modules = [Extension('sendfile', sources=['sendfilemodule.c'], libraries=libraries)], ) main() pysendfile-2.0.0/sendfilemodule.c0000664000175000017500000002523611703630147020774 0ustar giampaologiampaolo00000000000000/* * $Id: sendfilemodule.c 101 2012-01-10 23:27:51Z g.rodola@gmail.com $ */ /* * pysendfile * * A Python module interface to sendfile(2) * * Original author: * Copyright (C) 2005 Ben Woolley * * The AIX support code is: * Copyright (C) 2008,2009 Niklas Edmundsson * * Rewritten from scratch and maintained by Giampaolo Rodola' * Copyright (C) 2009,2012 * * * The MIT License * * Copyright (c) <2011> * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ #include #include // force a compilation error if platform is not supported #if !defined(__FreeBSD__) && \ !defined(__DragonFly__) && \ !defined(__APPLE__) && \ !defined(_AIX) && \ !defined(__linux__) && \ !defined(__sun) #error platfom not supported #endif static int _parse_off_t(PyObject* arg, void* addr) { #if !defined(HAVE_LARGEFILE_SUPPORT) *((off_t*)addr) = PyLong_AsLong(arg); #else *((off_t*)addr) = PyLong_AsLongLong(arg); #endif if (PyErr_Occurred()) return 0; return 1; } /* --- begin FreeBSD / Dragonfly / OSX --- */ #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) #include #include #include // needed on OSX - Python < 2.6 #if defined(__APPLE__) && defined(_POSIX_C_SOURCE) struct sf_hdtr { struct iovec *headers; int hdr_cnt; struct iovec *trailers; int trl_cnt; }; #endif static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int fd; int sock; int flags = 0; off_t offset; Py_ssize_t ret; Py_ssize_t nbytes; char * head = NULL; char * tail = NULL; Py_ssize_t head_len = 0; Py_ssize_t tail_len = 0; off_t sent; struct sf_hdtr hdtr; PyObject *offobj; static char *keywords[] = {"out", "in", "offset", "nbytes", "header", "trailer", "flags", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiOn|s#s#i:sendfile", keywords, &fd, &sock, &offobj, &nbytes, &head, &head_len, &tail, &tail_len, &flags)) { return NULL; } if (!_parse_off_t(offobj, &offset)) return NULL; #ifdef __APPLE__ sent = nbytes; #endif if (head_len != 0 || tail_len != 0) { struct iovec ivh = {head, head_len}; struct iovec ivt = {tail, tail_len}; hdtr.headers = &ivh; hdtr.hdr_cnt = 1; hdtr.trailers = &ivt; hdtr.trl_cnt = 1; Py_BEGIN_ALLOW_THREADS #ifdef __APPLE__ sent += head_len; ret = sendfile(sock, fd, offset, &sent, &hdtr, flags); #else ret = sendfile(sock, fd, offset, nbytes, &hdtr, &sent, flags); #endif Py_END_ALLOW_THREADS } else { Py_BEGIN_ALLOW_THREADS #ifdef __APPLE__ ret = sendfile(sock, fd, offset, &sent, NULL, flags); #else ret = sendfile(sock, fd, offset, nbytes, NULL, &sent, flags); #endif Py_END_ALLOW_THREADS } if (ret < 0) { if ((errno == EAGAIN) || (errno == EBUSY) || (errno == EWOULDBLOCK)) { if (sent != 0) { // some data has been sent errno = 0; goto done; } else { // no data has been sent; upper application is supposed // to retry on EAGAIN / EBUSY / EWOULDBLOCK PyErr_SetFromErrno(PyExc_OSError); return NULL; } } PyErr_SetFromErrno(PyExc_OSError); return NULL; } goto done; done: #if defined(HAVE_LARGEFILE_SUPPORT) return Py_BuildValue("L", sent); #else return Py_BuildValue("l", sent); #endif } /* --- end OSX / FreeBSD / Dragonfly --- */ /* --- begin AIX --- */ #elif defined(_AIX) #include static PyObject * method_sendfile(PyObject *self, PyObject *args) { int out_fd, in_fd; off_t offset; size_t nbytes; char *hdr=NULL, *trail=NULL; int hdrsize, trailsize; ssize_t sts=0; struct sf_parms sf_iobuf; int rc; if (!PyArg_ParseTuple(args, "iiLk|s#s#", &out_fd, &in_fd, &offset, &nbytes, &hdr, &hdrsize, &trail, &trailsize)) return NULL; if(hdr != NULL) { sf_iobuf.header_data = hdr; sf_iobuf.header_length = hdrsize; } else { sf_iobuf.header_data = NULL; sf_iobuf.header_length = 0; } if(trail != NULL) { sf_iobuf.trailer_data = trail; sf_iobuf.trailer_length = trailsize; } else { sf_iobuf.trailer_data = NULL; sf_iobuf.trailer_length = 0; } sf_iobuf.file_descriptor = in_fd; sf_iobuf.file_offset = offset; sf_iobuf.file_bytes = nbytes; Py_BEGIN_ALLOW_THREADS; do { sf_iobuf.bytes_sent = 0; /* Really needed? */ rc = send_file(&out_fd, &sf_iobuf, SF_DONT_CACHE); sts += sf_iobuf.bytes_sent; } while( rc == 1 || (rc == -1 && errno == EINTR) ); Py_END_ALLOW_THREADS; offset = sf_iobuf.file_offset; if (rc == -1) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } else { #if defined(HAVE_LARGEFILE_SUPPORT) return Py_BuildValue("L", sts); #else return Py_BuildValue("l", sts); #endif } } /* --- end AIX --- */ /* --- start Linux --- */ #elif defined (__linux__) #include static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int out_fd, in_fd; off_t offset; Py_ssize_t nbytes; Py_ssize_t sent; PyObject *offobj; if (!PyArg_ParseTuple(args, "iiOn", &out_fd, &in_fd, &offobj, &nbytes)) { return NULL; } if (offobj == Py_None) { Py_BEGIN_ALLOW_THREADS; sent = sendfile(out_fd, in_fd, NULL, nbytes); Py_END_ALLOW_THREADS; } else { if (!_parse_off_t(offobj, &offset)) return NULL; Py_BEGIN_ALLOW_THREADS; sent = sendfile(out_fd, in_fd, &offset, nbytes); Py_END_ALLOW_THREADS; } if (sent == -1) return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("n", sent); } /* --- end Linux --- */ /* --- begin SUN OS --- */ #elif defined(__sun) #include static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int out_fd; int in_fd; off_t offset; Py_ssize_t nbytes; Py_ssize_t sent; PyObject *offobj; if (!PyArg_ParseTuple(args, "iiOn", &out_fd, &in_fd, &offobj, &nbytes)) { return NULL; } if (!_parse_off_t(offobj, &offset)) return NULL; sent = sendfile(out_fd, in_fd, &offset, nbytes); if (sent == -1) return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("n", sent); } #else /* --- end SUN OS --- */ #error platfom not supported #endif /* --- module initialization --- */ struct module_state { PyObject *error; }; #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif static PyMethodDef sendfile_methods[] = { {"sendfile", (PyCFunction)method_sendfile, METH_VARARGS | METH_KEYWORDS, "sendfile(out, in, offset, nbytes, header=\"\", trailer=\"\", flags=0)\n\n" "Copy nbytes bytes from file descriptor in (a regular file) to\n" "file descriptor out (a socket) starting at offset.\n" "Return the number of bytes just being sent. When the end of\n" "file is reached return 0.\n" "On Linux, if offset is given as None, the bytes are read from\n" "the current position of in and the position of in is updated.\n" "headers and trailers are strings that are written before and\n" "after the data from in is written. In cross platform applications\n" "their usage is discouraged (socket.send() or socket.sendall()\n" "can be used instead).\n" "On Solaris, out may be the file descriptor of a regular file\n" "or the file descriptor of a socket. On all other platforms,\n" "out must be the file descriptor of an open socket.\n" "flags argument is only supported on FreeBSD.\n" }, {NULL, NULL} }; #if PY_MAJOR_VERSION >= 3 static int sendfile_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->error); return 0; } static int sendfile_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "sendfile", NULL, sizeof(struct module_state), sendfile_methods, NULL, sendfile_traverse, sendfile_clear, NULL }; #define INITERROR return NULL PyObject * PyInit_sendfile(void) #else #define INITERROR return void initsendfile(void) #endif { #if PY_MAJOR_VERSION >= 3 PyObject *module = PyModule_Create(&moduledef); #else PyObject *module = Py_InitModule("sendfile", sendfile_methods); #endif #ifdef SF_NODISKIO PyModule_AddIntConstant(module, "SF_NODISKIO", SF_NODISKIO); #endif #ifdef SF_MNOWAIT PyModule_AddIntConstant(module, "SF_MNOWAIT", SF_MNOWAIT); #endif #ifdef SF_SYNC PyModule_AddIntConstant(module, "SF_SYNC", SF_SYNC); #endif if (module == NULL) INITERROR; struct module_state *st = GETSTATE(module); st->error = PyErr_NewException("sendfile.Error", NULL, NULL); if (st->error == NULL) { Py_DECREF(module); INITERROR; } #if PY_MAJOR_VERSION >= 3 return module; #endif } pysendfile-2.0.0/HISTORY0000664000175000017500000000162511703642277016720 0ustar giampaologiampaolo00000000000000Bug tracker at http://code.google.com/p/py-sendfile/issues/list Version 2.0.0 - 2012-01-12 ========================== (Giampaolo Rodolà took over maintenance) ##: complete rewriting except AIX code ##: non blocking sockets support ##: threads support (release GIL) #1: unit tests #2: python 3 support #3: FreeBSD implementation is broken #4: python large file support #5: header/trailer are now keyword arguments #6: exposed SF_NODISKIO, SF_MNOWAIT and SF_SYNC constants on FreeBSD #8: benchmark script #10: Mac OSX support #13: Sun OS support Version 1.2.4 - 2009-03-06 ========================== (Stephan Peijnik took over maintenance) ## Add AIX support. Version 1.2.3 - 2008-04-09 ========================== ## Use setuptools instead of distutils. Version 1.2.2 - 2008-03-29 ========================== (Ben Woolley) ## First release including support for Linux, FreeBSD and DragonflyBSD platforms. pysendfile-2.0.0/LICENSE0000664000175000017500000000264111703621773016636 0ustar giampaologiampaolo00000000000000====================================================================== This software is distributed under the MIT license reproduced below: Original author: Copyright (C) 2005-2008 Ben Woolley AIX support code by: Copyright (C) 2008-2009 Niklas Edmundsson Rewritten from scratch and maintained by: Copyright (C) 2009-2012 Giampaolo Rodola' Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Giampaolo Rodola' not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. Ben Woolley, Niklas Edmundsson and Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ====================================================================== pysendfile-2.0.0/test/0000775000175000017500000000000011703650243016577 5ustar giampaologiampaolo00000000000000pysendfile-2.0.0/test/benchmark.py0000664000175000017500000001721111703627777021125 0ustar giampaologiampaolo00000000000000#!/usr/bin/env python # $Id: benchmark.py 109 2012-01-12 19:09:48Z g.rodola@gmail.com $ # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2012 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """ A simle benchmark script which compares plain send() and sendfile() performances in terms of CPU time spent and bytes transmitted in one second. This is what I get on my Linux 2.6.38 box, AMD dual core 1.6 GHz: send() cpu: 28.41 usec/pass rate: 362.13 MB/sec sendfile() cpu: 11.25 usec/pass rate: 848.56 MB/sec Works with both python 2.X and 3.X. """ from __future__ import with_statement import socket import os import errno import timeit import time import atexit import sys import optparse import threading import itertools import signal import contextlib from multiprocessing import Process from sendfile import sendfile # overridable defaults HOST = "127.0.0.1" PORT = 8022 BIGFILE = "$testfile1" BIGFILE_SIZE = 1024 * 1024 * 1024 # 1 GB BUFFER_SIZE = 65536 # python 3 compatibility layer def b(s): return bytes(s, 'ascii') if sys.version_info >= (3,) else s # python 2.5 compatibility try: next except NameError: def next(iterator): return iterator.next() def print_(s, hilite=False): if hilite: bold = '1' s = '\x1b[%sm%s\x1b[0m' % (';'.join([bold]), s) sys.stdout.write(s + "\n") sys.stdout.flush() def create_file(filename, size): with open(filename, 'wb') as f: bytes = 0 chunk = b("x") * BUFFER_SIZE while 1: f.write(chunk) bytes += len(chunk) if bytes >= size: break def safe_remove(file): try: os.remove(file) except OSError: pass class Spinner(threading.Thread): def run(self): self._exit = False self._spinner = itertools.cycle('-\|/') while not self._exit: sys.stdout.write(next(self._spinner) + "\b") sys.stdout.flush() time.sleep(.1) def stop(self): self._exit = True self.join() class Client: def __init__(self): self.sock = socket.socket() self.sock.connect((HOST, PORT)) self.sock.settimeout(1) def retr(self): with contextlib.closing(self.sock): while 1: data = self.sock.recv(BUFFER_SIZE) if not data: break def retr_for_1_sec(self): with contextlib.closing(self.sock): stop_at = time.time() + 1 bytes_recv = 0 while stop_at > time.time(): chunk = self.sock.recv(BUFFER_SIZE) if not chunk: assert 0 bytes_recv += len(chunk) return bytes_recv def start_server(use_sendfile, keep_sending=False): """A simple test server which sends a file once a client connects. use_sendfile decides whether using sendfile() or plain send(). If keep_sending is True restart sending file when EOF is reached. """ sock = socket.socket() sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((HOST, PORT)) sock.listen(1) conn, addr = sock.accept() sock.close() file = open(BIGFILE, 'rb') def on_exit(signum, fram): file.close(); conn.close() sys.exit(0) signal.signal(signal.SIGTERM, on_exit) signal.signal(signal.SIGINT, on_exit) if not use_sendfile: while 1: chunk = file.read(BUFFER_SIZE) if not chunk: # EOF if keep_sending: file.seek(0) continue else: break conn.sendall(chunk) else: offset = 0 sockno = conn.fileno() fileno = file.fileno() while 1: try: sent = sendfile(sockno, fileno, offset, BUFFER_SIZE) except OSError: err = sys.exc_info()[1] if err.errno in (errno.EAGAIN, errno.EBUSY): continue raise else: if not sent: # EOF if keep_sending: offset = 0 continue else: break else: offset += sent def main(): parser = optparse.OptionParser() parser.add_option('-k', '--keepfile', action="store_true", default=False, help="do not remove test file on exit") options, args = parser.parse_args() if not options.keepfile: atexit.register(lambda: safe_remove(BIGFILE)) if not os.path.exists(BIGFILE) or os.path.getsize(BIGFILE) < BIGFILE_SIZE: print_("creating big file...") create_file(BIGFILE, BIGFILE_SIZE) print_("starting benchmark...") # CPU time: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile":True}) server.start() time.sleep(0.1) t1 = timeit.Timer(setup="from __main__ import Client; client = Client()", stmt="client.retr()").timeit(number=1) server.terminate() server.join() # CPU time: use send() server = Process(target=start_server, kwargs={"use_sendfile":False}) server.start() time.sleep(0.1) t2 = timeit.Timer(setup="from __main__ import Client; client = Client()", stmt="client.retr()").timeit(number=1) server.terminate() server.join() # MB/sec: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile":True, "keep_sending":True}) server.start() time.sleep(0.1) client = Client() bytes1 = client.retr_for_1_sec() server.terminate() server.join() # MB/sec: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile":False, "keep_sending":True}) server.start() time.sleep(0.1) client = Client() bytes2 = client.retr_for_1_sec() server.terminate() server.join() print_(" ") print_("send()", hilite=True) print_(" cpu: %7.2f usec/pass" % (1000000 * t2 / 100000)) print_(" rate: %7.2f MB/sec" % round(bytes2 / 1024.0 / 1024.0, 2)) print_("") print_("sendfile()", hilite=True) print_(" cpu: %7.2f usec/pass" % (1000000 * t1 / 100000)) print_(" rate: %7.2f MB/sec" % round(bytes1 / 1024.0 / 1024.0, 2)) if __name__ == '__main__': s = Spinner() s.start() try: main() finally: s.stop() pysendfile-2.0.0/test/test_sendfile.py0000664000175000017500000004250411703627777022026 0ustar giampaologiampaolo00000000000000#!/usr/bin/env python # # $Id: test_sendfile.py 109 2012-01-12 19:09:48Z g.rodola@gmail.com $ # # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2012 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """ pysendfile test suite; works with both python 2.x and 3.x (no need to run 2to3 tool first). Accepts a -k cmdline option to avoid removing the big file created during tests. """ from __future__ import with_statement import unittest import os import sys import socket import asyncore import asynchat import threading import errno import time import atexit import warnings import optparse import sendfile PY3 = sys.version_info >= (3,) def b(x): if PY3: return bytes(x, 'ascii') return x TESTFN = "$testfile" TESTFN2 = TESTFN + "2" TESTFN3 = TESTFN + "3" DATA = b("12345abcde" * 1024 * 1024) # 10 Mb HOST = '127.0.0.1' BIGFILE_SIZE = 2500000000 # > 2GB file (2GB = 2147483648 bytes) BUFFER_LEN = 4096 try: sendfile.sendfile(0, 0, 0, 0, 0) except TypeError: SUPPORT_HEADER_TRAILER = False except Exception: SUPPORT_HEADER_TRAILER = True def safe_remove(file): try: os.remove(file) except OSError: pass def has_large_file_support(): # taken from Python's Lib/test/test_largefile.py with open(TESTFN, 'wb', buffering=0) as f: try: f.seek(BIGFILE_SIZE) # seeking is not enough of a test: you must write and flush too f.write(b('x')) f.flush() except (IOError, OverflowError): return False else: return True class Handler(asynchat.async_chat): ac_in_buffer_size = BUFFER_LEN ac_out_buffer_size = BUFFER_LEN def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.in_buffer = [] self.closed = False self.push(b("220 ready\r\n")) def handle_read(self): data = self.recv(BUFFER_LEN) self.in_buffer.append(data) def get_data(self): return b('').join(self.in_buffer) def handle_close(self): self.close() def close(self): asynchat.async_chat.close(self) self.closed = True def handle_error(self): raise class NoMemoryHandler(Handler): # same as above but doesn't store received data in memory ac_in_buffer_size = 65536 def __init__(self, conn): Handler.__init__(self, conn) self.in_buffer_len = 0 def handle_read(self): data = self.recv(self.ac_in_buffer_size) self.in_buffer_len += len(data) def get_data(self): raise NotImplementedError class Server(asyncore.dispatcher, threading.Thread): handler = Handler def __init__(self, address): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.bind(address) self.listen(5) self.host, self.port = self.socket.getsockname()[:2] self.handler_instance = None self._active = False self._active_lock = threading.Lock() # --- public API @property def running(self): return self._active def start(self): assert not self.running self.__flag = threading.Event() threading.Thread.start(self) self.__flag.wait() def stop(self): assert self.running self._active = False self.join() assert not asyncore.socket_map, asyncore.socket_map def wait(self): # wait for handler connection to be closed, then stop the server while not getattr(self.handler_instance, "closed", True): time.sleep(0.001) self.stop() # --- internals def run(self): self._active = True self.__flag.set() while self._active and asyncore.socket_map: self._active_lock.acquire() asyncore.loop(timeout=0.001, count=1) self._active_lock.release() asyncore.close_all() def handle_accept(self): conn, addr = self.accept() self.handler_instance = self.handler(conn) def handle_connect(self): self.close() handle_read = handle_connect def writable(self): return 0 def handle_error(self): raise def sendfile_wrapper(sock, file, offset, nbytes=BUFFER_LEN, header="", trailer=""): """A higher level wrapper representing how an application is supposed to use sendfile(). """ while 1: try: if SUPPORT_HEADER_TRAILER: sent = sendfile.sendfile(sock, file, offset, nbytes, header, trailer) else: sent = sendfile.sendfile(sock, file, offset, nbytes) except OSError: err = sys.exc_info()[1] if err.errno == errno.EAGAIN: # retry continue raise else: assert sent <= nbytes, (sent, nbytes) return sent class TestSendfile(unittest.TestCase): def setUp(self): self.server = Server((HOST, 0)) self.server.start() self.client = socket.socket() self.client.connect((self.server.host, self.server.port)) self.client.settimeout(1) # synchronize by waiting for "220 ready" response self.client.recv(1024) self.sockno = self.client.fileno() self.file = open(TESTFN, 'rb') self.fileno = self.file.fileno() def tearDown(self): safe_remove(TESTFN2) self.file.close() self.client.close() if self.server.running: self.server.stop() self.server = None # allow garbage collection def test_send_whole_file(self): # normal send total_sent = 0 offset = 0 while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break total_sent += sent offset += sent self.assertEqual(offset, total_sent) self.assertEqual(total_sent, len(DATA)) self.client.close() if "sunos" in sys.platform: time.sleep(.1) self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) def test_send_from_certain_offset(self): # start sending a file at a certain offset total_sent = 0 offset = int(len(DATA) / 2) while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break total_sent += sent offset += sent self.client.close() if "sunos" in sys.platform: time.sleep(.1) self.server.wait() data = self.server.handler_instance.get_data() expected = DATA[int(len(DATA) / 2):] self.assertEqual(total_sent, len(expected)) self.assertEqual(hash(data), hash(expected)) if SUPPORT_HEADER_TRAILER: def test_header(self): total_sent = 0 header = b("x") * 512 sent = sendfile.sendfile(self.sockno, self.fileno, 0, header=header) total_sent += sent offset = BUFFER_LEN while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break offset += sent total_sent += sent expected_data = header + DATA self.assertEqual(total_sent, len(expected_data)) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(expected_data)) self.assertEqual(hash(data), hash(expected_data)) def test_trailer(self): with open(TESTFN2, 'wb') as f: f.write(b("abcde")) with open(TESTFN2, 'rb') as f: sendfile.sendfile(self.sockno, f.fileno(), 0, trailer=b("12345")) time.sleep(.1) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(data, b("abcde12345")) def test_non_socket(self): fd_in = open(TESTFN, 'rb') fd_out = open(TESTFN2, 'wb') try: sendfile.sendfile(fd_in.fileno(), fd_out.fileno(), 0, BUFFER_LEN) except OSError: err = sys.exc_info()[1] self.assertEqual(err.errno, errno.EBADF) else: self.fail("exception not raised") finally: fd_in.close() fd_out.close() if sys.platform.startswith('freebsd'): def test_send_nbytes_0(self): # On Mac OS X and FreeBSD a value of 0 for nbytes # is supposed to send the whole file in one shot. # OSX implementation appears to be just broken. # On *BSD this works most of the times: sometimes # EAGAIN is returned internally and here we get the # number of bytes sent. sent = sendfile.sendfile(self.sockno, self.fileno, 0, 0) if sent == len(DATA): self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) if hasattr(sendfile, "SF_NODISKIO"): def test_flags(self): try: sendfile.sendfile(self.sockno, self.fileno, 0, BUFFER_LEN, flags=sendfile.SF_NODISKIO) except OSError: err = sys.exc_info()[1] if err.errno not in (errno.EBUSY, errno.EAGAIN): raise # --- corner cases def test_offset_overflow(self): # specify an offset > file size offset = len(DATA) + BUFFER_LEN sent = sendfile.sendfile(self.sockno, self.fileno, offset, BUFFER_LEN) self.assertEqual(sent, 0) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(data, b('')) if "sunos" not in sys.platform: def test_invalid_offset(self): try: sendfile.sendfile(self.sockno, self.fileno, -1, BUFFER_LEN) except OSError: err = sys.exc_info()[1] self.assertEqual(err.errno, errno.EINVAL) else: self.fail("exception not raised") def test_small_file(self): data = b('foo bar') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: offset = 0 while 1: sent = sendfile_wrapper(self.sockno, f.fileno(), offset) if sent == 0: break offset += sent self.client.close() time.sleep(.1) self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, data) def test_small_file_and_offset_overflow(self): data = b('foo bar') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: sendfile_wrapper(self.sockno, f.fileno(), 4096) self.client.close() self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, b('')) def test_empty_file(self): data = b('') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: sendfile_wrapper(self.sockno, f.fileno(), 0) self.client.close() self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, data) if "linux" in sys.platform: def test_offset_none(self): # on Linux offset == None sendfile() call is supposed # to update the file offset while 1: sent = sendfile_wrapper(self.sockno, self.fileno, None) if sent == 0: break self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) class RepeatedTimer: def __init__(self, timeout, fun): self.timeout = timeout self.fun = fun def start(self): def main(): self.fun() self.start() self.timer = threading.Timer(1, main) self.timer.start() def stop(self): self.timer.cancel() class TestLargeFile(unittest.TestCase): def setUp(self): self.server = Server((HOST, 0)) self.server.handler = NoMemoryHandler self.server.start() self.client = socket.socket() self.client.connect((self.server.host, self.server.port)) self.client.settimeout(1) # synchronize by waiting for "220 ready" response self.client.recv(1024) self.sockno = self.client.fileno() sys.stdout.write("\ncreating file:\n") sys.stdout.flush() self.create_file() self.file = open(TESTFN3, 'rb') self.fileno = self.file.fileno() sys.stdout.write("\nstarting transfer:\n") sys.stdout.flush() def tearDown(self): if hasattr(self, 'file'): self.file.close() self.client.close() if self.server.running: self.server.stop() def print_percent(self, a, b): percent = ((a * 100) / b) sys.stdout.write("\r%2d%%" % percent) sys.stdout.flush() def create_file(self): if os.path.isfile(TESTFN3) and os.path.getsize(TESTFN3) >= BIGFILE_SIZE: return f = open(TESTFN3, 'wb') chunk_len = 65536 chunk = b('x' * chunk_len) total = 0 timer = RepeatedTimer(1, lambda: self.print_percent(total, BIGFILE_SIZE)) timer.start() try: while 1: f.write(chunk) total += chunk_len if total >= BIGFILE_SIZE: break finally: f.close() timer.stop() def test_big_file(self): total_sent = 0 offset = 0 nbytes = 65536 file_size = os.path.getsize(TESTFN3) timer = RepeatedTimer(1, lambda: self.print_percent(total_sent, file_size)) timer.start() try: while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) if sent == 0: break total_sent += sent offset += sent self.assertEqual(offset, total_sent) except Exception: print raise finally: sys.stdout.write("\n") sys.stdout.flush() timer.stop() self.assertEqual(total_sent, file_size) self.client.close() if "sunos" in sys.platform: time.sleep(1) self.server.wait() data_len = self.server.handler_instance.in_buffer_len file_size = os.path.getsize(TESTFN3) self.assertEqual(file_size, data_len) def test_main(): parser = optparse.OptionParser() parser.add_option('-k', '--keepfile', action="store_true", default=False, help="do not remove test big file on exit") options, args = parser.parse_args() if not options.keepfile: atexit.register(lambda: safe_remove(TESTFN3)) def cleanup(): safe_remove(TESTFN) safe_remove(TESTFN2) atexit.register(cleanup) test_suite = unittest.TestSuite() test_suite.addTest(unittest.makeSuite(TestSendfile)) if has_large_file_support(): test_suite.addTest(unittest.makeSuite(TestLargeFile)) else: atexit.register(warnings.warn, "large files unsupported", RuntimeWarning) cleanup() with open(TESTFN, "wb") as f: f.write(DATA) unittest.TextTestRunner(verbosity=2).run(test_suite) if __name__ == '__main__': test_main() pysendfile-2.0.0/README0000664000175000017500000000205411703642163016503 0ustar giampaologiampaolo00000000000000=========== Quick links =========== * Home page: http://code.google.com/p/pysendfile * Download: http://code.google.com/p/pysendfile/downloads/list ===== About ===== A python interface to sendfile(2) system call. ======= Install ======= $ sudo setup.py install ...or: $ easy_install pysendfile =================== Supported platforms =================== * Linux * OSX * FreeBSD * Dragon Fly BSD * SunOS * AIX (non properly tested) Python versions from 2.5 to 3.3 by using a single code base. ============= Example usage ============= :: import socket import errno from sendfile import sendfile file = open("somefile", "rb") sock = socket.socket() sock.connect(("127.0.0.1", 8021)) offset = 0 while 1: try: sent = sendfile(sock.fileno(), file.fileno(), offset, 4096) except OSError, err: if err.errno == (errno.EAGAIN, errno.EBUSY): # retry continue raise else: if sent == 0: break # done offset += sent