pax_global_header00006660000000000000000000000064123410707160014513gustar00rootroot0000000000000052 comment=98e57424ea36c050afda19deaaec3a1b3bc49e84 pysendfile-release-2.0.1/000077500000000000000000000000001234107071600152735ustar00rootroot00000000000000pysendfile-release-2.0.1/.travis.yml000066400000000000000000000006501234107071600174050ustar00rootroot00000000000000language: python python: - 2.6 - 2.7 - 3.2 - 3.3 - 3.4 - pypy install: - if [[ $TRAVIS_PYTHON_VERSION == '2.5' ]]; then pip install unittest2; fi - if [[ $TRAVIS_PYTHON_VERSION == '2.6' ]]; then pip install unittest2; fi script: - pip install flake8 - python setup.py build - python setup.py install - python test/test_sendfile.py - make flake8 os: - linux - osx pysendfile-release-2.0.1/HISTORY.rst000066400000000000000000000024011234107071600171630ustar00rootroot00000000000000Bug tracker at https://github.com/giampaolo/pysendfile/issues Version 2.0.1 - 2014-05-27 ========================== - #20: host tarball on PYPI - #21: project migrated from google code to github - #21: project migrated from SVN to GIT - #22: pysendfile won't compile on python 3.4 - #23: add a Makefile - #24: use of travis continuous integration - #25: use tox for multiple python versions testing Version 2.0.0 - 2012-01-12 ========================== (Giampaolo Rodolà took over maintenance) - ##: complete rewriting except AIX code - ##: non blocking sockets support - ##: threads support (release GIL) - #1: unit tests - #2: python 3 support - #3: FreeBSD implementation is broken - #4: python large file support - #5: header/trailer are now keyword arguments - #6: exposed SF_NODISKIO, SF_MNOWAIT and SF_SYNC constants on FreeBSD - #8: benchmark script - #10: Mac OSX support - #13: Sun OS support Version 1.2.4 - 2009-03-06 ========================== (Stephan Peijnik took over maintenance) - ## Add AIX support. Version 1.2.3 - 2008-04-09 ========================== - ## Use setuptools instead of distutils. Version 1.2.2 - 2008-03-29 ========================== (Ben Woolley) - ## First release including support for Linux, FreeBSD and DragonflyBSD platforms. pysendfile-release-2.0.1/LICENSE000066400000000000000000000026411234107071600163030ustar00rootroot00000000000000====================================================================== This software is distributed under the MIT license reproduced below: Original author: Copyright (C) 2005-2008 Ben Woolley AIX support code by: Copyright (C) 2008-2009 Niklas Edmundsson Rewritten from scratch and maintained by: Copyright (C) 2009-2014 Giampaolo Rodola' Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Giampaolo Rodola' not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. Ben Woolley, Niklas Edmundsson and Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ====================================================================== pysendfile-release-2.0.1/MANIFEST.in000066400000000000000000000002661234107071600170350ustar00rootroot00000000000000include .travis.yml include HISTORY.rst include LICENSE include Makefile include MANIFEST.in include README.rst include sendfilemodule.c include setup.py recursive-include test *.py pysendfile-release-2.0.1/Makefile000066400000000000000000000026551234107071600167430ustar00rootroot00000000000000# Shortcuts for various tasks (UNIX only). # To use a specific Python version run: # $ make install PYTHON=python3.3 # You can set these variables from the command line. PYTHON = python TSCRIPT = test/test_sendfile.py all: test clean: rm -f `find . -type f -name \*.py[co]` rm -f `find . -type f -name \*.so` rm -f `find . -type f -nam1e .\*~` rm -f `find . -type f -name \*.orig` rm -f `find . -type f -name \*.bak` rm -f `find . -type f -name \*.rej` rm -rf `find . -type d -name __pycache__` rm -rf *.egg-info rm -rf *\$testfile* rm -rf .tox rm -rf build rm -rf dist build: clean $(PYTHON) setup.py build install: build if test $(PYTHON) = python2.5; then \ $(PYTHON) setup.py install; \ else \ $(PYTHON) setup.py install --user; \ fi uninstall: cd ..; $(PYTHON) -m pip uninstall -y -v pysendfile; \ test: install $(PYTHON) $(TSCRIPT) # requires "pip install pep8" pep8: @git ls-files | grep \\.py$ | xargs pep8 # requires "pip install pyflakes" pyflakes: @export PYFLAKES_NODOCTEST=1 && \ git ls-files | grep \\.py$ | xargs pyflakes # requires "pip install flake8" flake8: @git ls-files | grep \\.py$ | xargs flake8 # upload source tarball on https://pypi.python.org/pypi/pysendfile. upload-src: clean $(PYTHON) setup.py sdist upload # git-tag a new release git-tag-release: git tag -a release-`python -c "import setup; print(setup.VERSION)"` -m `git rev-list HEAD --count`:`git rev-parse --short HEAD` pysendfile-release-2.0.1/README.rst000066400000000000000000000201231234107071600167600ustar00rootroot00000000000000.. image:: https://pypip.in/d/pysendfile/badge.png :target: https://crate.io/packages/pysendfile/ :alt: Download this month .. image:: https://pypip.in/v/pysendfile/badge.png :target: https://pypi.python.org/pypi/pysendfile/ :alt: Latest version .. image:: https://pypip.in/license/pysendfile/badge.png :target: https://pypi.python.org/pypi/pysendfile/ :alt: License .. image:: https://api.travis-ci.org/giampaolo/pysendfile.png?branch=master :target: https://travis-ci.org/giampaolo/pysendfile :alt: Travis =========== Quick links =========== - `Home page `_ - `Mailing list `_ - `Blog `_ - `What's new `_ ===== About ===== `sendfile(2) `__ is a system call which provides a "zero-copy" way of copying data from one file descriptor to another (a socket). The phrase "zero-copy" refers to the fact that all of the copying of data between the two descriptors is done entirely by the kernel, with no copying of data into userspace buffers. This is particularly useful when sending a file over a socket (e.g. FTP). The normal way of sending a file over a socket involves reading data from the file into a userspace buffer, then write that buffer to the socket via `send() `__ or `sendall() `__: .. code-block:: python # how a file is tipically sent import socket file = open("somefile", "rb") sock = socket.socket() sock.connect(("127.0.0.1", 8021)) while True: chunk = file.read(65536) if not chunk: break # EOF sock.sendall(chunk) This copying of the data twice (once into the userland buffer, and once out from that userland buffer) imposes some performance and resource penalties. `sendfile(2) `__ syscall avoids these penalties by avoiding any use of userland buffers; it also results in a single system call (and thus only one context switch), rather than the series of `read(2) `__ / `write(2) `__ system calls (each system call requiring a context switch) used internally for the data copying. .. code-block:: python import socket from sendfile import sendfile file = open("somefile", "rb") blocksize = os.path.getsize("somefile") sock = socket.socket() sock.connect(("127.0.0.1", 8021)) offset = 0 while True: sent = sendfile(sock.fileno(), file.fileno(), offset, blocksize) if sent == 0: break # EOF offset += sent ================== A simple benchmark ================== This `benchmark script `__ implements the two examples above and compares plain socket.send() and sendfile() performances in terms of CPU time spent and bytes transmitted per second resulting in sendfile() being about **2.5x faster**. These are the results I get on my Linux 2.6.38 box, AMD dual-core 1.6 GHz: *send()* +---------------+-----------------+ | CPU time | 28.84 usec/pass | +---------------+-----------------+ | transfer rate | 359.38 MB/sec | +---------------+-----------------+ *sendfile()* +---------------+-----------------+ | CPU time | 11.28 usec/pass | +---------------+-----------------+ | transfer rate | 860.88 MB/sec | +---------------+-----------------+ =========================== When do you want to use it? =========================== Basically any application sending files over the network can take advantage of sendfile(2). HTTP and FTP servers are a typical example. `proftpd `__ and `vsftpd `__ are known to use it, so is `pyftpdlib `__. ================= API documentation ================= sendfile module provides a single function: sendfile(). - ``sendfile.sendfile(out, in, offset, nbytes, header="", trailer="", flags=0)`` Copy *nbytes* bytes from file descriptor *in* (a regular file) to file descriptor *out* (a socket) starting at *offset*. Return the number of bytes just being sent. When the end of file is reached return 0. On Linux, if *offset* is given as *None*, the bytes are read from the current position of *in* and the position of *in* is updated. *headers* and *trailers* are strings that are written before and after the data from *in* is written. In cross platform applications their usage is discouraged (`send() `__ or `sendall() `__ can be used instead). On Solaris, _out_ may be the file descriptor of a regular file or the file descriptor of a socket. On all other platforms, *out* must be the file descriptor of an open socket. *flags* argument is only supported on FreeBSD. - ``sendfile.SF_NODISKIO`` - ``sendfile.SF_MNOWAIT`` - ``sendfile.SF_SYNC`` Parameters for the _flags_ argument, if the implementation supports it. They are available on FreeBSD platforms. See `FreeBSD's man sendfile(2) `__. ======================= Differences with send() ======================= - sendfile(2) works with regular (mmap-like) files only (e.g. you can't use it with a `StringIO `__ object). - Also, it must be clear that the file can only be sent "as is" (e.g. you can't modify the content while transmitting). There might be problems with non regular filesystems such as NFS, SMBFS/Samba and CIFS. For this please refer to `proftpd documentation `__. - `OSError `__ is raised instead of `socket.error `__. The accompaining `error codes `__ have the same meaning though: EAGAIN, EWOULDBLOCK, EBUSY meaning you are supposed to retry, ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED in case of disconnection. Some examples: `benchmark script `__, `test suite `__, `pyftpdlib wrapper `__. =================== Supported platforms =================== This module works with Python versions from **2.5** to **3.4**. The supported platforms are: - **Linux** - **Mac OSX** - **FreeBSD** - **Dragon Fly BSD** - **Sun OS** - **AIX** (not properly tested) ======= Support ======= Feel free to mail me at *g.rodola [AT] gmail [DOT] com* or post on the the mailing list: http://groups.google.com/group/py-sendfile. ====== Status ====== As of now the code includes a solid `test suite `__ and its ready for production use. It's been included in `pyftpdlib `__ project and used in production environments for almost a year now without any problem being reported so far. ======= Authors ======= pysendfile was originally written by *Ben Woolley* including Linux, FreeBSD and DragonFly BSD support. Later on *Niklas Edmundsson* took over maintenance and added AIX support. After a couple of years of project stagnation `Giampaolo Rodola' `__ took over maintenance and rewrote it from scratch adding support for: - Python 3 - non-blocking sockets - `large file `__ support - Mac OSX - Sun OS - FreeBSD flag argument - multiple threads (release GIL) - a simple benchmark suite - unit tests - documentation pysendfile-release-2.0.1/sendfilemodule.c000066400000000000000000000246131234107071600204440ustar00rootroot00000000000000/* * pysendfile * * A Python module interface to sendfile(2) * * Original author: * Copyright (C) 2005 Ben Woolley * * The AIX support code is: * Copyright (C) 2008,2009 Niklas Edmundsson * * Rewritten from scratch and maintained by Giampaolo Rodola' * Copyright (C) 2009,2014 * * * The MIT License * * Copyright (c) <2011> * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal * in the Software without restriction, including without limitation the rights * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell * copies of the Software, and to permit persons to whom the Software is * furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN * THE SOFTWARE. */ #include #include // force a compilation error if platform is not supported #if !defined(__FreeBSD__) && \ !defined(__DragonFly__) && \ !defined(__APPLE__) && \ !defined(_AIX) && \ !defined(__linux__) && \ !defined(__sun) #error platfom not supported #endif static int _parse_off_t(PyObject* arg, void* addr) { #if !defined(HAVE_LARGEFILE_SUPPORT) *((off_t*)addr) = PyLong_AsLong(arg); #else *((off_t*)addr) = PyLong_AsLongLong(arg); #endif if (PyErr_Occurred()) return 0; return 1; } /* --- begin FreeBSD / Dragonfly / OSX --- */ #if defined(__FreeBSD__) || defined(__DragonFly__) || defined(__APPLE__) #include #include #include // needed on OSX - Python < 2.6 #if defined(__APPLE__) && defined(_POSIX_C_SOURCE) struct sf_hdtr { struct iovec *headers; int hdr_cnt; struct iovec *trailers; int trl_cnt; }; #endif static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int fd; int sock; int flags = 0; off_t offset; Py_ssize_t ret; Py_ssize_t nbytes; char * head = NULL; char * tail = NULL; Py_ssize_t head_len = 0; Py_ssize_t tail_len = 0; off_t sent; struct sf_hdtr hdtr; PyObject *offobj; static char *keywords[] = {"out", "in", "offset", "nbytes", "header", "trailer", "flags", NULL}; if (!PyArg_ParseTupleAndKeywords(args, kwdict, "iiOn|s#s#i:sendfile", keywords, &fd, &sock, &offobj, &nbytes, &head, &head_len, &tail, &tail_len, &flags)) { return NULL; } if (!_parse_off_t(offobj, &offset)) return NULL; #ifdef __APPLE__ sent = nbytes; #endif if (head_len != 0 || tail_len != 0) { struct iovec ivh = {head, head_len}; struct iovec ivt = {tail, tail_len}; hdtr.headers = &ivh; hdtr.hdr_cnt = 1; hdtr.trailers = &ivt; hdtr.trl_cnt = 1; Py_BEGIN_ALLOW_THREADS #ifdef __APPLE__ sent += head_len; ret = sendfile(sock, fd, offset, &sent, &hdtr, flags); #else ret = sendfile(sock, fd, offset, nbytes, &hdtr, &sent, flags); #endif Py_END_ALLOW_THREADS } else { Py_BEGIN_ALLOW_THREADS #ifdef __APPLE__ ret = sendfile(sock, fd, offset, &sent, NULL, flags); #else ret = sendfile(sock, fd, offset, nbytes, NULL, &sent, flags); #endif Py_END_ALLOW_THREADS } if (ret < 0) { if ((errno == EAGAIN) || (errno == EBUSY) || (errno == EWOULDBLOCK)) { if (sent != 0) { // some data has been sent errno = 0; goto done; } else { // no data has been sent; upper application is supposed // to retry on EAGAIN / EBUSY / EWOULDBLOCK PyErr_SetFromErrno(PyExc_OSError); return NULL; } } PyErr_SetFromErrno(PyExc_OSError); return NULL; } goto done; done: #if defined(HAVE_LARGEFILE_SUPPORT) return Py_BuildValue("L", sent); #else return Py_BuildValue("l", sent); #endif } /* --- end OSX / FreeBSD / Dragonfly --- */ /* --- begin AIX --- */ #elif defined(_AIX) #include static PyObject * method_sendfile(PyObject *self, PyObject *args) { int out_fd, in_fd; off_t offset; size_t nbytes; char *hdr=NULL, *trail=NULL; int hdrsize, trailsize; ssize_t sts=0; struct sf_parms sf_iobuf; int rc; if (!PyArg_ParseTuple(args, "iiLk|s#s#", &out_fd, &in_fd, &offset, &nbytes, &hdr, &hdrsize, &trail, &trailsize)) return NULL; if(hdr != NULL) { sf_iobuf.header_data = hdr; sf_iobuf.header_length = hdrsize; } else { sf_iobuf.header_data = NULL; sf_iobuf.header_length = 0; } if(trail != NULL) { sf_iobuf.trailer_data = trail; sf_iobuf.trailer_length = trailsize; } else { sf_iobuf.trailer_data = NULL; sf_iobuf.trailer_length = 0; } sf_iobuf.file_descriptor = in_fd; sf_iobuf.file_offset = offset; sf_iobuf.file_bytes = nbytes; Py_BEGIN_ALLOW_THREADS; do { sf_iobuf.bytes_sent = 0; /* Really needed? */ rc = send_file(&out_fd, &sf_iobuf, SF_DONT_CACHE); sts += sf_iobuf.bytes_sent; } while( rc == 1 || (rc == -1 && errno == EINTR) ); Py_END_ALLOW_THREADS; offset = sf_iobuf.file_offset; if (rc == -1) { PyErr_SetFromErrno(PyExc_OSError); return NULL; } else { #if defined(HAVE_LARGEFILE_SUPPORT) return Py_BuildValue("L", sts); #else return Py_BuildValue("l", sts); #endif } } /* --- end AIX --- */ /* --- start Linux --- */ #elif defined (__linux__) #include static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int out_fd, in_fd; off_t offset; Py_ssize_t nbytes; Py_ssize_t sent; PyObject *offobj; if (!PyArg_ParseTuple(args, "iiOn", &out_fd, &in_fd, &offobj, &nbytes)) { return NULL; } if (offobj == Py_None) { Py_BEGIN_ALLOW_THREADS; sent = sendfile(out_fd, in_fd, NULL, nbytes); Py_END_ALLOW_THREADS; } else { if (!_parse_off_t(offobj, &offset)) return NULL; Py_BEGIN_ALLOW_THREADS; sent = sendfile(out_fd, in_fd, &offset, nbytes); Py_END_ALLOW_THREADS; } if (sent == -1) return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("n", sent); } /* --- end Linux --- */ /* --- begin SUN OS --- */ #elif defined(__sun) #include static PyObject * method_sendfile(PyObject *self, PyObject *args, PyObject *kwdict) { int out_fd; int in_fd; off_t offset; Py_ssize_t nbytes; Py_ssize_t sent; PyObject *offobj; if (!PyArg_ParseTuple(args, "iiOn", &out_fd, &in_fd, &offobj, &nbytes)) { return NULL; } if (!_parse_off_t(offobj, &offset)) return NULL; sent = sendfile(out_fd, in_fd, &offset, nbytes); if (sent == -1) return PyErr_SetFromErrno(PyExc_OSError); return Py_BuildValue("n", sent); } #else /* --- end SUN OS --- */ #error platfom not supported #endif /* --- module initialization --- */ struct module_state { PyObject *error; }; #if PY_MAJOR_VERSION >= 3 #define GETSTATE(m) ((struct module_state*)PyModule_GetState(m)) #else #define GETSTATE(m) (&_state) static struct module_state _state; #endif static PyMethodDef sendfile_methods[] = { {"sendfile", (PyCFunction)method_sendfile, METH_VARARGS | METH_KEYWORDS, "sendfile(out, in, offset, nbytes, header=\"\", trailer=\"\", flags=0)\n\n" "Copy nbytes bytes from file descriptor in (a regular file) to\n" "file descriptor out (a socket) starting at offset.\n" "Return the number of bytes just being sent. When the end of\n" "file is reached return 0.\n" "On Linux, if offset is given as None, the bytes are read from\n" "the current position of in and the position of in is updated.\n" "headers and trailers are strings that are written before and\n" "after the data from in is written. In cross platform applications\n" "their usage is discouraged (socket.send() or socket.sendall()\n" "can be used instead).\n" "On Solaris, out may be the file descriptor of a regular file\n" "or the file descriptor of a socket. On all other platforms,\n" "out must be the file descriptor of an open socket.\n" "flags argument is only supported on FreeBSD.\n" }, {NULL, NULL} }; #if PY_MAJOR_VERSION >= 3 static int sendfile_traverse(PyObject *m, visitproc visit, void *arg) { Py_VISIT(GETSTATE(m)->error); return 0; } static int sendfile_clear(PyObject *m) { Py_CLEAR(GETSTATE(m)->error); return 0; } static struct PyModuleDef moduledef = { PyModuleDef_HEAD_INIT, "sendfile", NULL, sizeof(struct module_state), sendfile_methods, NULL, sendfile_traverse, sendfile_clear, NULL }; #define INITERROR return NULL PyObject * PyInit_sendfile(void) #else #define INITERROR return void initsendfile(void) #endif { #if PY_MAJOR_VERSION >= 3 PyObject *module = PyModule_Create(&moduledef); #else PyObject *module = Py_InitModule("sendfile", sendfile_methods); #endif #ifdef SF_NODISKIO PyModule_AddIntConstant(module, "SF_NODISKIO", SF_NODISKIO); #endif #ifdef SF_MNOWAIT PyModule_AddIntConstant(module, "SF_MNOWAIT", SF_MNOWAIT); #endif #ifdef SF_SYNC PyModule_AddIntConstant(module, "SF_SYNC", SF_SYNC); #endif if (module == NULL) INITERROR; #if PY_MAJOR_VERSION >= 3 return module; #endif } pysendfile-release-2.0.1/setup.py000066400000000000000000000066701234107071600170160ustar00rootroot00000000000000#!/usr/bin/env python # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== import sys try: from setuptools import setup, Extension except ImportError: from distutils.core import setup, Extension NAME = 'pysendfile' VERSION = '2.0.1' if sys.version_info < (2, 5): sys.exit('python version not supported (< 2.5)') if 'sunos' in sys.platform: libraries = ["sendfile"] else: libraries = [] def main(): setup(name=NAME, url='https://github.com/giampaolo/pysendfile', version=VERSION, description='A Python interface to sendfile(2)', long_description=open('README.rst', 'r').read(), author='Giampaolo Rodola', author_email='g.rodola@gmail.com', platforms='UNIX', license='MIT', keywords=['sendfile', 'python', 'performance', 'ftp'], classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Operating System :: POSIX :: Linux', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX :: BSD', 'Operating System :: POSIX :: BSD :: FreeBSD', 'Operating System :: POSIX :: SunOS/Solaris', 'Operating System :: POSIX :: AIX', 'Programming Language :: C', 'Programming Language :: Python :: 2.5', 'Programming Language :: Python :: 2.6', 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.0', 'Programming Language :: Python :: 3.1', 'Programming Language :: Python :: 3.2', 'Programming Language :: Python :: 3.3', 'Programming Language :: Python :: 3.4', 'Topic :: System :: Networking', 'Topic :: System :: Operating System', 'Topic :: Internet :: File Transfer Protocol (FTP)', 'Topic :: Internet :: WWW/HTTP', 'License :: OSI Approved :: MIT License', ], ext_modules=[Extension('sendfile', sources=['sendfilemodule.c'], libraries=libraries)]) if __name__ == '__main__': main() pysendfile-release-2.0.1/test/000077500000000000000000000000001234107071600162525ustar00rootroot00000000000000pysendfile-release-2.0.1/test/benchmark.py000066400000000000000000000171231234107071600205620ustar00rootroot00000000000000#!/usr/bin/env python # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """ A simle benchmark script which compares plain send() and sendfile() performances in terms of CPU time spent and bytes transmitted in one second. This is what I get on my Linux 2.6.38 box, AMD dual core 1.6 GHz: send() cpu: 28.41 usec/pass rate: 362.13 MB/sec sendfile() cpu: 11.25 usec/pass rate: 848.56 MB/sec Works with both python 2.X and 3.X. """ from __future__ import with_statement import atexit import contextlib import errno import itertools import optparse import os import signal import socket import sys import threading import time import timeit from multiprocessing import Process from sendfile import sendfile # overridable defaults HOST = "127.0.0.1" PORT = 8022 BIGFILE = "$testfile1" BIGFILE_SIZE = 1024 * 1024 * 1024 # 1 GB BUFFER_SIZE = 65536 # python 3 compatibility layer def b(s): return bytes(s, 'ascii') if sys.version_info >= (3, ) else s # python 2.5 compatibility try: next except NameError: def next(iterator): return iterator.next() def print_(s, hilite=False): if hilite: bold = '1' s = '\x1b[%sm%s\x1b[0m' % (';'.join([bold]), s) sys.stdout.write(s + "\n") sys.stdout.flush() def create_file(filename, size): with open(filename, 'wb') as f: bytes = 0 chunk = b("x") * BUFFER_SIZE while 1: f.write(chunk) bytes += len(chunk) if bytes >= size: break def safe_remove(file): try: os.remove(file) except OSError: pass class Spinner(threading.Thread): def run(self): self._exit = False self._spinner = itertools.cycle('-\|/') while not self._exit: sys.stdout.write(next(self._spinner) + "\b") sys.stdout.flush() time.sleep(.1) def stop(self): self._exit = True self.join() class Client: def __init__(self): self.sock = socket.socket() self.sock.connect((HOST, PORT)) self.sock.settimeout(1) def retr(self): with contextlib.closing(self.sock): while 1: data = self.sock.recv(BUFFER_SIZE) if not data: break def retr_for_1_sec(self): with contextlib.closing(self.sock): stop_at = time.time() + 1 bytes_recv = 0 while stop_at > time.time(): chunk = self.sock.recv(BUFFER_SIZE) if not chunk: assert 0 bytes_recv += len(chunk) return bytes_recv def start_server(use_sendfile, keep_sending=False): """A simple test server which sends a file once a client connects. use_sendfile decides whether using sendfile() or plain send(). If keep_sending is True restart sending file when EOF is reached. """ sock = socket.socket() sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.bind((HOST, PORT)) sock.listen(1) conn, addr = sock.accept() sock.close() file = open(BIGFILE, 'rb') def on_exit(signum, fram): file.close() conn.close() sys.exit(0) signal.signal(signal.SIGTERM, on_exit) signal.signal(signal.SIGINT, on_exit) if not use_sendfile: while 1: chunk = file.read(BUFFER_SIZE) if not chunk: # EOF if keep_sending: file.seek(0) continue else: break conn.sendall(chunk) else: offset = 0 sockno = conn.fileno() fileno = file.fileno() while 1: try: sent = sendfile(sockno, fileno, offset, BUFFER_SIZE) except OSError: err = sys.exc_info()[1] if err.errno in (errno.EAGAIN, errno.EBUSY): continue raise else: if not sent: # EOF if keep_sending: offset = 0 continue else: break else: offset += sent def main(): parser = optparse.OptionParser() parser.add_option('-k', '--keepfile', action="store_true", default=False, help="do not remove test file on exit") options, args = parser.parse_args() if not options.keepfile: atexit.register(lambda: safe_remove(BIGFILE)) if not os.path.exists(BIGFILE) or os.path.getsize(BIGFILE) < BIGFILE_SIZE: print_("creating big file...") create_file(BIGFILE, BIGFILE_SIZE) print_("starting benchmark...") # CPU time: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile": True}) server.start() time.sleep(0.1) t1 = timeit.Timer(setup="from __main__ import Client; client = Client()", stmt="client.retr()").timeit(number=1) server.terminate() server.join() # CPU time: use send() server = Process(target=start_server, kwargs={"use_sendfile": False}) server.start() time.sleep(0.1) t2 = timeit.Timer(setup="from __main__ import Client; client = Client()", stmt="client.retr()").timeit(number=1) server.terminate() server.join() # MB/sec: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile": True, "keep_sending": True}) server.start() time.sleep(0.1) client = Client() bytes1 = client.retr_for_1_sec() server.terminate() server.join() # MB/sec: use sendfile() server = Process(target=start_server, kwargs={"use_sendfile": False, "keep_sending": True}) server.start() time.sleep(0.1) client = Client() bytes2 = client.retr_for_1_sec() server.terminate() server.join() print_(" ") print_("send()", hilite=True) print_(" cpu: %7.2f usec/pass" % (1000000 * t2 / 100000)) print_(" rate: %7.2f MB/sec" % round(bytes2 / 1024.0 / 1024.0, 2)) print_("") print_("sendfile()", hilite=True) print_(" cpu: %7.2f usec/pass" % (1000000 * t1 / 100000)) print_(" rate: %7.2f MB/sec" % round(bytes1 / 1024.0 / 1024.0, 2)) if __name__ == '__main__': s = Spinner() s.start() try: main() finally: s.stop() pysendfile-release-2.0.1/test/test_sendfile.py000066400000000000000000000432011234107071600214540ustar00rootroot00000000000000#!/usr/bin/env python # ====================================================================== # This software is distributed under the MIT license reproduced below: # # Copyright (C) 2009-2014 Giampaolo Rodola' # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # Giampaolo Rodola' not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # Giampaolo Rodola' DISCLAIM ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT Giampaolo Rodola' BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """ pysendfile test suite; works with both python 2.x and 3.x (no need to run 2to3 tool first). Accepts a -k cmdline option to avoid removing the big file created during tests. """ from __future__ import with_statement import asynchat import asyncore import atexit import errno import optparse import os import socket import sys import threading import time import unittest import warnings import sendfile PY3 = sys.version_info >= (3, ) def b(x): if PY3: return bytes(x, 'ascii') return x TESTFN = "$testfile" TESTFN2 = TESTFN + "2" TESTFN3 = TESTFN + "3" DATA = b("12345abcde" * 1024 * 1024) # 10 Mb HOST = '127.0.0.1' BIGFILE_SIZE = 2500000000 # > 2GB file (2GB = 2147483648 bytes) BUFFER_LEN = 4096 try: sendfile.sendfile(0, 0, 0, 0, 0) except TypeError: SUPPORT_HEADER_TRAILER = False except Exception: SUPPORT_HEADER_TRAILER = True def safe_remove(file): try: os.remove(file) except OSError: pass def has_large_file_support(): # taken from Python's Lib/test/test_largefile.py with open(TESTFN, 'wb', buffering=0) as f: try: f.seek(BIGFILE_SIZE) # seeking is not enough of a test: you must write and flush too f.write(b('x')) f.flush() except (IOError, OverflowError): return False else: return True class Handler(asynchat.async_chat): ac_in_buffer_size = BUFFER_LEN ac_out_buffer_size = BUFFER_LEN def __init__(self, conn): asynchat.async_chat.__init__(self, conn) self.in_buffer = [] self.closed = False self.push(b("220 ready\r\n")) def handle_read(self): data = self.recv(BUFFER_LEN) self.in_buffer.append(data) def get_data(self): return b('').join(self.in_buffer) def handle_close(self): self.close() def close(self): asynchat.async_chat.close(self) self.closed = True def handle_error(self): raise class NoMemoryHandler(Handler): # same as above but doesn't store received data in memory ac_in_buffer_size = 65536 def __init__(self, conn): Handler.__init__(self, conn) self.in_buffer_len = 0 def handle_read(self): data = self.recv(self.ac_in_buffer_size) self.in_buffer_len += len(data) def get_data(self): raise NotImplementedError class Server(asyncore.dispatcher, threading.Thread): handler = Handler def __init__(self, address): threading.Thread.__init__(self) asyncore.dispatcher.__init__(self) self.create_socket(socket.AF_INET, socket.SOCK_STREAM) self.bind(address) self.listen(5) self.host, self.port = self.socket.getsockname()[:2] self.handler_instance = None self._active = False self._active_lock = threading.Lock() # --- public API @property def running(self): return self._active def start(self): assert not self.running self.__flag = threading.Event() threading.Thread.start(self) self.__flag.wait() def stop(self): assert self.running self._active = False self.join() assert not asyncore.socket_map, asyncore.socket_map def wait(self): # wait for handler connection to be closed, then stop the server while not getattr(self.handler_instance, "closed", True): time.sleep(0.001) self.stop() # --- internals def run(self): self._active = True self.__flag.set() while self._active and asyncore.socket_map: self._active_lock.acquire() asyncore.loop(timeout=0.001, count=1) self._active_lock.release() asyncore.close_all() def handle_accept(self): conn, addr = self.accept() self.handler_instance = self.handler(conn) def handle_connect(self): self.close() handle_read = handle_connect def writable(self): return 0 def handle_error(self): raise def sendfile_wrapper(sock, file, offset, nbytes=BUFFER_LEN, header="", trailer=""): """A higher level wrapper representing how an application is supposed to use sendfile(). """ while 1: try: if SUPPORT_HEADER_TRAILER: sent = sendfile.sendfile(sock, file, offset, nbytes, header, trailer) else: sent = sendfile.sendfile(sock, file, offset, nbytes) except OSError: err = sys.exc_info()[1] if err.errno == errno.EAGAIN: # retry continue raise else: assert sent <= nbytes, (sent, nbytes) return sent class TestSendfile(unittest.TestCase): def setUp(self): self.server = Server((HOST, 0)) self.server.start() self.client = socket.socket() self.client.connect((self.server.host, self.server.port)) self.client.settimeout(1) # synchronize by waiting for "220 ready" response self.client.recv(1024) self.sockno = self.client.fileno() self.file = open(TESTFN, 'rb') self.fileno = self.file.fileno() def tearDown(self): safe_remove(TESTFN2) self.file.close() self.client.close() if self.server.running: self.server.stop() self.server = None # allow garbage collection def test_send_whole_file(self): # normal send total_sent = 0 offset = 0 while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break total_sent += sent offset += sent self.assertEqual(offset, total_sent) self.assertEqual(total_sent, len(DATA)) self.client.close() if "sunos" in sys.platform: time.sleep(.1) self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) def test_send_from_certain_offset(self): # start sending a file at a certain offset total_sent = 0 offset = int(len(DATA) / 2) while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break total_sent += sent offset += sent self.client.close() if "sunos" in sys.platform: time.sleep(.1) self.server.wait() data = self.server.handler_instance.get_data() expected = DATA[int(len(DATA) / 2):] self.assertEqual(total_sent, len(expected)) self.assertEqual(hash(data), hash(expected)) if SUPPORT_HEADER_TRAILER: def test_header(self): total_sent = 0 header = b("x") * 512 sent = sendfile.sendfile(self.sockno, self.fileno, 0, header=header) total_sent += sent offset = BUFFER_LEN while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset) if sent == 0: break offset += sent total_sent += sent expected_data = header + DATA self.assertEqual(total_sent, len(expected_data)) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(expected_data)) self.assertEqual(hash(data), hash(expected_data)) def test_trailer(self): with open(TESTFN2, 'wb') as f: f.write(b("abcde")) with open(TESTFN2, 'rb') as f: sendfile.sendfile(self.sockno, f.fileno(), 0, trailer=b("12345")) time.sleep(.1) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(data, b("abcde12345")) def test_non_socket(self): fd_in = open(TESTFN, 'rb') fd_out = open(TESTFN2, 'wb') try: sendfile.sendfile(fd_in.fileno(), fd_out.fileno(), 0, BUFFER_LEN) except OSError: err = sys.exc_info()[1] self.assertEqual(err.errno, errno.EBADF) else: self.fail("exception not raised") finally: fd_in.close() fd_out.close() if sys.platform.startswith('freebsd'): def test_send_nbytes_0(self): # On Mac OS X and FreeBSD a value of 0 for nbytes # is supposed to send the whole file in one shot. # OSX implementation appears to be just broken. # On *BSD this works most of the times: sometimes # EAGAIN is returned internally and here we get the # number of bytes sent. sent = sendfile.sendfile(self.sockno, self.fileno, 0, 0) if sent == len(DATA): self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) if hasattr(sendfile, "SF_NODISKIO"): def test_flags(self): try: sendfile.sendfile(self.sockno, self.fileno, 0, BUFFER_LEN, flags=sendfile.SF_NODISKIO) except OSError: err = sys.exc_info()[1] if err.errno not in (errno.EBUSY, errno.EAGAIN): raise # --- corner cases def test_offset_overflow(self): # specify an offset > file size offset = len(DATA) + BUFFER_LEN sent = sendfile.sendfile(self.sockno, self.fileno, offset, BUFFER_LEN) self.assertEqual(sent, 0) self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(data, b('')) if "sunos" not in sys.platform: def test_invalid_offset(self): try: sendfile.sendfile(self.sockno, self.fileno, -1, BUFFER_LEN) except OSError: err = sys.exc_info()[1] self.assertEqual(err.errno, errno.EINVAL) else: self.fail("exception not raised") def test_small_file(self): data = b('foo bar') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: offset = 0 while 1: sent = sendfile_wrapper(self.sockno, f.fileno(), offset) if sent == 0: break offset += sent self.client.close() time.sleep(.1) self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, data) def test_small_file_and_offset_overflow(self): data = b('foo bar') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: sendfile_wrapper(self.sockno, f.fileno(), 4096) self.client.close() self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, b('')) def test_empty_file(self): data = b('') with open(TESTFN2, 'wb') as f: f.write(data) with open(TESTFN2, 'rb') as f: sendfile_wrapper(self.sockno, f.fileno(), 0) self.client.close() self.server.wait() data_sent = self.server.handler_instance.get_data() self.assertEqual(data_sent, data) if "linux" in sys.platform: def test_offset_none(self): # on Linux offset == None sendfile() call is supposed # to update the file offset while 1: sent = sendfile_wrapper(self.sockno, self.fileno, None) if sent == 0: break self.client.close() self.server.wait() data = self.server.handler_instance.get_data() self.assertEqual(len(data), len(DATA)) self.assertEqual(hash(data), hash(DATA)) class RepeatedTimer: def __init__(self, timeout, fun): self.timeout = timeout self.fun = fun def start(self): def main(): self.fun() self.start() self.timer = threading.Timer(1, main) self.timer.start() def stop(self): self.timer.cancel() class TestLargeFile(unittest.TestCase): def setUp(self): self.server = Server((HOST, 0)) self.server.handler = NoMemoryHandler self.server.start() self.client = socket.socket() self.client.connect((self.server.host, self.server.port)) self.client.settimeout(1) # synchronize by waiting for "220 ready" response self.client.recv(1024) self.sockno = self.client.fileno() sys.stdout.write("\ncreating file:\n") sys.stdout.flush() self.create_file() self.file = open(TESTFN3, 'rb') self.fileno = self.file.fileno() sys.stdout.write("\nstarting transfer:\n") sys.stdout.flush() def tearDown(self): if hasattr(self, 'file'): self.file.close() self.client.close() if self.server.running: self.server.stop() def print_percent(self, a, b): percent = ((a * 100) / b) sys.stdout.write("\r%2d%%" % percent) sys.stdout.flush() def create_file(self): if (os.path.isfile(TESTFN3) and os.path.getsize(TESTFN3) >= BIGFILE_SIZE): return f = open(TESTFN3, 'wb') chunk_len = 65536 chunk = b('x' * chunk_len) total = 0 timer = RepeatedTimer(1, lambda: self.print_percent(total, BIGFILE_SIZE)) timer.start() try: while 1: f.write(chunk) total += chunk_len if total >= BIGFILE_SIZE: break finally: f.close() timer.stop() def test_big_file(self): total_sent = 0 offset = 0 nbytes = 65536 file_size = os.path.getsize(TESTFN3) timer = RepeatedTimer(1, lambda: self.print_percent(total_sent, file_size)) timer.start() try: while 1: sent = sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) if sent == 0: break total_sent += sent offset += sent self.assertEqual(offset, total_sent) except Exception: print raise finally: sys.stdout.write("\n") sys.stdout.flush() timer.stop() self.assertEqual(total_sent, file_size) self.client.close() if "sunos" in sys.platform: time.sleep(1) self.server.wait() data_len = self.server.handler_instance.in_buffer_len file_size = os.path.getsize(TESTFN3) self.assertEqual(file_size, data_len) def cleanup(): safe_remove(TESTFN) safe_remove(TESTFN2) atexit.register(cleanup) def test_main(): parser = optparse.OptionParser() parser.add_option('-k', '--keepfile', action="store_true", default=False, help="do not remove test big file on exit") options, args = parser.parse_args() if not options.keepfile: atexit.register(lambda: safe_remove(TESTFN3)) test_suite = unittest.TestSuite() test_suite.addTest(unittest.makeSuite(TestSendfile)) if has_large_file_support(): test_suite.addTest(unittest.makeSuite(TestLargeFile)) else: atexit.register(warnings.warn, "large files unsupported", RuntimeWarning) cleanup() with open(TESTFN, "wb") as f: f.write(DATA) result = unittest.TextTestRunner(verbosity=2).run(test_suite) return result.wasSuccessful() if __name__ == '__main__': try: if not test_main(): sys.exit(1) except (KeyboardInterrupt, SystemExit): # this will make the threaded server exit immediately asyncore.socket_map.clear() raise pysendfile-release-2.0.1/tox.ini000066400000000000000000000011251234107071600166050ustar00rootroot00000000000000# Tox (http://tox.testrun.org/) is a tool for running tests # in multiple virtualenvs. This configuration file will run the # test suite on all supported python versions. # To use it run "pip install tox" and then run "tox" from this # directory. [tox] envlist = py26, py27, py32, py33, py34 [testenv] deps = pytest flake8 setenv = PYTHONPATH = {toxinidir}/test commands = python test/test_sendfile.py flake8 --exclude=build,.tox,.git [testenv:py26] deps = flake8 pytest unittest2 commands = python test/test_sendfile.py flake8 --exclude=build,.tox,.git