medusa-0.5.4/0040775000076400007640000000000007725426233011130 5ustar amkamkmedusa-0.5.4/debian/0040775000076400007640000000000007725426233012352 5ustar amkamkmedusa-0.5.4/debian/changelog0100664000076400007640000000021107721413711014204 0ustar amkamkpython2.2-medusa (0.5.4-1) unstable; urgency=low * Initial Release. -- A.M. Kuchling Fri, 22 Aug 2003 08:54:11 -0400 medusa-0.5.4/debian/control0100664000076400007640000000062707721413711013750 0ustar amkamkSource: python2.2-medusa Priority: optional Maintainer: A.M. Kuchling Build-Depends: debhelper (>> 3.0.0) Standards-Version: 3.5.8 Section: net Package: python2.2-medusa Section: net Architecture: all Depends: python2.2 Description: Medusa is a 'server platform' -- it provides a framework for implementing asynchronous socket-based servers (TCP/IP and on Unix, Unix domain, sockets). medusa-0.5.4/debian/copyright0100664000076400007640000000212707721413711014275 0ustar amkamkThis package was debianized by A.M. Kuchling on Fri, 22 Aug 2003 08:54:11 -0400. It was downloaded from www.amk.ca/python/code/medusa.html Upstream Author: A.M. Kuchling Copyright: Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Sam Rushing not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. medusa-0.5.4/debian/postinst0100775000076400007640000000262307721413711014154 0ustar amkamk#! /bin/sh # postinst script for medusa # # see: dh_installdeb(1) set -e # summary of how this script can be called: # * `configure' # * `abort-upgrade' # * `abort-remove' `in-favour' # # * `abort-deconfigure' `in-favour' # `removing' # # for details, see http://www.debian.org/doc/debian-policy/ or # the debian-policy package # # quoting from the policy: # Any necessary prompting should almost always be confined to the # post-installation script, and should be protected with a conditional # so that unnecessary prompting doesn't happen if a package's # installation fails and the `postinst' is called with `abort-upgrade', # `abort-remove' or `abort-deconfigure'. PACKAGE=python2.2-medusa VERSION=2.2 LIB="/usr/lib/python$VERSION" DIRLIST="$LIB/site-packages/medusa" case "$1" in configure|abort-upgrade|abort-remove|abort-deconfigure) for i in $DIRLIST ; do /usr/bin/python$VERSION -O $LIB/compileall.py -q $i /usr/bin/python$VERSION $LIB/compileall.py -q $i done ;; *) echo "postinst called with unknown argument \`$1'" >&2 exit 1 ;; esac exit 0 medusa-0.5.4/debian/prerm0100664000076400007640000000063107721413711013410 0ustar amkamk#! /bin/sh # prerm script for medusa set -e PACKAGE=python2.2-medusa VERSION=2.2 LIB="/usr/lib/python$VERSION" DIRLIST="$LIB/site-packages/medusa" case "$1" in remove|upgrade|failed-upgrade) for i in $DIRLIST ; do find $i -name '*.py[co]' -exec rm \{\} \; done ;; *) echo "prerm called with unknown argument \`$1'" >&2 exit 1 ;; esac exit 0 medusa-0.5.4/debian/rules0100775000076400007640000000153507721413711013424 0ustar amkamk#!/usr/bin/make -f # Sample debian/rules that uses debhelper. # GNU copyright 1997 to 1999 by Joey Hess. # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 # This is the debhelper compatibility version to use. export DH_COMPAT=4 build: build-stamp /usr/bin/python2.2 setup.py build build-stamp: touch build-stamp configure: # Do nothing clean: dh_testdir dh_testroot rm -f build-stamp -rm -rf build dh_clean install: build dh_testdir dh_testroot dh_clean -k /usr/bin/python2.2 setup.py install --no-compile --prefix=$(CURDIR)/debian/python2.2-medusa/usr # Build architecture-independent files here. binary-indep: install dh_testdir dh_testroot dh_installdocs dh_installdeb dh_gencontrol dh_md5sums dh_builddeb # We have nothing to do by default. binary: binary-indep .PHONY: build clean binary-indep binary install medusa-0.5.4/demo/0040775000076400007640000000000007725426233012054 5ustar amkamkmedusa-0.5.4/demo/publish.py0100644000076400007640000000300307450620727014062 0ustar amkamk# -*- Mode: Python -*- # Demonstrates use of the auth and put handlers to support publishing # web pages via HTTP. # It is also possible to set up the ftp server to do essentially the # same thing. # Security Note: Using HTTP with the 'Basic' authentication scheme is # only slightly more secure than using FTP: both techniques involve # sending a unencrypted password of the network (http basic auth # base64-encodes the username and password). The 'Digest' scheme is # much more secure, but not widely supported yet. import asyncore from medusa import default_handler from medusa import http_server from medusa import put_handler from medusa import auth_handler from medusa import filesys # For this demo, we'll just use a dictionary of usernames/passwords. # You can of course use anything that supports the mapping interface, # and it would be pretty easy to set this up to use the crypt module # on unix. users = { 'mozart' : 'jupiter', 'beethoven' : 'pastoral' } # The filesystem we will be giving access to fs = filesys.os_filesystem('/home/medusa') # The 'default' handler - delivers files for the HTTP GET method. dh = default_handler.default_handler(fs) # Supports the HTTP PUT method... ph = put_handler.put_handler(fs, '/.*') # ... but be sure to wrap it with an auth handler: ah = auth_handler.auth_handler(users, ph) # Create a Web Server hs = http_server.http_server(ip='', port=8080) # install the handlers we created: hs.install_handler(dh) # for GET hs.install_handler(ah) # for PUT asyncore.loop() medusa-0.5.4/demo/script_server.py0100644000076400007640000000234007515074257015314 0ustar amkamk# -*- Mode: Python -*- import re, sys import asyncore from medusa import http_server from medusa import default_handler from medusa import logger from medusa import script_handler from medusa import filesys PUBLISHING_ROOT='/home/medusa' CONTENT_LENGTH = re.compile ('Content-Length: ([0-9]+)', re.IGNORECASE) class sample_input_collector: def __init__ (self, request, length): self.request = request self.length = length def collect_incoming_data (self, data): print 'data from %s: <%s>' % (self.request, repr(data)) class post_script_handler (script_handler.script_handler): def handle_request (self, request): if request.command == 'post': cl = default_handler.get_header(CONTENT_LENGTH, request.header) ic = sample_input_collector(request, cl) request.collector = ic print request.header return script_handler.script_handler.handle_request (self, request) lg = logger.file_logger (sys.stdout) fs = filesys.os_filesystem (PUBLISHING_ROOT) dh = default_handler.default_handler (fs) ph = post_script_handler (fs) hs = http_server.http_server ('', 8081, logger_object = lg) hs.install_handler (dh) hs.install_handler (ph) asyncore.loop() medusa-0.5.4/demo/simple_anon_ftpd.py0100644000076400007640000000124107450620015015725 0ustar amkamk# -*- Mode: Python -*- import asyncore from medusa import ftp_server # create a 'dummy' authorizer (one that lets everyone in) that returns # a read-only filesystem rooted at '/home/ftp' authorizer = ftp_server.dummy_authorizer('/home/ftp') # Create an ftp server using this authorizer, running on port 8021 # [the standard port is 21, but you are probably already running # a server there] fs = ftp_server.ftp_server(authorizer, port=8021) # Run the async main loop asyncore.loop() # to test this server, try # $ ftp myhost 8021 # when using the standard bsd ftp client, # $ ncftp -p 8021 myhost # when using ncftp, and # ftp://myhost:8021/ # from a web browser. medusa-0.5.4/demo/start_medusa.py0100644000076400007640000001563407446201166015121 0ustar amkamk# -*- Mode: Python -*- # # Sample/Template Medusa Startup Script. # # This file acts as a configuration file and startup script for Medusa. # # You should make a copy of this file, then add, change or comment out # appropriately. Then you can start up the server by simply typing # # $ python start_medusa.py # import os import sys import asyncore from medusa import http_server from medusa import ftp_server from medusa import chat_server from medusa import monitor from medusa import filesys from medusa import default_handler from medusa import status_handler from medusa import resolver from medusa import logger if len(sys.argv) > 1: # process a few convenient arguments [HOSTNAME, IP_ADDRESS, PUBLISHING_ROOT] = sys.argv[1:] else: HOSTNAME = 'www.nightmare.com' # This is the IP address of the network interface you want # your servers to be visible from. This can be changed to '' # to listen on all interfaces. IP_ADDRESS = '205.160.176.5' # Root of the http and ftp server's published filesystems. PUBLISHING_ROOT = '/home/www' HTTP_PORT = 8080 # The standard port is 80 FTP_PORT = 8021 # The standard port is 21 CHAT_PORT = 8888 MONITOR_PORT = 9999 # =========================================================================== # Caching DNS Resolver # =========================================================================== # The resolver is used to resolve incoming IP address (for logging), # and also to resolve hostnames for HTTP Proxy requests. I recommend # using a nameserver running on the local machine, but you can also # use a remote nameserver. rs = resolver.caching_resolver ('127.0.0.1') # =========================================================================== # Logging. # =========================================================================== # There are several types of logging objects. Multiple loggers may be combined, # See 'logger.py' for more details. # This will log to stdout: lg = logger.file_logger (sys.stdout) # This will log to syslog: #lg = logger.syslog_logger ('/dev/log') # This will wrap the logger so that it will # 1) keep track of the last 500 entries # 2) display an entry in the status report with a hyperlink # to view these log entries. # # If you decide to comment this out, be sure to remove the # logger object from the list of status objects below. # lg = status_handler.logger_for_status (lg) # =========================================================================== # Filesystem Object. # =========================================================================== # An abstraction for the file system. Filesystem objects can be # combined and implemented in interesting ways. The default type # simply remaps a directory to root. fs = filesys.os_filesystem (PUBLISHING_ROOT) # =========================================================================== # Default HTTP handler # =========================================================================== # The 'default' handler for the HTTP server is one that delivers # files normally - this is the expected behavior of a web server. # Note that you needn't use it: Your web server might not want to # deliver files! # This default handler uses the filesystem object we just constructed. dh = default_handler.default_handler (fs) # =========================================================================== # HTTP Server # =========================================================================== hs = http_server.http_server (IP_ADDRESS, HTTP_PORT, rs, lg) # Here we install the default handler created above. hs.install_handler (dh) # =========================================================================== # Unix user `public_html' directory support # =========================================================================== if os.name == 'posix': from medusa import unix_user_handler uh = unix_user_handler.unix_user_handler ('public_html') hs.install_handler (uh) # =========================================================================== # FTP Server # =========================================================================== # Here we create an 'anonymous' ftp server. # Note: the ftp server is read-only by default. [in this mode, all # 'write-capable' commands are unavailable] ftp = ftp_server.ftp_server ( ftp_server.anon_authorizer ( PUBLISHING_ROOT ), ip=IP_ADDRESS, port=FTP_PORT, resolver=rs, logger_object=lg ) # =========================================================================== # Monitor Server: # =========================================================================== # This creates a secure monitor server, binding to the loopback # address on port 9999, with password 'fnord'. The monitor server # can be used to examine and control the server while it is running. # If you wish to access the server from another machine, you will # need to use '' or some other IP instead of '127.0.0.1'. ms = monitor.secure_monitor_server ('fnord', '127.0.0.1', MONITOR_PORT) # =========================================================================== # Chat Server # =========================================================================== # The chat server is a simple IRC-like server: It is meant as a # demonstration of how to write new servers and plug them into medusa. # It's a very simple server (it took about 2 hours to write), but it # could be easily extended. For example, it could be integrated with # the web server, perhaps providing navigational tools to browse # through a series of discussion groups, listing the number of current # users, authentication, etc... cs = chat_server.chat_server (IP_ADDRESS, CHAT_PORT) # =========================================================================== # Status Handler # =========================================================================== # These are objects that can report their status via the HTTP server. # You may comment out any of these, or add more of your own. The only # requirement for a 'status-reporting' object is that it have a method # 'status' that will return a producer, which will generate an HTML # description of the status of the object. status_objects = [ hs, ftp, ms, cs, rs, lg ] # Create a status handler. By default it binds to the URI '/status'... sh = status_handler.status_extension(status_objects) # ... and install it on the web server. hs.install_handler (sh) # become 'nobody' if os.name == 'posix': if hasattr (os, 'seteuid'): import pwd [uid, gid] = pwd.getpwnam ('nobody')[2:4] os.setegid (gid) os.seteuid (uid) # Finally, start up the server loop! This loop will not exit until # all clients and servers are closed. You may cleanly shut the system # down by sending SIGINT (a.k.a. KeyboardInterrupt). asyncore.loop() medusa-0.5.4/demo/winFTPserver.py0100664000076400007640000000400707643110212015004 0ustar amkamk# # winFTPServer.py -- FTP server that uses Win32 user API # # Contributed by John Abel # # For it to authenticate users correctly, the user running the # script must be added to the security policy "Act As Part Of The OS". # This is needed for the LogonUser to work. A pain, but something that MS # forgot to mention in the API. import win32security, win32con, win32api, win32net import ntsecuritycon, pywintypes import asyncore from medusa import ftp_server, filesys class Win32Authorizer: def authorize (self, channel, userName, passWord): self.AdjustPrivilege( ntsecuritycon.SE_CHANGE_NOTIFY_NAME ) self.AdjustPrivilege( ntsecuritycon.SE_ASSIGNPRIMARYTOKEN_NAME ) self.AdjustPrivilege( ntsecuritycon.SE_TCB_NAME ) try: logonHandle = win32security.LogonUser( userName, None, passWord, win32con.LOGON32_LOGON_INTERACTIVE, win32con.LOGON32_PROVIDER_DEFAULT ) except pywintypes.error, ErrorMsg: return 0, ErrorMsg[ 2 ], None userInfo = win32net.NetUserGetInfo( None, userName, 1 ) return 1, 'Login successful', filesys.os_filesystem( userInfo[ 'home_dir' ] ) def AdjustPrivilege( self, priv ): flags = ntsecuritycon.TOKEN_ADJUST_PRIVILEGES | ntsecuritycon.TOKEN_QUERY htoken = win32security.OpenProcessToken(win32api.GetCurrentProcess(), flags) id = win32security.LookupPrivilegeValue(None, priv) newPrivileges = [(id, ntsecuritycon.SE_PRIVILEGE_ENABLED)] win32security.AdjustTokenPrivileges(htoken, 0, newPrivileges) def start_Server(): # ftpServ = ftp_server.ftp_server( ftp_server.anon_authorizer( "D:\MyDocuments\MyDownloads"), port=21 ) ftpServ = ftp_server.ftp_server( Win32Authorizer(), port=21 ) asyncore.loop() if __name__ == "__main__": print "Starting FTP Server" start_Server() medusa-0.5.4/docs/0040775000076400007640000000000007725426233012060 5ustar amkamkmedusa-0.5.4/docs/async_blurbs.txt0100644000076400007640000000452107445723327015307 0ustar amkamk [from the win32 sdk named pipe documentation] ================================================== The simplest server process can use the CreateNamedPipe function to create a single instance of a pipe, connect to a single client, communicate with the client, disconnect the pipe, close the pipe handle, and terminate. Typically, however, a server process must communicate with multiple client processes. A server process can use a single pipe instance by connecting to and disconnecting from each client in sequence, but performance would be poor. To handle multiple clients simultaneously, the server process must create multiple pipe instances. There are three basic strategies for servicing multiple pipe instances. Create multiple threads (and/or processes) with a separate thread for each instance of the pipe. For an example of a multithreaded server process, see Multithreaded Server. Overlap operations by specifying an OVERLAPPED structure in the ReadFile, WriteFile, and ConnectNamedPipe functions. For an example of a server process that uses overlapped operations, see Server Using Overlapped Input and Output. Overlap operations by using the ReadFileEx and WriteFileEx functions, which specify a completion routine to be executed when the operation is complete. For an example of a server process that uses completion routines, see Server Using Completion Routines. The multithreaded server strategy is easy to write, because the thread for each instance handles communications for only a single client. The system allocates processor time to each thread as needed. But each thread uses system resources, which is a potential disadvantage for a server that handles a large number of clients. Other complications occur if the actions of one client necessitate communications with other clients (as for a network game program, where a move by one player must be communicated to the other players). With a single-threaded server, it is easier to coordinate operations that affect multiple clients, and it is easier to protect shared resources (for example, a database file) from simultaneous access by multiple clients. The challenge of a single-threaded server is that it requires coordination of overlapped operations in order to allocate processor time for handling the simultaneous needs of the clients. ================================================== medusa-0.5.4/docs/composing_producers.gif0100755000076400007640000000522307445723327016636 0ustar amkamkGIF87a,H*\ȰÇ#JHŋ3jȱǏ CIɓ(S\ɲ˗0cʜI͛8sɳϟ@JѣH*]ʴӧPpիXjʵׯ`Ê `سhӪ]@ّTʝKַ"˷/Y~ 7N0|2V<٫㏐Όs媗=z뙨UɦAVy(hWWqt[z>nXu=p__%6[侇7Nv?~Uw淃7zenϺuu/7 yhɩh>)U맍yjmnyE l ic:lF݊NJ*Ԗ .f)Rj^/^{nVK룟|XG kÀ>,Ņ[w̦2rȔR$ɘޅ&42.׬s ߘ'@-47KN &-LtDPGUmt^`')n=YG~q$HI>cH8Qwd("N-d!AQ!w`rZ3?sL(PxnӟP2s[bFͥ4'D$W`iNER74:g *FݗljFҢ(BudVVMW\RFU+5*T*uNGOI ]JR庩>i0zQ+aXzՓv1EF1:"|hX b%j(ǎB+QAY~ki[E*Q>M^HiTkjZr;~R'Ub-չVrZt-rj6xd]f"nFXے+_e/ԋPz%YϻXիTE؂A |kE؜X6}k.cfcٖ66j[iódQd1dO8c#h{#X?}##WW.Rjb޺% d˔e {yeJ4vs|9&˹f|57&KAoІˎq,ڇn"DS:2LϠ5TBRԨNWVհgMZָεw^=;medusa-0.5.4/docs/data_flow.gif0100755000076400007640000000777307445723327014526 0ustar amkamkGIF87a , H*\ȰÇ#JHŋ3jȱǏ CIɓ(S\ɲ˗0cʜI͛8sɳϟ@ JѣH*]ʴӧPJJիXjʵׯ`ÊKٳhӪ]˶۷pʝKݻx˷߿ LÈ+^̸ǐL˘3ks䷝CM2SGQnh뤯^vvPۮSfgoٺwKwO¿"߹7f~9tofgn5OiZxRTo}BїuL=/vw G~TK т`e#AaX:i%߆M~ HW^6bqxU蟊2n6"eI?S ZYD)dx16P9RVHS eayWnYU2щe9f袖jNEXfGsƩҎDogT4 &(Ox'fhfR袚i= i Ezv꫰"*֊Eޚm鐽:lB*JgvlDɭbQFf-D貫nj-b <]nK-K0wn.pKBձZ cݻ"3k<(kr社LA:W3&П Fq&34MGKEM- OC3\ V'uCf 7]vU IPf+DAr|lڜy;uoM{y8ւJn5B@ms8wN[zC8ڜ-cf^svWzدmm;%:9aOry7^1jo土>[3GYzC]N}`򐧼~ ޯ, :i[oex"WR椷m X o#=f[)vc ƐLVbq"_b~p@:8tO7D,QvD# 51|?毐Au "R)!yEBjv2VkIIV>2$KÎUn\e,aYFv2bYl8Twt//F66N,s"JLcZQ&trM(-M0)WkĹIrz3ռbKL2Ԥiw;q}nyMFf]ZZ[^fw%/Aˡ].{ߗӖ|MxՋ`WV;倭yD:P&4X RU$dHjj?l7F/{Xb'LaZJz0yId8-.4$rk<3x͡SU{F}utHKvOz[HR.4(˲;!LjW4QFT^'q\;W*pxaįz_>JXXCKccM8: \gS#Le,^Td6 TȨ^Ly>ǾuZE 䪯5.]^]5oSsh^zl˃槜$㯲9(-7?/bǽߞ8kXxf 9؀Mb*HuxHxp؁ lyQ&xU*1!0/! X'TA:%>8#<"(CHqHL7؄r(AVaZxC]vR`b/gX#jHiemhoxKqxcBu=8y8!SH4| bVwO7x!|~A&xch艜XJxU(('Ȋ_@RȋHxh`1ьo苀1)vXNN^811'JgsrX%h}g{(m'=lHTm"XDn}x%Bׂ9i:h8y{tsgu:j9;O$B`W388fTjhX\;iuUTipz]kq0dByvLFLTfඖqbef8\sE kysvA71&KIiVYqtGxk*ugW\]CalVivAbO`)h}95СxC甅.whS՘MiFŘVVl{|9R7 Grjəv^g'iRh ?y6}֛ yL9yd)oηnQnTw @ق5Y9:i*i&O"}ɎAřv!S }2X!(O).Zs04Z3zi:9ڣ:G@B:£FrLKGReTZyX ђj[:Yڥd0!xlڦnpr:tZx;medusa-0.5.4/docs/data_flow.html0100644000076400007640000000727507445723327014717 0ustar amkamk

Data Flow in Medusa

Data flow, both input and output, is asynchronous. This is signified by the request and reply queues in the above diagram. This means that both requests and replies can get 'backed up', and are still handled correctly. For instance, HTTP/1.1 supports the concept of pipelined requests, where a series of requests are sent immediately to a server, and the replies are sent as they are processed. With a synchronous request, the client would have to wait for a reply to each request before sending the next.

The input data is partitioned into requests by looking for a terminator. A terminator is simply a protocol-specific delimiter - often simply CRLF (carriage-return line-feed), though it can be longer (for example, MIME multi-part boundaries can be specified as terminators). The protocol handler is notified whenever a complete request has been received.

The protocol handler then generates a reply, which is enqueued for output back to the client. Sometimes, instead of queuing the actual data, an object that will generate this data is used, called a producer.

The use of producers gives the programmer extraordinary control over how output is generated and inserted into the output queue. Though they are simple objects (requiring only a single method, more(), to be defined), they can be composed - simple producers can be wrapped around each other to create arbitrarily complex behaviors. [now would be a good time to browse through some of the producer classes in producers.py.]

The HTTP/1.1 producers make an excellent example. HTTP allows replies to be encoded in various ways - for example a reply consisting of dynamically-generated output might use the 'chunked' transfer encoding to send data that is compressed on-the-fly.

In the diagram, green producers actually generate output, and grey ones transform it in some manner. This producer might generate output looking like this: 

                            HTTP/1.1 200 OK
                            Content-Encoding: gzip
                            Transfer-Encoding: chunked
              Header ==>    Date: Mon, 04 Aug 1997 21:31:44 GMT
                            Content-Type: text/html
                            Server: Medusa/3.0
                            
             Chunking ==>   0x200
            Compression ==> <512 bytes of compressed html>
                            0x200
                            <512 bytes of compressed html>
                            ...
                            0

Still more can be done with this output stream: For the purpose of efficiency, it makes sense to send output in large, fixed-size chunks: This transformation can be applied by wrapping a 'globbing' producer around the whole thing.

An important feature of Medusa's producers is that they are actually rather small objects that do not expand into actual output data until the moment they are needed: The async_chat class will only call on a producer for output when the outgoing socket has indicated that it is ready for data. Thus Medusa is extremely efficient when faced with network delays, 'hiccups', and low bandwidth clients.

One final note: The mechanisms described above are completely general - although the examples given demonstrate application to the http protocol, Medusa's asynchronous core has been applied to many different protocols, including smtp, pop3, ftp, and even dns. medusa-0.5.4/docs/debugging.txt0100644000076400007640000000551507445723327014560 0ustar amkamk=========================================================================== The Monitor Server =========================================================================== The monitor server gives the developer a way to get into a server while it's running. Here's a quick demonstration of how to get to your server objects while in the monitor: [rushing@gnome medusa]$ python monitor_client.py 127.0.0.1 9999 Enter Password: Python 1.5 (#47, Jul 27 1998, 00:59:35) [GCC egcs-2.90.29 980515 (egcs-1.0.3 release)] Copyright 1991-1995 Stichting Mathematisch Centrum, Amsterdam Welcome to >>> from __main__ import * >>> dir() ['CHAT_PORT', 'FTP_PORT', 'HOSTNAME', 'HTTP_PORT', 'IP_ADDRESS', 'MONITOR_PORT', 'PUBLISHING_ROOT', '__builtins__', 'asyncore', 'chat_server', 'cs', 'debug_mode', 'default_handler', 'dh', 'filesys', 'fs', 'ftp', 'ftp_server', 'hs', 'http_server', 'lg', 'logger', 'monitor', 'ms', 'os', 'resolver', 'rs', 'sh', 'status_handler', 'status_objects', 'sys', 'uh', 'unix_user_handler'] >>> ms >>> rs >>> hs >>> dir(hs) ['_fileno', 'accept', 'accepting', 'addr', 'bind', 'bytes_in', 'bytes_out', 'close', 'connect', 'connect_ex', 'dup', 'exceptions', 'family_and_type', 'fileno', 'getpeername', 'getsockname', 'getsockopt', 'handlers', 'ip', 'listen', 'logger', 'makefile', 'port', 'recv', 'recvfrom', 'send', 'sendto', 'server_name', 'server_port', 'setblocking', 'setsockopt', 'shutdown', 'socket', 'total_clients', 'total_requests'] >>> hs.total_clients >>> cs >>> cs.close() log: closing channel 9: >>> Use 'exit' or 'Ctrl-D' to close a monitor session. =========================================================================== Emergency Debug Mode =========================================================================== Bugs and memory leaks in long-running servers may take weeks to show up. A useful debugging technique is to leave a crippled or bloated server running, but to move the servers' sockets to different ports so that you can start up another fresh copy in the meanwhile. This allows you to debug the servers without forcing you to leave your site broken while you do it. Here's how: Uncomment the support for '/status/emergency_debug' in When your server goes ballistic, simply hit this URL. This will shut down the servers, and restart them on new ports. Add 10000 to the old port number, so if you were running a web server on port 80, it will be moved to 10080. Start up another copy of your system, which will run on the original ports. Now you can debug the errant servers in isolation. Beats using GDB! medusa-0.5.4/docs/producers.gif0100755000076400007640000001326607445723327014566 0ustar amkamkGIF87a),)H*\ȰÇ#JHŋ3jȱǏ CIɓ(S\ɲ˗0cʜI͛8sɳϟ@ JѣH*]ʴӧPJJիXjʵׯ`ÊKٳhӪ]˖)pʝKݻxm˷ߋKa&≯1R!;L9d+k,3Ϡfv::BKEͺujϮc˾ziٸs3>:<U̇FN=׫k=[ݷG>9ܕǫ眞a\7oCO[5 Me-f ^5!LVaT چwhb_҇'I*Ģ4!+֨#Y3_(#Dr%E&Ց'1䓻5唫U$XnteYvI^Ӎ3n) xh8"5Y&x"f|)ZgdHe 1(zPNQmqR{B榄fZ*4hju h*ǵ@z]}TbZs",iʚE{дV;-bb$ֆoɕ-.A.rZ/;0` W,k0n 1Olrm"Ʋur3/1Ҳ2:-mxAlJZs/ݚޢ{ў34tRI64<'ەz)|𽓎 ?|p|1QOqcގׇʎGNw>۟xM7 vhgC歮n3 ڷiI˛ڴA jDPAS{'g]lǸf2fo1k aHBɰ_: q P5>xDF!/R9,i `&0idT}4<юh#n8?=S~ ^HH15H)2'ld"YJ{K(y m}T_N:2x?Su䇸:іt /C&B`:KL \`_X?Sy aX21Όio{b)!d~"f/]\r~L&ECbʏ猏7gr)1z3p% IЂg)͎qBJQDM"PD!O=5R(::q?UGϸY43L{uъp0bdd˧2Ks X<- Pơ2;N}jl*'N5V)}d7PIY*DQrjQ]mZvrtdEKÚQf$Bj#AΪ&5TX["i`gWe;k6*mcK Zge\]KXֲIE)mW+ʴ.`m[J- YEk#&lJ7;5qgkݻUMnw]6WU.s_{+4zYqW%oxz_j,o\׾Y **`BXo|a v+4|UX8i ឦ8qE<~1{8·v1bgΏ\\!3Yb@0R1'Ư19[g_C -N:kjC+Y1N Q>׷evgv'1_ѥމO*RfTDV!k[S׹m_ lNظ6=jeMv}JӾo;9zmizk^c56|m5rbP`_hr3 oS^N ny; xɩUڶ_ϊeuXlĺPCSz"/_>*ZwEV~l~9eCvr>-2Uc (|7%=+7d>t-ח桽WOVoo]%;YipW`&}u\pGܣv_u mUYm2in6}~D92?@A IA#pw,|J('x%X/z{ԀTggnӲzmyrzVi|$cRCh9{ (w*ɓn||#'{m )=d4hoG—wLGXy]:Wx٘w GX`uw|0Z"7mphDX+D m VVYocɝYٝZ Dy36郅ɆtNR(yi}}ٟ3y~(A(빘LyaNw@]hC:w9̙Fcw =u i'#k 鎫)y{-Z^ %x7)<Yy Hꀃ,)vBXe5%:iꘞj? @6]jiԹ3FDd؈SʧʥVQʨu'xyXe:xJiUꐃJ{Cy ss誯jmuZvCj :"JH*_[RJʫz"61ԪU:= a"݊v]H[t7|ȮVڏAw3 zZ:׵W K[(BaUuw7űu)R$z&ʪC~ԱJ|dh{2%4+QS1{+KHy@IK+:f;uzXXOKQq&Z^dgSKVl+#n۲bz[ѷ<۳KktXw[:P6kcp;)a{;;5S;[ TaZۨ, ۸1۱Z`[^#r;K"[;eR  뱼yضܫ>]S)ڛ[p k%V$1%#c'(u֑ |JK!⛲ ;ha \l {*\{U $\;F ,^)L!,K^$/+|<,ڿ>;PßA|H<=|ś뷻+P#\,:I&ŔAĘ b3,E#ƈǘrl e\gǜZ,th{ך&k <ȄvJm||}-:~%^"M.1M^F6m^o}PԶ㴬[O>Y~ zv拼IIh.J hgSZn΄N~Sv+Gn6w^_'X7 dzqκB?Gy }蝘W tuh'NˈeAUn_>>^~؞ھ>^˸3;medusa-0.5.4/docs/programming.html0100644000076400007640000006631607600125604015265 0ustar amkamk Programming in Python with Medusa and the Async Sockets Library

Programming in Python with Medusa and the Async Sockets Library

Introduction

Why Asynchronous?

There are only two ways to have a program on a single processor do 'more than one thing at a time'. Multi-threaded programming is the simplest and most popular way to do it, but there is another very different technique, that lets you have nearly all the advantages of multi-threading, without actually using multiple threads. It's really only practical if your program is I/O bound (I/O is the principle bottleneck). If your program is CPU bound, then pre-emptive scheduled threads are probably what you really need. Network servers are rarely CPU-bound, however.

If your operating system supports the select() system call in its I/O library (and nearly all do), then you can use it to juggle multiple communication channels at once; doing other work while your I/O is taking place in the "background". Although this strategy can seem strange and complex (especially at first), it is in many ways easier to understand and control than multi-threaded programming. The library documented here solves many of the difficult problems for you, making the task of building sophisticated high-performance network servers and clients a snap.

Select-based multiplexing in the real world

Several well-known Web servers (and other programs) are written using exactly this technique: the thttpd and Zeus, and Squid Internet Object Cache servers are excellent examples.. The InterNet News server (INN) used this technique for several years before the web exploded.

An interesting web server comparison chart is available at the thttpd web site

Variations on a Theme: poll() and WaitForMultipleObjects

Of similar (but better) design is the poll() system call. The main advantage of poll() (for our purposes) is that it does not used fixed-size file-descriptor tables, and is thus more easily scalable than select(). poll() is only recently becoming widely available, so you need to check for availability on your particular operating system.

In the Windows world, the Win32 API provides a bewildering array of features for multiplexing. Although slightly different in semantics, the combination of Event objects and the WaitForMultipleObjects() interface gives essentially the same power as select() on Unix. A version of this library specific to Win32 has not been written yet, mostly because Win32 also provides select() (at least for sockets). If such an interface were written, it would have the advantage of allowing us to multiplex on other objects types, like named pipes and files.

select()

Here's what select() does: you pass in a set of file descriptors, in effect asking the operating system, "let me know when anything happens to any of these descriptors". (A descriptor is simply a numeric handle used by the operating system to keep track of a file, socket, pipe, or other I/O object. It is usually an index into a system table of some kind). You can also use a timeout, so that if nothing happens in the allotted period, select() will return control to your program.

select() takes three fd_set arguments; one for each of the following possible states/events: readability, writability, and exceptional conditions. The last set is less useful than it sounds; in the context of TCP/IP it refers to the presence of out-of-band (OOB) data. OOB is a relatively unportable and poorly used feature that you can (and should) ignore unless you really need it.

So that leaves only two types of events to build our programs around; read events and write events. As it turns out, this is actually enough to get by with, because other types of events can be implied by the sequencing of these two. It also keeps the low-level interface as simple as possible - always a good thing in my book.

The polling loop

Now that you know what select() does, you're ready for the final piece of the puzzle: the main polling loop. This is nothing more than a simple while loop that continually calls select() with a timeout (I usually use a 30-second timeout). Such a program will use virtually no CPU if your server is idle; it spends most of its time letting the operating system do the waiting for it. This is much more efficient than a busy-wait loop.

Here is a pseudo-code example of a polling loop: 

while (any_descriptors_left):
  events = select (descriptors, timeout)
  for event in events:
    handle_event (event)

If you take a look at the code used by the library, it looks very similar to this. (see the file asyncore.py, the functions poll() and loop()). Now, on to the magic that must take place to handle the events...

The Code

Blocking vs. Non-Blocking

File descriptors can be in either blocking or non-blocking mode. A descriptor in blocking mode will stop (or 'block') your entire program until the requested event takes place. For example, if you ask to read 64 bytes from a descriptor attached to a socket which is ultimately connected to a modem deep in the backwaters of the Internet, you may wait a while for those 64 bytes.

If you put the descriptor in non-blocking mode, then one of two things might happen: if the data is sitting in a local buffer, it will be returned to you immediately; otherwise you will get back a code (usually EWOULDBLOCK) telling you that the read is in progress, and you should check back later to see if it's done.

sockets vs. other kinds of descriptors

Although most of our discussion will be about TCP/IP sockets, on Unix you can use select() to multiplex other kinds of communications objects, like pipes and ttys. (Unfortunately, select() cannot be used to do non-blocking file I/O. Please correct me if you have information to the contrary!)

The socket_map

We use a global dictionary (asyncore.socket_map) to keep track of all the active socket objects. The keys for this dictionary are the objects themselves. Nothing is stored in the value slot. Each time through the loop, this dictionary is scanned. Each object is asked which fd_sets it wants to be in. These sets are then passed on to select().

asyncore.dispatcher

The first class we'll introduce you to is the dispatcher class. This is a thin wrapper around a low-level socket object. We have attached a few methods for event-handling to it. Otherwise, it can be treated as a normal non-blocking socket object.

The direct interface between the select loop and the socket object are the handle_read_event and handle_write_event methods. These are called whenever an object 'fires' that event.

The firing of these low-level events can tell us whether certain higher-level events have taken place, depending on the timing and state of the connection. For example, if we have asked for a socket to connect to another host, we know that the connection has been made when the socket fires a write event (at this point you know that you may write to it with the expectation of success).
The implied events are

  • handle_connect.
    implied by a write event.
  • handle_close
    implied by a read event with no data available.
  • handle_accept
    implied by a read event on a listening socket.

Thus, the set of user-level events is a little larger than simply readable and writeable. The full set of events your code may handle are:

  • handle_read
  • handle_write
  • handle_expt (OOB data)
  • handle_connect
  • handle_close
  • handle_accept

A quick terminology note: In order to distinguish between low-level socket objects and those based on the async library classes, I call these higher-level objects channels.

Enough Gibberish, let's write some code

Ok, that's enough abstract talk. Let's do something useful and concrete with this stuff. We'll write a simple HTTP client that demonstrates how easy it is to build a powerful tool in only a few lines of code. 

# -*- Mode: Python; tab-width: 4 -*-

import asyncore
import socket
import string

class http_client (asyncore.dispatcher):

    def __init__ (self, host, path):
        asyncore.dispatcher.__init__ (self)
        self.path = path
        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
        self.connect ((host, 80))

    def handle_connect (self):
        self.send ('GET %s HTTP/1.0\r\n\r\n' % self.path)

    def handle_read (self):
        data = self.recv (8192)
        print data

    def handle_write (self):
        pass

if __name__ == '__main__':
    import sys
    import urlparse
    for url in sys.argv[1:]:
        parts = urlparse.urlparse (url)
        if parts[0] != 'http':
            raise ValueError, "HTTP URL's only, please"
        else:
            host = parts[1]
            path = parts[2]
            http_client (host, path)
    asyncore.loop()

HTTP is (in theory, at least) a very simple protocol. You connect to the web server, send the string "GET /some/path HTTP/1.0", and the server will send a short header, followed by the file you asked for. It will then close the connection.

We have defined a single new class, http_client, derived from the abstract class asyncore.dispatcher. There are three event handlers defined.

  • handle_connect
    Once we have made the connection, we send the request string.
  • handle_read
    As the server sends data back to us, we simply print it out.
  • handle_write
    Ignore this for the moment, I'm brushing over a technical detail we'll clean up in a moment.

Go ahead and run this demo - giving a single URL as an argument, like this:

$ python asynhttp.py http://www.nightmare.com/

You should see something like this: 

[rushing@gnome demo]$ python asynhttp.py http://www.nightmare.com/
log: adding channel <http_client  at 80ef3e8>
HTTP/1.0 200 OK
Server: Medusa/3.19
Content-Type: text/html
Content-Length: 1649
Last-Modified: Sun, 26 Jul 1998 23:57:51 GMT
Date: Sat, 16 Jan 1999 13:04:30 GMT

[... body of the file ...]

log: unhandled close event
log: closing channel 4:<http_client connected at 80ef3e8>

The 'log' messages are there to help, they are useful when debugging but you will want to disable them later. The first log message tells you that a new http_client object has been added to the socket map. At the end, you'll notice there's a warning that you haven't bothered to handle the close event. No big deal, for now.

Now at this point we haven't seen anything revolutionary, but that's because we've only looked at one URL. Go ahead and add a few other URL's to the argument list; as many as you like - and make sure they're on different hosts...

Now you begin to see why select() is so powerful. Depending on your operating system (and its configuration), select() can be fed hundreds, or even thousands of descriptors like this. (I've recently tested select() on a FreeBSD box with over 10,000 descriptors).

A really good way to understand select() is to put a print statement into the asyncore.poll() function: 

        [...]
        (r,w,e) = select.select (r,w,e, timeout)
        print '---'
        print 'read', r
        print 'write', w
        [...]

Each time through the loop you will see which channels have fired which events. If you haven't skipped ahead, you'll also notice a pointless barrage of events, with all your http_client objects in the 'writable' set. This is because we were a bit lazy earlier; sweeping some ugliness under the rug. Let's fix that now.

Buffered Output

In our handle_connect, we cheated a bit by calling send without examining its return code. In truth, since we are using a non-blocking socket, it's (theoretically) possible that our data didn't get sent. To do this correctly, we actually need to set up a buffer of outgoing data, and then send as much of the buffer as we can whenever we see a write event: 


class http_client (asyncore.dispatcher):

    def __init__ (self, host, path):
        asyncore.dispatcher.__init__ (self)
        self.path = path
        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
        self.connect ((host, 80))
        self.buffer = 'GET %s HTTP/1.0\r\n\r\n' % self.path

    def handle_connect (self):
        pass

    def handle_read (self):
        data = self.recv (8192)
        print data

    def writable (self):
        return (len(self.buffer) > 0)

    def handle_write (self):
        sent = self.send (self.buffer)
        self.buffer = self.buffer[sent:]

The handle_connect method no longer assumes it can send its request string successfully. We move its work over to handle_write; which trims self.buffer as pieces of it are sent succesfully.

We also introduce the writable method. Each time through the loop, the set of sockets is scanned, the readable and writable methods of each object are called to see if are interested in those events. The default methods simply return 1, indicating that by default all channels will be in both sets. In this case, however, we are only interested in writing as long as we have something to write. So we override this method, making its behavior dependent on the length of self.buffer.

If you try the client now (with the print statements in asyncore.poll()), you'll see that select is firing more efficiently.

asynchat.py

The dispatcher class is useful, but somewhat limited in capability. As you might guess, managing input and output buffers manually can get complex, especially if you're working with a protocol more complicated than HTTP.

The async_chat class does a lot of the heavy lifting for you. It automatically handles the buffering of both input and output, and provides a "line terminator" facility that partitions an input stream into logical lines for you. It is also carefully designed to support pipelining - a nice feature that we'll explain later.

There are four new methods to introduce:

  • set_terminator (self, <eol-string>)
    Set the string used to identify end-of-line. For most Internet protocols, this is the string \r\n, that is; a carriage return followed by a line feed. To turn off input scanning, use None
  • collect_incoming_data (self, data)
    Called whenever data is available from a socket. Usually, your implementation will accumulate this data into a buffer of some kind.
  • found_terminator (self)
    Called whenever an end-of-line marker has been seen. Typically your code will process and clear the input buffer.
  • push (data)
    This is a buffered version of send. It will place the data in an outgoing buffer.

These methods build on the underlying capabilities of dispatcher by providing implementations of handle_read handle_write, etc... handle_read collects data into an input buffer, which is continually scanned for the terminator string. Data in between terminators is feed to your collect_incoming_data method.

The implementation of handle_write and writable examine an outgoing-data queue, and automatically send data whenever possible.

A Proxy Server

In order to demonstrate the async_chat class, we will put together a simple proxy server. A proxy server combines a server and a client together, in effect sitting between the real server and client. You can use this to monitor or debug protocol traffic. 

# -*- Mode: Python; tab-width: 4 -*-

import asynchat
import asyncore
import socket
import string

class proxy_server (asyncore.dispatcher):
    
    def __init__ (self, host, port):
        asyncore.dispatcher.__init__ (self)
        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
        self.set_reuse_addr()
        self.there = (host, port)
        here = ('', port + 8000)
        self.bind (here)
        self.listen (5)

    def handle_accept (self):
        proxy_receiver (self, self.accept())

class proxy_sender (asynchat.async_chat):

    def __init__ (self, receiver, address):
        asynchat.async_chat.__init__ (self)
        self.receiver = receiver
        self.set_terminator (None)
        self.create_socket (socket.AF_INET, socket.SOCK_STREAM)
        self.buffer = ''
        self.set_terminator ('\n')
        self.connect (address)

    def handle_connect (self):
        print 'Connected'

    def collect_incoming_data (self, data):
        self.buffer = self.buffer + data

    def found_terminator (self):
        data = self.buffer
        self.buffer = ''
        print '==> (%d) %s' % (self.id, repr(data))
        self.receiver.push (data + '\n')

     def handle_close (self):
         self.receiver.close()
         self.close()

class proxy_receiver (asynchat.async_chat):

    channel_counter = 0

    def __init__ (self, server, (conn, addr)):
        asynchat.async_chat.__init__ (self, conn)
        self.set_terminator ('\n')
        self.server = server
        self.id = self.channel_counter
        self.channel_counter = self.channel_counter + 1
        self.sender = proxy_sender (self, server.there)
        self.sender.id = self.id
        self.buffer = ''

    def collect_incoming_data (self, data):
        self.buffer = self.buffer + data
        
    def found_terminator (self):
        data = self.buffer
        self.buffer = ''
        print '<== (%d) %s' % (self.id, repr(data))
        self.sender.push (data + '\n')

     def handle_close (self):
         print 'Closing'
         self.sender.close()
         self.close()

if __name__ == '__main__':
    import sys
    import string
    if len(sys.argv) < 3:
        print 'Usage: %s <server-host> <server-port>' % sys.argv[0]
    else:
        ps = proxy_server (sys.argv[1], string.atoi (sys.argv[2]))
        asyncore.loop()

To try out the proxy, find a server (any SMTP, NNTP, or HTTP server should do fine), and give its hostname and port as arguments: 

python proxy.py localhost 25

The proxy server will start up its server on port n + 8000, in this case port 8025. Now, use a telnet program to connect to that port on your server host. Issue a few commands. See how the whole session is being echoed by your proxy server. Try opening up several simultaneous connections through your proxy. You might also try pointing a real client (a news reader [port 119] or web browser [port 80]) at your proxy.

Pipelining

Pipelining refers to a protocol capability. Normally, a conversation with a server has a back-and-forth quality to it. The client sends a command, and waits for the response. If a client needs to send many commands over a high-latency connection, waiting for each response can take a long time.

For example, when sending a mail message to many recipients with SMTP, the client will send a series of RCPT commands, one for each recipient. For each of these commands, the server will send back a reply indicating whether the mailbox specified is valid. If you want to send a message to several hundred recipients, this can be rather tedious if the round-trip time for each command is long. You'd like to be able to send a bunch of RCPT commands in one batch, and then count off the responses to them as they come.

I have a favorite visual when explaining the advantages of pipelining. Imagine each request to the server is a boxcar on a train. The client is in Los Angeles, and the server is in New York. Pipelining lets you hook all your cars in one long chain; send them to New York, where they are filled and sent back to you. Without pipelining you have to send one car at a time.

Not all protocols allow pipelining. Not all servers support it; Sendmail, for example, does not support pipelining because it tends to fork unpredictably, leaving buffered data in a questionable state. A recent extension to the SMTP protocol allows a server to specify whether it supports pipelining. HTTP/1.1 explicitly requires that a server support pipelining.

Servers built on top of async_chat automatically support pipelining. It is even possible to change the terminator repeatedly when processing data already in the input buffer. See the handle_read method if you're interested in the gory details.

Producers

async_chat supports a sophisticated output buffering model, using a queue of data-producing objects. For most purposes, you will use the push() method to send string data - but for more sophisticated usage you can push a producer

A producer is a very simple object, requiring only a single method in its implementation, more(). See the code for simple_producer in asynchat.py for an example. Many more examples are available in the Medusa distribution, in the file producers.py

Samual M. Rushing
Last modified: Fri Apr 30 21:42:52 PDT 1999 medusa-0.5.4/docs/proxy_notes.txt0100644000076400007640000000335607445723327015217 0ustar amkamk # we can build 'promises' to produce external data. Each producer # contains a 'promise' to fetch external data (or an error # message). writable() for that channel will only return true if the # top-most producer is ready. This state can be flagged by the dns # client making a callback. # So, say 5 proxy requests come in, we can send out DNS queries for # them immediately. If the replies to these come back before the # promises get to the front of the queue, so much the better: no # resolve delay. 8^) # # ok, there's still another complication: # how to maintain replies in order? # say three requests come in, (to different hosts? can this happen?) # yet the connections happen third, second, and first. We can't buffer # the entire request! We need to be able to specify how much to buffer. # # =========================================================================== # # the current setup is a 'pull' model: whenever the channel fires FD_WRITE, # we 'pull' data from the producer fifo. what we need is a 'push' option/mode, # where # 1) we only check for FD_WRITE when data is in the buffer # 2) whoever is 'pushing' is responsible for calling 'refill_buffer()' # # what is necessary to support this 'mode'? # 1) writable() only fires when data is in the buffer # 2) refill_buffer() is only called by the 'pusher'. # # how would such a mode affect things? with this mode could we support # a true http/1.1 proxy? [i.e, support pipelined proxy requests, possibly # to different hosts, possibly even mixed in with non-proxy requests?] For # example, it would be nice if we could have the proxy automatically apply the # 1.1 chunking for 1.0 close-on-eof replies when feeding it to the client. This # would let us keep our persistent connection. medusa-0.5.4/docs/README.html0100644000076400007640000001766607575741035013721 0ustar amkamk

What is Medusa?

Medusa is an architecture for very-high-performance TCP/IP servers (like HTTP, FTP, and NNTP). Medusa is different from most other servers because it runs as a single process, multiplexing I/O with its various client and server connections within a single process/thread.

It is capable of smoother and higher performance than most other servers, while placing a dramatically reduced load on the server machine. The single-process, single-thread model simplifies design and enables some new persistence capabilities that are otherwise difficult or impossible to implement.

Medusa is supported on any platform that can run Python and includes a functional implementation of the <socket> and <select> modules. This includes the majority of Unix implementations.

During development, it is constantly tested on Linux and Win32 [Win95/WinNT], but the core asynchronous capability has been shown to work on several other platforms, including the Macintosh. It might even work on VMS.

The Power of Python

A distinguishing feature of Medusa is that it is written entirely in Python. Python (http://www.python.org/) is a 'very-high-level' object-oriented language developed by Guido van Rossum (currently at CNRI). It is easy to learn, and includes many modern programming features such as storage management, dynamic typing, and an extremely flexible object system. It also provides convenient interfaces to C and C++.

The rapid prototyping and delivery capabilities are hard to exaggerate; for example

  • It took me longer to read the documentation for persistent HTTP connections (the 'Keep-Alive' connection token) than to add the feature to Medusa.
  • A simple IRC-like chat server system was written in about 90 minutes.

I've heard similar stories from alpha test sites, and other users of the core async library.

Server Notes

Both the FTP and HTTP servers use an abstracted 'filesystem object' to gain access to a given directory tree. One possible server extension technique would be to build behavior into this filesystem object, rather than directly into the server: Then the extension could be shared with both the FTP and HTTP servers.

HTTP

The core HTTP server itself is quite simple - all functionality is provided through 'extensions'. Extensions can be plugged in dynamically. [i.e., you could log in to the server via the monitor service and add or remove an extension on the fly]. The basic file-delivery service is provided by a 'default' extension, which matches all URI's. You can build more complex behavior by replacing or extending this class.

The default extension includes support for the 'Connection: Keep-Alive' token, and will re-use a client channel when requested by the client.

FTP

On Unix, the ftp server includes support for 'real' users, so that it may be used as a drop-in replacement for the normal ftp server. Since most ftp servers on Unix use the 'forking' model, each child process changes its user/group persona after a successful login. This is a appears to be a secure design.

Medusa takes a different approach - whenever Medusa performs an operation for a particular user [listing a directory, opening a file], it temporarily switches to that user's persona _only_ for the duration of the operation. [and each such operation is protected by a try/finally exception handler].

To do this Medusa MUST run with super-user privileges. This is a HIGHLY experimental approach, and although it has been thoroughly tested on Linux, security problems may still exist. If you are concerned about the security of your server machine, AND YOU SHOULD BE, I suggest running Medusa's ftp server in anonymous-only mode, under an account with limited privileges ('nobody' is usually used for this purpose).

I am very interested in any feedback on this feature, most especially information on how the server behaves on different implementations of Unix, and of course any security problems that are found.

Monitor

The monitor server gives you remote, 'back-door' access to your server while it is running. It implements a remote python interpreter. Once connected to the monitor, you can do just about anything you can do from the normal python interpreter. You can examine data structures, servers, connection objects. You can enable or disable extensions, restart the server, reload modules, etc...

The monitor server is protected with an MD5-based authentication similar to that proposed in RFC1725 for the POP3 protocol. The server sends the client a timestamp, which is then appended to a secret password. The resulting md5 digest is sent back to the server, which then compares this to the expected result. Failed login attempts are logged and immediately disconnected. The password itself is not sent over the network (unless you have foolishly transmitted it yourself through an insecure telnet or X11 session. 8^)

For this reason telnet cannot be used to connect to the monitor server when it is in a secure mode (the default). A client program is provided for this purpose. You will be prompted for a password when starting up the server, and by the monitor client.

For extra added security on Unix, the monitor server will eventually be able to use a Unix-domain socket, which can be protected behind a 'firewall' directory (similar to the InterNet News server).

Performance Notes

The select() function

At the heart of Medusa is a single select() loop. This loop handles all open socket connections, both servers and clients. It is in effect constantly asking the system: 'which of these sockets has activity?'. Performance of this system call can vary widely between operating systems.

There are also often builtin limitations to the number of sockets ('file descriptors') that a single process, or a whole system, can manipulate at the same time. Early versions of Linux placed draconian limits (256) that have since been raised. Windows 95 has a limit of 64, while OSF/1 seems to allow up to 4096.

These limits don't affect only Medusa, you will find them described in the documentation for other web and ftp servers, too.

The documentation for the Apache web server has some excellent notes on tweaking performance for various Unix implementations. See http://www.apache.org/docs/misc/perf.html for more information.

Buffer sizes

The default buffer sizes used by Medusa are set with a bias toward Internet-based servers: They are relatively small, so that the buffer overhead for each connection is low. The assumption is that Medusa will be talking to a large number of low-bandwidth connections, rather than a smaller number of high bandwidth.

This choice trades run-time memory use for efficiency - the down side of this is that high-speed local connections (i.e., over a local ethernet) will transfer data at a slower rate than necessary.

This parameter can easily be tweaked by the site designer, and can in fact be adjusted on a per-server or even per-client basis. For example, you could have the FTP server use larger buffer sizes for connections from certain domains.

If there's enough interest, I have some rough ideas for how to make these buffer sizes automatically adjust to an optimal setting. Send email if you'd like to see this feature.

See ./medusa.html for a brief overview of some of the ideas behind Medusa's design, and for a description of current and upcoming features.

Enjoy!


-Sam Rushing
rushing@nightmare.com medusa-0.5.4/docs/threads.txt0100644000076400007640000000413207445723327014251 0ustar amkamk# -*- Mode: Text; tab-width: 4 -*- [note, a better solution is now available, see the various modules in the 'thread' directory (SMR 990105)] A Workable Approach to Mixing Threads and Medusa. --------------------------------------------------------------------------- When Medusa receives a request that needs to be handled by a separate thread, have the thread remove the socket from Medusa's control, by calling the 'del_channel()' method, and put the socket into blocking-mode: request.channel.del_channel() request.channel.socket.setblocking (0) Now your thread is responsible for managing the rest of the HTTP 'session'. In particular, you need to send the HTTP response, followed by any headers, followed by the response body. Since the most common need for mixing threads and Medusa is to support CGI, there's one final hurdle that should be pointed out: CGI scripts sometimes make use of a 'Status:' hack (oops, I meant to say 'header') in order to tell the server to return a reply other than '200 OK'. To support this it is necessary to scan the output _before_ it is sent. Here is a sample 'write' method for a file-like object that performs this scan: HEADER_LINE = regex.compile ('\([A-Za-z0-9-]+\): \(.*\)') def write (self, data): if self.got_header: self._write (data) else: # CGI scripts may optionally provide extra headers. # # If they do not, then the output is assumed to be # text/html, with an HTTP reply code of '200 OK'. # # If they do, we need to scan those headers for one in # particular: the 'Status:' header, which will tell us # to use a different HTTP reply code [like '302 Moved'] # self.buffer = self.buffer + data lines = string.split (self.buffer, '\n') # look for something un-header-like for i in range(len(lines)): if i == (len(lines)-1): if lines[i] == '': break elif HEADER_LINE.match (lines[i]) == -1: # this is not a header line. self.got_header = 1 self.buffer = self.build_header (lines[:i]) # rejoin the rest of the data self._write (string.join (lines[i:], '\n')) break medusa-0.5.4/docs/tkinter.txt0100644000076400007640000000161707445723327014304 0ustar amkamk Here are some notes on combining the Tk Event loop with the async lib and/or Medusa. Many thanks to Aaron Rhodes (alrhodes@cpis.net) for the info! > Sam, > > Just wanted to send you a quick message about how I managed to > finally integrate Tkinter with asyncore. This solution is pretty > straightforward. From the main tkinter event loop i simply added > a repeating alarm that calls asyncore.poll() every so often. So > the code looks like this: > > in main: > import asyncore > > self.socket_check() > > ... > > then, socket_check() is: > > def socket_check(self): > asyncore.poll(timeout=0.0) > self.after(100, self.socket_check) > > > This simply causes asyncore to poll all the sockets every 100ms > during the tkinter event loop. The GUI doesn't block on IO since > all the IO calls are now handled with asyncore. medusa-0.5.4/test/0040775000076400007640000000000007725426233012107 5ustar amkamkmedusa-0.5.4/test/asyn_http_bench.py0100755000076400007640000000506207445740204015626 0ustar amkamk#! /usr/local/bin/python1.4 # -*- Mode: Python -*- import asyncore import socket import string import sys def blurt (thing): sys.stdout.write (thing) sys.stdout.flush () total_sessions = 0 class http_client (asyncore.dispatcher_with_send): def __init__ (self, host='127.0.0.1', port=80, uri='/', num=10): asyncore.dispatcher_with_send.__init__ (self) self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.host = host self.port = port self.uri = uri self.num = num self.bytes = 0 self.connect ((host, port)) def log (self, *info): pass def handle_connect (self): self.connected = 1 # blurt ('o') self.send ('GET %s HTTP/1.0\r\n\r\n' % self.uri) def handle_read (self): # blurt ('.') d = self.recv (8192) self.bytes = self.bytes + len(d) def handle_close (self): global total_sessions # blurt ('(%d)' % (self.bytes)) self.close() total_sessions = total_sessions + 1 if self.num: http_client (self.host, self.port, self.uri, self.num-1) import time class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start from asyncore import socket_map, poll MAX = 0 def loop (timeout=30.0): global MAX while socket_map: if len(socket_map) > MAX: MAX = len(socket_map) poll (timeout) if __name__ == '__main__': if len(sys.argv) < 6: print 'usage: %s ' % sys.argv[0] else: [host, port, uri, hits, num] = sys.argv[1:] hits = string.atoi (hits) num = string.atoi (num) port = string.atoi (port) t = timer() clients = map (lambda x: http_client (host, port, uri, hits-1), range(num)) #import profile #profile.run ('loop') loop() total_time = t.end() print ( '\n%d clients\n%d hits/client\n' 'total_hits:%d\n%.3f seconds\ntotal hits/sec:%.3f' % ( num, hits, total_sessions, total_time, total_sessions / total_time ) ) print 'Max. number of concurrent sessions: %d' % (MAX) # linux 2.x, talking to medusa # 50 clients # 1000 hits/client # total_hits:50000 # 2255.858 seconds # total hits/sec:22.165 # Max. number of concurrent sessions: 50 medusa-0.5.4/test/max_sockets.py0100644000076400007640000000256507445740204015000 0ustar amkamk# -*- Mode: Python -*- import socket import select # several factors here we might want to test: # 1) max we can create # 2) max we can bind # 3) max we can listen on # 4) max we can connect def max_server_sockets(): sl = [] while 1: try: s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.bind (('',0)) s.listen(5) sl.append (s) except: break num = len(sl) for s in sl: s.close() del sl return num def max_client_sockets(): # make a server socket server = socket.socket (socket.AF_INET, socket.SOCK_STREAM) server.bind (('', 9999)) server.listen (5) sl = [] while 1: try: s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect (('', 9999)) conn, addr = server.accept() sl.append ((s,conn)) except: break num = len(sl) for s,c in sl: s.close() c.close() del sl return num def max_select_sockets(): sl = [] while 1: try: s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.bind (('',0)) s.listen(5) sl.append (s) select.select(sl,[],[],0) except: break num = len(sl) - 1 for s in sl: s.close() del sl return num medusa-0.5.4/test/test_11.py0100644000076400007640000000625607445740204013741 0ustar amkamk# -*- Mode: Python -*- import asyncore import asynchat import socket import string # get some performance figures for an HTTP/1.1 server. # use pipelining. class test_client (asynchat.async_chat): ac_in_buffer_size = 16384 ac_out_buffer_size = 16384 total_in = 0 concurrent = 0 max_concurrent = 0 def __init__ (self, addr, chain): asynchat.async_chat.__init__ (self) self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.set_terminator ('\r\n\r\n') self.connect (addr) self.push (chain) def handle_connect (self): test_client.concurrent = test_client.concurrent + 1 if (test_client.concurrent > test_client.max_concurrent): test_client.max_concurrent = test_client.concurrent def handle_expt (self): print 'unexpected FD_EXPT thrown. closing()' self.close() def close (self): test_client.concurrent = test_client.concurrent - 1 asynchat.async_chat.close(self) def collect_incoming_data (self, data): test_client.total_in = test_client.total_in + len(data) def found_terminator (self): pass def log (self, *args): pass import time class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start def build_request_chain (num, host, request_size): s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host) sl = [s] * (num-1) sl.append ( 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % ( request_size, host ) ) return string.join (sl, '') if __name__ == '__main__': import string import sys if len(sys.argv) != 6: print 'usage: %s \n' % sys.argv[0] else: host = sys.argv[1] ip = socket.gethostbyname (host) [port, request_size, num_requests, num_conns] = map ( string.atoi, sys.argv[2:] ) chain = build_request_chain (num_requests, host, request_size) t = timer() for i in range (num_conns): test_client ((host,port), chain) asyncore.loop() total_time = t.end() # ok, now do some numbers total_bytes = test_client.total_in num_trans = num_requests * num_conns throughput = float (total_bytes) / total_time trans_per_sec = num_trans / total_time sys.stderr.write ('total time: %.2f\n' % total_time) sys.stderr.write ('number of transactions: %d\n' % num_trans) sys.stderr.write ('total bytes sent: %d\n' % total_bytes) sys.stderr.write ('total throughput (bytes/sec): %.2f\n' % throughput) sys.stderr.write ('transactions/second: %.2f\n' % trans_per_sec) sys.stderr.write ('max concurrent connections: %d\n' % test_client.max_concurrent) sys.stdout.write ( string.join ( map (str, (num_conns, num_requests, request_size, throughput, trans_per_sec)), ',' ) + '\n' ) medusa-0.5.4/test/test_lb.py0100644000076400007640000001115207445740204014104 0ustar amkamk# -*- Mode: Python -*- # Get a lower bound for Medusa performance with a simple async # client/server benchmark built on the async lib. The idea is to test # all the underlying machinery [select, asyncore, asynchat, etc...] in # a context where there is virtually no processing of the data. import socket import select import sys # ================================================== # server # ================================================== import asyncore import asynchat class test_channel (asynchat.async_chat): ac_in_buffer_size = 16384 ac_out_buffer_size = 16384 total_in = 0 def __init__ (self, conn, addr): asynchat.async_chat.__init__ (self, conn) self.set_terminator ('\r\n\r\n') self.buffer = '' def collect_incoming_data (self, data): self.buffer = self.buffer + data test_channel.total_in = test_channel.total_in + len(data) def found_terminator (self): # we've gotten the data, now send it back data = self.buffer self.buffer = '' self.push (data+'\r\n\r\n') def handle_close (self): sys.stdout.write ('.'); sys.stdout.flush() self.close() def log (self, *args): pass class test_server (asyncore.dispatcher): def __init__ (self, addr): if type(addr) == type(''): f = socket.AF_UNIX else: f = socket.AF_INET self.create_socket (f, socket.SOCK_STREAM) self.bind (addr) self.listen (5) print 'server started on',addr def handle_accept (self): conn, addr = self.accept() test_channel (conn, addr) # ================================================== # client # ================================================== # pretty much the same behavior, except that we kick # off the exchange and decide when to quit class test_client (test_channel): def __init__ (self, addr, packet, number): if type(addr) == type(''): f = socket.AF_UNIX else: f = socket.AF_INET asynchat.async_chat.__init__ (self) self.create_socket (f, socket.SOCK_STREAM) self.set_terminator ('\r\n\r\n') self.buffer = '' self.connect (addr) self.push (packet + '\r\n\r\n') self.number = number self.count = 0 def handle_connect (self): pass def found_terminator (self): self.count = self.count + 1 if self.count == self.number: sys.stdout.write('.'); sys.stdout.flush() self.close() else: test_channel.found_terminator (self) import time class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start if __name__ == '__main__': import string if '--poll' in sys.argv: sys.argv.remove ('--poll') use_poll=1 else: use_poll=0 if len(sys.argv) == 1: print 'usage: %s\n' \ ' (as a server) [--poll] -s \n' \ ' (as a client) [--poll] -c \n' % sys.argv[0] sys.exit(0) if sys.argv[1] == '-s': s = test_server ((sys.argv[2], string.atoi (sys.argv[3]))) asyncore.loop(use_poll=use_poll) elif sys.argv[1] == '-c': # create the packet packet = string.atoi(sys.argv[4]) * 'B' host = sys.argv[2] port = string.atoi (sys.argv[3]) num_packets = string.atoi (sys.argv[5]) num_conns = string.atoi (sys.argv[6]) t = timer() for i in range (num_conns): test_client ((host,port), packet, num_packets) asyncore.loop(use_poll=use_poll) total_time = t.end() # ok, now do some numbers bytes = test_client.total_in num_trans = num_packets * num_conns total_bytes = num_trans * len(packet) throughput = float (total_bytes) / total_time trans_per_sec = num_trans / total_time sys.stderr.write ('total time: %.2f\n' % total_time) sys.stderr.write ( 'number of transactions: %d\n' % num_trans) sys.stderr.write ( 'total bytes sent: %d\n' % total_bytes) sys.stderr.write ( 'total throughput (bytes/sec): %.2f\n' % throughput) sys.stderr.write ( ' [note, throughput is this amount in each direction]\n') sys.stderr.write ( 'transactions/second: %.2f\n' % trans_per_sec) sys.stdout.write ( string.join ( map (str, (num_conns, num_packets, len(packet), throughput, trans_per_sec)), ',' ) + '\n' ) medusa-0.5.4/test/test_medusa.py0100644000076400007640000000220307446201171014756 0ustar amkamk# -*- Mode: Python -*- import socket import string import time from medusa import http_date now = http_date.build_http_date (time.time()) cache_request = string.joinfields ( ['GET / HTTP/1.0', 'If-Modified-Since: %s' % now, ], '\r\n' ) + '\r\n\r\n' nocache_request = 'GET / HTTP/1.0\r\n\r\n' def get (request, host='', port=80): s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect((host, port)) s.send (request) while 1: d = s.recv (8192) if not d: break s.close() class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start def test_cache (n=1000): t = timer() for i in xrange (n): get(cache_request) end = t.end() print 'cache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end) def test_nocache (n=1000): t = timer() for i in xrange (n): get(nocache_request) end = t.end() print 'nocache: %d requests, %.2f seconds, %.2f hits/sec' % (n, end, n/end) if __name__ == '__main__': test_cache() test_nocache() medusa-0.5.4/test/test_single_11.py0100644000076400007640000000303007445740204015265 0ustar amkamk# -*- Mode: Python -*- # no-holds barred, test a single channel's pipelining speed import string import socket def build_request_chain (num, host, request_size): s = 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\n\r\n' % (request_size, host) sl = [s] * (num-1) sl.append ( 'GET /test%d.html HTTP/1.1\r\nHost: %s\r\nConnection: close\r\n\r\n' % ( request_size, host ) ) return string.join (sl, '') import time class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start if __name__ == '__main__': import sys if len(sys.argv) != 5: print 'usage: %s ' % (sys.argv[0]) else: host = sys.argv[1] [port, request_size, num_requests] = map ( string.atoi, sys.argv[2:] ) chain = build_request_chain (num_requests, host, request_size) import socket s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect ((host,port)) t = timer() s.send (chain) num_bytes = 0 while 1: data = s.recv(16384) if not data: break else: num_bytes = num_bytes + len(data) total_time = t.end() print 'total bytes received: %d' % num_bytes print 'total time: %.2f sec' % (total_time) print 'transactions/sec: %.2f' % (num_requests/total_time) medusa-0.5.4/test/tests.txt0100644000076400007640000000563007445723330014006 0ustar amkamk# server: linux, 486dx2/66 # client: win95, cyrix 6x86 p166+ # over ethernet. # # number of connections # | number of requests per connection # | | packet size # | | | throughput (bytes/sec) # | | | | transactions/sec # | | | | | 1 50 64 3440.86 53.76 1 100 64 3422.45 53.47 1 1 256 5120.00 20.00 1 50 256 13763.44 53.76 1 100 256 13333.33 52.08 1 1 1024 6400.00 6.25 1 50 1024 6909.58 6.74 1 100 1024 6732.41 6.57 1 1 4096 14628.56 3.57 1 50 4096 17181.20 4.19 1 100 4096 16835.18 4.11 5 1 64 1882.35 29.41 5 50 64 3990.02 62.34 5 100 64 3907.20 61.05 5 1 256 5818.18 22.72 5 50 256 15533.98 60.67 5 100 256 15744.15 61.50 5 1 1024 15515.14 15.15 5 50 1024 23188.40 22.64 5 100 1024 23659.88 23.10 5 1 4096 28444.44 6.94 5 50 4096 34913.05 8.52 5 100 4096 35955.05 8.77 10 1 64 191.04 2.98 10 50 64 4045.51 63.21 10 100 64 4045.51 63.21 10 1 256 764.17 2.98 10 50 256 15552.85 60.75 10 100 256 15581.25 60.86 10 1 1024 2959.53 2.89 10 50 1024 25061.18 24.47 10 100 1024 25498.00 24.90 10 1 4096 11314.91 2.76 10 50 4096 39002.09 9.52 10 100 4096 38780.53 9.46 15 1 64 277.45 4.33 15 50 64 4067.79 63.55 15 100 64 4083.36 63.80 15 1 256 386.31 1.50 15 50 256 15262.32 59.61 15 100 256 15822.00 61.80 15 1 1024 1528.35 1.49 15 50 1024 27263.04 26.62 15 100 1024 27800.90 27.14 15 1 4096 6047.24 1.47 15 50 4096 39695.05 9.69 15 100 4096 37112.65 9.06 20 1 64 977.09 15.26 20 50 64 2538.67 39.66 20 100 64 3377.30 52.77 20 1 256 221.93 0.86 20 50 256 10815.37 42.24 20 100 256 15880.89 62.03 20 1 1024 883.52 0.86 20 50 1024 29315.77 28.62 20 100 1024 29569.73 28.87 20 1 4096 7892.10 1.92 20 50 4096 40223.90 9.82 20 100 4096 41325.73 10.08 # # There's a big gap in trans/sec between 256 and 1024 bytes, we should # probably stick a 512 in there. # medusa-0.5.4/test/test_producers.py0100644000076400007640000000736707542157450015535 0ustar amkamk#!/usr/bin/env python # # Test script for producers.py # __revision__ = "$Id: test_producers.py,v 1.2 2002/09/18 20:16:40 akuchling Exp $" import StringIO, zlib from sancho.unittest import TestScenario, parse_args, run_scenarios tested_modules = ["medusa.producers"] from medusa import producers test_string = '' for i in range(16385): test_string += chr(48 + (i%10)) class ProducerTest (TestScenario): def setup (self): pass def shutdown (self): pass def _check_all (self, p, expected_string): # Check that a producer returns all of the string, # and that it's the unchanged string. count = 0 data = "" while 1: s = p.more() if s == "": break count += len(s) data += s self.test_val('count', len(expected_string)) self.test_val('data', expected_string) self.test_val('p.more()', '') return data def check_simple (self): p = producers.simple_producer(test_string) self.test_val('p.more()', test_string[:1024]) p = producers.simple_producer(test_string, buffer_size = 5) self._check_all(p, test_string) def check_scanning (self): p = producers.scanning_producer(test_string) self.test_val('p.more()', test_string[:1024]) p = producers.scanning_producer(test_string, buffer_size = 5) self._check_all(p, test_string) def check_lines (self): p = producers.lines_producer(['a']* 65) self._check_all(p, 'a\r\n'*65) def check_buffer (self): p = producers.buffer_list_producer(['a']* 1027) self._check_all(p, 'a'*1027) def check_file (self): f = StringIO.StringIO(test_string) p = producers.file_producer(f) self._check_all(p, test_string) def check_output (self): p = producers.output_producer() for i in range(0,66): p.write('a') for i in range(0,65): p.write('b\n') self._check_all(p, 'a'*66 + 'b\r\n'*65) def check_composite (self): p1 = producers.simple_producer('a'*66, buffer_size = 5) p2 = producers.lines_producer(['b']*65) p = producers.composite_producer([p1, p2]) self._check_all(p, 'a'*66 + 'b\r\n'*65) def check_glob (self): p1 = producers.simple_producer(test_string, buffer_size = 5) p = producers.globbing_producer(p1, buffer_size = 1024) self.test_true('1024 <= len(p.more())') def check_hooked (self): def f (num_bytes): self.test_val('num_bytes', len(test_string)) p1 = producers.simple_producer(test_string, buffer_size = 5) p = producers.hooked_producer(p1, f) self._check_all(p, test_string) def check_chunked (self): p1 = producers.simple_producer('the quick brown fox', buffer_size = 5) p = producers.chunked_producer(p1, footers=['FOOTER']) self._check_all(p, """5\r the q\r 5\r uick \r 5\r brown\r 4\r fox\r 0\r FOOTER\r \r\n""") def check_compressed (self): p1 = producers.simple_producer(test_string, buffer_size = 5) p = producers.compressed_producer(p1) compr_data = self._check_all(p, zlib.compress(test_string, 5)) self.test_val('zlib.decompress(compr_data)', test_string) def check_escaping (self): p1 = producers.simple_producer('the quick brown fox', buffer_size = 5) p = producers.escaping_producer(p1, esc_from = ' ', esc_to = '_') self._check_all(p, 'the_quick_brown_fox') # class ProducerTest if __name__ == "__main__": (scenarios, options) = parse_args() run_scenarios(scenarios, options) medusa-0.5.4/test/bench.py0100644000076400007640000000167007445740204013533 0ustar amkamk# -*- Mode: Python -*- # benchmark a single channel, pipelined request = 'GET /index.html HTTP/1.0\r\nConnection: Keep-Alive\r\n\r\n' last_request = 'GET /index.html HTTP/1.0\r\nConnection: close\r\n\r\n' import socket import time class timer: def __init__ (self): self.start = time.time() def end (self): return time.time() - self.start def bench (host, port=80, n=100): s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect ((host, port)) t = timer() s.send ((request * n) + last_request) while 1: d = s.recv(65536) if not d: break total = t.end() print 'time: %.2f seconds (%.2f hits/sec)' % (total, n/total) if __name__ == '__main__': import sys import string if len(sys.argv) < 3: print 'usage: %s ' % (sys.argv[0]) else: bench (sys.argv[1], string.atoi (sys.argv[2]), string.atoi (sys.argv[3])) medusa-0.5.4/thread/0040775000076400007640000000000007725426233012377 5ustar amkamkmedusa-0.5.4/thread/pi_module.py0100644000076400007640000000341007445740204014713 0ustar amkamk# -*- Mode: Python -*- # [reworking of the version in Python-1.5.1/Demo/scripts/pi.py] # Print digits of pi forever. # # The algorithm, using Python's 'long' integers ("bignums"), works # with continued fractions, and was conceived by Lambert Meertens. # # See also the ABC Programmer's Handbook, by Geurts, Meertens & Pemberton, # published by Prentice-Hall (UK) Ltd., 1990. import string StopException = "Stop!" def go (file): try: k, a, b, a1, b1 = 2L, 4L, 1L, 12L, 4L while 1: # Next approximation p, q, k = k*k, 2L*k+1L, k+1L a, b, a1, b1 = a1, b1, p*a+q*a1, p*b+q*b1 # Print common digits d, d1 = a/b, a1/b1 while d == d1: if file.write (str(int(d))): raise StopException a, a1 = 10L*(a%b), 10L*(a1%b1) d, d1 = a/b, a1/b1 except StopException: return class line_writer: "partition the endless line into 80-character ones" def __init__ (self, file, digit_limit=10000): self.file = file self.buffer = '' self.count = 0 self.digit_limit = digit_limit def write (self, data): self.buffer = self.buffer + data if len(self.buffer) > 80: line, self.buffer = self.buffer[:80], self.buffer[80:] self.file.write (line+'\r\n') self.count = self.count + 80 if self.count > self.digit_limit: return 1 else: return 0 def main (env, stdin, stdout): parts = string.split (env['REQUEST_URI'], '/') if len(parts) >= 3: ndigits = string.atoi (parts[2]) else: ndigits = 5000 stdout.write ('Content-Type: text/plain\r\n\r\n') go (line_writer (stdout, ndigits)) medusa-0.5.4/thread/select_trigger.py0100644000076400007640000002455107701142271015744 0ustar amkamk# -*- Mode: Python -*- ############################################################################## # # Copyright (c) 2001, 2002 Zope Corporation and Contributors. # All Rights Reserved. # # This software is subject to the provisions of the Zope Public License, # Version 2.0 (ZPL). A copy of the ZPL should accompany this distribution. # THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED # WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED # WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS # FOR A PARTICULAR PURPOSE # ############################################################################## __revision__ = "$Id: select_trigger.py,v 1.4 2003/01/09 15:49:15 akuchling Exp $" import asyncore import asynchat import os import socket import string import thread if os.name == 'posix': class trigger (asyncore.file_dispatcher): "Wake up a call to select() running in the main thread" # This is useful in a context where you are using Medusa's I/O # subsystem to deliver data, but the data is generated by another # thread. Normally, if Medusa is in the middle of a call to # select(), new output data generated by another thread will have # to sit until the call to select() either times out or returns. # If the trigger is 'pulled' by another thread, it should immediately # generate a READ event on the trigger object, which will force the # select() invocation to return. # A common use for this facility: letting Medusa manage I/O for a # large number of connections; but routing each request through a # thread chosen from a fixed-size thread pool. When a thread is # acquired, a transaction is performed, but output data is # accumulated into buffers that will be emptied more efficiently # by Medusa. [picture a server that can process database queries # rapidly, but doesn't want to tie up threads waiting to send data # to low-bandwidth connections] # The other major feature provided by this class is the ability to # move work back into the main thread: if you call pull_trigger() # with a thunk argument, when select() wakes up and receives the # event it will call your thunk from within that thread. The main # purpose of this is to remove the need to wrap thread locks around # Medusa's data structures, which normally do not need them. [To see # why this is true, imagine this scenario: A thread tries to push some # new data onto a channel's outgoing data queue at the same time that # the main thread is trying to remove some] def __init__ (self): r, w = self._fds = os.pipe() self.trigger = w asyncore.file_dispatcher.__init__(self, r) self.lock = thread.allocate_lock() self.thunks = [] self._closed = 0 # Override the asyncore close() method, because it seems that # it would only close the r file descriptor and not w. The # constructor calls file_dispatcher.__init__ and passes r, # which would get stored in a file_wrapper and get closed by # the default close. But that would leave w open... def close(self): if not self._closed: self._closed = 1 self.del_channel() for fd in self._fds: os.close(fd) self._fds = [] def __repr__ (self): return '' % id(self) def readable (self): return 1 def writable (self): return 0 def handle_connect (self): pass def handle_close(self): self.close() def pull_trigger (self, thunk=None): # print 'PULL_TRIGGER: ', len(self.thunks) if thunk: self.lock.acquire() try: self.thunks.append(thunk) finally: self.lock.release() os.write(self.trigger, 'x') def handle_read (self): try: self.recv(8192) except socket.error: return self.lock.acquire() try: for thunk in self.thunks: try: thunk() except: nil, t, v, tbinfo = asyncore.compact_traceback() print ('exception in trigger thunk:' ' (%s:%s %s)' % (t, v, tbinfo)) self.thunks = [] finally: self.lock.release() else: # win32-safe version # XXX Should define a base class that has the common methods and # then put the platform-specific in a subclass named trigger. HOST = '127.0.0.1' MINPORT = 19950 NPORTS = 50 class trigger (asyncore.dispatcher): portoffset = 0 def __init__ (self): a = socket.socket(socket.AF_INET, socket.SOCK_STREAM) w = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # set TCP_NODELAY to true to avoid buffering w.setsockopt(socket.IPPROTO_TCP, 1, 1) # tricky: get a pair of connected sockets for i in range(NPORTS): trigger.portoffset = (trigger.portoffset + 1) % NPORTS port = MINPORT + trigger.portoffset address = (HOST, port) try: a.bind(address) except socket.error: continue else: break else: raise RuntimeError, 'Cannot bind trigger!' a.listen(1) w.setblocking(0) try: w.connect(address) except: pass r, addr = a.accept() a.close() w.setblocking(1) self.trigger = w asyncore.dispatcher.__init__(self, r) self.lock = thread.allocate_lock() self.thunks = [] self._trigger_connected = 0 def __repr__ (self): return '' % id(self) def readable (self): return 1 def writable (self): return 0 def handle_connect (self): pass def pull_trigger (self, thunk=None): if thunk: self.lock.acquire() try: self.thunks.append(thunk) finally: self.lock.release() self.trigger.send('x') def handle_read (self): try: self.recv(8192) except socket.error: return self.lock.acquire() try: for thunk in self.thunks: try: thunk() except: nil, t, v, tbinfo = asyncore.compact_traceback() print ('exception in trigger thunk:' ' (%s:%s %s)' % (t, v, tbinfo)) self.thunks = [] finally: self.lock.release() the_trigger = None class trigger_file: "A 'triggered' file object" buffer_size = 4096 def __init__ (self, parent): global the_trigger if the_trigger is None: the_trigger = trigger() self.parent = parent self.buffer = '' def write (self, data): self.buffer = self.buffer + data if len(self.buffer) > self.buffer_size: d, self.buffer = self.buffer, '' the_trigger.pull_trigger ( lambda d=d,p=self.parent: p.push (d) ) def writeline (self, line): self.write (line+'\r\n') def writelines (self, lines): self.write ( string.joinfields ( lines, '\r\n' ) + '\r\n' ) def flush (self): if self.buffer: d, self.buffer = self.buffer, '' the_trigger.pull_trigger ( lambda p=self.parent,d=d: p.push (d) ) def softspace (self, *args): pass def close (self): # in a derived class, you may want to call trigger_close() instead. self.flush() self.parent = None def trigger_close (self): d, self.buffer = self.buffer, '' p, self.parent = self.parent, None the_trigger.pull_trigger ( lambda p=p,d=d: (p.push(d), p.close_when_done()) ) if __name__ == '__main__': import time def thread_function (output_file, i, n): print 'entering thread_function' while n: time.sleep (5) output_file.write ('%2d.%2d %s\r\n' % (i, n, output_file)) output_file.flush() n = n - 1 output_file.close() print 'exiting thread_function' class thread_parent (asynchat.async_chat): def __init__ (self, conn, addr): self.addr = addr asynchat.async_chat.__init__ (self, conn) self.set_terminator ('\r\n') self.buffer = '' self.count = 0 def collect_incoming_data (self, data): self.buffer = self.buffer + data def found_terminator (self): data, self.buffer = self.buffer, '' if not data: asyncore.close_all() print "done" return n = string.atoi (string.split (data)[0]) tf = trigger_file (self) self.count = self.count + 1 thread.start_new_thread (thread_function, (tf, self.count, n)) class thread_server (asyncore.dispatcher): def __init__ (self, family=socket.AF_INET, address=('', 9003)): asyncore.dispatcher.__init__ (self) self.create_socket (family, socket.SOCK_STREAM) self.set_reuse_addr() self.bind (address) self.listen (5) def handle_accept (self): conn, addr = self.accept() tp = thread_parent (conn, addr) thread_server() #asyncore.loop(1.0, use_poll=1) try: asyncore.loop () except: asyncore.close_all() medusa-0.5.4/thread/test_module.py0100644000076400007640000000043107445740204015262 0ustar amkamk# -*- Mode: Python -*- import pprint def main (env, stdin, stdout): stdout.write ( '

Test CGI Module

\r\n' '
The Environment:
\r\n'
            )
    pprint.pprint (env, stdout)
    stdout.write ('
\r\n') medusa-0.5.4/thread/thread_channel.py0100644000076400007640000000714307445740204015704 0ustar amkamk# -*- Mode: Python -*- VERSION_STRING = "$Id: thread_channel.py,v 1.3 2002/03/19 22:49:40 amk Exp $" # This will probably only work on Unix. # The disadvantage to this technique is that it wastes file # descriptors (especially when compared to select_trigger.py) # May be possible to do it on Win32, using TCP localhost sockets. # [does winsock support 'socketpair'?] import asyncore import asynchat import fcntl import FCNTL import os import socket import string import thread # this channel slaves off of another one. it starts a thread which # pumps its output through the 'write' side of the pipe. The 'read' # side of the pipe will then notify us when data is ready. We push # this data on the owning data channel's output queue. class thread_channel (asyncore.file_dispatcher): buffer_size = 8192 def __init__ (self, channel, function, *args): self.parent = channel self.function = function self.args = args self.pipe = rfd, wfd = os.pipe() asyncore.file_dispatcher.__init__ (self, rfd) def start (self): rfd, wfd = self.pipe # The read side of the pipe is set to non-blocking I/O; it is # 'owned' by medusa. flags = fcntl.fcntl (rfd, FCNTL.F_GETFL, 0) fcntl.fcntl (rfd, FCNTL.F_SETFL, flags | FCNTL.O_NDELAY) # The write side of the pipe is left in blocking mode; it is # 'owned' by the thread. However, we wrap it up as a file object. # [who wants to 'write()' to a number?] of = os.fdopen (wfd, 'w') thread.start_new_thread ( self.function, # put the output file in front of the other arguments (of,) + self.args ) def writable (self): return 0 def readable (self): return 1 def handle_read (self): data = self.recv (self.buffer_size) self.parent.push (data) def handle_close (self): # Depending on your intentions, you may want to close # the parent channel here. self.close() # Yeah, it's bad when the test code is bigger than the library code. if __name__ == '__main__': import time def thread_function (output_file, i, n): print 'entering thread_function' while n: time.sleep (5) output_file.write ('%2d.%2d %s\r\n' % (i, n, output_file)) output_file.flush() n = n - 1 output_file.close() print 'exiting thread_function' class thread_parent (asynchat.async_chat): def __init__ (self, conn, addr): self.addr = addr asynchat.async_chat.__init__ (self, conn) self.set_terminator ('\r\n') self.buffer = '' self.count = 0 def collect_incoming_data (self, data): self.buffer = self.buffer + data def found_terminator (self): data, self.buffer = self.buffer, '' n = string.atoi (string.split (data)[0]) tc = thread_channel (self, thread_function, self.count, n) self.count = self.count + 1 tc.start() class thread_server (asyncore.dispatcher): def __init__ (self, family=socket.AF_INET, address=('127.0.0.1', 9003)): asyncore.dispatcher.__init__ (self) self.create_socket (family, socket.SOCK_STREAM) self.set_reuse_addr() self.bind (address) self.listen (5) def handle_accept (self): conn, addr = self.accept() tp = thread_parent (conn, addr) thread_server() #asyncore.loop(1.0, use_poll=1) asyncore.loop () medusa-0.5.4/thread/thread_handler.py0100644000076400007640000002540107570555567015725 0ustar amkamk# -*- Mode: Python -*- import re import string import StringIO import sys import os import sys import time import select_trigger from medusa import counter from medusa import producers from medusa.default_handler import unquote, get_header import threading class request_queue: def __init__ (self): self.mon = threading.RLock() self.cv = threading.Condition (self.mon) self.queue = [] def put (self, item): self.cv.acquire() self.queue.append(item) self.cv.notify() self.cv.release() def get(self): self.cv.acquire() while not self.queue: self.cv.wait() result = self.queue.pop(0) self.cv.release() return result header2env= { 'Content-Length' : 'CONTENT_LENGTH', 'Content-Type' : 'CONTENT_TYPE', 'Referer' : 'HTTP_REFERER', 'User-Agent' : 'HTTP_USER_AGENT', 'Accept' : 'HTTP_ACCEPT', 'Accept-Charset' : 'HTTP_ACCEPT_CHARSET', 'Accept-Language' : 'HTTP_ACCEPT_LANGUAGE', 'Host' : 'HTTP_HOST', 'Connection' : 'CONNECTION_TYPE', 'Authorization' : 'HTTP_AUTHORIZATION', 'Cookie' : 'HTTP_COOKIE', } # convert keys to lower case for case-insensitive matching for (key,value) in header2env.items(): del header2env[key] key=string.lower(key) header2env[key]=value class thread_output_file (select_trigger.trigger_file): def close (self): self.trigger_close() class script_handler: def __init__ (self, queue, document_root=""): self.modules = {} self.document_root = document_root self.queue = queue def add_module (self, module, *names): if not names: names = ["/%s" % module.__name__] for name in names: self.modules['/'+name] = module def match (self, request): uri = request.uri i = string.find(uri, "/", 1) if i != -1: uri = uri[:i] i = string.find(uri, "?", 1) if i != -1: uri = uri[:i] if self.modules.has_key (uri): request.module = self.modules[uri] return 1 else: return 0 def handle_request (self, request): [path, params, query, fragment] = request.split_uri() while path and path[0] == '/': path = path[1:] if '%' in path: path = unquote (path) env = {} env['REQUEST_URI'] = "/" + path env['REQUEST_METHOD'] = string.upper(request.command) env['SERVER_PORT'] = str(request.channel.server.port) env['SERVER_NAME'] = request.channel.server.server_name env['SERVER_SOFTWARE'] = request['Server'] env['DOCUMENT_ROOT'] = self.document_root parts = string.split(path, "/") # are script_name and path_info ok? env['SCRIPT_NAME'] = "/" + parts[0] if query and query[0] == "?": query = query[1:] env['QUERY_STRING'] = query try: path_info = "/" + string.join(parts[1:], "/") except: path_info = '' env['PATH_INFO'] = path_info env['GATEWAY_INTERFACE']='CGI/1.1' # what should this really be? env['REMOTE_ADDR'] =request.channel.addr[0] env['REMOTE_HOST'] =request.channel.addr[0] # TODO: connect to resolver for header in request.header: [key,value]=string.split(header,": ",1) key=string.lower(key) if header2env.has_key(key): if header2env[key]: env[header2env[key]]=value else: key = 'HTTP_' + string.upper( string.join( string.split (key,"-"), "_" ) ) env[key]=value ## remove empty environment variables for key in env.keys(): if env[key]=="" or env[key]==None: del env[key] try: httphost = env['HTTP_HOST'] parts = string.split(httphost,":") env['HTTP_HOST'] = parts[0] except KeyError: pass if request.command in ('put', 'post'): # PUT data requires a correct Content-Length: header # (though I bet with http/1.1 we can expect chunked encoding) request.collector = collector (self, request, env) request.channel.set_terminator (None) else: sin = StringIO.StringIO ('') self.continue_request (sin, request, env) def continue_request (self, stdin, request, env): stdout = header_scanning_file ( request, thread_output_file (request.channel) ) self.queue.put ( (request.module.main, (env, stdin, stdout)) ) HEADER_LINE = re.compile ('([A-Za-z0-9-]+): ([^\r\n]+)') # A file wrapper that handles the CGI 'Status:' header hack # by scanning the output. class header_scanning_file: def __init__ (self, request, file): self.buffer = '' self.request = request self.file = file self.got_header = 0 self.bytes_out = counter.counter() def write (self, data): if self.got_header: self._write (data) else: # CGI scripts may optionally provide extra headers. # # If they do not, then the output is assumed to be # text/html, with an HTTP reply code of '200 OK'. # # If they do, we need to scan those headers for one in # particular: the 'Status:' header, which will tell us # to use a different HTTP reply code [like '302 Moved'] # self.buffer = self.buffer + data lines = string.split (self.buffer, '\n') # ignore the last piece, it is either empty, or a partial line lines = lines[:-1] # look for something un-header-like for i in range(len(lines)): li = lines[i] if (not li) or (HEADER_LINE.match (li) is None): # this is either the header separator, or it # is not a header line. self.got_header = 1 h = self.build_header (lines[:i]) self._write (h) # rejoin the rest of the data d = string.join (lines[i:], '\n') self._write (d) self.buffer = '' break def build_header (self, lines): status = '200 OK' saw_content_type = 0 hl = HEADER_LINE for line in lines: mo = hl.match (line) if mo is not None: h = string.lower (mo.group(1)) if h == 'status': status = mo.group(2) elif h == 'content-type': saw_content_type = 1 lines.insert (0, 'HTTP/1.0 %s' % status) lines.append ('Server: ' + self.request['Server']) lines.append ('Date: ' + self.request['Date']) if not saw_content_type: lines.append ('Content-Type: text/html') lines.append ('Connection: close') return string.join (lines, '\r\n')+'\r\n\r\n' def _write (self, data): self.bytes_out.increment (len(data)) self.file.write (data) def writelines(self, list): self.write (string.join (list, '')) def flush(self): pass def close (self): if not self.got_header: # managed to slip through our header detectors self._write (self.build_header (['Status: 502', 'Content-Type: text/html'])) self._write ( '

Server Error

\r\n' 'Bad Gateway: No Header from CGI Script\r\n' '
Data: %s
' '\r\n' % (repr(self.buffer)) ) self.request.log (int(self.bytes_out.as_long())) self.file.close() self.request.channel.current_request = None class collector: "gathers input for PUT requests" def __init__ (self, handler, request, env): self.handler = handler self.env = env self.request = request self.data = StringIO.StringIO() # make sure there's a content-length header self.cl = request.get_header ('content-length') if not self.cl: request.error (411) return else: self.cl = string.atoi(self.cl) def collect_incoming_data (self, data): self.data.write (data) if self.data.tell() >= self.cl: self.data.seek(0) h=self.handler r=self.request # set the terminator back to the default self.request.channel.set_terminator ('\r\n\r\n') del self.handler del self.request h.continue_request (self.data, r, self.env) class request_loop_thread (threading.Thread): def __init__ (self, queue): threading.Thread.__init__ (self) self.setDaemon(1) self.queue = queue def run (self): while 1: function, (env, stdin, stdout) = self.queue.get() function (env, stdin, stdout) stdout.close() # =========================================================================== # Testing # =========================================================================== if __name__ == '__main__': import sys if len(sys.argv) < 2: print 'Usage: %s ' % sys.argv[0] else: nthreads = string.atoi (sys.argv[1]) import asyncore from medusa import http_server # create a generic web server hs = http_server.http_server ('', 7080) # create a request queue q = request_queue() # create a script handler sh = script_handler (q) # install the script handler on the web server hs.install_handler (sh) # get a couple of CGI modules import test_module import pi_module # install the module on the script handler sh.add_module (test_module, 'test') sh.add_module (pi_module, 'pi') # fire up the worker threads for i in range (nthreads): rt = request_loop_thread (q) rt.start() # start the main event loop asyncore.loop() medusa-0.5.4/auth_handler.py0100644000076400007640000001120207570555564014137 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1996-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: auth_handler.py,v 1.6 2002/11/25 19:40:23 akuchling Exp $' # support for 'basic' authenticaion. import base64 import md5 import re import string import time import counter import default_handler get_header = default_handler.get_header import producers # This is a 'handler' that wraps an authorization method # around access to the resources normally served up by # another handler. # does anyone support digest authentication? (rfc2069) class auth_handler: def __init__ (self, dict, handler, realm='default'): self.authorizer = dictionary_authorizer (dict) self.handler = handler self.realm = realm self.pass_count = counter.counter() self.fail_count = counter.counter() def match (self, request): # by default, use the given handler's matcher return self.handler.match (request) def handle_request (self, request): # authorize a request before handling it... scheme = get_header (AUTHORIZATION, request.header) if scheme: scheme = string.lower (scheme) if scheme == 'basic': cookie = get_header (AUTHORIZATION, request.header, 2) try: decoded = base64.decodestring (cookie) except: print 'malformed authorization info <%s>' % cookie request.error (400) return auth_info = string.split (decoded, ':') if self.authorizer.authorize (auth_info): self.pass_count.increment() request.auth_info = auth_info self.handler.handle_request (request) else: self.handle_unauthorized (request) #elif scheme == 'digest': # print 'digest: ',AUTHORIZATION.group(2) else: print 'unknown/unsupported auth method: %s' % scheme self.handle_unauthorized(request) else: # list both? prefer one or the other? # you could also use a 'nonce' here. [see below] #auth = 'Basic realm="%s" Digest realm="%s"' % (self.realm, self.realm) #nonce = self.make_nonce (request) #auth = 'Digest realm="%s" nonce="%s"' % (self.realm, nonce) #request['WWW-Authenticate'] = auth #print 'sending header: %s' % request['WWW-Authenticate'] self.handle_unauthorized (request) def handle_unauthorized (self, request): # We are now going to receive data that we want to ignore. # to ignore the file data we're not interested in. self.fail_count.increment() request.channel.set_terminator (None) request['Connection'] = 'close' request['WWW-Authenticate'] = 'Basic realm="%s"' % self.realm request.error (401) def make_nonce (self, request): "A digest-authentication , constructed as suggested in RFC 2069" ip = request.channel.server.ip now = str(long(time.time())) if now[-1:] == 'L': now = now[:-1] private_key = str (id (self)) nonce = string.join ([ip, now, private_key], ':') return self.apply_hash (nonce) def apply_hash (self, s): "Apply MD5 to a string , then wrap it in base64 encoding." m = md5.new() m.update (s) d = m.digest() # base64.encodestring tacks on an extra linefeed. return base64.encodestring (d)[:-1] def status (self): # Thanks to mwm@contessa.phone.net (Mike Meyer) r = [ producers.simple_producer ( '
  • Authorization Extension : ' 'Unauthorized requests: %s
      ' % self.fail_count ) ] if hasattr (self.handler, 'status'): r.append (self.handler.status()) r.append ( producers.simple_producer ('
    ') ) return producers.composite_producer(r) class dictionary_authorizer: def __init__ (self, dict): self.dict = dict def authorize (self, auth_info): [username, password] = auth_info if (self.dict.has_key (username)) and (self.dict[username] == password): return 1 else: return 0 AUTHORIZATION = re.compile ( # scheme challenge 'Authorization: ([^ ]+) (.*)', re.IGNORECASE ) medusa-0.5.4/CHANGES.txt0100644000076400007640000000504607722754301012736 0ustar amkamk Version 0.5.4: * Open syslog using datagram sockets, and only try streams if that fails (Alxy Sav) * Fix bug in http_server.crack_request() (Contributed by Canis Lupus) * Incorporate bugfixes to thread/select_trigger.py from ZEO's version (Zope Corporation) * Add demo/winFTPserver.py, an FTP server that uses Windows authorization to determine who can access the server. (Contributed by John Abel) * Add control files for creating a Debian package of Medusa (python2.2-medusa). Version 0.5.3: * Delete the broken and rather boring dual_server and simple_httpd demo scripts. start_medusa.py should be sufficient as an example. * Fix indentation bug in demo/script_server.py noted by Richard Philips * Fix bug in producers.composite_producer, spotted and fixed by Daniel Krech * Added test suite for producers.py * Fix timestamps in http_server logs * Fix unix_user_handler bug, spotted and fixed by Sergio Fernndez. * Fix auth_handler bug, spotted and fixed by Sergio Fernndez. * Delete unused http_server.fifo class and fifo.py module. Version 0.5.2: * Fix syntax error and missing import in default_handler.py * Fix various scripts in demo/ Version 0.5.1: * Apply cleanup patch from Donovan Baarda * Fix bug reported by Van Gale: counter.py and auth_handler.py did long(...)[:-1] to chop off a trailing L generated in earlier versions of Python. * Fix bug in ftp_server.py that I introduced in 0.5 * Remove some duplicated producer classes * Removed work_in_progress/ directory and the 'continuation' module * Remove MIME type table code and use the stdlib's mimelib module Version 0.5: * Added a setup.py installation script, which will install all the code under the package name 'medusa'. * Added README.txt and CHANGES.txt. * Fixed NameError in util/convert_mime_type_table.py * Fixed TypeError in test/test_medusa.py * Fixed several problems detected by PyChecker * Changed demos to use 'from medusa import ...' * Rearranged files to reduce the number of subdirectories. * Removed or updated uses of the obsolete regsub module * Removed asyncore.py and asynchat.py; these modules were added to Python's standard library with version 1.5.2, and Medusa now assumes that they're present. * Removed many obsolete files: poll/pollmodule.c, as Python's select module now supports poll() patches/posixmodule.c, as that patch was incorporated in Python old/*, script_handler_demo/*, sendfile/* The old ANNOUNCE files * Reindented all files to use four-space indents The last version of Medusa released by Sam Rushing was medusa-20010416. medusa-0.5.4/chat_server.py0100644000076400007640000001066007446144354014006 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1997-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: chat_server.py,v 1.4 2002/03/20 17:37:48 amk Exp $' import string VERSION = string.split(RCS_ID)[2] import socket import asyncore import asynchat import status_handler class chat_channel (asynchat.async_chat): def __init__ (self, server, sock, addr): asynchat.async_chat.__init__ (self, sock) self.server = server self.addr = addr self.set_terminator ('\r\n') self.data = '' self.nick = None self.push ('nickname?: ') def collect_incoming_data (self, data): self.data = self.data + data def found_terminator (self): line = self.data self.data = '' if self.nick is None: self.nick = string.split (line)[0] if not self.nick: self.nick = None self.push ('huh? gimmee a nickname: ') else: self.greet() else: if not line: pass elif line[0] != '/': self.server.push_line (self, line) else: self.handle_command (line) def greet (self): self.push ('Hello, %s\r\n' % self.nick) num_channels = len(self.server.channels)-1 if num_channels == 0: self.push ('[Kinda lonely in here... you\'re the only caller!]\r\n') else: self.push ('[There are %d other callers]\r\n' % (len(self.server.channels)-1)) nicks = map (lambda x: x.get_nick(), self.server.channels.keys()) self.push (string.join (nicks, '\r\n ') + '\r\n') self.server.push_line (self, '[joined]') def handle_command (self, command): import types command_line = string.split(command) name = 'cmd_%s' % command_line[0][1:] if hasattr (self, name): # make sure it's a method... method = getattr (self, name) if type(method) == type(self.handle_command): method (command_line[1:]) else: self.push ('unknown command: %s' % command_line[0]) def cmd_quit (self, args): self.server.push_line (self, '[left]') self.push ('Goodbye!\r\n') self.close_when_done() # alias for '/quit' - '/q' cmd_q = cmd_quit def push_line (self, nick, line): self.push ('%s: %s\r\n' % (nick, line)) def handle_close (self): self.close() def close (self): del self.server.channels[self] asynchat.async_chat.close (self) def get_nick (self): if self.nick is not None: return self.nick else: return 'Unknown' class chat_server (asyncore.dispatcher): SERVER_IDENT = 'Chat Server (V%s)' % VERSION channel_class = chat_channel spy = 1 def __init__ (self, ip='', port=8518): asyncore.dispatcher.__init__(self) self.port = port self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.bind ((ip, port)) print '%s started on port %d' % (self.SERVER_IDENT, port) self.listen (5) self.channels = {} self.count = 0 def handle_accept (self): conn, addr = self.accept() self.count = self.count + 1 print 'client #%d - %s:%d' % (self.count, addr[0], addr[1]) self.channels[self.channel_class (self, conn, addr)] = 1 def push_line (self, from_channel, line): nick = from_channel.get_nick() if self.spy: print '%s: %s' % (nick, line) for c in self.channels.keys(): if c is not from_channel: c.push ('%s: %s\r\n' % (nick, line)) def status (self): lines = [ '

    %s

    ' % self.SERVER_IDENT, '
    Listening on Port: %d' % self.port, '
    Total Sessions: %d' % self.count, '
    Current Sessions: %d' % (len(self.channels)) ] return status_handler.lines_producer (lines) def writable (self): return 0 if __name__ == '__main__': import sys if len(sys.argv) > 1: port = string.atoi (sys.argv[1]) else: port = 8518 s = chat_server ('', port) asyncore.loop() medusa-0.5.4/counter.py0100644000076400007640000000264007701142270013144 0ustar amkamk# -*- Mode: Python -*- # It is tempting to add an __int__ method to this class, but it's not # a good idea. This class tries to gracefully handle integer # overflow, and to hide this detail from both the programmer and the # user. Note that the __str__ method can be relied on for printing out # the value of a counter: # # >>> print 'Total Client: %s' % self.total_clients # # If you need to do arithmetic with the value, then use the 'as_long' # method, the use of long arithmetic is a reminder that the counter # will overflow. class counter: "general-purpose counter" def __init__ (self, initial_value=0): self.value = initial_value def increment (self, delta=1): result = self.value try: self.value = self.value + delta except OverflowError: self.value = long(self.value) + delta return result def decrement (self, delta=1): result = self.value try: self.value = self.value - delta except OverflowError: self.value = long(self.value) - delta return result def as_long (self): return long(self.value) def __nonzero__ (self): return self.value != 0 def __repr__ (self): return '' % (self.value, id(self)) def __str__ (self): s = str(long(self.value)) if s[-1:] == 'L': s = s[:-1] return s medusa-0.5.4/default_handler.py0100644000076400007640000001427207543176422014624 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1997 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: default_handler.py,v 1.8 2002/08/01 18:15:45 akuchling Exp $' # standard python modules import mimetypes import re import stat import string # medusa modules import http_date import http_server import status_handler import producers unquote = http_server.unquote # This is the 'default' handler. it implements the base set of # features expected of a simple file-delivering HTTP server. file # services are provided through a 'filesystem' object, the very same # one used by the FTP server. # # You can replace or modify this handler if you want a non-standard # HTTP server. You can also derive your own handler classes from # it. # # support for handling POST requests is available in the derived # class , defined below. # from counter import counter class default_handler: valid_commands = ['GET', 'HEAD'] IDENT = 'Default HTTP Request Handler' # Pathnames that are tried when a URI resolves to a directory name directory_defaults = [ 'index.html', 'default.html' ] default_file_producer = producers.file_producer def __init__ (self, filesystem): self.filesystem = filesystem # count total hits self.hit_counter = counter() # count file deliveries self.file_counter = counter() # count cache hits self.cache_counter = counter() hit_counter = 0 def __repr__ (self): return '<%s (%s hits) at %x>' % ( self.IDENT, self.hit_counter, id (self) ) # always match, since this is a default def match (self, request): return 1 # handle a file request, with caching. def handle_request (self, request): if request.command not in self.valid_commands: request.error (400) # bad request return self.hit_counter.increment() path, params, query, fragment = request.split_uri() if '%' in path: path = unquote (path) # strip off all leading slashes while path and path[0] == '/': path = path[1:] if self.filesystem.isdir (path): if path and path[-1] != '/': request['Location'] = 'http://%s/%s/' % ( request.channel.server.server_name, path ) request.error (301) return # we could also generate a directory listing here, # may want to move this into another method for that # purpose found = 0 if path and path[-1] != '/': path = path + '/' for default in self.directory_defaults: p = path + default if self.filesystem.isfile (p): path = p found = 1 break if not found: request.error (404) # Not Found return elif not self.filesystem.isfile (path): request.error (404) # Not Found return file_length = self.filesystem.stat (path)[stat.ST_SIZE] ims = get_header_match (IF_MODIFIED_SINCE, request.header) length_match = 1 if ims: length = ims.group (4) if length: try: length = string.atoi (length) if length != file_length: length_match = 0 except: pass ims_date = 0 if ims: ims_date = http_date.parse_http_date (ims.group (1)) try: mtime = self.filesystem.stat (path)[stat.ST_MTIME] except: request.error (404) return if length_match and ims_date: if mtime <= ims_date: request.reply_code = 304 request.done() self.cache_counter.increment() return try: file = self.filesystem.open (path, 'rb') except IOError: request.error (404) return request['Last-Modified'] = http_date.build_http_date (mtime) request['Content-Length'] = file_length self.set_content_type (path, request) if request.command == 'GET': request.push (self.default_file_producer (file)) self.file_counter.increment() request.done() def set_content_type (self, path, request): ext = string.lower (get_extension (path)) typ, encoding = mimetypes.guess_type(path) if typ is not None: request['Content-Type'] = typ else: # TODO: test a chunk off the front of the file for 8-bit # characters, and use application/octet-stream instead. request['Content-Type'] = 'text/plain' def status (self): return producers.simple_producer ( '
  • %s' % status_handler.html_repr (self) + '
      ' + '
    • Total Hits: %s' % self.hit_counter + '
    • Files Delivered: %s' % self.file_counter + '
    • Cache Hits: %s' % self.cache_counter + '
    ' ) # HTTP/1.0 doesn't say anything about the "; length=nnnn" addition # to this header. I suppose its purpose is to avoid the overhead # of parsing dates... IF_MODIFIED_SINCE = re.compile ( 'If-Modified-Since: ([^;]+)((; length=([0-9]+)$)|$)', re.IGNORECASE ) USER_AGENT = re.compile ('User-Agent: (.*)', re.IGNORECASE) CONTENT_TYPE = re.compile ( r'Content-Type: ([^;]+)((; boundary=([A-Za-z0-9\'\(\)+_,./:=?-]+)$)|$)', re.IGNORECASE ) get_header = http_server.get_header get_header_match = http_server.get_header_match def get_extension (path): dirsep = string.rfind (path, '/') dotsep = string.rfind (path, '.') if dotsep > dirsep: return path[dotsep+1:] else: return '' medusa-0.5.4/event_loop.py0100644000076400007640000000564507446144354013662 0ustar amkamk# -*- Mode: Python -*- # This is an alternative event loop that supports 'schedulable events'. # You can specify an event callback to take place after seconds. # Important usage note: The granularity of the time-check is limited # by the argument to 'go()'; if there is little or no # activity and you specify a 30-second timeout interval, then the # schedule of events may only be checked at those 30-second intervals. # In other words, if you need 1-second resolution, you will have to # poll at 1-second intervals. This facility is more useful for longer # timeouts ("if the channel doesn't close in 5 minutes, then forcibly # close it" would be a typical usage). import asyncore import bisect import time socket_map = asyncore.socket_map class event_loop: def __init__ (self): self.events = [] self.num_channels = 0 self.max_channels = 0 def go (self, timeout=30.0, granularity=15): global socket_map last_event_check = 0 while socket_map: now = int(time.time()) if (now - last_event_check) >= granularity: last_event_check = now fired = [] # yuck. i want my lisp. i = j = 0 while i < len(self.events): when, what = self.events[i] if now >= when: fired.append (what) j = i + 1 else: break i = i + 1 if fired: self.events = self.events[j:] for what in fired: what (self, now) # sample the number of channels n = len(asyncore.socket_map) self.num_channels = n if n > self.max_channels: self.max_channels = n asyncore.poll (timeout) def schedule (self, delta, callback): now = int (time.time()) bisect.insort (self.events, (now + delta, callback)) def __len__ (self): return len(self.events) class test (asyncore.dispatcher): def __init__ (self): asyncore.dispatcher.__init__ (self) def handle_connect (self): print 'Connected!' def writable (self): return not self.connected def connect_timeout_callback (self, event_loop, when): if not self.connected: print 'Timeout on connect' self.close() def periodic_thing_callback (self, event_loop, when): print 'A Periodic Event has Occurred!' # re-schedule it. event_loop.schedule (self, 15, self.periodic_thing_callback) if __name__ == '__main__': import socket el = event_loop() t = test () t.create_socket (socket.AF_INET, socket.SOCK_STREAM) el.schedule (10, t.connect_timeout_callback) el.schedule (15, t.periodic_thing_callback) t.connect (('squirl', 80)) el.go(1.0) medusa-0.5.4/filesys.py0100644000076400007640000002567707450101107013155 0ustar amkamk# -*- Mode: Python -*- # $Id: filesys.py,v 1.7 2002/03/26 14:14:31 amk Exp $ # Author: Sam Rushing # # Generic filesystem interface. # # We want to provide a complete wrapper around any and all # filesystem operations. # this class is really just for documentation, # identifying the API for a filesystem object. # opening files for reading, and listing directories, should # return a producer. class abstract_filesystem: def __init__ (self): pass def current_directory (self): "Return a string representing the current directory." pass def listdir (self, path, long=0): """Return a listing of the directory at 'path' The empty string indicates the current directory. If 'long' is set, instead return a list of (name, stat_info) tuples """ pass def open (self, path, mode): "Return an open file object" pass def stat (self, path): "Return the equivalent of os.stat() on the given path." pass def isdir (self, path): "Does the path represent a directory?" pass def isfile (self, path): "Does the path represent a plain file?" pass def cwd (self, path): "Change the working directory." pass def cdup (self): "Change to the parent of the current directory." pass def longify (self, path): """Return a 'long' representation of the filename [for the output of the LIST command]""" pass # standard wrapper around a unix-like filesystem, with a 'false root' # capability. # security considerations: can symbolic links be used to 'escape' the # root? should we allow it? if not, then we could scan the # filesystem on startup, but that would not help if they were added # later. We will probably need to check for symlinks in the cwd method. # what to do if wd is an invalid directory? import os import stat import re import string def safe_stat (path): try: return (path, os.stat (path)) except: return None import glob class os_filesystem: path_module = os.path # set this to zero if you want to disable pathname globbing. # [we currently don't glob, anyway] do_globbing = 1 def __init__ (self, root, wd='/'): self.root = root self.wd = wd def current_directory (self): return self.wd def isfile (self, path): p = self.normalize (self.path_module.join (self.wd, path)) return self.path_module.isfile (self.translate(p)) def isdir (self, path): p = self.normalize (self.path_module.join (self.wd, path)) return self.path_module.isdir (self.translate(p)) def cwd (self, path): p = self.normalize (self.path_module.join (self.wd, path)) translated_path = self.translate(p) if not self.path_module.isdir (translated_path): return 0 else: old_dir = os.getcwd() # temporarily change to that directory, in order # to see if we have permission to do so. try: can = 0 try: os.chdir (translated_path) can = 1 self.wd = p except: pass finally: if can: os.chdir (old_dir) return can def cdup (self): return self.cwd ('..') def listdir (self, path, long=0): p = self.translate (path) # I think we should glob, but limit it to the current # directory only. ld = os.listdir (p) if not long: return list_producer (ld, None) else: old_dir = os.getcwd() try: os.chdir (p) # if os.stat fails we ignore that file. result = filter (None, map (safe_stat, ld)) finally: os.chdir (old_dir) return list_producer (result, self.longify) # TODO: implement a cache w/timeout for stat() def stat (self, path): p = self.translate (path) return os.stat (p) def open (self, path, mode): p = self.translate (path) return open (p, mode) def unlink (self, path): p = self.translate (path) return os.unlink (p) def mkdir (self, path): p = self.translate (path) return os.mkdir (p) def rmdir (self, path): p = self.translate (path) return os.rmdir (p) # utility methods def normalize (self, path): # watch for the ever-sneaky '/+' path element path = re.sub('/+', '/', path) p = self.path_module.normpath (path) # remove 'dangling' cdup's. if len(p) > 2 and p[:3] == '/..': p = '/' return p def translate (self, path): # we need to join together three separate # path components, and do it safely. # // # use the operating system's path separator. path = string.join (string.split (path, '/'), os.sep) p = self.normalize (self.path_module.join (self.wd, path)) p = self.normalize (self.path_module.join (self.root, p[1:])) return p def longify (self, (path, stat_info)): return unix_longify (path, stat_info) def __repr__ (self): return '' % ( self.root, self.wd ) if os.name == 'posix': class unix_filesystem (os_filesystem): pass class schizophrenic_unix_filesystem (os_filesystem): PROCESS_UID = os.getuid() PROCESS_EUID = os.geteuid() PROCESS_GID = os.getgid() PROCESS_EGID = os.getegid() def __init__ (self, root, wd='/', persona=(None, None)): os_filesystem.__init__ (self, root, wd) self.persona = persona def become_persona (self): if self.persona is not (None, None): uid, gid = self.persona # the order of these is important! os.setegid (gid) os.seteuid (uid) def become_nobody (self): if self.persona is not (None, None): os.seteuid (self.PROCESS_UID) os.setegid (self.PROCESS_GID) # cwd, cdup, open, listdir def cwd (self, path): try: self.become_persona() return os_filesystem.cwd (self, path) finally: self.become_nobody() def cdup (self, path): try: self.become_persona() return os_filesystem.cdup (self) finally: self.become_nobody() def open (self, filename, mode): try: self.become_persona() return os_filesystem.open (self, filename, mode) finally: self.become_nobody() def listdir (self, path, long=0): try: self.become_persona() return os_filesystem.listdir (self, path, long) finally: self.become_nobody() # For the 'real' root, we could obtain a list of drives, and then # use that. Doesn't win32 provide such a 'real' filesystem? # [yes, I think something like this "\\.\c\windows"] class msdos_filesystem (os_filesystem): def longify (self, (path, stat_info)): return msdos_longify (path, stat_info) # A merged filesystem will let you plug other filesystems together. # We really need the equivalent of a 'mount' capability - this seems # to be the most general idea. So you'd use a 'mount' method to place # another filesystem somewhere in the hierarchy. # Note: this is most likely how I will handle ~user directories # with the http server. class merged_filesystem: def __init__ (self, *fsys): pass # this matches the output of NT's ftp server (when in # MSDOS mode) exactly. def msdos_longify (file, stat_info): if stat.S_ISDIR (stat_info[stat.ST_MODE]): dir = '' else: dir = ' ' date = msdos_date (stat_info[stat.ST_MTIME]) return '%s %s %8d %s' % ( date, dir, stat_info[stat.ST_SIZE], file ) def msdos_date (t): try: info = time.gmtime (t) except: info = time.gmtime (0) # year, month, day, hour, minute, second, ... if info[3] > 11: merid = 'PM' info[3] = info[3] - 12 else: merid = 'AM' return '%02d-%02d-%02d %02d:%02d%s' % ( info[1], info[2], info[0]%100, info[3], info[4], merid ) months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'] mode_table = { '0':'---', '1':'--x', '2':'-w-', '3':'-wx', '4':'r--', '5':'r-x', '6':'rw-', '7':'rwx' } import time def unix_longify (file, stat_info): # for now, only pay attention to the lower bits mode = ('%o' % stat_info[stat.ST_MODE])[-3:] mode = string.join (map (lambda x: mode_table[x], mode), '') if stat.S_ISDIR (stat_info[stat.ST_MODE]): dirchar = 'd' else: dirchar = '-' date = ls_date (long(time.time()), stat_info[stat.ST_MTIME]) return '%s%s %3d %-8d %-8d %8d %s %s' % ( dirchar, mode, stat_info[stat.ST_NLINK], stat_info[stat.ST_UID], stat_info[stat.ST_GID], stat_info[stat.ST_SIZE], date, file ) # Emulate the unix 'ls' command's date field. # it has two formats - if the date is more than 180 # days in the past, then it's like this: # Oct 19 1995 # otherwise, it looks like this: # Oct 19 17:33 def ls_date (now, t): try: info = time.gmtime (t) except: info = time.gmtime (0) # 15,600,000 == 86,400 * 180 if (now - t) > 15600000: return '%s %2d %d' % ( months[info[1]-1], info[2], info[0] ) else: return '%s %2d %02d:%02d' % ( months[info[1]-1], info[2], info[3], info[4] ) # =========================================================================== # Producers # =========================================================================== class list_producer: def __init__ (self, list, func=None): self.list = list self.func = func # this should do a pushd/popd def more (self): if not self.list: return '' else: # do a few at a time bunch = self.list[:50] if self.func is not None: bunch = map (self.func, bunch) self.list = self.list[50:] return string.joinfields (bunch, '\r\n') + '\r\n' medusa-0.5.4/ftp_server.py0100644000076400007640000011714207450243347013660 0ustar amkamk# -*- Mode: Python -*- # Author: Sam Rushing # Copyright 1996-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: ftp_server.py,v 1.10 2002/03/27 04:13:27 amk Exp $' # An extensible, configurable, asynchronous FTP server. # # All socket I/O is non-blocking, however file I/O is currently # blocking. Eventually file I/O may be made non-blocking, too, if it # seems necessary. Currently the only CPU-intensive operation is # getting and formatting a directory listing. [this could be moved # into another process/directory server, or another thread?] # # Only a subset of RFC 959 is implemented, but much of that RFC is # vestigial anyway. I've attempted to include the most commonly-used # commands, using the feature set of wu-ftpd as a guide. import asyncore import asynchat import os import socket import stat import string import sys import time from medusa.producers import file_producer # TODO: implement a directory listing cache. On very-high-load # servers this could save a lot of disk abuse, and possibly the # work of computing emulated unix ls output. # Potential security problem with the FTP protocol? I don't think # there's any verification of the origin of a data connection. Not # really a problem for the server (since it doesn't send the port # command, except when in PASV mode) But I think a data connection # could be spoofed by a program with access to a sniffer - it could # watch for a PORT command to go over a command channel, and then # connect to that port before the server does. # Unix user id's: # In order to support assuming the id of a particular user, # it seems there are two options: # 1) fork, and seteuid in the child # 2) carefully control the effective uid around filesystem accessing # methods, using try/finally. [this seems to work] VERSION = string.split(RCS_ID)[2] from counter import counter import producers import status_handler import logger class ftp_channel (asynchat.async_chat): # defaults for a reliable __repr__ addr = ('unknown','0') # unset this in a derived class in order # to enable the commands in 'self.write_commands' read_only = 1 write_commands = ['appe','dele','mkd','rmd','rnfr','rnto','stor','stou'] restart_position = 0 # comply with (possibly troublesome) RFC959 requirements # This is necessary to correctly run an active data connection # through a firewall that triggers on the source port (expected # to be 'L-1', or 20 in the normal case). bind_local_minus_one = 0 def __init__ (self, server, conn, addr): self.server = server self.current_mode = 'a' self.addr = addr asynchat.async_chat.__init__ (self, conn) self.set_terminator ('\r\n') # client data port. Defaults to 'the same as the control connection'. self.client_addr = (addr[0], 21) self.client_dc = None self.in_buffer = '' self.closing = 0 self.passive_acceptor = None self.passive_connection = None self.filesystem = None self.authorized = 0 # send the greeting self.respond ( '220 %s FTP server (Medusa Async V%s [experimental]) ready.' % ( self.server.hostname, VERSION ) ) # def __del__ (self): # print 'ftp_channel.__del__()' # -------------------------------------------------- # async-library methods # -------------------------------------------------- def handle_expt (self): # this is handled below. not sure what I could # do here to make that code less kludgish. pass def collect_incoming_data (self, data): self.in_buffer = self.in_buffer + data if len(self.in_buffer) > 4096: # silently truncate really long lines # (possible denial-of-service attack) self.in_buffer = '' def found_terminator (self): line = self.in_buffer if not len(line): return sp = string.find (line, ' ') if sp != -1: line = [line[:sp], line[sp+1:]] else: line = [line] command = string.lower (line[0]) # watch especially for 'urgent' abort commands. if string.find (command, 'abor') != -1: # strip off telnet sync chars and the like... while command and command[0] not in string.letters: command = command[1:] fun_name = 'cmd_%s' % command if command != 'pass': self.log ('<== %s' % repr(self.in_buffer)[1:-1]) else: self.log ('<== %s' % line[0]+' ') self.in_buffer = '' if not hasattr (self, fun_name): self.command_not_understood (line[0]) return fun = getattr (self, fun_name) if (not self.authorized) and (command not in ('user', 'pass', 'help', 'quit')): self.respond ('530 Please log in with USER and PASS') elif (not self.check_command_authorization (command)): self.command_not_authorized (command) else: try: result = apply (fun, (line,)) except: self.server.total_exceptions.increment() (file, fun, line), t,v, tbinfo = asyncore.compact_traceback() if self.client_dc: try: self.client_dc.close() except: pass self.respond ( '451 Server Error: %s, %s: file: %s line: %s' % ( t,v,file,line, ) ) closed = 0 def close (self): if not self.closed: self.closed = 1 if self.passive_acceptor: self.passive_acceptor.close() if self.client_dc: self.client_dc.close() self.server.closed_sessions.increment() asynchat.async_chat.close (self) # -------------------------------------------------- # filesystem interface functions. # override these to provide access control or perform # other functions. # -------------------------------------------------- def cwd (self, line): return self.filesystem.cwd (line[1]) def cdup (self, line): return self.filesystem.cdup() def open (self, path, mode): return self.filesystem.open (path, mode) # returns a producer def listdir (self, path, long=0): return self.filesystem.listdir (path, long) def get_dir_list (self, line, long=0): # we need to scan the command line for arguments to '/bin/ls'... args = line[1:] path_args = [] for arg in args: if arg[0] != '-': path_args.append (arg) else: # ignore arguments pass if len(path_args) < 1: dir = '.' else: dir = path_args[0] return self.listdir (dir, long) # -------------------------------------------------- # authorization methods # -------------------------------------------------- def check_command_authorization (self, command): if command in self.write_commands and self.read_only: return 0 else: return 1 # -------------------------------------------------- # utility methods # -------------------------------------------------- def log (self, message): self.server.logger.log ( self.addr[0], '%d %s' % ( self.addr[1], message ) ) def respond (self, resp): self.log ('==> %s' % resp) self.push (resp + '\r\n') def command_not_understood (self, command): self.respond ("500 '%s': command not understood." % command) def command_not_authorized (self, command): self.respond ( "530 You are not authorized to perform the '%s' command" % ( command ) ) def make_xmit_channel (self): # In PASV mode, the connection may or may _not_ have been made # yet. [although in most cases it is... FTP Explorer being # the only exception I've yet seen]. This gets somewhat confusing # because things may happen in any order... pa = self.passive_acceptor if pa: if pa.ready: # a connection has already been made. conn, addr = self.passive_acceptor.ready cdc = xmit_channel (self, addr) cdc.set_socket (conn) cdc.connected = 1 self.passive_acceptor.close() self.passive_acceptor = None else: # we're still waiting for a connect to the PASV port. cdc = xmit_channel (self) else: # not in PASV mode. ip, port = self.client_addr cdc = xmit_channel (self, self.client_addr) cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM) if self.bind_local_minus_one: cdc.bind (('', self.server.port - 1)) try: cdc.connect ((ip, port)) except socket.error, why: self.respond ("425 Can't build data connection") self.client_dc = cdc # pretty much the same as xmit, but only right on the verge of # being worth a merge. def make_recv_channel (self, fd): pa = self.passive_acceptor if pa: if pa.ready: # a connection has already been made. conn, addr = pa.ready cdc = recv_channel (self, addr, fd) cdc.set_socket (conn) cdc.connected = 1 self.passive_acceptor.close() self.passive_acceptor = None else: # we're still waiting for a connect to the PASV port. cdc = recv_channel (self, None, fd) else: # not in PASV mode. ip, port = self.client_addr cdc = recv_channel (self, self.client_addr, fd) cdc.create_socket (socket.AF_INET, socket.SOCK_STREAM) try: cdc.connect ((ip, port)) except socket.error, why: self.respond ("425 Can't build data connection") self.client_dc = cdc type_map = { 'a':'ASCII', 'i':'Binary', 'e':'EBCDIC', 'l':'Binary' } type_mode_map = { 'a':'t', 'i':'b', 'e':'b', 'l':'b' } # -------------------------------------------------- # command methods # -------------------------------------------------- def cmd_type (self, line): 'specify data transfer type' # ascii, ebcdic, image, local t = string.lower (line[1]) # no support for EBCDIC # if t not in ['a','e','i','l']: if t not in ['a','i','l']: self.command_not_understood (string.join (line)) elif t == 'l' and (len(line) > 2 and line[2] != '8'): self.respond ('504 Byte size must be 8') else: self.current_mode = t self.respond ('200 Type set to %s.' % self.type_map[t]) def cmd_quit (self, line): 'terminate session' self.respond ('221 Goodbye.') self.close_when_done() def cmd_port (self, line): 'specify data connection port' info = string.split (line[1], ',') ip = string.join (info[:4], '.') port = string.atoi(info[4])*256 + string.atoi(info[5]) # how many data connections at a time? # I'm assuming one for now... # TODO: we should (optionally) verify that the # ip number belongs to the client. [wu-ftpd does this?] self.client_addr = (ip, port) self.respond ('200 PORT command successful.') def new_passive_acceptor (self): # ensure that only one of these exists at a time. if self.passive_acceptor is not None: self.passive_acceptor.close() self.passive_acceptor = None self.passive_acceptor = passive_acceptor (self) return self.passive_acceptor def cmd_pasv (self, line): 'prepare for server-to-server transfer' pc = self.new_passive_acceptor() port = pc.addr[1] ip_addr = pc.control_channel.getsockname()[0] self.respond ( '227 Entering Passive Mode (%s,%d,%d)' % ( string.replace(ip_addr, '.', ','), port/256, port%256 ) ) self.client_dc = None def cmd_nlst (self, line): 'give name list of files in directory' # ncftp adds the -FC argument for the user-visible 'nlist' # command. We could try to emulate ls flags, but not just yet. if '-FC' in line: line.remove ('-FC') try: dir_list_producer = self.get_dir_list (line, 0) except os.error, why: self.respond ('550 Could not list directory: %s' % why) return self.respond ( '150 Opening %s mode data connection for file list' % ( self.type_map[self.current_mode] ) ) self.make_xmit_channel() self.client_dc.push_with_producer (dir_list_producer) self.client_dc.close_when_done() def cmd_list (self, line): 'give a list of files in a directory' try: dir_list_producer = self.get_dir_list (line, 1) except os.error, why: self.respond ('550 Could not list directory: %s' % why) return self.respond ( '150 Opening %s mode data connection for file list' % ( self.type_map[self.current_mode] ) ) self.make_xmit_channel() self.client_dc.push_with_producer (dir_list_producer) self.client_dc.close_when_done() def cmd_cwd (self, line): 'change working directory' if self.cwd (line): self.respond ('250 CWD command successful.') else: self.respond ('550 No such directory.') def cmd_cdup (self, line): 'change to parent of current working directory' if self.cdup(line): self.respond ('250 CDUP command successful.') else: self.respond ('550 No such directory.') def cmd_pwd (self, line): 'print the current working directory' self.respond ( '257 "%s" is the current directory.' % ( self.filesystem.current_directory() ) ) # modification time # example output: # 213 19960301204320 def cmd_mdtm (self, line): 'show last modification time of file' filename = line[1] if not self.filesystem.isfile (filename): self.respond ('550 "%s" is not a file' % filename) else: mtime = time.gmtime(self.filesystem.stat(filename)[stat.ST_MTIME]) self.respond ( '213 %4d%02d%02d%02d%02d%02d' % ( mtime[0], mtime[1], mtime[2], mtime[3], mtime[4], mtime[5] ) ) def cmd_noop (self, line): 'do nothing' self.respond ('200 NOOP command successful.') def cmd_size (self, line): 'return size of file' filename = line[1] if not self.filesystem.isfile (filename): self.respond ('550 "%s" is not a file' % filename) else: self.respond ( '213 %d' % (self.filesystem.stat(filename)[stat.ST_SIZE]) ) def cmd_retr (self, line): 'retrieve a file' if len(line) < 2: self.command_not_understood (string.join (line)) else: file = line[1] if not self.filesystem.isfile (file): self.log_info ('checking %s' % file) self.respond ('550 No such file') else: try: # FIXME: for some reason, 'rt' isn't working on win95 mode = 'r'+self.type_mode_map[self.current_mode] fd = self.open (file, mode) except IOError, why: self.respond ('553 could not open file for reading: %s' % (repr(why))) return self.respond ( "150 Opening %s mode data connection for file '%s'" % ( self.type_map[self.current_mode], file ) ) self.make_xmit_channel() if self.restart_position: # try to position the file as requested, but # give up silently on failure (the 'file object' # may not support seek()) try: fd.seek (self.restart_position) except: pass self.restart_position = 0 self.client_dc.push_with_producer ( file_producer (fd) ) self.client_dc.close_when_done() def cmd_stor (self, line, mode='wb'): 'store a file' if len (line) < 2: self.command_not_understood (string.join (line)) else: if self.restart_position: restart_position = 0 self.respond ('553 restart on STOR not yet supported') return file = line[1] # todo: handle that type flag try: fd = self.open (file, mode) except IOError, why: self.respond ('553 could not open file for writing: %s' % (repr(why))) return self.respond ( '150 Opening %s connection for %s' % ( self.type_map[self.current_mode], file ) ) self.make_recv_channel (fd) def cmd_abor (self, line): 'abort operation' if self.client_dc: self.client_dc.close() self.respond ('226 ABOR command successful.') def cmd_appe (self, line): 'append to a file' return self.cmd_stor (line, 'ab') def cmd_dele (self, line): if len (line) != 2: self.command_not_understood (string.join (line)) else: file = line[1] if self.filesystem.isfile (file): try: self.filesystem.unlink (file) self.respond ('250 DELE command successful.') except: self.respond ('550 error deleting file.') else: self.respond ('550 %s: No such file.' % file) def cmd_mkd (self, line): if len (line) != 2: self.command_not_understood (string.join (line)) else: path = line[1] try: self.filesystem.mkdir (path) self.respond ('257 MKD command successful.') except: self.respond ('550 error creating directory.') def cmd_rmd (self, line): if len (line) != 2: self.command_not_understood (string.join (line)) else: path = line[1] try: self.filesystem.rmdir (path) self.respond ('250 RMD command successful.') except: self.respond ('550 error removing directory.') def cmd_user (self, line): 'specify user name' if len(line) > 1: self.user = line[1] self.respond ('331 Password required.') else: self.command_not_understood (string.join (line)) def cmd_pass (self, line): 'specify password' if len(line) < 2: pw = '' else: pw = line[1] result, message, fs = self.server.authorizer.authorize (self, self.user, pw) if result: self.respond ('230 %s' % message) self.filesystem = fs self.authorized = 1 self.log_info('Successful login: Filesystem=%s' % repr(fs)) else: self.respond ('530 %s' % message) def cmd_rest (self, line): 'restart incomplete transfer' try: pos = string.atoi (line[1]) except ValueError: self.command_not_understood (string.join (line)) self.restart_position = pos self.respond ( '350 Restarting at %d. Send STORE or RETRIEVE to initiate transfer.' % pos ) def cmd_stru (self, line): 'obsolete - set file transfer structure' if line[1] in 'fF': # f == 'file' self.respond ('200 STRU F Ok') else: self.respond ('504 Unimplemented STRU type') def cmd_mode (self, line): 'obsolete - set file transfer mode' if line[1] in 'sS': # f == 'file' self.respond ('200 MODE S Ok') else: self.respond ('502 Unimplemented MODE type') # The stat command has two personalities. Normally it returns status # information about the current connection. But if given an argument, # it is equivalent to the LIST command, with the data sent over the # control connection. Strange. But wuftpd, ftpd, and nt's ftp server # all support it. # ## def cmd_stat (self, line): ## 'return status of server' ## pass def cmd_syst (self, line): 'show operating system type of server system' # Replying to this command is of questionable utility, because # this server does not behave in a predictable way w.r.t. the # output of the LIST command. We emulate Unix ls output, but # on win32 the pathname can contain drive information at the front # Currently, the combination of ensuring that os.sep == '/' # and removing the leading slash when necessary seems to work. # [cd'ing to another drive also works] # # This is how wuftpd responds, and is probably # the most expected. The main purpose of this reply is so that # the client knows to expect Unix ls-style LIST output. self.respond ('215 UNIX Type: L8') # one disadvantage to this is that some client programs # assume they can pass args to /bin/ls. # a few typical responses: # 215 UNIX Type: L8 (wuftpd) # 215 Windows_NT version 3.51 # 215 VMS MultiNet V3.3 # 500 'SYST': command not understood. (SVR4) def cmd_help (self, line): 'give help information' # find all the methods that match 'cmd_xxxx', # use their docstrings for the help response. attrs = dir(self.__class__) help_lines = [] for attr in attrs: if attr[:4] == 'cmd_': x = getattr (self, attr) if type(x) == type(self.cmd_help): if x.__doc__: help_lines.append ('\t%s\t%s' % (attr[4:], x.__doc__)) if help_lines: self.push ('214-The following commands are recognized\r\n') self.push_with_producer (producers.lines_producer (help_lines)) self.push ('214\r\n') else: self.push ('214-\r\n\tHelp Unavailable\r\n214\r\n') class ftp_server (asyncore.dispatcher): # override this to spawn a different FTP channel class. ftp_channel_class = ftp_channel SERVER_IDENT = 'FTP Server (V%s)' % VERSION def __init__ ( self, authorizer, hostname =None, ip ='', port =21, resolver =None, logger_object=logger.file_logger (sys.stdout) ): self.ip = ip self.port = port self.authorizer = authorizer if hostname is None: self.hostname = socket.gethostname() else: self.hostname = hostname # statistics self.total_sessions = counter() self.closed_sessions = counter() self.total_files_out = counter() self.total_files_in = counter() self.total_bytes_out = counter() self.total_bytes_in = counter() self.total_exceptions = counter() # asyncore.dispatcher.__init__ (self) self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind ((self.ip, self.port)) self.listen (5) if not logger_object: logger_object = sys.stdout if resolver: self.logger = logger.resolving_logger (resolver, logger_object) else: self.logger = logger.unresolving_logger (logger_object) self.log_info('FTP server started at %s\n\tAuthorizer:%s\n\tHostname: %s\n\tPort: %d' % ( time.ctime(time.time()), repr (self.authorizer), self.hostname, self.port) ) def writable (self): return 0 def handle_read (self): pass def handle_connect (self): pass def handle_accept (self): conn, addr = self.accept() self.total_sessions.increment() self.log_info('Incoming connection from %s:%d' % (addr[0], addr[1])) self.ftp_channel_class (self, conn, addr) # return a producer describing the state of the server def status (self): def nice_bytes (n): return string.join (status_handler.english_bytes (n)) return producers.lines_producer ( ['

    %s

    ' % self.SERVER_IDENT, '
    Listening on Host: %s' % self.hostname, 'Port: %d' % self.port, '
    Sessions', 'Total: %s' % self.total_sessions, 'Current: %d' % (self.total_sessions.as_long() - self.closed_sessions.as_long()), '
    Files', 'Sent: %s' % self.total_files_out, 'Received: %s' % self.total_files_in, '
    Bytes', 'Sent: %s' % nice_bytes (self.total_bytes_out.as_long()), 'Received: %s' % nice_bytes (self.total_bytes_in.as_long()), '
    Exceptions: %s' % self.total_exceptions, ] ) # ====================================================================== # Data Channel Classes # ====================================================================== # This socket accepts a data connection, used when the server has been # placed in passive mode. Although the RFC implies that we ought to # be able to use the same acceptor over and over again, this presents # a problem: how do we shut it off, so that we are accepting # connections only when we expect them? [we can't] # # wuftpd, and probably all the other servers, solve this by allowing # only one connection to hit this acceptor. They then close it. Any # subsequent data-connection command will then try for the default # port on the client side [which is of course never there]. So the # 'always-send-PORT/PASV' behavior seems required. # # Another note: wuftpd will also be listening on the channel as soon # as the PASV command is sent. It does not wait for a data command # first. # --- we need to queue up a particular behavior: # 1) xmit : queue up producer[s] # 2) recv : the file object # # It would be nice if we could make both channels the same. Hmmm.. # class passive_acceptor (asyncore.dispatcher): ready = None def __init__ (self, control_channel): # connect_fun (conn, addr) asyncore.dispatcher.__init__ (self) self.control_channel = control_channel self.create_socket (socket.AF_INET, socket.SOCK_STREAM) # bind to an address on the interface that the # control connection is coming from. self.bind (( self.control_channel.getsockname()[0], 0 )) self.addr = self.getsockname() self.listen (1) # def __del__ (self): # print 'passive_acceptor.__del__()' def log (self, *ignore): pass def handle_accept (self): conn, addr = self.accept() dc = self.control_channel.client_dc if dc is not None: dc.set_socket (conn) dc.addr = addr dc.connected = 1 self.control_channel.passive_acceptor = None else: self.ready = conn, addr self.close() class xmit_channel (asynchat.async_chat): # for an ethernet, you want this to be fairly large, in fact, it # _must_ be large for performance comparable to an ftpd. [64k] we # ought to investigate automatically-sized buffers... ac_out_buffer_size = 16384 bytes_out = 0 def __init__ (self, channel, client_addr=None): self.channel = channel self.client_addr = client_addr asynchat.async_chat.__init__ (self) # def __del__ (self): # print 'xmit_channel.__del__()' def log (self, *args): pass def readable (self): return not self.connected def writable (self): return 1 def send (self, data): result = asynchat.async_chat.send (self, data) self.bytes_out = self.bytes_out + result return result def handle_error (self): # usually this is to catch an unexpected disconnect. self.log_info ('unexpected disconnect on data xmit channel', 'error') try: self.close() except: pass # TODO: there's a better way to do this. we need to be able to # put 'events' in the producer fifo. to do this cleanly we need # to reposition the 'producer' fifo as an 'event' fifo. def close (self): c = self.channel s = c.server c.client_dc = None s.total_files_out.increment() s.total_bytes_out.increment (self.bytes_out) if not len(self.producer_fifo): c.respond ('226 Transfer complete') elif not c.closed: c.respond ('426 Connection closed; transfer aborted') del c del s del self.channel asynchat.async_chat.close (self) class recv_channel (asyncore.dispatcher): def __init__ (self, channel, client_addr, fd): self.channel = channel self.client_addr = client_addr self.fd = fd asyncore.dispatcher.__init__ (self) self.bytes_in = counter() def log (self, *ignore): pass def handle_connect (self): pass def writable (self): return 0 def recv (*args): result = apply (asyncore.dispatcher.recv, args) self = args[0] self.bytes_in.increment(len(result)) return result buffer_size = 8192 def handle_read (self): block = self.recv (self.buffer_size) if block: try: self.fd.write (block) except IOError: self.log_info ('got exception writing block...', 'error') def handle_close (self): s = self.channel.server s.total_files_in.increment() s.total_bytes_in.increment(self.bytes_in.as_long()) self.fd.close() self.channel.respond ('226 Transfer complete.') self.close() import filesys # not much of a doorman! 8^) class dummy_authorizer: def __init__ (self, root='/'): self.root = root def authorize (self, channel, username, password): channel.persona = -1, -1 channel.read_only = 1 return 1, 'Ok.', filesys.os_filesystem (self.root) class anon_authorizer: def __init__ (self, root='/'): self.root = root def authorize (self, channel, username, password): if username in ('ftp', 'anonymous'): channel.persona = -1, -1 channel.read_only = 1 return 1, 'Ok.', filesys.os_filesystem (self.root) else: return 0, 'Password invalid.', None # =========================================================================== # Unix-specific improvements # =========================================================================== if os.name == 'posix': class unix_authorizer: # return a trio of (success, reply_string, filesystem) def authorize (self, channel, username, password): import crypt import pwd try: info = pwd.getpwnam (username) except KeyError: return 0, 'No such user.', None mangled = info[1] if crypt.crypt (password, mangled[:2]) == mangled: channel.read_only = 0 fs = filesys.schizophrenic_unix_filesystem ( '/', info[5], persona = (info[2], info[3]) ) return 1, 'Login successful.', fs else: return 0, 'Password invalid.', None def __repr__ (self): return '' # simple anonymous ftp support class unix_authorizer_with_anonymous (unix_authorizer): def __init__ (self, root=None, real_users=0): self.root = root self.real_users = real_users def authorize (self, channel, username, password): if string.lower(username) in ['anonymous', 'ftp']: import pwd try: # ok, here we run into lots of confusion. # on some os', anon runs under user 'nobody', # on others as 'ftp'. ownership is also critical. # need to investigate. # linux: new linuxen seem to have nobody's UID=-1, # which is an illegal value. Use ftp. ftp_user_info = pwd.getpwnam ('ftp') if string.lower(os.uname()[0]) == 'linux': nobody_user_info = pwd.getpwnam ('ftp') else: nobody_user_info = pwd.getpwnam ('nobody') channel.read_only = 1 if self.root is None: self.root = ftp_user_info[5] fs = filesys.unix_filesystem (self.root, '/') return 1, 'Anonymous Login Successful', fs except KeyError: return 0, 'Anonymous account not set up', None elif self.real_users: return unix_authorizer.authorize ( self, channel, username, password ) else: return 0, 'User logins not allowed', None # usage: ftp_server /PATH/TO/FTP/ROOT PORT # for example: # $ ftp_server /home/users/ftp 8021 if os.name == 'posix': def test (port='8021'): fs = ftp_server ( unix_authorizer(), port=string.atoi (port) ) try: asyncore.loop() except KeyboardInterrupt: fs.log_info('FTP server shutting down. (received SIGINT)', 'warning') # close everything down on SIGINT. # of course this should be a cleaner shutdown. asyncore.close_all() if __name__ == '__main__': test (sys.argv[1]) # not unix else: def test (): fs = ftp_server (dummy_authorizer()) if __name__ == '__main__': test () # this is the command list from the wuftpd man page # '*' means we've implemented it. # '!' requires write access # command_documentation = { 'abor': 'abort previous command', #* 'acct': 'specify account (ignored)', 'allo': 'allocate storage (vacuously)', 'appe': 'append to a file', #*! 'cdup': 'change to parent of current working directory', #* 'cwd': 'change working directory', #* 'dele': 'delete a file', #! 'help': 'give help information', #* 'list': 'give list files in a directory', #* 'mkd': 'make a directory', #! 'mdtm': 'show last modification time of file', #* 'mode': 'specify data transfer mode', 'nlst': 'give name list of files in directory', #* 'noop': 'do nothing', #* 'pass': 'specify password', #* 'pasv': 'prepare for server-to-server transfer', #* 'port': 'specify data connection port', #* 'pwd': 'print the current working directory', #* 'quit': 'terminate session', #* 'rest': 'restart incomplete transfer', #* 'retr': 'retrieve a file', #* 'rmd': 'remove a directory', #! 'rnfr': 'specify rename-from file name', #! 'rnto': 'specify rename-to file name', #! 'site': 'non-standard commands (see next section)', 'size': 'return size of file', #* 'stat': 'return status of server', #* 'stor': 'store a file', #*! 'stou': 'store a file with a unique name', #! 'stru': 'specify data transfer structure', 'syst': 'show operating system type of server system', #* 'type': 'specify data transfer type', #* 'user': 'specify user name', #* 'xcup': 'change to parent of current working directory (deprecated)', 'xcwd': 'change working directory (deprecated)', 'xmkd': 'make a directory (deprecated)', #! 'xpwd': 'print the current working directory (deprecated)', 'xrmd': 'remove a directory (deprecated)', #! } # debugging aid (linux) def get_vm_size (): return string.atoi (string.split(open ('/proc/self/stat').readline())[22]) def print_vm(): print 'vm: %8dk' % (get_vm_size()/1024) medusa-0.5.4/http_date.py0100644000076400007640000000652707445740176013467 0ustar amkamk# -*- Mode: Python -*- import re import string import time def concat (*args): return ''.join (args) def join (seq, field=' '): return field.join (seq) def group (s): return '(' + s + ')' short_days = ['sun','mon','tue','wed','thu','fri','sat'] long_days = ['sunday','monday','tuesday','wednesday','thursday','friday','saturday'] short_day_reg = group (join (short_days, '|')) long_day_reg = group (join (long_days, '|')) daymap = {} for i in range(7): daymap[short_days[i]] = i daymap[long_days[i]] = i hms_reg = join (3 * [group('[0-9][0-9]')], ':') months = ['jan','feb','mar','apr','may','jun','jul','aug','sep','oct','nov','dec'] monmap = {} for i in range(12): monmap[months[i]] = i+1 months_reg = group (join (months, '|')) # From draft-ietf-http-v11-spec-07.txt/3.3.1 # Sun, 06 Nov 1994 08:49:37 GMT ; RFC 822, updated by RFC 1123 # Sunday, 06-Nov-94 08:49:37 GMT ; RFC 850, obsoleted by RFC 1036 # Sun Nov 6 08:49:37 1994 ; ANSI C's asctime() format # rfc822 format rfc822_date = join ( [concat (short_day_reg,','), # day group('[0-9][0-9]?'), # date months_reg, # month group('[0-9]+'), # year hms_reg, # hour minute second 'gmt' ], ' ' ) rfc822_reg = re.compile (rfc822_date) def unpack_rfc822 (m): g = m.group a = string.atoi return ( a(g(4)), # year monmap[g(3)], # month a(g(2)), # day a(g(5)), # hour a(g(6)), # minute a(g(7)), # second 0, 0, 0 ) # rfc850 format rfc850_date = join ( [concat (long_day_reg,','), join ( [group ('[0-9][0-9]?'), months_reg, group ('[0-9]+') ], '-' ), hms_reg, 'gmt' ], ' ' ) rfc850_reg = re.compile (rfc850_date) # they actually unpack the same way def unpack_rfc850 (m): g = m.group a = string.atoi return ( a(g(4)), # year monmap[g(3)], # month a(g(2)), # day a(g(5)), # hour a(g(6)), # minute a(g(7)), # second 0, 0, 0 ) # parsdate.parsedate - ~700/sec. # parse_http_date - ~1333/sec. def build_http_date (when): return time.strftime ('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(when)) def parse_http_date (d): d = string.lower (d) tz = time.timezone m = rfc850_reg.match (d) if m and m.end() == len(d): retval = int (time.mktime (unpack_rfc850(m)) - tz) else: m = rfc822_reg.match (d) if m and m.end() == len(d): retval = int (time.mktime (unpack_rfc822(m)) - tz) else: return 0 # Thanks to Craig Silverstein for pointing # out the DST discrepancy if time.daylight and time.localtime(retval)[-1] == 1: # DST correction retval = retval + (tz - time.altzone) return retval medusa-0.5.4/http_server.py0100644000076400007640000006123107701144442014036 0ustar amkamk#! /usr/local/bin/python # -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1996-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: http_server.py,v 1.11 2003/07/04 00:24:02 akuchling Exp $' # python modules import os import re import socket import string import sys import time # async modules import asyncore import asynchat # medusa modules import http_date import producers import status_handler import logger VERSION_STRING = string.split(RCS_ID)[2] from counter import counter from urllib import unquote, splitquery # =========================================================================== # Request Object # =========================================================================== class http_request: # default reply code reply_code = 200 request_counter = counter() # Whether to automatically use chunked encoding when # # HTTP version is 1.1 # Content-Length is not set # Chunked encoding is not already in effect # # If your clients are having trouble, you might want to disable this. use_chunked = 1 # by default, this request object ignores user data. collector = None def __init__ (self, *args): # unpack information about the request (self.channel, self.request, self.command, self.uri, self.version, self.header) = args self.outgoing = [] self.reply_headers = { 'Server' : 'Medusa/%s' % VERSION_STRING, 'Date' : http_date.build_http_date (time.time()) } self.request_number = http_request.request_counter.increment() self._split_uri = None self._header_cache = {} # -------------------------------------------------- # reply header management # -------------------------------------------------- def __setitem__ (self, key, value): self.reply_headers[key] = value def __getitem__ (self, key): return self.reply_headers[key] def has_key (self, key): return self.reply_headers.has_key (key) def build_reply_header (self): return string.join ( [self.response(self.reply_code)] + map ( lambda x: '%s: %s' % x, self.reply_headers.items() ), '\r\n' ) + '\r\n\r\n' # -------------------------------------------------- # split a uri # -------------------------------------------------- # ;?# path_regex = re.compile ( # path params query fragment r'([^;?#]*)(;[^?#]*)?(\?[^#]*)?(#.*)?' ) def split_uri (self): if self._split_uri is None: m = self.path_regex.match (self.uri) if m.end() != len(self.uri): raise ValueError, "Broken URI" else: self._split_uri = m.groups() return self._split_uri def get_header_with_regex (self, head_reg, group): for line in self.header: m = head_reg.match (line) if m.end() == len(line): return m.group (group) return '' def get_header (self, header): header = string.lower (header) hc = self._header_cache if not hc.has_key (header): h = header + ': ' hl = len(h) for line in self.header: if string.lower (line[:hl]) == h: r = line[hl:] hc[header] = r return r hc[header] = None return None else: return hc[header] # -------------------------------------------------- # user data # -------------------------------------------------- def collect_incoming_data (self, data): if self.collector: self.collector.collect_incoming_data (data) else: self.log_info( 'Dropping %d bytes of incoming request data' % len(data), 'warning' ) def found_terminator (self): if self.collector: self.collector.found_terminator() else: self.log_info ( 'Unexpected end-of-record for incoming request', 'warning' ) def push (self, thing): if type(thing) == type(''): self.outgoing.append(producers.simple_producer (thing)) else: self.outgoing.append(thing) def response (self, code=200): message = self.responses[code] self.reply_code = code return 'HTTP/%s %d %s' % (self.version, code, message) def error (self, code): self.reply_code = code message = self.responses[code] s = self.DEFAULT_ERROR_MESSAGE % { 'code': code, 'message': message, } self['Content-Length'] = len(s) self['Content-Type'] = 'text/html' # make an error reply self.push (s) self.done() # can also be used for empty replies reply_now = error def done (self): "finalize this transaction - send output to the http channel" # ---------------------------------------- # persistent connection management # ---------------------------------------- # --- BUCKLE UP! ---- connection = string.lower (get_header (CONNECTION, self.header)) close_it = 0 wrap_in_chunking = 0 if self.version == '1.0': if connection == 'keep-alive': if not self.has_key ('Content-Length'): close_it = 1 else: self['Connection'] = 'Keep-Alive' else: close_it = 1 elif self.version == '1.1': if connection == 'close': close_it = 1 elif not self.has_key ('Content-Length'): if self.has_key ('Transfer-Encoding'): if not self['Transfer-Encoding'] == 'chunked': close_it = 1 elif self.use_chunked: self['Transfer-Encoding'] = 'chunked' wrap_in_chunking = 1 else: close_it = 1 elif self.version is None: # Although we don't *really* support http/0.9 (because we'd have to # use \r\n as a terminator, and it would just yuck up a lot of stuff) # it's very common for developers to not want to type a version number # when using telnet to debug a server. close_it = 1 outgoing_header = producers.simple_producer (self.build_reply_header()) if close_it: self['Connection'] = 'close' if wrap_in_chunking: outgoing_producer = producers.chunked_producer ( producers.composite_producer (self.outgoing) ) # prepend the header outgoing_producer = producers.composite_producer( [outgoing_header, outgoing_producer] ) else: # prepend the header self.outgoing.insert(0, outgoing_header) outgoing_producer = producers.composite_producer (self.outgoing) # apply a few final transformations to the output self.channel.push_with_producer ( # globbing gives us large packets producers.globbing_producer ( # hooking lets us log the number of bytes sent producers.hooked_producer ( outgoing_producer, self.log ) ) ) self.channel.current_request = None if close_it: self.channel.close_when_done() def log_date_string (self, when): gmt = time.gmtime(when) if time.daylight and gmt[8]: tz = time.altzone else: tz = time.timezone if tz > 0: neg = 1 else: neg = 0 tz = -tz h, rem = divmod (tz, 3600) m, rem = divmod (rem, 60) if neg: offset = '-%02d%02d' % (h, m) else: offset = '+%02d%02d' % (h, m) return time.strftime ( '%d/%b/%Y:%H:%M:%S ', gmt) + offset def log (self, bytes): self.channel.server.logger.log ( self.channel.addr[0], '%d - - [%s] "%s" %d %d\n' % ( self.channel.addr[1], self.log_date_string (time.time()), self.request, self.reply_code, bytes ) ) responses = { 100: "Continue", 101: "Switching Protocols", 200: "OK", 201: "Created", 202: "Accepted", 203: "Non-Authoritative Information", 204: "No Content", 205: "Reset Content", 206: "Partial Content", 300: "Multiple Choices", 301: "Moved Permanently", 302: "Moved Temporarily", 303: "See Other", 304: "Not Modified", 305: "Use Proxy", 400: "Bad Request", 401: "Unauthorized", 402: "Payment Required", 403: "Forbidden", 404: "Not Found", 405: "Method Not Allowed", 406: "Not Acceptable", 407: "Proxy Authentication Required", 408: "Request Time-out", 409: "Conflict", 410: "Gone", 411: "Length Required", 412: "Precondition Failed", 413: "Request Entity Too Large", 414: "Request-URI Too Large", 415: "Unsupported Media Type", 500: "Internal Server Error", 501: "Not Implemented", 502: "Bad Gateway", 503: "Service Unavailable", 504: "Gateway Time-out", 505: "HTTP Version not supported" } # Default error message DEFAULT_ERROR_MESSAGE = string.join ( ['', 'Error response', '', '', '

    Error response

    ', '

    Error code %(code)d.', '

    Message: %(message)s.', '', '' ], '\r\n' ) # =========================================================================== # HTTP Channel Object # =========================================================================== class http_channel (asynchat.async_chat): # use a larger default output buffer ac_out_buffer_size = 1<<16 current_request = None channel_counter = counter() def __init__ (self, server, conn, addr): self.channel_number = http_channel.channel_counter.increment() self.request_counter = counter() asynchat.async_chat.__init__ (self, conn) self.server = server self.addr = addr self.set_terminator ('\r\n\r\n') self.in_buffer = '' self.creation_time = int (time.time()) self.check_maintenance() def __repr__ (self): ar = asynchat.async_chat.__repr__(self)[1:-1] return '<%s channel#: %s requests:%s>' % ( ar, self.channel_number, self.request_counter ) # Channel Counter, Maintenance Interval... maintenance_interval = 500 def check_maintenance (self): if not self.channel_number % self.maintenance_interval: self.maintenance() def maintenance (self): self.kill_zombies() # 30-minute zombie timeout. status_handler also knows how to kill zombies. zombie_timeout = 30 * 60 def kill_zombies (self): now = int (time.time()) for channel in asyncore.socket_map.values(): if channel.__class__ == self.__class__: if (now - channel.creation_time) > channel.zombie_timeout: channel.close() # -------------------------------------------------- # send/recv overrides, good place for instrumentation. # -------------------------------------------------- # this information needs to get into the request object, # so that it may log correctly. def send (self, data): result = asynchat.async_chat.send (self, data) self.server.bytes_out.increment (len(data)) return result def recv (self, buffer_size): try: result = asynchat.async_chat.recv (self, buffer_size) self.server.bytes_in.increment (len(result)) return result except MemoryError: # --- Save a Trip to Your Service Provider --- # It's possible for a process to eat up all the memory of # the machine, and put it in an extremely wedged state, # where medusa keeps running and can't be shut down. This # is where MemoryError tends to get thrown, though of # course it could get thrown elsewhere. sys.exit ("Out of Memory!") def handle_error (self): t, v = sys.exc_info()[:2] if t is SystemExit: raise t, v else: asynchat.async_chat.handle_error (self) def log (self, *args): pass # -------------------------------------------------- # async_chat methods # -------------------------------------------------- def collect_incoming_data (self, data): if self.current_request: # we are receiving data (probably POST data) for a request self.current_request.collect_incoming_data (data) else: # we are receiving header (request) data self.in_buffer = self.in_buffer + data def found_terminator (self): if self.current_request: self.current_request.found_terminator() else: header = self.in_buffer self.in_buffer = '' lines = string.split (header, '\r\n') # -------------------------------------------------- # crack the request header # -------------------------------------------------- while lines and not lines[0]: # as per the suggestion of http-1.1 section 4.1, (and # Eric Parker ), ignore a leading # blank lines (buggy browsers tack it onto the end of # POST requests) lines = lines[1:] if not lines: self.close_when_done() return request = lines[0] command, uri, version = crack_request (request) header = join_headers (lines[1:]) # unquote path if necessary (thanks to Skip Montanaro for pointing # out that we must unquote in piecemeal fashion). rpath, rquery = splitquery(uri) if '%' in rpath: if rquery: uri = unquote (rpath) + '?' + rquery else: uri = unquote (rpath) r = http_request (self, request, command, uri, version, header) self.request_counter.increment() self.server.total_requests.increment() if command is None: self.log_info ('Bad HTTP request: %s' % repr(request), 'error') r.error (400) return # -------------------------------------------------- # handler selection and dispatch # -------------------------------------------------- for h in self.server.handlers: if h.match (r): try: self.current_request = r # This isn't used anywhere. # r.handler = h # CYCLE h.handle_request (r) except: self.server.exceptions.increment() (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() self.log_info( 'Server Error: %s, %s: file: %s line: %s' % (t,v,file,line), 'error') try: r.error (500) except: pass return # no handlers, so complain r.error (404) def writable_for_proxy (self): # this version of writable supports the idea of a 'stalled' producer # [i.e., it's not ready to produce any output yet] This is needed by # the proxy, which will be waiting for the magic combination of # 1) hostname resolved # 2) connection made # 3) data available. if self.ac_out_buffer: return 1 elif len(self.producer_fifo): p = self.producer_fifo.first() if hasattr (p, 'stalled'): return not p.stalled() else: return 1 # =========================================================================== # HTTP Server Object # =========================================================================== class http_server (asyncore.dispatcher): SERVER_IDENT = 'HTTP Server (V%s)' % VERSION_STRING channel_class = http_channel def __init__ (self, ip, port, resolver=None, logger_object=None): self.ip = ip self.port = port asyncore.dispatcher.__init__ (self) self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.handlers = [] if not logger_object: logger_object = logger.file_logger (sys.stdout) self.set_reuse_addr() self.bind ((ip, port)) # lower this to 5 if your OS complains self.listen (1024) host, port = self.socket.getsockname() if not ip: self.log_info('Computing default hostname', 'warning') ip = socket.gethostbyname (socket.gethostname()) try: self.server_name = socket.gethostbyaddr (ip)[0] except socket.error: self.log_info('Cannot do reverse lookup', 'warning') self.server_name = ip # use the IP address as the "hostname" self.server_port = port self.total_clients = counter() self.total_requests = counter() self.exceptions = counter() self.bytes_out = counter() self.bytes_in = counter() if not logger_object: logger_object = logger.file_logger (sys.stdout) if resolver: self.logger = logger.resolving_logger (resolver, logger_object) else: self.logger = logger.unresolving_logger (logger_object) self.log_info ( 'Medusa (V%s) started at %s' '\n\tHostname: %s' '\n\tPort:%d' '\n' % ( VERSION_STRING, time.ctime(time.time()), self.server_name, port, ) ) def writable (self): return 0 def handle_read (self): pass def readable (self): return self.accepting def handle_connect (self): pass def handle_accept (self): self.total_clients.increment() try: conn, addr = self.accept() except socket.error: # linux: on rare occasions we get a bogus socket back from # accept. socketmodule.c:makesockaddr complains that the # address family is unknown. We don't want the whole server # to shut down because of this. self.log_info ('warning: server accept() threw an exception', 'warning') return except TypeError: # unpack non-sequence. this can happen when a read event # fires on a listening socket, but when we call accept() # we get EWOULDBLOCK, so dispatcher.accept() returns None. # Seen on FreeBSD3. self.log_info ('warning: server accept() threw EWOULDBLOCK', 'warning') return self.channel_class (self, conn, addr) def install_handler (self, handler, back=0): if back: self.handlers.append (handler) else: self.handlers.insert (0, handler) def remove_handler (self, handler): self.handlers.remove (handler) def status (self): def nice_bytes (n): return string.join (status_handler.english_bytes (n)) handler_stats = filter (None, map (maybe_status, self.handlers)) if self.total_clients: ratio = self.total_requests.as_long() / float(self.total_clients.as_long()) else: ratio = 0.0 return producers.composite_producer ( [producers.lines_producer ( ['

    %s

    ' % self.SERVER_IDENT, '
    Listening on: Host: %s' % self.server_name, 'Port: %d' % self.port, '

      ' '
    • Total Clients: %s' % self.total_clients, 'Requests: %s' % self.total_requests, 'Requests/Client: %.1f' % (ratio), '
    • Total Bytes In: %s' % (nice_bytes (self.bytes_in.as_long())), 'Bytes Out: %s' % (nice_bytes (self.bytes_out.as_long())), '
    • Total Exceptions: %s' % self.exceptions, '

    ' 'Extension List

      ', ])] + handler_stats + [producers.simple_producer('
    ')] ) def maybe_status (thing): if hasattr (thing, 'status'): return thing.status() else: return None CONNECTION = re.compile ('Connection: (.*)', re.IGNORECASE) # merge multi-line headers # [486dx2: ~500/sec] def join_headers (headers): r = [] for i in range(len(headers)): if headers[i][0] in ' \t': r[-1] = r[-1] + headers[i][1:] else: r.append (headers[i]) return r def get_header (head_reg, lines, group=1): for line in lines: m = head_reg.match (line) if m and m.end() == len(line): return m.group (group) return '' def get_header_match (head_reg, lines): for line in lines: m = head_reg.match (line) if m and m.end() == len(line): return m return '' REQUEST = re.compile ('([^ ]+) ([^ ]+)(( HTTP/([0-9.]+))$|$)') def crack_request (r): m = REQUEST.match (r) if m and m.end() == len(r): if m.group(3): version = m.group(5) else: version = None return m.group(1), m.group(2), version else: return None, None, None if __name__ == '__main__': import sys if len(sys.argv) < 2: print 'usage: %s ' % (sys.argv[0]) else: import monitor import filesys import default_handler import status_handler import ftp_server import chat_server import resolver import logger rs = resolver.caching_resolver ('127.0.0.1') lg = logger.file_logger (sys.stdout) ms = monitor.secure_monitor_server ('fnord', '127.0.0.1', 9999) fs = filesys.os_filesystem (sys.argv[1]) dh = default_handler.default_handler (fs) hs = http_server ('', string.atoi (sys.argv[2]), rs, lg) hs.install_handler (dh) ftp = ftp_server.ftp_server ( ftp_server.dummy_authorizer(sys.argv[1]), port=8021, resolver=rs, logger_object=lg ) cs = chat_server.chat_server ('', 7777) sh = status_handler.status_extension([hs,ms,ftp,cs,rs]) hs.install_handler (sh) if ('-p' in sys.argv): def profile_loop (): try: asyncore.loop() except KeyboardInterrupt: pass import profile profile.run ('profile_loop()', 'profile.out') else: asyncore.loop() medusa-0.5.4/__init__.py0100644000076400007640000000017107445740176013237 0ustar amkamk"""medusa.__init__ """ # created 2002/03/19, AMK __revision__ = "$Id: __init__.py,v 1.2 2002/03/19 22:49:34 amk Exp $" medusa-0.5.4/INSTALL.txt0100644000076400007640000001126107450622256012771 0ustar amkamk Medusa Installation. --------------------------------------------------------------------------- 1. INSTALL PYTHON Medusa is distributed as Python source code. Before using Medusa, you will need to install Python on your machine. The Python interpreter, source, documentation, etc... may be obtained from http://www.python.org/ Versions for many different operating systems are available, including Unix, 32-bit Windows (Win95 & NT), Macintosh, VMS, etc... Medusa has been tested on Unix and Windows, though it may very well work on other operating systems. You don't need to learn Python in order to use Medusa. However, if you are interested in extending Medusa, you should spend the hour or so that it will take you to go through the Python Tutorial: http://www.python.org/doc/tut/ Python is remarkably easy to learn, and I guarantee that it will be worth your while. After only about thirty minutes, you should know enough about Python to be able to start customizing and extending Medusa. 2. INSTALL MEDUSA The core Medusa code consists of a single package named 'medusa'. To install it in the site-packages directory of your Python installation, run the command "python setup.py install". After running this command, you should be able to run the Python interpreter and import things from the 'medusa' package: bash-2.05$ python >>> from medusa import ftp_server >>> 3. WRITE A MEDUSA STARTUP SCRIPT Once you have installed Python, you are ready to configure Medusa. Medusa does not use configuration files per se, or even command-line arguments. It is configured via a 'startup script', written in Python. A sample is provided in 'demo/start_medusa.py'. You should make a copy of this. The sample startup script is heavily commented. Many (though not all) of Medusa's features are made available in the startup script. You may modify this script by commenting out portions, adding or changing parameters, etc... Here is a section from the front of 'demo/start_medusa.py' | if len(sys.argv) > 1: | # process a few convenient arguments | [HOSTNAME, IP_ADDRESS, PUBLISHING_ROOT] = sys.argv[1:] | else: | HOSTNAME = 'www.nightmare.com' | # This is the IP address of the network interface you want | # your servers to be visible from. This can be changed to '' | # to listen on all interfaces. | IP_ADDRESS = '205.160.176.5' | | # Root of the http and ftp server's published filesystems. | PUBLISHING_ROOT = '/home/www' | | HTTP_PORT = 8080 # The standard port is 80 | FTP_PORT = 8021 # The standard port is 21 | CHAT_PORT = 8888 | MONITOR_PORT = 9999 If you are familiar with the process of configuring a web or ftp server, then these parameters should be fairly obvious: You will need to change the hostname, IP address, and port numbers for the server that you wish to run. A Medusa configuration does not need to be this complex - demo/start_medusa.py is bloated somewhat by its attempt to include most of the available features. Another example startup script, demo/publish.py, is also available for you to look at. Once you have made your own startup script, you may simply invoke the Python interpreter on it: [Unix] $ python start_medusa.py & [Win32] d:\medusa\> start python start_medusa.py Medusa (V3.8) started at Sat Jan 24 01:43:21 1998 Hostname: ziggurat.nightmare.com Port:8080 FTP server started at Sat Jan 24 01:43:21 1998 Authorizer: Hostname: ziggurat.nightmare.com Port: 21 192.168.200.40:1450 - - [24/Jan/1998:07:43:23 -0500] "GET /status HTTP/1.0" 200 1638 192.168.200.40:1451 - - [24/Jan/1998:07:43:23 -0500] "GET /status/medusa.gif HTTP/1.0" 200 1084 Documentation for specific Medusa servers is somewhat lacking, mostly because development continues to move rapidly. The best place to go to understand Medusa and how it works is to dive into the source code. Many of the more interesting features, especially the latest, are described only in the source code. Some notes on data flow in Medusa are available in 'docs/data_flow.html'. I encourage you to examine and experiment with Medusa. You may develop your own extensions, handlers, etc... I appreciate feedback from users and developers on desired features, and of course descriptions of your most splendid hacks. Medusa's design is somewhat novel compared to most other network servers. In fact, the asynchronous i/o capability seems to have attracted the majority of paying customers, who are often more interested in harnessing the i/o framework than the actual web and ftp servers. medusa-0.5.4/LICENSE.txt0100644000076400007640000000276607455335034012757 0ustar amkamkMedusa was once distributed under a 'free for non-commercial use' license, but in May of 2000 Sam Rushing changed the license to be identical to the standard Python license at the time. The standard Python license has always applied to the core components of Medusa, this change just frees up the rest of the system, including the http server, ftp server, utilities, etc. Medusa is therefore under the following license: ============================== Permission to use, copy, modify, and distribute this software and its documentation for any purpose and without fee is hereby granted, provided that the above copyright notice appear in all copies and that both that copyright notice and this permission notice appear in supporting documentation, and that the name of Sam Rushing not be used in advertising or publicity pertaining to distribution of the software without specific, written prior permission. SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. ============================== Sam would like to take this opportunity to thank all of the folks who supported Medusa over the years by purchasing commercial licenses. medusa-0.5.4/logger.py0100644000076400007640000001750007515074256012760 0ustar amkamk# -*- Mode: Python -*- import asynchat import socket import time # these three are for the rotating logger import os # | import stat # v # # three types of log: # 1) file # with optional flushing. Also, one that rotates the log. # 2) socket # dump output directly to a socket connection. [how do we # keep it open?] # 3) syslog # log to syslog via tcp. this is a per-line protocol. # # # The 'standard' interface to a logging object is simply # log_object.log (message) # # a file-like object that captures output, and # makes sure to flush it always... this could # be connected to: # o stdio file # o low-level file # o socket channel # o syslog output... class file_logger: # pass this either a path or a file object. def __init__ (self, file, flush=1, mode='a'): if type(file) == type(''): if (file == '-'): import sys self.file = sys.stdout else: self.file = open (file, mode) else: self.file = file self.do_flush = flush def __repr__ (self): return '' % self.file def write (self, data): self.file.write (data) self.maybe_flush() def writeline (self, line): self.file.writeline (line) self.maybe_flush() def writelines (self, lines): self.file.writelines (lines) self.maybe_flush() def maybe_flush (self): if self.do_flush: self.file.flush() def flush (self): self.file.flush() def softspace (self, *args): pass def log (self, message): if message[-1] not in ('\r', '\n'): self.write (message + '\n') else: self.write (message) # like a file_logger, but it must be attached to a filename. # When the log gets too full, or a certain time has passed, # it backs up the log and starts a new one. Note that backing # up the log is done via "mv" because anything else (cp, gzip) # would take time, during which medusa would do nothing else. class rotating_file_logger (file_logger): # If freq is non-None we back up "daily", "weekly", or "monthly". # Else if maxsize is non-None we back up whenever the log gets # to big. If both are None we never back up. def __init__ (self, file, freq=None, maxsize=None, flush=1, mode='a'): self.filename = file self.mode = mode self.file = open (file, mode) self.freq = freq self.maxsize = maxsize self.rotate_when = self.next_backup(self.freq) self.do_flush = flush def __repr__ (self): return '' % self.file # We back up at midnight every 1) day, 2) monday, or 3) 1st of month def next_backup (self, freq): (yr, mo, day, hr, min, sec, wd, jday, dst) = time.localtime(time.time()) if freq == 'daily': return time.mktime((yr,mo,day+1, 0,0,0, 0,0,-1)) elif freq == 'weekly': return time.mktime((yr,mo,day-wd+7, 0,0,0, 0,0,-1)) # wd(monday)==0 elif freq == 'monthly': return time.mktime((yr,mo+1,1, 0,0,0, 0,0,-1)) else: return None # not a date-based backup def maybe_flush (self): # rotate first if necessary self.maybe_rotate() if self.do_flush: # from file_logger() self.file.flush() def maybe_rotate (self): if self.freq and time.time() > self.rotate_when: self.rotate() self.rotate_when = self.next_backup(self.freq) elif self.maxsize: # rotate when we get too big try: if os.stat(self.filename)[stat.ST_SIZE] > self.maxsize: self.rotate() except os.error: # file not found, probably self.rotate() # will create a new file def rotate (self): (yr, mo, day, hr, min, sec, wd, jday, dst) = time.localtime(time.time()) try: self.file.close() newname = '%s.ends%04d%02d%02d' % (self.filename, yr, mo, day) try: open(newname, "r").close() # check if file exists newname = newname + "-%02d%02d%02d" % (hr, min, sec) except: # YEARMODY is unique pass os.rename(self.filename, newname) self.file = open(self.filename, self.mode) except: pass # syslog is a line-oriented log protocol - this class would be # appropriate for FTP or HTTP logs, but not for dumping stderr to. # TODO: a simple safety wrapper that will ensure that the line sent # to syslog is reasonable. # TODO: async version of syslog_client: now, log entries use blocking # send() import m_syslog syslog_logger = m_syslog.syslog_client class syslog_logger (m_syslog.syslog_client): def __init__ (self, address, facility='user'): m_syslog.syslog_client.__init__ (self, address) self.facility = m_syslog.facility_names[facility] self.address=address def __repr__ (self): return '' % (repr(self.address)) def log (self, message): m_syslog.syslog_client.log ( self, message, facility=self.facility, priority=m_syslog.LOG_INFO ) # log to a stream socket, asynchronously class socket_logger (asynchat.async_chat): def __init__ (self, address): if type(address) == type(''): self.create_socket (socket.AF_UNIX, socket.SOCK_STREAM) else: self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.connect (address) self.address = address def __repr__ (self): return '' % (self.address) def log (self, message): if message[-2:] != '\r\n': self.socket.push (message + '\r\n') else: self.socket.push (message) # log to multiple places class multi_logger: def __init__ (self, loggers): self.loggers = loggers def __repr__ (self): return '' % (repr(self.loggers)) def log (self, message): for logger in self.loggers: logger.log (message) class resolving_logger: """Feed (ip, message) combinations into this logger to get a resolved hostname in front of the message. The message will not be logged until the PTR request finishes (or fails).""" def __init__ (self, resolver, logger): self.resolver = resolver self.logger = logger class logger_thunk: def __init__ (self, message, logger): self.message = message self.logger = logger def __call__ (self, host, ttl, answer): if not answer: answer = host self.logger.log ('%s:%s' % (answer, self.message)) def log (self, ip, message): self.resolver.resolve_ptr ( ip, self.logger_thunk ( message, self.logger ) ) class unresolving_logger: "Just in case you don't want to resolve" def __init__ (self, logger): self.logger = logger def log (self, ip, message): self.logger.log ('%s:%s' % (ip, message)) def strip_eol (line): while line and line[-1] in '\r\n': line = line[:-1] return line class tail_logger: "Keep track of the last log messages" def __init__ (self, logger, size=500): self.size = size self.logger = logger self.messages = [] def log (self, message): self.messages.append (strip_eol (message)) if len (self.messages) > self.size: del self.messages[0] self.logger.log (message) medusa-0.5.4/Makefile0100644000076400007640000000015207445740350012557 0ustar amkamk# -*- Mode: Makefile -*- clean: find ./ -name '*.pyc' -exec rm {} \; find ./ -name '*~' -exec rm {} \; medusa-0.5.4/MANIFEST0100644000076400007640000000226307725426150012255 0ustar amkamkauth_handler.py CHANGES.txt chat_server.py counter.py default_handler.py demo/publish.py demo/script_server.py demo/simple_anon_ftpd.py demo/start_medusa.py demo/winFTPserver.py docs/async_blurbs.txt docs/composing_producers.gif docs/data_flow.gif docs/data_flow.html docs/debugging.txt docs/producers.gif docs/programming.html docs/proxy_notes.txt docs/README.html docs/threads.txt docs/tkinter.txt event_loop.py filesys.py ftp_server.py http_date.py http_server.py __init__.py INSTALL.txt LICENSE.txt logger.py Makefile MANIFEST medusa_gif.py monitor_client.py monitor_client_win32.py monitor.py m_syslog.py producers.py put_handler.py README.txt redirecting_handler.py resolver.py rpc_client.py rpc_server.py script_handler.py setup.py status_handler.py test/asyn_http_bench.py test/max_sockets.py test/test_11.py test/test_lb.py test/test_medusa.py test/test_single_11.py test/tests.txt thread/pi_module.py thread/select_trigger.py thread/test_module.py thread/thread_channel.py thread/thread_handler.py TODO.txt unix_user_handler.py test/test_producers.py test/bench.py virtual_handler.py xmlrpc_handler.py debian/changelog debian/control debian/copyright debian/postinst debian/prerm debian/rules medusa-0.5.4/medusa_gif.py0100644000076400007640000000532407445723327013607 0ustar amkamk# -*- Mode: Python -*- # the medusa icon as a python source file. width = 97 height = 61 data = 'GIF89aa\000=\000\204\000\000\000\000\000\255\255\255\245\245\245ssskkkccc111)))\326\326\326!!!\316\316\316\300\300\300\204\204\000\224\224\224\214\214\214\200\200\200RRR\377\377\377JJJ\367\367\367BBB\347\347\347\000\204\000\020\020\020\265\265\265\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000\000!\371\004\001\000\000\021\000,\000\000\000\000a\000=\000\000\005\376`$\216di\236h\252\256l\353\276p,\317tm\337x\256\357|m\001@\240E\305\000\364\2164\206R)$\005\201\214\007r\012{X\255\312a\004\260\\>\026\3240\353)\224n\001W+X\334\373\231~\344.\303b\216\024\027x<\273\307\255G,rJiWN\014{S}k"?ti\013EdPQ\207G@_%\000\026yy\\\201\202\227\224<\221Fs$pOjWz\241\272\002\325\307g\012(\007\205\312#j\317(\012A\200\224.\241\003\346GS\247\033\245\344\264\366\015L\'PXQl]\266\263\243\232\260?\245\316\371\362\225\035\332\243J\273\332Q\263\357-D\241T\327\270\265\013W&\330\010u\371b\322IW0\214\261]\003\033Va\365Z#\207\213a\030k\2647\262\014p\354\024[n\321N\363\346\317\003\037P\000\235C\302\000\3228(\244\363YaA\005\022\255_\237@\260\000A\212\326\256qbp\321\332\266\011\334=T\023\010"!B\005\003A\010\224\020\220 H\002\337#\020 O\276E\357h\221\327\003\\\000b@v\004\351A.h\365\354\342B\002\011\257\025\\ \220\340\301\353\006\000\024\214\200pA\300\353\012\364\241k/\340\033C\202\003\000\310fZ\011\003V\240R\005\007\354\376\026A\000\000\360\'\202\177\024\004\210\003\000\305\215\360\000\000\015\220\240\332\203\027@\'\202\004\025VpA\000%\210x\321\206\032J\341\316\010\262\211H"l\333\341\200\200>"]P\002\212\011\010`\002\0066FP\200\001\'\024p]\004\027(8B\221\306]\000\201w>\002iB\001\007\340\260"v7J1\343(\257\020\251\243\011\242i\263\017\215\337\035\220\200\221\365m4d\015\016D\251\341iN\354\346Ng\253\200I\240\031\35609\245\2057\311I\302\2007t\231"&`\314\310\244\011e\226(\236\010w\212\300\234\011\012HX(\214\253\311@\001\233^\222pg{% \340\035\224&H\000\246\201\362\215`@\001"L\340\004\030\234\022\250\'\015(V:\302\235\030\240q\337\205\224\212h@\177\006\000\250\210\004\007\310\207\337\005\257-P\346\257\367]p\353\203\271\256:\203\236\211F\340\247\010\3329g\244\010\307*=A\000\203\260y\012\304s#\014\007D\207,N\007\304\265\027\021C\233\207%B\366[m\353\006\006\034j\360\306+\357\274a\204\000\000;' medusa-0.5.4/monitor_client.py0100644000076400007640000000631107446144354014524 0ustar amkamk# -*- Mode: Python -*- # monitor client, unix version. import asyncore import asynchat import socket import string import sys import os import md5 class stdin_channel (asyncore.file_dispatcher): def handle_read (self): data = self.recv(512) if not data: print '\nclosed.' self.sock_channel.close() try: self.close() except: pass data = string.replace(data, '\n', '\r\n') self.sock_channel.push (data) def writable (self): return 0 def log (self, *ignore): pass class monitor_client (asynchat.async_chat): def __init__ (self, password, addr=('',8023), socket_type=socket.AF_INET): asynchat.async_chat.__init__ (self) self.create_socket (socket_type, socket.SOCK_STREAM) self.terminator = '\r\n' self.connect (addr) self.sent_auth = 0 self.timestamp = '' self.password = password def collect_incoming_data (self, data): if not self.sent_auth: self.timestamp = self.timestamp + data else: sys.stdout.write (data) sys.stdout.flush() def found_terminator (self): if not self.sent_auth: self.push (hex_digest (self.timestamp + self.password) + '\r\n') self.sent_auth = 1 else: print def handle_close (self): # close all the channels, which will make the standard main # loop exit. map (lambda x: x.close(), asyncore.socket_map.values()) def log (self, *ignore): pass class encrypted_monitor_client (monitor_client): "Wrap push() and recv() with a stream cipher" def init_cipher (self, cipher, key): self.outgoing = cipher.new (key) self.incoming = cipher.new (key) def push (self, data): # push the encrypted data instead return monitor_client.push (self, self.outgoing.encrypt (data)) def recv (self, block_size): data = monitor_client.recv (self, block_size) if data: return self.incoming.decrypt (data) else: return data def hex_digest (s): m = md5.md5() m.update (s) return string.join ( map (lambda x: hex (ord (x))[2:], map (None, m.digest())), '', ) if __name__ == '__main__': if len(sys.argv) == 1: print 'Usage: %s host port' % sys.argv[0] sys.exit(0) if ('-e' in sys.argv): encrypt = 1 sys.argv.remove ('-e') else: encrypt = 0 sys.stderr.write ('Enter Password: ') sys.stderr.flush() try: os.system ('stty -echo') p = raw_input() print finally: os.system ('stty echo') stdin = stdin_channel (0) if len(sys.argv) > 1: if encrypt: client = encrypted_monitor_client (p, (sys.argv[1], string.atoi (sys.argv[2]))) import sapphire client.init_cipher (sapphire, p) else: client = monitor_client (p, (sys.argv[1], string.atoi (sys.argv[2]))) else: # default to local host, 'standard' port client = monitor_client (p) stdin.sock_channel = client asyncore.loop() medusa-0.5.4/monitor_client_win32.py0100644000076400007640000000246607446121101015536 0ustar amkamk# -*- Mode: Python -*- # monitor client, win32 version # since we can't do select() on stdin/stdout, we simply # use threads and blocking sockets. import socket import string import sys import thread import md5 def hex_digest (s): m = md5.md5() m.update (s) return string.join ( map (lambda x: hex (ord (x))[2:], map (None, m.digest())), '', ) def reader (lock, sock, password): # first grab the timestamp ts = sock.recv (1024)[:-2] sock.send (hex_digest (ts+password) + '\r\n') while 1: d = sock.recv (1024) if not d: lock.release() print 'Connection closed. Hit to exit' thread.exit() sys.stdout.write (d) sys.stdout.flush() def writer (lock, sock, barrel="just kidding"): while lock.locked(): sock.send ( sys.stdin.readline()[:-1] + '\r\n' ) if __name__ == '__main__': if len(sys.argv) == 1: print 'Usage: %s host port' sys.exit(0) print 'Enter Password: ', p = raw_input() s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect ((sys.argv[1], string.atoi(sys.argv[2]))) l = thread.allocate_lock() l.acquire() thread.start_new_thread (reader, (l, s, p)) writer (l, s) medusa-0.5.4/monitor.py0100644000076400007640000002544707447115126013175 0ustar amkamk# -*- Mode: Python -*- # Author: Sam Rushing # # python REPL channel. # RCS_ID = '$Id: monitor.py,v 1.5 2002/03/23 15:08:06 amk Exp $' import md5 import socket import string import sys import time VERSION = string.split(RCS_ID)[2] import asyncore import asynchat from counter import counter import producers class monitor_channel (asynchat.async_chat): try_linemode = 1 def __init__ (self, server, sock, addr): asynchat.async_chat.__init__ (self, sock) self.server = server self.addr = addr self.set_terminator ('\r\n') self.data = '' # local bindings specific to this channel self.local_env = sys.modules['__main__'].__dict__.copy() self.push ('Python ' + sys.version + '\r\n') self.push (sys.copyright+'\r\n') self.push ('Welcome to %s\r\n' % self) self.push ("[Hint: try 'from __main__ import *']\r\n") self.prompt() self.number = server.total_sessions.as_long() self.line_counter = counter() self.multi_line = [] def handle_connect (self): # send IAC DO LINEMODE self.push ('\377\375\"') def close (self): self.server.closed_sessions.increment() asynchat.async_chat.close(self) def prompt (self): self.push ('>>> ') def collect_incoming_data (self, data): self.data = self.data + data if len(self.data) > 1024: # denial of service. self.push ('BCNU\r\n') self.close_when_done() def found_terminator (self): line = self.clean_line (self.data) self.data = '' self.line_counter.increment() # check for special case inputs... if not line and not self.multi_line: self.prompt() return if line in ['\004', 'exit']: self.push ('BCNU\r\n') self.close_when_done() return oldout = sys.stdout olderr = sys.stderr try: p = output_producer(self, olderr) sys.stdout = p sys.stderr = p try: # this is, of course, a blocking operation. # if you wanted to thread this, you would have # to synchronize, etc... and treat the output # like a pipe. Not Fun. # # try eval first. If that fails, try exec. If that fails, # hurl. try: if self.multi_line: # oh, this is horrible... raise SyntaxError co = compile (line, repr(self), 'eval') result = eval (co, self.local_env) method = 'eval' if result is not None: print repr(result) self.local_env['_'] = result except SyntaxError: try: if self.multi_line: if line and line[0] in [' ','\t']: self.multi_line.append (line) self.push ('... ') return else: self.multi_line.append (line) line = string.join (self.multi_line, '\n') co = compile (line, repr(self), 'exec') self.multi_line = [] else: co = compile (line, repr(self), 'exec') except SyntaxError, why: if why[0] == 'unexpected EOF while parsing': self.push ('... ') self.multi_line.append (line) return else: t,v,tb = sys.exc_info() del tb raise t,v exec co in self.local_env method = 'exec' except: method = 'exception' self.multi_line = [] (file, fun, line), t, v, tbinfo = asyncore.compact_traceback() self.log_info('%s %s %s' %(t, v, tbinfo), 'warning') finally: sys.stdout = oldout sys.stderr = olderr self.log_info('%s:%s (%s)> %s' % ( self.number, self.line_counter, method, repr(line)) ) self.push_with_producer (p) self.prompt() # for now, we ignore any telnet option stuff sent to # us, and we process the backspace key ourselves. # gee, it would be fun to write a full-blown line-editing # environment, etc... def clean_line (self, line): chars = [] for ch in line: oc = ord(ch) if oc < 127: if oc in [8,177]: # backspace chars = chars[:-1] else: chars.append (ch) return string.join (chars, '') class monitor_server (asyncore.dispatcher): SERVER_IDENT = 'Monitor Server (V%s)' % VERSION channel_class = monitor_channel def __init__ (self, hostname='127.0.0.1', port=8023): self.hostname = hostname self.port = port self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind ((hostname, port)) self.log_info('%s started on port %d' % (self.SERVER_IDENT, port)) self.listen (5) self.closed = 0 self.failed_auths = 0 self.total_sessions = counter() self.closed_sessions = counter() def writable (self): return 0 def handle_accept (self): conn, addr = self.accept() self.log_info('Incoming monitor connection from %s:%d' % addr) self.channel_class (self, conn, addr) self.total_sessions.increment() def status (self): return producers.simple_producer ( '

    %s

    ' % self.SERVER_IDENT + '
    Total Sessions: %s' % self.total_sessions + '
    Current Sessions: %d' % ( self.total_sessions.as_long()-self.closed_sessions.as_long() ) ) def hex_digest (s): m = md5.md5() m.update (s) return string.joinfields ( map (lambda x: hex (ord (x))[2:], map (None, m.digest())), '', ) class secure_monitor_channel (monitor_channel): authorized = 0 def __init__ (self, server, sock, addr): asynchat.async_chat.__init__ (self, sock) self.server = server self.addr = addr self.set_terminator ('\r\n') self.data = '' # local bindings specific to this channel self.local_env = {} # send timestamp string self.timestamp = str(time.time()) self.count = 0 self.line_counter = counter() self.number = int(server.total_sessions.as_long()) self.multi_line = [] self.push (self.timestamp + '\r\n') def found_terminator (self): if not self.authorized: if hex_digest ('%s%s' % (self.timestamp, self.server.password)) != self.data: self.log_info ('%s: failed authorization' % self, 'warning') self.server.failed_auths = self.server.failed_auths + 1 self.close() else: self.authorized = 1 self.push ('Python ' + sys.version + '\r\n') self.push (sys.copyright+'\r\n') self.push ('Welcome to %s\r\n' % self) self.prompt() self.data = '' else: monitor_channel.found_terminator (self) class secure_encrypted_monitor_channel (secure_monitor_channel): "Wrap send() and recv() with a stream cipher" def __init__ (self, server, conn, addr): key = server.password self.outgoing = server.cipher.new (key) self.incoming = server.cipher.new (key) secure_monitor_channel.__init__ (self, server, conn, addr) def send (self, data): # send the encrypted data instead ed = self.outgoing.encrypt (data) return secure_monitor_channel.send (self, ed) def recv (self, block_size): data = secure_monitor_channel.recv (self, block_size) if data: dd = self.incoming.decrypt (data) return dd else: return data class secure_monitor_server (monitor_server): channel_class = secure_monitor_channel def __init__ (self, password, hostname='', port=8023): monitor_server.__init__ (self, hostname, port) self.password = password def status (self): p = monitor_server.status (self) # kludge p.data = p.data + ('
    Failed Authorizations: %d' % self.failed_auths) return p # don't try to print from within any of the methods # of this object. 8^) class output_producer: def __init__ (self, channel, real_stderr): self.channel = channel self.data = '' # use _this_ for debug output self.stderr = real_stderr def check_data (self): if len(self.data) > 1<<16: # runaway output, close it. self.channel.close() def write (self, data): lines = string.splitfields (data, '\n') data = string.join (lines, '\r\n') self.data = self.data + data self.check_data() def writeline (self, line): self.data = self.data + line + '\r\n' self.check_data() def writelines (self, lines): self.data = self.data + string.joinfields ( lines, '\r\n' ) + '\r\n' self.check_data() def flush (self): pass def softspace (self, *args): pass def more (self): if self.data: result = self.data[:512] self.data = self.data[512:] return result else: return '' if __name__ == '__main__': if '-s' in sys.argv: sys.argv.remove ('-s') print 'Enter password: ', password = raw_input() else: password = None if '-e' in sys.argv: sys.argv.remove ('-e') encrypt = 1 else: encrypt = 0 if len(sys.argv) > 1: port = string.atoi (sys.argv[1]) else: port = 8023 if password is not None: s = secure_monitor_server (password, '', port) if encrypt: s.channel_class = secure_encrypted_monitor_channel import sapphire s.cipher = sapphire else: s = monitor_server ('', port) asyncore.loop(use_poll=1) medusa-0.5.4/m_syslog.py0100644000076400007640000001626707577763503013356 0ustar amkamk# -*- Mode: Python -*- # ====================================================================== # Copyright 1997 by Sam Rushing # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of Sam # Rushing not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # ====================================================================== """socket interface to unix syslog. On Unix, there are usually two ways of getting to syslog: via a local unix-domain socket, or via the TCP service. Usually "/dev/log" is the unix domain socket. This may be different for other systems. >>> my_client = syslog_client ('/dev/log') Otherwise, just use the UDP version, port 514. >>> my_client = syslog_client (('my_log_host', 514)) On win32, you will have to use the UDP version. Note that you can use this to log to other hosts (and indeed, multiple hosts). This module is not a drop-in replacement for the python extension module - the interface is different. Usage: >>> c = syslog_client() >>> c = syslog_client ('/strange/non_standard_log_location') >>> c = syslog_client (('other_host.com', 514)) >>> c.log ('testing', facility='local0', priority='debug') """ # TODO: support named-pipe syslog. # [see ftp://sunsite.unc.edu/pub/Linux/system/Daemons/syslog-fifo.tar.z] # from : # =========================================================================== # priorities/facilities are encoded into a single 32-bit quantity, where the # bottom 3 bits are the priority (0-7) and the top 28 bits are the facility # (0-big number). Both the priorities and the facilities map roughly # one-to-one to strings in the syslogd(8) source code. This mapping is # included in this file. # # priorities (these are ordered) LOG_EMERG = 0 # system is unusable LOG_ALERT = 1 # action must be taken immediately LOG_CRIT = 2 # critical conditions LOG_ERR = 3 # error conditions LOG_WARNING = 4 # warning conditions LOG_NOTICE = 5 # normal but significant condition LOG_INFO = 6 # informational LOG_DEBUG = 7 # debug-level messages # facility codes LOG_KERN = 0 # kernel messages LOG_USER = 1 # random user-level messages LOG_MAIL = 2 # mail system LOG_DAEMON = 3 # system daemons LOG_AUTH = 4 # security/authorization messages LOG_SYSLOG = 5 # messages generated internally by syslogd LOG_LPR = 6 # line printer subsystem LOG_NEWS = 7 # network news subsystem LOG_UUCP = 8 # UUCP subsystem LOG_CRON = 9 # clock daemon LOG_AUTHPRIV = 10 # security/authorization messages (private) # other codes through 15 reserved for system use LOG_LOCAL0 = 16 # reserved for local use LOG_LOCAL1 = 17 # reserved for local use LOG_LOCAL2 = 18 # reserved for local use LOG_LOCAL3 = 19 # reserved for local use LOG_LOCAL4 = 20 # reserved for local use LOG_LOCAL5 = 21 # reserved for local use LOG_LOCAL6 = 22 # reserved for local use LOG_LOCAL7 = 23 # reserved for local use priority_names = { "alert": LOG_ALERT, "crit": LOG_CRIT, "debug": LOG_DEBUG, "emerg": LOG_EMERG, "err": LOG_ERR, "error": LOG_ERR, # DEPRECATED "info": LOG_INFO, "notice": LOG_NOTICE, "panic": LOG_EMERG, # DEPRECATED "warn": LOG_WARNING, # DEPRECATED "warning": LOG_WARNING, } facility_names = { "auth": LOG_AUTH, "authpriv": LOG_AUTHPRIV, "cron": LOG_CRON, "daemon": LOG_DAEMON, "kern": LOG_KERN, "lpr": LOG_LPR, "mail": LOG_MAIL, "news": LOG_NEWS, "security": LOG_AUTH, # DEPRECATED "syslog": LOG_SYSLOG, "user": LOG_USER, "uucp": LOG_UUCP, "local0": LOG_LOCAL0, "local1": LOG_LOCAL1, "local2": LOG_LOCAL2, "local3": LOG_LOCAL3, "local4": LOG_LOCAL4, "local5": LOG_LOCAL5, "local6": LOG_LOCAL6, "local7": LOG_LOCAL7, } import socket class syslog_client: def __init__ (self, address='/dev/log'): self.address = address self.stream = 0 if isinstance(address, type('')): try: self.socket = socket.socket(socket.AF_UNIX, socket.SOCK_DGRAM) self.socket.connect(address) except socket.error: # Some Linux installations have /dev/log # a stream socket instead of a datagram socket. self.socket = socket.socket (socket.AF_UNIX, socket.SOCK_STREAM) self.stream = 1 else: self.socket = socket.socket (socket.AF_INET, socket.SOCK_DGRAM) # curious: when talking to the unix-domain '/dev/log' socket, a # zero-terminator seems to be required. this string is placed # into a class variable so that it can be overridden if # necessary. log_format_string = '<%d>%s\000' def log (self, message, facility=LOG_USER, priority=LOG_INFO): message = self.log_format_string % ( self.encode_priority (facility, priority), message ) if self.stream: self.socket.send (message) else: self.socket.sendto (message, self.address) def encode_priority (self, facility, priority): if type(facility) == type(''): facility = facility_names[facility] if type(priority) == type(''): priority = priority_names[priority] return (facility<<3) | priority def close (self): if self.stream: self.socket.close() medusa-0.5.4/producers.py0100644000076400007640000002233607701142271013500 0ustar amkamk# -*- Mode: Python -*- RCS_ID = '$Id: producers.py,v 1.8 2003/04/03 19:51:23 akuchling Exp $' """ A collection of producers. Each producer implements a particular feature: They can be combined in various ways to get interesting and useful behaviors. For example, you can feed dynamically-produced output into the compressing producer, then wrap this with the 'chunked' transfer-encoding producer. """ import string from asynchat import find_prefix_at_end class simple_producer: "producer for a string" def __init__ (self, data, buffer_size=1024): self.data = data self.buffer_size = buffer_size def more (self): if len (self.data) > self.buffer_size: result = self.data[:self.buffer_size] self.data = self.data[self.buffer_size:] return result else: result = self.data self.data = '' return result class scanning_producer: "like simple_producer, but more efficient for large strings" def __init__ (self, data, buffer_size=1024): self.data = data self.buffer_size = buffer_size self.pos = 0 def more (self): if self.pos < len(self.data): lp = self.pos rp = min ( len(self.data), self.pos + self.buffer_size ) result = self.data[lp:rp] self.pos = self.pos + len(result) return result else: return '' class lines_producer: "producer for a list of lines" def __init__ (self, lines): self.lines = lines def more (self): if self.lines: chunk = self.lines[:50] self.lines = self.lines[50:] return string.join (chunk, '\r\n') + '\r\n' else: return '' class buffer_list_producer: "producer for a list of strings" # i.e., data == string.join (buffers, '') def __init__ (self, buffers): self.index = 0 self.buffers = buffers def more (self): if self.index >= len(self.buffers): return '' else: data = self.buffers[self.index] self.index = self.index + 1 return data class file_producer: "producer wrapper for file[-like] objects" # match http_channel's outgoing buffer size out_buffer_size = 1<<16 def __init__ (self, file): self.done = 0 self.file = file def more (self): if self.done: return '' else: data = self.file.read (self.out_buffer_size) if not data: self.file.close() del self.file self.done = 1 return '' else: return data # A simple output producer. This one does not [yet] have # the safety feature builtin to the monitor channel: runaway # output will not be caught. # don't try to print from within any of the methods # of this object. class output_producer: "Acts like an output file; suitable for capturing sys.stdout" def __init__ (self): self.data = '' def write (self, data): lines = string.splitfields (data, '\n') data = string.join (lines, '\r\n') self.data = self.data + data def writeline (self, line): self.data = self.data + line + '\r\n' def writelines (self, lines): self.data = self.data + string.joinfields ( lines, '\r\n' ) + '\r\n' def flush (self): pass def softspace (self, *args): pass def more (self): if self.data: result = self.data[:512] self.data = self.data[512:] return result else: return '' class composite_producer: "combine a fifo of producers into one" def __init__ (self, producers): self.producers = producers def more (self): while len(self.producers): p = self.producers[0] d = p.more() if d: return d else: self.producers.pop(0) else: return '' class globbing_producer: """ 'glob' the output from a producer into a particular buffer size. helps reduce the number of calls to send(). [this appears to gain about 30% performance on requests to a single channel] """ def __init__ (self, producer, buffer_size=1<<16): self.producer = producer self.buffer = '' self.buffer_size = buffer_size def more (self): while len(self.buffer) < self.buffer_size: data = self.producer.more() if data: self.buffer = self.buffer + data else: break r = self.buffer self.buffer = '' return r class hooked_producer: """ A producer that will call when it empties,. with an argument of the number of bytes produced. Useful for logging/instrumentation purposes. """ def __init__ (self, producer, function): self.producer = producer self.function = function self.bytes = 0 def more (self): if self.producer: result = self.producer.more() if not result: self.producer = None self.function (self.bytes) else: self.bytes = self.bytes + len(result) return result else: return '' # HTTP 1.1 emphasizes that an advertised Content-Length header MUST be # correct. In the face of Strange Files, it is conceivable that # reading a 'file' may produce an amount of data not matching that # reported by os.stat() [text/binary mode issues, perhaps the file is # being appended to, etc..] This makes the chunked encoding a True # Blessing, and it really ought to be used even with normal files. # How beautifully it blends with the concept of the producer. class chunked_producer: """A producer that implements the 'chunked' transfer coding for HTTP/1.1. Here is a sample usage: request['Transfer-Encoding'] = 'chunked' request.push ( producers.chunked_producer (your_producer) ) request.done() """ def __init__ (self, producer, footers=None): self.producer = producer self.footers = footers def more (self): if self.producer: data = self.producer.more() if data: return '%x\r\n%s\r\n' % (len(data), data) else: self.producer = None if self.footers: return string.join ( ['0'] + self.footers, '\r\n' ) + '\r\n\r\n' else: return '0\r\n\r\n' else: return '' # Unfortunately this isn't very useful right now (Aug 97), because # apparently the browsers don't do on-the-fly decompression. Which # is sad, because this could _really_ speed things up, especially for # low-bandwidth clients (i.e., most everyone). try: import zlib except ImportError: zlib = None class compressed_producer: """ Compress another producer on-the-fly, using ZLIB [Unfortunately, none of the current browsers seem to support this] """ # Note: It's not very efficient to have the server repeatedly # compressing your outgoing files: compress them ahead of time, or # use a compress-once-and-store scheme. However, if you have low # bandwidth and low traffic, this may make more sense than # maintaining your source files compressed. # # Can also be used for compressing dynamically-produced output. def __init__ (self, producer, level=5): self.producer = producer self.compressor = zlib.compressobj (level) def more (self): if self.producer: cdata = '' # feed until we get some output while not cdata: data = self.producer.more() if not data: self.producer = None return self.compressor.flush() else: cdata = self.compressor.compress (data) return cdata else: return '' class escaping_producer: "A producer that escapes a sequence of characters" " Common usage: escaping the CRLF.CRLF sequence in SMTP, NNTP, etc..." def __init__ (self, producer, esc_from='\r\n.', esc_to='\r\n..'): self.producer = producer self.esc_from = esc_from self.esc_to = esc_to self.buffer = '' self.find_prefix_at_end = find_prefix_at_end def more (self): esc_from = self.esc_from esc_to = self.esc_to buffer = self.buffer + self.producer.more() if buffer: buffer = string.replace (buffer, esc_from, esc_to) i = self.find_prefix_at_end (buffer, esc_from) if i: # we found a prefix self.buffer = buffer[-i:] return buffer[:-i] else: # no prefix, return it all self.buffer = '' return buffer else: return buffer medusa-0.5.4/put_handler.py0100644000076400007640000000637307543176422014013 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1996-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: put_handler.py,v 1.4 2002/08/01 18:15:45 akuchling Exp $' import re import string import default_handler unquote = default_handler.unquote get_header = default_handler.get_header last_request = None class put_handler: def __init__ (self, filesystem, uri_regex): self.filesystem = filesystem if type (uri_regex) == type(''): self.uri_regex = re.compile (uri_regex) else: self.uri_regex = uri_regex def match (self, request): uri = request.uri if request.command == 'PUT': m = self.uri_regex.match (uri) if m and m.end() == len(uri): return 1 return 0 def handle_request (self, request): path, params, query, fragment = request.split_uri() # strip off leading slashes while path and path[0] == '/': path = path[1:] if '%' in path: path = unquote (path) # make sure there's a content-length header cl = get_header (CONTENT_LENGTH, request.header) if not cl: request.error (411) return else: cl = string.atoi (cl) # don't let the try to overwrite a directory if self.filesystem.isdir (path): request.error (405) return is_update = self.filesystem.isfile (path) try: output_file = self.filesystem.open (path, 'wb') except: request.error (405) return request.collector = put_collector (output_file, cl, request, is_update) # no terminator while receiving PUT data request.channel.set_terminator (None) # don't respond yet, wait until we've received the data... class put_collector: def __init__ (self, file, length, request, is_update): self.file = file self.length = length self.request = request self.is_update = is_update self.bytes_in = 0 def collect_incoming_data (self, data): ld = len(data) bi = self.bytes_in if (bi + ld) >= self.length: # last bit of data chunk = self.length - bi self.file.write (data[:chunk]) self.file.close() if chunk != ld: print 'orphaned %d bytes: <%s>' % (ld - chunk, repr(data[chunk:])) # do some housekeeping r = self.request ch = r.channel ch.current_request = None # set the terminator back to the default ch.set_terminator ('\r\n\r\n') if self.is_update: r.reply_code = 204 # No content r.done() else: r.reply_now (201) # Created # avoid circular reference del self.request else: self.file.write (data) self.bytes_in = self.bytes_in + ld def found_terminator (self): # shouldn't be called pass CONTENT_LENGTH = re.compile ('Content-Length: ([0-9]+)', re.IGNORECASE) medusa-0.5.4/README.txt0100644000076400007640000000331407455116577012631 0ustar amkamkMedusa is a 'server platform' -- it provides a framework for implementing asynchronous socket-based servers (TCP/IP and on Unix, Unix domain, sockets). An asynchronous socket server is a server that can communicate with many other clients simultaneously by multiplexing I/O within a single process/thread. In the context of an HTTP server, this means a single process can serve hundreds or even thousands of clients, depending only on the operating system's configuration and limitations. There are several advantages to this approach: o performance - no fork() or thread() start-up costs per hit. o scalability - the overhead per client can be kept rather small, on the order of several kilobytes of memory. o persistence - a single-process server can easily coordinate the actions of several different connections. This makes things like proxy servers and gateways easy to implement. It also makes it possible to share resources like database handles. Medusa includes HTTP, FTP, and 'monitor' (remote python interpreter) servers. Medusa can simultaneously support several instances of either the same or different server types - for example you could start up two HTTP servers, an FTP server, and a monitor server. Then you could connect to the monitor server to control and manipulate medusa while it is running. Other servers and clients have been written (SMTP, POP3, NNTP), and several are in the planning stages. Medusa was originally written by Sam Rushing , and its original Web page is at . After Sam moved on to other things, A.M. Kuchling took over maintenance of the Medusa package. --amk medusa-0.5.4/redirecting_handler.py0100644000076400007640000000256407446144354015501 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1996-2000 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: redirecting_handler.py,v 1.4 2002/03/20 17:37:48 amk Exp $' import re import counter class redirecting_handler: def __init__ (self, pattern, redirect, regex_flag=re.IGNORECASE): self.pattern = pattern self.redirect = redirect self.patreg = re.compile (pattern, regex_flag) self.hits = counter.counter() def match (self, request): m = self.patreg.match (request.uri) return (m and (m.end() == len(request.uri))) def handle_request (self, request): self.hits.increment() m = self.patreg.match (request.uri) part = m.group(1) request['Location'] = self.redirect % part request.error (302) # moved temporarily def __repr__ (self): return ' %s]>' % ( id(self), repr(self.pattern), repr(self.redirect) ) def status (self): import producers return producers.simple_producer ( '
  • Redirecting Handler %s => %s Hits: %s' % ( self.pattern, self.redirect, self.hits ) ) medusa-0.5.4/resolver.py0100644000076400007640000003630007446144354013341 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # RCS_ID = '$Id: resolver.py,v 1.4 2002/03/20 17:37:48 amk Exp $' # Fast, low-overhead asynchronous name resolver. uses 'pre-cooked' # DNS requests, unpacks only as much as it needs of the reply. # see rfc1035 for details import string import asyncore import socket import sys import time from counter import counter VERSION = string.split(RCS_ID)[2] # header # 1 1 1 1 1 1 # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | ID | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # |QR| Opcode |AA|TC|RD|RA| Z | RCODE | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | QDCOUNT | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | ANCOUNT | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | NSCOUNT | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | ARCOUNT | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # question # 1 1 1 1 1 1 # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | | # / QNAME / # / / # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | QTYPE | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | QCLASS | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # build a DNS address request, _quickly_ def fast_address_request (host, id=0): return ( '%c%c' % (chr((id>>8)&0xff),chr(id&0xff)) + '\001\000\000\001\000\000\000\000\000\000%s\000\000\001\000\001' % ( string.join ( map ( lambda part: '%c%s' % (chr(len(part)),part), string.split (host, '.') ), '' ) ) ) def fast_ptr_request (host, id=0): return ( '%c%c' % (chr((id>>8)&0xff),chr(id&0xff)) + '\001\000\000\001\000\000\000\000\000\000%s\000\000\014\000\001' % ( string.join ( map ( lambda part: '%c%s' % (chr(len(part)),part), string.split (host, '.') ), '' ) ) ) def unpack_name (r,pos): n = [] while 1: ll = ord(r[pos]) if (ll&0xc0): # compression pos = (ll&0x3f << 8) + (ord(r[pos+1])) elif ll == 0: break else: pos = pos + 1 n.append (r[pos:pos+ll]) pos = pos + ll return string.join (n,'.') def skip_name (r,pos): s = pos while 1: ll = ord(r[pos]) if (ll&0xc0): # compression return pos + 2 elif ll == 0: pos = pos + 1 break else: pos = pos + ll + 1 return pos def unpack_ttl (r,pos): return reduce ( lambda x,y: (x<<8)|y, map (ord, r[pos:pos+4]) ) # resource record # 1 1 1 1 1 1 # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | | # / / # / NAME / # | | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | TYPE | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | CLASS | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | TTL | # | | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ # | RDLENGTH | # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--| # / RDATA / # / / # +--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+--+ def unpack_address_reply (r): ancount = (ord(r[6])<<8) + (ord(r[7])) # skip question, first name starts at 12, # this is followed by QTYPE and QCLASS pos = skip_name (r, 12) + 4 if ancount: # we are looking very specifically for # an answer with TYPE=A, CLASS=IN (\000\001\000\001) for an in range(ancount): pos = skip_name (r, pos) if r[pos:pos+4] == '\000\001\000\001': return ( unpack_ttl (r,pos+4), '%d.%d.%d.%d' % tuple(map(ord,r[pos+10:pos+14])) ) # skip over TYPE, CLASS, TTL, RDLENGTH, RDATA pos = pos + 8 rdlength = (ord(r[pos])<<8) + (ord(r[pos+1])) pos = pos + 2 + rdlength return 0, None else: return 0, None def unpack_ptr_reply (r): ancount = (ord(r[6])<<8) + (ord(r[7])) # skip question, first name starts at 12, # this is followed by QTYPE and QCLASS pos = skip_name (r, 12) + 4 if ancount: # we are looking very specifically for # an answer with TYPE=PTR, CLASS=IN (\000\014\000\001) for an in range(ancount): pos = skip_name (r, pos) if r[pos:pos+4] == '\000\014\000\001': return ( unpack_ttl (r,pos+4), unpack_name (r, pos+10) ) # skip over TYPE, CLASS, TTL, RDLENGTH, RDATA pos = pos + 8 rdlength = (ord(r[pos])<<8) + (ord(r[pos+1])) pos = pos + 2 + rdlength return 0, None else: return 0, None # This is a UDP (datagram) resolver. # # It may be useful to implement a TCP resolver. This would presumably # give us more reliable behavior when things get too busy. A TCP # client would have to manage the connection carefully, since the # server is allowed to close it at will (the RFC recommends closing # after 2 minutes of idle time). # # Note also that the TCP client will have to prepend each request # with a 2-byte length indicator (see rfc1035). # class resolver (asyncore.dispatcher): id = counter() def __init__ (self, server='127.0.0.1'): asyncore.dispatcher.__init__ (self) self.create_socket (socket.AF_INET, socket.SOCK_DGRAM) self.server = server self.request_map = {} self.last_reap_time = int(time.time()) # reap every few minutes def writable (self): return 0 def log (self, *args): pass def handle_close (self): self.log_info('closing!') self.close() def handle_error (self): # don't close the connection on error (file,fun,line), t, v, tbinfo = asyncore.compact_traceback() self.log_info( 'Problem with DNS lookup (%s:%s %s)' % (t, v, tbinfo), 'error') def get_id (self): return (self.id.as_long() % (1<<16)) def reap (self): # find DNS requests that have timed out now = int(time.time()) if now - self.last_reap_time > 180: # reap every 3 minutes self.last_reap_time = now # update before we forget for k,(host,unpack,callback,when) in self.request_map.items(): if now - when > 180: # over 3 minutes old del self.request_map[k] try: # same code as in handle_read callback (host, 0, None) # timeout val is (0,None) except: (file,fun,line), t, v, tbinfo = asyncore.compact_traceback() self.log_info('%s %s %s' % (t,v,tbinfo), 'error') def resolve (self, host, callback): self.reap() # first, get rid of old guys self.socket.sendto ( fast_address_request (host, self.get_id()), (self.server, 53) ) self.request_map [self.get_id()] = ( host, unpack_address_reply, callback, int(time.time())) self.id.increment() def resolve_ptr (self, host, callback): self.reap() # first, get rid of old guys ip = string.split (host, '.') ip.reverse() ip = string.join (ip, '.') + '.in-addr.arpa' self.socket.sendto ( fast_ptr_request (ip, self.get_id()), (self.server, 53) ) self.request_map [self.get_id()] = ( host, unpack_ptr_reply, callback, int(time.time())) self.id.increment() def handle_read (self): reply, whence = self.socket.recvfrom (512) # for security reasons we may want to double-check # that is the server we sent the request to. id = (ord(reply[0])<<8) + ord(reply[1]) if self.request_map.has_key (id): host, unpack, callback, when = self.request_map[id] del self.request_map[id] ttl, answer = unpack (reply) try: callback (host, ttl, answer) except: (file,fun,line), t, v, tbinfo = asyncore.compact_traceback() self.log_info('%s %s %s' % ( t,v,tbinfo), 'error') class rbl (resolver): def resolve_maps (self, host, callback): ip = string.split (host, '.') ip.reverse() ip = string.join (ip, '.') + '.rbl.maps.vix.com' self.socket.sendto ( fast_ptr_request (ip, self.get_id()), (self.server, 53) ) self.request_map [self.get_id()] = host, self.check_reply, callback self.id.increment() def check_reply (self, r): # we only need to check RCODE. rcode = (ord(r[3])&0xf) self.log_info('MAPS RBL; RCODE =%02x\n %s' % (rcode, repr(r))) return 0, rcode # (ttl, answer) class hooked_callback: def __init__ (self, hook, callback): self.hook, self.callback = hook, callback def __call__ (self, *args): apply (self.hook, args) apply (self.callback, args) class caching_resolver (resolver): "Cache DNS queries. Will need to honor the TTL value in the replies" def __init__ (*args): apply (resolver.__init__, args) self = args[0] self.cache = {} self.forward_requests = counter() self.reverse_requests = counter() self.cache_hits = counter() def resolve (self, host, callback): self.forward_requests.increment() if self.cache.has_key (host): when, ttl, answer = self.cache[host] # ignore TTL for now callback (host, ttl, answer) self.cache_hits.increment() else: resolver.resolve ( self, host, hooked_callback ( self.callback_hook, callback ) ) def resolve_ptr (self, host, callback): self.reverse_requests.increment() if self.cache.has_key (host): when, ttl, answer = self.cache[host] # ignore TTL for now callback (host, ttl, answer) self.cache_hits.increment() else: resolver.resolve_ptr ( self, host, hooked_callback ( self.callback_hook, callback ) ) def callback_hook (self, host, ttl, answer): self.cache[host] = time.time(), ttl, answer SERVER_IDENT = 'Caching DNS Resolver (V%s)' % VERSION def status (self): import producers return producers.simple_producer ( '

    %s

    ' % self.SERVER_IDENT + '
    Server: %s' % self.server + '
    Cache Entries: %d' % len(self.cache) + '
    Outstanding Requests: %d' % len(self.request_map) + '
    Forward Requests: %s' % self.forward_requests + '
    Reverse Requests: %s' % self.reverse_requests + '
    Cache Hits: %s' % self.cache_hits ) #test_reply = """\000\000\205\200\000\001\000\001\000\002\000\002\006squirl\011nightmare\003com\000\000\001\000\001\300\014\000\001\000\001\000\001Q\200\000\004\315\240\260\005\011nightmare\003com\000\000\002\000\001\000\001Q\200\000\002\300\014\3006\000\002\000\001\000\001Q\200\000\015\003ns1\003iag\003net\000\300\014\000\001\000\001\000\001Q\200\000\004\315\240\260\005\300]\000\001\000\001\000\000\350\227\000\004\314\033\322\005""" # def test_unpacker (): # print unpack_address_reply (test_reply) # # import time # class timer: # def __init__ (self): # self.start = time.time() # def end (self): # return time.time() - self.start # # # I get ~290 unpacks per second for the typical case, compared to ~48 # # using dnslib directly. also, that latter number does not include # # picking the actual data out. # # def benchmark_unpacker(): # # r = range(1000) # t = timer() # for i in r: # unpack_address_reply (test_reply) # print '%.2f unpacks per second' % (1000.0 / t.end()) if __name__ == '__main__': if len(sys.argv) == 1: print 'usage: %s [-r] [-s ] host [host ...]' % sys.argv[0] sys.exit(0) elif ('-s' in sys.argv): i = sys.argv.index('-s') server = sys.argv[i+1] del sys.argv[i:i+2] else: server = '127.0.0.1' if ('-r' in sys.argv): reverse = 1 i = sys.argv.index('-r') del sys.argv[i] else: reverse = 0 if ('-m' in sys.argv): maps = 1 sys.argv.remove ('-m') else: maps = 0 if maps: r = rbl (server) else: r = caching_resolver(server) count = len(sys.argv) - 1 def print_it (host, ttl, answer): global count print '%s: %s' % (host, answer) count = count - 1 if not count: r.close() for host in sys.argv[1:]: if reverse: r.resolve_ptr (host, print_it) elif maps: r.resolve_maps (host, print_it) else: r.resolve (host, print_it) # hooked asyncore.loop() while asyncore.socket_map: asyncore.poll (30.0) print 'requests outstanding: %d' % len(r.request_map) medusa-0.5.4/rpc_client.py0100644000076400007640000002271307570555565013635 0ustar amkamk# -*- Mode: Python -*- # Copyright 1999, 2000 by eGroups, Inc. # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # eGroups not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # EGROUPS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT SHALL EGROUPS BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. import marshal import socket import string import exceptions import string import sys # # there are three clients in here. # # 1) rpc client # 2) fastrpc client # 3) async fastrpc client # # we hope that *whichever* choice you make, that you will enjoy the # excellent hand-made construction, and return to do business with us # again in the near future. # class RPC_Error (exceptions.StandardError): pass # =========================================================================== # RPC Client # =========================================================================== # request types: # 0 call # 1 getattr # 2 setattr # 3 repr # 4 del class rpc_proxy: DEBUG = 0 def __init__ (self, conn, oid): # route around __setattr__ self.__dict__['conn'] = conn self.__dict__['oid'] = oid # Warning: be VERY CAREFUL with attribute references, keep # this __getattr__ in mind! def __getattr__ (self, attr): # __getattr__ and __call__ if attr == '__call__': # 0 == __call__ return self.__remote_call__ elif attr == '__repr__': # 3 == __repr__ return self.__remote_repr__ elif attr == '__getitem__': return self.__remote_getitem__ elif attr == '__setitem__': return self.__remote_setitem__ elif attr == '__len__': return self.__remote_len__ else: # 1 == __getattr__ return self.__send_request__ (1, attr) def __setattr__ (self, attr, value): return self.__send_request__ (2, (attr, value)) def __del__ (self): try: self.__send_request__ (4, None) except: import who_calls info = who_calls.compact_traceback() print info def __remote_repr__ (self): r = self.__send_request__ (3, None) return '' % r[1:-1] def __remote_call__ (self, *args): return self.__send_request__ (0, args) def __remote_getitem__ (self, key): return self.__send_request__ (5, key) def __remote_setitem__ (self, key, value): return self.__send_request__ (6, (key, value)) def __remote_len__ (self): return self.__send_request__ (7, None) _request_types_ = ['call', 'getattr', 'setattr', 'repr', 'del', 'getitem', 'setitem', 'len'] def __send_request__ (self, *args): if self.DEBUG: kind = args[0] print ( 'RPC: ==> %s:%08x:%s:%s' % ( self.conn.address, self.oid, self._request_types_[kind], repr(args[1:]) ) ) packet = marshal.dumps ((self.oid,)+args) # send request self.conn.send_packet (packet) # get response data = self.conn.receive_packet() # types of response: # 0: proxy # 1: error # 2: marshal'd data kind, value = marshal.loads (data) if kind == 0: # proxy (value == oid) if self.DEBUG: print 'RPC: <== proxy(%08x)' % (value) return rpc_proxy (self.conn, value) elif kind == 1: raise RPC_Error, value else: if self.DEBUG: print 'RPC: <== %s' % (repr(value)) return value class rpc_connection: cache = {} def __init__ (self, address): self.address = address self.connect () def connect (self): s = socket.socket (socket.AF_INET, socket.SOCK_STREAM) s.connect (self.address) self.socket = s def receive_packet (self): packet_len = string.atoi (self.socket.recv (8), 16) packet = [] while packet_len: data = self.socket.recv (8192) packet.append (data) packet_len = packet_len - len(data) return string.join (packet, '') def send_packet (self, packet): self.socket.send ('%08x%s' % (len(packet), packet)) def rpc_connect (address = ('localhost', 8746)): if not rpc_connection.cache.has_key (address): conn = rpc_connection (address) # get oid of remote object data = conn.receive_packet() (oid,) = marshal.loads (data) rpc_connection.cache[address] = rpc_proxy (conn, oid) return rpc_connection.cache[address] # =========================================================================== # fastrpc client # =========================================================================== class fastrpc_proxy: def __init__ (self, conn, path=()): self.conn = conn self.path = path def __getattr__ (self, attr): if attr == '__call__': return self.__method_caller__ else: return fastrpc_proxy (self.conn, self.path + (attr,)) def __method_caller__ (self, *args): # send request packet = marshal.dumps ((self.path, args)) self.conn.send_packet (packet) # get response data = self.conn.receive_packet() error, result = marshal.loads (data) if error is None: return result else: raise RPC_Error, error def __repr__ (self): return '' % (string.join (self.path, '.'), id (self)) def fastrpc_connect (address = ('localhost', 8748)): if not rpc_connection.cache.has_key (address): conn = rpc_connection (address) rpc_connection.cache[address] = fastrpc_proxy (conn) return rpc_connection.cache[address] # =========================================================================== # async fastrpc client # =========================================================================== import asynchat class async_fastrpc_client (asynchat.async_chat): STATE_LENGTH = 'length state' STATE_PACKET = 'packet state' def __init__ (self, address=('idb', 3001)): asynchat.async_chat.__init__ (self) if type(address) is type(''): family = socket.AF_UNIX else: family = socket.AF_INET self.create_socket (family, socket.SOCK_STREAM) self.address = address self.request_fifo = [] self.buffer = [] self.pstate = self.STATE_LENGTH self.set_terminator (8) self._connected = 0 self.connect (self.address) def log (self, *args): pass def handle_connect (self): self._connected = 1 def close (self): self._connected = 0 self.flush_pending_requests ('lost connection to rpc server') asynchat.async_chat.close(self) def flush_pending_requests (self, why): f = self.request_fifo while len(f): callback = f.pop(0) callback (why, None) def collect_incoming_data (self, data): self.buffer.append (data) def found_terminator (self): self.buffer, data = [], string.join (self.buffer, '') if self.pstate is self.STATE_LENGTH: packet_length = string.atoi (data, 16) self.set_terminator (packet_length) self.pstate = self.STATE_PACKET else: # modified to fix socket leak in chat server, 2000-01-27, schiller@eGroups.net #self.set_terminator (8) #self.pstate = self.STATE_LENGTH error, result = marshal.loads (data) callback = self.request_fifo.pop(0) callback (error, result) self.close() # for chat server def call_method (self, method, args, callback): if not self._connected: # might be a unix socket... family, type = self.family_and_type self.create_socket (family, type) self.connect (self.address) # push the request out the socket path = string.split (method, '.') packet = marshal.dumps ((path, args)) self.push ('%08x%s' % (len(packet), packet)) self.request_fifo.append(callback) if __name__ == '__main__': if '-f' in sys.argv: connect = fastrpc_connect else: connect = rpc_connect print 'connecting...' c = connect() print 'calling .calc.sum (1,2,3)' print c.calc.sum (1,2,3) print 'calling .calc.nonexistent(), expect an exception!' print c.calc.nonexistent() medusa-0.5.4/rpc_server.py0100644000076400007640000002334107446144354013653 0ustar amkamk# -*- Mode: Python -*- # Copyright 1999, 2000 by eGroups, Inc. # # All Rights Reserved # # Permission to use, copy, modify, and distribute this software and # its documentation for any purpose and without fee is hereby # granted, provided that the above copyright notice appear in all # copies and that both that copyright notice and this permission # notice appear in supporting documentation, and that the name of # eGroups not be used in advertising or publicity pertaining to # distribution of the software without specific, written prior # permission. # # EGROUPS DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, # INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN # NO EVENT SHALL EGROUPS BE LIABLE FOR ANY SPECIAL, INDIRECT OR # CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS # OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, # NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. # There are two RPC implementations here. # The first ('rpc') attempts to be as transparent as possible, and # passes along 'internal' methods like __getattr__, __getitem__, and # __del__. It is rather 'chatty', and may not be suitable for a # high-performance system. # The second ('fastrpc') is less flexible, but has much less overhead, # and is easier to use from an asynchronous client. import marshal import socket import string import sys import types import asyncore import asynchat from producers import scanning_producer from counter import counter MY_NAME = string.split (socket.gethostname(), '.')[0] # =========================================================================== # RPC server # =========================================================================== # marshal is good for low-level data structures. # but when passing an 'object' (any non-marshallable object) # we really want to pass a 'reference', which will act on # the other side as a proxy. How transparent can we make this? class rpc_channel (asynchat.async_chat): 'Simple RPC server.' # a 'packet': NNNNNNNNmmmmmmmmmmmmmmmm # (hex length in 8 bytes, followed by marshal'd packet data) # same protocol used in both directions. STATE_LENGTH = 'length state' STATE_PACKET = 'packet state' ac_out_buffer_size = 65536 request_counter = counter() exception_counter = counter() client_counter = counter() def __init__ (self, root, conn, addr): self.root = root self.addr = addr asynchat.async_chat.__init__ (self, conn) self.pstate = self.STATE_LENGTH self.set_terminator (8) self.buffer = [] self.proxies = {} rid = id(root) self.new_reference (root) p = marshal.dumps ((rid,)) # send root oid to the other side self.push ('%08x%s' % (len(p), p)) self.client_counter.increment() def new_reference (self, object): oid = id(object) ignore, refcnt = self.proxies.get (oid, (None, 0)) self.proxies[oid] = (object, refcnt + 1) def forget_reference (self, oid): object, refcnt = self.proxies.get (oid, (None, 0)) if refcnt > 1: self.proxies[oid] = (object, refcnt - 1) else: del self.proxies[oid] def log (self, *ignore): pass def collect_incoming_data (self, data): self.buffer.append (data) def found_terminator (self): self.buffer, data = [], string.join (self.buffer, '') if self.pstate is self.STATE_LENGTH: packet_length = string.atoi (data, 16) self.set_terminator (packet_length) self.pstate = self.STATE_PACKET else: self.set_terminator (8) self.pstate = self.STATE_LENGTH oid, kind, arg = marshal.loads (data) obj, refcnt = self.proxies[oid] e = None reply_kind = 2 try: if kind == 0: # __call__ result = apply (obj, arg) elif kind == 1: # __getattr__ result = getattr (obj, arg) elif kind == 2: # __setattr__ key, value = arg setattr (obj, key, value) result = None elif kind == 3: # __repr__ result = repr(obj) elif kind == 4: # __del__ self.forget_reference (oid) result = None elif kind == 5: # __getitem__ result = obj[arg] elif kind == 6: # __setitem__ (key, value) = arg obj[key] = value result = None elif kind == 7: # __len__ result = len(obj) except: reply_kind = 1 (file,fun,line), t, v, tbinfo = asyncore.compact_traceback() result = '%s:%s:%s:%s (%s:%s)' % (MY_NAME, file, fun, line, t, str(v)) self.log_info (result, 'error') self.exception_counter.increment() self.request_counter.increment() # optimize a common case if type(result) is types.InstanceType: can_marshal = 0 else: can_marshal = 1 try: rb = marshal.dumps ((reply_kind, result)) except ValueError: can_marshal = 0 if not can_marshal: # unmarshallable object, return a reference rid = id(result) self.new_reference (result) rb = marshal.dumps ((0, rid)) self.push_with_producer ( scanning_producer ( ('%08x' % len(rb)) + rb, buffer_size = 65536 ) ) class rpc_server_root: pass class rpc_server (asyncore.dispatcher): def __init__ (self, root, address = ('', 8746)): self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind (address) self.listen (128) self.root = root def handle_accept (self): conn, addr = self.accept() rpc_channel (self.root, conn, addr) # =========================================================================== # Fast RPC server # =========================================================================== # no proxies, request consists # of a 'chain' of getattrs terminated by a __call__. # Protocol: # .. ( , , ... ) # => ( , , ... ) # # # (, ) # path: tuple of strings # params: tuple of objects class fastrpc_channel (asynchat.async_chat): 'Simple RPC server' # a 'packet': NNNNNNNNmmmmmmmmmmmmmmmm # (hex length in 8 bytes, followed by marshal'd packet data) # same protocol used in both directions. # A request consists of (, ) # where is a list of strings (eqv to string.split ('a.b.c', '.')) STATE_LENGTH = 'length state' STATE_PACKET = 'packet state' def __init__ (self, root, conn, addr): self.root = root self.addr = addr asynchat.async_chat.__init__ (self, conn) self.pstate = self.STATE_LENGTH self.set_terminator (8) self.buffer = [] def log (*ignore): pass def collect_incoming_data (self, data): self.buffer.append (data) def found_terminator (self): self.buffer, data = [], string.join (self.buffer, '') if self.pstate is self.STATE_LENGTH: packet_length = string.atoi (data, 16) self.set_terminator (packet_length) self.pstate = self.STATE_PACKET else: self.set_terminator (8) self.pstate = self.STATE_LENGTH (path, params) = marshal.loads (data) o = self.root e = None try: for p in path: o = getattr (o, p) result = apply (o, params) except: e = repr (asyncore.compact_traceback()) result = None rb = marshal.dumps ((e,result)) self.push (('%08x' % len(rb)) + rb) class fastrpc_server (asyncore.dispatcher): def __init__ (self, root, address = ('', 8748)): self.create_socket (socket.AF_INET, socket.SOCK_STREAM) self.set_reuse_addr() self.bind (address) self.listen (128) self.root = root def handle_accept (self): conn, addr = self.accept() fastrpc_channel (self.root, conn, addr) # =========================================================================== if __name__ == '__main__': class thing: def __del__ (self): print 'a thing has gone away %08x' % id(self) class sample_calc: def product (self, *values): return reduce (lambda a,b: a*b, values, 1) def sum (self, *values): return reduce (lambda a,b: a+b, values, 0) def eval (self, string): return eval (string) def make_a_thing (self): return thing() if '-f' in sys.argv: server_class = fastrpc_server address = ('', 8748) else: server_class = rpc_server address = ('', 8746) root = rpc_server_root() root.calc = sample_calc() root.sys = sys rs = server_class (root, address) asyncore.loop() medusa-0.5.4/script_handler.py0100644000076400007640000001463207543176422014504 0ustar amkamk# -*- Mode: Python -*- # This is a simple python server-side script handler. # A note about performance: This is really only suited for 'fast' # scripts: The script should generate its output quickly, since the # whole web server will stall otherwise. This doesn't mean you have # to write 'fast code' or anything, it simply means that you shouldn't # call any long-running code, [like say something that opens up an # internet connection, or a database query that will hold up the # server]. If you need this sort of feature, you can support it using # the asynchronous I/O 'api' that the rest of medusa is built on. [or # you could probably use threads] # Put your script into your web docs directory (like a cgi-bin # script), make sure it has the correct extension [see the overridable # script_handler.extension member below]. # # There's lots of things that can be done to tweak the restricted # execution model. Also, of course you could just use 'execfile' # instead (this is now the default, see class variable # script_handler.restricted) import rexec import re import string import StringIO import sys import counter import default_handler import producers unquote = default_handler.unquote class script_handler: extension = 'mpy' restricted = 0 script_regex = re.compile ( r'.*/([^/]+\.%s)' % extension, re.IGNORECASE ) def __init__ (self, filesystem): self.filesystem = filesystem self.hits = counter.counter() self.exceptions = counter.counter() def match (self, request): [path, params, query, fragment] = request.split_uri() m = self.script_regex.match (path) return (m and (m.end() == len(path))) def handle_request (self, request): [path, params, query, fragment] = request.split_uri() while path and path[0] == '/': path = path[1:] if '%' in path: path = unquote (path) if not self.filesystem.isfile (path): request.error (404) return else: self.hits.increment() request.script_filename = self.filesystem.translate (path) if request.command in ('PUT', 'POST'): # look for a Content-Length header. cl = request.get_header ('content-length') length = int(cl) if not cl: request.error (411) else: collector (self, length, request) else: self.continue_request ( request, StringIO.StringIO() # empty stdin ) def continue_request (self, request, stdin): temp_files = stdin, StringIO.StringIO(), StringIO.StringIO() old_files = sys.stdin, sys.stdout, sys.stderr if self.restricted: r = rexec.RExec() try: sys.request = request sys.stdin, sys.stdout, sys.stderr = temp_files try: if self.restricted: r.s_execfile (request.script_filename) else: execfile (request.script_filename) request.reply_code = 200 except: request.reply_code = 500 self.exceptions.increment() finally: sys.stdin, sys.stdout, sys.stderr = old_files del sys.request i,o,e = temp_files if request.reply_code != 200: s = e.getvalue() else: s = o.getvalue() request['Content-Length'] = len(s) request.push (s) request.done() def status (self): return producers.simple_producer ( '
  • Server-Side Script Handler' + '
      ' + '
    • Hits: %s' % self.hits + '
    • Exceptions: %s' % self.exceptions + '
    ' ) class persistent_script_handler: def __init__ (self): self.modules = {} self.hits = counter.counter() self.exceptions = counter.counter() def add_module (self, name, module): self.modules[name] = module def del_module (self, name): del self.modules[name] def match (self, request): [path, params, query, fragment] = request.split_uri() parts = string.split (path, '/') if (len(parts)>1) and self.modules.has_key (parts[1]): module = self.modules[parts[1]] request.module = module return 1 else: return 0 def handle_request (self, request): if request.command in ('PUT', 'POST'): # look for a Content-Length header. cl = request.get_header ('content-length') length = int(cl) if not cl: request.error (411) else: collector (self, length, request) else: self.continue_request (request, StringIO.StringIO()) def continue_request (self, request, input_data): temp_files = input_data, StringIO.StringIO(), StringIO.StringIO() old_files = sys.stdin, sys.stdout, sys.stderr try: sys.stdin, sys.stdout, sys.stderr = temp_files # provide a default request['Content-Type'] = 'text/html' try: request.module.main (request) request.reply_code = 200 except: request.reply_code = 500 self.exceptions.increment() finally: sys.stdin, sys.stdout, sys.stderr = old_files i,o,e = temp_files if request.reply_code != 200: s = e.getvalue() else: s = o.getvalue() request['Content-Length'] = len(s) request.push (s) request.done() class collector: def __init__ (self, handler, length, request): self.handler = handler self.request = request self.request.collector = self self.request.channel.set_terminator (length) self.buffer = StringIO.StringIO() def collect_incoming_data (self, data): self.buffer.write (data) def found_terminator (self): self.buffer.seek(0) self.request.collector = None self.request.channel.set_terminator ('\r\n\r\n') self.handler.continue_request ( self.request, self.buffer ) medusa-0.5.4/setup.py0100644000076400007640000000075707722754301012643 0ustar amkamk __revision__ = '$Id: setup.py,v 1.9 2003/08/22 13:07:07 akuchling Exp $' from distutils.core import setup setup( name = 'medusa', version = "0.5.4", description = "A framework for implementing asynchronous servers.", author = "Sam Rushing", author_email = "rushing@nightmare.com", maintainer = "A.M. Kuchling", maintainer_email = "amk@amk.ca", url = "http://oedipus.sourceforge.net/medusa/", packages = ['medusa'], package_dir = {'medusa':'.'}, ) medusa-0.5.4/status_handler.py0100644000076400007640000002220107450102215014474 0ustar amkamk# -*- Mode: Python -*- VERSION_STRING = "$Id: status_handler.py,v 1.6 2002/03/26 14:24:13 amk Exp $" # # medusa status extension # import string import time import re from cgi import escape import asyncore import http_server import medusa_gif import producers from counter import counter START_TIME = long(time.time()) class status_extension: hit_counter = counter() def __init__ (self, objects, statusdir='/status', allow_emergency_debug=0): self.objects = objects self.statusdir = statusdir self.allow_emergency_debug = allow_emergency_debug # We use /status instead of statusdir here because it's too # hard to pass statusdir to the logger, who makes the HREF # to the object dir. We don't need the security-through- # obscurity here in any case, because the id is obscurity enough self.hyper_regex = re.compile('/status/object/([0-9]+)/.*') self.hyper_objects = [] for object in objects: self.register_hyper_object (object) def __repr__ (self): return '' % ( self.hit_counter, id(self) ) def match (self, request): path, params, query, fragment = request.split_uri() # For reasons explained above, we don't use statusdir for /object return (path[:len(self.statusdir)] == self.statusdir or path[:len("/status/object/")] == '/status/object/') # Possible Targets: # /status # /status/channel_list # /status/medusa.gif # can we have 'clickable' objects? # [yes, we can use id(x) and do a linear search] # Dynamic producers: # HTTP/1.0: we must close the channel, because it's dynamic output # HTTP/1.1: we can use the chunked transfer-encoding, and leave # it open. def handle_request (self, request): [path, params, query, fragment] = request.split_uri() self.hit_counter.increment() if path == self.statusdir: # and not a subdirectory up_time = string.join (english_time (long(time.time()) - START_TIME)) request['Content-Type'] = 'text/html' request.push ( '' 'Medusa Status Reports' '' '

    Medusa Status Reports

    ' 'Up: %s' % up_time ) for i in range(len(self.objects)): request.push (self.objects[i].status()) request.push ('
    \r\n') request.push ( '

    Channel List' '

    ' '' '' % ( self.statusdir, self.statusdir, medusa_gif.width, medusa_gif.height ) ) request.done() elif path == self.statusdir + '/channel_list': request['Content-Type'] = 'text/html' request.push ('') request.push(channel_list_producer(self.statusdir)) request.push ( '
    ' '' % ( self.statusdir, medusa_gif.width, medusa_gif.height ) + '' ) request.done() elif path == self.statusdir + '/medusa.gif': request['Content-Type'] = 'image/gif' request['Content-Length'] = len(medusa_gif.data) request.push (medusa_gif.data) request.done() elif path == self.statusdir + '/close_zombies': message = ( '

    Closing all zombie http client connections...

    ' '

    Back to the status page' % self.statusdir ) request['Content-Type'] = 'text/html' request['Content-Length'] = len (message) request.push (message) now = int (time.time()) for channel in asyncore.socket_map.keys(): if channel.__class__ == http_server.http_channel: if channel != request.channel: if (now - channel.creation_time) > channel.zombie_timeout: channel.close() request.done() # Emergency Debug Mode # If a server is running away from you, don't KILL it! # Move all the AF_INET server ports and perform an autopsy... # [disabled by default to protect the innocent] elif self.allow_emergency_debug and path == self.statusdir + '/emergency_debug': request.push ('Moving All Servers...') request.done() for channel in asyncore.socket_map.keys(): if channel.accepting: if type(channel.addr) is type(()): ip, port = channel.addr channel.socket.close() channel.del_channel() channel.addr = (ip, port+10000) fam, typ = channel.family_and_type channel.create_socket (fam, typ) channel.set_reuse_addr() channel.bind (channel.addr) channel.listen(5) else: m = self.hyper_regex.match (path) if m: oid = string.atoi (m.group (1)) for object in self.hyper_objects: if id (object) == oid: if hasattr (object, 'hyper_respond'): object.hyper_respond (self, path, request) else: request.error (404) return def status (self): return producers.simple_producer ( '

  • Status Extension Hits : %s' % self.hit_counter ) def register_hyper_object (self, object): if not object in self.hyper_objects: self.hyper_objects.append (object) import logger class logger_for_status (logger.tail_logger): def status (self): return 'Last %d log entries for: %s' % ( len (self.messages), html_repr (self) ) def hyper_respond (self, sh, path, request): request['Content-Type'] = 'text/plain' messages = self.messages[:] messages.reverse() request.push (lines_producer (messages)) request.done() class lines_producer: def __init__ (self, lines): self.lines = lines def more (self): if self.lines: chunk = self.lines[:50] self.lines = self.lines[50:] return string.join (chunk, '\r\n') + '\r\n' else: return '' class channel_list_producer (lines_producer): def __init__ (self, statusdir): channel_reprs = map ( lambda x: '<' + repr(x)[1:-1] + '>', asyncore.socket_map.values() ) channel_reprs.sort() lines_producer.__init__ ( self, ['

    Active Channel List

    ', '
    '
                 ] + channel_reprs + [
                         '
    ', '

    Status Report' % statusdir ] ) def html_repr (object): so = escape (repr (object)) if hasattr (object, 'hyper_respond'): return '%s' % (id (object), so) else: return so def html_reprs (list, front='', back=''): reprs = map ( lambda x,f=front,b=back: '%s%s%s' % (f,x,b), map (lambda x: escape (html_repr(x)), list) ) reprs.sort() return reprs # for example, tera, giga, mega, kilo # p_d (n, (1024, 1024, 1024, 1024)) # smallest divider goes first - for example # minutes, hours, days # p_d (n, (60, 60, 24)) def progressive_divide (n, parts): result = [] for part in parts: n, rem = divmod (n, part) result.append (rem) result.append (n) return result # b,k,m,g,t def split_by_units (n, units, dividers, format_string): divs = progressive_divide (n, dividers) result = [] for i in range(len(units)): if divs[i]: result.append (format_string % (divs[i], units[i])) result.reverse() if not result: return [format_string % (0, units[0])] else: return result def english_bytes (n): return split_by_units ( n, ('','K','M','G','T'), (1024, 1024, 1024, 1024, 1024), '%d %sB' ) def english_time (n): return split_by_units ( n, ('secs', 'mins', 'hours', 'days', 'weeks', 'years'), ( 60, 60, 24, 7, 52), '%d %s' ) medusa-0.5.4/TODO.txt0100644000076400007640000000062607600125766012434 0ustar amkamkThings to do ============ Bring remaining code up to current standards Translate docs to RST Write README, INSTALL, docs What should __init__ import? Anything? Every single class? Use syslog module in m_syslog for the constants? Add abo's support for blocking producers Get all the producers into the producers module and write tests for them Test suites for protocols: how could that be implemented? medusa-0.5.4/unix_user_handler.py0100644000076400007640000000440707570265063015220 0ustar amkamk# -*- Mode: Python -*- # # Author: Sam Rushing # Copyright 1996, 1997 by Sam Rushing # All Rights Reserved. # RCS_ID = '$Id: unix_user_handler.py,v 1.4 2002/11/25 00:09:23 akuchling Exp $' # support for `~user/public_html'. import re import string import default_handler import filesys import os import pwd get_header = default_handler.get_header user_dir = re.compile ('/~([^/]+)(.*)') class unix_user_handler (default_handler.default_handler): def __init__ (self, public_html = 'public_html'): self.public_html = public_html default_handler.default_handler.__init__ (self, None) # cache userdir-filesystem objects fs_cache = {} def match (self, request): m = user_dir.match (request.uri) return m and (m.end() == len (request.uri)) def handle_request (self, request): # get the user name m = user_dir.match (request.uri) user = m.group(1) rest = m.group(2) # special hack to catch those lazy URL typers if not rest: request['Location'] = '/~%s/' % user request.error (301) return # have we already built a userdir fs for this user? if self.fs_cache.has_key (user): fs = self.fs_cache[user] else: # no, well then, let's build one. # first, find out where the user directory is try: info = pwd.getpwnam (user) except KeyError: request.error (404) return ud = info[5] + '/' + self.public_html if os.path.isdir (ud): fs = filesys.os_filesystem (ud) self.fs_cache[user] = fs else: request.error (404) return # fake out default_handler self.filesystem = fs # massage the request URI request.uri = '/' + rest return default_handler.default_handler.handle_request (self, request) def __repr__ (self): return '' % ( id(self), self.public_html, len(self.fs_cache) ) medusa-0.5.4/virtual_handler.py0100644000076400007640000000327407445740177014673 0ustar amkamk# -*- Mode: Python -*- import socket import default_handler import re HOST = re.compile ('Host: ([^:/]+).*', re.IGNORECASE) get_header = default_handler.get_header class virtual_handler: """HTTP request handler for an HTTP/1.0-style virtual host. Each Virtual host must have a different IP""" def __init__ (self, handler, hostname): self.handler = handler self.hostname = hostname try: self.ip = socket.gethostbyname (hostname) except socket.error: raise ValueError, "Virtual Hostname %s does not appear to be registered in the DNS" % hostname def match (self, request): if (request.channel.addr[0] == self.ip): return 1 else: return 0 def handle_request (self, request): return self.handler.handle_request (request) def __repr__ (self): return '' % self.hostname class virtual_handler_with_host: """HTTP request handler for HTTP/1.1-style virtual hosts. This matches by checking the value of the 'Host' header in the request. You actually don't _have_ to support HTTP/1.1 to use this, since many browsers now send the 'Host' header. This is a Good Thing.""" def __init__ (self, handler, hostname): self.handler = handler self.hostname = hostname def match (self, request): host = get_header (HOST, request.header) if host == self.hostname: return 1 else: return 0 def handle_request (self, request): return self.handler.handle_request (request) def __repr__ (self): return '' % self.hostname medusa-0.5.4/xmlrpc_handler.py0100644000076400007640000000557707543176422014515 0ustar amkamk# -*- Mode: Python -*- # See http://www.xml-rpc.com/ # http://www.pythonware.com/products/xmlrpc/ # Based on "xmlrpcserver.py" by Fredrik Lundh (fredrik@pythonware.com) VERSION = "$Id: xmlrpc_handler.py,v 1.5 2002/08/01 18:15:03 akuchling Exp $" import http_server import xmlrpclib import string import sys class xmlrpc_handler: def match (self, request): # Note: /RPC2 is not required by the spec, so you may override this method. if request.uri[:5] == '/RPC2': return 1 else: return 0 def handle_request (self, request): [path, params, query, fragment] = request.split_uri() if request.command == 'POST': request.collector = collector (self, request) else: request.error (400) def continue_request (self, data, request): params, method = xmlrpclib.loads (data) try: # generate response try: response = self.call (method, params) if type(response) != type(()): response = (response,) except: # report exception back to server response = xmlrpclib.dumps ( xmlrpclib.Fault (1, "%s:%s" % (sys.exc_type, sys.exc_value)) ) else: response = xmlrpclib.dumps (response, methodresponse=1) except: # internal error, report as HTTP server error request.error (500) else: # got a valid XML RPC response request['Content-Type'] = 'text/xml' request.push (response) request.done() def call (self, method, params): # override this method to implement RPC methods raise "NotYetImplemented" class collector: "gathers input for POST and PUT requests" def __init__ (self, handler, request): self.handler = handler self.request = request self.data = '' # make sure there's a content-length header cl = request.get_header ('content-length') if not cl: request.error (411) else: cl = string.atoi (cl) # using a 'numeric' terminator self.request.channel.set_terminator (cl) def collect_incoming_data (self, data): self.data = self.data + data def found_terminator (self): # set the terminator back to the default self.request.channel.set_terminator ('\r\n\r\n') self.handler.continue_request (self.data, self.request) if __name__ == '__main__': class rpc_demo (xmlrpc_handler): def call (self, method, params): print 'method="%s" params=%s' % (method, params) return "Sure, that works" import asyncore hs = http_server.http_server ('', 8000) rpc = rpc_demo() hs.install_handler (rpc) asyncore.loop() medusa-0.5.4/PKG-INFO0100664000076400007640000000043007725426233012217 0ustar amkamkMetadata-Version: 1.1 Name: medusa Version: 0.5.4 Summary: A framework for implementing asynchronous servers. Home-page: http://oedipus.sourceforge.net/medusa/ Author: A.M. Kuchling Author-email: amk@amk.ca License: UNKNOWN Description: UNKNOWN Platform: UNKNOWN Provides: medusa