nwsclient-1.6.4/0000755000175000017500000000000011261702410013142 5ustar westonwestonnwsclient-1.6.4/nws/0000755000175000017500000000000011261702410013751 5ustar westonwestonnwsclient-1.6.4/nws/util.py0000755000175000017500000000567211175420457015331 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import sys, os, string _iswin = sys.platform.startswith('win') _winexts = os.environ.get('PATHEXT', '').split(os.pathsep) _need_quoting = string.whitespace + '"' def _ismatch(p): return os.path.isfile(p) and os.access(p, os.X_OK) def _expand(n): s = [n] # only expand on Windows if _iswin: # only expand if not extension is specified b, e = os.path.splitext(n) if not e: for e in _winexts: s.append(b + e) return s def which(cmd, path=None): if os.path.split(cmd)[0]: cmd = os.path.abspath(cmd) # print "Checking for", cmd matches = [x for x in _expand(cmd) if _ismatch(x)] else: matches = [] if not path: path = os.environ.get('PATH', os.defpath).split(os.pathsep) if _iswin: path.insert(0, os.curdir) for pelem in path: full = os.path.join(os.path.normpath(pelem), cmd) # print "Checking for", full matches += [x for x in _expand(full) if _ismatch(x)] return matches # This is useful in conjunction with os.spawnv on Windows. # For example, # os.spawnv(os.P_WAIT, argv[0], [msc_quote(a) for a in argv]) # This is only useful on Windows. def msc_quote(cmd): if not [c for c in cmd if c in _need_quoting]: return cmd q = '"' nbs = 0 for c in cmd: if c == '\\': q += c nbs += 1 elif c == '"': q += (nbs + 1) * '\\' + c nbs = 0 else: q += c nbs = 0 q += nbs * '\\' + '"' return q # This is useful in conjunction with win32process.CreateProcess # and os.system on Windows. # This is only useful on Windows. def msc_argv2str(argv): return ' '.join([msc_quote(a) for a in argv]) def _qtest(argv): print "quoted command string:", msc_argv2str(argv) def _wtest(argv): for p in argv[1:]: plist = which(p) if not plist: print >> sys.stderr, "error: no matches found for", p continue if len(plist) > 1: print >> sys.stderr, "warning: more than one match found for", p print plist[0] if __name__ == '__main__': _qtest(sys.argv) nwsclient-1.6.4/nws/client.py0000644000175000017500000016620211175420457015624 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Python API for performing NetWorkSpace operations. NetWorkSpaces (NWS) is a powerful, open-source software package that makes it easy to use clusters from within scripting languages like Python, R, and Matlab. It uses a Space-based approach, similar to JavaSpaces (TM) for example, that makes it easier to write distributed applications. Example: First start up the NWS server, using the twistd command: % twistd -y /etc/nws.tac Now you can perform operations against it using this module: % python >>> from nws.client import NetWorkSpace >>> nws = NetWorkSpace("test") >>> nws.store("answer", 42) >>> count = nws.fetch("answer") >>> print "The answer is", count """ import string import cPickle, os, socket, stat try: import fcntl except ImportError: fcntl = None try: import errno except ImportError: errno = None try: import webbrowser except ImportError: webbrowser = None from types import StringType from warnings import warn, filterwarnings from urllib import quote_plus # Change the default warning filtering for this module filterwarnings('always', module=__name__) __all__ = ['NwsServer', 'NetWorkSpace', 'NwsValueIterator', 'NwsException', 'NwsServerException', 'NwsConnectException', 'NwsConnectionDroppedException', 'NwsUnsupportedOperationException', 'NwsOperationException', 'NwsNoWorkSpaceException', 'NwsDeclarationFailedException', 'NwsPicklingException', 'NwsUnpicklingException', 'NwsUnsupportedMethodException', 'FIFO', 'LIFO', 'MULTI', 'SINGLE', 'STRING', 'DICT', 'OPT_SSLKEY', 'OPT_SSLCERT', 'OPT_SSL', 'OPT_DEADMAN', 'OPT_METADATATOSERVER', 'OPT_METADATAFROMSERVER', 'WS_NAME', 'WS_MINE', 'WS_OWNER', 'WS_PERSISTENT', 'WS_NUMVARS', 'WS_VARLIST', 'V_VARIABLE', 'V_VALUES', 'V_FETCHERS', 'V_FINDERS', 'V_MODE', 'PROTOCOL_VERSION_OLD', 'PROTOCOL_VERSION_NEW_0'] _PythonFP = 0x01000000 _DirectString = 0x00000001 _OldProtocol = '2222' _NegotiationHandshakeInit = 'X000' _NegotiationHandshakeAdvertise = 'P000' _NegotiationHandshakeRequest = 'R000' _NegotiationHandshakeAccept = 'A000' PROTOCOL_VERSION_OLD = 'old' PROTOCOL_VERSION_NEW_0 = 'new-0000' NWS_PORT = 8765 FIFO = 'fifo' LIFO = 'lifo' MULTI = 'multi' SINGLE = 'single' STRING = 'string' DICT = 'dict' OPT_SSLKEY = "SSLKey" OPT_SSLCERT = "SSLCertificate" OPT_SSL = "SSL" OPT_DEADMAN = "KillServerOnClose" OPT_METADATATOSERVER = "MetadataToServer" OPT_METADATAFROMSERVER = "MetadataFromServer" WS_NAME = 0 WS_MINE = 1 WS_OWNER = 2 WS_PERSISTENT = 3 WS_NUMVARS = 4 WS_VARLIST = 5 V_VARIABLE = 0 V_VALUES = 1 V_FETCHERS = 2 V_FINDERS = 3 V_MODE = 4 _modes = [FIFO, LIFO, MULTI, SINGLE] _formats = [STRING, DICT] def _marshaldict(d): if d is None: return "0000" else: n = len(d) return ("%04d" % n) + ''.join(["%04d%s%04d%s" % (len(k), k, len(v), v) for k, v in d.items()]) def _marshallist(d, extraArgs=0): if d is None: return "%04d" % extraArgs else: n = len(d) + extraArgs return ("%04d" % n) + ''.join(["%020d%s" % (len(k), k) for k in d]) def _getintattr(obj, attr, defval=0): try: val = getattr(obj, attr) if callable(val): val = val() val = int(val) except: val = defval return val def _sanityFileObj(fobj, mode): if fobj is None: return try: # check the mode (but don't require "mode" attribute) if hasattr(fobj, 'mode'): # these can throw a ValueError fobj.mode.index('b') fobj.mode.index(mode) # must be open (but don't require "closed" attribute) if hasattr(fobj, 'closed'): assert not fobj.closed # must have either a read or write method, depending on mode if mode == 'w': assert hasattr(fobj, 'write') else: assert hasattr(fobj, 'read') except (AssertionError, ValueError): if mode == 'w': raise TypeError('fobj must be a binary mode file object opened for writing') else: raise TypeError('fobj must be a binary mode file object opened for reading') def _sanityStrObj(s, sname): try: assert len(s) == len(s.__str__()) except (AssertionError, AttributeError): raise TypeError('%s must be a string' % sname) class NwsException(Exception): """Base class for all exceptions raised by this module.""" class NwsServerException(NwsException): """Error communicating with the NWS server.""" class NwsConnectException(NwsServerException): """Unable to connect to the NWS server.""" class NwsConnectionDroppedException(NwsServerException): """NWS server connection dropped.""" class NwsUnsupportedOperationException(NwsServerException): """NWS server does not support this operation.""" class NwsOperationException(NwsException): """Error performing an NWS operation.""" class NwsNoWorkSpaceException(NwsOperationException): """No such workspace.""" class NwsDeclarationFailedException(NwsOperationException): """Variable declaration failed.""" class NwsUnsupportedMethodException(NwsException): """Unsupported method.""" class NwsPicklingException(NwsOperationException): """Error pickling an object.""" def __init__(self, exception, wsName, varName): Exception.__init__(self) self.exception = exception self._wsName = wsName self._varName = varName try: self._exname = exception.__class__.__name__ except: self._exname = '[unknown]' def __str__(self): return "caught %s while pickling object to store in '%s @ %s'" % \ (repr(self._exname), self._wsName, self._varName) class NwsUnpicklingException(NwsOperationException): """Error unpickling an object.""" def __init__(self, exception, wsName, varName, obj): Exception.__init__(self) self.exception = exception self._wsName = wsName self._varName = varName self.obj = obj try: self._exname = exception.__class__.__name__ except: self._exname = '[unknown]' def __str__(self): return "caught %s while unpickling object retrieved from '%s @ %s'" % \ (repr(self._exname), self._wsName, self._varName) class SslSocket(object): """Simple wrapper class giving an SSL object a minimal socket interface allowing seamless switching between plain sockets and SSL. """ def __init__(self, sock, keyfile=None, certfile=None): self.sock = sock self.sslobj = socket.ssl(sock, keyfile, certfile) def recv(self, len): return self.sslobj.read(len) def sendall(self, bytes): return self.sslobj.write(bytes) def shutdown(self, n): self.sock.shutdown(n) def close(self): self.sock.close() class NwsServer(object): """Perform operations against the NWS server. Operations against workspaces are performed by using the NetWorkSpace class. """ def __init__(self, host='localhost', port=NWS_PORT, connopts=None): """Create a connection to the NWS server. Arguments: host -- Host name of the NWS server. port -- Port of the NWS server. """ self.serverHost = host self.serverPort = port if connopts is not None: self.connopts = dict(connopts) else: self.connopts = None self.sslkey = None self.sslcert = None self.lastReceivedMetadata = None if self.connopts is not None: if self.connopts.has_key(OPT_SSLKEY): self.sslkey = self.connopts[OPT_SSLKEY] del self.connopts[OPT_SSLKEY] if self.connopts.has_key(OPT_SSLCERT): self.sslcert = self.connopts[OPT_SSLCERT] del self.connopts[OPT_SSLCERT] if (self.sslkey is None) != (self.sslcert is None): raise NwsConnectException("""invalid SSL configuration while connecting to the NWS server at %s:%d Either both or none of SSLKey and SSLCertificate must be specified in the connection options.""" % \ (self.serverHost, self.serverPort)) self._connect() def receive_name(self): len = int(self._recvN(4)) return self._recvN(len) def receive_dict(self): num_opts = int(self._recvN(4)) advertise_opts = {} for i in range(0, num_opts): name = self.receive_name() value = self.receive_name() advertise_opts[name] = value return advertise_opts def _connect(self): if self.sslkey is not None: if self.connopts is None: self.connopts = dict() self.connopts[OPT_SSL] = "" self.nwsSocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) try: self.nwsSocket.connect((self.serverHost, self.serverPort)) except socket.error: raise NwsConnectException('unable to connect to the NWS server at %s:%d' % \ (self.serverHost, self.serverPort)) # denagle the socket to improve performance self.nwsSocket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) # enable the keepalive option self.nwsSocket.setsockopt(socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) # try to set the 'close on exec' flag for the socket if fcntl and hasattr(fcntl, 'FD_CLOEXEC'): fd = self.nwsSocket.fileno() f = fcntl.fcntl(fd, fcntl.F_GETFD) | fcntl.FD_CLOEXEC fcntl.fcntl(fd, fcntl.F_SETFD, f) # tell the server that we're a new client self._sendAll(_NegotiationHandshakeInit) self._handshake = self._recvN(4) if self._handshake == _NegotiationHandshakeAdvertise: self._protocolVersion = PROTOCOL_VERSION_NEW_0 self._cookieProtocol = True advertise_opts = self.receive_dict() if self.connopts is None and len(advertise_opts) != 0: self.connopts = {} for k, v in self.connopts.items(): if not advertise_opts.has_key(k): raise NwsConnectException('requested connection option %s="%s" but this server does not support this option' % (k, v)) elif advertise_opts[k] != '' and advertise_opts[k] != v: raise NwsConnectException('requested connection option %s="%s" but this server requires %s="%s"' % (k, v, k, advertise_opts[k])) self._sendAll(_NegotiationHandshakeRequest + _marshaldict(self.connopts)) if self._recvN(4) != _NegotiationHandshakeAccept: raise NwsConnectException('failed options negotiation with NWS server at %s:%d' % \ (self.serverHost, self.serverPort)) if self.connopts is not None and self.connopts.has_key(OPT_SSL): self.nwsSocket = SslSocket(self.nwsSocket, self.sslkey, self.sslcert) else: self._protocolVersion = PROTOCOL_VERSION_OLD self._cookieProtocol = self._handshake != _OldProtocol def __getstate__(self): return {'serverHost': self.serverHost, 'serverPort': self.serverPort} def __setstate__(self, d): self.__dict__.update(d) self._connect() def __str__(self): return "NwsServer %s:%d" % (self.serverHost, self.serverPort) def get_protocol_version(self): return self._protocolVersion def _recvN(self, n, fobj=None): if n > 0: if fobj is not None: # determine the buffer size for writing to the file or # file-like object defval = 16 * 1024 blen = _getintattr(fobj, "blocksize", defval) if blen <= 0: blen = defval # read data from the socket, then write it to the file while n > 0: d = self.nwsSocket.recv(min(n, blen)) if not d: raise NwsConnectionDroppedException('NWS server connection dropped') fobj.write(d) n -= len(d) return '' else: b = self.nwsSocket.recv(n) m = len(b) n -= m if n <= 0: return b if m == 0: raise NwsConnectionDroppedException('NWS server connection dropped') r = [b] while 1: b = self.nwsSocket.recv(n) r.append(b) m = len(b) n -= m if n <= 0: break if m == 0: raise NwsConnectionDroppedException('NWS server connection dropped') return ''.join(r) else: return '' def _sendAll(self, b): return self.nwsSocket.sendall(b) def sendOp(self, *args, **metadata): if self.connopts.has_key(OPT_METADATATOSERVER): self._sendAll(_marshaldict(metadata) + _marshallist(args)) else: self._sendAll(_marshallist(args)) def sendOpStreaming(self, *args, **metadata): if self.connopts.has_key(OPT_METADATATOSERVER): self._sendAll(_marshaldict(metadata) + _marshallist(args, 1)) else: self._sendAll(_marshallist(args, 1)) def receiveMetadata(self): if self.connopts.has_key(OPT_METADATAFROMSERVER): return self.receive_dict() else: return None def _receiveMetadata(self): self.lastReceivedMetadata = self.receiveMetadata() def close(self): """Close the connection to the NWS server. s.close() This will indirectly cause the NWS server to purge all non-persistent workspaces owned by this client. Purging may not happen immediately, though, but will depend on the load on the server. """ try: self.nwsSocket.shutdown(2) self.nwsSocket.close() except: pass def deleteWs(self, wsName, metadata={}): """Delete the specfied workspace on the NWS server. s.deleteWs(wsName) Arguments: wsName -- Name of the workspace to delete. """ _sanityStrObj(wsName, 'workspace name') self.sendOp('delete ws', wsName, **metadata) self._receiveMetadata() status = int(self._recvN(4)) if status: raise NwsOperationException('deleteWs failed') def listWss(self, format=STRING, metadata={}): """Return a listing of all of the workspaces on the NWS server. s.listWss([format]) -> string or dictionary The listing is a string, consisting of lines, each ending with a newline. Each line consists of tab separated fields. The first field is the workspace name, prefixed with either a '>' or a space, indicating whether the client owns that workspace or not. """ _sanityStrObj(format, 'format') if format not in _formats: raise ValueError('illegal format: ' + format) self.sendOp('list wss', **metadata) self._receiveMetadata() status = int(self._recvN(4)) desc = self._recvN(20) # unused at the moment. if self._cookieProtocol: cookie = self._recvN(40) # unused at the moment. n = int(self._recvN(20)) listing = self._recvN(n) if status: raise NwsOperationException('listWss failed') if format == DICT: wss = {} if listing: for ws in listing[:-1].split('\n'): wsRec = ws.split('\t') mine = wsRec[0][0] == '>' wsRec[WS_NAME] = wsRec[WS_NAME][1:] wsRec.insert(WS_MINE, mine) wsRec[WS_PERSISTENT] = wsRec[WS_PERSISTENT].lower().startswith('t') wsRec[WS_NUMVARS] = int(wsRec[WS_NUMVARS]) wsRec[WS_VARLIST] = wsRec[WS_VARLIST] and wsRec[WS_VARLIST].split(',') or [] wss[wsRec[WS_NAME]] = tuple(wsRec) else: wss = listing return wss def mktempWs(self, wsName='__pyws__%d', metadata={}): """Make a temporary workspace on the NWS server. s.mktempWs([wsName]) -> string The workspace is named using the template and a number generated by the server that makes the workspace name unique. Note that the workspace will be created, but it will not be owned until some client that is willing to take ownership of it opens it. The return value is the actual name of workspace that was created. Arguments: wsName -- Template for the workspace name. This must be a legal 'format' string, containing only an integer format specifier. The default is '__pyws__%d'. Examples: Let's create an NwsServer, call mktempWs to make a workspace, and then use openWs to create a NetWorkSpace object for that workspace: >>> from nws.client import NwsServer >>> server = NwsServer() >>> name = server.mktempWs('example_%d') >>> workspace = server.openWs(name) """ _sanityStrObj(wsName, 'workspace name') self.sendOp('mktemp ws', wsName, **metadata) self._receiveMetadata() status = int(self._recvN(4)) desc = self._recvN(20) # unused at the moment. if self._cookieProtocol: cookie = self._recvN(40) # unused at the moment. n = int(self._recvN(20)) name = self._recvN(n) if status: raise NwsOperationException('mktempWs failed') return name def openWs(self, wsName, space=None, metadata={}, **opt): """Open a workspace. s.openWs(wsName[, space]) -> space If called without a space argument, this method will construct a NetWorkSpace object that will be associated with this NwsServer object, and then perform an open operation with it against the NWS server. The open operation tells the NWS server that this client wants to use that workspace, and is willing to take ownership of it, if it doesn't already exist. The space argument is only intended to be used from the NetWorkSpace constructor. The return value is the constructed NetWorkSpace object. Arguments: wsName -- Name of the workspace to open. If the space argument is not None, this value must match the space's name. space -- NetWorkSpace object to use for the open operation. If the value is None, then openWs will construct a NetWorkSpace object, specifying this NwsServer object as the space's server. Note that this argument is only intended to be used from the NetWorkSpace constructor. The default value is None. Keyword Arguments: persistent -- Boolean value indicating whether the workspace should be persistent or not. See the description of the persistent argument in the __init__ method of the NetWorkSpace class for more information. create -- Boolean value indicating whether the workspace should be created if it doesn't already exist. The default value is true. Examples: Let's create an NwsServer, and then use openWs to create an NetWorkSpace object for a workspace called 'foo': >>> from nws.client import NwsServer >>> server = NwsServer() >>> workspace = server.openWs('foo') Note that this is (nearly) equivalent to: >>> from nws.client import NetWorkSpace >>> workspace = NetWorkSpace('foo') """ # sanity check the keyword arguments unrecog = [a for a in opt.keys() if a not in ['create', 'persistent']] if unrecog: raise TypeError( "__init__() got an unexpected keyword argument '%s'" % \ unrecog[0]) _sanityStrObj(wsName, 'workspace name') # if invoked directly by user, we need to create a space # instance. if invoked via NetWorkSpace constructor, use the # space passed in. if not space: space = NetWorkSpace(wsName, server=self) elif space.currentWs() != wsName: raise ValueError('name of the specified workspace is incorrect') else: owner = '%d' % os.getpid() p = 'no' if opt.get('persistent', False): p = 'yes' if opt.get('create', True): self.sendOp('open ws', wsName, owner, p, **metadata) else: self.sendOp('open ws', wsName, owner, p, 'no', **metadata) self._receiveMetadata() status = int(self._recvN(4)) if status: raise NwsNoWorkSpaceException("workspace %s doesn't exist" % repr(wsName)) return space def useWs(self, wsName, space=None, metadata={}, **opt): """Use a NetWorkSpace object. s.useWs(wsName[, space]) -> space If called without a space argument, this method will construct a NetWorkSpace object that will be associated with this NwsServer object, and then perform a use operation with it against the NWS server. The use operation tells the NWS server that this client wants to use that workspace, but is not willing to take ownership of it. The space argument is only intended to be used from the NetWorkSpace constructor. The return value is the constructed NetWorkSpace object. Arguments: wsName -- Name of the workspace to use. If the space argument is not None, this value must match the space's name. space -- NetWorkSpace object to use for the use operation. If the value is None, then useWs will construct a NetWorkSpace object, specifying this NwsServer object as the space's server. Note that this argument is only intended to be used from the NetWorkSpace constructor. The default value is None. Keyword Arguments: create -- Boolean value indicating whether the workspace should be created if it doesn't already exist. The default value is true. Examples: Let's create an NwsServer, and then use useWs to create an NetWorkSpace object for a workspace called 'foo': >>> from nws.client import NwsServer >>> server = NwsServer() >>> workspace = server.useWs('foo') Note that this is (nearly) equivalent to: >>> from nws.client import NetWorkSpace >>> workspace = NetWorkSpace('foo', useUse=True) """ # sanity check the keyword arguments unrecog = [a for a in opt.keys() if a not in ['create', 'persistent']] if unrecog: raise TypeError( "__init__() got an unexpected keyword argument '%s'" % \ unrecog[0]) _sanityStrObj(wsName, 'workspace name') # if invoked directly by user, we need to create a space # instance. if invoked via networkspace constructor, use the # space passed in. if not space: space = NetWorkSpace(wsName, server=self) elif space.currentWs() != wsName: raise ValueError('name of the specified workspace is incorrect') else: owner = '' p = 'no' if opt.get('create', True): self.sendOp('use ws', wsName, owner, p, **metadata) else: self.sendOp('use ws', wsName, owner, p, 'no', **metadata) self._receiveMetadata() status = int(self._recvN(4)) if status: raise NwsNoWorkSpaceException("workspace %s doesn't exist" % repr(wsName)) return space def view(self, port=8766): if webbrowser: vstring = "http://%s:%d/doit?op=listWss" % \ (self.serverHost, port) webbrowser.open_new(vstring) else: raise NwsUnsupportedMethodException('cannot import webbrowser module') class NetWorkSpace(object): """Perform operations against workspaces on NWS servers. The NetWorkSpace object is the basic object used to perform operatations on workspaces. Variables can be declared, created, deleted, and the values of those variables can be manipulated. You can think of a workspace as a network accessible python dictionary, where the variable names are keys in the dictionary, and the associated values are lists of pickled python objects. The store method puts a value into the list associated with the specified variable. The find method returns a single value from a list. Which value it returns depends on the "mode" of the variable (see the declare method for more information on the variable mode). If the list is empty, the find method will not return until a value is stored in that list. The findTry method works like the find method, but doesn't wait, returning a default value instead (somewhat like the dictionary's get method). The fetch method works like the find method, but it will also remove the value from the list. If multiple clients are all blocked on a fetch operation, and a value is stored into that variable, the server guarantees that only one client will be able to fetch that value. The fetchTry method, not surprisingly, works like the fetch method, but doesn't wait, returning a default value instead. """ def __init__(self, wsName='__default', serverHost='localhost', serverPort=NWS_PORT, useUse=False, server=None, connopts=None, wsmetadata={}, **opt): """Construct a NetWorkSpace object for the specified NwsServer. Arguments: wsName -- Name of the workspace. There can only be one workspace on the server with a given name, so two clients can easily communicate with each other by both creating a NetWorkSpace object with the same name on the same server. The first client that creates a workspace that is willing to take ownership of it, will become the owner (see the description of the useUse argument below for more information on workspace ownership). serverHost -- Host name of the NWS server. This argument is ignored if the server argument is not None. The default value is 'localhost'. serverPort -- Port of the NWS server. This argument is ignored if the server argument is not None. The default value is 8765. useUse -- Boolean value indicating whether you only want to use this workspace, or if you want to open it (which means you're willing to take ownership of it, if it's not already owned). The default value is False, meaning you are willing to take ownership of this workspace. server -- This argument is only intended for internal use. Specifying a value other than None will result in a NetWorkSpace object that isn't properly initialized. connopts -- connection options for options negotiation with the server. Keyword Arguments: persistent -- Boolean value indicating whether the workspace should be persistent or not. If a workspace is persistent, it won't be purged when the owner disconnects from the NWS server. Note that only the client who actually takes ownership of the workspace can make the workspace persistent. The persistent argument is effectively ignored if useUse is True, since that client never becomes the owner of the workspace. If useUse is False, it is the client who actually becomes the owner that determines whether it is persistent or not. That is, after the workspace is owned, the persistent argument is ignored. The default value is false. create -- Boolean value indicating whether the workspace should be created if it doesn't already exist. The default value is true. """ # sanity check the keyword arguments unrecog = [a for a in opt.keys() if a not in ['create', 'persistent']] if unrecog: raise TypeError( "__init__() got an unexpected keyword argument '%s'" % \ unrecog[0]) _sanityStrObj(wsName, 'workspace name') self.curWs = wsName self._useUse = useUse self._opt = opt self.lastReceivedMetadata = None if len(wsmetadata) != 0: if connopts is None: connopts = {} connopts[OPT_METADATATOSERVER] = '1' # if invoked (indirectly) via a NwsServer openWs or useWs method, # the server will be passed in and used. if invoked directly, # need to create a new NwsServer instance. if not server: self.server = NwsServer(serverHost, serverPort, connopts=connopts) else: self.server = server # now give the server a chance to do its thing. try: if self._useUse: self.server.useWs(self.curWs, self, metadata=wsmetadata, **self._opt) else: self.server.openWs(self.curWs, self, metadata=wsmetadata, **self._opt) except Exception, e: # close the server and re-raise the exception try: self.server.close() except: pass raise e self.send = self.server._sendAll self.recv = self.server._recvN self._handshake = self.server._handshake self._cookieProtocol = self.server._cookieProtocol def __getstate__(self): return {'curWs': self.curWs, '_useUse': self._useUse, '_opt': self._opt, 'server': self.server} def __setstate__(self, d): self.__dict__.update(d) try: if self._useUse: self.server.useWs(self.curWs, self, **self._opt) else: self.server.openWs(self.curWs, self, **self._opt) except Exception, e: # close the server and re-raise the exception try: self.server.close() except: pass raise e self.send = self.server._sendAll self.recv = self.server._recvN self._handshake = self.server._handshake self._cookieProtocol = self.server._cookieProtocol def __str__(self): return "NetWorkspace '%s' [%s]" % (self.curWs, str(self.server)) def receiveMetadata(self): self.lastReceivedMetadata = self.server.receiveMetadata() def currentWs(self): """Return the name of the current workspace. ws.currentWs() -> string """ return self.curWs def declare(self, varName, mode, metadata={}): """Declare a variable in a workspace with the specified mode. ws.declare(varName, mode) This method is used to specify a mode other than the default mode of 'fifo'. Legal values for the mode are: 'fifo', 'lifo', 'multi', and 'single' In the first three modes, multiple value can be stored in a variable. If the mode is 'fifo', then values are retrieved in a "first-in, first-out" fashion. That is, the first value stored, will be the first value fetched. If the mode is 'lifo', then values are retrieved in a "last-in, first-out" fashion, as in a stack. If the mode is 'multi', then the order of retrieval is pseudorandom. The 'single' mode means that only a single value can be stored in the variable. Each new store operation will overwrite the current value of the variable. If a variable is created using a store operation, then the mode defaults to 'fifo'. The mode cannot be changed with subsequent calls to declare, regardless of whether the variable was originally created using store or declare. Arguments: varName -- Name of the workspace variable to declare. mode -- Mode of the variable. """ _sanityStrObj(varName, 'variable name') _sanityStrObj(mode, 'variable mode') # note that workspace variable extensions start with '__' if mode not in _modes and not mode.startswith('__'): raise ValueError('illegal mode: ' + str(mode)) self.server.sendOp('declare var', self.curWs, varName, mode, **metadata) self.receiveMetadata() status = int(self.recv(4)) if status: raise NwsDeclarationFailedException('variable declaration failed') def deleteVar(self, varName, metadata={}): """Delete a variable from a workspace. ws.deleteVar(varName) All values of the variable are destroyed, and all currently blocking fetch and find operations will be aborted. Arguments: varName -- Name of the workspace variable to delete. """ _sanityStrObj(varName, 'variable name') self.server.sendOp('delete var', self.curWs, varName, **metadata) self.receiveMetadata() status = int(self.recv(4)) if status: raise NwsOperationException('deleteVar failed') def __retrieve(self, varName, op, missing=None, fobj=None, metadata={}): _sanityStrObj(varName, 'variable name') _sanityFileObj(fobj, 'w') self.server.sendOp(op, self.curWs, varName, **metadata) self.receiveMetadata() status = int(self.recv(4)) # barely used at the moment. # even if failure status, read the rest of the bytes. desc = int(self.recv(20)) if self.server._cookieProtocol: cookie = self.recv(40) # unused at the moment. n = int(self.recv(20)) pVal = self.recv(n, fobj) if status: raise NwsOperationException('retrieval failed') if fobj is None: if desc & _DirectString: return pVal elif pVal: try: val = cPickle.loads(pVal) except Exception, e: raise NwsUnpicklingException(e, self.curWs, varName, pVal) return val else: return missing else: return n def fetch(self, varName, metadata={}): """Return and remove a value of a variable from a workspace. ws.fetch(varName) -> object If the variable has no values, the operation will not return until it does. In other words, this is a "blocking" operation. fetchTry is the "non-blocking" version of this method. Note that if many clients are all trying to fetch from the same variable, only one client can fetch a given value. Other clients may have previously seen that value using the find or findTry method, but only one client can ever fetch or fetchTry a given value. Arguments: varName -- Name of the workspace variable to fetch. """ return self.__retrieve(varName, 'fetch', None, metadata=metadata) def fetchTry(self, varName, missing=None, metadata={}): """Try to return and remove a value of a variable from a workspace. ws.fetchTry(varName[, missing]) -> object If the variable has no values, the operation will return the value specified by "missing", which defaults to None. Note that if many clients are all trying to fetchTry from the same variable, only one client can fetchTry a given value. Other clients may have previously seen that value using the find or findTry method, but only one client can ever fetch or fetchTry a given value. Arguments: varName -- Name of the workspace variable to fetch. missing -- Value to return if the variable has no values. """ try: return self.__retrieve(varName, 'fetchTry', missing, metadata=metadata) except NwsUnpicklingException: raise except NwsOperationException: return missing def find(self, varName, metadata={}): """Return a value of a variable from a workspace. ws.find(varName) -> object If the variable has no values, the operation will not return until it does. In other words, this is a "blocking" operation. findTry is the "non-blocking" version of this method. Note that (unlike fetch) find does not remove the value. The value remains in the variable. Arguments: varName -- Name of the workspace variable to find. """ return self.__retrieve(varName, 'find', None, metadata=metadata) def findTry(self, varName, missing=None, metadata={}): """Try to return a value of a variable in the workspace. ws.findTry(varName[, missing]) -> object If the variable has no values, the operation will return the value specified by "missing", which defaults to the value "None". Note that (unlike fetchTry) findTry does not remove the value. The value remains in the variable. Arguments: varName -- Name of the workspace variable to use. missing -- Value to return if the variable has no values. The default is None. """ try: return self.__retrieve(varName, 'findTry', missing, metadata=metadata) except NwsUnpicklingException: raise except NwsOperationException: return missing def listVars(self, wsName=None, format=STRING, metadata={}): """Return a listing of the variables in the workspace. ws.listVars([wsName[, format]]) -> string or dictionary Arguments: wsName -- Name of the workspace to list. The default is None, which means to use the current workspace. format -- Output format to return. Legal values include 'string' and 'dict'. The 'string' format returns a string which is suitable for printing. The 'dict' format returns a dictionary of tuples, where the first field is the variable name, the second is the number of values, the third is the number of fetchers, the fourth is the number of finders, and the fifth is the variables mode. The default value is 'string'. """ if wsName is not None: _sanityStrObj(wsName, 'workspace name') _sanityStrObj(format, 'format') if format not in _formats: raise ValueError('illegal format: ' + format) if not wsName: wsName = self.curWs self.server.sendOp('list vars', wsName, **metadata) self.receiveMetadata() status = int(self.recv(4)) desc = self.recv(20) # unused at the moment. if self.server._cookieProtocol: cookie = self.recv(40) # unused at the moment. n = int(self.recv(20)) listing = self.recv(n) if status != 0: raise NwsOperationException('listVars failed') if format == DICT: vars = {} if listing: for var in listing.split('\n'): varRec = var.split('\t') varRec[V_VALUES] = int(varRec[V_VALUES]) varRec[V_FETCHERS] = int(varRec[V_FETCHERS]) varRec[V_FINDERS] = int(varRec[V_FINDERS]) vars[varRec[V_VARIABLE]] = tuple(varRec) else: vars = listing return vars def store(self, varName, val, metadata={}): """Store a new value into a variable in the workspace. ws.store(varName, val) Arguments: varName -- Name of the workspace variable. val -- Value to store in the variable. """ _sanityStrObj(varName, 'variable name') desc = _PythonFP if isinstance(val, str): desc |= _DirectString descTxt = '%020u' % desc if desc & _DirectString: pVal = val else: try: pVal = cPickle.dumps(val, 1) except Exception, e: raise NwsPicklingException(e, self.curWs, varName) self.server.sendOpStreaming('store', self.curWs, varName, descTxt, **metadata) self.send('%020d' % len(pVal)) self.send(pVal) self.receiveMetadata() status = int(self.recv(4)) if status: raise NwsOperationException('store failed') def fetchFile(self, varName, fobj, metadata={}): """Return and remove a value of a variable from a workspace. ws.fetchFile(varName, fobj) -> number of bytes written to file Arguments: varName -- Name of the workspace variable to fetch. fobj -- File to write data to. """ return self.__retrieve(varName, 'fetch', fobj=fobj, metadata=metadata) def fetchTryFile(self, varName, fobj, metadata={}): """Try to return and remove a value of a variable from a workspace. ws.fetchTryFile(varName, fobj) -> number of bytes written to file Arguments: varName -- Name of the workspace variable to fetch. fobj -- File to write data to. """ try: return self.__retrieve(varName, 'fetchTry', fobj=fobj, metadata=metadata) except NwsUnpicklingException: raise except NwsOperationException: return False def findFile(self, varName, fobj, metadata={}): """Return a value of a variable from a workspace. ws.findFile(varName, fobj) -> number of bytes written to file Arguments: varName -- Name of the workspace variable to find. fobj -- File to write data to. """ return self.__retrieve(varName, 'find', fobj=fobj, metadata=metadata) def findTryFile(self, varName, fobj, metadata={}): """Try to return a value of a variable in the workspace. ws.findTryFile(varName, fobj) -> number of bytes written to file Arguments: varName -- Name of the workspace variable to use. fobj -- File to write data to. """ try: return self.__retrieve(varName, 'findTry', fobj=fobj, metadata=metadata) except NwsUnpicklingException: raise except NwsOperationException: return False def storeFile(self, varName, fobj, n=0, metadata={}): """Store a new value into a variable in the workspace from a file. ws.storeFile(varName, fobj[, n]) -> number of bytes read from file Note that if there is no more data to read from the file, storeFile returns 0, and does not store a value into the workspace variable. Arguments: varName -- Name of the workspace variable. fobj -- File to read data from. n -- Maximum number of bytes to read from the file. A value of zero means to read and store all of the data in the file. The default value is zero. """ _sanityStrObj(varName, 'variable name') _sanityFileObj(fobj, 'r') desc = _PythonFP | _DirectString descTxt = '%020u' % desc if hasattr(fobj, 'len'): # it's a file-like object. get the size from the # "len" attribute, which should be either an integer # (as in StringIO.StringIO) or a method fsize = _getintattr(fobj, 'len') else: # determine the length of the file using stat # if length not specified fsize = os.fstat(fobj.fileno())[stat.ST_SIZE] fpos = _getintattr(fobj, 'tell') fbytes = fsize - fpos if n <= 0: n = fbytes else: n = min(n, fbytes) if n <= 0: return 0 nread = n # save this for the return value self.server.sendOpStreaming('store', self.curWs, varName, descTxt, **metadata) self.send('%020d' % n) # allow file-like objects to tell us how much data to ask # for at a time. however, we also allow them return either # more or less data. this mechanism is more important when # receiving data. defval = 16 * 1024 blen = _getintattr(fobj, "blocksize", defval) if blen <= 0: blen = defval while n > 0: d = fobj.read(min(blen, n)) dlen = len(d) if dlen <= 0: break elif dlen > n: # we allow this for file-like objects dlen = n d = d[:n] self.send(d) n -= dlen if n > 0: # this can happen if a file-like object gives us # less data than the reported size warn('premature end-of-file reached: padding value with zeroes', stacklevel=2) blen = 1024 buffer = blen * "\0" while n > 0: if blen <= n: self.send(buffer) n -= dlen else: self.send(n * "\0") break self.receiveMetadata() status = int(self.recv(4)) if status: raise NwsOperationException('store file failed') return nread def _iretrieve(self, varName, op, varId, valIndex, fobj=None): _sanityFileObj(fobj, 'w') self.server.sendOp(op, self.curWs, varName, "%-20.20s" % varId, "%020d" % valIndex) self.receiveMetadata() status = int(self.recv(4)) # barely used at the moment. # even if failure status, read the rest of the bytes. desc = int(self.recv(20)) if self.server._cookieProtocol: varId = self.recv(20) valIndex = int(self.recv(20)) n = int(self.recv(20)) pVal = self.recv(n, fobj) if fobj is None: if desc & _DirectString: return (status, pVal, varId, valIndex, n) elif pVal: try: val = cPickle.loads(pVal) except Exception, e: raise NwsUnpicklingException(e, self.curWs, varName, pVal) return (status, val, varId, valIndex, n) else: raise StopIteration else: if desc & _DirectString or pVal: return (status, pVal, varId, valIndex, n) else: raise StopIteration def ifetch(self, varName): """Return a fetch iterator for a workspace variable. ws.ifetch(varName) -> iterator Unlike ifind, this method doesn't really provide any extra functionality over the fetch method. It is provided for completeness, and for those who just like iterators. Note that ifetch can be used on FIFO and SINGLE mode variables, but not LIFO and MULTI mode variables. Arguments: varName -- Name of the workspace variable to iterator over. """ if not self.server._cookieProtocol: raise NwsUnsupportedOperationException('NWS server does not support ifetch: ' + self._handshake) _sanityStrObj(varName, 'variable name') return NwsValueIterator(self, varName, 'ifetch') def ifetchTry(self, varName): """Return a fetchTry iterator for a workspace variable. ws.ifetchTry(varName) -> iterator Unlike ifindTry, this method doesn't really provide any extra functionality over the fetchTry method. It is provided for completeness, and for those who just like iterators. Note that ifetchTry can be used on FIFO and SINGLE mode variables, but not LIFO and MULTI mode variables. Arguments: varName -- Name of the workspace variable to iterator over. """ if not self.server._cookieProtocol: raise NwsUnsupportedOperationException('NWS server does not support ifetchTry: ' + self._handshake) _sanityStrObj(varName, 'variable name') return NwsValueIterator(self, varName, 'ifetchTry') def ifind(self, varName): """Return a find iterator for a workspace variable. ws.ifind(varName) -> iterator This is very useful if you want to see every value in a variable without destroying them. Unlike the find method, ifind won't return the same value repeatedly. When there are no more values to return, the iterator will block, waiting for a new value to arrive. Note that ifind can be used on FIFO and SINGLE mode variables, but not LIFO and MULTI mode variables. Arguments: varName -- Name of the workspace variable to iterate over. """ if not self.server._cookieProtocol: raise NwsUnsupportedOperationException('NWS server does not support ifind: ' + self._handshake) _sanityStrObj(varName, 'variable name') return NwsValueIterator(self, varName, 'ifind') def ifindTry(self, varName): """Return a findTry iterator for a workspace variable. ws.ifindTry(varName) -> iterator This is very useful if you want to see every value in a variable without destroying them. Unlike the findTry method, ifindTry won't return the same value repeatedly. When there are no more values to return, the iterator finishes. Note that ifindTry can be used on FIFO and SINGLE mode variables, but not LIFO and MULTI mode variables. Arguments: varName -- Name of the workspace variable to iterate over. """ if not self.server._cookieProtocol: raise NwsUnsupportedOperationException('NWS server does not support ifindTry: ' + self._handshake) _sanityStrObj(varName, 'variable name') return NwsValueIterator(self, varName, 'ifindTry') def makefile(self, varName, mode='r'): if mode in ['r', 'w', 'a']: return NwsTextFile(self, varName, mode) elif mode in ['rb', 'wb', 'ab']: raise ValueError('mode %r is not supported yet' % mode) else: raise ValueError('illegal mode specified: %r' % mode) def view(self, port=8766): if webbrowser: vstring = "http://%s:%d/doit?op=listVars&wsName=%s" % \ (self.server.serverHost, port, quote_plus(self.curWs)) webbrowser.open_new(vstring) else: raise NwsUnsupportedMethodException('cannot import webbrowser module') class NwsValueIterator(object): """Implements the iterated operations against a workspace variable. Instances of this class are returned from the NetWorkSpace ifetch, ifetchTry, ifind, and ifindTry methods. """ def __init__(self, ws, varName, op): """Create an iterator over a workspace varible. This constructor is intended for internal use only. Arguments: ws -- NetWorkSpace object containing the specified variable. varName -- Name of the workspace variable to iterate over. op -- Operation to perform. """ self._ws = ws self._varName = varName self._op = op self._varId = '' self._valIndex = 0 self._stopped = False def restart(self): """Allow the iterator to continue where it left off after stopping. The Python iterator protocol requires that iterators continue to raise StopIteration once they've raised it. The NwsValueIterator will do that unless and until you call this method. That can be useful for the "Try" iterators, where you might want to find all of the values in a variable, and at a later point see if there are any new values without having to see the previous ones again. """ self._stopped = False def reset(self): """Reset the iterator to the beginning of the workspace variable. This conveniently restores the state of the iterator to when it was first created. """ self._varId = '' self._valIndex = 0 self._stopped = False def __iter__(self): return self def next(self): if self._stopped: raise StopIteration try: status, val, self._varId, self._valIndex, n = \ self._ws._iretrieve(self._varName, self._op, self._varId, self._valIndex) except StopIteration, e: self._stopped = True raise e if status != 0: raise NwsOperationException('retrieval failed') return val def writeTo(self, fobj): """Write the next value to the specified file or file-like object. it.writeTo(fobj) -> number of bytes written to file This is very much like the "next" method, but unlike "next", is intended to be called explicitly. It provides the same kind of functionality as the various NetWorkSpace findFile/fetchTryFile methods. It is the easiest and most memory efficient way to non-destructively write all of the values of a variable to a file. Arguments: fobj -- File to write data to. """ if self._stopped: return 0 try: status, val, self._varId, self._valIndex, n = \ self._ws._iretrieve(self._varName, self._op, self._varId, self._valIndex, fobj) return n except StopIteration, e: self._stopped = True return 0 class NwsTextFile(object): def __init__(self, ws, varName, mode): assert mode in ['r', 'w', 'a'] self._ws = ws self._varName = varName self._wbuffer = [] self._rbuffer = '' self.closed = False self.mode = mode self.name = varName self.softspace = False if mode == 'w': # delete the variable if it exists try: ws.deleteVar(varName) except NwsOperationException: pass if mode == 'r': self._it = ws.ifindTry(varName) ws.declare(varName, FIFO) def __del__(self): try: self.close() except: pass def __iter__(self): if self.mode not in ['r', 'rb']: raise ValueError('iteration not allowed in mode %r' % self.mode) return self def __repr__(self): state = self.closed and "closed" or "open" return "<%s nws file %r, mode %r at %s>" % (state, self.name, self.mode, hex(id(self))) def __str__(self): return repr(self) def next(self): x = self.readline() if not x: raise StopIteration() return x def read(self, n=-1): if self.closed: raise ValueError('I/O operation on closed file') if self.mode not in ['r', 'rb']: raise ValueError('read operation not allowed in mode %r' % self.mode) navail = len(self._rbuffer) # read one or more values from the nws variable until # we have enough data, or we run out of values if n < 0 or navail < n: buffer = [self._rbuffer] try: while n < 0 or navail < n: x = self._it.next() if not isinstance(x, str): raise ValueError('read a non-string from %r: %r' % (self, x)) buffer.append(x) navail += len(x) except StopIteration: pass self._rbuffer = ''.join(buffer) if n < 0: x = self._rbuffer self._rbuffer = '' else: x = self._rbuffer[:n] self._rbuffer = self._rbuffer[n:] return x def readline(self): if self.closed: raise ValueError('I/O operation on closed file') if self.mode not in ['r', 'rb']: raise ValueError('read operation not allowed in mode %r' % self.mode) x = self._rbuffer buffer = [x] try: while not x.endswith('\n'): x = self._it.next() if not isinstance(x, str): raise ValueError('read a non-string from %r: %r' % (self, x)) buffer.append(x) except StopIteration: pass x = ''.join(buffer) self._rbuffer = '' return x def readlines(self): return [x for x in self] def write(self, s): if self.closed: raise ValueError('I/O operation on closed file') if self.mode not in ['w', 'a', 'wb', 'ab']: raise ValueError('write operation not allowed in mode %r' % self.mode) if s: x = s.split('\n') if len(x) > 1: self._wbuffer.append(x[0]) y = ''.join(self._wbuffer) self._ws.store(self._varName, y + '\n') for y in x[1:-1]: self._ws.store(self._varName, y + '\n') self._wbuffer = [] if x[-1]: self._wbuffer.append(x[-1]) else: self._wbuffer.append(s) def writelines(self, lst): for line in lst: self.write(line) def close(self): if not self.closed: try: if self.mode in ['w', 'wb', 'a', 'ab']: self.flush() finally: try: del self._wbuffer except AttributeError: pass try: del self._rbuffer except AttributeError: pass self.closed = True def flush(self): if self.closed: raise ValueError('I/O operation on closed file') if self.mode not in ['w', 'a', 'wb', 'ab']: raise ValueError('flush operation not allowed in mode %r' % self.mode) # flush any buffered data without a newline if self._wbuffer: y = ''.join(self._wbuffer) if y: self._ws.store(self._varName, y) self._wbuffer = [] def isatty(self): if self.closed: raise ValueError('I/O operation on closed file') return False nwsclient-1.6.4/nws/__init__.py0000644000175000017500000000145011175420457016076 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # __version__ = '1.6.4' import client, sleigh nwsclient-1.6.4/nws/babelfish.py0000755000175000017500000000456211175420457016270 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import sys, getopt, traceback from nws.client import NetWorkSpace from nws.client import NwsServerException, NwsConnectException from nws.client import NwsUnpicklingException MAXLEN = 1000 def babelfish(host, port): ws = NetWorkSpace('Python babelfish', host, port) while 1: try: v = ws.fetch('food') t = str(v) if len(t) > MAXLEN: t = t[:MAXLEN] + '[WARNING: output truncated]' elif not t: t = repr(v) except NwsUnpicklingException, e: t = 'Unpickling error: ' + str(e.exception) ws.store('doof', t) def main(argv): host = 'localhost' port = 8765 try: opts, args = getopt.getopt(argv, 'h:p:') for opt, arg in opts: if opt == '-h': host = arg elif opt == '-p': port = int(arg) else: raise 'internal error: out-of-sync with getopt' if len(args) > 0: print >> sys.stderr, 'ignoring unused arguments:', args babelfish(host, port) except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except NwsConnectException, e: # the server isn't running print >> sys.stderr, e.args[0] except (KeyboardInterrupt, NwsServerException, SystemExit): # either the user or the server is telling us to shutdown pass except: traceback.print_exc() if __name__ == '__main__': main(sys.argv[1:]) nwsclient-1.6.4/nws/sleigh.py0000644000175000017500000027530211175420457015623 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # """Python API for parallel programming using NetWorkSpaces. The sleigh module, built on top of NetWorkSpaces (NWS), makes it very easy to write simple parallel programs. It contains the Sleigh class, which provides two basic methods for executing tasks in parallel: eachElem and eachWorker. eachElem is used to execute a specified function multiple times in parallel with a varying set of arguments. eachWorker is used to execute a function exactly once on every worker in the sleigh with a fixed set of arguments. Example: First start up the NWS server, using the twistd command: % twistd -y /etc/nws.tac Now you can create a sleigh to execute python code in parallel: % python >>> from nws.sleigh import Sleigh >>> s = Sleigh() >>> import math >>> result = s.eachElem(math.exp, range(10)) >>> print "The answer is", result """ from __future__ import generators import sys, os, string, time, socket, random import cPickle, pickle, new, copy_reg, traceback try: import platform except: platform = None try: import subprocess except: pass try: import win32api _winuser = win32api.GetUserName() except ImportError: _winuser = None try: from itertools import izip, count except ImportError: # this is compatibility code for Python 2.2 support def izip(*iterables): iterables = map(iter, iterables) while iterables: result = [it.next() for it in iterables] yield tuple(result) def count(n=0): while True: yield n n += 1 try: import webbrowser except ImportError: webbrowser = None # We use this a lot, so provide convenience name. from os import environ as _Env from types import CodeType, FunctionType, BuiltinFunctionType, MethodType from cStringIO import StringIO from warnings import warn, filterwarnings from urllib import quote_plus from nws.client import NetWorkSpace, NwsServer, NwsNoWorkSpaceException from nws.client import FIFO, DICT, V_VARIABLE, V_VALUES, V_FETCHERS, V_FINDERS, V_MODE from nws.client import PROTOCOL_VERSION_OLD, PROTOCOL_VERSION_NEW_0 from nws.util import which, msc_quote from nws import __version__ # Change the default warning filtering for this module filterwarnings('always', module=__name__) __all__ = [ 'Sleigh', 'SleighPending', 'SleighResultIterator', 'SleighException', 'SleighNwsException', 'SleighGatheredException', 'SleighStoppedException', 'SleighOccupiedException', 'SleighIllegalValueException', 'SleighScriptException', 'SleighTaskException', 'SleighJoinException', 'SleighNotAllowedException', 'SleighUnsupportedMethodException' ] # We use alternating barriers to synchronize eachWorker # invocations. Their names are common to workers and sleighs. _barrierNames = ['barrier0', 'barrier1'] _lambdaName = (lambda: 0).__name__ # These are directories to look for the sleigh worker scripts # on the local machine. These are only searched if scriptDir # hasn't been set, and the worker script wasn't found using # the PATH environment variable. if sys.platform.startswith('win'): _pred = lambda p: True _rule = lambda p, t: os.path.join(*(p + t)) _exeRules = [ (_pred, (), _rule, (['scripts'],)), ] _scriptDirs = [ r'C:\python25\scripts', r'C:\python24\scripts', ] else: _pred = lambda p, prefix: _startsWith(p, prefix) _rule = lambda p, n, t: os.path.join(*(p[1:n] + t)) _exeRules = [ (_pred, (['', '/', 'Library', 'Frameworks', 'Python.framework', 'Versions'],), _rule, (7, ['bin'])), (_pred, (['', '/', 'System', 'Library', 'Frameworks', 'Python.framework', 'Versions'],), _rule, (8, ['bin'])), (_pred, (['', '/', 'usr', 'bin'],), _rule, (4, [])), (_pred, (['', '/', 'usr', 'local', 'bin'],), _rule, (5, [])), ] _scriptDirs = [ '/Library/Frameworks/Python.framework/Versions/Current/bin', '/System/Library/Frameworks/Python.framework/Versions/Current/bin', ] class _Result(object): tag = None value = None error = False def __str__(self): if self.error: return "Result Error %s: %s" % (repr(self.tag), repr(self.value)) else: return "Result %s: %s" % (repr(self.tag), repr(self.value)) class _SleighState(object): bx = 0 occupied = None stopped = False totalTasks = 0 class _Task(object): args = None barrier = None code = None module = None funcName = None methName = None tag = None type = None def __str__(self): if self.funcName: return "Function Task %s: %s(%s)" % \ (repr(self.tag), self.funcName, repr(self.args)) elif self.methName: return "Method Task %s: %s(%s)" % \ (repr(self.tag), self.methName, repr(self.args)) else: # XXX improve this return "Other Task %s: (%s)" % \ (repr(self.tag), repr(self.args)) class SleighException(Exception): """Base class for all exceptions raised by this module.""" def __str__(self): return '%s[%s]' % (self.__class__.__name__, ' '.join(map(str, self.args))) def __repr__(self): return '%s(%s)' % (self.__class__.__name__, ', '.join(map(repr, self.args))) class SleighNwsException(SleighException): """Error performing NWS operation.""" class SleighGatheredException(SleighException): """Results already gathered.""" class SleighStoppedException(SleighException): """Sleigh is stopped.""" class SleighOccupiedException(SleighException): """Sleigh is occupied.""" class SleighIllegalValueException(SleighException): """Illegal value specified.""" class SleighScriptException(SleighException): """Unable to find sleigh worker script.""" class SleighTaskException(SleighException): """Error executing a task.""" class SleighJoinException(SleighException): """Too late to join worker group.""" class SleighNotAllowedException(SleighException): """Sleigh construction not allowed.""" class SleighUnsupportedMethodException(SleighException): """Unsupported method.""" # functions needed for pickling and unpickling code objects def _code_ctor(*args): return new.code(*args) def _reduce_code(co): if co.co_freevars or co.co_cellvars: raise ValueError("cannot pickle code objects from closures") return _code_ctor, (co.co_argcount, co.co_nlocals, co.co_stacksize, co.co_flags, co.co_code, co.co_consts, co.co_names, co.co_varnames, co.co_filename, co.co_name, co.co_firstlineno, co.co_lnotab) def _tb_info(tb): while tb.tb_next is not None: tb = tb.tb_next return tb.tb_frame.f_code.co_filename, tb.tb_frame.f_lineno def _tb_stack(tb): sio = StringIO() traceback.print_tb(tb, None, sio) stack = sio.getvalue() sio.close() return stack # register the reductor to be used for pickling objects of type 'CodeType' copy_reg.pickle(CodeType, _reduce_code, _code_ctor) # test if an object is an iterable def _isiterable(x): try: iter(x) return True except TypeError: return False ############################################################################ # Worker code. # def cmdLaunch(verbose=0): """Execute sleigh tasks on behalf of the sleigh owner. cmdLaunch([verbose]) This function is intended for internal use only. Arguments: verbose -- Boolean flag used for displaying debug messages. The default value is 0. Environment Variables: PythonSleighNwsName -- Name of the sleigh workspace. PythonSleighUserNwsName -- Name of the user workspace. PythonSleighNwsHost -- Host name of the NWS server used by the sleigh master. The default value is 'localhost'. PythonSleighNwsPort -- Port number of the NWS server used by the sleigh master. The default value is 8765. PythonSleighName -- Name to use for monitoring/display purposes. The default value is the fully qualified domain name of the local machine. PythonSleighWorkerCount -- The total number of workers. PythonSleighRedirectOutput -- Specifies whether stdout and stderr should be redirected to a workspace variable. The default value is "True". """ # note that xbool works quite differently than bool. # xbool('False') -> False, whereas bool('False') -> True xbool = lambda s: s.lower().strip() == 'true' args = { 'PythonSleighNwsName': ('nwsName', str), 'PythonSleighUserNwsName': ('userNwsName', str), 'PythonSleighNwsHost': ('nwsHost', str), 'PythonSleighNwsPort': ('nwsPort', int), 'PythonSleighWorkerCount': ('maxWorkerCount', int), 'PythonSleighName': ('name', str), 'PythonSleighRank': ('rank', int), 'PythonSleighRedirectOutput': ('redirectOutput', xbool), } kw = {'verbose': verbose} for e, a in args.items(): try: kw[a[0]] = a[1](_Env[e]) except KeyError: pass print >> sys.stderr, "Calling launch:", kw launch(**kw) def launch(nwsName, nwsHost='localhost', nwsPort=8765, maxWorkerCount=-1, name=socket.getfqdn(), verbose=0, userNwsName='__default', rank=None, redirectOutput=True): """Execute sleigh tasks on behalf of the sleigh owner. launch(nwsName[, nwsHost, nwsPort, maxWorkerCount, name, verbose, userNwsName, redirectOutput]) This function is executed from the sleigh worker script, via cmdLaunch, but it can also be executed manually, using a command that was copied from the value of the 'runMe' variable in the sleigh workspace (this is refered to as 'web launch'). Arguments: nwsName -- Name of the sleigh workspace. nwsHost -- Host name of the NWS server used by the sleigh master. The default value is 'localhost'. nwsPort -- Port number of the NWS server used by the sleigh master. The default value is 8765. maxWorkerCount -- The maximum number of workers that should register themselves. The default value is -1, which means there is no limit, but that primarily intended for support 'web launch'. name -- Name to use for monitoring/display purposes. The default value is the fully qualified domain name of the local machine. rank -- worker rank, if we're using the new protocol verbose -- Boolean flag used for displaying debug messages. The default value is 0. userNwsName -- Name of the user workspace. The default value is '__default'. redirectOutput -- Boolean flag that indicates if stdout and stderr should be redirected to a workspace variable. """ nws = NetWorkSpace(nwsName, nwsHost, nwsPort, useUse=True, create=False) userNws = nws.server.useWs(userNwsName, create=False) if platform: worker_info = { 'host': platform.node(), 'os': platform.system(), 'pid': os.getpid(), 'python': platform.python_version(), 'nws': __version__, 'rank': rank, 'logfile': _Env.get('PythonSleighLogFile', '[unknown]')} else: worker_info = { 'host': socket.getfqdn(), 'os': sys.platform, 'pid': os.getpid(), 'python': sys.version, 'nws': __version__, 'rank': rank, 'logfile': _Env.get('PythonSleighLogFile', '[unknown]')} if rank is not None: new_protocol = True nws.store('worker_ids', str(rank), metadata=worker_info) else: new_protocol = False rank = nws.fetch('rankCount') worker_info['rank'] = rank nws.store('worker_info', worker_info) if rank < 0: nws.store('rankCount', rank) raise SleighJoinException('too late to join worker group') elif maxWorkerCount >= 0 and rank + 1 >= maxWorkerCount: # I think this handles a maxWorkerCount of zero, but it # shouldn't happen unless the user calls launch manually. nws.store('workerCount', maxWorkerCount) nws.store('rankCount', -1) else: nws.store('rankCount', rank + 1) # initialize for monitoring displayName = '%s@%d' % (name, rank) nws.declare(displayName, 'single') nws.store(displayName, '0') nodeList = nws.fetch('nodeList') if not nodeList: nodeList = displayName else: nodeList += ' ' + displayName nws.store('nodeList', nodeList) nws.store('workerDebug', displayName + ': Updated node list.') # wait for all workers to join (plugin does not wait) # XXX: Should I still do this with the plugin? I will answer your question # with another question. Do you feel like making fglob fetch # workerCount lazily? workerCount = nws.find('workerCount') nws.store('workerDebug', displayName + ': Getting ready to prepare to begin calling the worker loop.') # enter the main worker loop try: _workerLoop(nws, displayName, rank, workerCount, verbose, userNws, redirectOutput) except: nws.store('workerDebug', displayName + ': Something is rotten, my friends: ' + str(sys.exc_info())) class Logger(object): def __init__(self, ws, name, role): self._ws = ws self._name = name self._role = role def _logMsg(self, *s, **opt): try: msg = ' '.join([str(m) for m in s]) except Exception, e: msg = 'Exception while logging: %s' % str(e) print >> sys.__stderr__, msg if self._ws: try: logmsg = '[%s] %s %s -- %s' % \ (time.asctime(), self._name, self._role, msg) except Exception, e: logmsg = '[%s] %s %s -- Exception while logging: %s' % \ (time.asctime(), self._name, str(e)) self._ws.store(opt['var'], logmsg) def logError(self, *s): self._logMsg(*s, **{'var': 'logError'}) def logDebug(self, *s): self._logMsg(*s, **{'var': 'logDebug'}) def _workerLoop(nws, displayName, rank, workerCount, verbose, userNws, redirectOutput): """Repeatedly execute tasks on behalf of the sleigh. _workerLoop(nws, displayName, rank, workerCount, verbose, userNws) Arguments: nws -- NetWorkSpace object for the sleigh workspace. displayName -- Name to use for monitoring/display purposes. rank -- The integer rank of this worker. It acts as a unique ID within the sleigh. workerCount -- The total number of workers. A negative value means the number is unknown. verbose -- Boolean flag used for displaying debug messages. userNws -- NetWorkSpace object for the user workspace. redirectOutput -- Boolean flag that indicates if stdout and stderr should be redirected to a workspace variable. """ nws.store('workerDebug', displayName + ': Entering worker loop.') bx = 0 # monitoring stuff here tasks = 0 r = _Result() # create a global namespace fglob = { '__builtins__': globals()['__builtins__'], 'SleighRank': rank, 'SleighNws': nws, 'SleighUserNws': userNws, 'SleighWorkerCount': workerCount, } nws.store('workerDebug', displayName + ': Creating worker logger.') logger = Logger(nws, displayName, 'worker') if redirectOutput: f = nws.makefile('output@%d' % rank, 'w') sys.stdout = f sys.stderr = f # try to prevent Sleigh objects from being accidentally # created when modules are imported if not _Env.has_key('PythonSleighAllowed'): _Env['PythonSleighAllowed'] = 'FALSE' nws.store('workerDebug', displayName + ': Beginning task loop.') try: while 1: # update the number of tasks executed nws.store(displayName, str(tasks)) # get a task if verbose: logger.logDebug(nws, 'waiting for a task') t = nws.fetch('task') if verbose: logger.logDebug('got task %s' % repr(t.tag)) if not isinstance(t, _Task): # probably the sleigh modules are incompatible between # the master and this worker. return the task, because # maybe some other workers are compatible. logger.logError('found bad task: exiting') nws.store('task', t) break r.tag = t.tag r.value = None r.error = False if t.type == 'define' or t.type == 'invoke': # define the function if t.code: if t.funcName: try: # unpickle code object and put it into a function obj = cPickle.loads(t.code) f = new.function(obj, fglob) except Exception, e: r.value = 'Task definition from pickle failed: ' + str(e) r.error = True logger.logError(r.value) elif t.methName: try: # unpickle the object and get the method obj = cPickle.loads(t.code) f = getattr(obj, t.methName) except Exception, e: r.value = 'Task definition from pickle failed: ' + str(e) r.error = True logger.logError(r.value) else: logger.logError('found corrupt task: ignoring') continue else: try: # import the module __import__(t.module) module = sys.modules[t.module] # add our global variables to the function's global # namespace module.SleighRank = rank module.SleighNws = nws module.SleighUserNws = userNws module.SleighWorkerCount = workerCount # get the function from the module f = getattr(module, t.funcName) except ImportError, e: r.value = 'Unable to import module %s that defines worker ' \ 'function %s: %s' % \ (repr(t.module), repr(t.funcName), str(e)) r.error = True logger.logError(r.value) except AttributeError, e: r.value = 'Unable to find worker function %s at ' \ 'top level of module %s: %s' % \ (repr(t.funcName), repr(t.module), str(e)) r.error = True logger.logError(r.value) except Exception, e: try: exname = e.__class__.__name__ except: exname = '[unknown]' r.value = 'Error getting worker function %s from ' \ 'module %s - %s: %s' % \ (repr(t.funcName), repr(t.module), exname, str(e)) r.error = True logger.logError(r.value) if not r.error: if t.type == 'define': if t.funcName == _lambdaName: # XXX might want to support a way to specify # XXX an alternative name to use fglob['SleighFunction'] = f elif t.methName: # XXX kind of weird... fglob[t.methName] = f else: fglob[t.funcName] = f r.value = None else: # execute the function (or method) try: r.value = f(*t.args) except Exception, e: tb = sys.exc_info()[2] fname, lineno = _tb_info(tb) try: exname = e.__class__.__name__ except: exname = '[unknown]' r.value = 'Task invocation failed - %s: %s [%s:%d]' % \ (repr(exname), str(e), fname, lineno) r.error = True logger.logError(r.value) if verbose: # check if there is an attribute that provides # the "originating" exception if hasattr(e, 'exception'): logger.logError(str(e.exception)) logger.logError(_tb_stack(tb)) elif t.type == 'eval': try: # prepare the global namespace fglob['SleighArgs'] = t.args try: del fglob['SleighValue'] except: pass try: # see if we can treat it as an expression co = compile(t.code, '', 'eval') except: # it's not an expression, so compile it for use # with exec co = compile(t.code, '', 'exec') exec co in fglob # see if they want us to return a value r.value = fglob.get('SleighValue') else: # it compiled for 'eval', so now execute it r.value = eval(co, fglob) except Exception, e: tb = sys.exc_info()[2] fname, lineno = _tb_info(tb) r.value = 'Task evaluation failed: %s [%s:%d]' % \ (str(e), fname, lineno) r.error = True logger.logError(r.value) if verbose: logger.logError(_tb_stack(tb)) else: r.value = 'Task garbled: ' + t.type r.error = True logger.logError(r.value) # return the result try: if verbose: logger.logDebug('task %s completed' % repr(t.tag)) nws.store('result', r) except Exception, e: # maybe a pickle error: try to return an error message r.value = 'Error returning result: ' + str(e) r.error = True logger.logError(r.value) nws.store('result', r) tasks += 1 if t.barrier: if verbose: logger.logDebug('waiting at barrier') nws.find(_barrierNames[bx]) bx ^= 1 except Exception, e: # connection may corrupt, so just write to stderr sys.__stderr__.write('Worker exiting: %s\n' % str(e)) # traceback.print_exc() def getparam(name, opts, key, defkey='default', defval=None, coerce=str): try: p = opts[name] if isinstance(p, dict): try: p = p[key] except KeyError: p = p[defkey] except KeyError, e: if defval is None: raise e p = defval if coerce: p = coerce(p) return p def sshcmd(machine, **kw): socket.gethostbyname(machine) if not machine: raise SleighIllegalValueException('empty string specified in nodeList') try: ssh = which('ssh')[0] except IndexError: ssh = '/usr/bin/ssh' user = getparam('user', kw, machine, coerce=None) if user: return [ssh, '-n', '-x', '-l', str(user), machine] else: return [ssh, '-n', '-x', machine] def rshcmd(machine, **kw): socket.gethostbyname(machine) if not machine: raise SleighIllegalValueException('empty string specified in nodeList') try: rsh = which('rsh')[0] except IndexError: rsh = '/usr/bin/rsh' user = getparam('user', kw, machine, coerce=None) if user: return [rsh, machine, '-l', str(user), '-n'] else: return [rsh, machine, '-n'] def sshforwardcmd(machine, **kw): # this only seems to make sense if: # - we're ssh'ing to a different machine # - nwsHostRemote is one of remote machines's interfaces # (where localhost always is) socket.gethostbyname(machine) if not machine: raise SleighIllegalValueException('empty string specified in nodeList') r = '%s:%d:%s:%d' % \ (kw['nwsHostRemote'], kw['nwsPortRemote'], kw['nwsHost'], kw['nwsPort']) try: ssh = which('ssh')[0] except IndexError: ssh = '/usr/bin/ssh' user = getparam('user', kw, machine, coerce=None) if user: return [ssh, '-n', '-x', '-R', r, '-l', str(user), machine] else: return [ssh, '-n', '-x', '-R', r, machine] def rwincmd(machine, **kw): socket.gethostbyname(machine) if not machine: raise SleighIllegalValueException('empty string specified in nodeList') scriptDir = getparam('scriptDir', kw, machine, coerce=None) if scriptDir: vbscript = os.path.join(scriptDir, 'rwin.vbs') else: # scriptDir isn't set, so we'll try to guess where rwin.vbs is vbscript = _findScript('rwin.vbs') user = getparam('user', kw, machine, coerce=None) if not user: user = _Env.get('USER') or _Env.get('USERNAME') or _winuser or 'nwsuser' passwd = getparam('passwd', kw, machine, coerce=None) if not passwd: return ['cscript', '//nologo', vbscript, machine, '--'] else: return ['cscript', '//nologo', vbscript, machine, '-l', user, '-p', passwd, '--'] def _pathSplit(p): s = [] while True: p, b = os.path.split(p) if not b: if p: s.insert(0, p) break s.insert(0, b) return s def _startsWith(x, y): for a, b in izip(x, y): if a != b: return False return True # use heuristics to find the sleigh worker script def _findScript(scriptName): # first try to find scriptName by looking at sys.executable exedrive, exepath = os.path.splitdrive(sys.executable) exelist = [exedrive] + _pathSplit(exepath)[:-1] # remove file name, but include drive plist = [rule(exelist, *y) for pred, x, rule, y in _exeRules if pred(exelist, *x)] scriptPaths = which(scriptName, path=plist) if len(scriptPaths) > 0: scriptPath = scriptPaths[0] else: # next, try to find scriptName in your PATH scriptPaths = which(scriptName) if len(scriptPaths) > 0: scriptPath = scriptPaths[0] if len(scriptPaths) > 1: # don't give the warning if they're all the same file for p in scriptPaths[1:]: if not os.path.samefile(scriptPath, p): warn("found multiple copies of %s in PATH: using %s" % \ (scriptName, scriptPath)) break else: # finally, try to find scriptName in _scriptDirs scriptPaths = which(scriptName, path=_scriptDirs) if len(scriptPaths) > 0: scriptPath = scriptPaths[0] else: msg = 'unable to find %s in PATH: use Sleigh scriptDir option' % scriptName raise SleighScriptException(msg) return scriptPath def envcmd(machine, *envVars, **kw): # use the 'env' command so the script won't have to set it's # own environment. try: envcmd = which('env')[0] except IndexError: envcmd = '/usr/bin/env' envcmd = getparam('env', kw, machine, defval=envcmd) interp = getparam('interp', kw, machine, defval='') scriptDir = getparam('scriptDir', kw, machine, coerce=None) scriptName = getparam('scriptName', kw, machine) if interp: interp = [interp] else: interp = [] if scriptDir: scriptPath = os.path.join(scriptDir, scriptName) else: # they didn't tell us where to find the sleigh worker script # so we'll try to guess scriptPath = _findScript(scriptName) return [envcmd] + list(envVars) + interp + [scriptPath] def scriptcmd(machine, *envVars, **kw): # execute the script directly, passing environment # settings as arguments. # this is an example of how a script can define its own optional # parameters that can be machine specific. interp = getparam('interp', kw, machine, defval=sys.executable) scriptDir = getparam('scriptDir', kw, machine, coerce=None) scriptName = getparam('scriptName', kw, machine) if scriptDir: scriptPath = os.path.join(scriptDir, scriptName) else: # they didn't tell us where to find the sleigh worker script # so we'll try to guess scriptPath = _findScript(scriptName) return [interp, scriptPath] + list(envVars) def _getFuncInfo(fun): stacklevel = 4 # called from _newtask, eachElem funname = fun.__name__ try: modname = fun.__module__ except AttributeError: # should only be necessary in pre-python 2.3 modname = pickle.whichmodule(fun, funname) try: module = sys.modules[modname] except KeyError: warn('%s is not defined in an imported module' % funname, stacklevel=stacklevel) module = None if modname == '__main__': try: # try to get the name of the main program modpath, ext = os.path.splitext(sys.modules['__main__'].__file__) if ext == '.py': modname = os.path.basename(modpath) else: # can't import it if it doesn't have the '.py' extension warn('worker function defined in file without .py extension', stacklevel=stacklevel) warn('attempting to pickle the function code', stacklevel=stacklevel) modname = None except AttributeError: # __file__ attribute isn't defined when running interactively modname = None warn('cannot determine module from which to import %s' % funname, stacklevel=stacklevel) warn('attempting to pickle the function code', stacklevel=stacklevel) if modname and module and not hasattr(module, funname): # don't issue warning for lambdas if funname != _lambdaName: warn('module %s has no attribute named %s' % (modname, funname), stacklevel=stacklevel) warn('attempting to pickle the function code', stacklevel=stacklevel) modname = None return modname, funname # the following three functions are used for operating on # the accumulator objects def _accumresult(accum, results): try: try: accum.result(results) except AttributeError: accum(results) except Exception: traceback.print_exc() tags = ', '.join([str(r[0]) for r in results]) warn('error calling accumulator: result %s is lost' % tags, stacklevel=3) def _accumerror(accum, results): try: try: accum.error(results) except AttributeError: try: accum.result(results) except AttributeError: accum(results) except Exception: traceback.print_exc() tags = ', '.join([str(r[0]) for r in results]) warn('error calling accumulator: result %s is lost' % tags, stacklevel=3) def _accumvalue(accum): try: try: return accum.value() except AttributeError: return None except Exception: traceback.print_exc() warn('error calling value method of accumulator: returning None', stacklevel=3) return None # this is used by eachWorker, eachElem, etc, to create a new _Task object def _newtask(fun, taskType, barrier): if not isinstance(taskType, str): raise SleighIllegalValueException('"type" parameter must be a string') taskType = taskType.strip().lower() if taskType not in ['invoke', 'eval', 'define']: raise SleighIllegalValueException( '"type" parameter must be "invoke", "eval", or "define"') if isinstance(fun, str): if taskType != 'eval': raise SleighIllegalValueException( '"type" parameter must be "eval" for a string') else: # this may raise a SyntaxError fun = fun.lstrip() co = compile(fun, '', 'exec') elif isinstance(fun, (FunctionType, BuiltinFunctionType, MethodType)): if taskType == 'eval': raise SleighIllegalValueException( '"fun" argument must be a string if the task type is "eval"') if isinstance(fun, MethodType): # I think this is necessary if not hasattr(fun, '__module__'): raise SleighIllegalValueException( '"fun" argument is method without __module__ defined') if fun.__module__ == '__main__': raise SleighIllegalValueException( '"fun" argument is defined in module "__main__"') else: raise SleighIllegalValueException( '"fun" argument must be a function or a string') t = _Task() t.barrier = barrier t.type = taskType if taskType == 'eval': t.code = fun t.funcName = '' elif hasattr(fun, 'im_self') and hasattr(fun, 'im_func'): t.module = fun.__module__ if fun.im_self is None: raise SleighIllegalValueException( '"fun" argument cannot be an unbound method') t.code = cPickle.dumps(fun.im_self) t.methName = fun.im_func.__name__ else: t.module, t.funcName = _getFuncInfo(fun) # XXX we should require permission to do this if not t.module or t.funcName == _lambdaName: # try to pickle it t.code = cPickle.dumps(fun.func_code) return t defaultSleighOptions = { 'nwsHost': socket.getfqdn(), 'nwsHostRemote': None, 'nwsPort': 8765, 'nwsPortRemote': None, 'outfile': None, 'launch': 'local', 'workerCount': 3, 'nodeList': ['localhost', 'localhost', 'localhost'], 'scriptExec': (os.name == 'posix') and envcmd or scriptcmd, 'scriptDir': None, 'scriptName': 'PythonNWSSleighWorker.py', 'modulePath': None, 'workingDir': os.getcwd(), 'logDir': None, 'user': None, 'passwd': None, 'wsNameTemplate': 'sleigh_ride_%04d', 'userWsNameTemplate': 'sleigh_user_%04d', 'verbose': False, 'redirectOutput': True, } class Barrier(object): def __init__(self, ws, name): self.ws = ws self.name = name def leave(self, block=False): if block: self.ws.fetch(self.name) else: self.ws.fetchTry(self.name) def enter(self): self.ws.store(self.name, '1') def wait(self): self.ws.find(self.name) class NullBarrier(object): def __init__(self): pass def leave(self, block=False): pass def enter(self): pass def wait(self): pass class Accumulator(object): def __init__(self): self._val = {} def error(self, results): for tag, val in results: if self._val.has_key(tag): warn('got repeat result for task %d' % tag) else: self._val[tag] = SleighTaskException(val) def result(self, results): for tag, val in results: if self._val.has_key(tag): warn('got repeat result for task %d' % tag) else: self._val[tag] = val def value(self): return [self._val[i] for i in xrange(len(self._val))] class SleighPending(object): """Represents a sleigh eachWorker/eachElem invocation in progress. This is returned from the eachElem and eachWorker operations when executed asynchronously. It allows you to check for the number of tasks left to be executed, and to wait for the results of the operation to be returned. """ def __init__(self, nws, id, numTasks, barrier, my_sleigh, accumulator): """Create an object that represents the pending sleigh operation. This constructor is intended for internal use only. Arguments: nws -- Sleigh NetWorkSpace object. numTasks -- Number of tasks that were submitted. barrierName -- Name of the barrier to wait at when complete. sleighState -- Object representing the current state of the sleigh. accumulator -- Function to call with results as they arrive. """ self.nws = nws self.id = id self.numTasks = numTasks self.barrier = barrier self.my_sleigh = my_sleigh self.accumulator = accumulator self.done = False def __str__(self): return "SleighPending [%s]" % str(self.nws) def check(self): """Return the number of tasks still outstanding. p.check() -> integer """ if self.done: return 0 # could argue either way here... . # look for the count of result values. for l in self.nws.listVars().split('\n'): if l.startswith('result\t'): return self.numTasks - int(l.split('\t')[1]) # didn't find the variable 'result' --- assume no results have # yet been generated. return self.numTasks def wait(self): """Wait for and return the list of results. p.wait() -> list """ # note: a lot of code is duplicated here and in the # non-blocking sections of eachWorker and eachElem. refactor? if self.done: raise SleighGatheredException('results already gathered') self.sleigh.__retrieveResults(self.id, self.numTasks, self.accumulator) if self.barrier is not None: self.barrier.enter() self.sleigh.sleighState.occupied = False self.done = True return _accumvalue(self.accumulator) class Sleigh(object): """Represents a collection of python processes used to execute tasks. The sleigh allows python functions, methods, and expressions to be executed in parallel using the eachElem and eachWorker methods. The sleigh workers are started when the Sleigh object is constructed. When tasks are submitted to the sleigh, using the eachWorker and eachElem methods, the workers will execute the tasks, and return the results. When the stop method is called, the workers are stopped. Note that a given python program can create multiple Sleigh objects, which will each have it's own set of workers. This could be useful if tasks have different requirements. For example, you could create a Linux sleigh and a Windows sleigh, and submit Excel tasks only to your Windows sleigh. """ def __init__(self, *deprecated, **kw): """Start the remote python processes used to execute tasks. Keyword arguments: launch -- Specifies the method of starting workers. If this argument is set to the string 'local', then the workers are executed on the local machine. If it is set to the string 'web', then web launching is used. Otherwise, the launch argument must specify a function (such as nws.sleigh.sshcmd) that returns a list of command arguments use to execute the workers on the machines specified by the nodeList argument. The default value is 'local'. workerCount -- Number of workers to start if the launch argument is set to 'local' (which is the default value of launch). This argument is ignored if launch is not set to 'local'. The default value is 3. nodeList -- List of hosts on which to execute workers, if the launch argument is set to a function. This argument is ignored if launch is set to 'local' or 'web'. The default value is ['localhost', 'localhost', 'localhost']. nwsHost -- Host name of the machine where the NWS server is executing. nwsPort -- Port to connect to the NWS server. nwsHostRemote -- Host name of the machine that workers should use to connect to the NWS server. This is useful in conjunction with the sshforwardcmd function (see the description of scriptExec). The default is the value of the nwsHost argument. nwsPortRemote -- Port that workers should use to connect to the NWS server. This is useful in conjunction with the sshforwardcmd function (see the description of scriptExec). The default is the value of the nwsPort argument. scriptExec -- Python function returning a list of command arguments to execute the worker script. This list of command arguments is appended to the list returned by the launch function. The default value is the envcmd function (defined in this module), which uses the standard 'env' command to execute the script with the appropriate environment for the worker. scriptDir -- Directory on the worker that contains the execution script. scriptName -- Name of the script that executes the worker on the remote machines. This defaults to PythonNWSSleighWorker.sh on Unix, and PythonNWSSleighWorker.py on Windows. modulePath -- Directory path to add to sys.path on workers. This is often useful for giving the workers access to python modules that define the python function to be executed by eachWorker or eachElem. The default value is None. workingDir -- Directory path to use as the current working directory for the workers. The default value is the current working directory of the sleigh master. logDir -- Directory in which to create the worker log files. The default value is None, which leaves the decision to the sleigh worker scripts, which generally uses the remote system's temporary directory. user -- User name to use for remote execution of worker. This argument may be ignored, depending the specified launch function. The default is None, which allows the remote execution mechanism decide. wsNameTemplate -- Template for the sleigh workspace name. This must be a legal 'format' string, containing only an integer format specifier. The default is 'sleigh_ride_%04d'. userWsNameTemplate -- Template for the user's workspace name. This must be a legal 'format' string, containing only an integer format specifier. The default is 'sleigh_user_%04d'. verbose -- Boolean flag used for displaying debug messages. Debug messages will be sent to stderr. This will also cause the worker processes to write debug messages to files prefixed with 'sleigh_ride' in their current working directory (as controled by the workerDir argument). The default value is False. redirectOutput -- Boolean flag used to control whether messages written by the worker functions to stdout and stderr should be redirected to a workspace variable. If False, they will go to the output file if verbose is True, otherwise to the null device. The default value is True. """ # sanity check the keyword arguments unrecog = [a for a in kw.keys() if a not in defaultSleighOptions.keys()] if unrecog: raise TypeError( "__init__() got an unexpected keyword argument '%s'" % \ unrecog[0]) # check for a recursive sleigh construction if _Env.get('PythonSleighAllowed', 'TRUE') == 'FALSE': raise SleighNotAllowedException( 'Sleigh construction not allowed ' '(probably accidentally called from sleigh worker)') # join related self.rankCount = 0 # make a copy of the default sleigh options self.options = dict(defaultSleighOptions) # environment variables higher precedence than defaults try: self.options['nwsHost'] = _Env['PythonSleighNwsHost'] except: pass try: self.options['nwsPort'] = int(_Env['PythonSleighNwsPort']) except: pass # keyword arguments to constructor have highest precedence self.options.update(kw) # now that everyone has been heard from, if nwsHostRemote and # nwsPortRemote still aren't set, set them to the values of # nwsHost and nwsPort respectively if not self.options['nwsHostRemote']: self.options['nwsHostRemote'] = self.options['nwsHost'] if not self.options['nwsPortRemote']: self.options['nwsPortRemote'] = self.options['nwsPort'] opts = self.options # error check the options so we can throw any exceptions before # we've created the sleigh workspace, or caused any other side # effects. if not isinstance(opts['launch'], FunctionType) and \ opts['launch'] not in ('web', 'local', 'service'): raise SleighIllegalValueException('unknown launch protocol: ' + str(opts['launch'])) # check if a node list has been specified in the old way if len(deprecated) > 1: raise TypeError('__init__() takes exactly 1 argument (%d given)' % len(deprecated)) elif len(deprecated) == 1: if type(deprecated[0]) == type([]): warn('nodeList should be passed using the nodeList keyword', stacklevel=2) opts['nodeList'] = deprecated[0] if not isinstance(opts['launch'], FunctionType): warn('nodeList not used when launch = ' + opts['launch'], stacklevel=2) else: raise TypeError('__init__() takes exactly 1 argument (2 given)') try: socket.gethostbyname(opts['nwsHost']) except socket.gaierror, e: raise SleighIllegalValueException('bad nwsHost value "%s": %s' % (opts['nwsHost'], e.args[1])) # set up the sleigh's netWorkSpace. self.nwss = NwsServer(host=opts['nwsHost'], port=opts['nwsPort'], connopts={'MetadataToServer': '1', 'MetadataFromServer': '1'}) # last minute sanity checking of nodeList when using service launch if opts['launch'] == 'service': self.service = self.nwss.useWs('PySleighService', create=False) workers = self.service.listVars(format=DICT) for node in opts['nodeList']: try: w = workers[node] if w[V_MODE] != FIFO: self.nwss.close() raise SleighIllegalValueException('variable for node %s has illegal mode: %s' % (node, w[V_MODE])) if w[V_VALUES] > 0: warn('pending requests for node %s' % node, stacklevel=2) if w[V_FETCHERS] == 0: warn('worker may not be running on node %s' % node, stacklevel=2) elif w[V_FETCHERS] > 1: warn('multiple workers for node %s' % node, stacklevel=2) if w[V_FINDERS] > 0: warn('unknown finders present for node %s' % node, stacklevel=2) except KeyError: self.nwss.close() raise SleighIllegalValueException('bad node %s in nodeList: %s' % (node, 'no such worker registered')) self.nwsName = self.nwss.mktempWs(opts['wsNameTemplate'], metadata={'wstype': 'sleigh'}) if not self.nwsName: self.nwss.close() raise SleighIllegalValueException('bad wsNameTemplate value "%s": %s' % (opts['nwsHost'], e.args[1])) self.nws = self.nwss.openWs(self.nwsName) self.userNwsName = self.nwss.mktempWs(opts['userWsNameTemplate']) if not self.userNwsName: self.nwss.close() raise SleighIllegalValueException('bad userWsNameTemplate value "%s": %s' % (opts['nwsHost'], e.args[1])) self.userNws = self.nwss.openWs(self.userNwsName) self.__initialize(self.nwss.get_protocol_version()) self.__launch_workers(opts) self.state = _SleighState() self.state.bx = 0 self.state.occupied = False self.state.stopped = False self.state.totalTasks = 0 def __launch_workers(self, opts): if isinstance(opts['launch'], FunctionType): # remote launch self.nodeList = opts['nodeList'] self.workerCount = len(self.nodeList) for i in xrange(self.workerCount): if opts['verbose']: opts['outfile'] = '%s_%04d.txt' % (self.nwsName, i) self._addWorker(self.nodeList[i], i) elif opts['launch'] == 'service': # remote launch using the "Python Sleigh Service" self.nodeList = opts['nodeList'] self.workerCount = len(self.nodeList) b = lambda x: x and str(x) or '' # turn None into '' for i in xrange(self.workerCount): if opts['verbose']: opts['outfile'] = '%s_%04d.txt' % (self.nwsName, i) # XXX is '@' the best delimiter? # XXX this should include userNwsName request = "@%s@%d@%d@%s@%s@%s@%s@%s" % \ (self.nwsName, self.workerCount, i, b(opts['workingDir']), b(opts['outfile']), b(opts['logDir']), b(opts['user']), b(opts['modulePath'])) if opts['verbose']: print 'command:', request # XXX error check to make sure the worker is listed in workspace? # XXX or issue a warning message? self.service.store(self.nodeList[i], request) elif opts['launch'] == 'local': # local launch self.workerCount = int(opts['workerCount']) self.nodeList = self.workerCount * ['localhost'] for i in xrange(self.workerCount): if opts['verbose']: opts['outfile'] = '%s_%04d.txt' % (self.nwsName, i) self._addWorker(self.nodeList[i], i) elif opts['launch'] == 'web': # web launch self.nws.store('runMe', 'import nws.sleigh; nws.sleigh.launch(%s, %s, %s, userNwsName=%s)' % \ (repr(self.nwsName), repr(opts['nwsHost']), repr(opts['nwsPort']), repr(self.userNwsName))) # store some more values (probably only one) using # dotted-decimal ip addresses in case there are host name # resolution problems on the worker. try: ipaddrs = socket.gethostbyname_ex(opts['nwsHost'])[2] for ipaddr in ipaddrs: self.nws.store('runMe', 'import nws.sleigh; nws.sleigh.launch(%s, %s, %s, userNwsName=%s)' % \ (repr(self.nwsName), repr(ipaddr), repr(opts['nwsPort']), repr(self.userNwsName))) except: # this seems highly unlikely to happen, but it seems # wrong to allow something optional to blow us up pass try: self.nws.fetch('deleteMeWhenAllWorkersStarted') except: pass self.nws.deleteVar('runMe') self.workerCount = self.nws.fetch('rankCount') if not self.new_protocol: self.nws.store('workerCount', self.workerCount) self.nws.store('rankCount', -1) self.rankCount = -1 self.nodeList = self.nws.fetchTry('nodeList', '') else: # the previous error checking should prevent this from # ever happening warn("internal error: unknown launch protocol: " + str(opts['launch'])) self.workerCount = 0 self.rankCount = -1 self.nodeList = '' def __initialize(self, proto_version): v = self.nws.findTry('version') if v is None: proto_version = PROTOCOL_VERSION_OLD if proto_version == PROTOCOL_VERSION_OLD: self.__initialize_old() elif proto_version == PROTOCOL_VERSION_NEW_0: print 'New protocol' self.__initialize_new_0() else: raise Exception('Unknown protocol version') def __initialize_old(self): self.new_protocol = False # initialize for monitoring self.nws.declare('nodeList', 'single') self.nws.store('nodeList', '') self.nws.declare('totalTasks', 'single') self.nws.store('totalTasks', '0') self.nws.declare('rankCount', 'single') self.nws.store('rankCount', 0) self.nws.declare('workerCount', 'single') self.__statusImpl = self.__status_old self._syncWorkerCount = self.__syncWorkerCount_old self.__storeBroadcastTask = self.__storeBroadcastTask_old self.__newBatch = lambda: '' self.__barriers = [Barrier(self.nws, _barrierNames[i]) for i in range(2)] def __initialize_new_0(self): self.nws.declare('nodeList', 'single') self.nws.store('nodeList', '') self.__nextBatch = 0 self.new_protocol = True if self.options['launch'] != 'web': self.nws.store('workerCount', '%d' % len(self.options['nodeList'])) self.__statusImpl = self.__status_new_0 self._syncWorkerCount = self.__syncWorkerCount_new_0 self.__storeBroadcastTask = self.__storeBroadcastTask_new_0 self.__newBatch = self.__newBatch_new self.__barriers = [NullBarrier() for i in range(2)] def __newBatch_new(self): id = '%d' % self.__nextBatch self.__nextBatch += 1 return id def __storeBroadcastTask_old(self, wc, task): # update the total number of submitted tasks self.state.totalTasks += wc self.nws.store('totalTasks', str(self.state.totalTasks)) for i in xrange(wc): task.tag = i self.nws.store('task', task) # no batch ids in old versions return '' def __storeBroadcastTask_new_0(self, wc, task): id = self.__newBatch() self.nws.store('broadcast', task, metadata={'batchId': id}) return id def __retrieveResults(self, id, ntasks, accum): for i in xrange(ntasks): r = self.nws.fetch('result', metadata={'batchId': id}) if r.error: _accumerror(accum, [(r.tag, r.value)]) else: _accumresult(accum, [(r.tag, r.value)]) def __str__(self): return "Sleigh [%s] on nodes: %s" % \ (str(self.nws), " ".join(self.nodeList)) def _addWorker(self, machine, id): # basic idea is (or should be): if we can get the appropriate # sleighworker script running on the remote node, we # just need to give it enough env info to take care of the rest # prepare the environment envVars = [] envVars.append('PythonSleighNwsHost=' + self.options['nwsHostRemote']) envVars.append('PythonSleighNwsName=' + self.nwsName) envVars.append('PythonSleighUserNwsName=' + self.userNwsName) envVars.append('PythonSleighNwsPort=' + str(self.options['nwsPortRemote'])) if self.options['workingDir']: envVars.append('PythonSleighWorkingDir=' + self.options['workingDir']) if self.options['scriptDir']: envVars.append('PythonSleighScriptDir=' + self.options['scriptDir']) envVars.append('PythonSleighScriptName=' + self.options['scriptName']) if self.options['modulePath']: envVars.append('PythonSleighModulePath=' + self.options['modulePath']) if self.options['outfile']: envVars.append('PythonSleighWorkerOut=' + self.options['outfile']) if self.options['logDir']: envVars.append('PythonSleighLogDir=' + self.options['logDir']) envVars.append('PythonSleighName=' + machine) envVars.append('PythonSleighWorkerCount=' + str(self.workerCount)) envVars.append('PythonSleighID=' + str(id)) envVars.append('PythonSleighRedirectOutput=' + str(self.options['redirectOutput'])) # remote execution command if self.options['launch'] == 'local': argv = [] # this environment variable is only needed and used by # PythonNWSSleighWorker.sh, not the python version envVars.append('PythonProg=' + sys.executable) else: argv = self._getargs('launch', self.options['launch'], machine, **self.options) # environment setting command argv += self._getargs('scriptExec', self.options['scriptExec'], machine, *envVars, **self.options) # now actually execute the worker on the remote machine if not os.path.isabs(argv[0]): try: argv[0] = which(argv[0])[0] except IndexError: warn("unable to convert %s to absolute path" % argv[0], stacklevel=3) try: if self.options['verbose']: print 'command:', argv subprocess.Popen(argv) except NameError: app = argv[0] if sys.platform.startswith('win'): warn("Python 2.4 is _strongly_ recommended on Windows", stacklevel=3) argv = [msc_quote(a) for a in argv] if self.options['verbose']: print 'MS C quoted command:', argv os.spawnv(os.P_NOWAIT, app, argv) def _getargs(self, name, cmd, *args, **kw): if not isinstance(cmd, FunctionType): raise SleighIllegalValueException(name + ' must be set to a function') x = cmd(*args, **kw) if not isinstance(x, list): raise SleighIllegalValueException(name + ' does not yield a list') for i in x: if not isinstance(i, str): raise SleighIllegalValueException(name + ' does not yield a list of strings') return x def status(self, closeGroup=False, timeout=0.0): """Return the status of the worker group. s.status(closeGroup, timeout) -> numworkers, closed The status includes the number of workers that have joined the group so far, and a flag that indicates whether the group has been closed (meaning that no more workers can join). Normally, the group is automatically closed when all the workers that were listed in the constructor have joined. However, this method allows you to force the group to close after the timeout expires. This can be particularly useful if you are running on a large number of nodes, and some of the nodes are slow or unreliable. If some of the workers are never started, the group will never close, and no tasks will ever execute. Arguments: closeGroup -- Boolean flag indicating whether to close the group. If True, the group will be closed, after the specified timeout. The default value is False. timeout -- Number of seconds to wait for the group to close before returning. The default value is 0.0. """ if self.rankCount < 0: # join phase completed before return self.workerCount, 1 else: return self.__statusImpl(closeGroup, timeout) def __status_new_0(self, closeGroup, timeout): if closeGroup: self.workerCount = self.nws.fetch('status', metadata={'delay': str(timeout)}) self.rankCount = -1 return self.workerCount, 1 else: status = self.nws.fetch('join_status') if status == 'closed': self.workerCount = self.nws.fetch('status', metadata={'delay': str(timeout)}) self.rankCount = -1 return self.workerCount, 1 else: return self.nws.fetch('waitForWorkers', metadata={'delay': str(timeout)}), 0 def __status_old(self, closeGroup, timeout): sleepTime = initialSleep = min(2.0, timeout) repeatedSleep = 1.0 startTime = time.time() while 1: n = self.nws.fetch('rankCount') if n < 0: # all workers joined if self.options['verbose']: print 'all %d workers have started' % self.workerCount assert self.workerCount == self.nws.find('workerCount') self.rankCount = -1 self.nws.store('rankCount', self.rankCount) return self.workerCount, 1 else: # three choices: close now, return not closed, or # sleep and try again if time.time() - startTime >= timeout: # time is up, so either close the join, or # return the current status if closeGroup: if self.options['verbose']: print 'closing group: %d workers' % n self.workerCount = n self.nws.store('workerCount', self.workerCount) self.rankCount = -1 self.nws.store('rankCount', self.rankCount) return self.workerCount, 1 else: if self.options['verbose']: print 'group not formed: %d workers' %n self.rankCount = n self.nws.store('rankCount', self.rankCount) return self.rankCount, 0 else: if self.options['verbose']: print 'only %d workers: sleeping...' % n self.rankCount = n self.nws.store('rankCount', n) time.sleep(sleepTime) sleepTime = repeatedSleep def stop(self): """Stop the remote processes and delete the sleigh workspace. s.stop() """ if self.state.stopped: return self.nws.store('Sleigh ride over', 1) if self.options['launch'] != 'web': time.sleep(3) exitCount = 0 while self.nws.fetchTry('bye', None) != None: exitCount += 1 if exitCount < self.workerCount: warn('only %d of %d workers have exited' % \ (exitCount, self.workerCount), stacklevel=2) self.nwss.deleteWs(self.nwsName) self.nwss.close() self.state.stopped = True def __syncWorkerCount_old(self): if self.rankCount != -1: self.rankCount = self.nws.find('rankCount') if self.rankCount == -1: self.workerCount = int(self.nws.find('workerCount')) return self.workerCount def __syncWorkerCount_new_0(self): if self.rankCount != -1: self.workerCount = int(self.nws.find('workerCount')) return self.workerCount def eachWorker(self, fun, *workerArgs, **kw): """Execute a function, method, or expression on each worker of sleigh. s.eachWorker(fun[, ...]) -> list or SleighPending The results are normally returned as a list, unless the blocking arguments is set to False, in which case, a SleighPending object is returned. Arguments: fun -- Function, method, or python expression to execute. To execute a method, you must specify a bound method object. If the function or defining class is defined in a module, the workers will attempt to import that module. If that module isn't available to the worker (because it's a non-standard module, not in the PYTHONPATH), then the worker is not be able to execute those tasks. To execute a python expression, you must specify it as a string. Leading whitespace is automatically stripped to avoid a common source of python syntax errors. Optional arguments: *workerArgs -- Arguments to pass to the function or method. Specify whatever arguments the function requires, including no arguments. The exact same set of arguments will be used for each worker (unlike eachElem). For a python expression, these arguments are passed to the expression as a global variable named 'SleighArguments'. Keyword arguments: type -- Indicates the type of function invocation to perform. This can be either 'invoke', 'define', or 'eval'. If the fun argument is a function or bound method, then the default value is 'invoke'. If the fun argument is a string, then the default value is 'eval' (a value of 'invoke' or 'define' is illegal for python expressions). blocking -- A boolean value that indicates whether to wait for the results, or to return as soon as the tasks have been submitted. If set to False, eachWorker will return a SleighPending object that is used to monitor the status of the tasks, and to eventually retrieve the results. You must wait for the results to be complete before executing any further tasks on the sleigh, or a SleighOccupiedException will be raised. This argument is important if you want the master to be able to interact/communicate with the workers, via NWS operations, for example. This allows you to implement more complex parallel or distributed programs. The default value is True. accumulator -- A function or callable object that will be called for each result as they arrive. The first argument to the function is a list of result values, and the second argument is a list of indexes, which identifies which task. The arguments to the accumulator function are lists since in the future, we plan to allow tasks to be "chunked" to improve performance of small tasks. Note that bound methods can be very useful accumulators. """ if self.state.occupied: raise SleighOccupiedException('sleigh is occupied') if self.state.stopped: raise SleighStoppedException('sleigh is stopped') blocking = kw.get('blocking', True) accumulator = kw.get('accumulator', Accumulator()) if isinstance(fun, str): taskType = kw.get('type', 'eval') else: taskType = kw.get('type', 'invoke') # verify that accumulator is a callable object or an object with a result method if not callable(accumulator) and not hasattr(accumulator, 'result'): raise SleighIllegalValueException('accumulator must be callable or have a callable result attribute') t = _newtask(fun, taskType, True) self._syncWorkerCount() nws = self.nws wc = self.workerCount # use alternating barrier to sync eachWorker invocations with the workers. cur_barrier = self.__barriers[self.state.bx] self.state.bx ^= 1 cur_barrier.leave() t.args = list(workerArgs) id = self.__storeBroadcastTask(wc, t) if not blocking: self.state.occupied = not self.new_protocol return SleighPending(nws, id, wc, cur_barrier, self, accumulator) self.__retrieveResults(id, wc, accumulator) cur_barrier.enter() return _accumvalue(accumulator) # run fun once for each element of a vector. def eachElem(self, fun, elementArgs=[[]], fixedArgs=[], **kw): """Execute a function, method, or expression for each element in the specified list(s). s.eachElem(fun, elementArgs[, fixedArgs]) -> list or SleighPending The results are normally returned as a list, unless the blocking arguments is set to False, in which case, a SleighPending object is returned. Arguments: fun -- Function, method, or python expression to execute. To execute a method, you must specify a bound method object. If the function or defining class is defined in a module, the workers will attempt to import that module. If that module isn't available to the worker (because it's a non-standard module, not in the PYTHONPATH), then the worker is not be able to execute those tasks. To execute a python expression, you must specify it as a string. Leading whitespace is automatically stripped to avoid a common source of python syntax errors. elementArgs -- List of arguments to pass to the function or method that need to be different for different tasks. In general, this is a list of iterable objects, such as lists, each containing the values to use for a given argument of the different tasks. If your function needs only one varying argument of a simple type, you can specify it without the outer list. Note that for a python expression, the list of arguments is passed to the expression as a global variable named 'SleighArguments'. fixedArgs -- List of additional arguments that are fixed/constant for each task. Normally, they are appended to the arguments specified by elementArgs, but the order can be altered using the argPermute argument described below. The default value is an empty list, which means that no extra arguments are passed to the function. Note that for a python expression, the list of arguments is passed to the expression as a global variable named 'SleighArguments'. Keyword arguments: type -- Indicates the type of function invocation to perform. This can be either 'invoke', 'define', or 'eval'. If the fun argument is a function or bound method, then the default value is 'invoke'. If the fun argument is a string, then the default value is 'eval' (a value of 'invoke' or 'define' is illegal for python expressions). blocking -- A boolean value that indicates whether to wait for the results, or to return as soon as the tasks have been submitted. If set to False, eachElem will return a SleighPending object that is used to monitor the status of the tasks, and to eventually retrieve the results. You must wait for the results to be complete before executing any further tasks on the sleigh, or a SleighOccupiedException will be raised. If blocking is set to False, then the loadFactor argument is disabled and ignored. Note that it's unlikely that you'll need to turn off blocking in eachElem. Non-blocking mode is more useful in eachWorker. The default value is True. argPermute -- List that maps the specified arguments to the actual arguments of the execution function. By "specified arguments", I mean the items extracted from elementArgs, followed by fixedArgs. (Note that unless you are specifying both elementArgs and fixedArgs, you probably don't need to use argPermute.) The items in the argPermute list are used as indexes into the "specified arguments" list. The length of argPermute determines the number of arguments passed to the execution function, which would normally be the length of the specified arguments list, but this is not required. For example, setting argPermute to an empty list would cause the execution function to be called without any arguments (although elementArgs would still be required, and would be used to determine the number of tasks to execute). The default behaviour is to pass the execution function the arguments specified by elementArgs, followed by the arguments from fixedArgs, which is equivalent to setting argPermute to: n = len(elementArgs) + len(fixedArgs) argPermute = range(n) If you wished to reverse the order of the arguments, you could then modify argPermute: argPermute.reverse() But, more realistically, you need to interleave the fixed arguments with the varying arguments. For example, your execution function takes on fixed argument, followed by two that vary, you would set argPermute to: argPermute=[1,2,0] loadFactor -- Maximum number of tasks per worker to put into the workspace at one time. This can become important if you are executing a very large number of tasks. Setting loadFactor to 3 will probably keep enough tasks available in the workspace to keep the workers busy, without flooding the workspace and bogging down the NWS server. The default behaviour is to submit all of the tasks to the sleigh workspace immediately. accumulator -- A function or callable object that will be called for each result as they arrive. The first argument to the function is a list of result values, and the second argument is a list of indexes, which identifies which task. The arguments to the accumulator function are lists since in the future, we plan to allow tasks to be "chunked" to improve performance of small tasks. Note that bound methods can be very useful accumulators. """ if self.state.occupied: raise SleighOccupiedException('sleigh is occupied') if self.state.stopped: raise SleighStoppedException('sleigh is stopped') if isinstance(elementArgs, list): # it's a list, but if it doesn't contain an iterable, # then wrapped it in a list if len(elementArgs) > 0: if not _isiterable(elementArgs[0]): elementArgs = [elementArgs] else: raise SleighIllegalValueException('elementArgs is an empty list') else: # it's not a list, so wrap it in a list elementArgs = [elementArgs] # make sure that elementArgs is now a list of iterables for xv in elementArgs: if not _isiterable(xv): raise SleighIllegalValueException('elementArgs must be an iterable or a list of iterables') if not isinstance(fixedArgs, list): fixedArgs = [fixedArgs] argPermute = kw.get('argPermute', None) blocking = kw.get('blocking', True) loadFactor = kw.get('loadFactor', 0) accumulator = kw.get('accumulator', Accumulator()) if isinstance(fun, str): taskType = kw.get('type', 'eval') else: taskType = kw.get('type', 'invoke') if argPermute is not None and not isinstance(argPermute, list): raise SleighIllegalValueException('argPermute must be a list') # verify that accumulator is a callable object or an object with a result method if not callable(accumulator) and not hasattr(accumulator, 'result'): raise SleighIllegalValueException('accumulator must be callable or have a callable result attribute') t = _newtask(fun, taskType, False) nws = self.nws # create an "enumerator" for the task data taskIter = izip(count(), *elementArgs) # allocate a batch id for the tasks id = self.__newBatch() md = {'batchId': id} # loadFactor is ignored if non-blocking if blocking and loadFactor > 0: wc = self._syncWorkerCount() taskLimit = xrange(max(loadFactor * wc, wc)) else: taskLimit = count() # no limit # task args generator argsgen = ((taskData[0], (list(taskData[1:])+fixedArgs)) for taskData in taskIter) if argPermute is not None: argsgen = ((td[0], [td[1][j] for j in argPermute]) for td in argsgen) # submit tasks, possibly limited by loadFactor initialSubmission = 0 for ignore, taskData in izip(taskLimit, argsgen): t.tag = taskData[0] t.args = taskData[1] nws.store('task', t, md) initialSubmission += 1 # update the total number of submitted tasks if not self.new_protocol: self.state.totalTasks += initialSubmission nws.store('totalTasks', str(self.state.totalTasks)) # return a SleighPending object if we're non-blocking if not blocking: self.state.occupied = not self.new_protocol return SleighPending(nws, id, initialSubmission, None, self, accumulator) # start retrieving results and finish submitting tasks # which is needed when loadFactor > 0 numTasks = initialSubmission for taskData in argsgen: # fetch a result r = nws.fetch('result', md) if r.error: _accumerror(accumulator, [(r.tag, r.value)]) else: _accumresult(accumulator, [(r.tag, r.value)]) # submit next task t.tag = taskData[0] t.args = taskData[1] nws.store('task', t, md) numTasks += 1 # update the total number of submitted tasks if not self.new_protocol: self.state.totalTasks += 1 nws.store('totalTasks', str(self.state.totalTasks)) # finish retrieving results for i in xrange(initialSubmission): # fetch a result r = nws.fetch('result', md) if r.error: _accumerror(accumulator, [(r.tag, r.value)]) else: _accumresult(accumulator, [(r.tag, r.value)]) return _accumvalue(accumulator) def imap(self, fun, *iterables, **kw): """Return an iterator whose values are returned from the function evaluated with a argument tuple taken from the given iterable. Stops when the shortest of the iterables is exhausted. s.imap(fun, *iterables[, **kw]) -> SleighResultIterator This is intended to be very similar to the imap function in the itertools module. Other than being a method, rather than a function, this method takes additional, keyword arguments, and the iterator that is returned has special methods and properties. See the SleighResultIterator documentation for more information. Arguments: fun -- Function, method, or python expression to execute. To execute a method, you must specify a bound method object. If the function or defining class is defined in a module, the workers will attempt to import that module. If that module isn't available to the worker (because it's a non-standard module, not in the PYTHONPATH), then the worker is not be able to execute those tasks. To execute a python expression, you must specify it as a string. Leading whitespace is automatically stripped to avoid a common source of python syntax errors. *iterables -- One or more iterables, one for each argument needed by the function. Keyword arguments: type -- Indicates the type of function invocation to perform. This can be either 'invoke', 'define', or 'eval'. If the fun argument is a function or bound method, then the default value is 'invoke'. If the fun argument is a string, then the default value is 'eval' (a value of 'invoke' or 'define' is illegal for python expressions). loadFactor -- Maximum number of tasks per worker to put into the workspace at one time. This can become important if you are executing a very large number of tasks (and essential if submitting infinite tasks). The default value of 10 will probably keep enough tasks available in the workspace to keep the workers busy, without flooding the workspace and bogging down the NWS server. """ if self.state.occupied: raise SleighOccupiedException('sleigh is occupied') if self.state.stopped: raise SleighStoppedException('sleigh is stopped') # make sure that iterables is a list of iterables for it in iterables: if not _isiterable(it): raise SleighIllegalValueException('arguments must be iterables') loadFactor = kw.get('loadFactor', 10) if isinstance(fun, str): taskType = kw.get('type', 'eval') else: taskType = kw.get('type', 'invoke') t = _newtask(fun, taskType, False) nws = self.nws # create an "enumerator" for the task data taskIter = izip(count(), *iterables) # loadFactor is ignored if non-blocking if loadFactor > 0: wc = self._syncWorkerCount() taskLimit = xrange(max(loadFactor * wc, wc)) else: taskLimit = count() # no limit # submit tasks, possibly limited by loadFactor initialSubmission = 0 for ignore, taskData in izip(taskLimit, taskIter): t.tag = taskData[0] t.args = list(taskData[1:]) nws.store('task', t) initialSubmission += 1 # update the total number of submitted tasks self.state.totalTasks += initialSubmission nws.store('totalTasks', str(self.state.totalTasks)) self.state.occupied = True return SleighResultIterator(t, taskIter, False, nws, initialSubmission, self.state) def starmap(self, fun, iterable, **kw): """Return an iterator whose values are returned from the function evaluated with a argument tuple taken from the given iterable. Stops when the shortest of the iterables is exhausted. s.starmap(fun, iterable[, **kw]) -> SleighResultIterator This is intended to be very similar to the starmap function in the itertools module. Other than being a method, rather than a function, this method takes additional, optional arguments, and the iterator that is returned has special methods and properties. See the SleighResultIterator documentation for more information. Arguments: fun -- Function, method, or python expression to execute. To execute a method, you must specify a bound method object. If the function or defining class is defined in a module, the workers will attempt to import that module. If that module isn't available to the worker (because it's a non-standard module, not in the PYTHONPATH), then the worker is not be able to execute those tasks. To execute a python expression, you must specify it as a string. Leading whitespace is automatically stripped to avoid a common source of python syntax errors. iterable -- Returns argument tuples for the function. Keyword arguments: type -- Indicates the type of function invocation to perform. This can be either 'invoke', 'define', or 'eval'. If the fun argument is a function or bound method, then the default value is 'invoke'. If the fun argument is a string, then the default value is 'eval' (a value of 'invoke' or 'define' is illegal for python expressions). loadFactor -- Maximum number of tasks per worker to put into the workspace at one time. This can become important if you are executing a very large number of tasks (and essential if submitting infinite tasks). The default value of 10 will probably keep enough tasks available in the workspace to keep the workers busy, without flooding the workspace and bogging down the NWS server. """ if self.state.occupied: raise SleighOccupiedException('sleigh is occupied') if self.state.stopped: raise SleighStoppedException('sleigh is stopped') # make sure that iterable is an iterable if not _isiterable(iterable): raise SleighIllegalValueException('iterable must be an iterable') loadFactor = kw.get('loadFactor', 10) if isinstance(fun, str): taskType = kw.get('type', 'eval') else: taskType = kw.get('type', 'invoke') t = _newtask(fun, taskType, False) nws = self.nws # create an "enumerator" for the task data taskIter = izip(count(), iterable) # loadFactor is ignored if non-blocking if loadFactor > 0: wc = self._syncWorkerCount() taskLimit = xrange(max(loadFactor * wc, wc)) else: taskLimit = count() # no limit # submit tasks, possibly limited by loadFactor initialSubmission = 0 for ignore, taskData in izip(taskLimit, taskIter): t.tag = taskData[0] t.args = list(taskData[1]) nws.store('task', t) initialSubmission += 1 # update the total number of submitted tasks self.state.totalTasks += initialSubmission nws.store('totalTasks', str(self.state.totalTasks)) self.state.occupied = True return SleighResultIterator(t, taskIter, True, nws, initialSubmission, self.state) def workerInfo(self): """Return information about the sleigh workers. This returns a list of dictionaries, one for each worker. Note that if this is called before all of the workers have "joined", then you may get a partial list. This can be useful in conjunction with the status method. The keys in the dictionaries are: "nws": nws version used by the worker "python": Python version used by the worker "pid": process id of the worker process "rank": worker rank (0 to workerCount - 1) "host": name of worker machine (not necessarily the same as the value in nodeList) "logfile": path of the log file on the worker machine "os": operating system of the worker machine """ # get all of the "worker info" values info = list(self.nws.ifindTry('worker info')) # sort the list by rank to avoid surprising the caller aux = [(info[i]['rank'], i) for i in range(len(info))] aux.sort() return [info[i] for rank, i in aux] def view(self, port=8766, ws='system'): if webbrowser: if ws == 'system': nws = self.nws elif ws == 'user': nws = self.userNws else: raise SleighIllegalValueException( 'illegal value specified for "ws" argument: %s' % ws) vstring = "http://%s:%d/doit?op=listVars&wsName=%s" % \ (nws.server.serverHost, port, quote_plus(nws.curWs)) webbrowser.open_new(vstring) else: raise SleighUnsupportedMethodException('cannot import webbrowser module') class SleighResultIterator(object): """Returns results from tasks submitted to the sleigh. Instances of this class are returned from the Sleigh imap, and starmap methods. """ def __init__(self, task, taskIter, starIter, nws, numSubmitted, sleighState): """Create an iterator over the task results. This constructor is intended for internal use only. Arguments: task -- Partially initialized Task object. taskIter -- Iterator over the task arguments. starIter -- Is this a "star" iteration? nws -- Sleigh NetWorkSpace object. numSubmitted -- Number of tasks already submitted. sleighState -- Part of the Sleigh objects internal state. """ self._task = task self._taskIter = taskIter self._starIter = starIter self._nws = nws self._numSubmitted = numSubmitted self._sleighState = sleighState self._buffer = {} self._nextTag = 0 self._doneSubmitting = False def __iter__(self): return self def next(self): if self._nextTag >= self._numSubmitted: raise StopIteration() try: # see if we already have the next result r = self._buffer.pop(self._nextTag) except KeyError: while True: # fetch a result r = self._nws.fetch('result') if not self._doneSubmitting: try: # submit next task taskData = self._taskIter.next() self._task.tag = taskData[0] if self._starIter: self._task.args = list(taskData[1]) else: self._task.args = list(taskData[1:]) self._nws.store('task', self._task) # update the total number of submitted tasks self._sleighState.totalTasks += 1 self._nws.store('totalTasks', str(self._sleighState.totalTasks)) self._numSubmitted += 1 except StopIteration: self._doneSubmitting = True if r.tag == self._nextTag: break self._buffer[r.tag] = r self._nextTag += 1 if self._nextTag >= self._numSubmitted: self._sleighState.occupied = False if r.error: v = SleighTaskException(r.value) else: v = r.value return v def returned(self): return self._nextTag returned = property(returned, doc='Number of task results returned.') def buffered(self): return len(self._buffer) buffered = property(buffered, doc='Number of buffered task results.') def submitted(self): return self._numSubmitted submitted = property(submitted, doc='Number of submitted tasks.') def stopped(self): return self._nextTag >= self._numSubmitted stopped = property(stopped, doc='Is the iterator stopped?') def shutdown(self): """Stop submitting tasks from the iterator. This method is a less drastic version of "stop". It is expected that you will keep retrieving results that have already been submitted, but no new tasks will be submitted, regardless of what tasks were originally specified to imap or starmap. The sleigh object will continue to be "occupied" until all results of the pending tasks have been retreived. """ self._doneSubmitting = True def stop(self): """Stop the iterator, flushing any pending results. This method is useful if you're done with the iterator, and don't want to retrieve anymore results. After calling stop, you can submit more tasks to the sleigh (that is, it will no longer be "occupied". """ if self._nextTag < self._numSubmitted: self._nextTag += len(self._buffer) for i in xrange(self._numSubmitted - self._nextTag): self._nws.fetch('result') self._nextTag = self._numSubmitted self._doneSubmitting = True self._sleighState.occupied = False self._buffer.clear() nwsclient-1.6.4/examples/0000755000175000017500000000000011261702410014760 5ustar westonwestonnwsclient-1.6.4/examples/longvalue.py0000755000175000017500000000533111175420457017347 0ustar westonweston#!/usr/bin/env python import sys, os, getopt, time, traceback from nws.sleigh import Sleigh, sshcmd # this is the worker function def longvalue(numtasks, fname): status = [] print >> sys.stderr, "longvalue function starting" try: bname = os.path.basename(fname) + '_' + str(SleighRank) for i in xrange(numtasks): fobj = open(bname, 'wb') SleighUserNws.findFile('longvalue', fobj) fobj.close() s = os.system("cmp -s %s %s" % (fname, bname)) # XXX Unix specific if s: status.append(s) os.remove(bname) except: status = str(sys.exc_info()[1]) traceback.print_exc() return status if __name__ == '__main__': fname = None numtasks = 10 nkw = {} skw = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:c:f:v') for opt, arg in opts: if opt == '-h': nkw['host'] = skw['nwsHost'] = arg elif opt == '-p': nkw['port'] = skw['nwsPort'] = int(arg) elif opt == '-n': numtasks = int(arg) elif opt == '-c': skw['workerCount'] = int(arg) elif opt == '-f': fname = arg elif opt == '-v': skw['verbose'] = 1 else: raise 'internal error: out-of-sync with getopt' if not fname: print >> sys.stderr, 'error: must use -f option to specify a file' sys.exit(1) if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' # create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) fobj = open(fname, 'rb') print "Storing %s to server..." % fname s.userNws.declare('longvalue', 'single') s.userNws.storeFile('longvalue', fobj) fobj.close() print "Running with %d workers and %d tasks" % \ (s.workerCount, numtasks) # tell the workers to execute the longvalue function # defined in this file print s.eachWorker(longvalue, numtasks, fname) raw_input('Hit return to finish ') print "Finished" except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/ncp.py0000755000175000017500000001177511175420457016144 0ustar westonweston#!/usr/bin/env python import sys, os, getopt from nws.client import NetWorkSpace, DICT, V_VARIABLE, V_MODE, SINGLE def exists(ws, varName): return varName in listVars(ws) def listVars(ws): wsList = ws.listVars(format=DICT).values() return [t[V_VARIABLE] for t in wsList if t[V_MODE] == SINGLE] def usage(msg=''): print >> sys.stderr, "usage: %s [-h nwshost] [-p nwsport] [ls | get | put | rm] file ..." % sys.argv[0] if msg: print >> sys.stderr, msg sys.exit(1) if __name__ == '__main__': nkw = {'persistent': True} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:') for opt, arg in opts: if opt == '-h': nkw['serverHost'] = arg elif opt == '-p': nkw['serverPort'] = int(arg) else: raise 'internal error: out-of-sync with getopt' if len(args) < 1: op = 'ls' # default operation elif args[0] not in ['list', 'ls', 'dir', 'put', 'store', 'write', 'get', 'find', 'read', 'delete', 'del', 'remove', 'rm']: usage() else: op = args.pop(0) ws = NetWorkSpace('NWS Copy', **nkw) if op in ['list', 'ls', 'dir']: vars = listVars(ws) vars.sort() for var in vars: print var elif op in ['put', 'store', 'write']: if not args: usage() for n in args: try: f = open(n, 'rb') except IOError: print >> sys.stderr, "error reading file", n continue varName = os.path.basename(n) if exists(ws, varName): a = raw_input("overwrite existing variable %s? (y/n) " % varName) if not a.strip().lower().startswith('y'): print >> sys.stderr, "skipping", n continue else: print >> sys.stderr, "overwriting", n try: ws.declare(varName, 'single') ws.storeFile(varName, f) except: print >> sys.stderr, "error putting file", n try: f.close() except: pass elif op in ['get', 'find', 'read']: vars = args or listVars(ws) for n in vars: varName = os.path.basename(n) if not exists(ws, varName): print >> sys.stderr, "no variable named", varName continue # only confirm when wildcarding if not args: a = raw_input("get variable %s? (y/n) " % varName) if not a.strip().lower().startswith('y'): print >> sys.stderr, "skipping", varName continue else: print >> sys.stderr, "getting", varName if os.path.exists(n): a = raw_input("overwrite existing file %s? (y/n) " % n) if not a.strip().lower().startswith('y'): print >> sys.stderr, "skipping", n continue else: print >> sys.stderr, "overwriting", n try: f = open(n, 'wb') except IOError: print >> sys.stderr, "error writing file", n continue try: ws.findFile(varName, f) except: print >> sys.stderr, "error getting file", n try: f.close() except: pass elif op in ['delete', 'del', 'remove', 'rm']: vars = args or listVars(ws) for n in vars: varName = os.path.basename(n) if not exists(ws, varName): print >> sys.stderr, "no variable named", varName continue # always confirm when deleting a = raw_input("delete variable %s? (y/n) " % varName) if not a.strip().lower().startswith('y'): print >> sys.stderr, "skipping", varName continue else: print >> sys.stderr, "deleting", varName try: ws.deleteVar(varName) except: print >> sys.stderr, "error deleting", varName else: print >> sys.stderr, "internal error: unhandled operation:", op except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/ring.py0000755000175000017500000000647411175420457016323 0ustar westonweston#!/usr/bin/env python import sys, getopt, time from nws.sleigh import sshcmd from nws.sleigh import Sleigh # This is the "worker" function. Each worker waits for the token to # arrive in his variable. When it does, he increments it by one, and # puts it in the next workers variable. After he's done that "numtask" # times, he quit. def ring(numworkers, numtasks): mine = "worker_%d" % SleighRank next = (SleighRank + 1) % numworkers his = "worker_%d" % next for i in xrange(numtasks): j = SleighUserNws.fetch(mine) SleighUserNws.store(his, j + 1) if __name__ == '__main__': numtasks = 10 timeout = 10 nkw = {} skw = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:t:c:wv') for opt, arg in opts: if opt == '-h': nkw['host'] = skw['nwsHost'] = arg elif opt == '-p': nkw['port'] = skw['nwsPort'] = int(arg) elif opt == '-n': numtasks = int(arg) elif opt == '-t': timeout = int(arg) elif opt == '-c': skw['workerCount'] = int(arg) elif opt == '-w': skw['launch'] = 'web' elif opt == '-v': skw['verbose'] = 1 else: raise 'internal error: out-of-sync with getopt' if not skw.get('launch'): if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' else: if len(args) > 0: print >> sys.stderr, \ 'warning: ignoring unused arguments:', " ".join(args) print >> sys.stderr, 'starting sleigh in web launch mode' # create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) numworkers, complete = s.status(True, timeout) # include the master as one of the workers (which works even # when no workers join) numworkers += 1 # tell the workers to execute the ring function defined in this file s.eachWorker(ring, numworkers, numtasks, blocking=False) # the master becomes the last worker global SleighRank, SleighUserNws SleighRank = numworkers - 1 SleighUserNws = s.userNws print "Master assigned rank %d" % SleighRank # time how long it takes the token to go all the way around the # ring numtask times. start = time.time() s.userNws.store('worker_0', 0) ring(numworkers, numtasks) finish = time.time() token = s.userNws.fetch('worker_0') assert token == numtasks * numworkers print "The token was passed %d times in %f" % (token, finish - start) print "Seconds per operation:", (finish - start) / (2 * token) except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/nwsExample.py0000644000175000017500000000323511175420457017474 0ustar westonweston#!/usr/bin/env python import sys try: from nws.client import NetWorkSpace except ImportError, e: print >> sys.stderr, "unable to import the nws.client python module" print >> sys.stderr, "make sure you have nwsclient installed on this machine" sys.exit(1) try: ws = NetWorkSpace('snake pit') except ImportError, e: print >> sys.stderr, "make sure you're running the NWS server on this machine" print >> sys.stderr, str(e) sys.exit(1) print 'connected, listing contents of netWorkSpace (should be nothing there).' print ws.listVars() ws.store('x', 1) print 'should now see x.' print ws.listVars() print 'find (but don\'t consume) x.' print ws.find('x') print 'check that it is still there.' print ws.listVars() print 'associate another value with x.' ws.store('x', 2) print ws.listVars() print 'consume values for x, should see them in order saved.' print ws.fetch('x') print ws.fetch('x') print 'no more values for x... .' print ws.listVars() print 'so try to fetch and see what happens... .' print ws.fetchTry('x', 'no go') print 'create a single-value variable.' ws.declare('pi', 'single') print ws.listVars() print 'get rid of x.' ws.deleteVar('x') print ws.listVars() print 'try to store two values to pi.' ws.store('pi', 2.171828182) ws.store('pi', 3.141592654) print ws.listVars() print 'check that the right one was kept.' print ws.find('pi') print "store a dictionary." ws.store('dict', {'foo': 'spam', 'bar': 'eggs'}) d = ws.find('dict') print d print "store a list." ws.store('list', ['foo', 1, 'spam', 3.14159, 'bar', 42, 'eggs', 32764]) l = ws.find('list') print l print 'what about the rest of the world?' print ws.server.listWss() nwsclient-1.6.4/examples/exp.py0000755000175000017500000000531111175420457016145 0ustar westonweston#!/usr/bin/env python # XXX Might want to turn this into a sleigh test suite. # It's getting to be a bad example. import sys, os, math, operator, getopt from nws.sleigh import Sleigh, SleighIllegalValueException, sshcmd import worker verbose = 0 launch = sshcmd scriptDir = None try: opts, args = getopt.getopt(sys.argv[1:], 'vws:') except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) for opt, arg in opts: if opt == '-s': scriptDir = arg elif opt == '-v': verbose = 1 elif opt == '-w': launch = 'web' else: print >> sys.stderr, 'error: getopt should have prevented this' sys.exit(1) print "scriptDir =", scriptDir print "verbose =", verbose print "launch =", launch if launch == 'web': print "waiting for web launch workers..." s = Sleigh(scriptDir=scriptDir, launch=launch, verbose=verbose) t = map(math.exp, range(100)) print "testing a simple list:" r = s.eachElem(math.exp, range(100)) if r != t: print "error:", r else: print "success" print "testing list of one list:" r = s.eachElem(math.exp, [range(100)]) if r != t: print "error:", r else: print "success" print "testing a list of two lists:" r = s.eachElem(operator.add, [range(100), range(100)]) if r != range(0, 200, 2): print "error:", r else: print "success" print "testing a simple list and a fixed arg:" r = s.eachElem(operator.add, range(100), 100) if r != range(100, 200): print "error:", r else: print "success" print "testing a list of one list and a fixed arg:" r = s.eachElem(operator.add, [range(100)], 100) if r != range(100, 200): print "error:", r else: print "success" print "testing a list of one list, a fixed arg, and a permute vector:" r = s.eachElem(operator.add, [range(100)], 100, argPermute=[1, 0]) if r != range(100, 200): print "error:", r else: print "success" print "testing a zero length permutation vector:" r = s.eachElem(worker.answer, range(100), argPermute=[]) if r != [42] * 100: print "error:", r else: print "success" print "testing a short permutation vector:" r = s.eachElem(math.exp, [range(100), range(100,200), range(200,300)], argPermute=[0]) if r != t: print "error:", r else: print "success" print "testing a short permutation vector non-blocking:" p = s.eachElem(math.exp, [range(100), range(100,200), range(200,300)], argPermute=[0], blocking=False) print "check:", p.check() r = p.wait() if r != t: print "error:", r else: print "success" print "testing exception" try: s.eachElem(math.exp, 0) print "error: expected illegal value exception" except SleighIllegalValueException, e: print "success:", e raw_input("Hit return to stop sleigh: ") s.stop() nwsclient-1.6.4/examples/batchqueueing.py0000755000175000017500000000431311175420457020176 0ustar westonweston#!/usr/bin/env python # # For LSF, I think you can execute this script directly, # using bsub command: # # % bsub -n 10 batchqueueing.py -v -h `hostname` # # For PBS, I believe you need to write a shell script that # calls this script, using the proper annotations, and # execute that script using the qsub command. # import sys, os, getopt from nws.sleigh import Sleigh, sshcmd def getNodeList(): if os.environ.get('LSB_HOSTS'): print "Submitted via LSF" nodeList = os.environ['LSB_HOSTS'].split() elif os.environ.get('PBS_NODEFILE'): print "Submitted via PBS" nodeList = [n.strip() for n in open(os.environ['PBS_NODEFILE'])] else: print >> sys.stderr, "Can't figure out what nodes to use" print >> sys.stderr, "Has this been submitted using LSF or PBS?" sys.exit(1) return nodeList if __name__ == '__main__': host = None # no good default since this is executed on the cluster port = 8765 verbose = 0 try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:v') for opt, arg in opts: if opt == '-h': host = arg elif opt == '-p': port = int(arg) elif opt == '-v': verbose = 1 else: raise 'internal error: out-of-sync with getopt' if not host: print >> sys.stderr, 'the -h option is required' sys.exit(1) if len(args) > 0: print >> sys.stderr, 'ignoring unused arguments:', ' '.join(args) nodeList = getNodeList() print "Executing on nodes:", " ".join(nodeList) s = Sleigh(launch=sshcmd, nodeList=nodeList, nwsHost=host, nwsPort=port, verbose=verbose) # here's a personal favorite of mine... from math import exp r = s.eachElem(exp, range(100)) print "The results are:", r except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except SystemExit, e: sys.exit(e.code) except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/sleighdemo.py0000755000175000017500000000634211175420457017476 0ustar westonweston#!/usr/bin/env python import sys, getopt, time, math, socket, random from nws.sleigh import Sleigh, sshcmd, rshcmd, scriptcmd from nws.client import SINGLE defaultNodeList = [socket.gethostname] * 3 try: import win32api defaultNodeList = [win32api.GetUserName() + '@' + socket.gethostname] * 3 except: pass # define the function to execute def worker(x): time.sleep(abs(random.gauss(mu, 2.0))) return math.exp(x) # used by worker function mu = abs(random.gauss(10.0, 4.0)) if __name__ == '__main__': numtasks = 10 description = 'Simple Sleigh Demonstration' enableNwsUtility = False skw = {} eo = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:d:f:m:r:c:sve') for opt, arg in opts: if opt == '-h': skw['nwsHost'] = arg elif opt == '-p': skw['nwsPort'] = int(arg) elif opt == '-m': skw['modulePath'] = arg elif opt == '-n': numtasks = int(arg) elif opt == '-d': description = arg elif opt == '-f': eo['loadFactor'] = int(arg) elif opt == '-r': if arg == 'ssh': skw['launch'] = sshcmd elif arg == 'rsh': skw['launch'] = rshcmd elif arg in ['service']: skw['launch'] = arg else: print >> sys.stderr, \ "option -r takes a value of 'ssh', 'rsh' or 'service'" sys.exit(1) elif opt == '-s': skw['scriptExec'] = scriptcmd skw['scriptName'] = 'PythonNWSSleighWorker.py' elif opt == '-v': skw['verbose'] = 1 elif opt == '-e': enableNwsUtility = True elif opt == '-c': skw['workerCount'] = int(arg) else: raise 'internal error: out-of-sync with getopt' # if no nodes are specified, start three on the local machine if len(args) > 0: skw['nodeList'] = args elif skw.get('launch') == 'service': skw['nodeList'] = defaultNodeList print "warning: using default nodeList:", defaultNodeList # create a sleigh on the specified nodes s = Sleigh(**skw) if enableNwsUtility: s.nws.declare('enableNwsUtility', SINGLE) s.nws.declare('MainTitle', SINGLE) s.nws.store('MainTitle', description) s.nws.declare('SubTitle', SINGLE) submitted = s.workerCount # do the work for i in xrange(numtasks): submitted += numtasks s.nws.store('SubTitle', '%d tasks submitted' % submitted) s.eachElem(worker, range(numtasks), **eo) raw_input('Hit return to exit sleigh: ') s.stop() except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/bcmd.py0000755000175000017500000000277011175420457016264 0ustar westonweston#!/usr/bin/env python import sys, getopt, traceback from types import FunctionType, BuiltinFunctionType from nws.sleigh import Sleigh, sshcmd if __name__ == '__main__': host = 'localhost' port = 8765 verbose = 0 try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:v') for opt, arg in opts: if opt == '-h': host = arg elif opt == '-p': port = int(arg) elif opt == '-v': verbose = 1 else: raise 'internal error: out-of-sync with getopt' if len(args) == 0: print >> sys.stderr, 'no workers specified' sys.exit(1) s = Sleigh(launch=sshcmd, nodeList=args, nwsHost=host, nwsPort=port, verbose=verbose) while 1: print '>', cmd = raw_input() cmd = cmd.strip() if cmd: try: results = s.eachWorker(cmd) for i in results: if i: print i except SyntaxError, e: print >> sys.stderr, str(e) except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit, EOFError): print except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/ping.py0000755000175000017500000000172111175420457016307 0ustar westonweston#!/usr/bin/env python import sys, socket, time host = "localhost" port = 8765 wsname = "ping-pong" if len(sys.argv) == 2: wsname = sys.argv[1] elif len(sys.argv) != 1: print >> sys.stderr, "Usage: %s []" % sys.argv[0] sys.exit(1) try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port) nws.store("game", 0) print "Ping-pong server '%s' starting" % wsname while 1: request = nws.fetch("ping") pong = request['replyto'] request['time'] = time.asctime() print "Got a ping from", pong nws.store(pong, request) except KeyboardInterrupt: print "Ping-pong server '%s' exiting" % wsname except ImportError: print "Unable to import the nwsClient module" print "You may need to set or correct your PYTHONPATH" except socket.error: print "NWS server not running on %s, port %d" % (host, port) except: print "An unexpected error occurred:", sys.exc_info()[1] nwsclient-1.6.4/examples/pping.py0000755000175000017500000000713411175420457016473 0ustar westonweston#!/usr/bin/env python import sys, getopt, time, socket from nws.sleigh import sshcmd, rshcmd from nws.sleigh import Sleigh defaultNodeList = [socket.getfqdn()] * 3 try: import win32api defaultNodeList = [win32api.GetUserName() + '@' + socket.gethostname()] * 3 except: pass # this is the master function def ping(ws, totaltasks): for i in xrange(totaltasks): r = ws.fetch('ping') assert len(r) == 2 ws.store(r[0], r[1]) # this is the worker function def pong(numtasks, size): pong = "pong_%d" % SleighRank payload = "#" * size for i in xrange(numtasks): SleighUserNws.store('ping', [pong, payload]) j = SleighUserNws.fetch(pong) if __name__ == '__main__': numtasks = 10 timeout = 10 size = 10 nkw = {} skw = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:m:r:s:t:c:u:v') for opt, arg in opts: if opt == '-h': nkw['host'] = skw['nwsHost'] = arg elif opt == '-p': nkw['port'] = skw['nwsPort'] = int(arg) elif opt == '-m': skw['modulePath'] = arg elif opt == '-n': numtasks = int(arg) elif opt == '-r': if arg == 'ssh': skw['launch'] = sshcmd elif arg == 'rsh': skw['launch'] = rshcmd elif arg in ['service']: skw['launch'] = arg else: print >> sys.stderr, \ "option -r takes a value of 'ssh', 'rsh' or 'service'" sys.exit(1) elif opt == '-s': size = int(arg) elif opt == '-t': timeout = int(arg) elif opt == '-v': skw['verbose'] = 1 elif opt == '-c': skw['workerCount'] = int(arg) elif opt == '-u': skw['user'] = arg else: raise 'internal error: out-of-sync with getopt' if skw.get('launch') and skw.get('workerCount'): print >> sys.stderr, 'warning: workerCount is ignored when not using local launch' if len(args) > 0: if skw.get('launch'): skw['nodeList'] = args else: print >> sys.stderr, \ 'warning: ignoring unused arguments:', " ".join(args) elif skw.get('launch') == 'service': skw['nodeList'] = defaultNodeList print "warning: using default nodeList:", defaultNodeList # create a Sleigh s = Sleigh(**skw) numworkers, complete = s.status(True, timeout) if numworkers == 0: print >> sys.stderr, "no workers successfully started " \ "within %d seconds" % timeout sys.exit(1) # tell the workers to execute the ring function defined in this file s.eachWorker(pong, numtasks, size, blocking=False) totaltasks = numworkers * numtasks start = time.time() ping(s.userNws, totaltasks) totaltime = time.time() - start print "Seconds per operation:", totaltime / (4 * totaltasks) print "Payload size is approximately %d bytes" % size except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/chat2.py0000755000175000017500000000327511175420457016361 0ustar westonweston#!/usr/bin/env python import os, sys, threading, Queue from Tkinter import * from ScrolledText import ScrolledText from nws.client import NetWorkSpace, FIFO class Chat(threading.Thread): def __init__(self, user, varName, wsName, **opt): threading.Thread.__init__(self, name='Chat') self._user, self._varName, self._wsName = user, varName, wsName self._ws = NetWorkSpace(wsName, persistent=True, **opt) self._ws.declare(varName, FIFO) self._queue = Queue.Queue() recvWs = NetWorkSpace(wsName, **opt) self._ifind = recvWs.ifind(varName) self.setDaemon(True) self.start() def sendMessage(self, ent): msg = ent.get().strip() if len(msg) > 0: self._ws.store(self._varName, "%s: %s" % (self._user, msg)) ent.delete(0, END) def checkForNewMessages(self, txt): txt.config(state=NORMAL) while not self._queue.empty(): txt.insert(END, self._queue.get() + '\n') txt.see(END) txt.config(state=DISABLED) txt.after(1000, self.checkForNewMessages, txt) def run(self): for val in self._ifind: self._queue.put(val) if __name__ == '__main__': wsName, varName = 'chatroom', 'chat' user = os.environ.get('USER') or os.environ.get('USERNAME', 'anonymous') root = Tk() root.title('NWS Chat Client') ent = Entry(root) ent.pack(side=BOTTOM, fill=X, padx=4, pady=4) ent.focus() txt = ScrolledText(root) txt.pack(expand=YES, fill=BOTH, padx=4, pady=4) chat = Chat(user, varName, wsName) ent.bind('', lambda event: chat.sendMessage(ent)) chat.checkForNewMessages(txt) root.mainloop() nwsclient-1.6.4/examples/nonblocking.py0000755000175000017500000000405511175420457017660 0ustar westonweston#!/usr/bin/env python import sys, getopt, traceback from nws.sleigh import Sleigh, sshcmd def worker(): while 1: task = SleighUserNws.fetch('task') result = 2 * task SleighUserNws.store('result', (task, result)) if __name__ == '__main__': numtasks = 10 nkw = {} skw = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:c:wv') for opt, arg in opts: if opt == '-h': nkw['host'] = skw['nwsHost'] = arg elif opt == '-p': nkw['port'] = skw['nwsPort'] = int(arg) elif opt == '-n': numtasks = int(arg) elif opt == '-c': skw['workerCount'] = int(arg) elif opt == '-w': skw['launch'] = 'web' elif opt == '-v': skw['verbose'] = 1 else: raise 'internal error: out-of-sync with getopt' if not skw.get('launch'): if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' else: if len(args) > 0: print >> sys.stderr, \ 'warning: ignoring unused arguments:', " ".join(args) print >> sys.stderr, 'starting sleigh in web launch mode' s = Sleigh(**skw) s.eachWorker(worker, blocking=False) for i in range(numtasks): s.userNws.store('task', i) for i in range(numtasks): result = s.userNws.fetch('result') print result[0], 'times 2 is', result[1] except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/hello.py0000755000175000017500000000142711175420457016460 0ustar westonweston#!/usr/bin/env python import sys, socket host = "localhost" port = 8765 wsname = "hello" try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port) count = int(raw_input("Number of times to loop? ")) print "hello: iterations:", count nws.store("hello example", count) for i in range(count): nws.store("hello", i) j = nws.fetch("hello") nws.fetch("hello example") print "Success" except KeyboardInterrupt: print "hello exiting" except ImportError: print "Unable to import the nwsClient module" print "You may need to set or correct your PYTHONPATH" except socket.error: print "NWS server not running on %s, port %d" % (host, port) except: print "An unexpected error occurred:", sys.exc_info()[1] nwsclient-1.6.4/examples/store.py0000755000175000017500000000350511175420457016510 0ustar westonweston#!/usr/bin/env python import sys, getopt, time from nws.sleigh import Sleigh, sshcmd # this is the worker function def store(numtasks): for i in xrange(numtasks): SleighUserNws.store('store', str(SleighRank) + ':' + str(i)) if __name__ == '__main__': numtasks = 1000 nkw = {} skw = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:c:v') for opt, arg in opts: if opt == '-h': nkw['host'] = skw['nwsHost'] = arg elif opt == '-p': nkw['port'] = skw['nwsPort'] = int(arg) elif opt == '-n': numtasks = int(arg) elif opt == '-c': skw['workerCount'] = int(arg) elif opt == '-v': skw['verbose'] = 1 else: raise 'internal error: out-of-sync with getopt' if len(args) > 0: skw['launch'] = sshcmd skw['nodeList'] = args if skw.has_key('workerCount'): print >> sys.stderr, 'warning: ignoring -c option since workers are specified' # create a Sleigh, and compute the number of workers. # this is necessary because of web launch. s = Sleigh(**skw) s.userNws.declare('store', 'single') print "Running with %d workers and %d tasks" % \ (s.workerCount, numtasks) # tell the workers to execute the store function defined in this file s.eachWorker(store, numtasks) print "Finished" except getopt.GetoptError, e: print >> sys.stderr, e.msg sys.exit(1) except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt sys.exit(1) except (KeyboardInterrupt, SystemExit): pass except: ex = sys.exc_info() print >> sys.stderr, ex[0], ex[1] nwsclient-1.6.4/examples/chat3.py0000755000175000017500000000576511175420457016370 0ustar westonweston#!/usr/bin/env python import os, sys, threading, Queue, getopt from Tkinter import * from ScrolledText import ScrolledText from nws.client import NetWorkSpace, FIFO class Chat(threading.Thread): def __init__(self, user, varName, wsName, **opt): threading.Thread.__init__(self, name='Chat') self._user, self._varName, self._wsName = user, varName, wsName self._ws = NetWorkSpace(wsName, persistent=True, **opt) self._ws.declare(varName, FIFO) self._queue = Queue.Queue() recvWs = NetWorkSpace(wsName, **opt) self._ifind = recvWs.ifind(varName) self.setDaemon(True) self.start() def sendMessage(self, ent): msg = ent.get().strip() if len(msg) > 0: self._ws.store(self._varName, "%s: %s" % (self._user, msg)) ent.delete(0, END) def checkForNewMessages(self, txt): txt.config(state=NORMAL) while not self._queue.empty(): txt.insert(END, self._queue.get() + '\n') txt.see(END) txt.config(state=DISABLED) txt.after(1000, self.checkForNewMessages, txt) def run(self): for val in self._ifind: self._queue.put(val) def usage(): print >> sys.stderr, "usage: %s [option ...]" % os.path.basename(sys.argv[0]) print >> sys.stderr, "where option can be:" print >> sys.stderr, " -h " print >> sys.stderr, " -p " print >> sys.stderr, " -n " print >> sys.stderr, " -u " sys.exit(1) if __name__ == '__main__': wsName, varName = 'chatroom', 'chat' user = os.environ.get('USER') or os.environ.get('USERNAME' , 'anonymous') nwsargs = {} try: opts, args = getopt.getopt(sys.argv[1:], 'h:p:n:u:') for opt, arg in opts: if opt == '-h': nwsargs['serverHost'] = arg elif opt == '-p': nwsargs['serverPort'] = int(arg) elif opt == '-n': wsName = arg elif opt == '-u': user = arg else: raise 'internal error: out-of-sync with getopt' if args: print >> sys.stderr, "illegal argument(s) specified:", " ".join(args) usage() root = Tk() root.title('NWS Chat Client') ent = Entry(root) ent.pack(side=BOTTOM, fill=X, padx=4, pady=4) ent.focus() txt = ScrolledText(root) txt.pack(expand=YES, fill=BOTH, padx=4, pady=4) chat = Chat(user, varName, wsName, **nwsargs) ent.bind('', lambda event: chat.sendMessage(ent)) chat.checkForNewMessages(txt) root.mainloop() except getopt.GetoptError, e: print >> sys.stderr, e.msg usage() except ValueError, e: print >> sys.stderr, "option %s requires an integer argument" % opt usage() except SystemExit, e: sys.exit(e) except KeyboardInterrupt: pass except Exception, e: print >> sys.stderr, str(e) nwsclient-1.6.4/examples/parallel.py0000755000175000017500000000221711175420457017147 0ustar westonweston#!/usr/bin/env python import sys, os, time import nws.client, nws.sleigh import worker wsname, host, port = 'foo', 'localhost', 8765 ws = nws.client.NetWorkSpace(wsname, host, port) verbose = 1 scriptDir = None if len(sys.argv) > 1: scriptDir = sys.argv[1] print "using scriptDir =", scriptDir s = nws.sleigh.Sleigh(scriptDir=scriptDir, verbose=verbose) p = s.eachWorker(worker.worker, wsname, host, port, blocking=False) try: s.eachWorker(worker.worker, wsname, host, port, blocking=False) print "Error: should have generated a SleighOccupiedException" except: print "Success" r = p.wait() print "Results from execution 1:", r r = s.eachWorker(worker.worker, wsname, host, port) print "Results from execution 2:", r r = s.eachWorker(worker.worker, wsname, host, port) print "Results from execution 3:", r r = s.eachWorker(worker.worker, wsname, host, port) print "Results from execution 4:", r r = s.eachWorker(worker.worker, wsname, host, port) print "Results from execution 5:", r print "Loop over about 1 minute of work" for i in xrange(60): r = s.eachWorker(time.sleep, 1) print "Variable listing:" print ws.listVars(), s.stop() nwsclient-1.6.4/examples/worker.py0000644000175000017500000000057411175420457016665 0ustar westonwestonimport socket, nws.client, time def worker(*s, **kw): ws = nws.client.NetWorkSpace(*s, **kw) id = SleighRank ws.store("foo_%d" % id, socket.getfqdn()) time.sleep(1) return (id, socket.getfqdn()) + s def answer(): time.sleep(1) return 42 if __name__ == '__main__': global SleighRank SleighRank = 72 print worker('foo', 'localhost', 8765) nwsclient-1.6.4/examples/chat.py0000755000175000017500000000264311175420457016275 0ustar westonweston#!/usr/bin/env python import os, sys from Tkinter import * from ScrolledText import ScrolledText from nws.client import NetWorkSpace, FIFO class Chat: def __init__(self, user, varName, wsName, **opt): self._user, self._varName, self._wsName = user, varName, wsName self._ws = NetWorkSpace(wsName, persistent=True, **opt) self._ws.declare(varName, FIFO) self._ifindTry = self._ws.ifindTry(varName) def sendMessage(self, ent): msg = ent.get().strip() if len(msg) > 0: self._ws.store(self._varName, "%s: %s" % (self._user, msg)) ent.delete(0, END) def checkForNewMessages(self, txt): txt.config(state=NORMAL) self._ifindTry.restart() for msg in self._ifindTry: txt.insert(END, msg + '\n') txt.see(END) txt.config(state=DISABLED) txt.after(1000, self.checkForNewMessages, txt) if __name__ == '__main__': wsName, varName = 'chatroom', 'chat' user = os.environ.get('USER') or os.environ.get('USERNAME', 'anonymous') root = Tk() root.title('NWS Chat Client') ent = Entry(root) ent.pack(side=BOTTOM, fill=X, padx=4, pady=4) ent.focus() txt = ScrolledText(root) txt.pack(expand=YES, fill=BOTH, padx=4, pady=4) chat = Chat(user, varName, wsName) ent.bind('', lambda event: chat.sendMessage(ent)) chat.checkForNewMessages(txt) root.mainloop() nwsclient-1.6.4/examples/pong.py0000755000175000017500000000245211175420457016317 0ustar westonweston#!/usr/bin/env python import sys, socket host = "localhost" port = 8765 wsname = "ping-pong" loops = 10 if len(sys.argv) in [2, 3]: try: loops = int(sys.argv[1]) except: print >> sys.stderr, "Usage: %s [ []]" % sys.argv[0] sys.exit(1) if len(sys.argv) == 3: wsname = sys.argv[2] elif len(sys.argv) != 1: print >> sys.stderr, "Usage: %s [ []]" % sys.argv[0] sys.exit(1) try: from nws.client import NetWorkSpace nws = NetWorkSpace(wsname, host, port, useUse=True) game = nws.fetch("game") nws.store("game", game + 1) pong = "pong_%d" % game request = {'replyto': pong} print "Starting a ping-pong game", game for i in xrange(loops): request['i'] = i nws.store("ping", request) reply = nws.fetch(pong) print reply if i != reply['i']: print "Error: expected %d, received %d" % (i, reply['i']) nws.deleteVar(pong) except KeyboardInterrupt: print "pong exiting" except ImportError: print "Unable to import the nwsClient module" print "You may need to set or correct your PYTHONPATH" except socket.error: print "NWS server not running on %s, port %d" % (host, port) except: print "An unexpected error occurred:", sys.exc_info()[1] nwsclient-1.6.4/test/0000755000175000017500000000000011261702410014121 5ustar westonwestonnwsclient-1.6.4/test/test_barrier.py0000755000175000017500000000141011175420457017173 0ustar westonweston#!/usr/bin/env python import os from nws.sleigh import Sleigh, sshcmd from socket import getfqdn slargs = {} try: slargs['nwsHost'] = os.environ['NWS_HOST'] except: pass try: slargs['nwsPort'] = int(os.environ['NWS_PORT']) except: pass try: slargs['nodeList'] = os.environ['NWS_NODES'].split() slargs['launch'] = sshcmd print "using ssh to launch workers on machines:", slargs['nodeList'] except: print "using default local workers" s = Sleigh(**slargs) # use os.getloadavg if they have it (it's not in Python 2.2 or on Windows) # because socket.getfqdn runs terribly slowly on Mac OS X if hasattr(os, 'getloadavg'): for i in xrange(1000): print i, s.eachWorker(os.getloadavg) else: for i in xrange(1000): print i, s.eachWorker(getfqdn) s.stop() nwsclient-1.6.4/test/test_long.py0000755000175000017500000000523211175420457016512 0ustar westonweston#!/usr/bin/env python import unittest from nws.client import * try: from logging import getLogger, basicConfig, DEBUG except ImportError: DEBUG = 1 class Logger: def __init__(self): self._level = 0 def info(self, msg): if self._level: print msg def setLevel(self, level): self._level = level def getLogger(ign): return Logger() def basicConfig(): pass class LongTest(unittest.TestCase): log = getLogger('NWSTest') wsname = 'unit test ws' def setUp(self): self.log.info('Setting up') self.ws = NetWorkSpace(self.wsname, **nwsargs) def tearDown(self): self.log.info('Tearing down') try: self.ws.server.close() except: pass def testLongValues(self): """Test sending lots of long values of the same name to the server.""" self.log.info('Storing lots of long values (this will take awhile)') for i in xrange(10): for j in xrange(1000): self.ws.store('Long', range(10000)) for j in xrange(1000): self.ws.fetch('Long') def testDifferentLongValues(self): """Test sending lots of long values of different names to the server.""" self.log.info('Storing lots of different long values (this will take awhile)') for i in xrange(10): for j in xrange(1000): self.ws.store('Long %d' % j, range(10000)) for j in xrange(1000): self.ws.fetch('Long %d' % j) def testDeleteVarLongValues(self): """Test if long values are cleaned up when you delete a variable""" for i in xrange(100): # consume a lot of file descriptors for j in xrange(1000): self.ws.store('Long', range(10000)) self.ws.deleteVar('Long') def testDeleteWsLongValues(self): """Test if long values are cleaned up when you delete the workspace""" for i in xrange(100): twsname = self.ws.server.mktempWs("delws_%d") tws = self.ws.server.openWs(twsname) # consume a lot of file descriptors for j in xrange(1000): tws.store('Long', range(10000)) tws.server.deleteWs(twsname) if __name__ == '__main__': import sys, os nwsargs = {} try: nwsargs['serverHost'] = os.environ['NWS_HOST'] except: pass try: nwsargs['serverPort'] = int(os.environ['NWS_PORT']) except: pass verbose = os.environ.get('NWS_VERBOSE', False) basicConfig() log = getLogger('LongTest') if verbose: log.setLevel(DEBUG) unittest.main() nwsclient-1.6.4/test/test_file.py0000755000175000017500000001331011175420457016466 0ustar westonweston#!/usr/bin/env python import unittest from itertools import izip, count from StringIO import StringIO from nws.client import * try: from logging import getLogger, basicConfig, DEBUG except ImportError: DEBUG = 1 class Logger: def __init__(self): self._level = 0 def info(self, msg): if self._level: print msg def setLevel(self, level): self._level = level def getLogger(ign): return Logger() def basicConfig(): pass class FileTest(unittest.TestCase): log = getLogger('FileTest') wsname = 'FileTest' def setUp(self): self.log.info('Setting up') self.ws = NetWorkSpace(self.wsname, **nwsargs) def tearDown(self): self.log.info('Tearing down') try: self.ws.server.close() except: pass def testReadFile(self): """Test reading the values of a variable using a read mode file.""" for i in range(10): self.ws.store('TestReadFile', "%d\n" % i) for i, x in izip(count(), self.ws.makefile('TestReadFile')): self.assertEqual("%d\n" % i, x) def testWriteAndRead(self): """Test the line splitting of the write method.""" n = 10 s = StringIO() for i in xrange(n): s.write("%d\n" % i) w = self.ws.makefile('TestWriteAndRead', 'w') exp = s.getvalue() w.write(exp) w.close() for i, x in izip(count(), self.ws.makefile('TestWriteAndRead', 'r')): self.assertEqual("%d\n" % i, x) act = self.ws.makefile('TestWriteAndRead', 'r').read() self.assertEqual(act, exp) def testReadline(self): """Test the readline method.""" n = 10 s = StringIO() for i in xrange(n): s.write("%d\n" % i) w = self.ws.makefile('TestReadline', 'w') exp = s.getvalue() w.write(exp) w.close() r = self.ws.makefile('TestReadline', 'r') for i in xrange(n): x = r.readline() self.assertEqual("%d\n" % i, x) r.close() act = self.ws.makefile('TestReadline', 'r').read() self.assertEqual(act, exp) def testReadlines(self): """Test the readlines method.""" w = self.ws.makefile('TestReadlines', 'w') s = "This is a test" for c in s: w.write(c) # test that nothing is there because no newline and we didn't flush or close r1 = self.ws.makefile('TestReadlines') self.assertEqual(len(r1.readlines()), 0) w.close() # test that we still read nothing because we hit EOF self.assertEqual(len(r1.readlines()), 0) r2 = self.ws.makefile('TestReadlines') x = r2.readlines() self.assert_(len(x) == 1 and x[0] == s) w = self.ws.makefile('TestReadlines', 'w') exp = ['This is a test\n', 'that we get three\n', 'lines when we call readline\n'] s = ''.join(exp) + 'some extra junk that is buffered but not written' for c in s: w.write(c) act = self.ws.makefile('TestReadlines').readlines() self.assertEqual(act, exp) def testExceptions(self): """Test various exceptions.""" w = self.ws.makefile('TestExceptions', 'w') self.assertRaises(ValueError, iter, w) w.close() self.assertRaises(ValueError, w.read) self.ws.store('TestExceptions', 1) r = self.ws.makefile('TestExceptions') r.read(0) # this doesn't read anything self.assertRaises(ValueError, r.read, 1) def testFlush(self): """Test the flush method.""" w = self.ws.makefile('TestFlush', 'w') w.write("hello,") self.assertEqual([], list(self.ws.ifindTry('TestFlush'))) w.flush() self.assertEqual(['hello,'], list(self.ws.ifindTry('TestFlush'))) w.write(" world") self.assertEqual(['hello,'], list(self.ws.ifindTry('TestFlush'))) w.write('\n') self.assertEqual(['hello,', ' world\n'], list(self.ws.ifindTry('TestFlush'))) r = self.ws.makefile('TestFlush') self.assertEqual('hello, world\n', r.read()) def testSmallReads(self): """Test reads of one byte at a time.""" w = self.ws.makefile('TestSmallReads', 'w') s = 'abcdefg\n' w.write(s) r = self.ws.makefile('TestSmallReads') for c in s: self.assertEqual(c, r.read(1)) def testSmallWrites(self): """Test writing one byte at a time.""" w = self.ws.makefile('TestSmallReads', 'w') s = 'abcdefg\n' for c in s: w.write(c) r = self.ws.makefile('TestSmallReads') self.assertEqual(s, r.read()) def testAttributes(self): """Test that various attributes are set correctly.""" w = self.ws.makefile('TestAttributes', 'r') self.assertEqual(w.mode, 'r') self.assert_(not w.isatty()) self.assertEqual(w.name, 'TestAttributes') self.assert_(not w.closed) w.close() self.assert_(w.closed) w = self.ws.makefile('TestAttributes', 'w') self.assertEqual(w.mode, 'w') self.assert_(not w.isatty()) self.assertEqual(w.name, 'TestAttributes') self.assert_(not w.closed) w.close() self.assert_(w.closed) if __name__ == '__main__': import sys, os nwsargs = {} try: nwsargs['serverHost'] = os.environ['NWS_HOST'] except: pass try: nwsargs['serverPort'] = int(os.environ['NWS_PORT']) except: pass verbose = os.environ.get('NWS_VERBOSE', False) basicConfig() log = getLogger('LongTest') if verbose: log.setLevel(DEBUG) unittest.main() nwsclient-1.6.4/test/test_sleigh.py0000755000175000017500000000651611175420457017034 0ustar westonweston#!/usr/bin/env python import os, sys, glob from math import exp from nws.sleigh import Sleigh, defaultSleighOptions from nws.sleigh import envcmd, scriptcmd, sshcmd from nws.sleigh import _findScript # this is like scriptcmd, but uses 'python' explicitly, # and therefore requires a specified scriptDir def pythoncmd(machine, *env, **opts): if opts['scriptDir']: script = os.path.join(opts['scriptDir'], opts['scriptName']) else: script = opts['scriptName'] return [sys.executable, script] + list(env) # this is like envcmd, but uses '/bin/sh' explicitly, # and therefore requires a specified scriptDir def shellcmd(machine, *env, **opts): if opts['scriptDir']: script = os.path.join(opts['scriptDir'], opts['scriptName']) else: script = opts['scriptName'] return ['/usr/bin/env'] + list(env) + ['/bin/sh', script] # XXX this needs some thought BINDIR = os.path.dirname(_findScript('PythonNWSSleighWorker')) def test(verbose=0): s = Sleigh(verbose=verbose, **slargs) r = s.eachElem(exp, range(100)) q = [exp(i) for i in range(100)] s.stop() if r == q: sys.stderr.write('.') sys.stderr.flush() return 0 else: sys.stderr.write('E') sys.stderr.flush() return 1 slargs = {} try: slargs['nwsHost'] = os.environ['NWS_HOST'] except: pass try: slargs['nwsPort'] = int(os.environ['NWS_PORT']) except: pass orig = dict(defaultSleighOptions) pyopts = [ { 'scriptExec': pythoncmd, 'scriptDir': BINDIR, 'scriptName': 'PythonNWSSleighWorker.py', }, { 'scriptExec': scriptcmd, 'scriptDir': '', 'scriptName': 'PythonNWSSleighWorker.py', }, { 'scriptExec': scriptcmd, 'scriptDir': BINDIR, 'scriptName': 'PythonNWSSleighWorker.py', }, { 'scriptExec': envcmd, 'scriptDir': '', 'scriptName': 'PythonNWSSleighWorker.py', 'interp': sys.executable, }, { 'scriptExec': envcmd, 'scriptDir': BINDIR, 'scriptName': 'PythonNWSSleighWorker.py', 'interp': sys.executable, }, ] shopts = [ { 'scriptExec': shellcmd, 'scriptDir': BINDIR, 'scriptName': 'PythonNWSSleighWorker', }, { 'scriptExec': envcmd, 'scriptDir': '', 'scriptName': 'PythonNWSSleighWorker', }, { 'scriptExec': envcmd, 'scriptDir': BINDIR, 'scriptName': 'PythonNWSSleighWorker', }, ] failures = 0 successes = 0 if test(0) == 0: successes += 1 else: failures += 1 for o in pyopts + shopts: for wd in ('/tmp', os.getcwd()): for ssh in [sshcmd]: defaultSleighOptions.clear() defaultSleighOptions.update(orig) defaultSleighOptions.update(o) defaultSleighOptions['launch'] = ssh defaultSleighOptions['workingDir'] = wd # print 'testing:', defaultSleighOptions if test(0) == 0: successes += 1 else: failures += 1 print 'successes: %d; failures: %d' % (successes, failures) def rmglobs(*globs): for g in globs: for f in glob.glob(g): os.remove(f) globs = [ "PythonSleighSentinelLog_*", "PythonNWSSleighWorkerLog_*", "sleigh_ride_*", "/tmp/PythonSleighSentinelLog_*", "/tmp/PythonNWSSleighWorkerLog_*", "/tmp/sleigh_ride_*", ] rmglobs(*globs) print 'finished' nwsclient-1.6.4/test/test_sleigh2.py0000755000175000017500000002050611175420457017111 0ustar westonweston#!/usr/bin/env python import unittest, time, string from nws.client import SINGLE from nws.sleigh import Sleigh, sshcmd from random import randint from math import sqrt from itertools import repeat, imap, starmap, cycle from operator import mul try: from logging import getLogger, basicConfig, DEBUG except ImportError: DEBUG = 1 class Logger: def __init__(self): self._level = 0 def info(self, msg): if self._level: print msg def setLevel(self, level): self._level = level def getLogger(ign): return Logger() def basicConfig(): pass def broadcastWorker(n): for i in xrange(n): SleighNws.store('broadcast', randint(1, 1000)) workers = SleighNws.fetch('workers') - 1 SleighNws.store('workers', workers) if workers == 0: SleighNws.store('broadcast', 0) total = 0 for t in SleighNws.ifind('broadcast'): if t == 0: break total += t return total def monitorWorker(): return list(SleighNws.ifind('monitor')) def nothing(n): return n class SleighTest(unittest.TestCase): log = getLogger('SleighTest') wsname = 'unit test ws' def setUp(self): self.log.info('Setting up') self.s = Sleigh(**slargs) def tearDown(self): self.log.info('Tearing down') try: self.s.stop() except: pass def testBroadcast(self): """Test all-to-all broadcast using ifind""" self.s.nws.store('workers', self.s.workerCount) n = 1000 results = self.s.eachWorker(broadcastWorker, n) self.assertEquals(len(results), self.s.workerCount) # Check the results by calling ifindTry values = list(self.s.nws.ifindTry('broadcast')) self.assertEquals(len(values), n * self.s.workerCount + 1) self.assertEquals(values[-1], 0) total = sum(values) self.assertEquals(results, [total] * len(results)) # Now let's verify the results using a standard fetchTry loop check = 0 while True: t = self.s.nws.fetchTry('broadcast') if t is None: break check += t self.assertEquals(total, check) def testMonitor(self): """Test one-to-all broadcast with a single mode variable""" for n, s in [(1000, 0.0), (100, 0.1), (10, 1.0)]: pending = self.s.eachWorker(monitorWorker, blocking=False) time.sleep(3) # not necessary for test to pass self.s.nws.declare('monitor', SINGLE) for i in xrange(n): self.s.nws.store('monitor', i) time.sleep(s) # not necessary for test to pass time.sleep(3) # not necessary for test to pass self.s.nws.deleteVar('monitor') results = pending.wait() self.assertEquals(len(results), self.s.workerCount) for r in results: self.assert_(len(r) <= n) if len(r) > 0: p = r.pop(0) for t in r: self.assert_(t > p) p = t else: print 'WARNING: got zero length result' def testString(self): """Test when elementArgs is a string""" s = "Hello, world" act = self.s.eachElem(ord, s) exp = map(ord, s) self.assertEquals(act, exp) def testListOfString(self): """Test when elementArgs is a list containing a string""" s = "This is another string." act = self.s.eachElem(ord, [s]) exp = map(ord, s) self.assertEquals(act, exp) def testIterables(self): """Test when elementArgs is a list of unequal length iterables""" act = self.s.eachElem(cmp, [xrange(10), xrange(20)]) exp = [0] * 10 self.assertEquals(act, exp) def testDefaultElementArgs(self): """Test the default value of elementArgs""" act = self.s.eachElem(cmp) exp = [] self.assertEquals(act, exp) def testElementArgsIFind(self): """Test using ifindTry to specify elementArgs for eachElem""" n = 10 for i in xrange(n): self.s.userNws.store('i', i) exp = map(lambda x: x + x, xrange(n)) for i in xrange(3): act = self.s.eachElem(lambda x: x + x, self.s.userNws.ifindTry('i')) self.assertEqual(act, exp) def testElementArgsStoreFile(self): """Test the use of storeFile with an elementArgs ifindTry""" f = open("test_sleigh2.py", "rb") try: # write values of up to 100 bytes until EOF while self.s.userNws.storeFile('test_sleigh2.py', f, n=100): pass finally: f.close() exp = string.upper(open("test_sleigh2.py", "rb").read()) for i in xrange(3): act = "".join(self.s.eachElem(string.upper, self.s.userNws.ifindTry('test_sleigh2.py'))) self.assertEqual(act, exp) def testAccumulator(self): """Test the use of the accumulator argument to eachElem""" exp = string.uppercase # do it the standard way act = "".join(self.s.eachElem(string.upper, string.lowercase)) self.assertEqual(act, exp) # do it with an accumulator accum = Accum_1() self.s.eachElem(string.upper, string.lowercase, accumulator=accum.setresults) act = "".join(accum.getresults()) self.assertEqual(act, exp) def testEachWorkerAccum(self): """Test the use of the accumulator argument to eachWorker""" n = 42 exp = [n] * self.s.workerCount accum = Accum_1() self.s.eachWorker(nothing, n, accumulator=accum.setresults) act = accum.getresults() self.assertEqual(act, exp) def testNewStyleAccum(self): """Test the new-style accumulators""" exp = sum(xrange(300)) accum = Accum_2() act = self.s.eachElem(sum, [[range(100), range(100, 200), range(200, 300)]], accumulator=accum) self.assertEqual(act, exp) def testImap(self): """Test imap method""" self.assertEqual(list(self.s.imap(sqrt, xrange(10))), map(sqrt, xrange(10))) self.assertEqual(list(self.s.imap(mul, xrange(10), repeat(6))), list(imap(mul, xrange(10), repeat(6)))) axpy = lambda a, x, y: a * x + y self.assertEqual(list(self.s.imap(axpy, repeat(10), xrange(100), cycle(xrange(10)))), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10))))) def testStarmap(self): """Test starmap method""" def gen(a, x, y): x = iter(x) y = iter(y) try: while True: yield (a, x.next(), y.next()) except StopIteration: pass axpy = lambda a, x, y: a * x + y it = gen(10, xrange(100), cycle(xrange(10))) self.assertEqual(list(self.s.starmap(axpy, it)), list(imap(axpy, repeat(10), xrange(100), cycle(xrange(10))))) # general purpose, but useless, accumulator class class Accum_1: def __init__(self): self.results = {} def setresults(self, results): assert len(results) == 1 tag = results[0][0] val = results[0][1] assert not self.results.has_key(tag) self.results[tag] = val def getresults(self): return [self.results[i] for i in xrange(len(self.results))] # a "new-style" accumulator class that does result reduction class Accum_2: def __init__(self): self._sum = 0 def result(self, results): self._sum += sum([r[1] for r in results]) def value(self): return self._sum if __name__ == '__main__': import sys, os slargs = {} try: slargs['nwsHost'] = os.environ['NWS_HOST'] except: pass try: slargs['nwsPort'] = int(os.environ['NWS_PORT']) except: pass slargs['verbose'] = verbose = os.environ.get('NWS_VERBOSE', False) try: slargs['nodeList'] = os.environ['NWS_NODES'].split() slargs['launch'] = sshcmd print "using ssh to launch workers on machines:", slargs['nodeList'] except: print "using default local workers" basicConfig() log = getLogger('SleighTest') if verbose: log.setLevel(DEBUG) unittest.main() nwsclient-1.6.4/test/test_nws.py0000755000175000017500000004662111175420457016371 0ustar westonweston#!/usr/bin/env python import unittest, time, string, StringIO from nws.client import * try: from logging import getLogger, basicConfig, DEBUG except ImportError: DEBUG = 1 class Logger: def __init__(self): self._level = 0 def info(self, msg): if self._level: print msg def setLevel(self, level): self._level = level def getLogger(ign): return Logger() def basicConfig(): pass def getdict(**kw): return kw class NWSTest(unittest.TestCase): log = getLogger('NWSTest') wsname = 'unit test ws' def setUp(self): self.log.info('Setting up') self.ws = NetWorkSpace(self.wsname, **nwsargs) def tearDown(self): self.log.info('Tearing down') try: self.ws.server.close() except: pass # Get a list of all workspaces from a listing string generated by # the listWss method. def getWss(self, listing): x = {} self.assertEqual(listing[-1], '\n') listing = listing[:-1] for line in listing.split('\n'): t = line.split('\t') self.assertEqual(len(t), 5) self.assert_(len(t[0]) > 1) if t[0][0] == '>': mine = 1 else: self.assertEqual(t[0][0], ' ') mine = 0 name = t[0][1:] owner = t[1] if t[2] == 'True': persistent = 1 else: self.assertEqual(t[2], 'False') persistent = 0 nvars = int(t[3]) self.assert_(nvars >= 0) if t[4]: vars = t[4].split(',') else: vars = [] self.assertEqual(nvars, len(vars)) x[name] = getdict(name=name, mine=mine, owner=owner, persistent=persistent, nvars=nvars, vars=vars) return x # Get a list of all variables from a listing string generated by # the listVars method. def getVars(self, listing): x = {} if listing: self.assert_(listing[-1] != '\n') lines = listing.split('\n') for line in lines: t = line.split('\t') self.assertEqual(len(t), 5) self.assert_(len(t[0]) > 0) variable = t[0] values = int(t[1]) fetchers = int(t[2]) finders = int(t[3]) mode = t[4] x[variable] = getdict(variable=variable, values=values, fetchers=fetchers, finders=finders, mode=mode) return x def getVarsDict(self, varDict): x = {} for k in varDict: v = varDict[k] x[v[V_VARIABLE]] = getdict(variable=v[V_VARIABLE], values=v[V_VALUES], fetchers=v[V_FETCHERS], finders=v[V_FINDERS], mode=v[V_MODE]) return x def testFifo(self): """A basic test of using a fifo mode variable""" values = range(10) self.log.info('Storing variables in FIFO mode') for i in values: self.ws.store('FIFO', i) self.log.info('Fetching values for FIFO variable') for i in values: t = self.ws.fetchTry('FIFO') self.assertEqual(t, i) self.assertEqual(self.ws.fetchTry('FIFO'), None) def testLifo(self): """A basic test of using a lifo mode variable""" self.ws.declare('LIFO', LIFO) values = range(10) self.log.info('Storing variables in LIFO mode') for i in values: self.ws.store('LIFO', i) self.log.info('Fetching values for LIFO variable') values.reverse() for i in values: t = self.ws.fetchTry('LIFO') self.assertEqual(t, i) self.assertEqual(self.ws.fetchTry('LIFO'), None) def testSingle(self): """A basic test of using a single mode variable""" self.ws.declare('SINGLE', SINGLE) values = range(10) self.log.info('Storing variables in SINGLE mode') for i in values: self.ws.store('SINGLE', i) self.log.info('Fetching values for SINGLE variable') self.assertEqual(self.ws.fetchTry('SINGLE'), values[-1]) for i in values: self.assertEqual(self.ws.fetchTry('SINGLE'), None) def testCurrentWs(self): """Test the output of the currentWs method""" wsname = self.ws.currentWs() self.assertEqual(self.wsname, wsname) def testUseUseTrue(self): """Test setting useUse to True""" usews = NetWorkSpace(self.wsname, useUse=True, **nwsargs) val = 42 self.ws.store('foo', val) self.assertEqual(usews.fetch('foo'), val) def testUseUseFalse(self): """Test that explicitly setting useUse=False works""" usews = NetWorkSpace(self.wsname + ' 2', useUse=False, **nwsargs) val = 7 usews.store('bar', val) self.assertEqual(usews.findTry('bar'), val) self.assertEqual(usews.fetch('bar'), val) def testConstants(self): """Test the value of the mode constants""" self.assertEqual(FIFO, 'fifo') self.assertEqual(LIFO, 'lifo') self.assertEqual(MULTI, 'multi') self.assertEqual(SINGLE, 'single') def testDeleteWs(self): """Test deleting a workspace using deleteWs""" host = self.ws.server.serverHost port = self.ws.server.serverPort server = NwsServer(host, port) self.log.info('Deleting workspace %s' % repr(self.wsname)) server.deleteWs(self.wsname) self.assertRaises(NwsOperationException, self.ws.store, 'fail', 0) def testListWss(self): """Test output from listWss""" wss = self.getWss(self.ws.server.listWss()) self.assert_(wss.has_key('__default')) n = 0 for k, v in wss.items(): if v['mine']: n += 1 self.assertEqual(n, 1) self.assert_(wss[self.wsname]['mine']) def testPersistent(self): """Test creation of persistent workspace""" wsname = 'persistent' t = NetWorkSpace(wsname, persistent=True, **nwsargs) try: wss = self.getWss(self.ws.server.listWss()) self.assert_(wss.has_key(wsname)) self.assert_(wss[wsname]['persistent']) t.server.close() # give the server a chance to notice that the connection was # dropped time.sleep(1) wss = self.getWss(self.ws.server.listWss()) self.assert_(wss.has_key(wsname)) self.assert_(wss[wsname]['persistent']) finally: self.ws.server.deleteWs(wsname) wss = self.getWss(self.ws.server.listWss()) self.assert_(not wss.has_key(wsname)) def testNonPersistent(self): """Test creation of non-persistent workspace""" wsname = 'non-persistent' t = NetWorkSpace(wsname, persistent=False, **nwsargs) wss = self.getWss(self.ws.server.listWss()) self.assert_(wss.has_key(wsname)) self.assert_(not wss[wsname]['persistent']) t.server.close() # give the server a chance to notice that the connection was # dropped time.sleep(1) wss = self.getWss(self.ws.server.listWss()) self.assert_(not wss.has_key(wsname)) def testMktempWs(self): """Test mktempWs""" self.log.info('calling mktempWs incorrectly...') self.assertRaises(NwsOperationException, self.ws.server.mktempWs, 'foo') self.log.info('calling mktempWs incorrectly again...') self.assertRaises(NwsOperationException, self.ws.server.mktempWs, 'foo') self.log.info('calling mktempWs a third time...') t = self.ws.server.mktempWs('foo%d') self.assert_(t.startswith('foo')) self.log.info('mktempWs returned: ' + t) # must explicitly delete, because nobody owns it self.ws.server.deleteWs(t) def testListVars(self): """Test listVars""" expected = { 'foo': getdict(variable='foo', values=1, fetchers=0, finders=0, mode=SINGLE), 'bar': getdict(variable='bar', values=2, fetchers=0, finders=0, mode=MULTI), 'baz': getdict(variable='baz', values=1, fetchers=0, finders=0, mode=LIFO), 'buz': getdict(variable='buz', values=0, fetchers=0, finders=0, mode=FIFO), 'bif': getdict(variable='bif', values=0, fetchers=0, finders=0, mode=LIFO), } self.ws.declare('foo', SINGLE) self.ws.store('foo', 'hello, world') self.ws.store('foo', 'goodbye, world') self.ws.store('foo', 0) self.ws.store('foo', 222) self.ws.declare('bar', MULTI) self.ws.store('bar', 'spam') self.ws.store('bar', 'spam') self.ws.declare('baz', LIFO) self.ws.store('baz', 'eggs') self.ws.store('buz', 100) self.ws.fetchTry('buz') self.ws.declare('bif', LIFO) actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) actual2 = self.getVarsDict(self.ws.listVars(format=DICT)) self.assertEqual(actual2, expected) # now delete foo and bif self.ws.deleteVar('foo') self.ws.deleteVar('bif') del expected['foo'] del expected['bif'] actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) actual2 = self.getVarsDict(self.ws.listVars(format=DICT)) self.assertEqual(actual2, expected) # now delete baz self.ws.deleteVar('baz') del expected['baz'] actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) actual2 = self.getVarsDict(self.ws.listVars(format=DICT)) self.assertEqual(actual2, expected) # now try the same thing again actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) # now try to redeclare bar and buz self.assertRaises(NwsDeclarationFailedException, self.ws.declare, 'bar', FIFO) self.assertRaises(NwsDeclarationFailedException, self.ws.declare, 'buz', SINGLE) actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) actual2 = self.getVarsDict(self.ws.listVars(format=DICT)) self.assertEqual(actual2, expected) # now delete bar and buz self.ws.deleteVar('bar') self.ws.deleteVar('buz') del expected['bar'] del expected['buz'] actual = self.getVars(self.ws.listVars()) self.assertEqual(actual, expected) actual2 = self.getVarsDict(self.ws.listVars(format=DICT)) self.assertEqual(actual2, expected) self.assertEqual(len(actual), 0) self.assertEqual(len(actual2), 0) self.assertEqual(len(expected), 0) def testEmptyString(self): """Test that empty strings are handled correctly""" self.ws.store('Empty', '') self.assertEqual(self.ws.fetch('Empty'), '') self.assertEqual(self.ws.fetchTry('Empty'), None) def testDeclare(self): """Test variable declaration""" self.ws.declare('FIFO', FIFO) self.assertRaises(NwsDeclarationFailedException, self.ws.declare, 'FIFO', SINGLE) self.assertRaises(NwsDeclarationFailedException, self.ws.declare, 'FIFO', LIFO) self.assertRaises(NwsDeclarationFailedException, self.ws.declare, 'FIFO', MULTI) self.assertRaises(ValueError, self.ws.declare, 'FIFO', 'garbage') def testIfindTry(self): """Test ifindTry""" for i in range(10): self.ws.store('ifindTry', i) it = self.ws.ifindTry('ifindTry') self.assertEqual(list(it), range(10)) self.assertRaises(StopIteration, it.next) self.assertEqual(list(it), []) it.restart() self.assertEqual(list(it), []) it.reset() self.assertEqual(list(it), range(10)) self.assertEqual(list(it), []) it.restart() for i in range(10): self.ws.store('ifindTry', i + 10) self.assertEqual(list(it), range(10, 20)) self.assertEqual(list(it), []) it.restart() self.assertEqual(list(it), []) it.reset() self.assertEqual(list(it), range(20)) self.assertEqual(list(it), []) def testIfetchTry(self): """Test ifetchTry""" for i in range(10): self.ws.store('ifetchTry', i) it = self.ws.ifetchTry('ifetchTry') self.assertEqual(list(it), range(10)) self.assertRaises(StopIteration, it.next) self.assertEqual(list(it), []) it.restart() self.assertEqual(list(it), []) self.assertEqual(list(it), []) it.reset() self.assertEqual(list(it), []) self.assertEqual(list(it), []) it.restart() for i in range(10): self.ws.store('ifetchTry', i + 10) self.assertEqual(list(it), range(10, 20)) self.assertEqual(list(it), []) it.restart() self.assertEqual(list(it), []) self.assertEqual(list(it), []) it.reset() self.assertEqual(list(it), []) self.assertEqual(list(it), []) def testIfind(self): """Test ifind""" # XXX not a great test because it never blocks for i in range(10): self.ws.store('ifind', i) it = self.ws.ifind('ifind') self.assertEqual([it.next() for i in range(10)], range(10)) for i in range(10): self.ws.store('ifind', i + 10) self.assertEqual([it.next() for i in range(10)], range(10, 20)) it.reset() self.assertEqual([it.next() for i in range(20)], range(20)) self.ws.deleteVar('ifind') self.assertRaises(StopIteration, it.next) self.assertEqual(list(it), []) def testIfetch(self): """Test ifetch""" # XXX not a great test because it never blocks for i in range(10): self.ws.store('ifetch', i) it = self.ws.ifetch('ifetch') self.assertEqual([it.next() for i in range(10)], range(10)) for i in range(10): self.ws.store('ifetch', i + 10) self.assertEqual([it.next() for i in range(10)], range(10, 20)) self.ws.deleteVar('ifetch') self.assertRaises(StopIteration, it.next) self.assertEqual(list(it), []) def testConcurrentIfetchTry(self): """Test multiple concurrent ifetchTry iterators""" for i in range(10): self.ws.store('ifetchTry', i) it_1 = self.ws.ifetchTry('ifetchTry') it_2 = self.ws.ifetchTry('ifetchTry') self.assertEqual(it_1.next(), 0) self.assertEqual(it_2.next(), 1) self.assertEqual(it_1.next(), 2) self.assertEqual(it_2.next(), 3) self.assertEqual(it_1.next(), 4) self.assertEqual(it_1.next(), 5) self.assertEqual(it_2.next(), 6) self.assertEqual(it_2.next(), 7) self.assertEqual(it_1.next(), 8) self.assertEqual(it_2.next(), 9) self.assertRaises(StopIteration, it_1.next) self.assertRaises(StopIteration, it_2.next) self.ws.deleteVar('ifetchTry') def testConcurrentIfindTry(self): """Test multiple concurrent ifindTry iterators""" for i in range(5): self.ws.store('conifindTry', i) it_1 = self.ws.ifindTry('conifindTry') it_2 = self.ws.ifindTry('conifindTry') self.assertEqual(it_1.next(), 0) self.assertEqual(it_1.next(), 1) self.assertEqual(it_2.next(), 0) self.assertEqual(it_2.next(), 1) self.assertEqual(it_1.next(), 2) self.assertEqual(it_2.next(), 2) self.assertEqual(it_1.next(), 3) self.assertEqual(it_1.next(), 4) self.assertEqual(it_2.next(), 3) self.assertEqual(it_2.next(), 4) self.assertRaises(StopIteration, it_1.next) self.assertRaises(StopIteration, it_2.next) self.ws.deleteVar('conifindTry') def testIfindTrySingle(self): """Test ifindTry with single mode variables""" self.ws.declare('ifindTrySingle', SINGLE) it = self.ws.ifindTry('ifindTrySingle') self.ws.store('ifindTrySingle', 100) self.assertEqual(it.next(), 100) self.ws.store('ifindTrySingle', 101) self.assertEqual(it.next(), 101) self.ws.store('ifindTrySingle', 102) self.ws.store('ifindTrySingle', 103) self.assertEqual(it.next(), 103) self.assertRaises(StopIteration, it.next) it.restart() self.ws.store('ifindTrySingle', 104) self.ws.store('ifindTrySingle', 105) self.assertEqual(it.next(), 105) self.assertRaises(StopIteration, it.next) it.reset() self.assertEqual(it.next(), 105) self.assertRaises(StopIteration, it.next) self.ws.deleteVar('ifindTrySingle') def testIfetchTrySingle(self): """Test ifetchTry with single mode variables""" self.ws.declare('ifetchTrySingle', SINGLE) it = self.ws.ifetchTry('ifetchTrySingle') self.ws.store('ifetchTrySingle', 100) self.assertEqual(it.next(), 100) self.ws.store('ifetchTrySingle', 101) self.assertEqual(it.next(), 101) self.ws.store('ifetchTrySingle', 102) self.ws.store('ifetchTrySingle', 103) self.assertEqual(it.next(), 103) self.assertRaises(StopIteration, it.next) it.restart() self.ws.store('ifetchTrySingle', 104) self.ws.store('ifetchTrySingle', 105) self.assertEqual(it.next(), 105) self.assertRaises(StopIteration, it.next) it.reset() self.assertRaises(StopIteration, it.next) self.ws.deleteVar('ifetchTrySingle') def testFetchTryFile(self): """Test concatenating string using fetchTryFile""" for i in string.ascii_lowercase: self.ws.store('fetchTryFile test', i) r = StringIO.StringIO() while self.ws.fetchTryFile('fetchTryFile test', r): pass self.assertEqual(r.getvalue(), string.ascii_lowercase) def testStoreFile(self): """Test splitting up a file using storeFile""" s = StringIO.StringIO(string.ascii_lowercase) while self.ws.storeFile('storeFile test', s, n=1): pass for i in string.ascii_lowercase: self.assertEqual(i, self.ws.fetchTry('storeFile test')) def testWriteTo(self): """Test writeTo method on object returned by ifindTry""" for i in string.ascii_lowercase: self.ws.store('writeTo', i) for i in range(3): r = StringIO.StringIO() it = self.ws.ifindTry('writeTo') while it.writeTo(r): pass self.assertEqual(r.getvalue(), string.ascii_lowercase) r = StringIO.StringIO() it = self.ws.ifetchTry('writeTo') while it.writeTo(r): pass self.assertEqual(r.getvalue(), string.ascii_lowercase) for i in range(3): r = StringIO.StringIO() it = self.ws.ifindTry('writeTo') while it.writeTo(r): pass self.assertEqual(r.getvalue(), "") if __name__ == '__main__': import sys, os nwsargs = {} try: nwsargs['serverHost'] = os.environ['NWS_HOST'] except: pass try: nwsargs['serverPort'] = int(os.environ['NWS_PORT']) except: pass verbose = os.environ.get('NWS_VERBOSE', False) basicConfig() log = getLogger('NWSTest') if verbose: log.setLevel(DEBUG) unittest.main() nwsclient-1.6.4/test/test_local.py0000755000175000017500000000222311214274541016635 0ustar westonweston#!/usr/bin/env python from nwss.local import NwsLocalServer from nws.client import NetWorkSpace from nws.sleigh import Sleigh from math import sqrt # start the local nws server srv = NwsLocalServer(daemon=False) srv.wait_until_started() # try an nws example ws = NetWorkSpace('foo', serverPort=srv.port) print ws for i in range(10): ws.store('x', 'Value %d' % i) print list(ws.ifetchTry('x')) # try a sleigh example s = Sleigh(nwsPort=srv.port) print s for i in xrange(10): print s.eachElem(sqrt, range(10)) # shutdown the local nws server print 'shutting it down' srv.shutdown() assert not srv.isAlive() try: ws.store('y', 1) print 'did not get expected exception' except: print 'got expected exception' # This hangs on shutdown if False: # start the local nws server print 'starting another server' srv = NwsLocalServer(daemon=False) print 'waiting for it to start' srv.waitUntilStarted() # a quick test ws = NetWorkSpace('bar', serverPort=srv.port) ws.store('x', 'hello') print ws.listVars() # shutdown the local nws server print 'shutting it down, too' srv.shutdown() print 'finished' nwsclient-1.6.4/test/test_quote.py0000755000175000017500000000220611175420457016706 0ustar westonweston#!/usr/bin/env python import unittest from nws.util import msc_quote try: from logging import getLogger, basicConfig, DEBUG except ImportError: DEBUG = 1 class Logger: def __init__(self): self._level = 0 def info(self, msg): if self._level: print msg def setLevel(self, level): self._level = level def getLogger(ign): return Logger() def basicConfig(): pass class QuoteTest(unittest.TestCase): log = getLogger('QuoteTest') def testQuote(self): """Test Windows style quoting.""" examples = [('a', 'a'), (' ', '" "'), ('\\', '\\'), ('a"', '"a\\""'), (' \\', '" \\\\"'), (' \\"', '" \\\\\\""'), ('', '')] for r, q in examples: self.assertEqual(msc_quote(r), q) if __name__ == '__main__': import sys, os verbose = os.environ.get('NWS_VERBOSE', False) basicConfig() log = getLogger('QuoteTest') if verbose: log.setLevel(DEBUG) unittest.main() nwsclient-1.6.4/README.Sleigh0000644000175000017500000001050011175420457015244 0ustar westonwestonPython Sleigh ------------- Sleigh is a Python package, built on top of NetWorkSpaces, that makes it easy to write simple parallel programs. In order to get started with Sleigh, you first have to install the NetWorkSpaces server on at least one machine, and then you'll need to install NetWorkSpaces for Python on each of the machines that you'd like to execute your parallel program. To get started, you'll probably want to work on a single machine, but eventually you'll need to use multiple machines. Of course, if you have a machine with multiple CPUs, Sleigh can be used to take advantage of those CPUs. If you haven't read the primary README file (which covers NetWorkSpaces in general, but not Sleigh in particular), then please read that first. You don't really have to know anything about the basic NetWorkSpaces Python API (NetWorkSpace, NwsServer, etc) to do useful things with Sleigh (as the Sleigh tutorial demonstrates), but just about all of the information in the primary README file will be important to the Sleigh user (installation, online documentation, babelfish, etc). Sleigh isn't separately installed: it is installed as part of NetWorkSpaces for Python. But there are a few things that need to be done in order to run Sleigh scripts, that aren't needed to use the basic NetWorkSpaces API. Most of the issues have to do with the way that Sleigh starts workers. Here are some things to be aware of: 1) If you do a private installation, try to install NetWorkSpaces in the same directory path on all of the machines. On Posix systems, installing NetWorkSpaces into an NFS mounted directory is a good idea, as long as it is mounted with the same name everywhere. 2) Make sure that the directory containing the Python interpreter is in the same directory on all of the workers. It's possible to configure a Sleigh to use different paths on different machines, but that's a more advanced feature. 3) Make sure that Python can find the nws modules on each of the worker machines. With a System installation, that shouldn't be a problem. With a private installation, you'll need to use some method like setting the PYTHONPATH environment variable to point to your nws directory (see README for more information). You'll need to set PYTHONPATH in your shell's startup file (.bashrc, .zshenv, .cshrc, etc) so that it is set when ssh executes PythonNWSSleighWorker.sh. 4) If you're using the sshcmd method for launching remote workers, then you should configure ssh to allow password-less remote execution. By default, ssh will prompt you for a password for every worker that it starts (which could be a lot). Typing the same password even three times gets very boring, so password-less remote execution is worth setting up, if you haven't already done that. See the documentation for ssh-keygen, ssh-agent, and ssh-add for instructions on how to do this. Web Launch ---------- If you need to start workers on machines that don't have an ssh daemon installed on them, you may wish to use the "web launch" method of starting workers. It is more work, since it basically involves manually starting each worker. To use web launch, you need to set the Sleigh launch argument to the string 'web'. The constructor won't return until you've started all of the workers that you wish to use. To start a worker, you have to login to each machine, and perform the following series of steps: 1) Start the Python interpreter. 2) Using a web browser, go to the web interface of the NWS server that is used by the Sleigh master (http://nwsmachine:8766). 3) Click on the workspace of your Sleigh (Sleigh workspaces start with the prefix "sleigh_ride", unless you specified otherwise). 4) Click on the "runMe" variable. 5) Copy the text of one of the values. The text will be something like: import nws.sleigh; nws.sleigh.launch('sleigh_ride_004_tmp', 'n1.xyz.com', 8765) 6) Paste that text into your Python session, executing it as a Python command. When you've started up all the workers you want, go to the Sleigh workspace with a web browser, and click on the "deleteMeWhenAllWorkersStarted" variable, and then click on the "Remove Variable?" link to delete that variable. This signals to the Sleigh constructor that all the workers have been started, and your Sleigh script can continue executing. nwsclient-1.6.4/doc/0000755000175000017500000000000011261702410013707 5ustar westonwestonnwsclient-1.6.4/doc/Python_Introduction0000644000175000017500000001162111175420457017671 0ustar westonwestonIntroduction NetWorkSpaces (NWS) provides a framework to coordinate programs written in scripting languages; NWS currently supports the languages Python, Matlab, and R. This User Guide is for the Python language, and it assumes that the reader is familiar with Python. A Python program uses variables to communicate data from one part of the program to another. For example, 'x = 123' assigns the value 123 to the variable named x. Later portions of the program can reference "x" to use the value 123. This mechanism is generally known as binding. In this case, the binding associates the value 123 with the name "x". The collection of name-value bindings in use by a program is often referred to as its "workspace." Two or more Python programs use NWS to communicate data by means of name-value bindings stored in a network-based workspace (a NetWorkSpace, which in Python is an instance of a NetWorkSpace object). One program creates a binding, while another program reads the value the binding associated with the name. This is clearly quite similar to a traditional workspace; however, a NetWorkSpace differs from a traditional wprkspace in two important ways. First, in a setting in which two or more Python programs are interacting, it would not be unusual for one to attempt to "read" the value of a name before that name has been bound to a value. Rather than receiving an "unbound variable" error, the reading program (by default) simply waits (or "blocks") until the binding occurs. Second, a common usage paradigm involves processing a sequence of values for a given name. One Python program carries out a computation based on the first value, while another might carry out a computation on the second, and so on. To facilitate this paradigm, more than one value may be bound to a name in a workspace and values may be "removed" (fetch) as opposed to read (find). By default, values bound to a name are consumed in first-in-first-out (FIFO) order, but other modes are supported: last-in-first-out (LIFO), multiset (no ordering implied) and single (only the last value bound is retained). Since all its values could be removed, a name can, in fact, have no values associated with it. A NetWorkSpace provides five basic operations: store(), fetch(), fetchTry(), find(), and findTry(). - store() introduces a new binding for a specific name in a given workspace. - fetch() fetches (removes) a value associated with a name. - find() reads a value without removing it. Note that fetch() and find() block if no value is bound to the name. fetchTry() and findTry() are non-blocking; they return an empty value or user-supplied default if no value is available. There are several additional NetWorkSpace operations: - declare() declares a variable name with a specific mode. - deleteVar() deletes a name from a workspace. - listVars() provides a list of variables (bindings) in a workspace. - currentWs() returns the name of the specified workspace. In addition to a NetWorkSpace, a Python client of NWS also uses an NWSServer object. This object is created automatically when a new NetWorkSpace object is created, so you don't need to interact directly with it. However, server object is an attribute of the NetWorkSpace, and you can access it using the Python syntax netWorkSpace_object@server. A NwsServer object supports the following actions: - openWs() connects to a workspace or creates one if the specified workspace does not exist. - useWs() uses a NetWorkSpace without claiming ownership. - deleteWs() explicitly deletes a workspace. - listWss() provides a list of workspaces in the server. - close() closes the connection to a workspace. Depending on the ownership, closing a connection to a workspace can result in removing the workspace. The operations above enable coordination of different programs using NWS. There is also a mechanism, built on top of NWS, called Sleigh (inspired by R's SNOW package) to enable parallel function evaluation. Sleigh is especially useful for running "embarrassingly parallel" programs on multiple networked computers. Once a sleigh is created by specifying the nodes that are participating in computations, you can use: - eachElem() to invoke a function on each element of a list, with the invocations being evaluated concurrently by the sleigh pariticipants. - eachWorker() to have each participant in a sleigh invoke a function. You can use this operation, for example, to build up requisite state prior to an eachElem() invocation. Various data structures (workspaces, name-value bindings, etc.) in a NWS server can be monitored and even modified using a Web interface. In distributed programming, even among cooperating programs, the state of shared data can be hard to understand. The Web interface presents a simple way of checking current state remotely. This tool can also be used for learning and debugging purposes, and for monitoring Python programs as well. nwsclient-1.6.4/doc/nws.client.txt0000644000175000017500000014255311261702407016554 0ustar westonwestonHelp on module nws.client in nws: NAME nws.client - Python API for performing NetWorkSpace operations. FILE /usr/lib/python2.6/dist-packages/nws/client.py MODULE DOCS http://docs.python.org/library/nws.client DESCRIPTION NetWorkSpaces (NWS) is a powerful, open-source software package that makes it easy to use clusters from within scripting languages like Python, R, and Matlab. It uses a Space-based approach, similar to JavaSpaces (TM) for example, that makes it easier to write distributed applications. Example: First start up the NWS server, using the twistd command: % twistd -y /etc/nws.tac Now you can perform operations against it using this module: % python >>> from nws.client import NetWorkSpace >>> nws = NetWorkSpace("test") >>> nws.store("answer", 42) >>> count = nws.fetch("answer") >>> print "The answer is", count CLASSES __builtin__.object NetWorkSpace NwsServer NwsValueIterator exceptions.Exception(exceptions.BaseException) NwsException NwsOperationException NwsDeclarationFailedException NwsNoWorkSpaceException NwsPicklingException NwsUnpicklingException NwsServerException NwsConnectException NwsConnectionDroppedException NwsUnsupportedOperationException class NetWorkSpace(__builtin__.object) | Perform operations against workspaces on NWS servers. | | The NetWorkSpace object is the basic object used to perform | operatations on workspaces. Variables can be declared, created, | deleted, and the values of those variables can be manipulated. You | can think of a workspace as a network accessible python dictionary, | where the variable names are keys in the dictionary, and the | associated values are lists of pickled python objects. The store | method puts a value into the list associated with the specified | variable. The find method returns a single value from a list. | Which value it returns depends on the "mode" of the variable (see | the declare method for more information on the variable mode). If | the list is empty, the find method will not return until a value is | stored in that list. The findTry method works like the find method, | but doesn't wait, returning a default value instead (somewhat like | the dictionary's get method). The fetch method works like the find | method, but it will also remove the value from the list. If | multiple clients are all blocked on a fetch operation, and a value | is stored into that variable, the server guarantees that only one | client will be able to fetch that value. The fetchTry method, not | surprisingly, works like the fetch method, but doesn't wait, | returning a default value instead. | | Methods defined here: | | __getstate__(self) | | __init__(self, wsName='__default', serverHost='localhost', serverPort=8765, useUse=False, server=None, **opt) | Construct a NetWorkSpace object for the specified NwsServer. | | Arguments: | | wsName -- Name of the workspace. There can only be one | workspace on the server with a given name, so two | clients can easily communicate with each other by both | creating a NetWorkSpace object with the same name on the | same server. The first client that creates a workspace | that is willing to take ownership of it, will become the | owner (see the description of the useUse argument below | for more information on workspace ownership). | | serverHost -- Host name of the NWS server. This argument is | ignored if the server argument is not None. The default | value is 'localhost'. | | serverPort -- Port of the NWS server. This argument is ignored | if the server argument is not None. The default value | is 8765. | | useUse -- Boolean value indicating whether you only want to use | this workspace, or if you want to open it (which means | you're willing to take ownership of it, if it's not | already owned). | | The default value is False, meaning you are willing to | take ownership of this workspace. | | server -- NwsServer object to associate with this object. If | the value is None (the default value), then a NwsServer | object will be constructed, using the host and port | specified with the serverHost and serverPort arguments. | | The default value is None. | | Keyword Arguments: | | persistent -- Boolean value indicating whether the workspace | should be persistent or not. If a workspace is | persistent, it won't be purged when the owner | disconnects from the NWS server. Note that only the | client who actually takes ownership of the workspace | can make the workspace persistent. The persistent | argument is effectively ignored if useUse is True, since | that client never becomes the owner of the workspace. | If useUse is False, it is the client who actually | becomes the owner that determines whether it is | persistent or not. That is, after the workspace is | owned, the persistent argument is ignored. The default | value is false. | | create -- Boolean value indicating whether the workspace should | be created if it doesn't already exist. The default | value is true. | | __setstate__(self, d) | | __str__(self) | | currentWs(self) | Return the name of the current workspace. | | ws.currentWs() -> string | | declare(self, varName, mode) | Declare a variable in a workspace with the specified mode. | | ws.declare(varName, mode) | | This method is used to specify a mode other than the default | mode of 'fifo'. Legal values for the mode are: | | 'fifo', 'lifo', 'multi', and 'single' | | In the first three modes, multiple value can be stored in | a variable. If the mode is 'fifo', then values are retrieved | in a "first-in, first-out" fashion. That is, the first value | stored, will be the first value fetched. If the mode is 'lifo', | then values are retrieved in a "last-in, first-out" fashion, | as in a stack. If the mode is 'multi', then the order of | retrieval is pseudorandom. | | The 'single' mode means that only a single value can be | stored in the variable. Each new store operation will overwrite | the current value of the variable. | | If a variable is created using a store operation, then the | mode defaults to 'fifo'. The mode cannot be changed with | subsequent calls to declare, regardless of whether the variable | was originally created using store or declare. | | Arguments: | | varName -- Name of the workspace variable to declare. | | mode -- Mode of the variable. | | deleteVar(self, varName) | Delete a variable from a workspace. | | ws.deleteVar(varName) | | All values of the variable are destroyed, and all currently | blocking fetch and find operations will be aborted. | | Arguments: | | varName -- Name of the workspace variable to delete. | | fetch(self, varName) | Return and remove a value of a variable from a workspace. | | ws.fetch(varName) -> object | | If the variable has no values, the operation will not return | until it does. In other words, this is a "blocking" operation. | fetchTry is the "non-blocking" version of this method. | | Note that if many clients are all trying to fetch from the same | variable, only one client can fetch a given value. Other | clients may have previously seen that value using the find or | findTry method, but only one client can ever fetch or fetchTry a | given value. | | Arguments: | | varName -- Name of the workspace variable to fetch. | | fetchFile(self, varName, fobj) | Return and remove a value of a variable from a workspace. | | ws.fetchFile(varName, fobj) -> number of bytes written to file | | Arguments: | | varName -- Name of the workspace variable to fetch. | | fobj -- File to write data to. | | fetchTry(self, varName, missing=None) | Try to return and remove a value of a variable from a workspace. | | ws.fetchTry(varName[, missing]) -> object | | If the variable has no values, the operation will return | the value specified by "missing", which defaults to None. | | Note that if many clients are all trying to fetchTry from the | same variable, only one client can fetchTry a given value. | Other clients may have previously seen that value using the find | or findTry method, but only one client can ever fetch or | fetchTry a given value. | | Arguments: | | varName -- Name of the workspace variable to fetch. | | missing -- Value to return if the variable has no values. | | fetchTryFile(self, varName, fobj) | Try to return and remove a value of a variable from a workspace. | | ws.fetchTryFile(varName, fobj) -> number of bytes written to file | | Arguments: | | varName -- Name of the workspace variable to fetch. | | fobj -- File to write data to. | | find(self, varName) | Return a value of a variable from a workspace. | | ws.find(varName) -> object | | If the variable has no values, the operation will not return | until it does. In other words, this is a "blocking" operation. | findTry is the "non-blocking" version of this method. | | Note that (unlike fetch) find does not remove the value. The | value remains in the variable. | | Arguments: | | varName -- Name of the workspace variable to find. | | findFile(self, varName, fobj) | Return a value of a variable from a workspace. | | ws.findFile(varName, fobj) -> number of bytes written to file | | Arguments: | | varName -- Name of the workspace variable to find. | | fobj -- File to write data to. | | findTry(self, varName, missing=None) | Try to return a value of a variable in the workspace. | | ws.findTry(varName[, missing]) -> object | | If the variable has no values, the operation will return | the value specified by "missing", which defaults to the value | "None". | | Note that (unlike fetchTry) findTry does not remove the value. | The value remains in the variable. | | Arguments: | | varName -- Name of the workspace variable to use. | | missing -- Value to return if the variable has no values. The | default is None. | | findTryFile(self, varName, fobj) | Try to return a value of a variable in the workspace. | | ws.findTryFile(varName, fobj) -> number of bytes written to file | | Arguments: | | varName -- Name of the workspace variable to use. | | fobj -- File to write data to. | | ifetch(self, varName) | Return a fetch iterator for a workspace variable. | | ws.ifetch(varName) -> iterator | | Unlike ifind, this method doesn't really provide any extra | functionality over the fetch method. It is provided for | completeness, and for those who just like iterators. | | Note that ifetch can be used on FIFO and SINGLE mode variables, | but not LIFO and MULTI mode variables. | | Arguments: | | varName -- Name of the workspace variable to iterator over. | | ifetchTry(self, varName) | Return a fetchTry iterator for a workspace variable. | | ws.ifetchTry(varName) -> iterator | | Unlike ifindTry, this method doesn't really provide any extra | functionality over the fetchTry method. It is provided for | completeness, and for those who just like iterators. | | Note that ifetchTry can be used on FIFO and SINGLE mode | variables, but not LIFO and MULTI mode variables. | | Arguments: | | varName -- Name of the workspace variable to iterator over. | | ifind(self, varName) | Return a find iterator for a workspace variable. | | ws.ifind(varName) -> iterator | | This is very useful if you want to see every value in a variable | without destroying them. Unlike the find method, ifind won't | return the same value repeatedly. When there are no more values | to return, the iterator will block, waiting for a new value to | arrive. | | Note that ifind can be used on FIFO and SINGLE mode variables, | but not LIFO and MULTI mode variables. | | Arguments: | | varName -- Name of the workspace variable to iterate over. | | ifindTry(self, varName) | Return a findTry iterator for a workspace variable. | | ws.ifindTry(varName) -> iterator | | This is very useful if you want to see every value in a variable | without destroying them. Unlike the findTry method, ifindTry | won't return the same value repeatedly. When there are no more | values to return, the iterator finishes. | | Note that ifindTry can be used on FIFO and SINGLE mode | variables, but not LIFO and MULTI mode variables. | | Arguments: | | varName -- Name of the workspace variable to iterate over. | | listVars(self, wsName=None, format='string') | Return a listing of the variables in the workspace. | | ws.listVars([wsName[, format]]) -> string or dictionary | | Arguments: | | wsName -- Name of the workspace to list. The default is | None, which means to use the current workspace. | | format -- Output format to return. Legal values include | 'string' and 'dict'. The 'string' format returns | a string which is suitable for printing. | The 'dict' format returns a dictionary of tuples, where | the first field is the variable name, the second is | the number of values, the third is the number of | fetchers, the fourth is the number of finders, and | the fifth is the variables mode. The default value | is 'string'. | | makefile(self, varName, mode='r') | | store(self, varName, val) | Store a new value into a variable in the workspace. | | ws.store(varName, val) | | Arguments: | | varName -- Name of the workspace variable. | | val -- Value to store in the variable. | | storeFile(self, varName, fobj, n=0) | Store a new value into a variable in the workspace from a file. | | ws.storeFile(varName, fobj[, n]) -> number of bytes read from file | | Note that if there is no more data to read from the file, | storeFile returns 0, and does not store a value into the | workspace variable. | | Arguments: | | varName -- Name of the workspace variable. | | fobj -- File to read data from. | | n -- Maximum number of bytes to read from the file. A value of | zero means to read and store all of the data in the | file. The default value is zero. | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class NwsConnectException(NwsServerException) | Unable to connect to the NWS server. | | Method resolution order: | NwsConnectException | NwsServerException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsConnectionDroppedException(NwsServerException) | NWS server connection dropped. | | Method resolution order: | NwsConnectionDroppedException | NwsServerException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsDeclarationFailedException(NwsOperationException) | Variable declaration failed. | | Method resolution order: | NwsDeclarationFailedException | NwsOperationException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsException(exceptions.Exception) | Base class for all exceptions raised by this module. | | Method resolution order: | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors defined here: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsNoWorkSpaceException(NwsOperationException) | No such workspace. | | Method resolution order: | NwsNoWorkSpaceException | NwsOperationException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsOperationException(NwsException) | Error performing an NWS operation. | | Method resolution order: | NwsOperationException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsPicklingException(NwsOperationException) | Error pickling an object. | | Method resolution order: | NwsPicklingException | NwsOperationException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods defined here: | | __init__(self, exception, wsName, varName) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsServer(__builtin__.object) | Perform operations against the NWS server. | | Operations against workspaces are performed | by using the NetWorkSpace class. | | Methods defined here: | | __getstate__(self) | | __init__(self, host='localhost', port=8765) | Create a connection to the NWS server. | | Arguments: | | host -- Host name of the NWS server. | | port -- Port of the NWS server. | | __setstate__(self, d) | | __str__(self) | | close(self) | Close the connection to the NWS server. | | s.close() | | This will indirectly cause the NWS server to purge all | non-persistent workspaces owned by this client. Purging may | not happen immediately, though, but will depend on the load on | the server. | | deleteWs(self, wsName) | Delete the specfied workspace on the NWS server. | | s.deleteWs(wsName) | | Arguments: | | wsName -- Name of the workspace to delete. | | listWss(self, format='string') | Return a listing of all of the workspaces on the NWS server. | | s.listWss([format]) -> string or dictionary | | The listing is a string, consisting of lines, each ending with a | newline. Each line consists of tab separated fields. The first | field is the workspace name, prefixed with either a '>' or a | space, indicating whether the client owns that workspace or | not. | | mktempWs(self, wsName='__pyws__%d') | Make a temporary workspace on the NWS server. | | s.mktempWs([wsName]) -> string | | The workspace is named using the template and a number | generated by the server that makes the workspace name unique. | | Note that the workspace will be created, but it will not be | owned until some client that is willing to take ownership of it | opens it. | | The return value is the actual name of workspace that was | created. | | Arguments: | | wsName -- Template for the workspace name. This must be a legal | 'format' string, containing only an integer format | specifier. The default is '__pyws__%d'. | | Examples: | | Let's create an NwsServer, call mktempWs to make a | workspace, and then use openWs to create a NetWorkSpace | object for that workspace: | | >>> from nws.client import NwsServer | >>> server = NwsServer() | >>> name = server.mktempWs('example_%d') | >>> workspace = server.openWs(name) | | openWs(self, wsName, space=None, **opt) | Open a workspace. | | s.openWs(wsName[, space]) -> space | | If called without a space argument, this method will construct a | NetWorkSpace object that will be associated with this NwsServer | object, and then perform an open operation with it against the NWS | server. The open operation tells the NWS server that this | client wants to use that workspace, and is willing to take | ownership of it, if it doesn't already exist. | | The space argument is only intended to be used from the | NetWorkSpace constructor. | | The return value is the constructed NetWorkSpace object. | | Arguments: | | wsName -- Name of the workspace to open. If the space argument | is not None, this value must match the space's name. | | space -- NetWorkSpace object to use for the open operation. | If the value is None, then openWs will construct a | NetWorkSpace object, specifying this NwsServer object as | the space's server. Note that this argument is only | intended to be used from the NetWorkSpace constructor. | The default value is None. | | Keyword Arguments: | | persistent -- Boolean value indicating whether the workspace | should be persistent or not. See the description of the | persistent argument in the __init__ method of the | NetWorkSpace class for more information. | | create -- Boolean value indicating whether the workspace should | be created if it doesn't already exist. The default | value is true. | | Examples: | | Let's create an NwsServer, and then use openWs to create an | NetWorkSpace object for a workspace called 'foo': | | >>> from nws.client import NwsServer | >>> server = NwsServer() | >>> workspace = server.openWs('foo') | | Note that this is (nearly) equivalent to: | | >>> from nws.client import NetWorkSpace | >>> workspace = NetWorkSpace('foo') | | useWs(self, wsName, space=None, **opt) | Use a NetWorkSpace object. | | s.useWs(wsName[, space]) -> space | | If called without a space argument, this method will construct a | NetWorkSpace object that will be associated with this NwsServer | object, and then perform a use operation with it against the NWS | server. The use operation tells the NWS server that this client | wants to use that workspace, but is not willing to take | ownership of it. | | The space argument is only intended to be used from the | NetWorkSpace constructor. | | The return value is the constructed NetWorkSpace object. | | Arguments: | | wsName -- Name of the workspace to use. If the space argument | is not None, this value must match the space's name. | | space -- NetWorkSpace object to use for the use operation. | If the value is None, then useWs will construct a | NetWorkSpace object, specifying this NwsServer object as | the space's server. Note that this argument is only | intended to be used from the NetWorkSpace constructor. | The default value is None. | | Keyword Arguments: | | create -- Boolean value indicating whether the workspace should | be created if it doesn't already exist. The default | value is true. | | Examples: | | Let's create an NwsServer, and then use useWs to create an | NetWorkSpace object for a workspace called 'foo': | | >>> from nws.client import NwsServer | >>> server = NwsServer() | >>> workspace = server.useWs('foo') | | Note that this is (nearly) equivalent to: | | >>> from nws.client import NetWorkSpace | >>> workspace = NetWorkSpace('foo', useUse=True) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class NwsServerException(NwsException) | Error communicating with the NWS server. | | Method resolution order: | NwsServerException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsUnpicklingException(NwsOperationException) | Error unpickling an object. | | Method resolution order: | NwsUnpicklingException | NwsOperationException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods defined here: | | __init__(self, exception, wsName, varName, obj) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsUnsupportedOperationException(NwsServerException) | NWS server does not support this operation. | | Method resolution order: | NwsUnsupportedOperationException | NwsServerException | NwsException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Data descriptors inherited from NwsException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __repr__(...) | x.__repr__() <==> repr(x) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __str__(...) | x.__str__() <==> str(x) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class NwsValueIterator(__builtin__.object) | Implements the iterated operations against a workspace variable. | | Instances of this class are returned from the NetWorkSpace ifetch, | ifetchTry, ifind, and ifindTry methods. | | Methods defined here: | | __init__(self, ws, varName, op) | Create an iterator over a workspace varible. | | This constructor is intended for internal use only. | | Arguments: | | ws -- NetWorkSpace object containing the specified variable. | | varName -- Name of the workspace variable to iterate over. | | op -- Operation to perform. | | __iter__(self) | | next(self) | | reset(self) | Reset the iterator to the beginning of the workspace variable. | | This conveniently restores the state of the iterator to when it | was first created. | | restart(self) | Allow the iterator to continue where it left off after stopping. | | The Python iterator protocol requires that iterators continue to | raise StopIteration once they've raised it. The | NwsValueIterator will do that unless and until you call this | method. That can be useful for the "Try" iterators, where you | might want to find all of the values in a variable, and at a | later point see if there are any new values without having to | see the previous ones again. | | writeTo(self, fobj) | Write the next value to the specified file or file-like object. | | it.writeTo(fobj) -> number of bytes written to file | | This is very much like the "next" method, but unlike "next", is | intended to be called explicitly. It provides the same kind of | functionality as the various NetWorkSpace findFile/fetchTryFile | methods. It is the easiest and most memory efficient way to | non-destructively write all of the values of a variable to a | file. | | Arguments: | | fobj -- File to write data to. | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) DATA DICT = 'dict' FIFO = 'fifo' LIFO = 'lifo' MULTI = 'multi' SINGLE = 'single' STRING = 'string' V_FETCHERS = 2 V_FINDERS = 3 V_MODE = 4 V_VALUES = 1 V_VARIABLE = 0 WS_MINE = 1 WS_NAME = 0 WS_NUMVARS = 4 WS_OWNER = 2 WS_PERSISTENT = 3 WS_VARLIST = 5 __all__ = ['NwsServer', 'NetWorkSpace', 'NwsValueIterator', 'NwsExcept... nwsclient-1.6.4/doc/Python_NWS_Tutorial0000644000175000017500000001042411175420457017542 0ustar westonwestonPython NWS Tutorial ------------------- NetWorkSpaces (NWS) is a Python package that makes it very easy for different Python programs running on (potentially) different machines to communicate and coordinate with one another. To get started with Python NWS, you'll first have to install the NWS server, and the Python NWS client. Next, an NWS server must be started. This can be done from a shell using the nws command, as follows: % twistd -ny /etc/nws.tac From another window, start an interactive Python session, and import the NetWorkSpace class: % python >>> from nws.client import NetWorkSpace Next, create a work space called 'bayport': >>> ws = NetWorkSpace('bayport') This is using the NWS server on the local machine. Additional arguments can be used to specify the hostname and port used by the NWS server if neccessary. Once we have a work space, we can write data into a variable using the store method: >>> ws.store('joe', 17) The variable 'joe' now has a value of 17 in our work space. To read that variable we use the find method: >>> age = ws.find('joe') which sets 'age' to 17. Note that the find method will block until the variable 'joe' has a value. That is important when we're trying to read that variable from a different machine. If it didn't block, you might have to repeatedly try to read the variable until it succeeded. Of course, there are times when you don't want to block, but just want to see if some variable has a value. That is done with the findTry method. Let's try reading a variable that doesn't exist using findTry: >>> age = ws.findTry('chet') That assigns 'None' to age, since we haven't stored any value to that variable. If you'd rather have findTry return some other value when the variable doesn't exist (or has no value), you can use the command: >>> age = ws.findTry('chet', 0) which assigns 0 to age. So far, we've been using variables in work spaces in much the same way that global variables are used within a single program. This is certainly useful, but NWS work spaces can also be used to send a sequence of messages from from one program to another. If the store method is executed multiple times, the new values don't overwrite the previous values, they are all saved. Now the find method will only be able to read the first value that was written, but this is where another method, called 'fetch' is useful. The fetch method works the same as find, but in addition, it removes the value. Let's try write multiple values to a variable: >>> n = [16, 19, 25, 22] >>> for i in n: ... ws.store('biff', i) ... To read the values, we just call fetch repeatedly: >>> n = [] >>> for i in range(4): ... n.append(ws.fetch('biff')) ... If we didn't know how many values were stored in a variable, we could have done the following: >>> n = [] >>> while 1: ... t = ws.fetchTry('biff') ... if not t: break ... n.append(t) ... This uses fetchTry, which works like fetch, except that it is non-blocking, just like findTry. Those are the basic operations provided by NWS. It's a good idea to play around with these operations using two Python sessions. That way, you can really transfer data between two different programs. Also, you can see how the blocking operations work. We were pretty careful never to block in any of the examples above because we were only using one Python session. To use two Python session, just execute Python in another window, import NetWorkSpace, and then open the 'bayport' work space. This can be done with the same commands that we used previously, but this time, the command: >>> ws = NetWorkSpace('bayport') won't create the 'bayport' work space, since it already exists. Now you can execute a operation such as: >>> x = ws.fetch('frank') in one session, watch it block for a minute, and then execute store in the other session: >>> ws.store('frank', 18) and see that the fetch in the first session completes. While you're experimenting with these operations, it can be very helpful to use the NWS server's web interface to see what's going on in your work spaces. Just point your web browser to the URL: http://localhost:8766 (If you're using a browser on another machine from the NWS server you'll have to use the appropriate hostname, rather than 'localhost'.) That's all you need to get started. Have fun! nwsclient-1.6.4/doc/nws.client.html0000644000175000017500000030017211261702407016672 0ustar westonweston Python: module nws.client
 
 
nws.client		index
/usr/lib/python2.6/dist-packages/nws/client.py
Module Docs

Python API for performing NetWorkSpace operations.
 
NetWorkSpaces (NWS) is a powerful, open-source software package that
makes it easy to use clusters from within scripting languages like
Python, R, and Matlab.  It uses a Space-based approach, similar to
JavaSpaces (TM) for example, that makes it easier to write distributed
applications.
 
Example:
 
First start up the NWS server, using the twistd command:
 
    % twistd -y /etc/nws.tac
 
Now you can perform operations against it using this module:
 
    % python
    >>> from nws.client import NetWorkSpace
    >>> nws = NetWorkSpace("test")
    >>> nws.store("answer", 42)
    >>> count = nws.fetch("answer")
    >>> print "The answer is", count

 
Modules
       
cPickle
errno
fcntl
os
socket
stat

 
Classes
       
__builtin__.object
NetWorkSpace
NwsServer
NwsValueIterator
exceptions.Exception(exceptions.BaseException)
NwsException
NwsOperationException
NwsDeclarationFailedException
NwsNoWorkSpaceException
NwsPicklingException
NwsUnpicklingException
NwsServerException
NwsConnectException
NwsConnectionDroppedException
NwsUnsupportedOperationException

 
class NetWorkSpace(__builtin__.object)
    Perform operations against workspaces on NWS servers.
 
The NetWorkSpace object is the basic object used to perform
operatations on workspaces.  Variables can be declared, created,
deleted, and the values of those variables can be manipulated.  You
can think of a workspace as a network accessible python dictionary,
where the variable names are keys in the dictionary, and the
associated values are lists of pickled python objects.  The store
method puts a value into the list associated with the specified
variable.  The find method returns a single value from a list.
Which value it returns depends on the "mode" of the variable (see
the declare method for more information on the variable mode).  If
the list is empty, the find method will not return until a value is
stored in that list.  The findTry method works like the find method,
but doesn't wait, returning a default value instead (somewhat like
the dictionary's get method).  The fetch method works like the find
method, but it will also remove the value from the list.  If
multiple clients are all blocked on a fetch operation, and a value
is stored into that variable, the server guarantees that only one
client will be able to fetch that value.  The fetchTry method, not
surprisingly, works like the fetch method, but doesn't wait,
returning a default value instead.
 
  Methods defined here:
__getstate__(self)
__init__(self, wsName='__default', serverHost='localhost', serverPort=8765, useUse=False, server=None, **opt)
Construct a NetWorkSpace object for the specified NwsServer.
 
Arguments:
 
wsName -- Name of the workspace.  There can only be one
        workspace on the server with a given name, so two
        clients can easily communicate with each other by both
        creating a NetWorkSpace object with the same name on the
        same server.  The first client that creates a workspace
        that is willing to take ownership of it, will become the
        owner (see the description of the useUse argument below
        for more information on workspace ownership).
 
serverHost -- Host name of the NWS server.  This argument is
        ignored if the server argument is not None.  The default
        value is 'localhost'.
 
serverPort -- Port of the NWS server.  This argument is ignored
        if the server argument is not None.  The default value
        is 8765.
 
useUse -- Boolean value indicating whether you only want to use
        this workspace, or if you want to open it (which means
        you're willing to take ownership of it, if it's not
        already owned).
 
        The default value is False, meaning you are willing to
        take ownership of this workspace.
 
server -- NwsServer object to associate with this object.  If
        the value is None (the default value), then a NwsServer
        object will be constructed, using the host and port
        specified with the serverHost and serverPort arguments.
 
        The default value is None.
 
Keyword Arguments:
 
persistent -- Boolean value indicating whether the workspace
        should be persistent or not.  If a workspace is
        persistent, it won't be purged when the owner
        disconnects from the NWS server.  Note that only the
        client who actually takes ownership of the workspace
        can make the workspace persistent.  The persistent
        argument is effectively ignored if useUse is True, since
        that client never becomes the owner of the workspace.
        If useUse is False, it is the client who actually
        becomes the owner that determines whether it is
        persistent or not.  That is, after the workspace is
        owned, the persistent argument is ignored.  The default
        value is false.
 
create -- Boolean value indicating whether the workspace should
        be created if it doesn't already exist.  The default
        value is true.
__setstate__(self, d)
__str__(self)
currentWs(self)
Return the name of the current workspace.
 
ws.currentWs() -> string
declare(self, varName, mode)
Declare a variable in a workspace with the specified mode.
 
ws.declare(varName, mode)
 
This method is used to specify a mode other than the default
mode of 'fifo'.  Legal values for the mode are:
 
    'fifo', 'lifo', 'multi', and 'single'
 
In the first three modes, multiple value can be stored in
a variable.  If the mode is 'fifo', then values are retrieved
in a "first-in, first-out" fashion.  That is, the first value
stored, will be the first value fetched.  If the mode is 'lifo',
then values are retrieved in a "last-in, first-out" fashion,
as in a stack.  If the mode is 'multi', then the order of
retrieval is pseudorandom.
 
The 'single' mode means that only a single value can be
stored in the variable.  Each new store operation will overwrite
the current value of the variable.
 
If a variable is created using a store operation, then the
mode defaults to 'fifo'.  The mode cannot be changed with
subsequent calls to declare, regardless of whether the variable
was originally created using store or declare.
 
Arguments:
 
varName -- Name of the workspace variable to declare.
 
mode -- Mode of the variable.
deleteVar(self, varName)
Delete a variable from a workspace.
 
ws.deleteVar(varName)
 
All values of the variable are destroyed, and all currently
blocking fetch and find operations will be aborted.
 
Arguments:
 
varName -- Name of the workspace variable to delete.
fetch(self, varName)
Return and remove a value of a variable from a workspace.
 
ws.fetch(varName) -> object
 
If the variable has no values, the operation will not return
until it does.  In other words, this is a "blocking" operation.
fetchTry is the "non-blocking" version of this method.
 
Note that if many clients are all trying to fetch from the same
variable, only one client can fetch a given value.  Other
clients may have previously seen that value using the find or
findTry method, but only one client can ever fetch or fetchTry a
given value.
 
Arguments:
 
varName -- Name of the workspace variable to fetch.
fetchFile(self, varName, fobj)
Return and remove a value of a variable from a workspace.
 
ws.fetchFile(varName, fobj) -> number of bytes written to file
 
Arguments:
 
varName -- Name of the workspace variable to fetch.
 
fobj -- File to write data to.
fetchTry(self, varName, missing=None)
Try to return and remove a value of a variable from a workspace.
 
ws.fetchTry(varName[, missing]) -> object
 
If the variable has no values, the operation will return
the value specified by "missing", which defaults to None.
 
Note that if many clients are all trying to fetchTry from the
same variable, only one client can fetchTry a given value.
Other clients may have previously seen that value using the find
or findTry method, but only one client can ever fetch or
fetchTry a given value.
 
Arguments:
 
varName -- Name of the workspace variable to fetch.
 
missing -- Value to return if the variable has no values.
fetchTryFile(self, varName, fobj)
Try to return and remove a value of a variable from a workspace.
 
ws.fetchTryFile(varName, fobj) -> number of bytes written to file
 
Arguments:
 
varName -- Name of the workspace variable to fetch.
 
fobj -- File to write data to.
find(self, varName)
Return a value of a variable from a workspace.
 
ws.find(varName) -> object
 
If the variable has no values, the operation will not return
until it does.  In other words, this is a "blocking" operation.
findTry is the "non-blocking" version of this method.
 
Note that (unlike fetch) find does not remove the value.  The
value remains in the variable.
 
Arguments:
 
varName -- Name of the workspace variable to find.
findFile(self, varName, fobj)
Return a value of a variable from a workspace.
 
ws.findFile(varName, fobj) -> number of bytes written to file
 
Arguments:
 
varName -- Name of the workspace variable to find.
 
fobj -- File to write data to.
findTry(self, varName, missing=None)
Try to return a value of a variable in the workspace.
 
ws.findTry(varName[, missing]) -> object
 
If the variable has no values, the operation will return
the value specified by "missing", which defaults to the value
"None".
 
Note that (unlike fetchTry) findTry does not remove the value.
The value remains in the variable.
 
Arguments:
 
varName -- Name of the workspace variable to use.
 
missing -- Value to return if the variable has no values.  The
        default is None.
findTryFile(self, varName, fobj)
Try to return a value of a variable in the workspace.
 
ws.findTryFile(varName, fobj) -> number of bytes written to file
 
Arguments:
 
varName -- Name of the workspace variable to use.
 
fobj -- File to write data to.
ifetch(self, varName)
Return a fetch iterator for a workspace variable.
 
ws.ifetch(varName) -> iterator
 
Unlike ifind, this method doesn't really provide any extra
functionality over the fetch method.  It is provided for
completeness, and for those who just like iterators.
 
Note that ifetch can be used on FIFO and SINGLE mode variables,
but not LIFO and MULTI mode variables.
 
Arguments:
 
varName -- Name of the workspace variable to iterator over.
ifetchTry(self, varName)
Return a fetchTry iterator for a workspace variable.
 
ws.ifetchTry(varName) -> iterator
 
Unlike ifindTry, this method doesn't really provide any extra
functionality over the fetchTry method.  It is provided for
completeness, and for those who just like iterators.
 
Note that ifetchTry can be used on FIFO and SINGLE mode
variables, but not LIFO and MULTI mode variables.
 
Arguments:
 
varName -- Name of the workspace variable to iterator over.
ifind(self, varName)
Return a find iterator for a workspace variable.
 
ws.ifind(varName) -> iterator
 
This is very useful if you want to see every value in a variable
without destroying them.  Unlike the find method, ifind won't
return the same value repeatedly.  When there are no more values
to return, the iterator will block, waiting for a new value to
arrive.
 
Note that ifind can be used on FIFO and SINGLE mode variables,
but not LIFO and MULTI mode variables.
 
Arguments:
 
varName -- Name of the workspace variable to iterate over.
ifindTry(self, varName)
Return a findTry iterator for a workspace variable.
 
ws.ifindTry(varName) -> iterator
 
This is very useful if you want to see every value in a variable
without destroying them.  Unlike the findTry method, ifindTry
won't return the same value repeatedly.  When there are no more
values to return, the iterator finishes.
 
Note that ifindTry can be used on FIFO and SINGLE mode
variables, but not LIFO and MULTI mode variables.
 
Arguments:
 
varName -- Name of the workspace variable to iterate over.
listVars(self, wsName=None, format='string')
Return a listing of the variables in the workspace.
 
ws.listVars([wsName[, format]]) -> string or dictionary
 
Arguments:
 
wsName -- Name of the workspace to list.  The default is
        None, which means to use the current workspace.
 
format -- Output format to return.  Legal values include
        'string' and 'dict'.  The 'string' format returns
        a string which is suitable for printing.
        The 'dict' format returns a dictionary of tuples, where
        the first field is the variable name, the second is
        the number of values, the third is the number of
        fetchers, the fourth is the number of finders, and
        the fifth is the variables mode.  The default value
        is 'string'.
makefile(self, varName, mode='r')
store(self, varName, val)
Store a new value into a variable in the workspace.
 
ws.store(varName, val)
 
Arguments:
 
varName -- Name of the workspace variable.
 
val -- Value to store in the variable.
storeFile(self, varName, fobj, n=0)
Store a new value into a variable in the workspace from a file.
 
ws.storeFile(varName, fobj[, n]) -> number of bytes read from file
 
Note that if there is no more data to read from the file,
storeFile returns 0, and does not store a value into the
workspace variable.
 
Arguments:
 
varName -- Name of the workspace variable.
 
fobj -- File to read data from.
 
n -- Maximum number of bytes to read from the file.  A value of
        zero means to read and store all of the data in the
        file.  The default value is zero.
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)

 
class NwsConnectException(NwsServerException)
    Unable to connect to the NWS server.
 
 
Method resolution order:
NwsConnectException
NwsServerException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsConnectionDroppedException(NwsServerException)
    NWS server connection dropped.
 
 
Method resolution order:
NwsConnectionDroppedException
NwsServerException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsDeclarationFailedException(NwsOperationException)
    Variable declaration failed.
 
 
Method resolution order:
NwsDeclarationFailedException
NwsOperationException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsException(exceptions.Exception)
    Base class for all exceptions raised by this module.
 
 
Method resolution order:
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors defined here:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsNoWorkSpaceException(NwsOperationException)
    No such workspace.
 
 
Method resolution order:
NwsNoWorkSpaceException
NwsOperationException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsOperationException(NwsException)
    Error performing an NWS operation.
 
 
Method resolution order:
NwsOperationException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsPicklingException(NwsOperationException)
    Error pickling an object.
 
 
Method resolution order:
NwsPicklingException
NwsOperationException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods defined here:
__init__(self, exception, wsName, varName)
__str__(self)
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsServer(__builtin__.object)
    Perform operations against the NWS server.
 
Operations against workspaces are performed
by using the NetWorkSpace class.
 
  Methods defined here:
__getstate__(self)
__init__(self, host='localhost', port=8765)
Create a connection to the NWS server.
 
Arguments:
 
host -- Host name of the NWS server.
 
port -- Port of the NWS server.
__setstate__(self, d)
__str__(self)
close(self)
Close the connection to the NWS server.
 
s.close()
 
This will indirectly cause the NWS server to purge all
non-persistent workspaces owned by this client.  Purging may
not happen immediately, though, but will depend on the load on
the server.
deleteWs(self, wsName)
Delete the specfied workspace on the NWS server.
 
s.deleteWs(wsName)
 
Arguments:
 
wsName -- Name of the workspace to delete.
listWss(self, format='string')
Return a listing of all of the workspaces on the NWS server.
 
s.listWss([format]) -> string or dictionary
 
The listing is a string, consisting of lines, each ending with a
newline.  Each line consists of tab separated fields.  The first
field is the workspace name, prefixed with either a '>' or a
space, indicating whether the client owns that workspace or
not.
mktempWs(self, wsName='__pyws__%d')
Make a temporary workspace on the NWS server.
 
s.mktempWs([wsName]) -> string
 
The workspace is named using the template and a number
generated by the server that makes the workspace name unique.
 
Note that the workspace will be created, but it will not be
owned until some client that is willing to take ownership of it
opens it.
 
The return value is the actual name of workspace that was
created.
 
Arguments:
 
wsName -- Template for the workspace name.  This must be a legal
        'format' string, containing only an integer format
        specifier.  The default is '__pyws__%d'.
 
Examples:
 
    Let's create an NwsServer, call mktempWs to make a
    workspace, and then use openWs to create a NetWorkSpace
    object for that workspace:
 
        >>> from nws.client import NwsServer
        >>> server = NwsServer()
        >>> name = server.mktempWs('example_%d')
        >>> workspace = server.openWs(name)
openWs(self, wsName, space=None, **opt)
Open a workspace.
 
s.openWs(wsName[, space]) -> space
 
If called without a space argument, this method will construct a
NetWorkSpace object that will be associated with this NwsServer
object, and then perform an open operation with it against the NWS
server.  The open operation tells the NWS server that this
client wants to use that workspace, and is willing to take
ownership of it, if it doesn't already exist.
 
The space argument is only intended to be used from the
NetWorkSpace constructor.
 
The return value is the constructed NetWorkSpace object.
 
Arguments:
 
wsName -- Name of the workspace to open.  If the space argument
        is not None, this value must match the space's name.
 
space -- NetWorkSpace object to use for the open operation.
        If the value is None, then openWs will construct a
        NetWorkSpace object, specifying this NwsServer object as
        the space's server.  Note that this argument is only
        intended to be used from the NetWorkSpace constructor.
        The default value is None.
 
Keyword Arguments:
 
persistent -- Boolean value indicating whether the workspace
        should be persistent or not.  See the description of the
        persistent argument in the __init__ method of the
        NetWorkSpace class for more information.
 
create -- Boolean value indicating whether the workspace should
        be created if it doesn't already exist.  The default
        value is true.
 
Examples:
 
    Let's create an NwsServer, and then use openWs to create an
    NetWorkSpace object for a workspace called 'foo':
 
        >>> from nws.client import NwsServer
        >>> server = NwsServer()
        >>> workspace = server.openWs('foo')
 
    Note that this is (nearly) equivalent to:
 
        >>> from nws.client import NetWorkSpace
        >>> workspace = NetWorkSpace('foo')
useWs(self, wsName, space=None, **opt)
Use a NetWorkSpace object.
 
s.useWs(wsName[, space]) -> space
 
If called without a space argument, this method will construct a
NetWorkSpace object that will be associated with this NwsServer
object, and then perform a use operation with it against the NWS
server.  The use operation tells the NWS server that this client
wants to use that workspace, but is not willing to take
ownership of it.
 
The space argument is only intended to be used from the
NetWorkSpace constructor.
 
The return value is the constructed NetWorkSpace object.
 
Arguments:
 
wsName -- Name of the workspace to use.  If the space argument
        is not None, this value must match the space's name.
 
space -- NetWorkSpace object to use for the use operation.
        If the value is None, then useWs will construct a
        NetWorkSpace object, specifying this NwsServer object as
        the space's server.  Note that this argument is only
        intended to be used from the NetWorkSpace constructor.
        The default value is None.
 
Keyword Arguments:
 
create -- Boolean value indicating whether the workspace should
        be created if it doesn't already exist.  The default
        value is true.
 
Examples:
 
    Let's create an NwsServer, and then use useWs to create an
    NetWorkSpace object for a workspace called 'foo':
 
        >>> from nws.client import NwsServer
        >>> server = NwsServer()
        >>> workspace = server.useWs('foo')
 
    Note that this is (nearly) equivalent to:
 
        >>> from nws.client import NetWorkSpace
        >>> workspace = NetWorkSpace('foo', useUse=True)
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)

 
class NwsServerException(NwsException)
    Error communicating with the NWS server.
 
 
Method resolution order:
NwsServerException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsUnpicklingException(NwsOperationException)
    Error unpickling an object.
 
 
Method resolution order:
NwsUnpicklingException
NwsOperationException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods defined here:
__init__(self, exception, wsName, varName, obj)
__str__(self)
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsUnsupportedOperationException(NwsServerException)
    NWS server does not support this operation.
 
 
Method resolution order:
NwsUnsupportedOperationException
NwsServerException
NwsException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Data descriptors inherited from NwsException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__repr__(...)
x.__repr__() <==> repr(x)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__str__(...)
x.__str__() <==> str(x)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class NwsValueIterator(__builtin__.object)
    Implements the iterated operations against a workspace variable.
 
Instances of this class are returned from the NetWorkSpace ifetch,
ifetchTry, ifind, and ifindTry methods.
 
  Methods defined here:
__init__(self, ws, varName, op)
Create an iterator over a workspace varible.
 
This constructor is intended for internal use only.
 
Arguments:
 
ws -- NetWorkSpace object containing the specified variable.
 
varName -- Name of the workspace variable to iterate over.
 
op -- Operation to perform.
__iter__(self)
next(self)
reset(self)
Reset the iterator to the beginning of the workspace variable.
 
This conveniently restores the state of the iterator to when it
was first created.
restart(self)
Allow the iterator to continue where it left off after stopping.
 
The Python iterator protocol requires that iterators continue to
raise StopIteration once they've raised it.  The
NwsValueIterator will do that unless and until you call this
method.  That can be useful for the "Try" iterators, where you
might want to find all of the values in a variable, and at a
later point see if there are any new values without having to
see the previous ones again.
writeTo(self, fobj)
Write the next value to the specified file or file-like object.
 
it.writeTo(fobj) -> number of bytes written to file
 
This is very much like the "next" method, but unlike "next", is
intended to be called explicitly.  It provides the same kind of
functionality as the various NetWorkSpace findFile/fetchTryFile
methods.  It is the easiest and most memory efficient way to
non-destructively write all of the values of a variable to a
file.
 
Arguments:
 
fobj -- File to write data to.
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)

 
Data
        DICT = 'dict'
FIFO = 'fifo'
LIFO = 'lifo'
MULTI = 'multi'
SINGLE = 'single'
STRING = 'string'
V_FETCHERS = 2
V_FINDERS = 3
V_MODE = 4
V_VALUES = 1
V_VARIABLE = 0
WS_MINE = 1
WS_NAME = 0
WS_NUMVARS = 4
WS_OWNER = 2
WS_PERSISTENT = 3
WS_VARLIST = 5
__all__ = ['NwsServer', 'NetWorkSpace', 'NwsValueIterator', 'NwsException', 'NwsServerException', 'NwsConnectException', 'NwsConnectionDroppedException', 'NwsUnsupportedOperationException', 'NwsOperationException', 'NwsNoWorkSpaceException', 'NwsDeclarationFailedException', 'NwsPicklingException', 'NwsUnpicklingException', 'FIFO', 'LIFO', 'MULTI', 'SINGLE', 'STRING', 'DICT', 'WS_NAME', ...]
nwsclient-1.6.4/doc/nws.sleigh.html0000644000175000017500000031354311261702407016675 0ustar westonweston Python: module nws.sleigh
 
 
nws.sleigh (version 1.6.3)		index
/usr/lib/python2.6/dist-packages/nws/sleigh.py
Module Docs

Python API for parallel programming using NetWorkSpaces.
 
The sleigh module, built on top of NetWorkSpaces (NWS), makes it very
easy to write simple parallel programs.  It contains the Sleigh class,
which provides two basic methods for executing tasks in parallel:
eachElem and eachWorker.
 
eachElem is used to execute a specified function multiple times in
parallel with a varying set of arguments.  eachWorker is used to execute
a function exactly once on every worker in the sleigh with a fixed set
of arguments.
 
Example:
 
First start up the NWS server, using the twistd command:
 
    % twistd -y /etc/nws.tac
 
Now you can create a sleigh to execute python code in parallel:
 
    % python
    >>> from nws.sleigh import Sleigh
    >>> s = Sleigh()
    >>> import math
    >>> result = s.eachElem(math.exp, range(10))
    >>> print "The answer is", result

 
Modules
       
cPickle
copy_reg
new
os
pickle
platform
random
socket
string
subprocess
sys
time
traceback

 
Classes
       
__builtin__.object
Sleigh
SleighPending
SleighResultIterator
exceptions.Exception(exceptions.BaseException)
SleighException
SleighGatheredException
SleighIllegalValueException
SleighJoinException
SleighNotAllowedException
SleighNwsException
SleighOccupiedException
SleighScriptException
SleighStoppedException
SleighTaskException

 
class Sleigh(__builtin__.object)
    Represents a collection of python processes used to execute tasks.
 
The sleigh allows python functions, methods, and expressions to be
executed in parallel using the eachElem and eachWorker methods.
 
The sleigh workers are started when the Sleigh object is
constructed.  When tasks are submitted to the sleigh, using the
eachWorker and eachElem methods, the workers will execute the tasks,
and return the results.  When the stop method is called, the workers
are stopped.
 
Note that a given python program can create multiple Sleigh objects,
which will each have it's own set of workers.  This could be useful
if tasks have different requirements.  For example, you could create
a Linux sleigh and a Windows sleigh, and submit Excel tasks only to
your Windows sleigh.
 
  Methods defined here:
__init__(self, *deprecated, **kw)
Start the remote python processes used to execute tasks.
 
Keyword arguments:
 
launch -- Specifies the method of starting workers.  If this
        argument is set to the string 'local', then the workers
        are executed on the local machine.  If it is set to the
        string 'web', then web launching is used.  Otherwise,
        the launch argument must specify a function (such as
        nws.sleigh.sshcmd) that returns a list of command
        arguments use to execute the workers on the machines
        specified by the nodeList argument.  The default value
        is 'local'.
 
workerCount -- Number of workers to start if the launch argument
        is set to 'local' (which is the default value of launch).
        This argument is ignored if launch is not set to
        'local'.  The default value is 3.
 
nodeList -- List of hosts on which to execute workers, if the
        launch argument is set to a function.  This argument is
        ignored if launch is set to 'local' or 'web'.  The
        default value is ['localhost', 'localhost',
        'localhost'].
 
nwsHost -- Host name of the machine where the NWS server is
        executing.
 
nwsPort -- Port to connect to the NWS server.
 
nwsHostRemote -- Host name of the machine that workers should
        use to connect to the NWS server.  This is useful in
        conjunction with the sshforwardcmd function (see the
        description of scriptExec).  The default is the value of
        the nwsHost argument.
 
nwsPortRemote -- Port that workers should use to connect to the
        NWS server.  This is useful in conjunction with the
        sshforwardcmd function (see the description of
        scriptExec).  The default is the value of the nwsPort
        argument.
 
scriptExec -- Python function returning a list of command
        arguments to execute the worker script.  This list of
        command arguments is appended to the list returned by
        the launch function.  The default value is the envcmd
        function (defined in this module), which uses the
        standard 'env' command to execute the script with the
        appropriate environment for the worker.
 
scriptDir -- Directory on the worker that contains the execution
        script.
 
scriptName -- Name of the script that executes the worker on the
        remote machines.  This defaults to
        PythonNWSSleighWorker.sh on Unix, and
        PythonNWSSleighWorker.py on Windows.
 
modulePath -- Directory path to add to sys.path on workers.
        This is often useful for giving the workers access to
        python modules that define the python function to be
        executed by eachWorker or eachElem.  The default value
        is None.
 
workingDir -- Directory path to use as the current working
        directory for the workers.  The default value is the
        current working directory of the sleigh master.
 
logDir -- Directory in which to create the worker log files.
        The default value is None, which leaves the decision
        to the sleigh worker scripts, which generally uses the
        remote system's temporary directory.
 
user -- User name to use for remote execution of worker.  This
        argument may be ignored, depending the specified
        launch function.  The default is None, which allows the
        remote execution mechanism decide.
 
wsNameTemplate -- Template for the sleigh workspace name.  This
        must be a legal 'format' string, containing only an
        integer format specifier.  The default is
        'sleigh_ride_%04d'.
 
userWsNameTemplate -- Template for the user's workspace name.
        This must be a legal 'format' string, containing only
        an integer format specifier.  The default is
        'sleigh_user_%04d'.
 
verbose -- Boolean flag used for displaying debug messages.
        Debug messages will be sent to stderr.  This will also
        cause the worker processes to write debug messages to
        files prefixed with 'sleigh_ride' in their current
        working directory (as controled by the workerDir
        argument).  The default value is False.
 
redirectOutput -- Boolean flag used to control whether messages
        written by the worker functions to stdout and stderr
        should be redirected to a workspace variable.  If False,
        they will go to the output file if verbose is True,
        otherwise to the null device.  The default value is True.
__str__(self)
eachElem(self, fun, elementArgs=[[]], fixedArgs=[], **kw)
Execute a function, method, or expression for each element in
the specified list(s).
 
s.eachElem(fun, elementArgs[, fixedArgs]) -> list or SleighPending
 
The results are normally returned as a list, unless the blocking
arguments is set to False, in which case, a SleighPending object
is returned.
 
Arguments:
 
fun -- Function, method, or python expression to execute.  To
        execute a method, you must specify a bound method
        object.  If the function or defining class is defined in
        a module, the workers will attempt to import that
        module.  If that module isn't available to the worker
        (because it's a non-standard module, not in the
        PYTHONPATH), then the worker is not be able to execute
        those tasks.
 
        To execute a python expression, you must specify it as a
        string.  Leading whitespace is automatically stripped to
        avoid a common source of python syntax errors.
 
elementArgs -- List of arguments to pass to the function or method
        that need to be different for different tasks.  In
        general, this is a list of iterable objects, such as lists,
        each containing the values to use for a given argument of
        the different tasks.
 
        If your function needs only one varying argument of a
        simple type, you can specify it without the outer list.
 
        Note that for a python expression, the list of arguments
        is passed to the expression as a global variable named
        'SleighArguments'.
 
fixedArgs -- List of additional arguments that are
        fixed/constant for each task.  Normally, they are
        appended to the arguments specified by elementArgs, but
        the order can be altered using the argPermute argument
        described below.
 
        The default value is an empty list, which means that no
        extra arguments are passed to the function.
 
        Note that for a python expression, the list of arguments
        is passed to the expression as a global variable named
        'SleighArguments'.
 
Keyword arguments:
 
type -- Indicates the type of function invocation to perform.
        This can be either 'invoke', 'define', or 'eval'.
        If the fun argument is a function or bound method, then
        the default value is 'invoke'.  If the fun argument is
        a string, then the default value is 'eval' (a value of
        'invoke' or 'define' is illegal for python expressions).
 
blocking -- A boolean value that indicates whether to wait for
        the results, or to return as soon as the tasks have been
        submitted.  If set to False, eachElem will return a
        SleighPending object that is used to monitor the status
        of the tasks, and to eventually retrieve the results.
        You must wait for the results to be complete before
        executing any further tasks on the sleigh, or a
        SleighOccupiedException will be raised.
 
        If blocking is set to False, then the loadFactor
        argument is disabled and ignored.  Note that it's
        unlikely that you'll need to turn off blocking in
        eachElem.  Non-blocking mode is more useful in
        eachWorker.
 
        The default value is True.
 
argPermute -- List that maps the specified arguments to the
        actual arguments of the execution function.  By
        "specified arguments", I mean the items extracted from
        elementArgs, followed by fixedArgs.  (Note that unless
        you are specifying both elementArgs and fixedArgs, you
        probably don't need to use argPermute.)  The items in
        the argPermute list are used as indexes into the
        "specified arguments" list.  The length of argPermute
        determines the number of arguments passed to the
        execution function, which would normally be the length
        of the specified arguments list, but this is not
        required.  For example, setting argPermute to an empty
        list would cause the execution function to be called
        without any arguments (although elementArgs would still
        be required, and would be used to determine the number
        of tasks to execute).
 
        The default behaviour is to pass the execution function
        the arguments specified by elementArgs, followed by the
        arguments from fixedArgs, which is equivalent to setting
        argPermute to:
 
            n = len(elementArgs) + len(fixedArgs)
            argPermute = range(n)
 
        If you wished to reverse the order of the arguments, you
        could then modify argPermute:
 
            argPermute.reverse()
 
        But, more realistically, you need to interleave the
        fixed arguments with the varying arguments.  For
        example, your execution function takes on fixed
        argument, followed by two that vary, you would set
        argPermute to:
 
            argPermute=[1,2,0]
 
loadFactor -- Maximum number of tasks per worker to put into the
        workspace at one time.  This can become important if you
        are executing a very large number of tasks.  Setting
        loadFactor to 3 will probably keep enough tasks
        available in the workspace to keep the workers busy,
        without flooding the workspace and bogging down the NWS
        server.
 
        The default behaviour is to submit all of the tasks
        to the sleigh workspace immediately.
 
accumulator -- A function or callable object that will be called
        for each result as they arrive.  The first argument to
        the function is a list of result values, and the second
        argument is a list of indexes, which identifies which task.
 
        The arguments to the accumulator function are lists since
        in the future, we plan to allow tasks to be "chunked" to
        improve performance of small tasks.
 
        Note that bound methods can be very useful accumulators.
eachWorker(self, fun, *workerArgs, **kw)
Execute a function, method, or expression on each worker of sleigh.
 
s.eachWorker(fun[, ...]) -> list or SleighPending
 
The results are normally returned as a list, unless
the blocking arguments is set to False, in which case, a
SleighPending object is returned.
 
Arguments:
 
fun -- Function, method, or python expression to execute.  To
        execute a method, you must specify a bound method
        object.  If the function or defining class is defined in
        a module, the workers will attempt to import that
        module.  If that module isn't available to the worker
        (because it's a non-standard module, not in the
        PYTHONPATH), then the worker is not be able to execute
        those tasks.
 
        To execute a python expression, you must specify it as a
        string.  Leading whitespace is automatically stripped to
        avoid a common source of python syntax errors.
 
Optional arguments:
 
*workerArgs -- Arguments to pass to the function or method.
        Specify whatever arguments the function requires,
        including no arguments.  The exact same set of arguments
        will be used for each worker (unlike eachElem).  For a
        python expression, these arguments are passed to the
        expression as a global variable named 'SleighArguments'.
 
Keyword arguments:
 
type -- Indicates the type of function invocation to perform.
        This can be either 'invoke', 'define', or 'eval'.
        If the fun argument is a function or bound method, then
        the default value is 'invoke'.  If the fun argument is
        a string, then the default value is 'eval' (a value of
        'invoke' or 'define' is illegal for python expressions).
 
blocking -- A boolean value that indicates whether to wait for
        the results, or to return as soon as the tasks have been
        submitted.  If set to False, eachWorker will return a
        SleighPending object that is used to monitor the status
        of the tasks, and to eventually retrieve the results.
        You must wait for the results to be complete before
        executing any further tasks on the sleigh, or a
        SleighOccupiedException will be raised.
 
        This argument is important if you want the master to be
        able to interact/communicate with the workers, via NWS
        operations, for example.  This allows you to implement
        more complex parallel or distributed programs.
 
        The default value is True.
 
accumulator -- A function or callable object that will be called
        for each result as they arrive.  The first argument to
        the function is a list of result values, and the second
        argument is a list of indexes, which identifies which task.
 
        The arguments to the accumulator function are lists since
        in the future, we plan to allow tasks to be "chunked" to
        improve performance of small tasks.
 
        Note that bound methods can be very useful accumulators.
imap(self, fun, *iterables, **kw)
Return an iterator whose values are returned from the function
evaluated with a argument tuple taken from the given iterable.
Stops when the shortest of the iterables is exhausted.
 
s.imap(fun, *iterables[, **kw]) -> SleighResultIterator
 
This is intended to be very similar to the imap function
in the itertools module.  Other than being a method, rather than
a function, this method takes additional, keyword arguments, and
the iterator that is returned has special methods and properties.
See the SleighResultIterator documentation for more information.
 
Arguments:
 
fun -- Function, method, or python expression to execute.  To
        execute a method, you must specify a bound method
        object.  If the function or defining class is defined in
        a module, the workers will attempt to import that
        module.  If that module isn't available to the worker
        (because it's a non-standard module, not in the
        PYTHONPATH), then the worker is not be able to execute
        those tasks.
 
        To execute a python expression, you must specify it as a
        string.  Leading whitespace is automatically stripped to
        avoid a common source of python syntax errors.
 
 
*iterables -- One or more iterables, one for each argument needed
        by the function.
 
Keyword arguments:
 
type -- Indicates the type of function invocation to perform.
        This can be either 'invoke', 'define', or 'eval'.
        If the fun argument is a function or bound method, then
        the default value is 'invoke'.  If the fun argument is
        a string, then the default value is 'eval' (a value of
        'invoke' or 'define' is illegal for python expressions).
 
loadFactor -- Maximum number of tasks per worker to put into the
        workspace at one time.  This can become important if you
        are executing a very large number of tasks (and essential
        if submitting infinite tasks).  The default value of 10
        will probably keep enough tasks available in the workspace
        to keep the workers busy, without flooding the workspace
        and bogging down the NWS server.
starmap(self, fun, iterable, **kw)
Return an iterator whose values are returned from the function
evaluated with a argument tuple taken from the given iterable.
Stops when the shortest of the iterables is exhausted.
 
s.starmap(fun, iterable[, **kw]) -> SleighResultIterator
 
This is intended to be very similar to the starmap function
in the itertools module.  Other than being a method, rather than
a function, this method takes additional, optional arguments, and
the iterator that is returned has special methods and properties.
See the SleighResultIterator documentation for more information.
 
Arguments:
 
fun -- Function, method, or python expression to execute.  To
        execute a method, you must specify a bound method
        object.  If the function or defining class is defined in
        a module, the workers will attempt to import that
        module.  If that module isn't available to the worker
        (because it's a non-standard module, not in the
        PYTHONPATH), then the worker is not be able to execute
        those tasks.
 
        To execute a python expression, you must specify it as a
        string.  Leading whitespace is automatically stripped to
        avoid a common source of python syntax errors.
 
 
iterable -- Returns argument tuples for the function.
 
Keyword arguments:
 
type -- Indicates the type of function invocation to perform.
        This can be either 'invoke', 'define', or 'eval'.
        If the fun argument is a function or bound method, then
        the default value is 'invoke'.  If the fun argument is
        a string, then the default value is 'eval' (a value of
        'invoke' or 'define' is illegal for python expressions).
 
loadFactor -- Maximum number of tasks per worker to put into the
        workspace at one time.  This can become important if you
        are executing a very large number of tasks (and essential
        if submitting infinite tasks).  The default value of 10
        will probably keep enough tasks available in the workspace
        to keep the workers busy, without flooding the workspace
        and bogging down the NWS server.
status(self, closeGroup=False, timeout=0.0)
Return the status of the worker group.
 
s.status(closeGroup, timeout) -> numworkers, closed
 
The status includes the number of workers that have joined the
group so far, and a flag that indicates whether the group has
been closed (meaning that no more workers can join).  Normally,
the group is automatically closed when all the workers that were
listed in the constructor have joined.  However, this method
allows you to force the group to close after the timeout
expires.  This can be particularly useful if you are running
on a large number of nodes, and some of the nodes are slow or
unreliable.  If some of the workers are never started, the group
will never close, and no tasks will ever execute.
 
Arguments:
 
closeGroup -- Boolean flag indicating whether to close the
        group.  If True, the group will be closed, after the
        specified timeout.  The default value is False.
 
timeout -- Number of seconds to wait for the group to close
        before returning.  The default value is 0.0.
stop(self)
Stop the remote processes and delete the sleigh workspace.
 
s.stop()
workerInfo(self)
Return information about the sleigh workers.
 
This returns a list of dictionaries, one for each worker.
Note that if this is called before all of the workers have
"joined", then you may get a partial list.  This can be
useful in conjunction with the status method.
 
The keys in the dictionaries are:
    "nws":     nws version used by the worker
    "python":  Python version used by the worker
    "pid":     process id of the worker process
    "rank":    worker rank (0 to workerCount - 1)
    "host":    name of worker machine (not necessarily the same as 
               the value in nodeList)
    "logfile": path of the log file on the worker machine
    "os":      operating system of the worker machine
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)

 
class SleighException(exceptions.Exception)
    Base class for all exceptions raised by this module.
 
 
Method resolution order:
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods defined here:
__repr__(self)
__str__(self)
Data descriptors defined here:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighGatheredException(SleighException)
    Results already gathered.
 
 
Method resolution order:
SleighGatheredException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighIllegalValueException(SleighException)
    Illegal value specified.
 
 
Method resolution order:
SleighIllegalValueException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighJoinException(SleighException)
    Too late to join worker group.
 
 
Method resolution order:
SleighJoinException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighNotAllowedException(SleighException)
    Sleigh construction not allowed.
 
 
Method resolution order:
SleighNotAllowedException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighNwsException(SleighException)
    Error performing NWS operation.
 
 
Method resolution order:
SleighNwsException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighOccupiedException(SleighException)
    Sleigh is occupied.
 
 
Method resolution order:
SleighOccupiedException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighPending(__builtin__.object)
    Represents a sleigh eachWorker/eachElem invocation in progress.
 
This is returned from the eachElem and eachWorker operations
when executed asynchronously.  It allows you to check for
the number of tasks left to be executed, and to wait for
the results of the operation to be returned.
 
  Methods defined here:
__init__(self, nws, numTasks, barrierName, sleighState, accumulator)
Create an object that represents the pending sleigh operation.
 
This constructor is intended for internal use only.
 
Arguments:
 
nws -- Sleigh NetWorkSpace object.
 
numTasks -- Number of tasks that were submitted.
 
barrierName -- Name of the barrier to wait at when complete.
 
sleighState -- Object representing the current state of the
        sleigh.
 
accumulator -- Function to call with results as they arrive.
__str__(self)
check(self)
Return the number of tasks still outstanding.
 
p.check() -> integer
wait(self)
Wait for and return the list of results.
 
p.wait() -> list
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)

 
class SleighResultIterator(__builtin__.object)
    Returns results from tasks submitted to the sleigh.
 
Instances of this class are returned from the Sleigh imap, and
starmap methods.
 
  Methods defined here:
__init__(self, task, taskIter, starIter, nws, numSubmitted, sleighState)
Create an iterator over the task results.
 
This constructor is intended for internal use only.
 
Arguments:
 
task -- Partially initialized Task object.
 
taskIter -- Iterator over the task arguments.
 
starIter -- Is this a "star" iteration?
 
nws -- Sleigh NetWorkSpace object.
 
numSubmitted -- Number of tasks already submitted.
 
sleighState -- Part of the Sleigh objects internal state.
__iter__(self)
next(self)
shutdown(self)
Stop submitting tasks from the iterator.
 
This method is a less drastic version of "stop".  It is expected
that you will keep retrieving results that have already been
submitted, but no new tasks will be submitted, regardless of
what tasks were originally specified to imap or starmap.  The
sleigh object will continue to be "occupied" until all results of
the pending tasks have been retreived.
stop(self)
Stop the iterator, flushing any pending results.
 
This method is useful if you're done with the iterator, and
don't want to retrieve anymore results.  After calling stop,
you can submit more tasks to the sleigh (that is, it will
no longer be "occupied".
Data descriptors defined here:
__dict__
dictionary for instance variables (if defined)
__weakref__
list of weak references to the object (if defined)
buffered
Number of buffered task results.
returned
Number of task results returned.
stopped
Is the iterator stopped?
submitted
Number of submitted tasks.

 
class SleighScriptException(SleighException)
    Unable to find sleigh worker script.
 
 
Method resolution order:
SleighScriptException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighStoppedException(SleighException)
    Sleigh is stopped.
 
 
Method resolution order:
SleighStoppedException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
class SleighTaskException(SleighException)
    Error executing a task.
 
 
Method resolution order:
SleighTaskException
SleighException
exceptions.Exception
exceptions.BaseException
__builtin__.object
Methods inherited from SleighException:
__repr__(self)
__str__(self)
Data descriptors inherited from SleighException:
__weakref__
list of weak references to the object (if defined)
Methods inherited from exceptions.Exception:
__init__(...)
x.__init__(...) initializes x; see x.__class__.__doc__ for signature
Data and other attributes inherited from exceptions.Exception:
__new__ = <built-in method __new__ of type object at 0x8178c0>
T.__new__(S, ...) -> a new object with type S, a subtype of T
Methods inherited from exceptions.BaseException:
__delattr__(...)
x.__delattr__('name') <==> del x.name
__getattribute__(...)
x.__getattribute__('name') <==> x.name
__getitem__(...)
x.__getitem__(y) <==> x[y]
__getslice__(...)
x.__getslice__(i, j) <==> x[i:j]
 
Use of negative indices is not supported.
__reduce__(...)
__setattr__(...)
x.__setattr__('name', value) <==> x.name = value
__setstate__(...)
__unicode__(...)
Data descriptors inherited from exceptions.BaseException:
__dict__
args
message

 
Data
        __all__ = ['Sleigh', 'SleighPending', 'SleighResultIterator', 'SleighException', 'SleighNwsException', 'SleighGatheredException', 'SleighStoppedException', 'SleighOccupiedException', 'SleighIllegalValueException', 'SleighScriptException', 'SleighTaskException', 'SleighJoinException', 'SleighNotAllowedException']
__version__ = '1.6.3'
nwsclient-1.6.4/doc/nws.sleigh.txt0000644000175000017500000015124311261702407016545 0ustar westonwestonHelp on module nws.sleigh in nws: NAME nws.sleigh - Python API for parallel programming using NetWorkSpaces. FILE /usr/lib/python2.6/dist-packages/nws/sleigh.py MODULE DOCS http://docs.python.org/library/nws.sleigh DESCRIPTION The sleigh module, built on top of NetWorkSpaces (NWS), makes it very easy to write simple parallel programs. It contains the Sleigh class, which provides two basic methods for executing tasks in parallel: eachElem and eachWorker. eachElem is used to execute a specified function multiple times in parallel with a varying set of arguments. eachWorker is used to execute a function exactly once on every worker in the sleigh with a fixed set of arguments. Example: First start up the NWS server, using the twistd command: % twistd -y /etc/nws.tac Now you can create a sleigh to execute python code in parallel: % python >>> from nws.sleigh import Sleigh >>> s = Sleigh() >>> import math >>> result = s.eachElem(math.exp, range(10)) >>> print "The answer is", result CLASSES __builtin__.object Sleigh SleighPending SleighResultIterator exceptions.Exception(exceptions.BaseException) SleighException SleighGatheredException SleighIllegalValueException SleighJoinException SleighNotAllowedException SleighNwsException SleighOccupiedException SleighScriptException SleighStoppedException SleighTaskException class Sleigh(__builtin__.object) | Represents a collection of python processes used to execute tasks. | | The sleigh allows python functions, methods, and expressions to be | executed in parallel using the eachElem and eachWorker methods. | | The sleigh workers are started when the Sleigh object is | constructed. When tasks are submitted to the sleigh, using the | eachWorker and eachElem methods, the workers will execute the tasks, | and return the results. When the stop method is called, the workers | are stopped. | | Note that a given python program can create multiple Sleigh objects, | which will each have it's own set of workers. This could be useful | if tasks have different requirements. For example, you could create | a Linux sleigh and a Windows sleigh, and submit Excel tasks only to | your Windows sleigh. | | Methods defined here: | | __init__(self, *deprecated, **kw) | Start the remote python processes used to execute tasks. | | Keyword arguments: | | launch -- Specifies the method of starting workers. If this | argument is set to the string 'local', then the workers | are executed on the local machine. If it is set to the | string 'web', then web launching is used. Otherwise, | the launch argument must specify a function (such as | nws.sleigh.sshcmd) that returns a list of command | arguments use to execute the workers on the machines | specified by the nodeList argument. The default value | is 'local'. | | workerCount -- Number of workers to start if the launch argument | is set to 'local' (which is the default value of launch). | This argument is ignored if launch is not set to | 'local'. The default value is 3. | | nodeList -- List of hosts on which to execute workers, if the | launch argument is set to a function. This argument is | ignored if launch is set to 'local' or 'web'. The | default value is ['localhost', 'localhost', | 'localhost']. | | nwsHost -- Host name of the machine where the NWS server is | executing. | | nwsPort -- Port to connect to the NWS server. | | nwsHostRemote -- Host name of the machine that workers should | use to connect to the NWS server. This is useful in | conjunction with the sshforwardcmd function (see the | description of scriptExec). The default is the value of | the nwsHost argument. | | nwsPortRemote -- Port that workers should use to connect to the | NWS server. This is useful in conjunction with the | sshforwardcmd function (see the description of | scriptExec). The default is the value of the nwsPort | argument. | | scriptExec -- Python function returning a list of command | arguments to execute the worker script. This list of | command arguments is appended to the list returned by | the launch function. The default value is the envcmd | function (defined in this module), which uses the | standard 'env' command to execute the script with the | appropriate environment for the worker. | | scriptDir -- Directory on the worker that contains the execution | script. | | scriptName -- Name of the script that executes the worker on the | remote machines. This defaults to | PythonNWSSleighWorker.sh on Unix, and | PythonNWSSleighWorker.py on Windows. | | modulePath -- Directory path to add to sys.path on workers. | This is often useful for giving the workers access to | python modules that define the python function to be | executed by eachWorker or eachElem. The default value | is None. | | workingDir -- Directory path to use as the current working | directory for the workers. The default value is the | current working directory of the sleigh master. | | logDir -- Directory in which to create the worker log files. | The default value is None, which leaves the decision | to the sleigh worker scripts, which generally uses the | remote system's temporary directory. | | user -- User name to use for remote execution of worker. This | argument may be ignored, depending the specified | launch function. The default is None, which allows the | remote execution mechanism decide. | | wsNameTemplate -- Template for the sleigh workspace name. This | must be a legal 'format' string, containing only an | integer format specifier. The default is | 'sleigh_ride_%04d'. | | userWsNameTemplate -- Template for the user's workspace name. | This must be a legal 'format' string, containing only | an integer format specifier. The default is | 'sleigh_user_%04d'. | | verbose -- Boolean flag used for displaying debug messages. | Debug messages will be sent to stderr. This will also | cause the worker processes to write debug messages to | files prefixed with 'sleigh_ride' in their current | working directory (as controled by the workerDir | argument). The default value is False. | | redirectOutput -- Boolean flag used to control whether messages | written by the worker functions to stdout and stderr | should be redirected to a workspace variable. If False, | they will go to the output file if verbose is True, | otherwise to the null device. The default value is True. | | __str__(self) | | eachElem(self, fun, elementArgs=[[]], fixedArgs=[], **kw) | Execute a function, method, or expression for each element in | the specified list(s). | | s.eachElem(fun, elementArgs[, fixedArgs]) -> list or SleighPending | | The results are normally returned as a list, unless the blocking | arguments is set to False, in which case, a SleighPending object | is returned. | | Arguments: | | fun -- Function, method, or python expression to execute. To | execute a method, you must specify a bound method | object. If the function or defining class is defined in | a module, the workers will attempt to import that | module. If that module isn't available to the worker | (because it's a non-standard module, not in the | PYTHONPATH), then the worker is not be able to execute | those tasks. | | To execute a python expression, you must specify it as a | string. Leading whitespace is automatically stripped to | avoid a common source of python syntax errors. | | elementArgs -- List of arguments to pass to the function or method | that need to be different for different tasks. In | general, this is a list of iterable objects, such as lists, | each containing the values to use for a given argument of | the different tasks. | | If your function needs only one varying argument of a | simple type, you can specify it without the outer list. | | Note that for a python expression, the list of arguments | is passed to the expression as a global variable named | 'SleighArguments'. | | fixedArgs -- List of additional arguments that are | fixed/constant for each task. Normally, they are | appended to the arguments specified by elementArgs, but | the order can be altered using the argPermute argument | described below. | | The default value is an empty list, which means that no | extra arguments are passed to the function. | | Note that for a python expression, the list of arguments | is passed to the expression as a global variable named | 'SleighArguments'. | | Keyword arguments: | | type -- Indicates the type of function invocation to perform. | This can be either 'invoke', 'define', or 'eval'. | If the fun argument is a function or bound method, then | the default value is 'invoke'. If the fun argument is | a string, then the default value is 'eval' (a value of | 'invoke' or 'define' is illegal for python expressions). | | blocking -- A boolean value that indicates whether to wait for | the results, or to return as soon as the tasks have been | submitted. If set to False, eachElem will return a | SleighPending object that is used to monitor the status | of the tasks, and to eventually retrieve the results. | You must wait for the results to be complete before | executing any further tasks on the sleigh, or a | SleighOccupiedException will be raised. | | If blocking is set to False, then the loadFactor | argument is disabled and ignored. Note that it's | unlikely that you'll need to turn off blocking in | eachElem. Non-blocking mode is more useful in | eachWorker. | | The default value is True. | | argPermute -- List that maps the specified arguments to the | actual arguments of the execution function. By | "specified arguments", I mean the items extracted from | elementArgs, followed by fixedArgs. (Note that unless | you are specifying both elementArgs and fixedArgs, you | probably don't need to use argPermute.) The items in | the argPermute list are used as indexes into the | "specified arguments" list. The length of argPermute | determines the number of arguments passed to the | execution function, which would normally be the length | of the specified arguments list, but this is not | required. For example, setting argPermute to an empty | list would cause the execution function to be called | without any arguments (although elementArgs would still | be required, and would be used to determine the number | of tasks to execute). | | The default behaviour is to pass the execution function | the arguments specified by elementArgs, followed by the | arguments from fixedArgs, which is equivalent to setting | argPermute to: | | n = len(elementArgs) + len(fixedArgs) | argPermute = range(n) | | If you wished to reverse the order of the arguments, you | could then modify argPermute: | | argPermute.reverse() | | But, more realistically, you need to interleave the | fixed arguments with the varying arguments. For | example, your execution function takes on fixed | argument, followed by two that vary, you would set | argPermute to: | | argPermute=[1,2,0] | | loadFactor -- Maximum number of tasks per worker to put into the | workspace at one time. This can become important if you | are executing a very large number of tasks. Setting | loadFactor to 3 will probably keep enough tasks | available in the workspace to keep the workers busy, | without flooding the workspace and bogging down the NWS | server. | | The default behaviour is to submit all of the tasks | to the sleigh workspace immediately. | | accumulator -- A function or callable object that will be called | for each result as they arrive. The first argument to | the function is a list of result values, and the second | argument is a list of indexes, which identifies which task. | | The arguments to the accumulator function are lists since | in the future, we plan to allow tasks to be "chunked" to | improve performance of small tasks. | | Note that bound methods can be very useful accumulators. | | eachWorker(self, fun, *workerArgs, **kw) | Execute a function, method, or expression on each worker of sleigh. | | s.eachWorker(fun[, ...]) -> list or SleighPending | | The results are normally returned as a list, unless | the blocking arguments is set to False, in which case, a | SleighPending object is returned. | | Arguments: | | fun -- Function, method, or python expression to execute. To | execute a method, you must specify a bound method | object. If the function or defining class is defined in | a module, the workers will attempt to import that | module. If that module isn't available to the worker | (because it's a non-standard module, not in the | PYTHONPATH), then the worker is not be able to execute | those tasks. | | To execute a python expression, you must specify it as a | string. Leading whitespace is automatically stripped to | avoid a common source of python syntax errors. | | Optional arguments: | | *workerArgs -- Arguments to pass to the function or method. | Specify whatever arguments the function requires, | including no arguments. The exact same set of arguments | will be used for each worker (unlike eachElem). For a | python expression, these arguments are passed to the | expression as a global variable named 'SleighArguments'. | | Keyword arguments: | | type -- Indicates the type of function invocation to perform. | This can be either 'invoke', 'define', or 'eval'. | If the fun argument is a function or bound method, then | the default value is 'invoke'. If the fun argument is | a string, then the default value is 'eval' (a value of | 'invoke' or 'define' is illegal for python expressions). | | blocking -- A boolean value that indicates whether to wait for | the results, or to return as soon as the tasks have been | submitted. If set to False, eachWorker will return a | SleighPending object that is used to monitor the status | of the tasks, and to eventually retrieve the results. | You must wait for the results to be complete before | executing any further tasks on the sleigh, or a | SleighOccupiedException will be raised. | | This argument is important if you want the master to be | able to interact/communicate with the workers, via NWS | operations, for example. This allows you to implement | more complex parallel or distributed programs. | | The default value is True. | | accumulator -- A function or callable object that will be called | for each result as they arrive. The first argument to | the function is a list of result values, and the second | argument is a list of indexes, which identifies which task. | | The arguments to the accumulator function are lists since | in the future, we plan to allow tasks to be "chunked" to | improve performance of small tasks. | | Note that bound methods can be very useful accumulators. | | imap(self, fun, *iterables, **kw) | Return an iterator whose values are returned from the function | evaluated with a argument tuple taken from the given iterable. | Stops when the shortest of the iterables is exhausted. | | s.imap(fun, *iterables[, **kw]) -> SleighResultIterator | | This is intended to be very similar to the imap function | in the itertools module. Other than being a method, rather than | a function, this method takes additional, keyword arguments, and | the iterator that is returned has special methods and properties. | See the SleighResultIterator documentation for more information. | | Arguments: | | fun -- Function, method, or python expression to execute. To | execute a method, you must specify a bound method | object. If the function or defining class is defined in | a module, the workers will attempt to import that | module. If that module isn't available to the worker | (because it's a non-standard module, not in the | PYTHONPATH), then the worker is not be able to execute | those tasks. | | To execute a python expression, you must specify it as a | string. Leading whitespace is automatically stripped to | avoid a common source of python syntax errors. | | | *iterables -- One or more iterables, one for each argument needed | by the function. | | Keyword arguments: | | type -- Indicates the type of function invocation to perform. | This can be either 'invoke', 'define', or 'eval'. | If the fun argument is a function or bound method, then | the default value is 'invoke'. If the fun argument is | a string, then the default value is 'eval' (a value of | 'invoke' or 'define' is illegal for python expressions). | | loadFactor -- Maximum number of tasks per worker to put into the | workspace at one time. This can become important if you | are executing a very large number of tasks (and essential | if submitting infinite tasks). The default value of 10 | will probably keep enough tasks available in the workspace | to keep the workers busy, without flooding the workspace | and bogging down the NWS server. | | starmap(self, fun, iterable, **kw) | Return an iterator whose values are returned from the function | evaluated with a argument tuple taken from the given iterable. | Stops when the shortest of the iterables is exhausted. | | s.starmap(fun, iterable[, **kw]) -> SleighResultIterator | | This is intended to be very similar to the starmap function | in the itertools module. Other than being a method, rather than | a function, this method takes additional, optional arguments, and | the iterator that is returned has special methods and properties. | See the SleighResultIterator documentation for more information. | | Arguments: | | fun -- Function, method, or python expression to execute. To | execute a method, you must specify a bound method | object. If the function or defining class is defined in | a module, the workers will attempt to import that | module. If that module isn't available to the worker | (because it's a non-standard module, not in the | PYTHONPATH), then the worker is not be able to execute | those tasks. | | To execute a python expression, you must specify it as a | string. Leading whitespace is automatically stripped to | avoid a common source of python syntax errors. | | | iterable -- Returns argument tuples for the function. | | Keyword arguments: | | type -- Indicates the type of function invocation to perform. | This can be either 'invoke', 'define', or 'eval'. | If the fun argument is a function or bound method, then | the default value is 'invoke'. If the fun argument is | a string, then the default value is 'eval' (a value of | 'invoke' or 'define' is illegal for python expressions). | | loadFactor -- Maximum number of tasks per worker to put into the | workspace at one time. This can become important if you | are executing a very large number of tasks (and essential | if submitting infinite tasks). The default value of 10 | will probably keep enough tasks available in the workspace | to keep the workers busy, without flooding the workspace | and bogging down the NWS server. | | status(self, closeGroup=False, timeout=0.0) | Return the status of the worker group. | | s.status(closeGroup, timeout) -> numworkers, closed | | The status includes the number of workers that have joined the | group so far, and a flag that indicates whether the group has | been closed (meaning that no more workers can join). Normally, | the group is automatically closed when all the workers that were | listed in the constructor have joined. However, this method | allows you to force the group to close after the timeout | expires. This can be particularly useful if you are running | on a large number of nodes, and some of the nodes are slow or | unreliable. If some of the workers are never started, the group | will never close, and no tasks will ever execute. | | Arguments: | | closeGroup -- Boolean flag indicating whether to close the | group. If True, the group will be closed, after the | specified timeout. The default value is False. | | timeout -- Number of seconds to wait for the group to close | before returning. The default value is 0.0. | | stop(self) | Stop the remote processes and delete the sleigh workspace. | | s.stop() | | workerInfo(self) | Return information about the sleigh workers. | | This returns a list of dictionaries, one for each worker. | Note that if this is called before all of the workers have | "joined", then you may get a partial list. This can be | useful in conjunction with the status method. | | The keys in the dictionaries are: | "nws": nws version used by the worker | "python": Python version used by the worker | "pid": process id of the worker process | "rank": worker rank (0 to workerCount - 1) | "host": name of worker machine (not necessarily the same as | the value in nodeList) | "logfile": path of the log file on the worker machine | "os": operating system of the worker machine | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class SleighException(exceptions.Exception) | Base class for all exceptions raised by this module. | | Method resolution order: | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods defined here: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighGatheredException(SleighException) | Results already gathered. | | Method resolution order: | SleighGatheredException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighIllegalValueException(SleighException) | Illegal value specified. | | Method resolution order: | SleighIllegalValueException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighJoinException(SleighException) | Too late to join worker group. | | Method resolution order: | SleighJoinException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighNotAllowedException(SleighException) | Sleigh construction not allowed. | | Method resolution order: | SleighNotAllowedException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighNwsException(SleighException) | Error performing NWS operation. | | Method resolution order: | SleighNwsException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighOccupiedException(SleighException) | Sleigh is occupied. | | Method resolution order: | SleighOccupiedException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighPending(__builtin__.object) | Represents a sleigh eachWorker/eachElem invocation in progress. | | This is returned from the eachElem and eachWorker operations | when executed asynchronously. It allows you to check for | the number of tasks left to be executed, and to wait for | the results of the operation to be returned. | | Methods defined here: | | __init__(self, nws, numTasks, barrierName, sleighState, accumulator) | Create an object that represents the pending sleigh operation. | | This constructor is intended for internal use only. | | Arguments: | | nws -- Sleigh NetWorkSpace object. | | numTasks -- Number of tasks that were submitted. | | barrierName -- Name of the barrier to wait at when complete. | | sleighState -- Object representing the current state of the | sleigh. | | accumulator -- Function to call with results as they arrive. | | __str__(self) | | check(self) | Return the number of tasks still outstanding. | | p.check() -> integer | | wait(self) | Wait for and return the list of results. | | p.wait() -> list | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) class SleighResultIterator(__builtin__.object) | Returns results from tasks submitted to the sleigh. | | Instances of this class are returned from the Sleigh imap, and | starmap methods. | | Methods defined here: | | __init__(self, task, taskIter, starIter, nws, numSubmitted, sleighState) | Create an iterator over the task results. | | This constructor is intended for internal use only. | | Arguments: | | task -- Partially initialized Task object. | | taskIter -- Iterator over the task arguments. | | starIter -- Is this a "star" iteration? | | nws -- Sleigh NetWorkSpace object. | | numSubmitted -- Number of tasks already submitted. | | sleighState -- Part of the Sleigh objects internal state. | | __iter__(self) | | next(self) | | shutdown(self) | Stop submitting tasks from the iterator. | | This method is a less drastic version of "stop". It is expected | that you will keep retrieving results that have already been | submitted, but no new tasks will be submitted, regardless of | what tasks were originally specified to imap or starmap. The | sleigh object will continue to be "occupied" until all results of | the pending tasks have been retreived. | | stop(self) | Stop the iterator, flushing any pending results. | | This method is useful if you're done with the iterator, and | don't want to retrieve anymore results. After calling stop, | you can submit more tasks to the sleigh (that is, it will | no longer be "occupied". | | ---------------------------------------------------------------------- | Data descriptors defined here: | | __dict__ | dictionary for instance variables (if defined) | | __weakref__ | list of weak references to the object (if defined) | | buffered | Number of buffered task results. | | returned | Number of task results returned. | | stopped | Is the iterator stopped? | | submitted | Number of submitted tasks. class SleighScriptException(SleighException) | Unable to find sleigh worker script. | | Method resolution order: | SleighScriptException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighStoppedException(SleighException) | Sleigh is stopped. | | Method resolution order: | SleighStoppedException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message class SleighTaskException(SleighException) | Error executing a task. | | Method resolution order: | SleighTaskException | SleighException | exceptions.Exception | exceptions.BaseException | __builtin__.object | | Methods inherited from SleighException: | | __repr__(self) | | __str__(self) | | ---------------------------------------------------------------------- | Data descriptors inherited from SleighException: | | __weakref__ | list of weak references to the object (if defined) | | ---------------------------------------------------------------------- | Methods inherited from exceptions.Exception: | | __init__(...) | x.__init__(...) initializes x; see x.__class__.__doc__ for signature | | ---------------------------------------------------------------------- | Data and other attributes inherited from exceptions.Exception: | | __new__ = | T.__new__(S, ...) -> a new object with type S, a subtype of T | | ---------------------------------------------------------------------- | Methods inherited from exceptions.BaseException: | | __delattr__(...) | x.__delattr__('name') <==> del x.name | | __getattribute__(...) | x.__getattribute__('name') <==> x.name | | __getitem__(...) | x.__getitem__(y) <==> x[y] | | __getslice__(...) | x.__getslice__(i, j) <==> x[i:j] | | Use of negative indices is not supported. | | __reduce__(...) | | __setattr__(...) | x.__setattr__('name', value) <==> x.name = value | | __setstate__(...) | | __unicode__(...) | | ---------------------------------------------------------------------- | Data descriptors inherited from exceptions.BaseException: | | __dict__ | | args | | message DATA __all__ = ['Sleigh', 'SleighPending', 'SleighResultIterator', 'SleighE... __version__ = '1.6.3' VERSION 1.6.3 nwsclient-1.6.4/doc/Python_Sleigh_Tutorial0000644000175000017500000001267311175420457020316 0ustar westonwestonPython Sleigh Tutorial ---------------------- Sleigh is a python package, built on top of NWS, that makes it very easy to write simple parallel programs. It provides two basic functions for executing tasks in parallel: eachElem and eachWorker. eachElem is used to execute a specified function multiple times in parallel with a varying set of arguments. eachWorker is used to execute a function exactly once on every worker in the sleigh with a fixed set of arguments. eachElem is all that is needed for most basic programs, so that is what we will start with. First, we need to start up a sleigh, so we'll import the sleigh module, and then construct a Sleigh object: >>> from nws.sleigh import Sleigh >>> s = Sleigh() This starts three sleigh workers on the local machine, but workers can be started on other machines by specifying a launch method and a list of machines. Let's shut down the sleigh so we can start workers on some other machines. >>> s.stop() This deletes the sleigh's NWS workspace, and shuts down all of the sleigh worker processes. Now we'll make a new sleigh, starting workers on node1 and node2 using the ssh command, and we'll use an NWS server that's running on node10: >>> from nws.sleigh import sshcmd >>> s = Sleigh(launch=sshcmd, nodeList=['node1', 'node2'], nwsHost='node10') If you're just starting to use NWS, the first method is very simple and convenient, and you don't have to worry about configuring ssh. Once you've got your NWS program written, and want to run it a network, you just change the way that you construct your Sleigh, and you're ready to run. If you're running on a machine that has multiple processors, then you can use the simpler method of constructing your Sleigh, but you'll probably want to control the number of workers that get started. To do that, you use the workerCount option: >>> s = Sleigh(workerCount=8) There are more options, but that's more than enough to get you started. So now that we know how to create a sleigh, let actually run a parallel program. Here's how we do it: >>> result = s.eachElem(abs, range(-10, 0)) In that simple command, we have defined a set of data that is processed by multiple workers in parallel, and returned each of the results in a list. (Of course, you would never really bother to do such a trivial amount of work with a parallel program, but you get the idea.) This eachElem command puts 10 tasks into the sleigh workspace. Each task contains one value from -10 to -1. This value is passed as the argument to the absolute value function. The return value of the function is put into the sleigh workspace. The eachElem command waits for all of the results to be put into the workspace, and returns them as a list, which are the numbers from 10 to 1. As a second example, let's add two lists together. First, we'll define an add2 function, and then we'll use it with eachElem: >>> def add2(x, y): return x + y ... >>> result = s.eachElem(add2, [range(10), range(10, 0, -1)]) This is the parallel equivalent to the python command: >>> result = map(add2, range(10), range(10, 0, -1)) We can keep adding more list arguments in this way, but there is also a way to add arguments that are the same for every task, which we call fixed arguments: >>> result = s.eachElem(add2, range(10), 20) This is equivalent to the python command: >>> result = map(add2, range(10), [20] * 10) The order of the arguments passed to the function are normally in the specified order, which means that the fixed arguments always come after the varying arguments. To change this order, a permutation list can be specified. The permutation list is specified using the "argPermute" keyword parameter. For example, to perform the parallel equivalent of the python operation "map(sub, [20]*20, range(20))", we do the following: >>> def sub(x, y): return x - y ... >>> result = s.eachElem(sub, range(20), 20, argPermute=[1,0]) This permutation list says to first use the second argument, and then use the first, thus reversing the order of the two arguments. There is another keyword argument, called "blocking", which, if set to 0, will make eachElem return immediately after submitting the tasks, thus making it non-blocking. A "pending" object is returned, which can be used periodically to check how many of the tasks are complete, and also to wait until all tasks are finished. Here's a quick example: >>> p = s.eachElem(add2, [range(20), range(20, 0, -1)], blocking=0) >>> while p.check() > 0: ... # Do something useful for a little while ... pass ... >>> result = p.wait() There is also a keyword argument called "loadFactor" that enables watermarking. This limits the number of tasks that are put into the workspace at the same time. That could be important if you're executing a lot of tasks. Setting the load factor to 3 limits the number of tasks in the workspace to 3 times the number of workers in the sleigh. Here's how to do it: >>> result = s.eachElem(add2, [range(1000), range(1000)], loadFactor=3) The results are exactly the same as not using a load factor. Setting this option only changes the way that tasks are submitted by the eachElem command. As you can see, Sleigh makes it easy to write simple parallel programs. But you're not limited to simple programs. You can use NWS operations to communicate between the worker processes, allowing you to write message passing parallel programs much more easily than using MPI or PVM, for example. See the examples directory for more ideas on how to use Sleigh. nwsclient-1.6.4/PKG-INFO0000644000175000017500000000224711261702410014244 0ustar westonwestonMetadata-Version: 1.0 Name: nwsclient Version: 1.6.4 Summary: Python NetWorkSpaces Home-page: http://nws-py.sourceforge.net/ Author: REvolution Computing, Inc. Author-email: sbweston@users.sourceforge.net License: GPL version 2 or later Description: NetWorkSpaces (NWS) is a system that makes it very easy for different scripts and programs running (potentially) on different machines to communicate and coordinate with one another. The requirements for the NWS for Python are: Python 2.2 or later on Linux, Mac OS X, and other Unix systems. Python 2.4 or later on Windows. Platform: any Classifier: Topic :: System :: Clustering Classifier: Topic :: System :: Distributed Computing Classifier: License :: OSI Approved :: GNU General Public License (GPL) Classifier: Development Status :: 5 - Production/Stable Classifier: Environment :: Console Classifier: Intended Audience :: Developers Classifier: Intended Audience :: Science/Research Classifier: Intended Audience :: System Administrators Classifier: Operating System :: OS Independent Classifier: Programming Language :: Python Classifier: Natural Language :: English nwsclient-1.6.4/scripts/0000755000175000017500000000000011261702410014631 5ustar westonwestonnwsclient-1.6.4/scripts/com.revolution-computing.pybabelfish.plist0000644000175000017500000000167311175420457025206 0ustar westonweston Label com.revolution-computing.pybabelfish ProgramArguments /usr/bin/python /usr/local/bin/pybabelfish RunAtLoad UserName nobody GroupName nogroup WorkingDirectory /private/tmp KeepAlive OtherJobEnabled com.revolution-computing.nwsserver nwsclient-1.6.4/scripts/PyBabelfishService.py0000644000175000017500000001346011175420457020734 0ustar westonweston# # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import threading import win32serviceutil, win32service, win32event, win32api from nws.client import NetWorkSpace, NwsException # babelfish states _BABELFISH_RUNNING = 100 _BABELFISH_DIED = 101 _BABELFISH_RESTARTED = 102 # timeouts _DIED_TIMEOUT = 10 * 1000 # milliseconds to wait before restarting # a dead babelfish _RESTARTED_TIMEOUT = 10 * 1000 # milliseconds to wait before considering a # restarted babelfish to be running class PyBabelfishService(win32serviceutil.ServiceFramework): _svc_name_ = 'PyBabelfishService' _svc_display_name_ = 'Babelfish Service for Python NetWorkSpaces' _svc_description_ = 'Translates Python objects in NWS variables into ' \ 'human readable form so that the values of variables in a ' \ 'workspace can be displayed in the NWS server\'s web interface.' def __init__(self, args): win32serviceutil.ServiceFramework.__init__(self, args) self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) def BabelfishDied(self): win32event.SetEvent(self.babelfishDied) def SvcStop(self): self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) win32event.SetEvent(self.hWaitStop) def SvcDoRun(self): import servicemanager self.sm = servicemanager self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, self.sm.PYS_SERVICE_STARTED, (self._svc_name_, '')) self._startBabelfish('localhost', 8765) handles = (self.hWaitStop, self.babelfishDied) timeout = win32event.INFINITE state = _BABELFISH_RUNNING while True: # wait for a stop request, a babelfish death, or possibly a timeout s = win32event.WaitForMultipleObjects(handles, 0, timeout) if s == win32event.WAIT_TIMEOUT: if state == _BABELFISH_RESTARTED: self._info("babelfish restarted successfully") timeout = win32event.INFINITE state = _BABELFISH_RUNNING elif state == _BABELFISH_DIED: self._startBabelfish('localhost', 8765) handles = (self.hWaitStop, self.babelfishDied) timeout = _RESTARTED_TIMEOUT state = _BABELFISH_RESTARTED else: self._error("got an unexpected timeout while in state %d" % state) break elif s == win32event.WAIT_OBJECT_0: # a shutdown was requested, so break out of the while loop self._info("shutdown requested") break elif s == win32event.WAIT_OBJECT_0 + 1: # the babelfish exited by itself, which probably means # that the NWS server shutdown. we want to reconnect # when it comes back up, so sleep awhile, and then # start another babelfish. this will probably happen # over and over again, so don't do it too frequently. if state == _BABELFISH_RUNNING: self._info("babelfish died: restarting in a bit") win32api.CloseHandle(self.babelfishDied) self.babelfishDied = None handles = (self.hWaitStop,) timeout = _DIED_TIMEOUT state = _BABELFISH_DIED else: self._error("illegal status from WaitForMultipleObjects: stopping") break self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, self.sm.PYS_SERVICE_STOPPED, (self._svc_name_, '')) def _info(self, msg): self.sm.LogMsg(self.sm.EVENTLOG_INFORMATION_TYPE, 1, (msg,)) def _error(self, msg): self.sm.LogMsg(self.sm.EVENTLOG_ERROR_TYPE, 1, (msg,)) def _startBabelfish(self, nwsHost, nwsPort): self.babelfish = Babelfish(nwsHost, nwsPort, self.BabelfishDied) self.babelfishDied = win32event.CreateEvent(None, 0, 0, None) self.babelfish.setDaemon(True) self.babelfish.start() class Babelfish(threading.Thread): def __init__(self, nwsHost, nwsPort, died): threading.Thread.__init__(self, name='babelfish') self.nwsHost = nwsHost self.nwsPort = nwsPort self.died = died def __str__(self): return "Babelfish[%s:%d]" % (self.nwsHost, self.nwsPort) def run(self): try: try: ws = NetWorkSpace('Python babelfish', self.nwsHost, self.nwsPort) while True: try: v = ws.fetch('food') t = str(v) or repr(v) except ImportError, e: t = 'Import Error: ' + str(e) ws.store('doof', t) except NwsException: # probably the server went down pass finally: try: ws.server.close() except: pass self.died() if __name__ == '__main__': win32serviceutil.HandleCommandLine(PyBabelfishService) nwsclient-1.6.4/scripts/pybabelfish0000755000175000017500000000151611175420457017066 0ustar westonweston#!/usr/bin/env python # # Copyright (c) 2005-2008, REvolution Computing, Inc. # # NetWorkSpaces is free software; you can redistribute it and/or # modify it under the terms of the GNU General Public License as published # by the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 # USA # import sys from nws.babelfish import main main(sys.argv[1:]) nwsclient-1.6.4/scripts/rwin.vbs0000644000175000017500000001330211175420457016337 0ustar westonwestonOption Explicit Dim args, arg, icmd, i Dim host, user, passwd, verbose, simplequote Dim locator, server, wmiservice, processSU, config Dim cmd, ret ' Process the arguments Set args = WScript.Arguments ' Set default option values verbose = False simplequote = False user = "" passwd = "" If args.Count = 0 Then usage() End If host = args.Item(0) If host = "-h" Then usage() i = 1 icmd = 0 Do While i < args.Count arg = args.Item(i) If Left(arg, 1) = "-" Then Select Case arg Case "-h" usage() Case "-v" verbose = True WScript.Echo("Setting verbose option") Case "-n" ' Ignore this option If verbose Then WScript.Echo("Ignoring -n option") Case "-l" i = i + 1 If i >= args.Count Then WScript.Echo("The -l options takes a required argument") usage() End If user = args.Item(i) If verbose Then WScript.Echo("User name: " & user) Case "-p" i = i + 1 If i >= args.Count Then WScript.Echo("The -p options takes a required argument") usage() End If passwd = args.Item(i) If verbose Then WScript.Echo("Passwd: " & passwd) Case "-P" ' XXX Some way to avoid echoing? passwd = InputBox("Please enter password", "Password Entry") If verbose Then WScript.Echo("Passwd: " & passwd) Case "-s" simplequote = True If verbose Then WScript.Echo("Using simple quoting") Case "--" icmd = i + 1 Exit Do Case Else WScript.Echo("Illegal option specified: " & arg) usage() End Select i = i + 1 Else icmd = i Exit Do End If Loop If icmd = 0 Or icmd >= args.Count Then WScript.Echo("No command specified") usage() End If ' Cannot specified alternate user without a password If passwd = "" Then user = "" End If ' Cannot use alternative credentials on local machine If isLocal(host) Then user = "" passwd = "" If verbose Then WScript.Echo("Local target computer detected") End If If verbose Then If passwd = "" Then WScript.Echo("Not using alternative credentials") Else WScript.Echo("Using alternative credentials") End If End If Set locator = CreateObject("WbemScripting.SWbemLocator") Set server = locator.ConnectServer(host, "root\cimv2", user, passwd, "", "") Set wmiservice = server.Get("Win32_Process") Set processSU = server.Get("win32_ProcessStartUP") Set config = processSU.SpawnInstance_ config.ShowWindow = 0 ' Hide the window ' Create the command to execute on the target computer If simplequote Then cmd = simple_quote(args.Item(icmd)) For i = icmd + 1 To args.Count - 1 cmd = cmd & " " & simple_quote(args.Item(i)) Next Else cmd = msc_quote(args.Item(icmd)) For i = icmd + 1 To args.Count - 1 cmd = cmd & " " & msc_quote(args.Item(i)) Next End If ' Execute the command on the target computer asynchronously If verbose Then WScript.Echo("Executing remote command: " & cmd) ret = wmiservice.create(cmd, Null, config) ' Check for errors If ret <> 0 Then error("Error creating remote process: " & ret) End If Function error(msg) WScript.Echo(msg) WScript.Quit(1) End Function Function usage() error("Usage: rwin [